Switch main to the Python (pysceptre) pipeline

.githooks/pre-commit

@@ -12,6 +12,7 @@
 #   * any file larger than 512 KB
 #   * filenames that are not snake_case
 #   * camelCase / PascalCase identifiers in R code
+#   * ruff check and ruff format on Python, when ruff is on PATH
 #
 # Install:  pixi run install-hooks     (sets core.hooksPath to .githooks)
 # Lint all: pixi run lint              (same checks across the whole repo, read-only)
@@ -140,6 +141,27 @@ while IFS= read -r file; do
         rm -f "$scratch"
     fi
 
+    # --- Python lint and format ------------------------------------------------------------------
+    # ruff, not a bespoke regex. The R check below exists because nothing else was going to
+    # enforce R style; Python has a linter that already knows the rules, and the same one the
+    # project's `pixi run format` task uses, so the hook and the task cannot disagree.
+    #
+    # Read-only here even under --fix: ruff format rewrites whole files, and a hook that silently
+    # reformats what you are committing makes `git diff --cached` stop describing your change.
+    case "$file" in
+        *.py)
+            if command -v ruff >/dev/null 2>&1; then
+                if ! ruff check --quiet "$file"; then
+                    red "REJECT  $file fails ruff check"
+                    failures=$((failures + 1))
+                elif ! ruff format --check --quiet "$file" >/dev/null 2>&1; then
+                    red "REJECT  $file is not ruff-formatted; run 'pixi run -e dev format'"
+                    failures=$((failures + 1))
+                fi
+            fi
+            ;;
+    esac
+
     # --- R identifier convention ---------------------------------------------------------------
     # Flags camelCase and PascalCase assignments. Deliberately does not demand strict snake_case:
     # SCREAMING_SNAKE constants (PERT_LEVELS) and dotted S3 methods (print.sim_input) are correct

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+# The tests that need neither R nor a real screen, which is all of them by default: the two that
+# do are behind `-m realdata` and are skipped here.
+#
+# Not a pixi environment. pixi resolves pysceptre from git and would rebuild the whole conda
+# environment on every push; uv installs the same dependencies from pyproject.toml in seconds. What
+# that gives up is checking that pixi.lock still resolves, which is what the container build does
+# and which does not need to happen on every commit.
+
+name: tests
+
+on:
+  push:
+    branches: [main, 'feat/**']
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install
+        run: |
+          uv venv --python 3.12
+          # pysceptre is not on PyPI; the pin matches pixi.toml's and has to move with it.
+          uv pip install "pysceptre @ git+https://github.com/broadinstitute/pysceptre.git@v0.2.0"
+          uv pip install -e ".[dev]"
+
+      - name: Lint
+        run: |
+          uv run ruff check src tests workflow
+          uv run ruff format --check src tests workflow
+
+      - name: Test
+        run: uv run pytest -q
+
+      - name: Nextflow DAG
+        # Catches a rewiring mistake without running anything. The stub needs the synthetic
+        # dataset, which is generated rather than committed.
+        run: |
+          curl -fsSL https://get.nextflow.io | bash
+          uv run watteg-make-test-data --out tests/data/synthetic.h5mu
+          ./nextflow run . -profile standard -params-file config/test.yml -stub-run \
+            --samplesheet assets/samplesheet_synthetic.csv --outdir /tmp/stub

.gitignore

@@ -88,3 +88,7 @@ tests/data/
 
 # Created by tests/config/run.sbatch -- sparse files whose SIZE is the fixture, 3.3 GB of nothing.
 tests/config/inputs/
+.venv/
+__pycache__/
+*.egg-info/
+tests/data/*.h5mu

README.md

@@ -1,7 +1,8 @@
 # WattEG
 
 **W**atts for **E**lement–**G**ene pairs: power analysis for element–gene pairs in single-cell
-CRISPR screens, built on [sceptre](https://katsevich-lab.github.io/sceptre/).
+CRISPR screens, built on [pysceptre](https://github.com/broadinstitute/pysceptre) — a Python port
+of [sceptre](https://katsevich-lab.github.io/sceptre/)'s statistical engine.
 
 **📖 [Documentation](https://engreitzlab.github.io/WattEG/)**
 
@@ -22,67 +23,84 @@ written for one specific analysis. This repository generalises it.
 
 ```sh
 pixi install
-pixi run setup       # installs sceptre from a pinned commit
-pixi run check-api   # verifies that pin against the internals the pipeline uses
 ```
 
-sceptre is not on conda-forge or bioconda, so it is installed from a pinned commit rather than
-captured in `pixi.lock`. The pipeline reads several of its unexported S4 slots, which is why the
-version is pinned and checked.
+That is the whole of it. The environment is one Python package plus Nextflow: no R, no pinned
+sceptre commit, no patch applied to it, and no separate install step. pysceptre enters as a git
+dependency pinned to a commit, because a sweep has to be re-runnable against the engine that
+produced it.
+
+**The R implementation has not been deleted.** It is the reference the Python path was validated
+against and what the methods paper describes, and it still lives in `src/*.R` and `lib/*.R`. What
+is gone is the environment that ran it; to run it, use the **`r-implementation`** branch.
+
+That branch is **maintained, not frozen**. Two bugs found during the port change results, and both
+were fixed in R as well as in Python rather than being quarantined on a snapshot -- nothing was
+published, so there were no numbers owed a bit-for-bit reproduction, and freezing the branch would
+only have preserved the bugs. Sweeps produced before 2026-09-21 predate both fixes.
+(`legacy` is something else again -- the Snakemake implementation that preceded both.)
 
 ## Input
 
-**One file: a sceptre object** (`.rds`) on which `assign_grnas()` and `run_qc()` have been called,
-using `grna_integration_strategy = "union"`.
+**One file: a `.h5mu` dataset** exported from a sceptre object on which `assign_grnas()` and
+`run_qc()` have been called, using `grna_integration_strategy = "union"`.
 
 Everything else is derived from it — the discovery pairs, the gRNA-to-target mapping and the
-significance threshold all already live inside the object.
+significance threshold all travel with the export.
+
+Making one is a one-off step per dataset, run wherever R and sceptre are available. It is the only
+place R appears at all, and it is not part of the pipeline:
+
+```sh
+Rscript pysceptre/scripts/export_sceptre_dataset.R \
+  --sceptre-object results/sample1/sceptre_object.rds \
+  --out-dir export/ --all-genes --all-cells
+python  pysceptre/scripts/make_h5mu.py export/
+```
 
-If the object's response matrix is odm-backed (out-of-core), pass `--response-odm` on
-`prepare_sim_input.R` with the path to the backing `.odm` file — see [Usage](https://engreitzlab.github.io/WattEG/usage/).
+**`--all-cells` is required, not optional.** DESeq2 "poscounts" size factors are a per-cell
+reduction against a per-gene geometric mean taken over every cell in the object, so an export
+restricted to the QC-passing cells gives different size factors for the cells that remain. The
+export handles odm-backed (out-of-core) matrices itself, which is why there is no longer an
+`--response-odm` flag anywhere in the pipeline.
 
 List your samples in a CSV (see `assets/samplesheet.csv`):
 
 ```csv
-sample,sceptre_object
-sample1,results/sample1/sceptre_object.rds
+sample,dataset
+sample1,export/dataset.h5mu
 ```
 
-Add an optional `response_odm` column (see `assets/samplesheet_seqera_test.csv`) for any sample whose
-sceptre object's response matrix is odm-backed — same file `prepare_sim_input.R --response-odm` takes
-standalone. Leave it blank, or omit the column entirely, for in-memory-backed objects.
-
 ## Quickstart
 
-Each step is a standalone executable in `src/` with `--help`.
+```sh
+nextflow run . -profile sherlock -params-file config/config.yml
+```
+
+Each step is also a console script with `--help`, so a sweep can be driven by hand or by another
+runner:
 
 ```sh
 # derive the simulation inputs (once per sample)
-Rscript src/prepare_sim_input.R \
-  --sceptre-object results/sample1/sceptre_object.rds \
-  --outdir prepared/
+watteg-prepare-sim-input --dataset export/dataset.h5mu --outdir prepared/
 
 # split targets into per-task chunks
-Rscript src/split_pairs.R --pairs prepared/pairs.tsv --n-splits 280 --outdir splits/
+watteg-split-pairs --pairs prepared/pairs.tsv --n-splits 280 --outdir splits/
 
 # simulate (once per split x effect size)
-Rscript src/run_power_simulation.R \
-  --sim-input prepared/sim_input.rds \
-  --sceptre-template prepared/sceptre_template.rds \
-  --pairs splits/split_001.tsv \
-  --grna-targets prepared/grna_targets.tsv \
+watteg-run-power-simulation \
+  --prepared prepared/ --pairs splits/split_001.tsv \
   --effect-size 0.15 --reps 100 --seed 20250812 \
   --out sim/split_001_es0.15.tsv
 
 # power per pair, then one table across effect sizes
-Rscript src/compute_power.R \
-  --simulations "$(ls sim/*_es0.15.tsv | paste -sd, -)" \
-  --threshold-file prepared/discovery_threshold.txt \
+watteg-compute-power \
+  --simulations sim/ --threshold-file prepared/discovery_threshold.txt \
   --out power_es0.15.tsv
 
-Rscript src/summarize_power.R \
-  --power power_es0.15.tsv,power_es0.2.tsv \
-  --out power_summary.tsv
+watteg-summarize-power \
+  --power power_es0.15.tsv power_es0.2.tsv \
+  --sim-input prepared/sim_input.h5 --out power_summary.tsv
 ```
 
 Parameters live in `config/config.yml`.

assets/samplesheet.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object
-day0_grna20_no_shuffle,results/day0_grna20_no_shuffle/sceptre_object.rds
+sample,dataset
+day0_grna20_no_shuffle,results/day0_grna20_no_shuffle/dataset.h5mu

assets/samplesheet_all_samples.csv

@@ -1,5 +1,5 @@
-sample,sceptre_object
-day0_grna20,results/day0_grna20/sceptre_object.rds
-day0_grna20_no_shuffle,results/day0_grna20_no_shuffle/sceptre_object.rds
-day2_grna20,results/day2_grna20/sceptre_object.rds
-day4_grna20,results/day4_grna20/sceptre_object.rds
+sample,dataset
+day0_grna20,results/day0_grna20/dataset.h5mu
+day0_grna20_no_shuffle,results/day0_grna20_no_shuffle/dataset.h5mu
+day2_grna20,results/day2_grna20/dataset.h5mu
+day4_grna20,results/day4_grna20/dataset.h5mu

assets/samplesheet_day0_grna20.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object
-day0_grna20,results/day0_grna20/sceptre_object.rds
+sample,dataset
+day0_grna20,results/day0_grna20/dataset.h5mu

assets/samplesheet_moi5_cis.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object,response_odm
-moi5,gs://hgrm-seqera/element-gene-power-analysis-union/moi5/sceptre_object.rds,gs://hgrm-seqera/element-gene-power-analysis-union/moi5/response.odm
+sample,dataset
+moi5,gs://hgrm-seqera/element-gene-power-analysis-union/moi5/dataset.h5mu

assets/samplesheet_moi5_trans.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object,response_odm
-moi5_trans_discovery,gs://hgrm-seqera/sceptre-nf-ondisc-test/moi5-trans-output/sceptre_object.rds,gs://hgrm-seqera/20260508-wtc11-sceptre/grna-single-target-moi5-ondisc/gene.odm
+sample,dataset
+moi5_trans_discovery,gs://hgrm-seqera/sceptre-nf-ondisc-test/moi5-trans-output/dataset.h5mu

assets/samplesheet_seqera_test.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object,response_odm
-wtc11_union_odm,gs://hgrm-seqera/element-gene-power-analysis-test/sceptre_object.rds,gs://hgrm-seqera/element-gene-power-analysis-test/response.odm
+sample,dataset
+wtc11_union_odm,gs://hgrm-seqera/element-gene-power-analysis-test/dataset.h5mu

assets/samplesheet_synthetic.csv

@@ -1,2 +1,2 @@
-sample,sceptre_object
-synthetic,tests/data/sceptre_object_high_default.rds
+sample,dataset
+synthetic,tests/data/synthetic.h5mu