From 8e59225e1c0ac11eadd9903e2f7e81b483c66a2a Mon Sep 17 00:00:00 2001 From: Chin Hao <60098604+ch99l@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:41:18 +0800 Subject: [PATCH] add Visium HD support (#41) --- .github/workflows/build_container.yml | 170 ++++++--- .github/workflows/smoke_test.yml | 15 +- CHANGELOG.md | 26 ++ README.md | 340 +++++++++++++----- bin/bambu_discovery_to_unique_counts.R | 30 ++ bin/create_seurat_object.R | 15 + bin/map_spots_to_clusters.R | 23 ++ bin/save_counts.R | 34 +- bin/visium_hd_aggregate_resolution.R | 57 +++ bin/visium_hd_cluster.R | 66 ++++ bin/visium_hd_convert_barcode_mappings.py | 40 +++ bin/visium_hd_convert_tissue_positions.py | 30 ++ bin/visium_hd_spot_bin_mappings.py | 45 +++ conf/base.config | 2 +- conf/containers.config | 6 +- conf/dev.config | 2 +- conf/smoke_test.config | 33 ++ containers/{r => bambu}/Dockerfile | 8 +- containers/seurat/Dockerfile | 13 + examples/barcode_mappings_synthetic.parquet | Bin 0 -> 7768 bytes examples/bins_visium_hd_synthetic.csv | 4 + examples/clusters_sc_example.rds | Bin 0 -> 77561 bytes examples/clusters_visium_hd_synthetic.rds | Bin 0 -> 342 bytes examples/extended_annotations_sc_example.rds | Bin 0 -> 285968 bytes ...tended_annotations_visium_hd_synthetic.rds | Bin 0 -> 300400 bytes examples/quant_data_sc_example.rds | Bin 0 -> 1058845 bytes examples/quant_data_visium_hd_synthetic.rds | Bin 0 -> 318807 bytes examples/samplesheet_test_sc_quant_data.csv | 2 + examples/samplesheet_visium_hd_quant_data.csv | 2 + examples/samplesheet_visium_hd_synthetic.csv | 2 + .../tissue_positions_002um_synthetic.parquet | Bin 0 -> 22129 bytes .../tissue_positions_008um_synthetic.parquet | Bin 0 -> 3321 bytes .../tissue_positions_016um_synthetic.parquet | Bin 0 -> 2284 bytes examples/visium_hd_synthetic.bam | Bin 0 -> 7900866 bytes lib/Validation.groovy | 81 ++++- main.nf | 255 ++++++++++--- .../cluster_level_quantification.nf} | 33 +- .../{ => shared}/construct_read_class.nf | 2 +- .../bambu/{ => shared}/prepare_annotation.nf | 2 +- .../standard/single_cell_quantification.nf | 41 +++ .../{ => standard}/transcript_discovery.nf | 42 +-- modules/bambu/visium_hd/aggregate_bins.nf | 46 +++ .../bambu/visium_hd/extract_spot_barcodes.nf | 29 ++ .../visium_hd/map_clusters_to_2um_spots.nf | 33 ++ .../visium_hd/spot_level_quantification.nf | 63 ++++ .../bambu/visium_hd/transcript_discovery.nf | 62 ++++ .../standard}/extract_barcodes.nf | 0 .../standard}/extract_spatial_coordinates.nf | 0 .../visium_hd/convert_barcode_mappings.nf | 26 ++ .../visium_hd/convert_tissue_positions.nf | 27 ++ .../visium_hd/filter_barcoded_bam.nf | 27 ++ .../visium_hd/spot_bin_mappings.nf | 27 ++ .../multi_sample_clustering.nf} | 31 +- .../single_sample_clustering.nf} | 24 +- modules/seurat/visium_hd/clustering.nf | 55 +++ nextflow.config | 50 ++- subworkflows/clustering.nf | 22 -- subworkflows/clustering_standard.nf | 24 ++ subworkflows/prepare_input_standard.nf | 4 +- subworkflows/prepare_input_visium_hd.nf | 53 +++ 60 files changed, 1691 insertions(+), 333 deletions(-) create mode 100755 bin/bambu_discovery_to_unique_counts.R create mode 100755 bin/create_seurat_object.R create mode 100755 bin/map_spots_to_clusters.R create mode 100755 bin/visium_hd_aggregate_resolution.R create mode 100755 bin/visium_hd_cluster.R create mode 100755 bin/visium_hd_convert_barcode_mappings.py create mode 100755 bin/visium_hd_convert_tissue_positions.py create mode 100755 bin/visium_hd_spot_bin_mappings.py rename containers/{r => bambu}/Dockerfile (84%) create mode 100644 containers/seurat/Dockerfile create mode 100644 examples/barcode_mappings_synthetic.parquet create mode 100644 examples/bins_visium_hd_synthetic.csv create mode 100644 examples/clusters_sc_example.rds create mode 100644 examples/clusters_visium_hd_synthetic.rds create mode 100644 examples/extended_annotations_sc_example.rds create mode 100644 examples/extended_annotations_visium_hd_synthetic.rds create mode 100644 examples/quant_data_sc_example.rds create mode 100644 examples/quant_data_visium_hd_synthetic.rds create mode 100644 examples/samplesheet_test_sc_quant_data.csv create mode 100644 examples/samplesheet_visium_hd_quant_data.csv create mode 100644 examples/samplesheet_visium_hd_synthetic.csv create mode 100644 examples/tissue_positions_002um_synthetic.parquet create mode 100644 examples/tissue_positions_008um_synthetic.parquet create mode 100644 examples/tissue_positions_016um_synthetic.parquet create mode 100644 examples/visium_hd_synthetic.bam rename modules/bambu/{EM_quant.nf => shared/cluster_level_quantification.nf} (55%) rename modules/bambu/{ => shared}/construct_read_class.nf (98%) rename modules/bambu/{ => shared}/prepare_annotation.nf (97%) create mode 100644 modules/bambu/standard/single_cell_quantification.nf rename modules/bambu/{ => standard}/transcript_discovery.nf (64%) create mode 100644 modules/bambu/visium_hd/aggregate_bins.nf create mode 100644 modules/bambu/visium_hd/extract_spot_barcodes.nf create mode 100644 modules/bambu/visium_hd/map_clusters_to_2um_spots.nf create mode 100644 modules/bambu/visium_hd/spot_level_quantification.nf create mode 100644 modules/bambu/visium_hd/transcript_discovery.nf rename modules/{prepare_input_standard => prepare_input/standard}/extract_barcodes.nf (100%) rename modules/{prepare_input_standard => prepare_input/standard}/extract_spatial_coordinates.nf (100%) create mode 100644 modules/prepare_input/visium_hd/convert_barcode_mappings.nf create mode 100644 modules/prepare_input/visium_hd/convert_tissue_positions.nf create mode 100644 modules/prepare_input/visium_hd/filter_barcoded_bam.nf create mode 100644 modules/prepare_input/visium_hd/spot_bin_mappings.nf rename modules/seurat/{multi_sample.nf => standard/multi_sample_clustering.nf} (60%) rename modules/seurat/{single_sample.nf => standard/single_sample_clustering.nf} (67%) create mode 100644 modules/seurat/visium_hd/clustering.nf delete mode 100644 subworkflows/clustering.nf create mode 100644 subworkflows/clustering_standard.nf create mode 100644 subworkflows/prepare_input_visium_hd.nf diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index 1b97a6a..7c81d4a 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -1,13 +1,16 @@ -name: Build R container +name: Build R containers -# The image tag is read from conf/containers.config, so that single line is both what the -# pipeline pulls and what this workflow publishes -- the two cannot drift. -# Runs on pushes to a feature branch, whenever the container changes. +# The image tag of the container is read from conf/containers.config, so that the tag is +# what the pipeline pulls and what this workflow publishes. Each container is built independently, +# so updating one tag in conf/containers.config rebuilds only that image: +# feature branch - rebuilds its tag on every push, unless main or devel is pinned to that tag +# main or devel - builds only a tag that has never been published, so most merges are a no-op +# manual run - 'force' rebuilds and overwrites the tag on any branch on: push: paths: - - 'containers/r/**' + - 'containers/**' - 'conf/containers.config' - '.github/workflows/build_container.yml' workflow_dispatch: @@ -22,81 +25,130 @@ concurrency: cancel-in-progress: true jobs: - build: - name: Build and publish + discover: + name: Select containers to build runs-on: ubuntu-latest permissions: contents: read - packages: write + packages: read + outputs: + containers: ${{ steps.select.outputs.containers }} steps: - uses: actions/checkout@v4 - - name: Resolve image from conf/containers.config - id: image + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Select containers to build + id: select run: | - IMAGE=$(sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p" conf/containers.config) - if [ -z "$IMAGE" ]; then - echo "::error::could not parse the 'r' container from conf/containers.config" - exit 1 + RELEASED=false + case "${{ github.ref_name }}" in main|devel) RELEASED=true ;; esac + + # Each base branch is fetched into its own ref so both stay readable in the loop + BASES="" + for BASE in devel main; do + if git fetch --no-tags --depth=1 origin "$BASE:refs/base/$BASE" 2>/dev/null; then + BASES="$BASES refs/base/$BASE" + fi + done + + # devel is at or ahead of main, so it is the baseline for "did this branch edit a container" + DEVEL=refs/base/devel + + # a container is only rebuilt by a push that touches it, so the previous push is the + # baseline for that. devel is the fallback when that commit cannot be fetched, which is + # the case on the first push of a branch, on a force push and on a manual run + PUSH_BASE=$DEVEL + BEFORE="${{ github.event.before }}" + if [ -n "$BEFORE" ] && git fetch --no-tags --depth=1 origin "$BEFORE" 2>/dev/null; then + PUSH_BASE=$BEFORE fi - case "$IMAGE" in - ghcr.io/goekelab/*) ;; - *) echo "::error::refusing to push outside ghcr.io/goekelab: $IMAGE"; exit 1 ;; - esac - echo "image=$IMAGE" >> "$GITHUB_OUTPUT" - echo "Resolved container: $IMAGE" - - - name: Refuse to overwrite a tag main or devel is pinned to - if: github.ref_name != 'main' && github.ref_name != 'devel' - run: | - IMAGE="${{ steps.image.outputs.image }}" - for BASE in main devel; do - git fetch --no-tags --depth=1 origin "$BASE" 2>/dev/null || continue - PINNED=$(git show "FETCH_HEAD:conf/containers.config" 2>/dev/null \ - | sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p") - if [ "$IMAGE" = "$PINNED" ]; then - echo "::error::$BASE is pinned to $IMAGE -- bump the tag in conf/containers.config before rebuilding it" + + # Every containers/