Trim environments and containers

.github/project-filters.yaml

@@ -1,7 +1,10 @@
 workflow: &workflow
-  - '.github/lib-filters.yaml'
-  - '.github/workflows/lib-tests.yaml'
+  - '.github/project-filters.yaml'
+  - '.github/workflows/project-build-test.yaml'
   - 'container_templates/**'
+  - 'scripts/build_containers.py'
+  - 'pyproject.toml'
+  - 'uv.lock'
 
 data:
   - *workflow
@@ -34,11 +37,12 @@ plots:
   - *workflow
   - 'projects/plots/**'
   - 'libs/ledger/**'
-  - 'libs/prior/**'
+  - 'libs/priors/**'
   - 'libs/utils/**'
 
 train:
   - *workflow
   - 'projects/train/**'
   - 'libs/ledger/**'
   - 'libs/architectures/**'
+  - 'libs/utils/**'

.github/workflows/lib-tests.yaml

@@ -56,7 +56,7 @@ jobs:
           test_dir: ${{ github.workspace }}/libs/${{ matrix.library }}
         run: |
             cd $test_dir
-            uv sync --all-extras
+            uv sync --all-extras --group test
         shell: bash
 
       # run its unit tests inside that environment

.github/workflows/project-build-test.yaml

@@ -20,13 +20,11 @@ on:
     tags:
       - 'v*'
 
-# TODO: currently combining path for nvidia variables 
-# necessary to launch triton in export container
-# with path required by train / data projects. 
-# Should add conditionals in this workflow to set these dynamically
 env:
   REGISTRY: ghcr.io
-  PATH: /opt/env/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/cuda-11.8/bin:/opt/tritonserver/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin
+  # Base PATH for every image. Projects that need more than this 
+  # can extend it via their apptainer.env.
+  PATH: /opt/env/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/cuda/bin:/usr/local/nvidia/bin
 
 jobs:
   changes:
@@ -55,7 +53,7 @@ jobs:
       fail-fast: false
       matrix:
         # For tag pushes, we want to build all projects, not just changed ones
-        project: ${{ startsWith(github.ref, 'refs/tags/') && fromJSON('["data", "online", "train", "infer", "export"]') || fromJSON(needs.changes.outputs.projects) }}
+        project: ${{ startsWith(github.ref, 'refs/tags/') && fromJSON('["data", "online", "train", "infer", "export", "plots"]') || fromJSON(needs.changes.outputs.projects) }}
         exclude:
           - project: 'workflow' 
     steps:

.readthedocs.yaml

@@ -9,15 +9,14 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.12"
 
   jobs:
     post_create_environment:
       # Install uv
       - pip install uv
     post_install:
-      # Install dependencies
-      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv sync --group docs
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:

Snakefile

@@ -2,14 +2,14 @@
 
 Usage, from the root directory:
     snakemake -n                                   # dry-run
-    snakemake --profile pipeline/profiles/condor   # HTCondor
+    snakemake --profile pipeline/profiles/ldg   # HTCondor
     snakemake --profile pipeline/profiles/local    # local execution (dev)
 
 The pipeline config is loaded from pipeline/config/config.yaml by
 default. To use a different config, copy the original, make
 modifications, and run:
 
-    snakemake --configfile my_run.yaml --profile pipeline/profiles/condor
+    snakemake --configfile my_run.yaml --profile pipeline/profiles/ldg
 """
 
 from pathlib import Path

container_templates/micromamba.def

@@ -11,28 +11,39 @@ Stage: build
 # to PATH here makes the project commands findable.
 export PATH=/opt/env/bin:$PATH
 
+@@EXTRA_ENV@@
+
 %post
 mkdir -p /cvmfs /hdfs /gpfs /ceph /hadoop
 
-# install git for pip installation
 apt-get update
-apt-get install -y --no-install-recommends git
-apt-get install -y --no-install-recommends build-essential openssh-client
+apt-get install -y --no-install-recommends git build-essential
 rm -rf /var/lib/apt/lists/*
 
+@@EXTRA_POST@@
+
 # activate micromamba and create environment from lockfile
 micromamba create -p /opt/env -f /opt/aframe/projects/@@PROJECT@@/@@PROJECT@@.conda-lock.yml
 
 # install uv so we can install local deps of deps editably
 micromamba run -p /opt/env python -m \
-    pip install --no-build-isolation uv
+    pip install --no-build-isolation uv==0.9.30
 
-cd /opt/aframe/projects/@@PROJECT@@/
+# Export rather than `uv sync`, so that uv installs into the existing conda
+# env instead of managing its own. Export to pylock.toml rather than
+# requirements.txt so that the index each package was locked from is
+# recorded. This allows torch to be installed from the PyTorch CPU index. 
+cd /opt/aframe
 micromamba run -p /opt/env \
-    uv export --frozen --all-extras -o requirements.txt
+    @@UV_CMD@@ --no-cache -o pylock.toml
 
 micromamba run -p /opt/env \
-    uv pip install -r requirements.txt
+    uv pip install --no-cache -r pylock.toml
+
+rm pylock.toml
+micromamba clean -ay
+
+@@PURGE_BUILD_TOOLS@@
 
 # initialize our shell so that we can execute
 # commands in our environment at run time

container_templates/uv.def

@@ -1,5 +1,8 @@
 Bootstrap: docker
-From: ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+# Important: 0.9.30 is the last uv image built on Debian 12 (bookworm).
+# Every tag afterwards is Debian 13 (trixie), and NVIDIA's Debian 13 repo
+# doesn't have CUDA 12 support, which precludes running on V100s.
+From: ghcr.io/astral-sh/uv:0.9.30-python3.12-bookworm-slim
 
 %files
 @@FILES_BLOCK@@
@@ -16,8 +19,9 @@ cd /opt/aframe/projects/@@PROJECT@@
 # when binding the repo into the container
 export UV_PROJECT_ENVIRONMENT=/opt/env
 
-uv export --frozen --all-extras -o requirements.txt
-uv pip sync requirements.txt --system
+@@UV_CMD@@ --no-cache
+
+@@PURGE_BUILD_TOOLS@@
 
 %environment
 # Append venv dir to PATH so the

libs/architectures/pyproject.toml

@@ -7,20 +7,16 @@ requires-python = ">=3.11,<3.14"
 license = "MIT"
 dependencies = [
     "einops>=0.8,<0.9",
+    "jaxtyping>=0.2",
     "ml4gw>=0.7.2",
-    "h5py>=3.9.0,<4",
+    "h5py>=3.10,<4",
     "numpy>=2",
+    "torch",
 ]
 
 [dependency-groups]
-dev = ["pytest>=8.2.1,<9"]
-
-[tool.uv]
-
-[[tool.uv.index]]
-name = "torch"
-url = "https://download.pytorch.org/whl/cu118"
-explicit = true
+test = ["pytest>=8.2.1,<9"]
+dev = [{ include-group = "test" }, "jupyter>=1.0.0,<2"]
 
 [build-system]
 requires = ["hatchling"]