diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04bc352..824eea5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.18 hooks: - - id: ruff + - id: ruff-check args: [--fix, --exit-non-zero-on-fix] - id: ruff-format diff --git a/README.md b/README.md index aee5aeb..59c27c2 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,27 @@ A template for modular data workflows built with [`Snakemake`](https://snakemake.readthedocs.io/en/stable/). This template is part of the [Modelblocks](https://www.modelblocks.org/) toolset. -## Learning resources +> [!NOTE] +> +> Looking for general general information on Modelblocks and modular workflows? +> +> - Visit the Modelblocks [official website](https://www.modelblocks.org/) and [documentation](https://modelblocks.readthedocs.io/en). +> - Read about `Snakemake` modularisation in the [`Snakemake` documentation](https://snakemake.readthedocs.io/en/stable/snakefiles/modularization.html#modules). +> -Looking for general information on Modelblocks or how data modules work? +## Features -- Visit the [Modelblocks](https://www.modelblocks.org/) website and read our [documentation and guidelines](https://modelblocks.readthedocs.io/en). -- Check the auto-generated minimal example. You can find it in `tests/integration/Snakefile`. -- Read about `Snakemake` modularisation in the [`Snakemake` documentation](https://snakemake.readthedocs.io/en/stable/snakefiles/modularization.html#modules). +- Stable `Snakemake` development using `pixi`'s lockfile and conda-pinning functionality, with the following environments: + - `default`: the development environment, including `Snakemake` and `conda` as dependencies. This is never delivered to module users! + - `module`: the environment used by rules in the `Snakemake` workflow. It should only contain minimal dependencies needed by your module's processing steps. -## Features +> [!IMPORTANT] +> +> All software dependencies should be defined in `pixi.toml`. +> Before running your module for the first time, use the `export-snakemake-env` pixi command to export the required `Snakemake` environments to `conda`-compatible dependency files. This is necessary as long as Snakemake does not directly support the use of pixi. +> This must include at least the `module` environment, as well as any additional environments created for this purpose. +> See the [commands section](#pixi-task-commands) for more information. -- Standardised layout compliant with the [`Snakemake` workflow catalogue](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements, so modules can be included automatically once published. Read more about those requirements [here](https://snakemake.github.io/snakemake-workflow-catalog/docs/catalog.html#standardized-usage-workflows). - Standardised input-output structure across modules: - `resources/`: files needed for the module's processes. - `user/`: files that should be provided by users. Document them well! @@ -22,7 +32,7 @@ Looking for general information on Modelblocks or how data modules work? - Continuous Integration (CI) settings, ready for [pre-commit.ci](https://pre-commit.ci/). - Contributor recognition via [All Contributors](https://allcontributors.org/en/). - GitHub Actions to automate chores during pull requests and releases. - - Pre-made `pytest` setup. +- Fully compliant with the [`Snakemake` workflow catalogue](https://snakemake.github.io/snakemake-workflow-catalog/#) listing requirements, so modules can be included automatically once published. Read more about those requirements [here](https://snakemake.github.io/snakemake-workflow-catalog/docs/catalog.html#standardized-usage-workflows). > [!IMPORTANT] > @@ -65,7 +75,8 @@ This template uses [`pixi`](https://pixi.sh/) as its package manager. Once insta ```shell cd ./ # navigate to the new project - pixi install --all # install the project environment + pixi install --all # install the project's environments + pixi run export-snakemake-env module # initialise the Snakemake environment ``` 5. Register your project in [pre-commit.ci](https://pre-commit.ci/) and [allcontributors.org](https://allcontributors.org/en/) to benefit from CI and contributor task automation. 6. Extra: run the auto-generated example module! @@ -75,6 +86,19 @@ This template uses [`pixi`](https://pixi.sh/) as its package manager. Once insta pixi run snakemake --use-conda # run it! ``` +## `pixi` task commands + +### `pixi run export-snakemake-env ` + +Export `` to `conda`-compatible dependency files, saved in `workflow/envs`, allowing `Snakemake` to use them during rule execution. +This will generate both an `.yaml` file and platform-specific pin files for Windows, Linux and macOS (e.g., `.win-64.pin.txt`). + +### `pixi run test-integration` + +Run a minimal set of standardised tests to ensure your module complies with Modelblock requirements. +These are executed by Github's CI during pull requests. + + ## Contributors ✨ Thanks goes to these wonderful people, sorted alphabetically ([emoji key](https://allcontributors.org/en/reference/emoji-key/)): diff --git a/template/.gitignore.jinja b/template/.gitignore.jinja index f03928b..7210c19 100644 --- a/template/.gitignore.jinja +++ b/template/.gitignore.jinja @@ -12,7 +12,8 @@ __pycache__ *.pyc ### Environments -.pixi/ +.pixi/* +!.pixi/.gitignore ### Snakemake .snakemake/ diff --git a/template/.pixi/.gitignore.jinja b/template/.pixi/.gitignore.jinja new file mode 100644 index 0000000..8399391 --- /dev/null +++ b/template/.pixi/.gitignore.jinja @@ -0,0 +1,3 @@ +* +!.gitignore +!config.toml diff --git a/template/.pixi/config.toml b/template/.pixi/config.toml new file mode 100644 index 0000000..a6a577c --- /dev/null +++ b/template/.pixi/config.toml @@ -0,0 +1 @@ +pinning-strategy = "latest-up" diff --git a/template/.pre-commit-config.yaml b/template/.pre-commit-config.yaml index 222f40b..7ae919a 100644 --- a/template/.pre-commit-config.yaml +++ b/template/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.18 hooks: - - id: ruff + - id: ruff-check args: [--fix, --exit-non-zero-on-fix] - id: ruff-format diff --git a/template/INTERFACE.yaml b/template/INTERFACE.yaml index 1283095..4d51cce 100644 --- a/template/INTERFACE.yaml +++ b/template/INTERFACE.yaml @@ -1,4 +1,6 @@ # Module Input-Output structure for automated docs generation +convention_version: v1.0.0 + pathvars: snakemake_defaults: logs: diff --git a/template/README.md.jinja b/template/README.md.jinja index 2e072ef..d6cb8fa 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -44,6 +44,21 @@ cd {{module_short_name}} pixi install --all ``` +Please be aware that this is a multi-environment project (see [pixi.toml](./pixi.toml) for details). +- `default`: used for development and integration testing. +Because it contains `Snakemake`, `conda` and `pytest` as dependencies it **should not be used** in `Snakemake` rules. +- `module`: contains minimal dependencies used in `Snakemake` rules. +If modified, be sure to export it to `Snakemake` so it can be recreated by module users: + +```shell +# create module.yaml and conda-spec pin files in workflow/envs/ +pixi run export-snakemake-env module +``` + + +## Testing + + For testing, simply run: ```shell diff --git a/template/pixi.toml.jinja b/template/pixi.toml.jinja index 207e14e..80fdab3 100644 --- a/template/pixi.toml.jinja +++ b/template/pixi.toml.jinja @@ -9,7 +9,7 @@ platforms = ["win-64", "linux-64", "osx-arm64"] homepage = "https://www.modelblocks.org/" [dependencies] -clio-tools = ">=2026.03.30" +clio-tools = ">=2026.06.23" conda = ">=25.0.0" ipdb = ">=0.13.13" ipykernel = ">=6.29.5" @@ -22,5 +22,32 @@ snakefmt = ">=0.10.2" snakemake-minimal = ">=9.19.0" pytz = ">=2026.1.post1" +[feature.module.dependencies] +curl = ">=8.9.1" + +[environments] +module = { features = ["module"], no-default-feature = true } + [tasks] test-integration = {cmd = "pytest tests/integration_test.py"} + +{% raw %} +[tasks.export-snakemake-env] +description = "Export one Pixi environment as Snakemake-compatible conda files" +args = ["env", { arg = "outdir", default = "workflow/envs" }] +depends-on = [ + { task = "_export-snakemake-pin", args = ["{{ env }}", "win-64", "{{ outdir }}"] }, + { task = "_export-snakemake-pin", args = ["{{ env }}", "linux-64", "{{ outdir }}"] }, + { task = "_export-snakemake-pin", args = ["{{ env }}", "osx-arm64", "{{ outdir }}"] }, +] +cmd = "pixi workspace export conda-environment --environment '{{ env }}' '{{ outdir }}/{{ env }}.yaml'" + +[tasks._export-snakemake-pin] +description = "Export one Pixi environment/platform as a Snakemake-compatible pin file" +args = ["env", "platform", { arg = "outdir", default = "workflow/envs" }] +cmd = """ +set -e +pixi workspace export conda-explicit-spec --environment '{{ env }}' --platform '{{ platform }}' '{{ outdir }}' +mv '{{ outdir }}/{{ env }}_{{ platform }}_conda_spec.txt' '{{ outdir }}/{{ env }}.{{ platform }}.pin.txt' +""" +{% endraw %} diff --git a/template/tests/conftest.py b/template/tests/conftest.py new file mode 100644 index 0000000..f5a5699 --- /dev/null +++ b/template/tests/conftest.py @@ -0,0 +1,12 @@ +"""Shared pytest fixtures.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def module_path(): + """Parent directory of the project.""" + # If your module needs files in resources/user/, place automated downloads here. + return Path(__file__).parent.parent diff --git a/template/tests/integration_test.py.jinja b/template/tests/integration_test.py similarity index 52% rename from template/tests/integration_test.py.jinja rename to template/tests/integration_test.py index 1dfbcbc..e20dba3 100644 --- a/template/tests/integration_test.py.jinja +++ b/template/tests/integration_test.py @@ -4,6 +4,7 @@ Contents may be updated in future template updates. """ +import json import subprocess from pathlib import Path @@ -12,9 +13,43 @@ @pytest.fixture(scope="module") -def module_path(): - """Parent directory of the project.""" - return Path(__file__).parent.parent +def pixi_environments(module_path) -> dict: + """Pixi environments defined for this project.""" + process = subprocess.run( + ["pixi", "info", "--json"], + check=True, + cwd=module_path, + capture_output=True, + text=True, + ) + return { + environment["name"]: environment + for environment in json.loads(process.stdout)["environments_info"] + } + + +def test_snakemake_environments(module_path, pixi_environments, tmp_path): + """All Snakemake environment files should be based on pixi counterparts.""" + env_dir = module_path / "workflow/envs" + env_files = sorted(env_dir.glob("*.yaml")) + assert env_files, f"No conda environments found in {module_path}." + + for env_file in env_files: + env_name = env_file.stem + + output_dir = tmp_path / env_name + subprocess.run( + ["pixi", "run", "export-snakemake-env", env_name, str(output_dir)], + check=True, + cwd=module_path, + ) + + generated_yaml = output_dir / env_file.name + assert generated_yaml.read_text() == env_file.read_text() + + for platform in pixi_environments[env_name]["platforms"]: + pin_file = env_dir / f"{env_name}.{platform}.pin.txt" + assert pin_file.exists(), f"{env_name} has no conda pins for {platform}" def test_interface_file(module_path): diff --git a/template/workflow/Snakefile b/template/workflow/Snakefile index 1f8f266..7d35eb0 100644 --- a/template/workflow/Snakefile +++ b/template/workflow/Snakefile @@ -45,6 +45,6 @@ rule all: log: stderr="/all.stderr", conda: - "envs/shell.yaml" + "envs/module.yaml" shell: 'echo "This workflow must be called as a snakemake module." > {log.stderr}' diff --git a/template/workflow/envs/shell.yaml b/template/workflow/envs/shell.yaml deleted file mode 100644 index e4b351d..0000000 --- a/template/workflow/envs/shell.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: shell -channels: - - conda-forge - - nodefaults -dependencies: - - curl=8.9.1 diff --git a/template/workflow/rules/automatic.smk b/template/workflow/rules/automatic.smk index bdd8982..a7a37fe 100644 --- a/template/workflow/rules/automatic.smk +++ b/template/workflow/rules/automatic.smk @@ -11,6 +11,6 @@ rule dummy_download: log: "/dummy_download.log", conda: - "../envs/shell.yaml" + "../envs/module.yaml" shell: 'curl -sSLo {output.readme} "{params.url}"' diff --git a/template/workflow/rules/dummy.smk b/template/workflow/rules/dummy.smk index a0ab9a3..c7e7660 100644 --- a/template/workflow/rules/dummy.smk +++ b/template/workflow/rules/dummy.smk @@ -11,6 +11,6 @@ rule dummy_add_text: log: "/dummy_add_text.log", conda: - "../envs/shell.yaml" + "../envs/module.yaml" script: "../scripts/dummy_script.py" diff --git a/tests/template_test.py b/tests/template_test.py index a710ea6..c499056 100644 --- a/tests/template_test.py +++ b/tests/template_test.py @@ -37,6 +37,12 @@ def pixi_built(self, template_project): cwd=template_project, check=True, ) + subprocess.run( + "pixi run export-snakemake-env module", + shell=True, + cwd=template_project, + check=True, + ) return template_project def test_pytest(self, pixi_built):