From 9d029e9bffd1f41c10ea6d595638d7bbe4a50788 Mon Sep 17 00:00:00 2001 From: scverse-bot <108668866+scverse-bot@users.noreply.github.com> Date: Fri, 17 Jul 2026 07:16:47 +0000 Subject: [PATCH] Automated template update to v0.8.0 --- .codecov.yaml | 17 + .cruft.json | 46 +++ .editorconfig | 15 + .github/ISSUE_TEMPLATE/bug_report.yml | 93 ++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 11 + .github/dependabot.yml | 12 + .github/workflows/build.yaml | 30 ++ .github/workflows/release.yaml | 33 ++ .github/workflows/test.yaml | 111 +++++++ .gitignore | 21 ++ .pre-commit-config.yaml | 44 +++ .readthedocs.yaml | 16 + .vscode/extensions.json | 18 ++ .vscode/launch.json | 33 ++ .vscode/settings.json | 18 ++ atlas/__init__.py | 1 - atlas/mapping.py | 260 --------------- atlas/matching.py | 24 -- atlas/stats.py | 153 --------- atlas/wknn.py | 272 ---------------- biome.jsonc | 17 + docs/_static/.gitkeep | 0 docs/_static/css/custom.css | 4 + docs/_templates/.gitkeep | 0 docs/changelog.md | 3 + docs/conf.py | 137 ++++++++ docs/contributing.md | 347 +++++++++++++++++++++ pyproject.toml | 143 +++++++++ 29 files changed, 1174 insertions(+), 710 deletions(-) create mode 100644 .codecov.yaml create mode 100644 .cruft.json create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .readthedocs.yaml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json delete mode 100644 atlas/__init__.py delete mode 100644 atlas/mapping.py delete mode 100644 atlas/matching.py delete mode 100644 atlas/stats.py delete mode 100644 atlas/wknn.py create mode 100644 biome.jsonc create mode 100644 docs/_static/.gitkeep create mode 100644 docs/_static/css/custom.css create mode 100644 docs/_templates/.gitkeep create mode 100644 docs/changelog.md create mode 100644 docs/conf.py create mode 100644 docs/contributing.md create mode 100644 pyproject.toml diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 0000000..d0c0e29 --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,17 @@ +# Based on pydata/xarray +codecov: + require_ci_to_pass: no + +coverage: + status: + project: + default: + # Require 1% coverage, i.e., always succeed + target: 1 + patch: false + changes: false + +comment: + layout: diff, flags, files + behavior: once + require_base: no diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 0000000..c0e8b52 --- /dev/null +++ b/.cruft.json @@ -0,0 +1,46 @@ +{ + "template": "/tmp/tmpvf8ta0q7", + "commit": "6518dfa1abde7379ea7255daf0ce09c23f2b4c94", + "checkout": "v0.8.0", + "context": { + "cookiecutter": { + "project_name": "hnoca", + "package_name": "hnoca", + "project_description": "Human Neural Organoid Cell Atlas Toolbox", + "author_full_name": "Marius Lange", + "author_email": "marius.lange@bsse.ethz.ch", + "github_user": "devsystemslab", + "github_repo": "HNOCA-tools", + "license": "GPLv3", + "ide_integration": true, + "issue_categorization": "", + "_copy_without_render": [ + ".github/workflows/build.yaml", + ".github/workflows/test.yaml", + "docs/_templates/autosummary/**.rst" + ], + "_exclude_on_template_update": [ + "CHANGELOG.md", + "LICENSE", + "README.md", + "docs/api.md", + "docs/index.md", + "docs/notebooks/example.ipynb", + "docs/references.bib", + "docs/references.md", + "src/**", + "tests/**" + ], + "_render_devdocs": false, + "_jinja2_env_vars": { + "lstrip_blocks": true, + "trim_blocks": true + }, + "_template": "/tmp/tmpvf8ta0q7", + "_commit": "4d6aa34fd5be7523ca3bd49fbd81a882dde01d1f" + }, + "_commit": "6518dfa1abde7379ea7255daf0ce09c23f2b4c94", + "_template": "https://github.com/scverse/cookiecutter-scverse" + }, + "directory": null +} \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..66678e3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[{*.{yml,yaml,toml},.cruft.json}] +indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..cc32eda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,93 @@ +name: Bug report +description: Report something that is broken or incorrect +labels: bug +body: + - type: markdown + attributes: + value: | + **Note**: Please read [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) + detailing how to provide the necessary information for us to reproduce your bug. In brief: + * Please provide exact steps how to reproduce the bug in a clean Python environment. + * In case it's not clear what's causing this bug, please provide the data or the data generation procedure. + * Replicate problems on public datasets or share data subsets when full sharing isn't possible. + + - type: textarea + id: report + attributes: + label: Report + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: versions + attributes: + label: Versions + description: | + Which version of packages. + + Please install `session-info2`, run the following command in a notebook, + click the “Copy as Markdown” button, then paste the results into the text box below. + + ```python + In[1]: import session_info2; session_info2.session_info(dependencies=True) + ``` + + Alternatively, run this in a console: + + ```python + >>> import session_info2; print(session_info2.session_info(dependencies=True)._repr_mimebundle_()["text/markdown"]) + ``` + render: python + placeholder: | + anndata 0.11.3 + ---- ---- + charset-normalizer 3.4.1 + coverage 7.7.0 + psutil 7.0.0 + dask 2024.7.1 + jaraco.context 5.3.0 + numcodecs 0.15.1 + jaraco.functools 4.0.1 + Jinja2 3.1.6 + sphinxcontrib-jsmath 1.0.1 + sphinxcontrib-htmlhelp 2.1.0 + toolz 1.0.0 + session-info2 0.1.2 + PyYAML 6.0.2 + llvmlite 0.44.0 + scipy 1.15.2 + pandas 2.2.3 + sphinxcontrib-devhelp 2.0.0 + h5py 3.13.0 + tblib 3.0.0 + setuptools-scm 8.2.0 + more-itertools 10.3.0 + msgpack 1.1.0 + sparse 0.15.5 + wrapt 1.17.2 + jaraco.collections 5.1.0 + numba 0.61.0 + pyarrow 19.0.1 + pytz 2025.1 + MarkupSafe 3.0.2 + crc32c 2.7.1 + sphinxcontrib-qthelp 2.0.0 + sphinxcontrib-serializinghtml 2.0.0 + zarr 2.18.4 + asciitree 0.3.3 + six 1.17.0 + sphinxcontrib-applehelp 2.0.0 + numpy 2.1.3 + cloudpickle 3.1.1 + sphinxcontrib-bibtex 2.6.3 + natsort 8.4.0 + jaraco.text 3.12.1 + setuptools 76.1.0 + Deprecated 1.2.18 + packaging 24.2 + python-dateutil 2.9.0.post0 + ---- ---- + Python 3.13.2 | packaged by conda-forge | (main, Feb 17 2025, 14:10:22) [GCC 13.3.0] + OS Linux-6.11.0-109019-tuxedo-x86_64-with-glibc2.39 + Updated 2025-03-18 15:47 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..5b62547 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Scverse Community Forum + url: https://discourse.scverse.org/ + about: If you have questions about “How to do X”, please ask them here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..b988078 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,11 @@ +name: Feature request +description: Propose a new feature for hnoca +labels: enhancement +body: + - type: textarea + id: description + attributes: + label: Description of feature + description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered. + validations: + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..914ad69 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + actions-deps: + patterns: + - "*" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..6cca7d4 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,30 @@ +name: Check Build + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Build package + run: uv build + - name: Check package + run: uvx twine check --strict dist/*.whl diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..1dc8b03 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,33 @@ +name: Release + +on: + release: + types: [published] + +# Use "trusted publishing", see https://docs.pypi.org/trusted-publishers/ +permissions: {} + +jobs: + release: + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/hnoca + permissions: + contents: read + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + - name: Build package + run: uv build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..0075425 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,111 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 5 1,15 * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # Get the test environment from hatch as defined in pyproject.toml. + # This ensures that the pyproject.toml is the single point of truth for test definitions and the same tests are + # run locally and on continuous integration. + # Check [[tool.hatch.envs.hatch-test.matrix]] in pyproject.toml and https://hatch.pypa.io/latest/environment/ for + # more details. + get-environments: + runs-on: ubuntu-slim + outputs: + envs: ${{ steps.get-envs.outputs.envs }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Get test environments + id: get-envs + run: | + ENVS_JSON=$(uvx hatch env show --json | jq -c 'to_entries + | map( + select(.key | startswith("hatch-test")) + | { + name: .key, + label: (if (.key | contains("pre")) then .key + " (PRE-RELEASE DEPENDENCIES)" else .key end), + python: .value.python + } + )') + echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT + + # Run tests through hatch. Spawns a separate runner for each environment defined in the hatch matrix obtained above. + test: + needs: get-environments + permissions: + id-token: write # for codecov OIDC + contents: read + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + env: ${{ fromJSON(needs.get-environments.outputs.envs) }} + + name: ${{ matrix.env.label }} + runs-on: ${{ matrix.os }} + continue-on-error: ${{ contains(matrix.env.name, 'pre') }} # make "all-green" pass even if pre-release job fails + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + python-version: ${{ matrix.env.python }} + - name: create hatch environment + run: uvx hatch env create ${{ matrix.env.name }} + - name: list all all installed package versions + run: uvx hatch run ${{ matrix.env.name }}:uv pip list + - name: run tests using hatch + env: + MPLBACKEND: agg + PLATFORM: ${{ matrix.os }} + DISPLAY: :42 + run: uvx hatch run ${{ matrix.env.name }}:run-cov -v --color=yes -n auto + - name: generate coverage report + run: | + # See https://coverage.readthedocs.io/page/config.html#run-patch + test -f .coverage || uvx hatch run ${{ matrix.env.name }}:cov-combine + uvx hatch run ${{ matrix.env.name }}:cov-report # report visibly + uvx hatch run ${{ matrix.env.name }}:coverage xml # create report for upload + - name: Upload coverage + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + fail_ci_if_error: true + use_oidc: true + + # Check that all tests defined above pass. This makes it easy to set a single "required" test in branch + # protection instead of having to update it frequently. See https://github.com/re-actors/alls-green#why. + check: + name: Tests pass in all hatch environments + if: always() + needs: + - get-environments + - test + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bd24e4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Temp files +.DS_Store +*~ +buck-out/ + +# Compiled files +.venv/ +__pycache__/ +.*cache/ + +# Distribution / packaging +/dist/ + +# Tests and coverage +/data/ +/node_modules/ +/.coverage* + +# docs +/docs/generated/ +/docs/_build/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3a84322 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +fail_fast: false +default_language_version: + python: python3 +default_stages: + - pre-commit + - pre-push +minimum_pre_commit_version: 2.16.0 +repos: + - repo: https://github.com/biomejs/pre-commit + rev: v2.5.3 + hooks: + - id: biome-format + exclude: ^\.cruft\.json$ # inconsistent indentation with cruft - file never to be modified manually. + - repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.25.3 + hooks: + - id: pyproject-fmt + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.21 + hooks: + - id: ruff-check + types_or: [python, pyi, jupyter] + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + types_or: [python, pyi, jupyter] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: detect-private-key + - id: check-ast + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + - id: check-case-conflict + # Check that there are no merge conflicts (could be generated by template sync) + - id: check-merge-conflict + args: [--assume-in-merge] + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..67eb948 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +# https://docs.readthedocs.io/page/config-file/v2.html +version: 2 +build: + os: ubuntu-24.04 + tools: + python: "3.14" + nodejs: latest + jobs: + create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - uvx hatch run docs:build + - mv docs/_build $READTHEDOCS_OUTPUT diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..caaeb4f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,18 @@ +{ + "recommendations": [ + // GitHub integration + "github.vscode-github-actions", + "github.vscode-pull-request-github", + // Language support + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "tamasfe.even-better-toml", + // Dependency management + "ninoseki.vscode-mogami", + // Linting and formatting + "editorconfig.editorconfig", + "charliermarsh.ruff", + "biomejs.biome", + ], +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..36d1874 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,33 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Build Documentation", + "type": "debugpy", + "request": "launch", + "module": "sphinx", + "args": ["-M", "html", ".", "_build"], + "cwd": "${workspaceFolder}/docs", + "console": "internalConsole", + "justMyCode": false, + }, + { + "name": "Python: Debug Test", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "purpose": ["debug-test"], + "console": "internalConsole", + "justMyCode": false, + "env": { + "PYTEST_ADDOPTS": "--color=yes", + }, + "presentation": { + "hidden": true, + }, + }, + ], +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e034b91 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "[python][json][jsonc]": { + "editor.formatOnSave": true, + }, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.codeActionsOnSave": { + "source.fixAll": "always", + "source.organizeImports": "always", + }, + }, + "[json][jsonc]": { + "editor.defaultFormatter": "biomejs.biome", + }, + "python.analysis.typeCheckingMode": "basic", + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["-vv", "--color=yes"], +} diff --git a/atlas/__init__.py b/atlas/__init__.py deleted file mode 100644 index 2a52429..0000000 --- a/atlas/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .mapping import AtlasMapper diff --git a/atlas/mapping.py b/atlas/mapping.py deleted file mode 100644 index 6568d3e..0000000 --- a/atlas/mapping.py +++ /dev/null @@ -1,260 +0,0 @@ -import os -import sys - -import cloudpickle - -from typing import Literal, Optional - -import numpy as np -import pandas as pd - -import scvi -from scvi.model.utils import mde - -import scarches -import scanpy as sc - -from .wknn import get_wknn, transfer_labels, estimate_presence_score -from .matching import get_matched_transcriptome - - -class AtlasMapper: - def __init__(self, ref_model): - self.model_type = self._check_model_type(ref_model) - self.ref_model = ref_model - self.ref_adata = ref_model.adata - self.query_model = None - self.ref_trans_prob = None - - def map_query(self, query_adata, retrain="partial", **kwargs): - """ - Map a query dataset to the reference dataset - - Parameters - ---------- - query_adata : AnnData - The query dataset to map to the reference dataset - query_model : str - The model to use for the query dataset - retrain : str - Whether to retrain the query model. Options are "partial", "full" or "none" - """ - if retrain in ["partial", "full"]: - if self.model_type == "scanvi": - self._train_scanvi(query_adata, retrain, **kwargs) - if self.model_type == "scvi": - self._train_scvi(query_adata, retrain, **kwargs) - if self.model_type == "scpoli": - self._train_scpoli(query_adata, **kwargs) - self.query_adata = self.query_model.adata - else: - self.query_model = self.ref_model - self.query_adata = query_adata - - def _train_scanvi(self, query_adata, retrain="partial", **kwargs): - """ - Train a new scanvi model on the query data - """ - unfrozen = retrain == "full" - scvi.model.SCANVI.prepare_query_anndata(query_adata, self.ref_model) - vae_q = scvi.model.SCANVI.load_query_data( - query_adata, self.ref_model, unfrozen=unfrozen - ) - vae_q.train(**kwargs) - - self.query_model = vae_q - - def _train_scvi(self, query_adata, query_model, retrain="partial", **kwargs): - """ - Train a new scvi model on the query data - """ - unfrozen = retrain == "full" - scvi.model.SCVI.prepare_query_anndata(query_adata, self.ref_model) - vae_q = scvi.model.SCVI.load_query_data( - query_adata, self.ref_model, unfrozen=unfrozen - ) - vae_q.train(**kwargs) - - self.query_model = vae_q - - def _train_scpoli(self, query_adata, query_model, **kwargs): - """ - Train a new scpoli model on the query data - """ - freeze = retrain != "full" - labeled_indices = np.arange(query_adata.X.shape[0]) - - vae_q = scarches.models.scPoli.load_query_data( - query_adata, - reference_model=self.ref_model, - unknown_ct_names=["Unknown"], - labeled_indices=labeled_indices, - ) - - vae_q.train(**kwargs) - - self.query_model = vae_q - - def _check_model_type(self, model): - if isinstance(model, scvi.model._scanvi.SCANVI): - return "scanvi" - elif isinstance(model, scvi.model._scvi.SCVI): - return "scvi" - elif isinstance(model, scarches.models.scpoli.scPoli): - return "scpoli" - else: - raise RuntimeError("This VAE model is currently not supported") - - def _get_latent(self, model, adata, **kwargs): - if self.model_type in ["scanvi", "scanvi"]: - return model.get_latent_representation(adata, **kwargs) - if self.model_type == "scpoli": - return model.get_latent(adata, **kwargs) - - def compute_wknn( - self, - ref_adata=None, - k: int = 100, - query2ref: bool = True, - ref2query: bool = False, - weighting_scheme: Literal[ - "n", "top_n", "jaccard", "jaccard_square", "gaussian", "dist" - ] = "jaccard_square", - top_n: Optional[int] = None, - ): - """ - Compute the weighted k-nearest neighbors graph between the reference and query datasets - - Parameters - ---------- - k : int - Number of neighbors per cell - query2ref : bool - Consider query-to-ref neighbors - ref2query : bool - Consider ref-to-query neighbors - weighting_scheme : str - How to weight edges in the ref-query neighbor graph - top_n : int - The number of top neighbors to consider - """ - - self.ref_adata = ref_adata if ref_adata is not None else self.ref_adata - ref_latent = self._get_latent(self.query_model, self.ref_adata) - query_latent = self._get_latent(self.query_model, self.query_adata) - - wknn = get_wknn( - ref=ref_latent, - query=query_latent, - k=k, - query2ref=query2ref, - ref2query=ref2query, - weighting_scheme=weighting_scheme, - top_n=top_n, - ) - - self.wknn = wknn - self.ref_adata.obsm["X_latent"] = ref_latent - self.query_adata.obsm["X_latent"] = query_latent - - def estimate_presence_scores( - self, split_by=None, random_walk=True, alpha=0.1, n_rounds=100, log=True - ): - """ - Estimate the presence score of the query dataset - - Parameters - ---------- - split_by : str - The column in the query dataset to split by - random_walk : bool - Whether to use random walk to estimate presence score - alpha : float - The heat diffusion parameter for the random walk - n_rounds : int - The number of rounds for the random walk - log : bool - Whether to log the presence score - """ - - scores = estimate_presence_score( - self.ref_adata, - self.query_adata, - self.wknn, - use_rep_ref_wknn="X_latent", - use_rep_query_wknn="X_latent", - ref_trans_prop=self.ref_trans_prob, - split_by=split_by, - alpha_random_walk=alpha, - num_rounds_random_walk=n_rounds, - log=log, - ) - - self.ref_trans_prob = scores["ref_trans_prop"] - return scores - - def transfer_labels(self, label_key): - """ - Transfer labels from the reference dataset to the query dataset - - Parameters - ---------- - label_key : str - The column in the reference dataset to transfer - """ - - scores = transfer_labels( - self.ref_adata, - self.query_adata, - self.wknn, - label_key=label_key, - ) - - return scores - - def get_matched_expression(self, rescale_factor=1): - """ - Get the expression of reference cells matched to query cells. This can be used for quantitative comparisons like DE analysis. - - Parameters - ---------- - layer: str - If not None, uses this as the key in adata.layers to return the reference transcriptome. - rescale_factor: str - Factor to rescale the log-normalized counts - """ - matched_adata = get_matched_transcriptome( - self.query_adata, - self.ref_adata, - self.wknn, - rescale_factor=rescale_factor, - ) - self.matched_adata = matched_adata - return matched_adata - - def save(self, output_dir): - """ - Save the mapper object to disk - - Parameters - ---------- - output_dir : str - The directory to save the mapper object - """ - os.makedirs(output_dir, exist_ok=True) - with open(os.path.join(output_dir, "mapper.pkl"), "wb") as f: - cloudpickle.dump(self, f) - - @classmethod - def load(cls, input_dir): - """ - Load the mapper object from disk - - Parameters - ---------- - input_dir : str - The directory to load the mapper object - """ - with open(os.path.join(input_dir, "mapper.pkl"), "rb") as f: - mapper = cloudpickle.load(f) - return mapper diff --git a/atlas/matching.py b/atlas/matching.py deleted file mode 100644 index ab278d8..0000000 --- a/atlas/matching.py +++ /dev/null @@ -1,24 +0,0 @@ -import anndata -import numpy as np -import scanpy as sc -from scipy import sparse - - -def get_matched_transcriptome( - adata, - adata_ref, - wknn, - rescale_factor=1, -): - expr_ref = adata_ref.X.copy() - if rescale_factor != 1: - expr_ref = ((expr_ref.expm1()) * rescale_factor).log1p() - - normmat = sparse.diags(1 / np.array(wknn.sum(axis=1)).flatten()) - softmax_wknn = normmat.dot(wknn) - expr_bg = softmax_wknn @ expr_ref - - adata_bg = anndata.AnnData( - expr_bg, obs=adata.obs.copy(), var=adata_ref.var.copy(), obsm=adata.obsm.copy() - ) - return adata_bg diff --git a/atlas/stats.py b/atlas/stats.py deleted file mode 100644 index d1c2546..0000000 --- a/atlas/stats.py +++ /dev/null @@ -1,153 +0,0 @@ -import numpy as np -import pandas as pd -from scipy.sparse import issparse -from scipy.stats import f -from statsmodels.formula.api import ols -from statsmodels.stats.anova import anova_lm -from joblib import Parallel, delayed -import multiprocessing -from tqdm import tqdm - - -def anova(e, group, covar=None, return_coef_group=None): - if issparse(e): - e = e.todense() - e = np.array(e).flatten() - - if covar is None: - m0 = ols("e ~ 1", data=pd.DataFrame({"e": e})).fit() - m1 = ols("e ~ zz_group", data=pd.DataFrame({"e": e, "zz_group": group})).fit() - else: - data = pd.DataFrame({"e": e, **covar, "zz_group": group}) - m0 = ols("e ~ " + " + ".join(covar.columns), data=data).fit() - m1 = ols("e ~ " + " + ".join(covar.columns) + " + zz_group", data=data).fit() - - a0 = anova_lm(m0) - a1 = anova_lm(m1) - a01 = anova_lm(m0, m1) - p1 = a01.loc[:, "Pr(>F)"].iloc[-1] - p2_cdf = f.cdf( - a0.loc["Residual", "mean_sq"] / a1.loc["Residual", "mean_sq"], - a0.loc["Residual", "df"], - a1.loc["Residual", "df"], - ) - p2 = 1 - np.abs(0.5 - p2_cdf) * 2 - - var_tot = a0["sum_sq"].sum() - var_covar = a0.iloc[:-1]["sum_sq"].sum() - var_g = a1.loc["zz_group", "sum_sq"] - - if ( - return_coef_group is not None - and return_coef_group in pd.Series(group).unique() - and f"zz_group[T.{return_coef_group}]" in m1.params.index - ): - coef_g = m1.params[f"zz_group[T.{return_coef_group}]"] - else: - coef_g = np.nan - - return (var_g, var_covar, var_tot, p1, p2, coef_g) - - -def ancova_group_test( - expr, group, covar=None, num_threads=1, return_coef_group=None, var_names=None -): - if not isinstance(group, pd.Series): - group = pd.Series(group) - idx = np.where(~group.isnull())[0] - expr = expr[idx, :] - group = group[idx] - if covar is not None: - covar = pd.DataFrame(covar).iloc[idx, :] - - if group.dtype.name != "category": - group = group.astype("category") - group = group.cat.remove_unused_categories() - g_lev = np.array(pd.Series(group).astype("category").cat.categories) - if return_coef_group is not None and return_coef_group in g_lev: - g_lev = np.concatenate([g_lev[g_lev != return_coef_group], [return_coef_group]]) - group = group.cat.reorder_categories(g_lev) - - num_cores = min(num_threads, multiprocessing.cpu_count()) - if num_cores > 1: - results = Parallel(n_jobs=num_cores)( - delayed(anova)(expr[:, i], group, covar, return_coef_group) - for i in range(expr.shape[1]) - ) - else: - results = list() - for i in tqdm(range(expr.shape[1])): - results.append( - anova( - expr[:, i], group, covar=covar, return_coef_group=return_coef_group - ) - ) - - df_res = pd.DataFrame( - results, - columns=["var_group", "var_covar", "var_total", "p_ANCOVA", "p_Resi", "coef"], - ) - if var_names is not None and len(var_names) == df_res.shape[0]: - df_res.index = var_names - else: - df_res.index = pd.RangeIndex(start=0, stop=expr.shape[1], step=1) - return df_res - - -def f_nonzero(e, covar=None): - if issparse(e): - e = e.todense() - e = np.array(e).flatten() - - if covar is None: - m1 = ols("e ~ 1", data=pd.DataFrame({"e": e})).fit() - a1 = anova_lm(m1) - fval = (np.sum(e**2) / len(e)) / a1.loc["Residual", "mean_sq"] - p = 1 - np.abs(0.5 - f.cdf(fval, len(e), a1.loc["Residual", "df"])) * 2 - coef = m1.params["Intercept"] - else: - data = pd.DataFrame({"e": e, **covar}) - m0 = ols("e ~ " + " + ".join(covar.columns) + " -1", data=data).fit() - a0 = anova_lm(m0) - m1 = ols("e ~ " + " + ".join(covar.columns), data=data).fit() - a1 = anova_lm(m1) - sumsq0 = a0.loc["Residual", "sum_sq"] + ( - a0.iloc[:-1]["sum_sq"].sum() - a1.iloc[:-1]["sum_sq"].sum() - ) - meansq0 = sumsq0 / (a0.loc["Residual", "df"] + 1) - meansq1 = a1.loc["Residual", "mean_sq"] - fval = meansq0 / meansq1 - p = ( - 1 - - np.abs( - 0.5 - - f.cdf(fval, a0.loc["Residual", "df"] + 1, a1.loc["Residual", "df"]) - ) - * 2 - ) - coef = m1.params["Intercept"] - - return (fval, coef, p) - - -def f_nonzero_test(expr, covar=None, num_threads=1, var_names=None): - if covar is not None: - covar = pd.DataFrame(covar) - - num_cores = min(num_threads, multiprocessing.cpu_count()) - if num_cores > 1: - results = Parallel(n_jobs=num_cores)( - delayed(f_nonzero)(expr[:, i], covar) for i in range(expr.shape[1]) - ) - else: - results = list() - for i in tqdm(range(expr.shape[1])): - results.append(f_nonzero(expr[:, i], covar=covar)) - - df_res = pd.DataFrame(results, columns=["f", "coef", "pval"]) - if var_names is not None and len(var_names) == df_res.shape[0]: - df_res.index = var_names - else: - df_res.index = pd.RangeIndex(start=0, stop=expr.shape[1], step=1) - return df_res - diff --git a/atlas/wknn.py b/atlas/wknn.py deleted file mode 100644 index fde541b..0000000 --- a/atlas/wknn.py +++ /dev/null @@ -1,272 +0,0 @@ -import scanpy as sc -import numpy as np -import pandas as pd -import anndata as ad -import torch -from pynndescent import NNDescent - -from scipy import sparse -from typing import Optional, Union, Mapping, Literal -import warnings -import sys -import os -import importlib.util -import argparse - -warnings.filterwarnings("ignore") - - -def nn2adj(nn, n1=None, n2=None): - if n1 is None: - n1 = nn[1].shape[0] - if n2 is None: - n2 = np.max(nn[1].flatten()) - - df = pd.DataFrame( - { - "i": np.repeat(range(nn[1].shape[0]), nn[1].shape[1]), - "j": nn[1].flatten(), - "x": nn[0].flatten(), - } - ) - adj = sparse.csr_matrix( - (np.repeat(1, df.shape[0]), (df["i"], df["j"])), shape=(n1, n2) - ) - return adj - - -def build_nn( - ref, - query=None, - k=100, - weight: Literal["unweighted", "dist", "gaussian_kernel"] = "unweighted", - sigma=None, -): - if query is None: - query = ref - - if torch.cuda.is_available() and importlib.util.find_spec("cuml"): - print("GPU detected and cuml installed. Use cuML for neighborhood estimation.") - from cuml.neighbors import NearestNeighbors - - model = NearestNeighbors(n_neighbors=k) - model.fit(ref) - knn = model.kneighbors(query) - else: - print( - "Failed calling cuML. Falling back to neighborhood estimation using CPU with pynndescent." - ) - index = NNDescent(ref) - knn = index.query(query, k=k) - - adj = nn2adj(knn, n1=query.shape[0], n2=ref.shape[0]) - return adj - - -def build_mutual_nn(dat1, dat2=None, k1=100, k2=None): - if dat2 is None: - dat2 = dat1 - if k2 is None: - k2 = k1 - - adj_12 = build_nn(dat1, dat2, k=k2) - adj_21 = build_nn(dat2, dat1, k=k1) - - adj_mnn = adj_12.multiply(adj_21.T) - return adj_mnn - - -def get_transition_prob_mat(dat, k=50, symm=True): - adj = build_nn(dat, k=k) - if symm: - adj = adj + adj.transpose() - prob = sparse.diags(1 / np.array(adj.sum(1)).flatten()) @ adj.transpose() - return prob - - -def random_walk_with_restart(init, transition_prob, alpha=0.5, num_rounds=100): - init = np.array(init).flatten() - heat = init[:, None] - for i in range(num_rounds): - heat = init[:, None] * alpha + (1 - alpha) * ( - transition_prob.transpose() @ heat - ) - return heat - - -def get_wknn( - ref, - query, - ref2=None, - k: int = 100, - query2ref: bool = True, - ref2query: bool = True, - weighting_scheme: Literal[ - "n", "top_n", "jaccard", "jaccard_square", "gaussian", "dist" - ] = "jaccard_square", - top_n: Optional[int] = None, - return_adjs: bool = False, -): - """ - Compute the weighted k-nearest neighbors graph between the reference and query datasets - - Parameters - ---------- - ref : np.ndarray - The reference representation to build ref-query neighbor graph - query : np.ndarray - The query representation to build ref-query neighbor graph - ref2 : np.ndarray - The reference representation to build ref-ref neighbor graph - k : int - Number of neighbors per cell - query2ref : bool - Consider query-to-ref neighbors - ref2query : bool - Consider ref-to-query neighbors - weighting_scheme : str - How to weight edges in the ref-query neighbor graph - top_n : int - The number of top neighbors to consider - return_adjs : bool - Whether to return the adjacency matrices of ref-query, query-ref, ref-ref, and ref-ref for weighting - """ - adj_q2r = build_nn(ref=ref, query=query, k=k) - - adj_r2q = None - if ref2query: - adj_r2q = build_nn(ref=query, query=ref, k=k) - - if query2ref and not ref2query: - adj_knn = adj_q2r.T - elif ref2query and not query2ref: - adj_knn = adj_r2q - elif ref2query and query2ref: - adj_knn = ((adj_r2q + adj_q2r.T) > 0) + 0 - else: - warnings.warn( - "At least one of query2ref and ref2query should be True. Reset to default with both being True." - ) - adj_knn = ((adj_r2q + adj_q2r.T) > 0) + 0 - - if ref2 is None: - ref2 = ref - adj_ref = build_nn(ref=ref2, k=k) - num_shared_neighbors = adj_q2r @ adj_ref.T - num_shared_neighbors_nn = num_shared_neighbors.multiply(adj_knn.T) - - wknn = num_shared_neighbors_nn.copy() - if weighting_scheme == "top_n": - if top_n is None: - top_n = k // 4 if k > 4 else 1 - wknn = (wknn > top_n) * 1 - elif weighting_scheme == "jaccard": - wknn.data = wknn.data / (k + k - wknn.data) - elif weighting_scheme == "jaccard_square": - wknn.data = (wknn.data / (k + k - wknn.data)) ** 2 - - if return_adjs: - adjs = {"q2r": adj_q2r, "r2q": adj_r2q, "knn": adj_knn, "r2r": adj_ref} - return (wknn, adjs) - else: - return wknn - - -def estimate_presence_score( - ref_adata, - query_adata, - wknn=None, - use_rep_ref_wknn="X_latent", - use_rep_query_wknn="X_latent", - k_wknn=100, - query2ref_wknn=True, - ref2query_wknn=False, - weighting_scheme_wknn="jaccard_square", - ref_trans_prop=None, - use_rep_ref_trans_prop=None, - k_ref_trans_prop=50, - symm_ref_trans_prop=True, - split_by=None, - do_random_walk=True, - alpha_random_walk=0.1, - num_rounds_random_walk=100, - log=True, -): - if wknn is None: - ref = ref_adata.obsm[use_rep_ref_wknn] - query = query_adata.obsm[use_rep_query_wknn] - wknn = get_wknn( - ref=ref, - query=query, - k=k_wknn, - query2ref=query2ref_wknn, - ref2query=ref2query_wknn, - weighting_scheme=weighting_scheme_wknn, - ) - - if ref_trans_prop is None and do_random_walk: - if use_rep_ref_trans_prop is None: - use_rep_ref_trans_prop = use_rep_ref_wknn - ref = ref_adata.obsm[use_rep_ref_trans_prop] - ref_trans_prop = get_transition_prob_mat(ref, k=k_ref_trans_prop) - - if split_by and split_by in query_adata.obs.columns: - presence_split = [ - np.array(wknn[query_adata.obs[split_by] == x, :].sum(axis=0)).flatten() - for x in query_adata.obs[split_by].unique() - ] - else: - presence_split = [np.array(wknn.sum(axis=0)).flatten()] - - if do_random_walk: - presence_split_sm = [ - random_walk_with_restart( - init=x, - transition_prob=ref_trans_prop, - alpha=alpha_random_walk, - num_rounds=num_rounds_random_walk, - ) - for x in presence_split - ] - else: - presence_split_sm = [x[:, None] for x in presence_split] - - columns = ( - query_adata.obs[split_by].unique() - if split_by and split_by in query_adata.obs.columns - else ["query"] - ) - if len(columns) > 1: - df_presence = pd.DataFrame( - np.concatenate(presence_split_sm, axis=1), - columns=columns, - index=ref_adata.obs_names, - ) - else: - df_presence = pd.DataFrame({columns[0]: presence_split_sm[0]}).set_index( - ref_adata.obs_names - ) - - if log: - df_presence = df_presence.apply(lambda x: np.log1p(x), axis=0) - df_presence_norm = df_presence.apply( - lambda x: np.clip(x, np.percentile(x, 1), np.percentile(x, 99)), axis=0 - ).apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)), axis=0) - max_presence = df_presence_norm.max(1) - - return { - "max": max_presence, - "per_group": df_presence_norm, - "ref_trans_prop": ref_trans_prop, - } - - -def transfer_labels(ref_adata, query_adata, wknn, label_key="celltype"): - scores = pd.DataFrame( - wknn @ pd.get_dummies(ref_adata.obs[label_key]), - columns=pd.get_dummies(ref_adata.obs[label_key]).columns, - index=query_adata.obs_names, - ) - scores["best_label"] = scores.idxmax(1) - scores["best_score"] = scores.max(1) - return scores diff --git a/biome.jsonc b/biome.jsonc new file mode 100644 index 0000000..9f8f220 --- /dev/null +++ b/biome.jsonc @@ -0,0 +1,17 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.2.0/schema.json", + "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, + "formatter": { "useEditorconfig": true }, + "overrides": [ + { + "includes": ["./.vscode/*.json", "**/*.jsonc"], + "json": { + "formatter": { "trailingCommas": "all" }, + "parser": { + "allowComments": true, + "allowTrailingCommas": true, + }, + }, + }, + ], +} diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 0000000..b8c8d47 --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,4 @@ +/* Reduce the font size in data frames - See https://github.com/scverse/cookiecutter-scverse/issues/193 */ +div.cell_output table.dataframe { + font-size: 0.8em; +} diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..d9e79ba --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,3 @@ +```{include} ../CHANGELOG.md + +``` diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..905ba69 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,137 @@ +# Configuration file for the Sphinx documentation builder. + +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/page/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- +import shutil +import sys +from datetime import datetime +from importlib.metadata import metadata +from pathlib import Path + +from sphinxcontrib import katex + +HERE = Path(__file__).parent +sys.path.insert(0, str(HERE / "extensions")) + + +# -- Project information ----------------------------------------------------- + +# NOTE: If you installed your project in editable mode, this might be stale. +# If this is the case, reinstall it to refresh the metadata +info = metadata("hnoca") +project = info["Name"] +author = info["Author"] +copyright = f"{datetime.now():%Y}, {author}." +version = info["Version"] +urls = dict(pu.split(", ") for pu in info.get_all("Project-URL")) +repository_url = urls["Source"] + +# The full version, including alpha/beta/rc tags +release = info["Version"] + +bibtex_bibfiles = ["references.bib"] +templates_path = ["_templates"] +nitpicky = True # Warn about broken links +needs_sphinx = "4.0" + +html_context = { + "display_github": True, # Integrate GitHub + "github_user": "devsystemslab", + "github_repo": "HNOCA-tools", + "github_version": "main", + "conf_py_path": "/docs/", +} + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. +# They can be extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + "myst_nb", + "sphinx_copybutton", + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinxcontrib.bibtex", + "sphinxcontrib.katex", + "sphinx_autodoc_typehints", + "sphinx_design", + "IPython.sphinxext.ipython_console_highlighting", + "sphinxext.opengraph", + "scverse_misc.sphinx_ext", + *[p.stem for p in (HERE / "extensions").glob("*.py")], +] + +autosummary_generate = True +autodoc_member_order = "groupwise" +default_role = "literal" +napoleon_google_docstring = False +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False +napoleon_use_rtype = True # having a separate entry generally helps readability +napoleon_use_param = True +myst_heading_anchors = 6 # create anchors for h1-h6 +myst_enable_extensions = [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "html_image", + "html_admonition", +] +myst_url_schemes = ("http", "https", "mailto") +nb_output_stderr = "remove" +nb_execution_mode = "off" +nb_merge_streams = True +typehints_defaults = "braces" +always_use_bars_union = True # use `|` instead of `Union` in types even when building with Python ≤3.14 + +source_suffix = { + ".rst": "restructuredtext", + ".ipynb": "myst-nb", + ".myst": "myst-nb", +} + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "anndata": ("https://anndata.scverse.org/en/stable/", None), + "scanpy": ("https://scanpy.scverse.org/en/stable/", None), + "numpy": ("https://numpy.org/doc/stable/", None), +} + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_book_theme" +html_static_path = ["_static"] +html_css_files = ["css/custom.css"] + +html_title = project + +html_theme_options = { + "repository_url": repository_url, + "use_repository_button": True, + "path_to_docs": "docs/", + "navigation_with_keys": False, +} + +pygments_style = "default" +katex_prerender = shutil.which(katex.NODEJS_BINARY) is not None + +nitpick_ignore = [ + # If building the documentation fails because of a missing link that is outside your control, + # you can add an exception to this list. + # ("py:class", "igraph.Graph"), +] diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..33afd7d --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,347 @@ +# Contributing guide + +This document aims at summarizing the most important information for getting you started on contributing to this project. +We assume that you are already familiar with git and with making pull requests on GitHub. + +For more extensive tutorials, that also cover the absolute basics, +please refer to other resources such as the [pyopensci tutorials][], +the [scientific Python tutorials][], or the [scanpy developer guide][]. + +[pyopensci tutorials]: https://www.pyopensci.org/learn.html +[scientific Python tutorials]: https://learn.scientific-python.org/development/tutorials/ +[scanpy developer guide]: https://scanpy.scverse.org/page/dev/ + +:::{tip} The *hatch* project manager + +We highly recommend to familiarize yourself with [`hatch`][hatch]. +Hatch is a Python project manager that + +- manages virtual environments, separately for development, testing and building the documentation. + Separating the environments is useful to avoid dependency conflicts. +- allows to run tests locally in different environments (e.g. different python versions) +- allows to run tasks defined in `pyproject.toml`, e.g. to build documentation. + +While the project is setup with `hatch` in mind, +it is still possible to use different tools to manage dependencies, such as `uv` or `pip`. + +::: + +[hatch]: https://hatch.pypa.io/latest/ + +## Installing dev dependencies + +In addition to the packages needed to _use_ this package, +you need additional python packages to [run tests](#writing-tests) and [build the documentation](#docs-building). + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch + +On the command line, you typically interact with hatch through its command line interface (CLI). +Running one of the following commands will automatically resolve the environments for testing and +building the documentation in the background: + +```bash +hatch test # defined in the table [tool.hatch.envs.hatch-test] in pyproject.toml +hatch run docs:build # defined in the table [tool.hatch.envs.docs] +``` + +### VS Code + +If you are using VS code, install the [hatch-code][] extension. +Additionally, make sure that the `vscode-python-environments` extension is installed (should be by default) +and `"python.useEnvironmentsExtension": true` is activated in your `settings.json`. + +Next, open the "Python Environment Managers" sidebar. +You can do so by opening the command palette (Ctrl+Shift+P) and searching for `Python: Focus on Environment Managers View`. +It will show a collapsible list where you can expand "Hatch" +and activate an environment by clicking on the checkmark next to it. +As the main development environment, we recommend to use `hatch-test` with the latest supported Python version. + +### Other IDEs + +For other IDEs, you’ll have to point the editor at the paths to the virtual environments manually. +To get a list of all environments for your projects, run + +```bash +hatch env show -i +``` + +This will list “Standalone” environments and a table of “Matrix” environments like the following: + +``` ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +| Name | Type | Envs | Features | Dependencies | Scripts | ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +| hatch-test | virtual | hatch-test.py3.12-stable | dev | coverage-enable-subprocess==1.0 | cov-combine | +| | | hatch-test.py3.14-stable | test | coverage[toml]~=7.4 | cov-report | +| | | hatch-test.py3.14-pre | | pytest-mock~=3.12 | run | +| | | | | pytest-randomly~=3.15 | run-cov | +| | | | | pytest-rerunfailures~=14.0 | | +| | | | | pytest-xdist[psutil]~=3.5 | | +| | | | | pytest~=8.1 | | ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +``` + +From the `Envs` column, select the environment name you want to use for development. +As the main development environment, we recommend to use `hatch-test` with the latest supported Python version. +In this example, it would be `hatch-test.py3.14-stable`. + +Next, create the environment with + +```bash +hatch env create hatch-test.py3.14-stable +``` + +Then, obtain the path to the environment using + +```bash +hatch env find hatch-test.py3.14-stable +``` + +and manually point it to the python binary. + + +:::: + +::::{tab-item} uv +:sync: uv + +A popular choice for managing virtual environments is [uv][]. +The main disadvantage compared to hatch is that it supports only a single environment per project at a time, +which requires you to mix the dependencies for running tests and building docs. +This can have undesired side-effects, +such as requiring to install a lower version of a library your project depends on, +only because an outdated sphinx plugin pins an older version. + +To initialize a virtual environment in the `.venv` directory of your project, simply run + +```bash +uv sync --all-extras +``` + +The `.venv` directory is typically automatically discovered by IDEs such as VS Code. + +:::: + +::::{tab-item} Pip +:sync: pip + +Pip is nowadays mostly superseded by environment manager such as [hatch][]. +However, for the sake of completeness, and since it’s ubiquitously available, +we describe how you can manage environments manually using `pip`: + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -e ".[dev,test,doc]" +``` + +The `.venv` directory is typically automatically discovered by IDEs such as VS Code. + +:::: +::::: + +[hatch environments]: https://hatch.pypa.io/latest/tutorials/environment/basic-usage/ +[hatch-code]: https://marketplace.visualstudio.com/items?itemName=PyPA.hatch +[uv]: https://docs.astral.sh/uv/ + +## Code-style + +This package uses [pre-commit][]-style hooks to enforce consistent code-styles. +We recommend running them with [prek][], a fast, drop-in replacement for `pre-commit` that reads the same `.pre-commit-config.yaml`. +On every commit, the checks will either automatically fix issues with the code, or raise an error message. + +To enable the checks locally, install [prek][] (e.g. with `uv tool install prek`) and run + +```bash +prek install +``` + +in the root of the repository. +prek will automatically download all dependencies when it is run for the first time. + +Alternatively, you can rely on the [pre-commit.ci][] service enabled on GitHub. +If you didn’t run the checks before pushing changes to GitHub it will automatically commit fixes to your pull request, or show an error message. + +If pre-commit.ci added a commit on a branch you still have been working on locally, simply use + +```bash +git pull --rebase +``` + +to integrate the changes into yours. +While the [pre-commit.ci][] is useful, we strongly encourage installing and running the checks locally first to understand their usage. + +Finally, most editors have an _autoformat on save_ feature. +Consider enabling this option for [ruff][ruff-editors] and [biome][biome-editors]. + +[pre-commit]: https://pre-commit.com/ +[prek]: https://prek.j178.dev/ +[pre-commit.ci]: https://pre-commit.ci/ +[ruff-editors]: https://docs.astral.sh/ruff/integrations/ +[biome-editors]: https://biomejs.dev/guides/integrate-in-editor/ + +(writing-tests)= + +## Writing tests + +This package uses [pytest][] for automated testing. +Please write {doc}`scanpy:dev/testing` for every function added to the package. + +Most IDEs integrate with pytest and provide a GUI to run tests. +If you set up your virtual environments as described in [installing dev dependencies](#installing-dev-dependencies), +test cases should be automatically discovered by your IDE. + +Alternatively, you can run all tests from the command line by executing + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch + +```bash +hatch test # test with the highest supported Python version +# or +hatch test --all # test with all supported Python versions +``` + +:::: + +::::{tab-item} uv +:sync: uv + +```bash +uv run pytest +``` + +:::: + +::::{tab-item} Pip +:sync: pip + +```bash +source .venv/bin/activate +pytest +``` + +:::: +::::: + +in the root of the repository. + +[pytest]: https://docs.pytest.org/ + +### Continuous integration + +Continuous integration via GitHub actions will automatically run the tests on all pull requests and test +against the minimum and maximum supported Python version. + +Additionally, there’s a CI job that tests against pre-releases of all dependencies (if there are any). +The purpose of this check is to detect incompatibilities of new package versions early on and +gives you time to fix the issue or reach out to the developers of the dependency before the package +is released to a wider audience. + +The CI job is defined in `.github/workflows/test.yaml`, +however the single point of truth for CI jobs is the Hatch test matrix defined in `pyproject.toml`. +This means that local testing via hatch and remote testing on CI tests against the same python versions and uses the same environments. + +## Publishing a release + +### Updating the version number + +Before making a release, you need to update the version number in the `pyproject.toml` file. +Please adhere to [Semantic Versioning][semver], in brief + +> Given a version number MAJOR.MINOR.PATCH, increment the: +> +> 1. MAJOR version when you make incompatible API changes, +> 2. MINOR version when you add functionality in a backwards compatible manner, and +> 3. PATCH version when you make backwards compatible bug fixes. +> +> Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. + +Once you are done, commit and push your changes and navigate to the "Releases" page of this project on GitHub. +Specify `vX.X.X` as a tag name and create a release. +For more information, see [managing GitHub releases][]. +This will automatically create a git tag and trigger a Github workflow that creates a release on [PyPI][]. + +[semver]: https://semver.org/ +[managing GitHub releases]: https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository +[pypi]: https://pypi.org/ + +## Writing documentation + +Please write documentation for new or changed features and use-cases. +This project uses [sphinx][] with the following features: + +- The [myst][] extension allows to write documentation in markdown/Markedly Structured Text +- [Numpy-style docstrings][numpydoc] (through the [napoloen][numpydoc-napoleon] extension). +- Jupyter notebooks as tutorials through [myst-nb][] (See [Tutorials with myst-nb](#tutorials-with-myst-nb-and-jupyter-notebooks)) +- [sphinx-autodoc-typehints][], to automatically reference annotated input and output types +- Citations (like {cite:p}`Virshup_2023`) can be included with [sphinxcontrib-bibtex](https://sphinxcontrib-bibtex.readthedocs.io/) + +See scanpy’s {doc}`scanpy:dev/documentation` for more information on how to write your own. + +[sphinx]: https://www.sphinx-doc.org/ +[myst]: https://myst-parser.readthedocs.io/page/intro.html +[myst-nb]: https://myst-nb.readthedocs.io/ +[numpydoc-napoleon]: https://www.sphinx-doc.org/page/usage/extensions/napoleon.html +[numpydoc]: https://numpydoc.readthedocs.io/page/format.html +[sphinx-autodoc-typehints]: https://github.com/tox-dev/sphinx-autodoc-typehints + +### Tutorials with myst-nb and jupyter notebooks + +The documentation is set-up to render jupyter notebooks stored in the `docs/notebooks` directory using [myst-nb][]. +Currently, only notebooks in `.ipynb` format are supported that will be included with both their input and output cells. +It is your responsibility to update and re-run the notebook whenever necessary. + +If you are interested in automatically running notebooks as part of the continuous integration, +please check out [this feature request][issue-render-notebooks] in the `cookiecutter-scverse` repository. + +[issue-render-notebooks]: https://github.com/scverse/cookiecutter-scverse/issues/40 + +#### Hints + +- If you refer to objects from other packages, please add an entry to `intersphinx_mapping` in `docs/conf.py`. + Only if you do so can sphinx automatically create a link to the external documentation. +- If building the documentation fails because of a missing link that is outside your control, + you can add an entry to the `nitpick_ignore` list in `docs/conf.py` + +(docs-building)= + +### Building the docs locally + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch + +```bash +hatch run docs:build +hatch run docs:open +``` + +:::: + +::::{tab-item} uv +:sync: uv + +```bash +cd docs +uv run sphinx-build -M html . _build -W +(xdg-)open _build/html/index.html +``` + +:::: + +::::{tab-item} Pip +:sync: pip + +```bash +source .venv/bin/activate +cd docs +sphinx-build -M html . _build -W +(xdg-)open _build/html/index.html +``` + +:::: +::::: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b9a61af --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,143 @@ +[build-system] +build-backend = "hatchling.build" +requires = [ "hatchling" ] + +[project] +name = "hnoca" +version = "0.0.1" +description = "Human Neural Organoid Cell Atlas Toolbox" +readme = "README.md" +license = { file = "LICENSE" } +maintainers = [ + { name = "Marius Lange", email = "marius.lange@bsse.ethz.ch" }, +] +authors = [ + { name = "Marius Lange" }, +] +requires-python = ">=3.12" +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "anndata", + # for debug logging (referenced from the issue template) + "session-info2", +] +# https://docs.pypi.org/project_metadata/#project-urls +urls.Documentation = "https://hnoca.readthedocs.io/" +urls.Homepage = "https://github.com/devsystemslab/HNOCA-tools" +urls.Source = "https://github.com/devsystemslab/HNOCA-tools" + +[dependency-groups] +dev = [ + "prek", + "twine>=4.0.2", +] +test = [ + "coverage>=7.10", + "pytest", + "pytest-cov", # For VS Code’s coverage functionality +] +doc = [ + "ipykernel", + "ipython", + "myst-nb>=1.1", + "pandas", + "scverse-misc[sphinx]>=0.1.2", + "sphinx>=8.1", + "sphinx-autodoc-typehints", + "sphinx-book-theme>=1", + "sphinx-copybutton", + "sphinx-design", + "sphinxcontrib-bibtex>=1", + "sphinxcontrib-katex", + "sphinxext-opengraph", +] + +[tool.hatch] +envs.default.installer = "uv" +envs.default.dependency-groups = [ "dev" ] +envs.docs.scripts.build = "sphinx-build -M html docs docs/_build -W {args}" +envs.docs.scripts.clean = "git clean -fdX -- {args:docs}" +envs.docs.scripts.open = "python -m webbrowser -t docs/_build/html/index.html" +envs.docs.dependency-groups = [ "doc" ] +envs.hatch-test.matrix = [ + # Test the lowest and highest supported Python versions with normal deps + { deps = [ "stable" ], python = [ "3.12", "3.14" ] }, + # Test the newest supported Python version also with pre-release deps + { deps = [ "pre" ], python = [ "3.14" ] }, +] +# If the matrix variable `deps` is set to "pre", +# set the environment variable `UV_PRERELEASE` to "allow". +envs.hatch-test.overrides.matrix.deps.env-vars = [ + { value = "allow", key = "UV_PRERELEASE", if = [ "pre" ] }, +] +envs.hatch-test.dependency-groups = [ "dev", "test" ] + +[tool.ruff] +line-length = 120 +src = [ "src" ] +extend-include = [ "*.ipynb" ] +format.docstring-code-format = true +lint.select = [ + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "D", # pydocstyle + "E", # Error detected by Pycodestyle + "F", # Errors detected by Pyflakes + "I", # isort + "RUF100", # Report unused noqa directives + "TID", # flake8-tidy-imports + "UP", # pyupgrade + "W", # Warning detected by Pycodestyle +] +lint.ignore = [ + "B008", # Errors from function calls in argument defaults. These are fine when the result is immutable. + "D100", # Missing docstring in public module + "D104", # Missing docstring in public package + "D105", # __magic__ methods are often self-explanatory, allow missing docstrings + "D107", # Missing docstring in __init__ + # Disable one in each pair of mutually incompatible rules + "D203", # We don’t want a blank line before a class docstring + "D213", # <> We want docstrings to start immediately after the opening triple quote + "D400", # first line should end with a period [Bug: doesn’t work with single-line docstrings] + "D401", # First line should be in imperative mood; try rephrasing + "E501", # line too long -> we accept long comment lines; formatter gets rid of long code lines + "E731", # Do not assign a lambda expression, use a def -> lambda expression assignments are convenient + "E741", # allow I, O, l as variable names -> I is the identity matrix +] +lint.per-file-ignores."*/__init__.py" = [ "F401" ] +lint.per-file-ignores."docs/*" = [ "I" ] +lint.per-file-ignores."tests/*" = [ "D" ] +lint.pydocstyle.convention = "numpy" + +[tool.pytest] +addopts = [ + "--import-mode=importlib", # allow using test files with same name +] +strict = true +testpaths = [ "tests" ] + +[tool.coverage] +run.omit = [ + "**/test_*.py", +] +run.patch = [ "subprocess" ] +run.source = [ "hnoca" ] + +[tool.cruft] +skip = [ + ".git", + "tests", + "src/**/__init__.py", + "src/**/basic.py", + "docs/api.md", + "docs/changelog.md", + "docs/references.bib", + "docs/references.md", + "docs/notebooks/example.ipynb", +]