diff --git a/.github/workflows/cicd_tests.yml b/.github/workflows/cicd_tests.yml index e9b207951f..19f5c0d44c 100644 --- a/.github/workflows/cicd_tests.yml +++ b/.github/workflows/cicd_tests.yml @@ -52,6 +52,34 @@ env: # When support is dropped for a version it is important to update these as appropriate. jobs: + pre-commit: # Run the hooks pre-commit.ci skips, using the tools pyproject.toml pins + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # This job executes the hooks named by a PR's own .pre-commit-config.yaml, so + # it must not leave a usable token in .git/config for those hooks to reach. + - uses: actions/checkout@v7 + with: + persist-credentials: false + # reads the pins below with tomllib, which needs 3.11+; the hooks take their + # target from pyproject.toml, so the interpreter version does not affect them + - name: Set up Python ${{ env.PYTHON_VER3 }} + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VER3 }} + cache: 'pip' + - name: Install lint tools + run: | + # the lint extra alone, so this job needs neither torch nor the optional dependencies + python -m pip install --upgrade pip + python -c "import tomllib; print('\n'.join(tomllib.load(open('pyproject.toml','rb'))['project']['optional-dependencies']['lint']))" > lint-requirements.txt + cat lint-requirements.txt + python -m pip install -r lint-requirements.txt + rm lint-requirements.txt + - name: Run pre-commit + run: python -m pre_commit run --all-files --show-diff-on-failure + static-checks: # Perform static type and other checks using runtests.sh runs-on: ubuntu-latest strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d78b08041..e3a27e079c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,9 @@ ci: autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' autoupdate_schedule: quarterly # submodules: true + # The hooks below run the tools from the environment pyproject.toml defines, which + # pre-commit.ci does not build; the pre-commit job in cicd_tests.yml runs them instead. + skip: [black, isort, pycln, ruff-check] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -26,19 +29,33 @@ repos: args: ['--autofix', '--no-sort-keys', '--indent=4'] - id: end-of-file-fixer - id: mixed-line-ending - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 - hooks: - - id: ruff-check - args: ["--fix"] - exclude: | - (?x)( - ^versioneer.py| - ^monai/_version.py - ) - - repo: https://github.com/hadialqattan/pycln - rev: v2.6.0 + # Versions come from [project.optional-dependencies].lint and settings from the + # [tool.*] tables, so these hooks and runtests.sh cannot resolve a different tool. + # The --force-exclude/--filter-files flags apply those settings to the explicit + # filenames pre-commit passes. + - repo: local hooks: + - id: ruff-check + name: ruff check + entry: python -m ruff check --force-exclude --fix + language: system + types_or: [python, pyi] + + - id: isort + name: isort + entry: python -m isort --filter-files + language: system + types_or: [python, pyi] + + - id: black + name: black + entry: python -m black + language: system + types_or: [python, pyi] + - id: pycln - args: [--config=pyproject.toml] + name: pycln + entry: python -m pycln --config=pyproject.toml + language: system + types_or: [python, pyi] diff --git a/monai/config/print_dependencies.py b/monai/config/print_dependencies.py index a099949eca..7b8957983c 100644 --- a/monai/config/print_dependencies.py +++ b/monai/config/print_dependencies.py @@ -18,6 +18,7 @@ from __future__ import annotations +import re import sys from collections.abc import Collection @@ -25,10 +26,49 @@ PROJ_KEY = "project" OPTS_KEY = "optional-dependencies" DEP_KEY = "dependencies" +NAME_KEY = "name" REQ_KEY = "requires" TOML_FILE = "pyproject.toml" +def _expand_self_extras(dependencies: list[str], name: str, opts: dict) -> list[str]: + """ + Replace self-referential requirements such as ``monai[lint]`` with the contents of that group. + + pip resolves such a requirement against the package index, so leaving one in a generated + requirements file installs the published release instead of the checkout being worked on. + + Args: + dependencies: requirement strings, some of which may be self-references. + name: this project's name, the only one treated as a self-reference. + opts: the "optional-dependencies" table the groups are read from. + + Returns: + List of requirements with every self-reference replaced by the group it names. + + Raises: + KeyError: If a self-reference names a group absent from `opts`. + """ + pattern = re.compile(rf"^{re.escape(name)}\s*\[([^\]]+)\]$", re.IGNORECASE) + expanded: list[str] = [] + pending = list(dependencies) + seen: set[str] = set() + + while pending: + req = pending.pop(0) + match = pattern.match(req.strip()) + if match is None: + expanded.append(req) + continue + for group in (g.strip() for g in match.group(1).split(",")): + if group in seen: # a group already expanded, or a cycle + continue + seen.add(group) + pending.extend(opts[group]) + + return expanded + + def parse_dependencies(filename: str | None = None, sections: Collection[str] | None = None) -> list[str]: """ Parse the toml file given by `filename` and return the dependency sections selected by `sections`. @@ -68,6 +108,8 @@ def parse_dependencies(filename: str | None = None, sections: Collection[str] | for s in sections: dependencies += opts[s] + dependencies = _expand_self_extras(dependencies, proj[NAME_KEY], opts) + return sorted(set(dependencies)) diff --git a/pyproject.toml b/pyproject.toml index 9c5f892283..2e9c360030 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -157,20 +157,25 @@ torchvision = ["torchvision"] tqdm = ["tqdm>=4.47.0"] transformers = ["transformers>=4.53.0, <5.0"] # 5.x references torch.float8_e8m0fnu absent in older PyTorch builds zarr = ["zarr"] +# the tools .pre-commit-config.yaml and runtests.sh invoke; installable without torch +lint = [ + "black>=26.3.1", + "isort>=5.1,<6,!=6.0.0", + "pre-commit", + "pycln==2.6.0", + "ruff==0.16.4" +] # these dependencies are for testing/building only, they aren't needed for regular use so don't appear in "all" testing = [ - "black>=26.3.1", + "monai[lint]", "coverage>=5.5", - "isort>=5.1,<6,!=6.0.0", "mccabe", "packaging", "parameterized", "pep8-naming", - "pre-commit", "pycodestyle", "pyflakes", "pyrefly>=1.0.0", - "ruff>=0.14.11,<0.15", "tomli", # used in print_dependencies.py for Python<3.11 "typeguard<3", # https://github.com/microsoft/nni/issues/5457 "types-PyYAML", @@ -266,7 +271,9 @@ line-length = 120 target-version = ['py310'] skip-magic-trailing-comma = true include = '\.pyi?$' -exclude = ''' +# force-exclude, not exclude: black ignores exclude for filenames passed explicitly, +# which is how pre-commit invokes it. +force-exclude = ''' ( /( # exclude a few common directories in the root of the project @@ -296,6 +303,8 @@ exclude = "monai/bundle/__main__.py" [tool.ruff] line-length = 120 target-version = "py310" +# Vendored/generated; matches [tool.black] exclude and [tool.pyrefly] project-excludes. +extend-exclude = ["versioneer.py", "monai/_version.py"] [tool.ruff.lint] select = [ diff --git a/runtests.sh b/runtests.sh index 73508a093b..d74abebf62 100755 --- a/runtests.sh +++ b/runtests.sh @@ -595,13 +595,13 @@ then then install_deps fi - ruff --version + ${cmdPrefix}"${PY_EXE}" -m ruff --version if [ $doRuffFix = true ] then - ruff check --fix --unsafe-fixes --exclude versioneer.py --exclude "monai/_version.py" "$homedir" + ${cmdPrefix}"${PY_EXE}" -m ruff check --fix --unsafe-fixes "$homedir" else - ruff check --exclude versioneer.py --exclude "monai/_version.py" "$homedir" + ${cmdPrefix}"${PY_EXE}" -m ruff check "$homedir" fi ruff_status=$? diff --git a/tests/config/test_print_dependencies.py b/tests/config/test_print_dependencies.py index bbf8c4c7cd..628ff2cb45 100644 --- a/tests/config/test_print_dependencies.py +++ b/tests/config/test_print_dependencies.py @@ -32,14 +32,18 @@ [project.optional-dependencies] all = ["something", "another"] -testing = ["coverage", "black"] +lint = ["ruff", "black"] +testing = ["test[lint]", "coverage"] +cyclic = ["test[cyclic]", "spam"] """ PARSE_CASES = [ ([], ["numpy", "torch"]), - (["testing"], ["black", "coverage", "numpy", "torch"]), + # "test[lint]" expands rather than reaching pip as a requirement on the published package + (["testing"], ["black", "coverage", "numpy", "ruff", "torch"]), (["build-system"], ["numpy", "setuptools", "torch", "wheel"]), - (["*"], ["another", "black", "coverage", "numpy", "something", "torch"]), + (["*"], ["another", "black", "coverage", "numpy", "ruff", "something", "spam", "torch"]), + (["cyclic"], ["numpy", "spam", "torch"]), ]