MNT Rework workflows: split concerns (alternative)

.github/workflows/wheels.yml → .github/workflows/build-wheels.yml

@@ -1,37 +1,39 @@
-name: Publish sdist and wheels
+name: Build sdist and wheels
 
 on:
-  schedule:
-    # 01:01 UTC every day
-    - cron: "1 1 * * *"
   pull_request:
     branches:
       - main
-      - "[0-9]+.[0-9]+.X"
-  push:
-    branches:
-      - main
-  workflow_dispatch:
+  # Reusable entry point. Callers pass `ref` (sklearn SHA/branch/PR merge ref):
+  # - nightly.yml: sklearn@main, then uploads to Anaconda
+  # - scikit-learn wheels.yml: PR head when the commit has [cd build]
+  workflow_call:
     inputs:
       ref:
-        description: The scikit-learn branch, tag, or commit to build
+        description: The scikit-learn branch, tag, SHA, or refs/pull/N/merge to build
         required: false
         default: main
         type: string
-      environment:
-        description: Which PyPI environment to upload to, if any
+  # Manual "Run workflow" on this repo, usually after the sklearn release PR is merged.
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: The scikit-learn branch, tag, SHA, or refs/pull/N/merge to build
         required: true
-        type: choice
-        options: ["none", "chain", "testpypi", "pypi"]
+        type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: build-wheels-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+run-name: Build sklearn ${{ inputs.ref || 'main' }}
+
+permissions:
+  contents: read
+
 jobs:
   build_wheels:
     name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}
-    if: github.repository == 'scikit-learn/scikit-learn-release'
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -165,13 +167,17 @@ jobs:
       - name: Checkout scikit-learn-release
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
+          # This workflow may be called from scikit-learn ([cd build]). Make sure to
+          # checkout the repository and commit this workflow file comes from.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
           persist-credentials: false
 
       - name: Checkout scikit-learn
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           repository: scikit-learn/scikit-learn
-          ref: ${{ github.event.inputs.ref || 'main' }}
+          ref: ${{ inputs.ref || 'main' }}
           path: scikit-learn-src
           persist-credentials: false
 
@@ -196,9 +202,9 @@ jobs:
           PYTHONHASHSEED: "0"
           CIBW_ENVIRONMENT_PASS_LINUX: SOURCE_DATE_EPOCH PYTHONHASHSEED RUNNER_OS
         with:
-           package-dir: scikit-learn-src
-           output-dir: dist
-           config-file: cibuildwheel.toml
+          package-dir: scikit-learn-src
+          output-dir: dist
+          config-file: cibuildwheel.toml
 
       - name: Test Windows wheel in a minimal Docker image
         # Currently Windows ARM64 runners do not have Docker support and there
@@ -214,7 +220,9 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cibw-wheels-cp${{ matrix.python }}-${{ matrix.platform_id }}
-          path: ./dist/*.whl
+          # `**/` is what keeps the dist/ prefix inside the artifact: the artifact root
+          # is the path prefix before the first wildcard.
+          path: "**/dist/*.whl"
 
   build_sdist:
     name: Source distribution
@@ -224,18 +232,22 @@ jobs:
       - name: Checkout scikit-learn-release
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
+          # This workflow may be called from scikit-learn ([cd build]). Make sure to
+          # checkout the repository and commit this workflow file comes from.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
           persist-credentials: false
 
       - name: Checkout scikit-learn
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           repository: scikit-learn/scikit-learn
-          ref: ${{ github.event.inputs.ref || 'main' }}
+          ref: ${{ inputs.ref || 'main' }}
           path: scikit-learn-src
           persist-credentials: false
 
       - name: Setup Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # 6.2.0
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0
         with:
           python-version: "3.12"
 
@@ -247,100 +259,14 @@ jobs:
         env:
           SKLEARN_SKIP_NETWORK_TESTS: 1
 
+      # publish.yml checks this against the release tag before uploading to PyPI.
+      - name: Record scikit-learn commit
+        run: git -C scikit-learn-src rev-parse HEAD | tee sklearn-sha.txt
+
       - name: Store artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cibw-sdist
-          path: dist/*.tar.gz
-
-  testpypi-publish:
-    name: Publish release to TestPyPI
-    if: github.event_name == 'workflow_dispatch' && (github.event.inputs.environment == 'testpypi' || github.event.inputs.environment == 'chain')
-    runs-on: ubuntu-latest
-    needs: [build_wheels, build_sdist]
-    environment:
-      name: testpypi
-      url: https://test.pypi.org/p/scikit-learn
-    permissions:
-      id-token: write  # mandatory for trusted publishing
-    steps:
-      - name: Download sdist and wheels
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: dist
-          merge-multiple: true
-
-      - name: View artifacts
-        run: ls dist
-
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
-        with:
-          repository-url: https://test.pypi.org/legacy/
-          skip-existing: true
-          print-hash: true
-          attestations: true
-
-  nightly_upload:
-    name: Nightly upload
-    if: github.repository == 'scikit-learn/scikit-learn-release' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'none'))
-    runs-on: ubuntu-latest
-    environment: upload_anaconda
-    needs: [build_wheels, build_sdist]
-
-    steps:
-      - name: Download sdist and wheels
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: dist
-          merge-multiple: true
-
-      - name: Install micromamba
-        uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
-        with:
-          # For installation of anaconda-client, required for upload to anaconda.org
-          init-shell: bash
-          environment-name: upload-env
-          create-args: >-
-            anaconda-client
-
-      - name: Upload to anaconda.org
-        shell: bash -el {0}  # required for micromamba
-        env:
-          TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }}
-        run: |
-          anaconda -q -t ${TOKEN} upload --force -u scientific-python-nightly-wheels ./dist/*
-
-  pypi-publish:
-    name: Publish release to PyPI
-    if: >-
-      github.event_name == 'workflow_dispatch'
-      && (
-        github.event.inputs.environment == 'pypi'
-        || (
-          github.event.inputs.environment == 'chain'
-          && (needs.testpypi-publish.result == 'success' || needs.testpypi-publish.result == 'skipped')
-        )
-      )
-    runs-on: ubuntu-latest
-    needs: [build_wheels, build_sdist, testpypi-publish]
-    environment:
-      name: pypi
-      url: https://pypi.org/p/scikit-learn
-    permissions:
-      id-token: write  # mandatory for trusted publishing
-    steps:
-      - name: Download sdist and wheels
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          path: dist
-          merge-multiple: true
-
-      - name: View artifacts
-        run: ls dist
-
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
-        with:
-          print-hash: true
-          attestations: true
+          path: |
+            dist
+            sklearn-sha.txt

.github/workflows/nightly.yml

@@ -0,0 +1,63 @@
+name: Nightly wheels
+
+on:
+  schedule:
+    # 01:01 UTC every day
+    - cron: "1 1 * * *"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build sdist and wheels
+    if: github.repository == 'scikit-learn/scikit-learn-release'
+    uses: ./.github/workflows/build-wheels.yml
+    with:
+      ref: main
+
+  nightly_upload:
+    name: Nightly upload
+    runs-on: ubuntu-latest
+    environment: upload_anaconda
+    needs: [build]
+
+    steps:
+      - name: Download sdist and wheels
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          merge-multiple: true
+
+      - name: Install micromamba
+        uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
+        with:
+          # For installation of anaconda-client, required for upload to anaconda.org
+          init-shell: bash
+          environment-name: upload-env
+          create-args: >-
+            anaconda-client
+
+      - name: Upload to anaconda.org
+        shell: bash -el {0}
+        env:
+          # anaconda-client uses this env var as the default token.
+          ANACONDA_API_TOKEN: ${{ secrets.SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN }}
+        run: |
+          anaconda -q upload --force -u scientific-python-nightly-wheels ./dist/*
+
+  # Open/update a tracking issue on scikit-learn when this nightly fails.
+  # Uncomment when:
+  # - BOT_GITHUB_TOKEN is set on this repo (issues: write on scikit-learn)
+  # - scikit-learn/.github/workflows/update_tracking_issue.yml accepts
+  #   issue_repo, always checkouts scikit-learn/scikit-learn for the script,
+  #   and no longer requires github.repository == 'scikit-learn/scikit-learn'
+  #
+  # update-tracker:
+  #   needs: [build]
+  #   if: ${{ always() && github.repository == 'scikit-learn/scikit-learn-release' }}
+  #   uses: scikit-learn/scikit-learn/.github/workflows/update_tracking_issue.yml@main
+  #   with:
+  #     job_status: ${{ needs.build.result }}
+  #     issue_repo: scikit-learn/scikit-learn
+  #   secrets:
+  #     BOT_GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}

.github/workflows/publish.yml

@@ -0,0 +1,102 @@
+name: Publish to TestPyPI and PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: The scikit-learn maintenance branch to release from, e.g. 1.10.X
+        required: true
+        type: string
+
+run-name: Publish scikit-learn from ${{ inputs.ref }}
+
+permissions:
+  contents: read
+
+jobs:
+  check_ref:
+    name: Check ref is a maintenance branch
+    runs-on: ubuntu-latest
+    if: github.repository == 'scikit-learn/scikit-learn-release'
+    steps:
+      - env:
+          REF: ${{ inputs.ref }}
+        run: |
+          if [[ ! "$REF" =~ ^[0-9]+\.[0-9]+\.X$ ]]; then
+            echo "ref must be a maintenance branch, e.g. 1.10.X (got '$REF')"
+            exit 1
+          fi
+
+  build:
+    name: Build sdist and wheels
+    needs: [check_ref]
+    uses: ./.github/workflows/build-wheels.yml
+    with:
+      ref: ${{ inputs.ref }}
+
+  testpypi-publish:
+    name: Publish release to TestPyPI
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/scikit-learn
+    permissions:
+      id-token: write  # mandatory for trusted publishing
+    steps:
+      - name: Download sdist and wheels
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          merge-multiple: true
+
+      - name: View artifacts
+        run: ls -l dist
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          print-hash: true
+          attestations: true
+
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    needs: [testpypi-publish]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/scikit-learn
+    permissions:
+      id-token: write  # mandatory for trusted publishing
+    steps:
+      - name: Download sdist and wheels
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          merge-multiple: true
+
+      - name: View artifacts
+        run: ls -l dist
+
+      # Check that the sdist version is already a tag, pointing at the commit these
+      # artifacts were built from.
+      - name: Check tag matches the built commit
+        run: |
+          set -euo pipefail
+          version=$(basename dist/scikit_learn-*.tar.gz .tar.gz)
+          version=${version#scikit_learn-}
+          built=$(cat sklearn-sha.txt)
+          echo "publishing scikit-learn $version, built from $built"
+          git clone --depth 1 --branch "$version" https://github.com/scikit-learn/scikit-learn.git sklearn-tag
+          tagged=$(git -C sklearn-tag rev-parse HEAD)
+          echo "tag $version: $tagged"
+          if [[ "$built" != "$tagged" ]]; then
+            echo "tag $version must point at the commit that produced these artifacts."
+            exit 1
+          fi
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
+        with:
+          print-hash: true
+          attestations: true