release-python.yml

name: Release Python

on:
  workflow_dispatch:
    inputs:
      # Latest commit to include with the release. If omitted, use the latest commit on the main branch.
      sha:
        description: Commit SHA
        type: string
      # Create the sdist and build the wheels, but do not publish to PyPI / GitHub.
      dry-run:
        description: Dry run
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  PYTHON_VERSION: '3.10'
  PYTHON_VERSION_WIN_ARM64: '3.11' # ARM64 Windows doesn't have older versions
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  RUSTUP_MAX_RETRIES: 10

  COMPAT_TUNE_CPU: ''
  COMPAT_FEATURES: '+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b'
  COMPAT_CC_FEATURES: '-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16'

  NONCOMPAT_TUNE_CPU: 'skylake'
  NONCOMPAT_FEATURES: '+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe'
  NONCOMPAT_CC_FEATURES: '-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe'

defaults:
  run:
    shell: bash

jobs:
  base-package:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false

    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          ref: ${{ inputs.sha }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install yq
        run: pip install yq

      - name: Check runtime versions
        env:
          POLARS_PYPROJECT_TOML: py-polars/pyproject.toml
        run: |
          set -e

          VERSION=$(tomlq '.project.version' "$POLARS_PYPROJECT_TOML" | tr -d '"')
          CARGO_32_VERSION=$(tomlq '.package.version' py-polars/runtime/polars-runtime-32/Cargo.toml | tr -d '"')
          CARGO_64_VERSION=$(tomlq '.package.version' py-polars/runtime/polars-runtime-64/Cargo.toml | tr -d '"')
          CARGO_COMPAT_VERSION=$(tomlq '.package.version' py-polars/runtime/polars-runtime-compat/Cargo.toml | tr -d '"')

          RT32_VERSION=($(tomlq '.project.dependencies[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"'))
          RT64_VERSION=($(tomlq '.project."optional-dependencies".rt64[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"'))
          COMPAT_VERSION=($(tomlq '.project."optional-dependencies".rtcompat[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"'))

          [[ "$VERSION" == "$(echo $CARGO_32_VERSION | sed 's/-beta\./b/')" ]] || (echo "Incorrect cargo 32 version" && exit 1)
          [[ "$VERSION" == "$(echo $CARGO_64_VERSION | sed 's/-beta\./b/')" ]] || (echo "Incorrect cargo 64 version" && exit 1)
          [[ "$VERSION" == "$(echo $CARGO_COMPAT_VERSION | sed 's/-beta\./b/')" ]] || (echo "Incorrect cargo compat version" && exit 1)
          [[ "$VERSION" == "${RT32_VERSION[2]}" ]] || (echo "Incorrect rt32 version" && exit 1)
          [[ "$VERSION" == "${RT64_VERSION[2]}" ]] || (echo "Incorrect rt64 version" && exit 1)
          [[ "$VERSION" == "${COMPAT_VERSION[2]}" ]] || (echo "Incorrect rtcompat version" && exit 1)

      - name: Install uv
        run: pip install uv

      - name: Build sdist and wheel
        run: |
          uv build py-polars --out-dir dist

      - name: Upload base package
        uses: actions/upload-artifact@v6
        with:
          path: dist

  create-sdist:
    needs: [base-package]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        package: [polars-runtime-32, polars-runtime-64, polars-runtime-compat]

    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.sha }}

      - name: Download base package
        uses: actions/download-artifact@v7
        with:
          path: dist
          merge-multiple: true

      # Avoid potential out-of-memory errors
      - name: Set swap space for Linux
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install yq
        run: |
          pip install --force-reinstall --no-dependencies dist/polars-*.whl
          rm -rf dist/*
          pip install yq

      - name: Create source distribution
        uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: >
            --manifest-path py-polars/runtime/${{ matrix.package }}/Cargo.toml
            --out dist
          maturin-version: 1.8.3

      - name: Test sdist
        run: |
          pip install --force-reinstall --verbose dist/*.tar.gz
          python -c 'import polars; polars.show_versions()'

      - name: Upload sdist
        uses: actions/upload-artifact@v6
        with:
          name: sdist-${{ matrix.package }}
          path: dist/*.tar.gz

  build-wheels:
    needs: [base-package]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        package: [polars-runtime-32, polars-runtime-64, polars-runtime-compat]
        job_config: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, macos-x64, macos-arm64, win-x64, win-arm64]
        include:
          - job_config: linux-x64
            os: ubuntu-latest
            architecture: x86-64
            target: x86_64-unknown-linux-gnu
          - job_config: linux-arm64
            os: linux-arm-large
            architecture: aarch64
            target: aarch64-unknown-linux-gnu
          - job_config: linux-musl-x64
            os: ubuntu-latest
            architecture: x86-64
            target: x86_64-unknown-linux-musl
          - job_config: linux-musl-arm64
            os: linux-arm-large
            architecture: aarch64
            target: aarch64-unknown-linux-musl
          - job_config: macos-x64
            os: macos-15-intel
            architecture: x86-64
            target: x86_64-apple-darwin
          - job_config: macos-arm64
            os: macos-15
            architecture: aarch64
            target: aarch64-apple-darwin
          - job_config: win-x64
            os: windows-latest
            architecture: x86-64
            target: x86_64-pc-windows-msvc
          - job_config: win-arm64
            os: windows-11-arm
            architecture: aarch64
            target: aarch64-pc-windows-msvc

    steps:
      - name: Setup build environment (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        shell:
          powershell
          # Notes
          # * We update `Expand-Archive` to avoid "" is not a supported archive file format when extracting
          #   files that don't end in `.zip`
        run: |
          Write-Output "> Update Expand-Archive (Microsoft.PowerShell.Archive)"
          Install-PackageProvider -Name NuGet -Force
          Install-Module -Name Microsoft.PowerShell.Archive -Force

          Write-Output "> Setup bash.exe (git-for-windows/PortableGit)"
          Invoke-WebRequest "https://github.com/git-for-windows/git/releases/download/v2.53.0.windows.2/PortableGit-2.53.0.2-arm64.7z.exe" -OutFile /git.7z.exe
          /git.7z.exe -o/git -y

          Write-Output "> Setup Rust"
          Invoke-WebRequest "https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe" -OutFile /rustup-init.exe
          /rustup-init.exe --default-host aarch64-pc-windows-msvc -y

          Write-Output "> Setup VS Build Tools"
          Invoke-WebRequest "https://aka.ms/vs/17/release/vs_BuildTools.exe" -OutFile /vs_BuildTools.exe
          Start-Process C:/vs_BuildTools.exe -ArgumentList " `
              --add Microsoft.VisualStudio.Workload.NativeDesktop `
              --add Microsoft.VisualStudio.Workload.VCTools `
              --add Microsoft.VisualStudio.Component.VC.Tools.arm64 `
              --add Microsoft.VisualStudio.Component.VC.Llvm.Clang `
              --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset `
              --includeRecommended --quiet --norestart --wait" -Wait

          Write-Output "> Setup CMake"
          Invoke-WebRequest "https://github.com/Kitware/CMake/releases/download/v4.2.3/cmake-4.2.3-windows-arm64.zip" -OutFile /cmake.zip
          Expand-Archive /cmake.zip -DestinationPath /

          Write-Output "> Download jq.exe (github.com/jqlang) (needed for tomlq / yq)"
          Invoke-WebRequest https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-windows-i386.exe -OutFile /jq.exe

          Write-Output "> Update GITHUB_PATH"
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/git/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + $Env:USERPROFILE + "/.cargo/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/bin/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/cmake-4.2.3-windows-arm64/bin")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/")
          [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n")

          Get-Content $Env:GITHUB_PATH | Out-Host

      - name: Check build environment (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        run: |
          set -x
          bash --version
          rustup show
          clang -v
          cmake --version

      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.sha }}

      - name: Free some disk space (remove unused packages)
        uses: jlumbroso/free-disk-space@v1.3.1
        if: ${{ runner.os == 'Linux' }}

      - name: Download base package
        uses: actions/download-artifact@v7
        with:
          path: dist
          merge-multiple: true

      # Avoid potential out-of-memory errors
      - name: Set swap space for Linux
        if: runner.os == 'Linux'
        uses: pierotofy/set-swap-space@master
        with:
          swap-size-gb: 10

      - name: Set up Python
        if: matrix.os != 'windows-11-arm'
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Set up Python (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION_WIN_ARM64 }}

      # Otherwise can't find `tomlq` after `pip install yq`
      - name: Add Python scripts folder to GITHUB_PATH (ARM64 Windows)
        if: matrix.os == 'windows-11-arm'
        run: |
          python -c "import sysconfig; print(sysconfig.get_path('scripts'))" >> $GITHUB_PATH

      - name: Install yq
        run: |
          pip install --force-reinstall --no-dependencies dist/polars-*.whl
          rm -rf dist/*
          pip install yq

      - name: Determine CPU features for x86-64
        id: features
        if: matrix.architecture == 'x86-64'
        env:
          IS_RT_COMPAT: ${{ matrix.package == 'polars-runtime-compat' }}
        # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py
        run: |
          if [[ "$IS_RT_COMPAT" = true ]]; then
            TUNE_CPU="$COMPAT_TUNE_CPU"
            FEATURES="$COMPAT_FEATURES"
            CC_FEATURES="$COMPAT_CC_FEATURES"
          else
            TUNE_CPU="$NONCOMPAT_TUNE_CPU"
            FEATURES="$NONCOMPAT_FEATURES"
            CC_FEATURES="$NONCOMPAT_CC_FEATURES"
          fi
          echo "features=$FEATURES" >> $GITHUB_OUTPUT
          echo "tune_cpu=$TUNE_CPU" >> $GITHUB_OUTPUT
          echo "cc_features=$CC_FEATURES" >> $GITHUB_OUTPUT

      - name: Set RUSTFLAGS for x86-64
        if: matrix.architecture == 'x86-64'
        env:
          FEATURES: ${{ steps.features.outputs.features }}
          TUNE_CPU: ${{ steps.features.outputs.tune_cpu }}
          CC_FEATURES: ${{ steps.features.outputs.cc_features }}
          CFG: ${{ matrix.package == 'polars-runtime-compat' && '--cfg allocator="default"' || '' }}
        run: |
          if [[ -z "$TUNE_CPU" ]]; then
            echo "RUSTFLAGS=-C target-feature=$FEATURES $CFG" >> $GITHUB_ENV
            echo "CFLAGS=$CC_FEATURES" >> $GITHUB_ENV
          else
            echo "RUSTFLAGS=-C target-feature=$FEATURES -Z tune-cpu=$TUNE_CPU $CFG" >> $GITHUB_ENV
            echo "CFLAGS=$CC_FEATURES -mtune=$TUNE_CPU" >> $GITHUB_ENV
          fi

      - name: Set jemalloc for aarch64 Linux
        if: matrix.architecture == 'aarch64' && runner.os == 'Linux'
        run: |
          echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV

      - name: Copy toolchain to py-polars/ (ARM64 Windows)
        # Manual fix for:
        #   TomlError: Unknown character "46" at row 1, col 2, pos 1:
        #   1> ../rust-toolchain.toml
        if: matrix.os == 'windows-11-arm'
        run: cp rust-toolchain.toml py-polars/runtime/${{ matrix.package }}/rust-toolchain.toml

      - name: Build wheel
        uses: PyO3/maturin-action@v1
        with:
          command: build
          target: ${{ matrix.target }}
          args: >
            --profile ${{ inputs.dry-run == true && 'mindebug-dev' || 'dist-release' }}
            --manifest-path py-polars/runtime/${{ matrix.package }}/Cargo.toml
            --out dist
          manylinux: auto
          maturin-version: 1.8.3

      - name: Test wheel
        # Testing on entire matrix except musl targets, which are missing on all ubuntu-x images.
        if: endsWith(matrix.target, '-musl') == false
        run: |
          pip install --force-reinstall --verbose dist/*.whl
          python -c 'import polars; polars.show_versions()'

      - name: Upload wheel
        uses: actions/upload-artifact@v6
        with:
          name: wheel-${{ matrix.package }}-${{ matrix.job_config }}
          path: dist/*.whl

  publish-to-pypi:
    needs: [base-package, create-sdist, build-wheels]
    environment:
      name: release-python
      url: https://pypi.org/project/polars
    runs-on: ubuntu-latest
    permissions:
      id-token: write

    steps:
      - name: Download sdists and wheels
        uses: actions/download-artifact@v7
        with:
          path: dist
          merge-multiple: true

      - name: Remove Emscripten wheel
        run: rm -f dist/*emscripten*.whl

      - name: Publish to PyPI
        if: inputs.dry-run == false
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          skip-existing: true
          verbose: true

  publish-to-github:
    needs: [publish-to-pypi]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ inputs.sha }}

      - name: Get version from Cargo.toml
        id: version
        run: |
          VERSION=$(grep -m 1 -oP 'version = "\K[^"]+' py-polars/runtime/template/Cargo.template.toml)
          if [[ "$VERSION" == *"-"* ]]; then
            IS_PRERELEASE=true
          else
            IS_PRERELEASE=false
          fi
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "is_prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT

      - name: Create GitHub release
        id: github-release
        uses: release-drafter/release-drafter@v6
        with:
          config-name: release-drafter-python.yml
          name: Python Polars ${{ steps.version.outputs.version }}
          tag: py-${{ steps.version.outputs.version }}
          version: ${{ steps.version.outputs.version }}
          prerelease: ${{ steps.version.outputs.is_prerelease }}
          commitish: ${{ inputs.sha || github.sha }}
          disable-autolabeler: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Publish GitHub release
        if: inputs.dry-run == false
        run: gh release edit $TAG --draft=false
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TAG: ${{ steps.github-release.outputs.tag_name }}

      - name: Trigger other workflows related to the release
        if: inputs.dry-run == false
        uses: peter-evans/repository-dispatch@v4
        with:
          event-type: python-release
          client-payload: >
            {
              "version": "${{ steps.version.outputs.version }}",
              "is_prerelease": "${{ steps.version.outputs.is_prerelease }}",
              "tag": "${{ steps.github-release.outputs.tag_name }}",
              "sha": "${{ inputs.sha || github.sha }}"
            }