ci-python.yml

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

name: CI / Python

on:
  workflow_call:

permissions:
  contents: read

env:
  JDK_VERSION: 8
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
  LUMINA_DATA_VERSION: 0.1.0


jobs:
  # Lint + test on selected versions only (3.6 / 3.7 low end, 3.10 / 3.11 / 3.12 / 3.13 current), not every Python version.
  test:
    name: Tests / Python ${{ matrix.python-version }}
    env:
      # Real native planning is covered by the Rust main job below.
      PYTEST_ADDOPTS: "-m 'not native_plan'"
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:${{ matrix.python-version }}-slim"
    strategy:
      fail-fast: false
      matrix:
        python-version: [ '3.6.15', '3.7', '3.10', '3.11', '3.12', '3.13' ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v5
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v5
        with:
          path: ~/.m2/repository
          key: ci-maven-python-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ci-maven-python-${{ runner.os }}-${{ runner.arch }}-

      - name: Set up Maven
        uses: stCarolas/setup-maven@v5
        with:
          maven-version: 3.8.8

      - name: Install system dependencies
        shell: bash
        run: |
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            # The Python 3.6 image uses EOL Debian Bullseye.
            sed -i '/security.debian.org\/debian-security/d' /etc/apt/sources.list
            sed -i 's|deb.debian.org/debian|archive.debian.org/debian|g' /etc/apt/sources.list
            echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive
          fi
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            pkg-config \
            libssl-dev \
            libclang-dev \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Verify requirements.txt dependencies can be installed
        shell: bash
        run: |
          cd paimon-python
          python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || {
            echo "ERROR: Failed to resolve dependencies from dev/requirements.txt"
            rm -rf "$TEMP_DIR"
            exit 1
          }
          rm -rf "$TEMP_DIR"
          echo "✓ dev/requirements.txt can be resolved on this Python"

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B -ntp install -DskipTests \
            -pl paimon-core,paimon-lance,paimon-vortex/paimon-vortex-format,paimon-full-text,paimon-lumina,paimon-vector -am

      - name: Install Python dependencies
        shell: bash
        run: |
          df -h
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            python -m pip install --upgrade pip==21.3.1
            python --version
            python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 datasketches==4.1.0 2>&1 >/dev/null
            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
          elif [[ "${{ matrix.python-version }}" == "3.7" ]]; then
            # 3.7 installs the version-pinned set declared for 3.7 in dev/requirements.txt.
            # ray/lance/daft/torch have no 3.7 wheels; those tests are importorskip-ed (as on 3.6.15).
            python -m pip install --upgrade pip
            python -m pip install -r paimon-python/dev/requirements.txt
            python -m pip install flake8==4.0.1 'pytest~=7.0' py4j==0.10.9.9 parameterized==0.9.0 datasketches==4.1.0
            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
          else
            python -m pip install --upgrade pip
            pip install torch --index-url https://download.pytorch.org/whl/cpu
            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 'isal>=1.8,<2' zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 pylance==0.39.0 cramjam pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=54,<55' datasketches
            if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)"; then
              python -m pip install pyarrow==24.0.0 numpy==2.4.6 pandas==2.3.3 flake8==7.1.2
            else
              python -m pip install pyarrow==16.0.0 numpy==1.24.3 pandas==2.0.3 flake8==4.0.1
            fi
            python -m pip install 'h5py>=3,<4'
            python -c "import h5py; print('h5py', h5py.__version__)"
            if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)"; then
              python -m pip install './paimon-python[rosbag]'
              python -c "import rosbags; print('rosbags installed')"
            fi
            if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
              python -m pip install './paimon-python[lerobot]'
              python -c "import datasets, lerobot; print('datasets', datasets.__version__, 'lerobot', lerobot.__version__)"
            fi

            # Required by SQL tests; native planner tests run against main below.
            python -m pip install pypaimon-rust
            if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
              # Run the RoboMIND pipeline tests with synthetic local HDF5 data.
              python -m pip install "./paimon-python[sql]"
            fi
            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
            if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then
              # Vortex requires pyarrow >= 17, while Pypaimon requires pyarrow < 20 and != 19.
              python -m pip install vortex-data==0.70.0 pyarrow==18.1.0
            fi
            python -m pip install 'paimon-mosaic>=0.1.0'
            if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
              python -m pip check
            fi
          fi
          df -h

      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -e pytest_torch

  rust-plan:
    name: Rust Plan
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:3.11-slim"
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          apt-get update && apt-get install -y --no-install-recommends \
            build-essential git curl cmake pkg-config libssl-dev libclang-dev \
            && rm -rf /var/lib/apt/lists/*

      - name: Install Rust toolchain
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - name: Build paimon-rust main
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade "git+https://github.com/apache/paimon-rust.git@main#subdirectory=bindings/python"

      - name: Install Python test dependencies
        run: |
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install "./paimon-python[sql,rosbag]" \
            readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \
            ray==2.54.0 fastavro==1.11.1 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \
            pylance==0.39.0 'pytest~=7.0' py4j==0.10.9.9 parameterized==0.9.0 \
            'daft>=0.7.6' datasketches pyarrow==18.1.0 numpy==1.24.3 pandas==2.0.3 \
            'h5py>=3,<4' 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' \
            vortex-data==0.70.0 'paimon-mosaic>=0.1.0'
          python -m pip check

      - name: Verify upstream native planning APIs
        run: |
          python - <<'PY'
          import importlib.metadata as metadata
          import json

          from pypaimon_rust.datafusion import PaimonCatalog, ReadBuilder, Split

          distribution = metadata.distribution('pypaimon-rust')
          source = json.loads(distribution.read_text('direct_url.json'))
          print('pypaimon-rust', distribution.version,
                'commit', source['vcs_info']['commit_id'])
          assert hasattr(PaimonCatalog, 'get_table'), 'Missing PaimonCatalog.get_table'
          assert hasattr(Split, 'serialize'), 'Missing Split.serialize'
          assert hasattr(Split, 'is_streaming'), 'Missing stream-aware Split.is_streaming'
          assert hasattr(ReadBuilder, 'new_incremental_scan'), 'Missing ReadBuilder.new_incremental_scan'
          assert hasattr(ReadBuilder, 'with_row_ranges'), 'Missing ReadBuilder.with_row_ranges'
          PY

      - name: Run Python tests with Rust planning
        shell: bash
        env:
          PYPAIMON_TEST_NATIVE_PLAN: '1'
        run: |
          bash paimon-python/dev/lint-python.sh -i pytest

  torch:
    name: PyTorch / Python 3.10
    timeout-minutes: 60
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Python version
        run: python --version

      - name: Install Python dependencies
        shell: bash
        run: |
            python -m pip install --upgrade pip
            pip install torch --index-url https://download.pytorch.org/whl/cpu
            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 datasketches 'datasets>=4,<4.1'
            python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -i pytest_torch

  # One job: check dev/requirements.txt on each Python version in sequence, then Ray version test on 3.10.
  compatibility:
    name: Requirements and Ray compatibility
    timeout-minutes: 60
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          sudo apt-get update && sudo apt-get install -y --no-install-recommends \
            build-essential git curl cmake \
            && sudo rm -rf /var/lib/apt/lists/*

      - name: Verify dev/requirements.txt on Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - name: Resolve requirements (3.8)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.8"

      - name: Verify dev/requirements.txt on Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Resolve requirements (3.9)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.9"

      - name: Verify dev/requirements.txt on Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Resolve requirements (3.10)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.10"

      - name: Verify dev/requirements.txt on Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Resolve requirements (3.11)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.11"

      - name: Verify dev/requirements.txt on Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Resolve requirements (3.12)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.12"

      - name: Verify dev/requirements.txt on Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - name: Resolve requirements (3.13)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.13"

      - name: Setup Python 3.10 for Ray test
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install base Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install --no-cache-dir \
            pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \
            fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \
            numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9 requests \
            parameterized==0.9.0 packaging datasketches
          python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
      - name: Test Ray version compatibility
        run: |
          cd paimon-python
          echo "=========================================="
          echo "Testing Ray version compatibility"
          echo "=========================================="
          for ray_version in 2.44.0 2.48.0 2.50.1 2.53.0; do
            echo "Testing Ray version: $ray_version"
            python -m pip install --no-cache-dir -q ray==$ray_version
            python -c "import ray; print(f'Ray version: {ray.__version__}')"
            python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
            tests="pypaimon/tests/ray_data_test.py::RayDataTest pypaimon/tests/ray_range_join_test.py::RayRangeJoinTest"
            if [ "$ray_version" = "2.50.1" ]; then
              tests="$tests pypaimon/tests/ray_update_by_row_id_test.py::RayUpdateByRowIdTest::test_empty_dataset_after_transform_is_noop"
            fi
            if [ "$ray_version" = "2.44.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_legacy_map_batches_cardinality_is_unknown"
            fi
            if [ "$ray_version" = "2.53.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_metadata_respects_transform_cardinality"
            fi
            python -m pytest $tests -v --tb=short || {
              echo "Tests failed for Ray $ray_version"; python -m pip uninstall -y ray; exit 1;
            }
            python -m pip uninstall -y ray
          done
        env:
          PYTHONPATH: ${{ github.workspace }}/paimon-python