Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 52 additions & 8 deletions .github/workflows/delta_spark_ut.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ on:
# Negated patterns are evaluated in order and override the positives
# above, so this must stay last.
- '!.github/workflows/util/delta-spark-ut/**/*.md'
# Force a run skipped by `paths:` without requiring label permissions.
issue_comment:
types: [created]
workflow_dispatch:
inputs:
delta_ref:
Expand Down Expand Up @@ -92,10 +95,17 @@ on:
schedule:
- cron: '0 5 * * *'

# Comment runs execute PR code; keep the token read-only. Stash needs artifacts access.
permissions:
contents: read
actions: read

env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MVN_CMD: 'build/mvn -ntp'
CCACHE_DIR: "${{ github.workspace }}/.ccache"
# `issue_comment` runs on the default branch, so explicitly select the PR merge ref.
DELTA_CHECKOUT_REF: ${{ github.event.issue.number && format('refs/pull/{0}/merge', github.event.issue.number) || '' }}
# Gluten profile / bundle naming for the build-gluten-bundle and
# delta-spark-test jobs. `spark_version` is the single source of truth for the
# Spark version: it drives the Gluten bundle profile (-Pspark-<v>), the bundle
Expand Down Expand Up @@ -136,22 +146,48 @@ env:
# granularity for the split to stay balanced.
DELTA_NUM_SHARDS: '8'

# Now that this is a standalone workflow (not called by velox_backend_x86.yml),
# `github.workflow` resolves to THIS workflow, so a concurrency group is both safe
# and necessary: without it every push to a PR branch stacks another full ~2.5 h
# Delta run instead of superseding the previous one. Keyed on the branch for PRs
# and the sha otherwise, matching velox_backend_x86.yml's group.
# Cancel older `pull_request` and `/delta-test` runs for the same PR.
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
group: ${{ github.repository }}-${{ github.event.pull_request.number || github.event.issue.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
delta-test-requested:
# Match `/delta-test` as the first token; pass other event types through.
if: >-
github.event_name != 'issue_comment' ||
(github.event.issue.pull_request &&
(github.event.comment.body == '/delta-test' ||
startsWith(github.event.comment.body, '/delta-test ') ||
startsWith(github.event.comment.body, format('/delta-test{0}', fromJSON('"\r"'))) ||
startsWith(github.event.comment.body, format('/delta-test{0}', fromJSON('"\n"')))))
runs-on: ubuntu-22.04
permissions:
pull-requests: write
steps:
- name: Acknowledge the request
if: ${{ github.event_name == 'issue_comment' }}
# The informational comment must not block the run.
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.issue.number }}
TRIGGER_USER: ${{ github.event.comment.user.login }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
gh pr comment "$PR_NUMBER" --repo "$REPO" --body \
"🔄 Delta Spark UT started by @${TRIGGER_USER} (~2.5 h). [View run](${RUN_URL})"

build-native-lib-centos-7:
needs: delta-test-requested
# This workflow always builds its own native lib -- there is no caller to
# provide one, and the `paths:` filter already decided whether the run happens.
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
ref: ${{ env.DELTA_CHECKOUT_REF }}
- name: Get Ccache from Apache Stash
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
Expand All @@ -172,7 +208,8 @@ jobs:
ccache -s
"
- name: Save Ccache to Apache Stash
if: always()
# Comment runs use main's Stash scope; never overwrite it with PR output.
if: ${{ always() && github.event_name != 'issue_comment' }}
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: '${{ env.CCACHE_DIR }}'
Expand All @@ -190,6 +227,8 @@ jobs:
container: apache/gluten:centos-9-jdk17
steps:
- uses: actions/checkout@v4
with:
ref: ${{ env.DELTA_CHECKOUT_REF }}
- name: Download native artifacts
uses: actions/download-artifact@v4
with:
Expand Down Expand Up @@ -232,6 +271,7 @@ jobs:
-Pbackends-velox -Pdelta \
-DskipTests -Dmaven.compiler.release=17
- name: Save Maven repository to Apache Stash
if: ${{ github.event_name != 'issue_comment' }}
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: /root/.m2/repository
Expand Down Expand Up @@ -282,6 +322,8 @@ jobs:
SHARD_ID: ${{ matrix.shard }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ env.DELTA_CHECKOUT_REF }}

- name: Resolve workflow inputs
id: resolve
Expand Down Expand Up @@ -395,7 +437,7 @@ jobs:
- name: Save sbt / Ivy / Coursier to Apache Stash
# All shards have the same dependencies; one writer avoids matrix jobs
# overwriting the same stash artifact.
if: ${{ success() && matrix.shard == 0 }}
if: ${{ success() && matrix.shard == 0 && github.event_name != 'issue_comment' }}
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
with:
path: |
Expand Down Expand Up @@ -478,6 +520,8 @@ jobs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
ref: ${{ env.DELTA_CHECKOUT_REF }}
- name: Download per-shard gate lists
uses: actions/download-artifact@v4
continue-on-error: true
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/util/delta-spark-ut/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,12 @@ longer shared between them, those PRs pay for the centos-7 native build twice
still caught daily. The nightly run enforces the baseline **and** fails on
now-passing tests (`fail_on_fixed=true`), so baseline drift surfaces as a red
nightly — the signal to refresh `known-failures.txt`.
- **On demand, from a PR** — anyone can comment **`/delta-test`** as the first
token to run the default configuration against the PR merge ref. This covers
PRs skipped by `paths:` without requiring label permissions. The workflow
posts the run link on the PR and enforces the committed baseline. Comment runs
restore but never save main-scoped Stash caches; use `workflow_dispatch` for
custom inputs or baseline updates.
- **Manually** — **Actions → Delta Spark UT (Gluten) → Run workflow**
(`workflow_dispatch`), e.g. to refresh the baseline (see below). This is also
how you validate a Velox/core change against Delta before merging: run it on
Expand Down
Loading