From 17d62e71579403444bb80500f5f1c6422e95b0f8 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 9 Jul 2026 16:28:53 +0800 Subject: [PATCH] feat(CI): Keep local and Github CI test in sync with `cargo xtask` --- .cargo/config.toml | 19 + .github/workflows/rust.yml | 440 ++++++++------------- Cargo.lock | 4 + Cargo.toml | 1 + xtask/Cargo.toml | 26 ++ xtask/src/ci_jobs.rs | 270 +++++++++++++ xtask/src/ci_timing.rs | 152 ++++++++ xtask/src/ci_workflow_rust.rs | 44 +++ xtask/src/ci_workflows.rs | 29 ++ xtask/src/main.rs | 361 +++++++++++++++++ xtask/src/step_runners.rs | 701 ++++++++++++++++++++++++++++++++++ xtask/src/steps.rs | 227 +++++++++++ xtask/src/utils.rs | 145 +++++++ 13 files changed, 2140 insertions(+), 279 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 xtask/Cargo.toml create mode 100644 xtask/src/ci_jobs.rs create mode 100644 xtask/src/ci_timing.rs create mode 100644 xtask/src/ci_workflow_rust.rs create mode 100644 xtask/src/ci_workflows.rs create mode 100644 xtask/src/main.rs create mode 100644 xtask/src/step_runners.rs create mode 100644 xtask/src/steps.rs create mode 100644 xtask/src/utils.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000000..3e143544580a4 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[alias] +xtask = "run --quiet --package datafusion-xtask --" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b317dbbf6241e..c0068527d3135 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,7 +15,91 @@ # specific language governing permissions and limitations # under the License. -# For some actions, we use Runs-On to run them on ASF infrastructure: https://datafusion.apache.org/contributor-guide/#ci-runners +# ------------ +# Quick start +# ------------ +# +# Reproduce this workflow locally: +# cargo xtask ci workflow rust +# +# Run one CI step locally: +# cargo xtask ci step clippy +# +# ------------- +# Architecture +# ------------- +# +# Each atomic CI check lives in an individual `xtask` step. The GitHub workflow +# is only a thin wrapper that calls a list of `xtask` steps. A `xtask` step looks like: +# +# cargo xtask ci step +# +# `xtask` also provides local orchestration for this `rust.yml` workflow. It similarly +# calls the same list of `xtask` steps. For example the below command is a local +# reproducer for this Github Workflow: +# +# cargo xtask ci workflow rust +# +# Example: +# +# (rust.yml) +# ... +# clippy: +# ... +# steps: +# - name: Run clippy +# run: cargo xtask ci step clippy # <--- Atomic CI check item +# ... +# +# +# ---------------------- +# Adding a new check +# ---------------------- +# +# First wrap the atomic check in an xtask step: +# cargo xtask ci step new-check +# +# Then add that step to the Rust workflow group in `xtask/src/job_groups.rs` so +# local reproduction includes it: +# cargo xtask ci workflow rust # it should include 'new-check' +# +# ---------------------- +# Job dependency graph +# ---------------------- +# +# (@ means no `needs` dependency.) +# +# @ -> linux-build-lib +# -> linux-datafusion-common-features +# -> linux-datafusion-substrait-features +# -> linux-datafusion-proto-features +# -> linux-cargo-check-datafusion +# -> linux-cargo-check-datafusion-functions +# -> linux-test +# -> linux-test-datafusion-cli +# -> linux-test-example +# -> linux-test-doc +# -> linux-rustdoc +# -> verify-benchmark-results +# -> sqllogictest-postgres +# -> sqllogictest-substrait +# -> clippy +# -> cargo-toml-formatting-checks +# -> config-docs-check +# -> examples-docs-check +# +# @ -> linux-wasm-pack +# @ -> macos-aarch64 +# @ -> vendor +# @ -> check-fmt +# @ -> msrv +# +# ------------ +# Runner info +# ------------ +# +# Some jobs use Runs-On to run on ASF infrastructure: +# https://datafusion.apache.org/contributor-guide/#ci-runners name: Rust @@ -44,7 +128,7 @@ permissions: contents: read jobs: - # Check crate compiles and base cargo check passes + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-build-lib: name: linux build test runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} @@ -63,16 +147,9 @@ jobs: shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs save-if: ${{ github.ref_name == 'main' }} - name: Prepare cargo build - run: | - # Adding `--locked` here to assert that the `Cargo.lock` file is up to - # date with the manifest. When this fails, please make sure to commit - # the changes to `Cargo.lock` after building with the updated manifest. - cargo check --profile ci --workspace --all-targets --features integration-tests --locked + run: cargo xtask ci step rust-check workspace - # Check datafusion-common features - # - # Ensure via `cargo check` that the crate can be built with a - # subset of the features packages enabled. + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-datafusion-common-features: name: cargo check datafusion-common features needs: linux-build-lib @@ -85,19 +162,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Check datafusion-common (default features) - run: cargo check --profile ci --all-targets -p datafusion-common - # - # Note: Only check libraries (not --all-targets) to cover end user APIs - # - - name: Check datafusion-common (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-common - # Note: don't check other feature flags as datafusion-common is not typically used standalone - - # Check datafusion-substrait features - # - # Ensure via `cargo check` that the crate can be built with a - # subset of the features packages enabled. + - name: Check datafusion-common features + run: cargo xtask ci step rust-check datafusion-common + + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-datafusion-substrait-features: name: cargo check datafusion-substrait features needs: linux-build-lib @@ -115,26 +183,14 @@ jobs: with: save-if: false # set in linux-test shared-key: "amd-ci" - - name: Check datafusion-substrait (default features) - run: cargo check --profile ci --all-targets -p datafusion-substrait - # - # Note: Only check libraries (not --all-targets) to cover end user APIs - # - - name: Check datafusion-substrait (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-substrait - - name: Check datafusion-substrait (physical) - run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=physical - name: Install cmake run: | # note the builder setup runs apt-get update / installs protobuf compiler apt-get install -y cmake - - name: Check datafusion-substrait (protoc) - run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=protoc + - name: Check datafusion-substrait features + run: cargo xtask ci step rust-check datafusion-substrait - # Check datafusion-proto features - # - # Ensure via `cargo check` that the crate can be built with a - # subset of the features packages enabled. + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-datafusion-proto-features: name: cargo check datafusion-proto features needs: linux-build-lib @@ -148,25 +204,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Check datafusion-proto (default features) - run: cargo check --profile ci --all-targets -p datafusion-proto - # - # Note: Only check libraries (not --all-targets) to cover end user APIs - # - - name: Check datafusion-proto (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-proto - - name: Check datafusion-proto (json) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=json - - name: Check datafusion-proto (parquet) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=parquet - - name: Check datafusion-proto (avro) - run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro - - - # Check datafusion crate features - # - # Ensure via `cargo check` that the crate can be built with a - # subset of the features packages enabled. + - name: Check datafusion-proto features + run: cargo xtask ci step rust-check datafusion-proto + + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-cargo-check-datafusion: name: cargo check datafusion features needs: linux-build-lib @@ -185,54 +226,10 @@ jobs: with: save-if: false # set in linux-test shared-key: "amd-ci" - - name: Check datafusion (default features) - run: cargo check --profile ci --all-targets -p datafusion - # - # Note: Only check libraries (not --all-targets) to cover end user APIs - # - - name: Check datafusion (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion - - name: Check datafusion (nested_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=nested_expressions - - name: Check datafusion (array_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=array_expressions - - name: Check datafusion (avro) - run: cargo check --profile ci --no-default-features -p datafusion --features=avro - - name: Check datafusion (backtrace) - run: cargo check --profile ci --no-default-features -p datafusion --features=backtrace - - name: Check datafusion (compression) - run: cargo check --profile ci --no-default-features -p datafusion --features=compression - - name: Check datafusion (crypto_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=crypto_expressions - - name: Check datafusion (datetime_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=datetime_expressions - - name: Check datafusion (encoding_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=encoding_expressions - - name: Check datafusion (force_hash_collisions) - run: cargo check --profile ci --no-default-features -p datafusion --features=force_hash_collisions - - name: Check datafusion (math_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=math_expressions - - name: Check datafusion (parquet) - run: cargo check --profile ci --no-default-features -p datafusion --features=parquet - - name: Check datafusion (regex_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=regex_expressions - - name: Check datafusion (recursive_protection) - run: cargo check --profile ci --no-default-features -p datafusion --features=recursive_protection - - name: Check datafusion (serde) - run: cargo check --profile ci --no-default-features -p datafusion --features=serde - - name: Check datafusion (sql) - run: cargo check --profile ci --no-default-features -p datafusion --features=sql - - name: Check datafusion (string_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions - - name: Check datafusion (unicode_expressions) - run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions - - name: Check parquet encryption (parquet_encryption) - run: cargo check --profile ci --no-default-features -p datafusion --features=parquet_encryption - - # Check datafusion-functions crate features - # - # Ensure via `cargo check` that the crate can be built with a - # subset of the features packages enabled. + - name: Check datafusion features + run: cargo xtask ci step rust-check datafusion + + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-cargo-check-datafusion-functions: name: cargo check datafusion-functions features needs: linux-build-lib @@ -245,29 +242,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Check datafusion-functions (default features) - run: cargo check --profile ci --all-targets -p datafusion-functions - # - # Note: Only check libraries (not --all-targets) to cover end user APIs - # - - name: Check datafusion-functions (no-default-features) - run: cargo check --profile ci --no-default-features -p datafusion-functions - - name: Check datafusion-functions (crypto_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=crypto_expressions - - name: Check datafusion-functions (datetime_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=datetime_expressions - - name: Check datafusion-functions (encoding_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=encoding_expressions - - name: Check datafusion-functions (math_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=math_expressions - - name: Check datafusion-functions (regex_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=regex_expressions - - name: Check datafusion-functions (string_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=string_expressions - - name: Check datafusion-functions (unicode_expressions) - run: cargo check --profile ci --no-default-features -p datafusion-functions --features=unicode_expressions - - # Library and integration tests + - name: Check datafusion-functions features + run: cargo xtask ci step rust-check datafusion-functions + + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-test: name: cargo test (amd64) needs: linux-build-lib @@ -292,32 +270,13 @@ jobs: save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci" - name: Run tests (excluding doctests and datafusion-cli) - env: - RUST_BACKTRACE: 1 - run: | - cargo test \ - --profile ci \ - --exclude datafusion-examples \ - --exclude ffi_example_table_provider \ - --exclude datafusion-cli \ - --workspace \ - --lib \ - --tests \ - --bins \ - --features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait + run: cargo xtask ci step rust-test - name: Verify Working Directory Clean - run: git diff --exit-code - # Check no temporary directories created during test. - # `false/` folder is excuded for rust cache. + run: cargo xtask ci step verify-clean - name: Verify Working Directory Clean (No Untracked Files) - run: | - STATUS="$(git status --porcelain | sed -e '/^?? false\/$/d' -e '/^?? false$/d')" - if [ -n "$STATUS" ]; then - echo "$STATUS" - exit 1 - fi + run: cargo xtask ci step verify-clean --untracked - # datafusion-cli tests + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) needs: linux-build-lib @@ -336,18 +295,12 @@ jobs: save-if: false # set in linux-test shared-key: "amd-ci" - name: Run tests (excluding doctests) - env: - RUST_BACKTRACE: 1 - AWS_ENDPOINT: http://127.0.0.1:9000 - AWS_ACCESS_KEY_ID: TEST-DataFusionLogin - AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword - TEST_STORAGE_INTEGRATION: 1 - AWS_ALLOW_HTTP: true - run: cargo test --features backtrace --profile ci -p datafusion-cli --lib --tests --bins + run: cargo xtask ci step rust-test datafusion-cli - name: Verify Working Directory Clean - run: git diff --exit-code + run: cargo xtask ci step verify-clean + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-test-example: name: cargo examples (amd64) needs: linux-build-lib @@ -370,15 +323,11 @@ jobs: save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-linux-test-example" - name: Run examples - run: | - # test datafusion-sql examples - cargo run --profile ci --example sql - # test datafusion-examples - ci/scripts/rust_example.sh + run: cargo xtask ci step rust-examples - name: Verify Working Directory Clean - run: git diff --exit-code + run: cargo xtask ci step verify-clean - # Run `cargo test doc` (test documentation examples) + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-test-doc: name: cargo test doc (amd64) needs: linux-build-lib @@ -396,11 +345,11 @@ jobs: with: rust-version: stable - name: Run doctests - run: cargo test --profile ci --doc --features avro,json + run: cargo xtask ci step rust-doctest - name: Verify Working Directory Clean - run: git diff --exit-code + run: cargo xtask ci step verify-clean - # Run `cargo doc` to ensure the rustdoc is clean + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-rustdoc: name: cargo doc needs: linux-build-lib @@ -415,8 +364,9 @@ jobs: with: rust-version: stable - name: Run cargo doc - run: ci/scripts/rust_docs.sh + run: cargo xtask ci step rust-doc + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. linux-wasm-pack: name: build and run with wasm-pack runs-on: ubuntu-24.04 @@ -434,14 +384,9 @@ jobs: with: tool: wasm-pack - name: Run tests with headless mode - working-directory: ./datafusion/wasmtest - run: | - # debuginfo=none because CI tests weren't completing successfully after this upstream PR: - # https://github.com/wasm-bindgen/wasm-bindgen/pull/4635 - RUSTFLAGS='--cfg getrandom_backend="wasm_js" -C debuginfo=none' wasm-pack test --headless --firefox - RUSTFLAGS='--cfg getrandom_backend="wasm_js" -C debuginfo=none' wasm-pack test --headless --chrome --chromedriver $CHROMEWEBDRIVER/chromedriver + run: cargo xtask ci step rust-wasm - # verify that the benchmark queries return the correct results + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. verify-benchmark-results: name: verify benchmark results (amd64) needs: linux-build-lib @@ -458,24 +403,12 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Generate benchmark data and expected query results - run: | - mkdir -p datafusion/sqllogictest/test_files/tpch/data - git clone https://github.com/databricks/tpch-dbgen.git - cd tpch-dbgen - make - ./dbgen -f -s 0.1 - mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data - name: Verify that benchmark queries return expected results - run: | - # increase stack size to fix stack overflow - export RUST_MIN_STACK=20971520 - export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` - cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests + run: cargo xtask ci step rust-benchmark-results - name: Verify Working Directory Clean - run: git diff --exit-code + run: cargo xtask ci step verify-clean + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" needs: linux-build-lib @@ -507,14 +440,9 @@ jobs: with: rust-version: stable - name: Run sqllogictest - run: | - cd datafusion/sqllogictest - PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests - env: - # use postgres for the host here because we have specified a container for the job - POSTGRES_HOST: postgres - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + run: cargo xtask ci step rust-sqllogictest postgres + # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. sqllogictest-substrait: name: "Run sqllogictest in Substrait round-trip mode" needs: linux-build-lib @@ -532,11 +460,7 @@ jobs: with: rust-version: stable - name: Run sqllogictest - # TODO: Right now several tests are failing in Substrait round-trip mode, so this - # command cannot be run for all the .slt files. Run it for just one that works (limit.slt) - # until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed - # and this command can be run without filters. - run: cargo test -p datafusion-sqllogictest --test sqllogictests --features substrait -- --substrait-round-trip limit.slt + run: cargo xtask ci step rust-sqllogictest substrait # Temporarily commenting out the Windows flow, the reason is enormously slow running build # Waiting for new Windows 2025 github runner @@ -557,11 +481,8 @@ jobs: # export PATH=$PATH:$HOME/d/protoc/bin # cargo test --lib --tests --bins --features avro,json,backtrace - # macOS scope is narrowed to `datafusion-ffi`: the only bug class amd64 - # cannot reproduce is FFI cdylib loading (`.dylib` vs `.so` resolution in - # datafusion/ffi/src/tests/utils.rs). All other macos-only failures - # historically came from datafusion-benchmarks (now covered on amd64) or - # flaky sqllogictest metrics. + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. macos-aarch64: name: cargo test (macos-aarch64) runs-on: macos-15 @@ -574,8 +495,10 @@ jobs: uses: ./.github/actions/setup-macos-aarch64-builder - name: Run datafusion-ffi tests shell: bash - run: cargo test --profile ci -p datafusion-ffi --lib --tests --features integration-tests + run: cargo xtask ci step rust-test datafusion-ffi + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. vendor: name: Verify Vendored Code runs-on: ubuntu-latest @@ -588,11 +511,10 @@ jobs: with: rust-version: stable - name: Run gen - run: ./regen.sh - working-directory: ./datafusion/proto - - name: Verify workspace clean (if this fails, run ./datafusion/proto/regen.sh and check in results) - run: git diff --exit-code + run: cargo xtask ci step rust-vendor + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. check-fmt: name: Check cargo fmt runs-on: ubuntu-latest @@ -605,8 +527,7 @@ jobs: with: rust-version: stable - name: Run - run: | - ci/scripts/rust_fmt.sh + run: cargo xtask ci step fmt # Coverage job disabled due to # https://github.com/apache/datafusion/issues/3678 @@ -650,6 +571,8 @@ jobs: # continue-on-error: true # run: bash <(curl -s https://codecov.io/bash) + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. clippy: name: clippy needs: linux-build-lib @@ -666,16 +589,16 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Install Clippy - run: rustup component add clippy - name: Rust Dependency Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-clippy" - name: Run clippy - run: ci/scripts/rust_clippy.sh + run: cargo xtask ci step clippy + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. cargo-toml-formatting-checks: name: check Cargo.toml formatting needs: linux-build-lib @@ -691,12 +614,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Install taplo - run: cargo +stable install taplo-cli --version ^0.9 --locked - # if you encounter an error, try running 'taplo format' to fix the formatting automatically. - name: Check Cargo.toml formatting - run: taplo format --check + run: cargo xtask ci step rust-toml-format + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. config-docs-check: name: check configs.md and ***_functions.md is up-to-date needs: linux-build-lib @@ -716,22 +638,11 @@ jobs: - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 with: node-version: "20" - - name: Check if configs.md has been modified - run: | - # If you encounter an error, run './dev/update_config_docs.sh' and commit - ./dev/update_config_docs.sh - git diff --exit-code - - name: Check if any of the ***_functions.md has been modified - run: | - # If you encounter an error, run './dev/update_function_docs.sh' and commit - ./dev/update_function_docs.sh - git diff --exit-code - -# This job ensures `datafusion-examples/README.md` stays in sync with the source code: -# 1. Generates README automatically using the Rust examples docs generator -# (parsing documentation from `examples//main.rs`) -# 2. Formats the generated Markdown using DataFusion's standard Prettier setup -# 3. Compares the result against the committed README.md and fails if out-of-date + - name: Check if generated docs have been modified + run: cargo xtask ci step rust-config-docs + + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. examples-docs-check: name: check example README is up-to-date needs: linux-build-lib @@ -756,14 +667,10 @@ jobs: node-version: '18' - name: Run examples docs check script - run: | - bash ci/scripts/check_examples_docs.sh + run: cargo xtask ci step rust-examples-docs - # Verify MSRV for the crates which are directly used by other projects: - # - datafusion - # - datafusion-substrait - # - datafusion-proto - # - datafusion-cli + # Keep step-specific comments in the corresponding `StepInfo` entry in + # `xtask/src/steps.rs`. msrv: name: Verify MSRV (Min Supported Rust Version) runs-on: ubuntu-latest @@ -779,36 +686,11 @@ jobs: tool: cargo-msrv - name: Check datafusion - working-directory: datafusion/core - run: | - # If you encounter an error with any of the commands below it means - # your code or some crate in the dependency tree has a higher MSRV - # (Min Supported Rust Version) than the one specified in the - # `rust-version` key of `Cargo.toml`. - # - # To reproduce: - # 1. Install the version of Rust that is failing. - # 2. Run the command that failed with that version. - # - # Example: - # # MSRV looks like "1.80.0" and is specified in Cargo.toml. We can read the value with the following command: - # msrv="$(cargo metadata --format-version=1 | jq '.packages[] | select( .name == "datafusion" ) | .rust_version' -r)" - # echo "MSRV: ${msrv}" - # rustup install "${msrv}" - # cargo "+${msrv}" check - # - # To resolve, either: - # 1. Change your code to use older Rust features, - # 2. Revert dependency update - # 3. Update the MSRV version in `Cargo.toml` - # - # Please see the DataFusion Rust Version Compatibility Policy before - # updating Cargo.toml. You may have to update the code instead. - # https://github.com/apache/datafusion/blob/main/README.md#rust-version-compatibility-policy - cargo msrv --output-format json --log-target stdout verify + run: cargo xtask ci step rust-msrv datafusion - name: Check datafusion-substrait - working-directory: datafusion/substrait - run: cargo msrv --output-format json --log-target stdout verify + run: cargo xtask ci step rust-msrv datafusion-substrait - name: Check datafusion-proto - working-directory: datafusion/proto - run: cargo msrv --output-format json --log-target stdout verify + run: cargo xtask ci step rust-msrv datafusion-proto + +# When adding a new step, see the "Adding a new check" comment section at the top of +# this file. diff --git a/Cargo.lock b/Cargo.lock index 57df1b848150e..ad4b1f96a5803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2704,6 +2704,10 @@ dependencies = [ "wasm-bindgen-test", ] +[[package]] +name = "datafusion-xtask" +version = "0.1.0" + [[package]] name = "deranged" version = "0.5.8" diff --git a/Cargo.toml b/Cargo.toml index 769e5bd710949..12784c2f49e50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ members = [ "benchmarks", "datafusion/macros", "datafusion/doc", + "xtask", ] exclude = ["dev/depcheck"] resolver = "2" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000000000..ab3670cf41d8f --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-xtask" +version = "0.1.0" +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true diff --git a/xtask/src/ci_jobs.rs b/xtask/src/ci_jobs.rs new file mode 100644 index 0000000000000..628325ae7ac4c --- /dev/null +++ b/xtask/src/ci_jobs.rs @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::steps::StepCall; + +/// One GitHub Actions job expressed as an ordered list of local xtask steps. +pub(crate) struct JobInfo { + pub(crate) name: &'static str, + pub(crate) help_description: &'static str, + pub(crate) steps: &'static [StepCall], +} + +pub(crate) const JOBS: &[JobInfo] = &[ + JobInfo { + name: "verify-clean", + help_description: "verify working tree is clean", + steps: &[ + StepCall { + command: "verify-clean", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &["--untracked"], + }, + ], + }, + JobInfo { + name: "linux-build-lib", + help_description: "linux build test", + steps: &[StepCall { + command: "rust-check", + args: &["workspace"], + }], + }, + JobInfo { + name: "linux-datafusion-common-features", + help_description: "cargo check datafusion-common features", + steps: &[StepCall { + command: "rust-check", + args: &["datafusion-common"], + }], + }, + JobInfo { + name: "linux-datafusion-substrait-features", + help_description: "cargo check datafusion-substrait features", + steps: &[StepCall { + command: "rust-check", + args: &["datafusion-substrait"], + }], + }, + JobInfo { + name: "linux-datafusion-proto-features", + help_description: "cargo check datafusion-proto features", + steps: &[StepCall { + command: "rust-check", + args: &["datafusion-proto"], + }], + }, + JobInfo { + name: "linux-cargo-check-datafusion", + help_description: "cargo check datafusion features", + steps: &[StepCall { + command: "rust-check", + args: &["datafusion"], + }], + }, + JobInfo { + name: "linux-cargo-check-datafusion-functions", + help_description: "cargo check datafusion-functions features", + steps: &[StepCall { + command: "rust-check", + args: &["datafusion-functions"], + }], + }, + JobInfo { + name: "linux-test", + help_description: "cargo test (amd64)", + steps: &[ + StepCall { + command: "rust-test", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &["--untracked"], + }, + ], + }, + JobInfo { + name: "linux-test-datafusion-cli", + help_description: "cargo test datafusion-cli (amd64)", + steps: &[ + StepCall { + command: "rust-test", + args: &["datafusion-cli"], + }, + StepCall { + command: "verify-clean", + args: &[], + }, + ], + }, + JobInfo { + name: "linux-test-example", + help_description: "cargo examples (amd64)", + steps: &[ + StepCall { + command: "rust-examples", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &[], + }, + ], + }, + JobInfo { + name: "linux-test-doc", + help_description: "cargo test doc (amd64)", + steps: &[ + StepCall { + command: "rust-doctest", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &[], + }, + ], + }, + JobInfo { + name: "linux-rustdoc", + help_description: "cargo doc", + steps: &[StepCall { + command: "rust-doc", + args: &[], + }], + }, + JobInfo { + name: "linux-wasm-pack", + help_description: "build and run with wasm-pack", + steps: &[StepCall { + command: "rust-wasm", + args: &[], + }], + }, + JobInfo { + name: "verify-benchmark-results", + help_description: "verify benchmark results (amd64)", + steps: &[ + StepCall { + command: "rust-benchmark-results", + args: &[], + }, + StepCall { + command: "verify-clean", + args: &[], + }, + ], + }, + JobInfo { + name: "sqllogictest-postgres", + help_description: "Run sqllogictest with Postgres runner", + steps: &[StepCall { + command: "rust-sqllogictest", + args: &["postgres"], + }], + }, + JobInfo { + name: "sqllogictest-substrait", + help_description: "Run sqllogictest in Substrait round-trip mode", + steps: &[StepCall { + command: "rust-sqllogictest", + args: &["substrait"], + }], + }, + JobInfo { + name: "macos-aarch64", + help_description: "cargo test (macos-aarch64)", + steps: &[StepCall { + command: "rust-test", + args: &["datafusion-ffi"], + }], + }, + JobInfo { + name: "vendor", + help_description: "Verify Vendored Code", + steps: &[StepCall { + command: "rust-vendor", + args: &[], + }], + }, + JobInfo { + name: "check-fmt", + help_description: "Check cargo fmt", + steps: &[StepCall { + command: "fmt", + args: &[], + }], + }, + JobInfo { + name: "clippy", + help_description: "clippy", + steps: &[StepCall { + command: "clippy", + args: &[], + }], + }, + JobInfo { + name: "cargo-toml-formatting-checks", + help_description: "check Cargo.toml formatting", + steps: &[StepCall { + command: "rust-toml-format", + args: &[], + }], + }, + JobInfo { + name: "config-docs-check", + help_description: "check configs.md and ***_functions.md is up-to-date", + steps: &[StepCall { + command: "rust-config-docs", + args: &[], + }], + }, + JobInfo { + name: "examples-docs-check", + help_description: "check example README is up-to-date", + steps: &[StepCall { + command: "rust-examples-docs", + args: &[], + }], + }, + JobInfo { + name: "msrv", + help_description: "Verify MSRV (Min Supported Rust Version)", + steps: &[ + StepCall { + command: "rust-msrv", + args: &["datafusion"], + }, + StepCall { + command: "rust-msrv", + args: &["datafusion-substrait"], + }, + StepCall { + command: "rust-msrv", + args: &["datafusion-proto"], + }, + ], + }, +]; diff --git a/xtask/src/ci_timing.rs b/xtask/src/ci_timing.rs new file mode 100644 index 0000000000000..97e564ee1f842 --- /dev/null +++ b/xtask/src/ci_timing.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::cmp::Reverse; +use std::time::Duration; + +use crate::ci_jobs::JobInfo; +use crate::ci_workflows::WorkflowInfo; +use crate::steps::StepInfo; + +pub(crate) struct JobTiming { + job_info: &'static JobInfo, + /// step_name, execution time + per_step_timing: Vec<(&'static StepInfo, Duration)>, +} + +impl JobTiming { + pub(crate) fn new(job: &'static JobInfo) -> Self { + Self { + job_info: job, + per_step_timing: Vec::new(), + } + } + + pub(crate) fn add_step_timing(&mut self, step: &'static StepInfo, timing: Duration) { + self.per_step_timing.push((step, timing)); + } + + fn total_time(&self) -> Duration { + self.per_step_timing.iter().map(|pair| pair.1).sum() + } + + /// Sort each step's execution time, and print like + /// + /// Job finished in 47s: + /// step b: 30s + /// step a: 20s + /// step f: 1s + /// ... + pub(crate) fn display(&self) { + let mut sorted_by_duration = self.per_step_timing.clone(); + sorted_by_duration.sort_by_key(|step| Reverse(step.1)); + + println!( + "\nci job `{}` total execution time: {:?}", + self.job_info.name, + self.total_time() + ); + + for (step, timing) in sorted_by_duration { + println!(" ci step `{}`: {timing:?}", step.command); + } + } +} + +pub(crate) struct WorkflowTiming { + workflow_info: &'static WorkflowInfo, + /// job, execution time + per_job_timing: Vec, +} + +impl WorkflowTiming { + pub(crate) fn new(workflow: &'static WorkflowInfo) -> Self { + Self { + workflow_info: workflow, + per_job_timing: Vec::new(), + } + } + + pub(crate) fn add_job_timing(&mut self, job_timing: JobTiming) { + self.per_job_timing.push(job_timing); + } + + fn total_time(&self) -> Duration { + self.per_job_timing.iter().map(JobTiming::total_time).sum() + } + + pub(crate) fn display(&self) { + let mut sorted_by_duration = self.per_job_timing.iter().collect::>(); + sorted_by_duration.sort_by_key(|job_timing| Reverse(job_timing.total_time())); + + println!( + "\nci workflow `{}` total execution time: {:?}", + self.workflow_info.name, + self.total_time() + ); + + for job_timing in sorted_by_duration { + println!( + " ci job `{}`: {:?}", + job_timing.job_info.name, + job_timing.total_time() + ); + } + } +} + +pub(crate) struct WorkflowsTiming { + per_workflow_timing: Vec, +} + +impl WorkflowsTiming { + pub(crate) fn new() -> Self { + Self { + per_workflow_timing: Vec::new(), + } + } + + pub(crate) fn add_workflow_timing(&mut self, workflow_timing: WorkflowTiming) { + self.per_workflow_timing.push(workflow_timing); + } + + fn total_time(&self) -> Duration { + self.per_workflow_timing + .iter() + .map(WorkflowTiming::total_time) + .sum() + } + + pub(crate) fn display(&self) { + let mut sorted_by_duration = self.per_workflow_timing.iter().collect::>(); + sorted_by_duration + .sort_by_key(|workflow_timing| Reverse(workflow_timing.total_time())); + + println!( + "\nci workflows total execution time: {:?}", + self.total_time() + ); + + for workflow_timing in sorted_by_duration { + println!( + " ci workflow `{}`: {:?}", + workflow_timing.workflow_info.name, + workflow_timing.total_time() + ); + } + } +} diff --git a/xtask/src/ci_workflow_rust.rs b/xtask/src/ci_workflow_rust.rs new file mode 100644 index 0000000000000..96aeaf6e2cc8d --- /dev/null +++ b/xtask/src/ci_workflow_rust.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::ci_jobs::{JOBS, JobInfo}; +use crate::ci_workflows::WorkflowInfo; + +const VERIFY_CLEAN_JOB_NAME: &str = "verify-clean"; +const SKIPPED_JOBS: &[&str] = &[ + // TODO: Enable these after local setup is fixed. + "linux-wasm-pack", + "verify-benchmark-results", + "sqllogictest-postgres", +]; + +pub(crate) const WORKFLOW_RUST: WorkflowInfo = WorkflowInfo { + name: "rust", + help_usage: "ci workflow rust", + help_description: "Run Rust CI jobs locally", + jobs, +}; + +fn jobs() -> Vec<&'static JobInfo> { + let verify_clean = JOBS + .iter() + .find(|job| job.name == VERIFY_CLEAN_JOB_NAME) + .unwrap_or_else(|| panic!("missing `{VERIFY_CLEAN_JOB_NAME}` job")); + let mut jobs = vec![verify_clean]; + jobs.extend(JOBS.iter().filter(|job| !SKIPPED_JOBS.contains(&job.name))); + jobs +} diff --git a/xtask/src/ci_workflows.rs b/xtask/src/ci_workflows.rs new file mode 100644 index 0000000000000..775340cbf06d3 --- /dev/null +++ b/xtask/src/ci_workflows.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::ci_jobs::JobInfo; +use crate::ci_workflow_rust; + +/// One GitHub Actions workflow expressed as an ordered list of local xtask jobs. +pub(crate) struct WorkflowInfo { + pub(crate) name: &'static str, + pub(crate) help_usage: &'static str, + pub(crate) help_description: &'static str, + pub(crate) jobs: fn() -> Vec<&'static JobInfo>, +} + +pub(crate) const WORKFLOWS: &[WorkflowInfo] = &[ci_workflow_rust::WORKFLOW_RUST]; diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000000000..81bc17e35865d --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,361 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! See `cargo xtask help` for usage and `.github/workflows/rust.yml` for architecture. + +mod ci_jobs; +mod ci_timing; +mod ci_workflow_rust; +mod ci_workflows; +mod step_runners; +mod steps; +mod utils; + +use ci_jobs::JOBS as CI_JOBS; +use ci_timing::{JobTiming, WorkflowTiming, WorkflowsTiming}; +use ci_workflows::{WORKFLOWS as CI_WORKFLOWS, WorkflowInfo}; +use std::env; +use std::path::PathBuf; +use std::time::SystemTime; +use steps::STEPS; + +pub(crate) type Result = std::result::Result; + +fn main() { + let args = env::args().skip(1).collect::>(); + if let Err(error) = Xtask::new().and_then(|xtask| xtask.run(&args)) { + eprintln!("error: {error}"); + std::process::exit(1); + } +} + +pub(crate) struct Xtask { + pub(crate) root: PathBuf, +} + +impl Xtask { + fn new() -> Result { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let root = manifest_dir + .parent() + .ok_or_else(|| { + format!( + "could not find workspace root from {}", + manifest_dir.display() + ) + })? + .to_path_buf(); + + Ok(Self { root }) + } + + fn run(&self, args: &[String]) -> Result<()> { + let Some((command, rest)) = args.split_first() else { + self.print_help(); + return Ok(()); + }; + + if matches!(command.as_str(), "-h" | "--help" | "help" | "list") { + self.print_help(); + return Ok(()); + } + + match command.as_str() { + "ci" => self.ci(rest), + unknown => Err(format!( + "unknown command `{unknown}`. Run `cargo xtask help`." + )), + } + } + + fn print_help(&self) { + println!("DataFusion local CI runner"); + println!(); + println!("Run the same CI checks locally that GitHub Actions schedules."); + println!(); + println!("Quick Start:"); + println!(" Syntax:"); + println!(" cargo xtask ci step [args]"); + println!(" cargo xtask ci job "); + println!(" cargo xtask ci workflow [name]"); + println!(); + println!(" Examples:"); + println!(" cargo xtask ci step rust-test"); + println!(" Run one atomic Rust test step."); + println!(" cargo xtask ci job linux-test"); + println!(" Run one GitHub Actions job locally."); + println!(" cargo xtask ci step fmt --write --allow-dirty"); + println!(" Run best-effort formatting auto-fixes."); + println!(" cargo xtask ci workflow rust"); + println!(" Reproduce `.github/workflows/rust.yml` locally."); + println!(); + println!("Concepts:"); + println!( + " step Atomic CI check item, runnable with `cargo xtask ci step `" + ); + println!(" job Ordered list of steps matching one GitHub Actions job"); + println!(" workflow Ordered list of jobs matching a GitHub workflow file"); + println!( + " Example: `cargo xtask ci workflow rust` reproduces `.github/workflows/rust.yml`" + ); + println!(); + println!("Compatibility:"); + println!( + " cargo xtask ci run is still accepted as an alias for `ci step`." + ); + println!(); + println!("Common flags:"); + println!( + " --write Perform best-effort auto-fixes for applicable lint errors (e.g. `cargo fmt`)" + ); + println!( + " --allow-dirty Allow `--write` to change local files with uncommitted changes" + ); + println!(); + self.print_commands(); + } + + fn print_commands(&self) { + println!("Steps:"); + let width = STEPS + .iter() + .map(|step| format!("ci step {}", step.help_usage).len()) + .max() + .unwrap_or_default(); + + for step in STEPS { + println!( + " {: Result<()> { + let Some((command, rest)) = args.split_first() else { + return Err("usage: cargo xtask ci ...".to_string()); + }; + + match command.as_str() { + "step" | "run" => self.ci_step(rest), + "job" => self.ci_job(rest), + "workflow" => self.ci_workflow(rest), + "profile" => Err( + "cargo xtask ci profile is reserved for future local profiles" + .to_string(), + ), + "-h" | "--help" | "help" => { + self.print_help(); + Ok(()) + } + unknown => Err(format!( + "unknown ci command `{unknown}`. Run `cargo xtask help`." + )), + } + } + + fn ci_step(&self, args: &[String]) -> Result<()> { + let Some((command, rest)) = args.split_first() else { + return Err("usage: cargo xtask ci step [args]".to_string()); + }; + + let step = steps::find_step(command).ok_or_else(|| { + format!("unknown command `{command}`. Run `cargo xtask help`.") + })?; + + let runner = step.runner; + let start = SystemTime::now(); + let result = runner(self, rest); + println!( + "\nci step `{command}` completed in {:?}", + start.elapsed().unwrap_or_default() + ); + + result.map_err(|error| { + format!( + "{}\n\nUnderlying command error:\n {error}", + step.error_message + ) + }) + } + + fn ci_job(&self, args: &[String]) -> Result<()> { + let [job_name] = args else { + return Err("usage: cargo xtask ci job ".to_string()); + }; + + let job = CI_JOBS + .iter() + .find(|job| job.name == job_name.as_str()) + .ok_or_else(|| "usage: cargo xtask ci job ".to_string())?; + + let mut job_timing = JobTiming::new(job); + for step_call in job.steps { + let step = step_call.step()?; + let args = step_call + .args + .iter() + .map(|arg| arg.to_string()) + .collect::>(); + let runner = step.runner; + let start = SystemTime::now(); + let result = runner(self, &args); + job_timing.add_step_timing(step, start.elapsed().unwrap_or_default()); + + if let Err(error) = result + .map_err(|error| { + format!( + "{}\n\nUnderlying command error:\n {error}", + step.error_message + ) + }) + .map_err(|error| { + format!( + "job `{}` failed while running `ci step {}`\n\n{error}", + job.name, + step_call.command_line() + ) + }) + { + job_timing.display(); + return Err(error); + } + } + + job_timing.display(); + Ok(()) + } + + fn ci_workflow(&self, args: &[String]) -> Result<()> { + match args { + [] => { + let mut workflows_timing = WorkflowsTiming::new(); + for workflow in CI_WORKFLOWS { + match self.execute_workflow(workflow) { + Ok(workflow_timing) => { + workflows_timing.add_workflow_timing(workflow_timing); + } + Err(error) => { + workflows_timing.display(); + return Err(error); + } + } + } + workflows_timing.display(); + Ok(()) + } + [workflow_name] => { + let workflow = CI_WORKFLOWS + .iter() + .find(|workflow| workflow.name == workflow_name.as_str()) + .ok_or_else(|| "usage: cargo xtask ci workflow [rust]".to_string())?; + + let workflow_timing = self.execute_workflow(workflow)?; + workflow_timing.display(); + Ok(()) + } + _ => Err("usage: cargo xtask ci workflow [rust]".to_string()), + } + } + + fn execute_workflow( + &self, + workflow: &'static WorkflowInfo, + ) -> Result { + let mut workflow_timing = WorkflowTiming::new(workflow); + for job in (workflow.jobs)() { + let mut job_timing = JobTiming::new(job); + for step_call in job.steps { + let step = step_call.step()?; + let args = step_call + .args + .iter() + .map(|arg| arg.to_string()) + .collect::>(); + let runner = step.runner; + let start = SystemTime::now(); + let result = runner(self, &args); + job_timing.add_step_timing(step, start.elapsed().unwrap_or_default()); + + if let Err(error) = result + .map_err(|error| { + format!( + "{}\n\nUnderlying command error:\n {error}", + step.error_message + ) + }) + .map_err(|error| { + format!( + "job `{}` failed while running `ci step {}`\n\n{error}", + job.name, + step_call.command_line() + ) + }) + .map_err(|error| { + format!( + "workflow `{}` failed while running `ci job {}`\n\n{error}", + workflow.name, job.name + ) + }) + { + workflow_timing.add_job_timing(job_timing); + workflow_timing.display(); + return Err(error); + } + } + + workflow_timing.add_job_timing(job_timing); + } + + Ok(workflow_timing) + } +} diff --git a/xtask/src/step_runners.rs b/xtask/src/step_runners.rs new file mode 100644 index 0000000000000..9df2fd0b323f8 --- /dev/null +++ b/xtask/src/step_runners.rs @@ -0,0 +1,701 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Result, Xtask}; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; +use std::process::Command; + +// --------------------------------------------- +// Leaf step runner implementations and utilities. +// --------------------------------------------- + +impl Xtask { + // Step runner entry point for `cargo xtask ci step fmt ...`. + // See the corresponding `fmt` StepInfo for supported args. + pub(crate) fn run_fmt(&self, args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_fmt.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step clippy ...`. + // See the corresponding `clippy` StepInfo for supported args. + pub(crate) fn run_clippy(&self, args: &[String]) -> Result<()> { + self.command("rustup") + .args(["component", "add", "clippy"]) + .run()?; + self.script("ci/scripts/rust_clippy.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-check ...`. + // See the corresponding `rust-check` StepInfo for supported args. + pub(crate) fn run_rust_check(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err("usage: cargo xtask ci step rust-check [target]".to_string()); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "check", + "--profile", + "ci", + "--workspace", + "--all-targets", + "--features", + "integration-tests", + "--locked", + ]) + .run(), + "datafusion-common" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-common", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-common", + ]) + .run() + } + "datafusion-substrait" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=physical", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-substrait", + "--features=protoc", + ]) + .run() + } + "datafusion-proto" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-proto", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .run()?; + for feature in ["json", "parquet", "avro"] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-proto", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .run()?; + for feature in [ + "nested_expressions", + "array_expressions", + "avro", + "backtrace", + "compression", + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "force_hash_collisions", + "math_expressions", + "parquet", + "regex_expressions", + "recursive_protection", + "serde", + "sql", + "string_expressions", + "unicode_expressions", + "parquet_encryption", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + "datafusion-functions" => { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--all-targets", + "-p", + "datafusion-functions", + ]) + .run()?; + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .run()?; + for feature in [ + "crypto_expressions", + "datetime_expressions", + "encoding_expressions", + "math_expressions", + "regex_expressions", + "string_expressions", + "unicode_expressions", + ] { + self.command("cargo") + .args([ + "check", + "--profile", + "ci", + "--no-default-features", + "-p", + "datafusion-functions", + ]) + .arg(format!("--features={feature}")) + .run()?; + } + Ok(()) + } + unknown => Err(format!("unknown rust-check target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-test ...`. + // See the corresponding `rust-test` StepInfo for supported args. + pub(crate) fn run_rust_test(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("workspace"); + if args.len() > 1 { + return Err( + "usage: cargo xtask ci step rust-test [workspace|datafusion-cli|datafusion-ffi]" + .to_string(), + ); + } + + match target { + "workspace" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "--exclude", + "datafusion-examples", + "--exclude", + "ffi_example_table_provider", + "--exclude", + "datafusion-cli", + "--workspace", + "--lib", + "--tests", + "--bins", + "--features", + "serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait", + ]) + .env("RUST_BACKTRACE", "1") + .run(), + "datafusion-cli" => self + .command("cargo") + .args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "-p", + "datafusion-cli", + "--lib", + "--tests", + "--bins", + ]) + .env("RUST_BACKTRACE", "1") + .env("AWS_ENDPOINT", "http://127.0.0.1:9000") + .env("AWS_ACCESS_KEY_ID", "TEST-DataFusionLogin") + .env("AWS_SECRET_ACCESS_KEY", "TEST-DataFusionPassword") + .env("TEST_STORAGE_INTEGRATION", "1") + .env("AWS_ALLOW_HTTP", "true") + .run(), + "datafusion-ffi" => self + .command("cargo") + .args([ + "test", + "--profile", + "ci", + "-p", + "datafusion-ffi", + "--lib", + "--tests", + "--features", + "integration-tests", + ]) + .run(), + unknown => Err(format!("unknown rust-test target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step rust-doctest ...`. + // See the corresponding `rust-doctest` StepInfo for supported args. + pub(crate) fn run_rust_doctest(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args([ + "test", + "--profile", + "ci", + "--doc", + "--features", + "avro,json", + ]) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-doc ...`. + // See the corresponding `rust-doc` StepInfo for supported args. + pub(crate) fn run_rust_doc(&self, _args: &[String]) -> Result<()> { + self.script("ci/scripts/rust_docs.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-examples ...`. + // See the corresponding `rust-examples` StepInfo for supported args. + pub(crate) fn run_rust_examples(&self, _args: &[String]) -> Result<()> { + self.command("cargo") + .args(["run", "--profile", "ci", "--example", "sql"]) + .run()?; + self.script("ci/scripts/rust_example.sh").run() + } + + // Step runner entry point for `cargo xtask ci step rust-wasm ...`. + // See the corresponding `rust-wasm` StepInfo for supported args. + pub(crate) fn run_rust_wasm(&self, _args: &[String]) -> Result<()> { + let wasm_dir = self.root.join("datafusion/wasmtest"); + let rustflags = r#"--cfg getrandom_backend="wasm_js" -C debuginfo=none"#; + self.command("wasm-pack") + .args(["test", "--headless", "--firefox"]) + .current_dir(&wasm_dir) + .env("RUSTFLAGS", rustflags) + .run()?; + + let chrome_driver = format!( + "{}/chromedriver", + env::var("CHROMEWEBDRIVER").unwrap_or_default() + ); + self.command("wasm-pack") + .args([ + "test", + "--headless", + "--chrome", + "--chromedriver", + chrome_driver.as_str(), + ]) + .current_dir(&wasm_dir) + .env("RUSTFLAGS", rustflags) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-benchmark-results ...`. + // See the corresponding `rust-benchmark-results` StepInfo for supported args. + // + // Generate TPCH data, then run checks that require `TPCH_DATA`: + // - benchmark Rust tests for TPCH plan round-trip coverage + // - sqllogictests for TPCH plans and query answers + pub(crate) fn run_rust_benchmark_results(&self, _args: &[String]) -> Result<()> { + let tpch_data = self + .root + .join("datafusion/sqllogictest/test_files/tpch/data"); + fs::create_dir_all(&tpch_data).map_err(|error| { + format!("failed to create {}: {error}", tpch_data.display()) + })?; + + self.command("git") + .args(["clone", "https://github.com/databricks/tpch-dbgen.git"]) + .run()?; + + let tpch_dbgen = self.root.join("tpch-dbgen"); + self.command("make").current_dir(&tpch_dbgen).run()?; + self.command("./dbgen") + .args(["-f", "-s", "0.1"]) + .current_dir(&tpch_dbgen) + .run()?; + self.move_tbl_files(&tpch_dbgen, &tpch_data)?; + + let tpch_data = fs::canonicalize(&tpch_data).map_err(|error| { + format!("failed to canonicalize {}: {error}", tpch_data.display()) + })?; + let tpch_data = tpch_data + .to_str() + .ok_or_else(|| format!("{} is not valid UTF-8", tpch_data.display()))?; + + self.command("cargo") + .args([ + "test", + "plan_q", + "--package", + "datafusion-benchmarks", + "--profile", + "ci", + "--features=ci", + "--", + "--test-threads=1", + ]) + .env("RUST_MIN_STACK", "20971520") + .env("TPCH_DATA", tpch_data) + .run()?; + self.command("cargo") + .args([ + "test", + "--features", + "backtrace,parquet_encryption,substrait", + "--profile", + "ci", + "--package", + "datafusion-sqllogictest", + "--test", + "sqllogictests", + ]) + .env("RUST_MIN_STACK", "20971520") + .env("TPCH_DATA", tpch_data) + .env("INCLUDE_TPCH", "true") + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-sqllogictest ...`. + // + // Supported targets: + // - postgres: reference tests for applicable SLTs against Postgres + // - substrait: Substrait round-trip tests for applicable SLTs + pub(crate) fn run_rust_sqllogictest(&self, args: &[String]) -> Result<()> { + match args { + [target] if target == "postgres" => { + let (host, port) = if matches!( + env::var("GITHUB_ACTIONS").as_deref(), + Ok("true") + ) { + // GitHub container jobs reach services by label. + ( + env::var("POSTGRES_HOST") + .unwrap_or_else(|_| "postgres".to_string()), + env::var("POSTGRES_PORT").unwrap_or_else(|_| "5432".to_string()), + ) + } else { + ( + env::var("POSTGRES_HOST").map_err(|_| { + "POSTGRES_HOST must be set for local Postgres sqllogictest runs" + .to_string() + })?, + env::var("POSTGRES_PORT").map_err(|_| { + "POSTGRES_PORT must be set for local Postgres sqllogictest runs" + .to_string() + })?, + ) + }; + let uri = format!("postgresql://postgres:postgres@{host}:{port}/db_test"); + self.command("cargo") + .args([ + "test", + "--features", + "backtrace", + "--profile", + "ci", + "--features=postgres", + "--test", + "sqllogictests", + ]) + .current_dir(self.root.join("datafusion/sqllogictest")) + .env("PG_COMPAT", "true") + .env("PG_URI", uri.as_str()) + .run() + } + [target] if target == "substrait" => self + .command("cargo") + .args([ + "test", + "-p", + "datafusion-sqllogictest", + "--test", + "sqllogictests", + "--features", + "substrait", + "--", + "--substrait-round-trip", + "limit.slt", + ]) + .run(), + _ => Err( + "usage: cargo xtask ci step rust-sqllogictest [postgres|substrait]" + .to_string(), + ), + } + } + + // Step runner entry point for `cargo xtask ci step rust-vendor ...`. + // See the corresponding `rust-vendor` StepInfo for supported args. + pub(crate) fn run_rust_vendor(&self, _args: &[String]) -> Result<()> { + self.command("./regen.sh") + .current_dir(self.root.join("datafusion/proto")) + .run()?; + self.run_verify_clean(&[]) + } + + // Step runner entry point for `cargo xtask ci step rust-toml-format ...`. + // See the corresponding `rust-toml-format` StepInfo for supported args. + pub(crate) fn run_rust_toml_format(&self, args: &[String]) -> Result<()> { + self.ensure_tool( + "taplo", + self.command("cargo").args([ + "+stable", + "install", + "taplo-cli", + "--version", + "^0.9", + "--locked", + ]), + )?; + self.script("ci/scripts/rust_toml_fmt.sh") + .args(args.iter().map(String::as_str)) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-config-docs ...`. + // See the corresponding `rust-config-docs` StepInfo for supported args. + pub(crate) fn run_rust_config_docs(&self, _args: &[String]) -> Result<()> { + self.command("./dev/update_config_docs.sh").run()?; + self.run_verify_clean(&[])?; + self.command("./dev/update_function_docs.sh").run()?; + self.run_verify_clean(&[]) + } + + // Step runner entry point for `cargo xtask ci step rust-examples-docs ...`. + // See the corresponding `rust-examples-docs` StepInfo for supported args. + pub(crate) fn run_rust_examples_docs(&self, _args: &[String]) -> Result<()> { + self.command("bash") + .args(["ci/scripts/check_examples_docs.sh"]) + .run() + } + + // Step runner entry point for `cargo xtask ci step rust-msrv ...`. + // See the corresponding `rust-msrv` StepInfo for supported args. + pub(crate) fn run_rust_msrv(&self, args: &[String]) -> Result<()> { + let target = args.first().map(String::as_str).unwrap_or("all"); + if args.len() > 1 { + return Err("usage: cargo xtask ci step rust-msrv [all|datafusion|datafusion-substrait|datafusion-proto]".to_string()); + } + + match target { + "all" => { + for target in ["datafusion", "datafusion-substrait", "datafusion-proto"] { + self.run_rust_msrv(&[target.to_string()])?; + } + Ok(()) + } + "datafusion" => self.cargo_msrv("datafusion/core"), + "datafusion-substrait" => self.cargo_msrv("datafusion/substrait"), + "datafusion-proto" => self.cargo_msrv("datafusion/proto"), + unknown => Err(format!("unknown rust-msrv target `{unknown}`")), + } + } + + // Step runner entry point for `cargo xtask ci step verify-clean ...`. + // See the corresponding `verify-clean` StepInfo for supported args. + pub(crate) fn run_verify_clean(&self, args: &[String]) -> Result<()> { + let include_untracked = match args { + [] => false, + [flag] if flag == "--untracked" => true, + _ => { + return Err( + "usage: cargo xtask ci step verify-clean [--untracked]".to_string() + ); + } + }; + + self.command("git").args(["diff", "--exit-code"]).run()?; + + if include_untracked { + let output = Command::new("git") + .args(["status", "--porcelain"]) + .current_dir(&self.root) + .output() + .map_err(|error| { + format!("failed to run `git status --porcelain`: {error}") + })?; + + if !output.status.success() { + return Err(format!( + "`git status --porcelain` failed with {}", + output.status + )); + } + + let status = String::from_utf8(output.stdout) + .map_err(|error| format!("git status output was not UTF-8: {error}"))?; + let unexpected = status + .lines() + .filter(|line| *line != "?? false/" && *line != "?? false") + .collect::>(); + + if !unexpected.is_empty() { + for line in unexpected { + println!("{line}"); + } + return Err("working directory contains untracked files".to_string()); + } + } + + Ok(()) + } + + fn cargo_msrv(&self, relative_dir: &str) -> Result<()> { + self.command("cargo") + .args([ + "msrv", + "--output-format", + "json", + "--log-target", + "stdout", + "verify", + // Avoid the default rust-changelog source, which is subject to GitHub rate limits. + "--release-source", + "rust-dist", + ]) + .current_dir(self.root.join(relative_dir)) + .run() + } + + fn move_tbl_files(&self, source_dir: &Path, destination_dir: &Path) -> Result<()> { + let mut moved_files = 0; + for entry in fs::read_dir(source_dir).map_err(|error| { + format!("failed to read {}: {error}", source_dir.display()) + })? { + let entry = entry + .map_err(|error| format!("failed to read directory entry: {error}"))?; + let source = entry.path(); + if source.extension() == Some(OsStr::new("tbl")) { + let destination = destination_dir.join(entry.file_name()); + fs::rename(&source, &destination).map_err(|error| { + format!( + "failed to move {} to {}: {error}", + source.display(), + destination.display() + ) + })?; + moved_files += 1; + } + } + + if moved_files == 0 { + return Err(format!("no .tbl files found in {}", source_dir.display())); + } + + Ok(()) + } +} diff --git a/xtask/src/steps.rs b/xtask/src/steps.rs new file mode 100644 index 0000000000000..8a6dd0866e33a --- /dev/null +++ b/xtask/src/steps.rs @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Result, Xtask}; + +pub(crate) type StepRunner = fn(&Xtask, &[String]) -> Result<()>; + +/// Definition of each leaf xtask step. +/// +/// Steps are the commands GitHub Actions should schedule directly. Higher +/// levels such as `ci job linux-test` and `ci workflow rust` compose these +/// definitions with concrete arguments. +/// +/// # Example +/// +/// StepInfo { +/// command: "fmt", +/// help_usage: "fmt [--write] [--allow-dirty]", +/// help_description: "Check or fix Rust formatting", +/// error_message: "Rust formatting failed", +/// runner: Xtask::run_fmt, +/// } +/// +/// After executing `cargo xtask ci step fmt`, the `runner` is called. If it +/// fails, `error_message` is displayed with guidance for fixing the failure. +/// +/// `help_usage` and `help_description` are displayed in `cargo xtask help`. +pub(crate) struct StepInfo { + pub(crate) command: &'static str, + pub(crate) runner: StepRunner, + pub(crate) error_message: &'static str, + pub(crate) help_usage: &'static str, + pub(crate) help_description: &'static str, +} + +/// An invocation of a step. +/// +/// * command: the step command, resolved through `STEPS` +/// * args: the arguments for this invocation +pub(crate) struct StepCall { + pub(crate) command: &'static str, + pub(crate) args: &'static [&'static str], +} + +impl StepCall { + pub(crate) fn step(&self) -> Result<&'static StepInfo> { + find_step(self.command).ok_or_else(|| { + format!("unknown ci step `{}` in job definition", self.command) + }) + } + + pub(crate) fn command_line(&self) -> String { + let mut command_line = self.command.to_string(); + for arg in self.args { + command_line.push(' '); + command_line.push_str(arg); + } + command_line + } +} + +pub(crate) fn find_step(command: &str) -> Option<&'static StepInfo> { + STEPS.iter().find(|step| step.command == command) +} + +pub(crate) const STEPS: &[StepInfo] = &[ + StepInfo { + command: "fmt", + help_usage: "fmt [--write] [--allow-dirty]", + help_description: "Check or fix Rust formatting", + error_message: "Rust formatting failed. Run `cargo xtask ci step fmt --write --allow-dirty` and commit the result.", + runner: Xtask::run_fmt, + }, + StepInfo { + command: "clippy", + help_usage: "clippy [--write] [--allow-dirty]", + help_description: "Run the CI clippy suite", + error_message: "Rust clippy failed. Run `cargo xtask ci step clippy` locally and fix all warnings.", + runner: Xtask::run_clippy, + }, + // GitHub uses this command for the base workspace cargo check and for the + // package feature-check steps. The workspace target also verifies + // `Cargo.lock` with `--locked`; the datafusion-common target intentionally + // checks only its limited standalone feature set. + StepInfo { + command: "rust-check", + help_usage: "rust-check [workspace|datafusion|datafusion-common|datafusion-functions|datafusion-proto|datafusion-substrait]", + help_description: "Run CI cargo check variants", + error_message: "Rust check failed. For workspace lockfile failures, regenerate and commit Cargo.lock; otherwise run the same `cargo xtask ci step rust-check ...` target locally and fix the reported compile error.", + runner: Xtask::run_rust_check, + }, + // Covers the Linux workspace test step, the datafusion-cli test step, and the + // macOS datafusion-ffi test step. macOS is narrowed to `datafusion-ffi` + // because amd64 cannot reproduce FFI cdylib loading differences (`.dylib` + // vs `.so` resolution in datafusion/ffi/src/tests/utils.rs). Other + // historically macOS-only failures came from datafusion-benchmarks (now + // covered on amd64) or flaky sqllogictest metrics. + StepInfo { + command: "rust-test", + help_usage: "rust-test [workspace|datafusion-cli|datafusion-ffi]", + help_description: "Run CI Rust test variants (defaults to workspace)", + error_message: "Rust tests failed. Re-run the same `cargo xtask ci step rust-test ...` target locally and fix the failing test.", + runner: Xtask::run_rust_test, + }, + // Runs documentation examples through `cargo test --doc`. + StepInfo { + command: "rust-doctest", + help_usage: "rust-doctest", + help_description: "Run Rust doctests", + error_message: "Rust doctests failed. Run `cargo xtask ci step rust-doctest` locally and fix the failing documentation example.", + runner: Xtask::run_rust_doctest, + }, + // Builds Rust documentation to ensure rustdoc is clean. + StepInfo { + command: "rust-doc", + help_usage: "rust-doc", + help_description: "Build Rust documentation with warnings denied", + error_message: "Rust docs failed. Run `cargo xtask ci step rust-doc` locally and fix rustdoc warnings or errors.", + runner: Xtask::run_rust_doc, + }, + StepInfo { + command: "rust-examples", + help_usage: "rust-examples", + help_description: "Run Rust examples checks", + error_message: "Rust examples failed. Run `cargo xtask ci step rust-examples` locally and fix the failing example.", + runner: Xtask::run_rust_examples, + }, + StepInfo { + command: "rust-wasm", + help_usage: "rust-wasm", + help_description: "Run wasm-pack tests", + error_message: "Rust wasm tests failed. Run `cargo xtask ci step rust-wasm` locally with wasm-pack installed and fix the wasm test failure.", + runner: Xtask::run_rust_wasm, + }, + // Verifies that the benchmark queries return the expected results. + StepInfo { + command: "rust-benchmark-results", + help_usage: "rust-benchmark-results", + help_description: "Generate TPC-H data (sf=0.1) and verify benchmark results", + error_message: "Rust benchmark result verification failed. Run `cargo xtask ci step rust-benchmark-results` locally and update or fix the affected benchmark expectation.", + runner: Xtask::run_rust_benchmark_results, + }, + // Specialized sqllogictest variants: + // - postgres: reference tests for applicable SLTs against Postgres + // - substrait: Substrait round-trip tests for applicable SLTs + // + // The Substrait round-trip variant is intentionally limited to `limit.slt` + // until https://github.com/apache/datafusion/issues/16248 is addressed. + StepInfo { + command: "rust-sqllogictest", + help_usage: "rust-sqllogictest [postgres|substrait]", + help_description: "Run specialized sqllogictests. (PostgreSQL reference test, or substrait round-trip test)", + error_message: "Rust sqllogictest failed. Re-run the same `cargo xtask ci step rust-sqllogictest ...` target locally and fix the failing sqllogictest.", + runner: Xtask::run_rust_sqllogictest, + }, + StepInfo { + command: "rust-vendor", + help_usage: "rust-vendor", + help_description: "Verify vendored generated code", + error_message: concat!( + "Rust vendor verification failed. Verify the workspace is clean.\n", + "If this fails, run `./datafusion/proto/regen.sh` and check in the results." + ), + runner: Xtask::run_rust_vendor, + }, + StepInfo { + command: "rust-toml-format", + help_usage: "rust-toml-format [--write] [--allow-dirty]", + help_description: "Check or fix Cargo.toml formatting", + error_message: "Cargo.toml formatting failed. Run `cargo xtask ci step rust-toml-format --write --allow-dirty`, then commit the result.", + runner: Xtask::run_rust_toml_format, + }, + StepInfo { + command: "rust-config-docs", + help_usage: "rust-config-docs", + help_description: "Verify generated config and function docs", + error_message: "Config or function docs verification failed. Run `cargo xtask ci step rust-config-docs` and commit the generated docs.", + runner: Xtask::run_rust_config_docs, + }, + // Keeps `datafusion-examples/README.md` in sync with the Rust examples by + // regenerating the README, formatting it with Prettier, and checking it + // against the committed file. + StepInfo { + command: "rust-examples-docs", + help_usage: "rust-examples-docs", + help_description: "Verify examples README generation", + error_message: "Examples docs verification failed. Run `cargo xtask ci step rust-examples-docs` and commit the updated datafusion-examples/README.md.", + runner: Xtask::run_rust_examples_docs, + }, + // Verifies MSRV for crates directly used by downstream projects: + // datafusion, datafusion-substrait, and datafusion-proto. + StepInfo { + command: "rust-msrv", + help_usage: "rust-msrv [all|datafusion|datafusion-substrait|datafusion-proto]", + help_description: "Run cargo-msrv verification", + error_message: "Rust MSRV verification failed. Some code or dependency may require a newer Rust version; review the DataFusion Rust Version Compatibility Policy before updating Cargo.toml.", + runner: Xtask::run_rust_msrv, + }, + // Used after tests and code generation to ensure CI did not leave temporary + // files or generated diffs. The untracked mode ignores rust-cache's `false/` + // marker directory. + StepInfo { + command: "verify-clean", + help_usage: "verify-clean [--untracked]", + help_description: "Verify the git worktree is clean", + error_message: concat!( + "Worktree cleanliness verification failed.\n", + " - For local runs, ensure all expected changes are committed before running this check.\n", + " - Next, try `git diff` and `git status --porcelain`; then remove temporary files or commit expected generated output." + ), + runner: Xtask::run_verify_clean, + }, +]; diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs new file mode 100644 index 0000000000000..06788e89e1d9e --- /dev/null +++ b/xtask/src/utils.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{Result, Xtask}; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +impl Xtask { + pub(crate) fn script(&self, relative_path: &str) -> XtaskCommand { + self.command(self.root.join(relative_path)) + } + + pub(crate) fn command(&self, program: impl Into) -> XtaskCommand { + XtaskCommand::new(program, &self.root) + } + + pub(crate) fn ensure_tool(&self, tool: &str, installer: XtaskCommand) -> Result<()> { + let installed = Command::new(tool) + .arg("--version") + .current_dir(&self.root) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false); + + if installed { + return Ok(()); + } + + installer.run() + } +} + +/// Builder for commands run from xtask. +/// +/// Keeps command setup in one fluent path instead of adding helper methods for +/// every combination of args, environment, and working directory. +/// +/// ```no_run +/// self.command("cargo") +/// .args(["test", "--profile", "ci"]) +/// .env("RUST_BACKTRACE", "1") +/// .current_dir(...) +/// .run() +/// ``` +pub(crate) struct XtaskCommand { + program: OsString, + args: Vec, + current_dir: PathBuf, + envs: Vec<(OsString, OsString)>, +} + +impl XtaskCommand { + fn new(program: impl Into, current_dir: impl AsRef) -> Self { + Self { + program: program.into(), + args: Vec::new(), + current_dir: current_dir.as_ref().to_path_buf(), + envs: Vec::new(), + } + } + + pub(crate) fn arg(mut self, arg: impl Into) -> Self { + self.args.push(arg.into()); + self + } + + pub(crate) fn args( + mut self, + args: impl IntoIterator>, + ) -> Self { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + pub(crate) fn current_dir(mut self, current_dir: impl AsRef) -> Self { + self.current_dir = current_dir.as_ref().to_path_buf(); + self + } + + pub(crate) fn env( + mut self, + name: impl Into, + value: impl Into, + ) -> Self { + self.envs.push((name.into(), value.into())); + self + } + + pub(crate) fn run(self) -> Result<()> { + let display = command_display(&self.program, &self.args); + println!("+ {display}"); + + let mut command = Command::new(&self.program); + command + .args(&self.args) + .current_dir(&self.current_dir) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + for (name, value) in self.envs { + command.env(name, value); + } + + let status = command.status().map_err(|error| { + format!( + "failed to run `{display}` in {}: {error}", + self.current_dir.display() + ) + })?; + + if status.success() { + Ok(()) + } else { + Err(format!("`{display}` failed with {status}")) + } + } +} + +fn command_display(program: &OsStr, args: &[OsString]) -> String { + let mut display = program.to_string_lossy().into_owned(); + for arg in args { + display.push(' '); + display.push_str(&arg.to_string_lossy()); + } + display +}