From 7cb535f9bb659dcfb3accf6f8e2a44c809649341 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Mon, 12 Oct 2026 11:17:25 -0700 Subject: [PATCH 1/5] Resolve GPU test dependencies before loading workers Adding a GPU backend can downgrade LLVM and Atomix. Install optional backends together before importing Dagger or starting workers, and launch workers in the active test project so Pkg.test sandbox dependencies match on every process. Validated on Julia 1.11.9 with a forced LLVM 10 to 9.13.2 downgrade and CUDA 5.8.5: the driver and three workers load matching versions from the same project. The focused CPU mapreduce suite passes all 421 tests. --- AGENTS.md | 8 ++++++++ test/runtests.jl | 24 ++++++++++++++++++------ test/setup_gpu.jl | 23 +---------------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 52879753c..9e2fac2ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -633,3 +633,11 @@ lesson. YAML but passes `{{array}}` to the benchmark worker, which rejects the suite before sampling. Check the expanded command arguments and artifact paths for every pipeline; parser and dependency checks do not catch this error. + +61. **Resolve optional test packages before loading dependencies or starting + workers, and share the active test project.** Installing a GPU backend can + downgrade LLVM and Atomix after Dagger/KernelAbstractions already loaded + them; GPUCompiler then sees the old module with the new package graph and + fails before any GPU test runs. `Pkg.test` uses a temporary project, so + workers must inherit `Base.active_project()`, not the repository project. + Install optional packages in one batch before imports and worker startup. diff --git a/test/runtests.jl b/test/runtests.jl index 628cc24fa..99b559de6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -99,11 +99,6 @@ if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__ Pkg.instantiate() catch end - if USE_FINCH - # Finch is not a default test dependency; add it on demand when signaled. - Pkg.add("Finch") - end - using ArgParse s = ArgParseSettings(description = "Dagger Testsuite") @eval begin @@ -202,11 +197,28 @@ else @info "Running all tests" end +# Resolve optional dependencies before loading Dagger (and its dependencies) or +# starting workers. Adding a GPU backend can downgrade LLVM/Atomix; a process +# that already loaded them cannot switch to the newly resolved versions. +if USE_GPU || USE_FINCH + using Pkg + optional_packages = String[] + USE_CUDA && push!(optional_packages, "CUDA") + USE_ROCM && push!(optional_packages, "AMDGPU") + USE_ONEAPI && push!(optional_packages, "oneAPI") + USE_METAL && push!(optional_packages, "Metal") + USE_OPENCL && append!(optional_packages, ["OpenCL", "pocl_jll"]) + USE_FINCH && push!(optional_packages, "Finch") + Pkg.add(optional_packages) +end + using Distributed if additional_workers > 0 # We put this inside a branch because addprocs() takes a minimum of 1s to # complete even if doing nothing, which is annoying. - addprocs(additional_workers; exeflags=`--project=$(joinpath(@__DIR__, "..")) -t $worker_threads`) + # Pkg.test runs in a temporary project containing the test-only backends. + # Workers must use that same resolution, rather than the repository project. + addprocs(additional_workers; exeflags=`--project=$(Base.active_project()) -t $worker_threads`) end include("imports.jl") diff --git a/test/setup_gpu.jl b/test/setup_gpu.jl index 931161850..eceef4ff5 100644 --- a/test/setup_gpu.jl +++ b/test/setup_gpu.jl @@ -1,25 +1,4 @@ -if USE_CUDA - using Pkg - Pkg.add("CUDA") -end -if USE_ROCM - using Pkg - Pkg.add("AMDGPU") -end -if USE_ONEAPI - using Pkg - Pkg.add("oneAPI") -end -if USE_METAL - using Pkg - Pkg.add("Metal") -end -if USE_OPENCL - using Pkg - Pkg.add("OpenCL") - Pkg.add("pocl_jll") -end - +# Optional packages are installed in runtests.jl before workers and imports. @everywhere begin if $USE_CUDA using CUDA From 05726b07250cbeaa3b331652a43b61ae96027535 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Mon, 12 Oct 2026 11:17:33 -0700 Subject: [PATCH 2/5] Initialize Python only when its test suite is selected PythonCall initializes its interpreter during test-environment precompilation. A Python library discovery failure was aborting the Metal GPU job before any tests ran. Install PythonCall and CondaPkg on demand before imports instead, preserving the default CPU Python suite while keeping GPU-only jobs independent of Python. Validated that the GPU bootstrap excludes Python and that selecting extlang/python installs both dependencies and enters that suite. The local Python suite reproduces the existing Python library discovery failure; no Python runtime behavior is changed. --- AGENTS.md | 7 +++++++ test/Project.toml | 4 ++-- test/runtests.jl | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9e2fac2ab..8f6c2afd9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -641,3 +641,10 @@ lesson. fails before any GPU test runs. `Pkg.test` uses a temporary project, so workers must inherit `Base.active_project()`, not the repository project. Install optional packages in one batch before imports and worker startup. + +62. **Optional language runtimes must not initialize in unrelated test jobs.** + Listing PythonCall/CondaPkg as unconditional test dependencies initializes + Python during `Pkg.test` precompilation, even in GPU-only jobs. A Python + library discovery failure then aborts Metal tests before the runner starts. + Install those packages before imports only when the Python suite is selected; + default CPU runs still select that suite, and GPU runs do not. diff --git a/test/Project.toml b/test/Project.toml index d54a0e7aa..b17cb51ef 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,11 +1,12 @@ [deps] +# PythonCall and CondaPkg are installed by runtests.jl only for extlang/python; +# GPU jobs should not initialize a Python environment during precompilation. # Direct dep so the [compat] pin below applies. AlgebraicMultigrid -> LinearSolve # -> SparseColumnPivotedQR's AMD extension reads AMD.SS_Int, which 0.5.4 removed. AMD = "14f7f29c-3bd6-536c-9a0b-7339e30b5a3e" AlgebraicMultigrid = "2169fc97-5a83-5252-b627-83903c6c433c" ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" -CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -27,7 +28,6 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Preferences = "21216c6a-2e73-6563-6e65-726566657250" PureKLU = "0c0d3e7f-3a8b-4f7e-b6f1-9a4d2e7c1f01" PureUMFPACK = "b7e1f0a2-3c4d-4e5f-9a0b-1c2d3e4f5a6b" -PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/test/runtests.jl b/test/runtests.jl index 99b559de6..ce7c874d6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -200,7 +200,7 @@ end # Resolve optional dependencies before loading Dagger (and its dependencies) or # starting workers. Adding a GPU backend can downgrade LLVM/Atomix; a process # that already loaded them cannot switch to the newly resolved versions. -if USE_GPU || USE_FINCH +if USE_GPU || USE_FINCH || "extlang/python" in to_test using Pkg optional_packages = String[] USE_CUDA && push!(optional_packages, "CUDA") @@ -209,6 +209,9 @@ if USE_GPU || USE_FINCH USE_METAL && push!(optional_packages, "Metal") USE_OPENCL && append!(optional_packages, ["OpenCL", "pocl_jll"]) USE_FINCH && push!(optional_packages, "Finch") + if "extlang/python" in to_test + append!(optional_packages, ["CondaPkg", "PythonCall"]) + end Pkg.add(optional_packages) end From 4c52cb38e8cf745b5aa49db568a5cf8b3838d7a8 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Mon, 12 Oct 2026 12:17:10 -0700 Subject: [PATCH 3/5] Run Finch tests on GitHub Actions instead of GPU agents Finch needs CPU workers rather than GPU hardware. Move its Julia 1.11 job to ubuntu-latest while preserving the 90-minute timeout, optional-dependency signal, skip condition, and coverage reporting. Validated the workflow YAML and confirmed CI_TEST_FINCH selects only array/linalg/matmul_finch in the test runner. --- .buildkite/pipeline.yml | 12 ------------ .github/workflows/CI.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c3c609227..26e3a7a2b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -128,18 +128,6 @@ steps: # julia --project=test/metalenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()' # julia --project=test/metalenv test/mpi/run.jl 2 2 test/mpi/metal.jl - - label: Julia 1.11 (Finch) - timeout_in_minutes: 90 - if: build.message !~ /\[skip tests\]/ - plugins: - - JuliaCI/julia#v1: - version: "1.11" - - JuliaCI/julia-test#v1: ~ - - JuliaCI/julia-coverage#v1: - codecov: true - env: - CI_TEST_FINCH: "1" - # env: # SECRET_CODECOV_TOKEN: "" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 23f33d938..16d1f2a20 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -92,6 +92,34 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false + finch: + name: Julia 1.11 (Finch) + runs-on: ubuntu-latest + timeout-minutes: 90 + if: ${{ !contains(github.event.head_commit.message, '[skip tests]') }} + env: + JULIA_NUM_THREADS: '1' + CI_TEST_FINCH: '1' + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1.11' + arch: x64 + - uses: julia-actions/cache@v3 + - name: Develop local TimespanLogging + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/TimespanLogging")' + - name: Instantiate + run: julia --project=. -e 'using Pkg; Pkg.instantiate()' + - name: Run Finch tests + run: julia --code-coverage=user test/runtests.jl + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v7 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + opencl: name: Julia 1.11 (OpenCL) runs-on: ubuntu-latest From 7bc9a436e9bcc2e541b1a046381c6894262ab91d Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Mon, 12 Oct 2026 12:17:10 -0700 Subject: [PATCH 4/5] Reserve both Buildkite pipelines for GPU workloads CPU test matrices, TimespanLogging, DaggerWebDash, and CPU benchmarks already run on GitHub Actions. Remove their duplicate Buildkite jobs and unused anchors so GPU agents only service GPU tests, benchmarks, and reports. Validated YAML, all remaining step dependencies, and GPU job classification in both pipelines; all six Buildkite report and matrix-expansion tests pass. --- .buildkite/pipeline-julia.yml | 154 +--------------------------------- 1 file changed, 2 insertions(+), 152 deletions(-) diff --git a/.buildkite/pipeline-julia.yml b/.buildkite/pipeline-julia.yml index 7f5e1d83d..7a6fbfe1f 100644 --- a/.buildkite/pipeline-julia.yml +++ b/.buildkite/pipeline-julia.yml @@ -1,117 +1,10 @@ -.test: &test - timeout_in_minutes: 120 - if: build.message !~ /\[skip tests\]/ - agents: - queue: "juliaecosystem" - sandbox_capable: "true" - os: linux - arch: x86_64 - command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\")'" - +# CPU tests and benchmarks run in .github/workflows/CI.yml. +# This pipeline contains the OpenCL GPU tests, benchmarks, and their report. .gputest: &gputest timeout_in_minutes: 60 if: build.message !~ /\[skip tests\]/ -.bench: &bench - # Run on every pull request (gating against >25% regressions vs master), and - # on demand elsewhere via a "[run benchmarks]" commit message. Use - # "[skip benchmarks]" to opt a PR out. - if: (build.pull_request.id != null && build.message !~ /\[skip benchmarks\]/) || build.message =~ /\[run benchmarks\]/ - agents: - queue: "juliaecosystem" - sandbox_capable: "true" - os: linux - arch: x86_64 - num_cpus: 16 - steps: - - label: Julia 1 (multithreaded) - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia --project -e 'using Pkg; Pkg.instantiate()'; julia -t $BUILDKITE_AGENT_META_DATA_NUM_CPUS test/runtests.jl -p 0" - - - label: Julia 1.10 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.10" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1.11 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.11" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1.12 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.12" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia nightly - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "nightly" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 (Windows) - <<: *test - agents: - queue: "juliaecosystem" - os: windows - arch: x86_64 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 (macOS) - <<: *test - agents: - queue: "juliaecosystem" - os: macos - arch: x86_64 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - label: Julia 1.11 (OpenCL) key: test-opencl <<: *gputest @@ -148,49 +41,6 @@ steps: julia --project=test/openclenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()' julia --project=test/openclenv test/mpi/run.jl 2 2 test/mpi/opencl.jl - - label: Julia 1 - TimespanLogging - <<: *test - timeout_in_minutes: 20 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\"); Pkg.instantiate(); Pkg.test(\"TimespanLogging\")'" - - - label: Julia 1 - DaggerWebDash - <<: *test - timeout_in_minutes: 20 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path=\"lib/TimespanLogging\"), PackageSpec(path=\"lib/DaggerWebDash\")]); include(\"lib/DaggerWebDash/test/runtests.jl\")'" - - - label: "Benchmarks (vs master)" - timeout_in_minutes: 120 - <<: *bench - plugins: - - JuliaCI/julia#v1: - version: "1" - command: "bash .buildkite/run_benchmarks.sh" - env: - JULIA_NUM_THREADS: "8" - BENCHMARK_BASE_REV: "master" - BENCHMARK_REGRESSION_THRESHOLD: "0.25" - BENCHMARK_CI_THREADS: "8" - # Keep CI runs bounded; tune as needed (see benchmark/benchmarks.jl). - BENCHMARK_SCALE: "[1024, 4096]" - BENCHMARK_SECONDS: "5" - BENCHMARK_SAMPLES: "20" - BENCHMARK_OUTPUT_DIR: "benchmark_results" - artifacts: - - "benchmark_results/**/*" - -# env: -# SECRET_CODECOV_TOKEN: "" - - label: "Benchmarks (OpenCL, gpu, {{matrix}})" key: "bench-opencl-gpu" # GPU timings must not compete with sibling benchmark shards. From c52f47e4b5dd7c7f814d6d4584cca2537f3118cf Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Mon, 12 Oct 2026 13:13:55 -0700 Subject: [PATCH 5/5] Document upstream causes of Python CI initialization failures --- AGENTS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 8f6c2afd9..2ee908bf8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -648,3 +648,14 @@ lesson. library discovery failure then aborts Metal tests before the runner starts. Install those packages before imports only when the Python suite is selected; default CPU runs still select that suite, and GPU runs do not. + +63. **Python setup failures can come from two independent upstream regressions.** + CondaPkg scans every environment in `LOAD_PATH`, including optional global + and stdlib environments whose `Project.toml` may not exist. CondaPkg 0.2.37 + tries to open those files and fails before Python initialization (fixed by + JuliaPy/CondaPkg.jl#235). Once resolution succeeds, PythonCall 0.9.36 can + still fail library discovery because conda-forge's Python 3.15 separates + `libpython` from the interpreter (fixed by JuliaPy/PythonCall.jl#839). + Compare unrelated PR logs and check upstream releases before changing the + test bootstrap: installing Python only for its suite does not cause either + failure. Both fixes together pass Dagger's Python suite on Julia 1.11.