Fix GPU CI startup and move CPU jobs to GitHub Actions

.buildkite/pipeline-julia.yml

@@ -1,117 +1,10 @@
-.test: &test
-  timeout_in_minutes: 120
-  if: build.message !~ /\[skip tests\]/
-  agents:
-    queue: "juliaecosystem"
-    sandbox_capable: "true"
-    os: linux
-    arch: x86_64
-  command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\")'"
-
+# CPU tests and benchmarks run in .github/workflows/CI.yml.
+# This pipeline contains the OpenCL GPU tests, benchmarks, and their report.
 .gputest: &gputest
   timeout_in_minutes: 60
   if: build.message !~ /\[skip tests\]/
 
-.bench: &bench
-  # Run on every pull request (gating against >25% regressions vs master), and
-  # on demand elsewhere via a "[run benchmarks]" commit message. Use
-  # "[skip benchmarks]" to opt a PR out.
-  if: (build.pull_request.id != null && build.message !~ /\[skip benchmarks\]/) || build.message =~ /\[run benchmarks\]/
-  agents:
-    queue: "juliaecosystem"
-    sandbox_capable: "true"
-    os: linux
-    arch: x86_64
-    num_cpus: 16
-
 steps:
-  - label: Julia 1 (multithreaded)
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-    command: "julia --project -e 'using Pkg; Pkg.instantiate()'; julia -t $BUILDKITE_AGENT_META_DATA_NUM_CPUS test/runtests.jl -p 0"
-
-  - label: Julia 1.10
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.10"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia 1.11
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.11"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia 1.12
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.12"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia 1
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia nightly
-    <<: *test
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "nightly"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia 1 (Windows)
-    <<: *test
-    agents:
-      queue: "juliaecosystem"
-      os: windows
-      arch: x86_64
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
-  - label: Julia 1 (macOS)
-    <<: *test
-    agents:
-      queue: "juliaecosystem"
-      os: macos
-      arch: x86_64
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-
   - label: Julia 1.11 (OpenCL)
     key: test-opencl
     <<: *gputest
@@ -148,49 +41,6 @@ steps:
       julia --project=test/openclenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()'
       julia --project=test/openclenv test/mpi/run.jl 2 2 test/mpi/opencl.jl
 
-  - label: Julia 1 - TimespanLogging
-    <<: *test
-    timeout_in_minutes: 20
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-    command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\"); Pkg.instantiate(); Pkg.test(\"TimespanLogging\")'"
-
-  - label: Julia 1 - DaggerWebDash
-    <<: *test
-    timeout_in_minutes: 20
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-    command: "julia -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path=\"lib/TimespanLogging\"), PackageSpec(path=\"lib/DaggerWebDash\")]); include(\"lib/DaggerWebDash/test/runtests.jl\")'"
-
-  - label: "Benchmarks (vs master)"
-    timeout_in_minutes: 120
-    <<: *bench
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1"
-    command: "bash .buildkite/run_benchmarks.sh"
-    env:
-      JULIA_NUM_THREADS: "8"
-      BENCHMARK_BASE_REV: "master"
-      BENCHMARK_REGRESSION_THRESHOLD: "0.25"
-      BENCHMARK_CI_THREADS: "8"
-      # Keep CI runs bounded; tune as needed (see benchmark/benchmarks.jl).
-      BENCHMARK_SCALE: "[1024, 4096]"
-      BENCHMARK_SECONDS: "5"
-      BENCHMARK_SAMPLES: "20"
-      BENCHMARK_OUTPUT_DIR: "benchmark_results"
-    artifacts:
-      - "benchmark_results/**/*"
-
-# env:
-#   SECRET_CODECOV_TOKEN: ""
-
   - label: "Benchmarks (OpenCL, gpu, {{matrix}})"
     key: "bench-opencl-gpu"
     # GPU timings must not compete with sibling benchmark shards.

.buildkite/pipeline.yml

@@ -128,18 +128,6 @@ steps:
   #     julia --project=test/metalenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()'
   #     julia --project=test/metalenv test/mpi/run.jl 2 2 test/mpi/metal.jl
 
-  - label: Julia 1.11 (Finch)
-    timeout_in_minutes: 90
-    if: build.message !~ /\[skip tests\]/
-    plugins:
-      - JuliaCI/julia#v1:
-          version: "1.11"
-      - JuliaCI/julia-test#v1: ~
-      - JuliaCI/julia-coverage#v1:
-          codecov: true
-    env:
-      CI_TEST_FINCH: "1"
-
 # env:
 #   SECRET_CODECOV_TOKEN: ""
 

.github/workflows/CI.yml

@@ -92,6 +92,34 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
+  finch:
+    name: Julia 1.11 (Finch)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    if: ${{ !contains(github.event.head_commit.message, '[skip tests]') }}
+    env:
+      JULIA_NUM_THREADS: '1'
+      CI_TEST_FINCH: '1'
+    steps:
+      - uses: actions/checkout@v7
+      - uses: julia-actions/setup-julia@v3
+        with:
+          version: '1.11'
+          arch: x64
+      - uses: julia-actions/cache@v3
+      - name: Develop local TimespanLogging
+        run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/TimespanLogging")'
+      - name: Instantiate
+        run: julia --project=. -e 'using Pkg; Pkg.instantiate()'
+      - name: Run Finch tests
+        run: julia --code-coverage=user test/runtests.jl
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v7
+        with:
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
+
   opencl:
     name: Julia 1.11 (OpenCL)
     runs-on: ubuntu-latest

AGENTS.md

@@ -633,3 +633,29 @@ lesson.
     YAML but passes `{{array}}` to the benchmark worker, which rejects the suite
     before sampling. Check the expanded command arguments and artifact paths
     for every pipeline; parser and dependency checks do not catch this error.
+
+61. **Resolve optional test packages before loading dependencies or starting
+    workers, and share the active test project.** Installing a GPU backend can
+    downgrade LLVM and Atomix after Dagger/KernelAbstractions already loaded
+    them; GPUCompiler then sees the old module with the new package graph and
+    fails before any GPU test runs. `Pkg.test` uses a temporary project, so
+    workers must inherit `Base.active_project()`, not the repository project.
+    Install optional packages in one batch before imports and worker startup.
+
+62. **Optional language runtimes must not initialize in unrelated test jobs.**
+    Listing PythonCall/CondaPkg as unconditional test dependencies initializes
+    Python during `Pkg.test` precompilation, even in GPU-only jobs. A Python
+    library discovery failure then aborts Metal tests before the runner starts.
+    Install those packages before imports only when the Python suite is selected;
+    default CPU runs still select that suite, and GPU runs do not.
+
+63. **Python setup failures can come from two independent upstream regressions.**
+    CondaPkg scans every environment in `LOAD_PATH`, including optional global
+    and stdlib environments whose `Project.toml` may not exist. CondaPkg 0.2.37
+    tries to open those files and fails before Python initialization (fixed by
+    JuliaPy/CondaPkg.jl#235). Once resolution succeeds, PythonCall 0.9.36 can
+    still fail library discovery because conda-forge's Python 3.15 separates
+    `libpython` from the interpreter (fixed by JuliaPy/PythonCall.jl#839).
+    Compare unrelated PR logs and check upstream releases before changing the
+    test bootstrap: installing Python only for its suite does not cause either
+    failure. Both fixes together pass Dagger's Python suite on Julia 1.11.

test/Project.toml

@@ -1,11 +1,12 @@
 [deps]
+# PythonCall and CondaPkg are installed by runtests.jl only for extlang/python;
+# GPU jobs should not initialize a Python environment during precompilation.
 # Direct dep so the [compat] pin below applies. AlgebraicMultigrid -> LinearSolve
 # -> SparseColumnPivotedQR's AMD extension reads AMD.SS_Int, which 0.5.4 removed.
 AMD = "14f7f29c-3bd6-536c-9a0b-7339e30b5a3e"
 AlgebraicMultigrid = "2169fc97-5a83-5252-b627-83903c6c433c"
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
-CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
@@ -27,7 +28,6 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 PureKLU = "0c0d3e7f-3a8b-4f7e-b6f1-9a4d2e7c1f01"
 PureUMFPACK = "b7e1f0a2-3c4d-4e5f-9a0b-1c2d3e4f5a6b"
-PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

test/runtests.jl

@@ -99,11 +99,6 @@ if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__
         Pkg.instantiate()
     catch
     end
-    if USE_FINCH
-        # Finch is not a default test dependency; add it on demand when signaled.
-        Pkg.add("Finch")
-    end
-
     using ArgParse
     s = ArgParseSettings(description = "Dagger Testsuite")
     @eval begin
@@ -202,11 +197,31 @@ else
     @info "Running all tests"
 end
 
+# Resolve optional dependencies before loading Dagger (and its dependencies) or
+# starting workers. Adding a GPU backend can downgrade LLVM/Atomix; a process
+# that already loaded them cannot switch to the newly resolved versions.
+if USE_GPU || USE_FINCH || "extlang/python" in to_test
+    using Pkg
+    optional_packages = String[]
+    USE_CUDA && push!(optional_packages, "CUDA")
+    USE_ROCM && push!(optional_packages, "AMDGPU")
+    USE_ONEAPI && push!(optional_packages, "oneAPI")
+    USE_METAL && push!(optional_packages, "Metal")
+    USE_OPENCL && append!(optional_packages, ["OpenCL", "pocl_jll"])
+    USE_FINCH && push!(optional_packages, "Finch")
+    if "extlang/python" in to_test
+        append!(optional_packages, ["CondaPkg", "PythonCall"])
+    end
+    Pkg.add(optional_packages)
+end
+
 using Distributed
 if additional_workers > 0
     # We put this inside a branch because addprocs() takes a minimum of 1s to
     # complete even if doing nothing, which is annoying.
-    addprocs(additional_workers; exeflags=`--project=$(joinpath(@__DIR__, "..")) -t $worker_threads`)
+    # Pkg.test runs in a temporary project containing the test-only backends.
+    # Workers must use that same resolution, rather than the repository project.
+    addprocs(additional_workers; exeflags=`--project=$(Base.active_project()) -t $worker_threads`)
 end
 
 include("imports.jl")

test/setup_gpu.jl

@@ -1,25 +1,4 @@
-if USE_CUDA
-    using Pkg
-    Pkg.add("CUDA")
-end
-if USE_ROCM
-    using Pkg
-    Pkg.add("AMDGPU")
-end
-if USE_ONEAPI
-    using Pkg
-    Pkg.add("oneAPI")
-end
-if USE_METAL
-    using Pkg
-    Pkg.add("Metal")
-end
-if USE_OPENCL
-    using Pkg
-    Pkg.add("OpenCL")
-    Pkg.add("pocl_jll")
-end
-
+# Optional packages are installed in runtests.jl before workers and imports.
 @everywhere begin
     if $USE_CUDA
         using CUDA