diff --git a/.buildkite/pipeline-julia.yml b/.buildkite/pipeline-julia.yml index 7f5e1d83d..7a6fbfe1f 100644 --- a/.buildkite/pipeline-julia.yml +++ b/.buildkite/pipeline-julia.yml @@ -1,117 +1,10 @@ -.test: &test - timeout_in_minutes: 120 - if: build.message !~ /\[skip tests\]/ - agents: - queue: "juliaecosystem" - sandbox_capable: "true" - os: linux - arch: x86_64 - command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\")'" - +# CPU tests and benchmarks run in .github/workflows/CI.yml. +# This pipeline contains the OpenCL GPU tests, benchmarks, and their report. .gputest: &gputest timeout_in_minutes: 60 if: build.message !~ /\[skip tests\]/ -.bench: &bench - # Run on every pull request (gating against >25% regressions vs master), and - # on demand elsewhere via a "[run benchmarks]" commit message. Use - # "[skip benchmarks]" to opt a PR out. - if: (build.pull_request.id != null && build.message !~ /\[skip benchmarks\]/) || build.message =~ /\[run benchmarks\]/ - agents: - queue: "juliaecosystem" - sandbox_capable: "true" - os: linux - arch: x86_64 - num_cpus: 16 - steps: - - label: Julia 1 (multithreaded) - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia --project -e 'using Pkg; Pkg.instantiate()'; julia -t $BUILDKITE_AGENT_META_DATA_NUM_CPUS test/runtests.jl -p 0" - - - label: Julia 1.10 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.10" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1.11 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.11" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1.12 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1.12" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia nightly - <<: *test - plugins: - - JuliaCI/julia#v1: - version: "nightly" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 (Windows) - <<: *test - agents: - queue: "juliaecosystem" - os: windows - arch: x86_64 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - - label: Julia 1 (macOS) - <<: *test - agents: - queue: "juliaecosystem" - os: macos - arch: x86_64 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-test#v1: - julia_args: "--threads=1" - - JuliaCI/julia-coverage#v1: - codecov: true - - label: Julia 1.11 (OpenCL) key: test-opencl <<: *gputest @@ -148,49 +41,6 @@ steps: julia --project=test/openclenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()' julia --project=test/openclenv test/mpi/run.jl 2 2 test/mpi/opencl.jl - - label: Julia 1 - TimespanLogging - <<: *test - timeout_in_minutes: 20 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia --project -e 'using Pkg; Pkg.develop(;path=\"lib/TimespanLogging\"); Pkg.instantiate(); Pkg.test(\"TimespanLogging\")'" - - - label: Julia 1 - DaggerWebDash - <<: *test - timeout_in_minutes: 20 - plugins: - - JuliaCI/julia#v1: - version: "1" - - JuliaCI/julia-coverage#v1: - codecov: true - command: "julia -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path=\"lib/TimespanLogging\"), PackageSpec(path=\"lib/DaggerWebDash\")]); include(\"lib/DaggerWebDash/test/runtests.jl\")'" - - - label: "Benchmarks (vs master)" - timeout_in_minutes: 120 - <<: *bench - plugins: - - JuliaCI/julia#v1: - version: "1" - command: "bash .buildkite/run_benchmarks.sh" - env: - JULIA_NUM_THREADS: "8" - BENCHMARK_BASE_REV: "master" - BENCHMARK_REGRESSION_THRESHOLD: "0.25" - BENCHMARK_CI_THREADS: "8" - # Keep CI runs bounded; tune as needed (see benchmark/benchmarks.jl). - BENCHMARK_SCALE: "[1024, 4096]" - BENCHMARK_SECONDS: "5" - BENCHMARK_SAMPLES: "20" - BENCHMARK_OUTPUT_DIR: "benchmark_results" - artifacts: - - "benchmark_results/**/*" - -# env: -# SECRET_CODECOV_TOKEN: "" - - label: "Benchmarks (OpenCL, gpu, {{matrix}})" key: "bench-opencl-gpu" # GPU timings must not compete with sibling benchmark shards. diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c3c609227..26e3a7a2b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -128,18 +128,6 @@ steps: # julia --project=test/metalenv -e 'using Pkg; Pkg.develop([PackageSpec(path=pwd()), PackageSpec(path="lib/TimespanLogging")]); Pkg.instantiate()' # julia --project=test/metalenv test/mpi/run.jl 2 2 test/mpi/metal.jl - - label: Julia 1.11 (Finch) - timeout_in_minutes: 90 - if: build.message !~ /\[skip tests\]/ - plugins: - - JuliaCI/julia#v1: - version: "1.11" - - JuliaCI/julia-test#v1: ~ - - JuliaCI/julia-coverage#v1: - codecov: true - env: - CI_TEST_FINCH: "1" - # env: # SECRET_CODECOV_TOKEN: "" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 23f33d938..16d1f2a20 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -92,6 +92,34 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false + finch: + name: Julia 1.11 (Finch) + runs-on: ubuntu-latest + timeout-minutes: 90 + if: ${{ !contains(github.event.head_commit.message, '[skip tests]') }} + env: + JULIA_NUM_THREADS: '1' + CI_TEST_FINCH: '1' + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1.11' + arch: x64 + - uses: julia-actions/cache@v3 + - name: Develop local TimespanLogging + run: julia --project=. -e 'using Pkg; Pkg.develop(path="lib/TimespanLogging")' + - name: Instantiate + run: julia --project=. -e 'using Pkg; Pkg.instantiate()' + - name: Run Finch tests + run: julia --code-coverage=user test/runtests.jl + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v7 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + opencl: name: Julia 1.11 (OpenCL) runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 52879753c..2ee908bf8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -633,3 +633,29 @@ lesson. YAML but passes `{{array}}` to the benchmark worker, which rejects the suite before sampling. Check the expanded command arguments and artifact paths for every pipeline; parser and dependency checks do not catch this error. + +61. **Resolve optional test packages before loading dependencies or starting + workers, and share the active test project.** Installing a GPU backend can + downgrade LLVM and Atomix after Dagger/KernelAbstractions already loaded + them; GPUCompiler then sees the old module with the new package graph and + fails before any GPU test runs. `Pkg.test` uses a temporary project, so + workers must inherit `Base.active_project()`, not the repository project. + Install optional packages in one batch before imports and worker startup. + +62. **Optional language runtimes must not initialize in unrelated test jobs.** + Listing PythonCall/CondaPkg as unconditional test dependencies initializes + Python during `Pkg.test` precompilation, even in GPU-only jobs. A Python + library discovery failure then aborts Metal tests before the runner starts. + Install those packages before imports only when the Python suite is selected; + default CPU runs still select that suite, and GPU runs do not. + +63. **Python setup failures can come from two independent upstream regressions.** + CondaPkg scans every environment in `LOAD_PATH`, including optional global + and stdlib environments whose `Project.toml` may not exist. CondaPkg 0.2.37 + tries to open those files and fails before Python initialization (fixed by + JuliaPy/CondaPkg.jl#235). Once resolution succeeds, PythonCall 0.9.36 can + still fail library discovery because conda-forge's Python 3.15 separates + `libpython` from the interpreter (fixed by JuliaPy/PythonCall.jl#839). + Compare unrelated PR logs and check upstream releases before changing the + test bootstrap: installing Python only for its suite does not cause either + failure. Both fixes together pass Dagger's Python suite on Julia 1.11. diff --git a/test/Project.toml b/test/Project.toml index d54a0e7aa..b17cb51ef 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,11 +1,12 @@ [deps] +# PythonCall and CondaPkg are installed by runtests.jl only for extlang/python; +# GPU jobs should not initialize a Python environment during precompilation. # Direct dep so the [compat] pin below applies. AlgebraicMultigrid -> LinearSolve # -> SparseColumnPivotedQR's AMD extension reads AMD.SS_Int, which 0.5.4 removed. AMD = "14f7f29c-3bd6-536c-9a0b-7339e30b5a3e" AlgebraicMultigrid = "2169fc97-5a83-5252-b627-83903c6c433c" ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" -CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -27,7 +28,6 @@ Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Preferences = "21216c6a-2e73-6563-6e65-726566657250" PureKLU = "0c0d3e7f-3a8b-4f7e-b6f1-9a4d2e7c1f01" PureUMFPACK = "b7e1f0a2-3c4d-4e5f-9a0b-1c2d3e4f5a6b" -PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" diff --git a/test/runtests.jl b/test/runtests.jl index 628cc24fa..ce7c874d6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -99,11 +99,6 @@ if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__ Pkg.instantiate() catch end - if USE_FINCH - # Finch is not a default test dependency; add it on demand when signaled. - Pkg.add("Finch") - end - using ArgParse s = ArgParseSettings(description = "Dagger Testsuite") @eval begin @@ -202,11 +197,31 @@ else @info "Running all tests" end +# Resolve optional dependencies before loading Dagger (and its dependencies) or +# starting workers. Adding a GPU backend can downgrade LLVM/Atomix; a process +# that already loaded them cannot switch to the newly resolved versions. +if USE_GPU || USE_FINCH || "extlang/python" in to_test + using Pkg + optional_packages = String[] + USE_CUDA && push!(optional_packages, "CUDA") + USE_ROCM && push!(optional_packages, "AMDGPU") + USE_ONEAPI && push!(optional_packages, "oneAPI") + USE_METAL && push!(optional_packages, "Metal") + USE_OPENCL && append!(optional_packages, ["OpenCL", "pocl_jll"]) + USE_FINCH && push!(optional_packages, "Finch") + if "extlang/python" in to_test + append!(optional_packages, ["CondaPkg", "PythonCall"]) + end + Pkg.add(optional_packages) +end + using Distributed if additional_workers > 0 # We put this inside a branch because addprocs() takes a minimum of 1s to # complete even if doing nothing, which is annoying. - addprocs(additional_workers; exeflags=`--project=$(joinpath(@__DIR__, "..")) -t $worker_threads`) + # Pkg.test runs in a temporary project containing the test-only backends. + # Workers must use that same resolution, rather than the repository project. + addprocs(additional_workers; exeflags=`--project=$(Base.active_project()) -t $worker_threads`) end include("imports.jl") diff --git a/test/setup_gpu.jl b/test/setup_gpu.jl index 931161850..eceef4ff5 100644 --- a/test/setup_gpu.jl +++ b/test/setup_gpu.jl @@ -1,25 +1,4 @@ -if USE_CUDA - using Pkg - Pkg.add("CUDA") -end -if USE_ROCM - using Pkg - Pkg.add("AMDGPU") -end -if USE_ONEAPI - using Pkg - Pkg.add("oneAPI") -end -if USE_METAL - using Pkg - Pkg.add("Metal") -end -if USE_OPENCL - using Pkg - Pkg.add("OpenCL") - Pkg.add("pocl_jll") -end - +# Optional packages are installed in runtests.jl before workers and imports. @everywhere begin if $USE_CUDA using CUDA