From b14e67362c30b47d7e3c4d5abfa33cdef590e4a8 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:31:42 -0700 Subject: [PATCH 01/19] Collect dashboard logs before checking its HTTP endpoint MultiEventLog aggregators run at collection, so emitting one task never started the renderer and curl failed to connect. Enable the logging runtime, collect the batch, and assert the actual HTTP status and HTML response. Keep the five-second window semantics and disable logging afterward. Validation: DaggerWebDash Basics passes all four assertions on Julia 1.12.4 with two threads. --- AGENTS.md | 7 +++++++ lib/DaggerWebDash/test/runtests.jl | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f501f5063..9fcbf854b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -859,3 +859,10 @@ lesson. the failing native call, not its root cause. Keep Intel MPI functional tests enabled separately from the benchmark jobs; a green small matmul test is insufficient evidence to enable the larger tiled benchmark matrix. + +82. **Dashboard aggregators run when logs are collected.** A `D3Renderer` + attached to a `MultiEventLog` does not start its HTTP listener when a task + emits an event. Call `get_logs!` to run the consumers and aggregators before + requesting its page. A short `LogWindow` may legitimately discard every + event emitted before a long first compilation; test the HTTP response + independently of whether that batch still contains recent events. diff --git a/lib/DaggerWebDash/test/runtests.jl b/lib/DaggerWebDash/test/runtests.jl index 8c0b9818d..4cd5a2f24 100644 --- a/lib/DaggerWebDash/test/runtests.jl +++ b/lib/DaggerWebDash/test/runtests.jl @@ -1,5 +1,5 @@ using DaggerWebDash -import DaggerWebDash: LinePlot, GanttPlot +import DaggerWebDash: LinePlot, GanttPlot, HTTP using TimespanLogging using Dagger using Test @@ -33,6 +33,18 @@ using Test ml.aggregators[:logwindow] = lw ctx = Context(; log_sink=ml) Dagger.Sch.EAGER_CONTEXT[] = ctx - fetch(Dagger.@spawn 1+1) - run(pipeline(`curl -s localhost:8080/index.html`; stdout=devnull)) + TimespanLogging.enable!() + try + @test fetch(Dagger.@spawn 1+1) == 2 + # Consumers and aggregators run when logs are collected, rather than + # during event emission. Collection starts the renderer's HTTP server. + logs = TimespanLogging.get_logs!(ml) + # The five-second window may prune events emitted before compilation. + @test haskey(logs[1], :core) + response = HTTP.get("http://localhost:8080/index.html") + @test response.status == 200 + @test occursin(" Date: Thu, 15 Oct 2026 09:35:29 -0700 Subject: [PATCH 02/19] Keep the first event inside a log retention window LogWindow used the first retained index as a deletion count, dropping an additional valid event from mixed-age batches. Remove only the preceding entries and preserve alignment across consumer columns. Validation: all 77 TimespanLogging assertions pass on Julia 1.12.4 with four threads, including mixed, entirely retained, and entirely expired batches. --- AGENTS.md | 7 +++++++ lib/TimespanLogging/src/extras.jl | 2 +- lib/TimespanLogging/test/runtests.jl | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9fcbf854b..453ff2d52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -866,3 +866,10 @@ lesson. requesting its page. A short `LogWindow` may legitimately discard every event emitted before a long first compilation; test the HTTP response independently of whether that batch still contains recent events. + +83. **A log-window cutoff is a retained index, not a removal count.** + `findfirst(timestamp >= cutoff)` names the first event that must survive; + deleting through that index discards one valid event whenever a batch mixes + old and recent records. Convert it to the preceding index, and use the + full batch length only when every record expired. Apply the same removal + count to every consumer column and deletion hook. diff --git a/lib/TimespanLogging/src/extras.jl b/lib/TimespanLogging/src/extras.jl index 5720307b4..c46cb0c43 100644 --- a/lib/TimespanLogging/src/extras.jl +++ b/lib/TimespanLogging/src/extras.jl @@ -125,7 +125,7 @@ function (lw::LogWindow)(logs::Dict) # Find entries outside the window window_start = time_ns() - lw.window_length idx = something(findfirst(ev->ev.timestamp >= window_start, core_log), - length(core_log)) + length(core_log) + 1) - 1 # Return if all events are in the window if length(core_log) == 0 || (core_log[1].timestamp >= window_start) diff --git a/lib/TimespanLogging/test/runtests.jl b/lib/TimespanLogging/test/runtests.jl index 6fad96d8e..307a0ebfd 100644 --- a/lib/TimespanLogging/test/runtests.jl +++ b/lib/TimespanLogging/test/runtests.jl @@ -41,6 +41,24 @@ end @test isempty(steal_legacy()) end + @testset "LogWindow retains the first unexpired event" begin + now = time_ns() + window = UInt64(60_000_000_000) + lw = Events.LogWindow(window, :core) + logs = Dict(:core => [(timestamp=now - 2window,), (timestamp=now,)], + :ids => [1, 2]) + lw(logs) + @test logs[:core] == [(timestamp=now,)] + @test logs[:ids] == [2] + + lw(logs) + @test logs[:ids] == [2] + expired = Dict(:core => [(timestamp=now - 2window,)], :ids => [1]) + lw(expired) + @test isempty(expired[:core]) + @test isempty(expired[:ids]) + end + @testset "Legacy Symbol API + LocalEventLog" begin reset!() ctx = Ctx(LocalEventLog(), false) From 7f8847ac6d97c4f28ac835490e2ab6e87f38465c Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:37:58 -0700 Subject: [PATCH 03/19] Run older Julia Distributed jobs on the selected transport consistently Use DistributedNext for the Julia 1.10/1.11 multiprocess CI jobs, where stdlib Distributed raises lock-must-be-held concurrency errors. Configure all three transport preferences before imports, use the same transport for test macros and worker management, and preserve it after preference tests. Workers created inside scheduler/processor tests now inherit the active test project. Validation: Julia 1.10.12, two driver threads and three workers with two threads each: thunk, scheduler, processors, options, task queues, datadeps (2143 passed), reuse (12615 passed), and preferences all pass. Existing broken tests remain reported as broken. Newer Julia jobs retain stdlib transport coverage. --- .github/workflows/CI.yml | 4 ++++ AGENTS.md | 9 +++++++++ test/gpu.jl | 2 +- test/gpu_distributed.jl | 2 +- test/imports.jl | 2 +- test/preferences.jl | 11 ++++++++--- test/processors.jl | 4 ++-- test/runtests.jl | 16 ++++++++++++++-- test/scheduler.jl | 14 ++++++++------ 9 files changed, 48 insertions(+), 16 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 16d1f2a20..2052a57d0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,6 +23,10 @@ jobs: if: ${{ !contains(github.event.head_commit.message, '[skip tests]') }} env: JULIA_NUM_THREADS: '1' + # Distributed's older releases race when used from multiple threads. + # Keep stdlib coverage on newer Julia while testing the supported + # DistributedNext transport on 1.10/1.11. + CI_DISTRIBUTED_PACKAGE: ${{ (matrix.version == '1.10' || matrix.version == '1.11') && 'DistributedNext' || 'Distributed' }} strategy: fail-fast: false matrix: diff --git a/AGENTS.md b/AGENTS.md index 453ff2d52..6f6f7a216 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -873,3 +873,12 @@ lesson. old and recent records. Convert it to the preceding index, and use the full batch length only when every record expired. Apply the same removal count to every consumer column and deletion hook. + +84. **Transport selection must cover the test harness and every extra worker.** + Dagger, MemPool, and TimespanLogging read their transport preference before + loading; setting only Dagger's preference or continuing to import stdlib + Distributed in tests creates separate process registries. Import the module + that owns `Dagger.myid` throughout the harness. Even workers launched inside + individual tests must inherit `Base.active_project()`; bare `--project` + selects the checkout instead and can lose DistributedNext and test-only + dependencies. Preference tests must restore the incoming transport. diff --git a/test/gpu.jl b/test/gpu.jl index bfb3c79f8..822e653d1 100644 --- a/test/gpu.jl +++ b/test/gpu.jl @@ -4,7 +4,7 @@ using SparseArrays using Krylov @everywhere begin - using Distributed, Dagger + using Dagger import Dagger: Kernel using KernelAbstractions end diff --git a/test/gpu_distributed.jl b/test/gpu_distributed.jl index f96742b0b..b65b49dfe 100644 --- a/test/gpu_distributed.jl +++ b/test/gpu_distributed.jl @@ -1,6 +1,6 @@ # Included by gpu.jl in every GPU CI job. Both workers select device 1, so # this exercises inter-process transfers even on agents with just one GPU. -using Distributed, Dagger, LinearAlgebra, Test +using .Distributed, Dagger, LinearAlgebra, Test @everywhere begin function distributed_gpu_add!(A) diff --git a/test/imports.jl b/test/imports.jl index 136f28250..2937f6502 100644 --- a/test/imports.jl +++ b/test/imports.jl @@ -1,5 +1,5 @@ using LinearAlgebra, SparseArrays, Random import Dagger: DArray, chunks, domainchunks, treereduce_nd -import Distributed: myid, procs +import .Distributed: myid, procs import Statistics: mean, var, std import OnlineStats diff --git a/test/preferences.jl b/test/preferences.jl index 74f778f16..199cb5500 100644 --- a/test/preferences.jl +++ b/test/preferences.jl @@ -15,6 +15,9 @@ import Preferences: load_preference, set_preferences! load_path = join(Base.load_path(), Sys.iswindows() ? ';' : ':') base_cmd = `$(Base.julia_cmd()) --startup-file=no -E 'using Dagger; parentmodule(Dagger.myid)'` cmd = addenv(base_cmd, "JULIA_LOAD_PATH" => load_path) + packages = (Dagger, Dagger.MemPool, Dagger.TimespanLogging) + distributed_packages = map(p -> load_preference(p, "distributed-package", nothing), packages) + precompile_workload = load_preference(Dagger, "precompile_workload", nothing) try # Disabling the precompilation workload shaves off over half the time # this test takes. @@ -28,8 +31,10 @@ import Preferences: load_preference, set_preferences! @test readchomp(cmd) == "DistributedNext" end finally - set_preferences!(Dagger, "precompile_workload" => true; force=true) - # Reset distributed package to the default to avoid affecting subsequent runs - Dagger.set_distributed_package!("Distributed") + set_preferences!(Dagger, "precompile_workload" => precompile_workload; force=true) + # Preserve the transport selected by the test environment. + for (package, value) in zip(packages, distributed_packages) + set_preferences!(package, "distributed-package" => value; force=true) + end end end diff --git a/test/processors.jl b/test/processors.jl index fa2f30551..0544da90a 100644 --- a/test/processors.jl +++ b/test/processors.jl @@ -1,4 +1,4 @@ -using Distributed +using .Distributed import Dagger: Context, Options, Processor, OSProc, ThreadProc, get_parent, get_processors @everywhere begin @@ -61,7 +61,7 @@ end end @testset "Modify workers in Context" begin - ps = addprocs(4, exeflags="--project") + ps = addprocs(4, exeflags=`--project=$(Base.active_project())`) @everywhere ps using Dagger ctx = Context(ps[1:2]) diff --git a/test/runtests.jl b/test/runtests.jl index 19fb41804..ba5225590 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -226,7 +226,20 @@ if USE_GPU || USE_FINCH || "extlang/python" in to_test Pkg.add(optional_packages) end -using Distributed +# Select the transport before loading Dagger, MemPool, or TimespanLogging: +# all three packages must resolve the same preference on every worker. +if haskey(ENV, "CI_DISTRIBUTED_PACKAGE") + using Preferences + distributed_package = ENV["CI_DISTRIBUTED_PACKAGE"] + distributed_package in ("Distributed", "DistributedNext") || + error("Invalid CI_DISTRIBUTED_PACKAGE: $distributed_package") + for package in ("Dagger", "MemPool", "TimespanLogging") + Preferences.set_preferences!(package, "distributed-package" => distributed_package; force=true) + end +end +using Dagger +const Distributed = parentmodule(Dagger.myid) +using .Distributed if additional_workers > 0 # We put this inside a branch because addprocs() takes a minimum of 1s to # complete even if doing nothing, which is annoying. @@ -240,7 +253,6 @@ include("util.jl") include("fakeproc.jl") using Test -using Dagger using UUIDs @everywhere import MemPool diff --git a/test/scheduler.jl b/test/scheduler.jl index 211abd0cb..aaad28cf7 100644 --- a/test/scheduler.jl +++ b/test/scheduler.jl @@ -179,7 +179,9 @@ end # ensure workers are actually modified while the scheduler is working. setup = quote - using Dagger, Distributed + using Dagger + const Distributed = parentmodule(Dagger.myid) + using .Distributed function _list_workers(ctx, state, task, tid, _) return procs(ctx) end @@ -209,7 +211,7 @@ end @testset "Add new workers" begin ps = [] try - ps1 = addprocs(2, exeflags="--project") + ps1 = addprocs(2, exeflags=`--project=$(Base.active_project())`) append!(ps, ps1) @everywhere vcat(ps1, myid()) $setup @@ -224,10 +226,10 @@ end end # Will not be added, so they should never appear in output - ps2 = addprocs(2, exeflags="--project") + ps2 = addprocs(2, exeflags=`--project=$(Base.active_project())`) append!(ps, ps2) - ps3 = addprocs(2, exeflags="--project") + ps3 = addprocs(2, exeflags=`--project=$(Base.active_project())`) append!(ps, ps3) @everywhere ps3 $setup addprocs!(ctx, ps3) @@ -251,7 +253,7 @@ end #=@testset "Remove workers" begin ps = [] try - ps1 = addprocs(4, exeflags="--project") + ps1 = addprocs(4, exeflags=`--project=$(Base.active_project())`) append!(ps, ps1) @everywhere vcat(ps1, myid()) $setup @@ -310,7 +312,7 @@ end @testset "Remove all workers throws" begin ps = [] try - ps1 = addprocs(2, exeflags="--project") + ps1 = addprocs(2, exeflags=`--project=$(Base.active_project())`) append!(ps, ps1) @everywhere vcat(ps1, myid()) $setup From 8117d316d851dab589cc2660a714930d137e78cd Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:38:50 -0700 Subject: [PATCH 04/19] Preserve distributed factorization solve results on Julia 1.14 LinearAlgebra now passes a Factorization to _zeros instead of its column count. Forward that signature to Dagger existing allocator so distributed RHS solves keep their storage and dispatch. Cover vectors and matrices, numerical results, RHS preservation, and allocation via the new protocol on every supported Julia. Validation: on Julia 1.14.0-DEV.3709 the Cholesky reproducer changes from Vector to DVector; all 25 allocation-suite checks pass with existing bounds. Python integration also passes all six checks on this nightly; the independent Pkg CommandSpec precompile failure is fixed upstream in JuliaLang/Pkg.jl#4863. --- AGENTS.md | 8 ++++++++ src/array/alloc.jl | 5 +++++ test/array/linalg/solve.jl | 21 ++++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 6f6f7a216..ac30e59ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -882,3 +882,11 @@ lesson. individual tests must inherit `Base.active_project()`; bare `--project` selects the checkout instead and can lose DistributedNext and test-only dependencies. Preference tests must restore the incoming transport. + +85. **Track LinearAlgebra's factorization allocation protocol across Julia versions.** + Dagger's `_zeros(T, rhs, n::Integer)` hook keeps factorization solves over + distributed right-hand sides distributed. Julia 1.14 now passes the + factorization itself as the third argument. Without the matching method, + Cholesky/QR solves silently allocate a local Array and bypass Dagger's + distributed solve path. Forward the new protocol to the existing column-count + method, and check both result type and values in solve tests. diff --git a/src/array/alloc.jl b/src/array/alloc.jl index 1497bec08..7552a3d7b 100644 --- a/src/array/alloc.jl +++ b/src/array/alloc.jl @@ -144,6 +144,11 @@ function LinearAlgebra._zeros(::Type{T}, B::DMatrix, n::Integer) where T return zeros(auto_blocks(sz), T, sz) end +# LinearAlgebra 1.14 passes the factorization rather than its column count. +# Preserve distributed RHS storage through that allocation path as well. +LinearAlgebra._zeros(::Type{T}, B::DVecOrMat, F::LinearAlgebra.Factorization) where T = + LinearAlgebra._zeros(T, B, size(F, 2)) + function Base.view(A::AbstractArray{T,N}, p::Blocks{N}) where {T,N} d = ArrayDomain(Base.index_shape(A)) dc = partition(p, d) diff --git a/test/array/linalg/solve.jl b/test/array/linalg/solve.jl index c29471270..57b72d90b 100644 --- a/test/array/linalg/solve.jl +++ b/test/array/linalg/solve.jl @@ -70,4 +70,23 @@ LinearAlgebra.ldiv!(chol_DA, Db) @test collect(Db) ≈ b_ref rtol=tol end -end \ No newline at end of file +end + +@testset "Factorization solves retain distributed RHS storage" begin + A = Matrix{Float64}(I, 8, 8) + fill(0.1, 8, 8) + for b in (collect(1.0:8.0), reshape(collect(1.0:16.0), 8, 2)) + Db = distribute(b, b isa AbstractVector ? Blocks(3) : Blocks(3, 1)) + for F in (cholesky(distribute(A, Blocks(4, 4))), + lu(distribute(A, Blocks(4, 4)))) + x = F \ Db + @test x isa Dagger.DArray + @test collect(x) ≈ A \ b + @test collect(Db) == b + # Exercise both allocation protocols even on older Julia. + dest = LinearAlgebra._zeros(Float64, Db, F) + @test dest isa Dagger.DArray + @test size(dest) == size(b) + @test iszero(sum(dest)) + end + end +end From 44463e0f18f04bd0d5e5102a9e1e8e85a74e51a8 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:39:40 -0700 Subject: [PATCH 05/19] Align LU pivot blocks with the returned factor matrix NoPivot allocated an automatically partitioned identity pivot vector, while row-swap kernels assume its blocks match the factors. This corrupted solves depending on processor count. Rectangular tiles also left pivoted LU pivots in the temporary square-panel layout. Allocate or repartition pivots to the returned factors row blocking. Validation: Julia 1.11.9 with four threads passes 136 LU and 44 solve assertions, including real/complex, square/rectangular tile layouts, partial edge tiles, and vector/matrix RHS. A deterministic old-layout reproducer has solution error 0.5249 versus 2.09e-16 after alignment. Memory: Julia 1.11.9, identical dependencies, pinned 16x16 fixture with 8x8 tiles, 10 warmups then GC and minima over five runs. NoPivot: 9464 allocations / 441440 bytes before, 9663 / 448496 after (two required pivot tiles instead of the incorrect single automatic tile). RowMaximum: 10937 / 503104 before, 10985 / 504384 after. Existing allocation bounds are unchanged. --- AGENTS.md | 9 +++++++++ src/array/lu.jl | 7 ++++++- test/array/linalg/lu.jl | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index ac30e59ad..3a0266e6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -890,3 +890,12 @@ lesson. Cholesky/QR solves silently allocate a local Array and bypass Dagger's distributed solve path. Forward the new protocol to the existing column-count method, and check both result type and values in solve tests. + +86. **LU pivot chunks must match the returned factors' row blocks.** + `_dagger_apply_ipiv_rows!` interprets every pivot chunk using the factors' + row block size. AutoBlocks for NoPivot can therefore permute an identity + permutation incorrectly, or run its unchecked swap kernel out of bounds. + Pivoted LU also needs to re-tile its pivots after square-panel buffering of + a non-square tile layout. Read the returned matrix's partitioning at that + point: the buffering closure reassigns the captured `mb`/`nb` locals to the + temporary square layout, so those locals no longer describe the original. diff --git a/src/array/lu.jl b/src/array/lu.jl index 9483b6838..3c0cc0f0a 100644 --- a/src/array/lu.jl +++ b/src/array/lu.jl @@ -43,7 +43,9 @@ function LinearAlgebra.lu!(A::DMatrix{T}, ::LinearAlgebra.NoPivot; check::Bool = end end - ipiv = DVector([i for i in 1:min(size(A)...)]) + # Row permutation kernels index pivot chunks using the factor row blocks. + # AutoBlocks depends on the processor count and need not match those blocks. + ipiv = DVector(collect(1:min(size(A)...)), Blocks(mb)) return LinearAlgebra.LU{T,DMatrix{T},DVector{Int}}(A, ipiv, 0) end @@ -381,5 +383,8 @@ function LinearAlgebra.lu!(A::DMatrix{T}, ::LinearAlgebra.RowMaximum; check::Boo return panel_info end + # Buffering a non-square tile layout factors square tiles internally, but + # row permutation kernels use the original factor matrix's row blocking. + ipiv = repartition(ipiv, Blocks(A.partitioning.blocksize[1])) return LinearAlgebra.LU{T,DMatrix{T},DVector{Int}}(A, ipiv, info_value) end diff --git a/test/array/linalg/lu.jl b/test/array/linalg/lu.jl index c7b660dca..0de5bab03 100644 --- a/test/array/linalg/lu.jl +++ b/test/array/linalg/lu.jl @@ -13,6 +13,24 @@ function factorization_rtol(T, pivot) end end +function test_lu_pivot_blocks(::Type{T}, part, pivot) where T + A = Matrix{T}(I, 13, 13) + rand(T, 13, 13) / 20 + F = lu(distribute(A, part), pivot) + @test F.ipiv.partitioning == Blocks(part.blocksize[1]) + pivot == NoPivot() && @test collect(F.ipiv) == collect(1:13) + b = rand(T, 13) + @test collect(F \ distribute(b, Blocks(2))) ≈ A \ b + B = rand(T, 13, 2) + @test collect(F \ distribute(B, Blocks(2, 1))) ≈ A \ B +end + +@testset "Pivot blocks follow factor rows" begin + # Deliberately differ from AutoBlocks, including a partial final tile. + for T in (Float64, ComplexF64), part in (Blocks(3, 3), Blocks(5, 3)), pivot in (NoPivot(), RowMaximum()) + test_lu_pivot_blocks(T, part, pivot) + end +end + @testset "$T with $pivot" for T in (Float64, ComplexF64), pivot in (NoPivot(), RowMaximum()) A = rand(T, 128, 128) B = copy(A) From 58a0e3dfb874015bb27a28d775186d4678cf98fa Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:43:44 -0700 Subject: [PATCH 06/19] Make sparse benchmark fixtures reproducible across revisions Sparse inputs still used arbitrary placement, allowing driver-local allocation reports to change with ownership. Use the existing CPU Distributed assignment helper and seeded input generators in setup and capability probes. MPI/GPU keep arbitrary placement; operations, result placement, samples, and regression thresholds are unchanged. Validation: Julia 1.12.4 with three one-thread workers runs SpMV, CG, and SpGEMM benchmark leaves with three samples each at N=64 and block size 16. Repeated seeded matrices collect identically and have the same balanced 4/4/4/4 tile-owner layout. Smoke allocation minima: SpMV 54387 / 2454224 bytes, CG 623058 / 27935464, SpGEMM 69602 / 3083344; these remain process-local measurements. --- AGENTS.md | 9 +++++++++ benchmark/suites/sparse.jl | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3a0266e6f..5c04c54fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -899,3 +899,12 @@ lesson. a non-square tile layout. Read the returned matrix's partitioning at that point: the buffering closure reassigns the captured `mb`/`nb` locals to the temporary square layout, so those locals no longer describe the original. + +87. **Sparse benchmark fixtures need fixed placement and fixed nonzeros.** + The sparse suite used arbitrary placement after the other suites adopted + `benchmark_assignment()`. Driver-only BenchmarkTools allocation counts then + changed with the driver's share of sparse tiles, even across the same + revision. Seed sparse matrices and right-hand sides in untimed setup and + use the shared assignment helper for both operands and capability probes. + Keep result placement and regression thresholds unchanged; MPI/GPU fixtures + must still use the helper's scope-aware arbitrary placement. diff --git a/benchmark/suites/sparse.jl b/benchmark/suites/sparse.jl index f2617527f..4669197fd 100644 --- a/benchmark/suites/sparse.jl +++ b/benchmark/suites/sparse.jl @@ -35,6 +35,9 @@ function sparse_suite(ctx; method, accels) @assert accel == "cpu" || haskey(GPU_BACKENDS, accel) "Unknown backend" T = benchmark_eltype() + # Driver-local allocation counts require the same tile placement in both + # revisions. Seed the untimed fixtures too, so sparse structure is fixed. + fixture_assignment = benchmark_assignment() atol, rtol = T(1e-8), T(1e-6) # Target a fixed number of nonzeros per row, so density shrinks with N and # the nonzero count stays ~O(N). @@ -48,17 +51,17 @@ function sparse_suite(ctx; method, accels) # operation present on one revision but not the other simply won't be # compared. This self-heals once the baseline gains the feature. spmv_ok = supported("sparse/spmv (S*x)") do - S = distribute(sprand(T, 8, 8, 0.5), Blocks(4, 4)) - x = distribute(rand(T, 8), Blocks(4)) + S = distribute(sprand(Random.Xoshiro(1234), T, 8, 8, 0.5), Blocks(4, 4), fixture_assignment) + x = distribute(rand(Random.Xoshiro(4321), T, 8), Blocks(4), fixture_assignment) wait(S * x) end spgemm_ok = supported("sparse/spgemm (S*S)") do - S = distribute(sprand(T, 8, 8, 0.5), Blocks(4, 4)) + S = distribute(sprand(Random.Xoshiro(1234), T, 8, 8, 0.5), Blocks(4, 4), fixture_assignment) wait(S * S) end cg_ok = KRYLOV_AVAILABLE && isdefined(Dagger, :cg) && supported("sparse/cg solve") do - A = distribute(laplacian_1d(T, 8), Blocks(4, 4)) - rhs = distribute(rand(T, 8), Blocks(4)) + A = distribute(laplacian_1d(T, 8), Blocks(4, 4), fixture_assignment) + rhs = distribute(rand(Random.Xoshiro(4321), T, 8), Blocks(4), fixture_assignment) wait(first(Dagger.cg(A, rhs; atol=atol, rtol=rtol, itmax=50))) end @@ -72,23 +75,23 @@ function sparse_suite(ctx; method, accels) # SpMV: sparse matrix tiles, dense vectors. if spmv_ok && fits_budget(sparse_bytes(N; nmats=2, density=density, T=T)) sub["spmv (S*x)"] = @benchmarkable(wait(S * x), - setup = (S = distribute(sprand($T, $N, $N, $density), Blocks($b, $b)); - x = distribute(rand($T, $N), Blocks($b)); wait(S); wait(x)), + setup = (S = distribute(sprand(Random.Xoshiro(1234), $T, $N, $N, $density), Blocks($b, $b), $fixture_assignment); + x = distribute(rand(Random.Xoshiro(4321), $T, $N), Blocks($b), $fixture_assignment); wait(S); wait(x)), teardown = (S = nothing; x = nothing; @everywhere GC.gc())) end # SpGEMM: the result fills in, so budget generously. if spgemm_ok && fits_budget(sparse_bytes(N; nmats=6, density=density, T=T)) sub["spgemm (S*S)"] = @benchmarkable(wait(S * S), - setup = (S = distribute(sprand($T, $N, $N, $density), Blocks($b, $b)); wait(S)), + setup = (S = distribute(sprand(Random.Xoshiro(1234), $T, $N, $N, $density), Blocks($b, $b), $fixture_assignment); wait(S)), teardown = (S = nothing; @everywhere GC.gc())) end # Iterative solve of an SPD system via conjugate gradients. if cg_ok && fits_budget(sparse_bytes(N; nmats=2, density=3 / N, T=T)) sub["cg solve (laplacian)"] = @benchmarkable(wait(first(Dagger.cg(A, rhs; atol=$atol, rtol=$rtol, itmax=200))), - setup = (A = distribute(laplacian_1d($T, $N), Blocks($b, $b)); - rhs = distribute(rand($T, $N), Blocks($b)); wait(A); wait(rhs)), + setup = (A = distribute(laplacian_1d($T, $N), Blocks($b, $b), $fixture_assignment); + rhs = distribute(rand(Random.Xoshiro(4321), $T, $N), Blocks($b), $fixture_assignment); wait(A); wait(rhs)), teardown = (A = nothing; rhs = nothing; @everywhere GC.gc())) end From f480e3da2cfd1df870cd63b7249ed70281e6111c Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:49:39 -0700 Subject: [PATCH 07/19] Handle generic sparse destinations in Finch tile multiplication Out-of-place Finch GEMM can start with a host CSC destination from similar. The fallback sent Finch tensors through generic collect and failed because Tensor has no length. Build the Finch result and replace the sparse wrapper storage; materialize an existing non-Finch destination before nonzero-beta accumulation. Remove the obsolete test LOAD_PATH injection since workers already inherit the active project. Validation: Julia 1.11.9, one driver thread and three one-thread workers: all 32 GEMM and 16 SpMV checks pass, including new mixed-storage beta checks. A separate two-thread run exposes Finch 1.6.0 global LogicExecutor.codes Dict races; no serialization workaround is introduced. Memory: native 2x2 Finch GEMM with nonzero beta, Julia 1.11.9, 10 warmups, GC, minimum over five measurements: 1530 allocations / 65472 bytes before and after. The previously failing CSC-destination path has no valid baseline allocation measurement. --- AGENTS.md | 11 +++++++++++ ext/FinchExt.jl | 12 ++++++------ test/array/linalg/matmul_finch.jl | 12 +++++++----- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5c04c54fd..61a590c5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -908,3 +908,14 @@ lesson. use the shared assignment helper for both operands and capability probes. Keep result placement and regression thresholds unchanged; MPI/GPU fixtures must still use the helper's scope-aware arbitrary placement. + +88. **A Finch multiply may receive a CSC destination from `similar`.** + Dagger's generic sparse allocator does not retain Finch's tensor format. + Forwarding such a destination to generic matmul also forwards raw Finch + tensors to `collect`, which requires an iteration/length interface they do + not implement. Compute the Finch result and replace the DSparseArray's + storage; for nonzero beta, materialize the previous destination into Finch + before accumulation. Native Finch output still uses the same path. Finch + 1.6.0 also has an independent thread-unsafe global `LogicExecutor.codes` + Dict: cold two-thread GEMM can raise a concurrent-write assertion even when + the one-thread-per-worker CI topology passes. diff --git a/ext/FinchExt.jl b/ext/FinchExt.jl index 927cb56ca..e7ccecaf0 100644 --- a/ext/FinchExt.jl +++ b/ext/FinchExt.jl @@ -116,10 +116,6 @@ function Dagger.matmatmul!( alpha, beta ) - if !isa(C.mat, Finch.Tensor) - # Not supported here, forward to generic matmatmul! - return Dagger.matmatmul!(C.mat, transA, transB, A, B, alpha, beta) - end transA in ('N', 'T', 'C') || throw(ArgumentError("Invalid transA: $transA")) transB in ('N', 'T', 'C') || throw(ArgumentError("Invalid transB: $transB")) @@ -136,7 +132,11 @@ function Dagger.matmatmul!( # sparse `Tensor` so accumulation and later iterations stay well-typed. C_out = _finch_materialize(C_out, eltype(Cm), size(Cm)) if beta != 0 - C_out = C_out + beta * Cm + # `similar(::DArray)` may initially allocate host CSC tiles. The + # wrapper permits replacing their storage with the Finch result, but + # accumulation must first bring the previous values into Finch too. + C_prev = Cm isa Finch.Tensor ? Cm : _finch_materialize(Cm, eltype(Cm), size(Cm)) + C_out = C_out + beta * C_prev end C.mat = C_out @@ -185,4 +185,4 @@ function Dagger.transpose_tile(B::Finch.Tensor, uplo=nothing) return _finch_materialize(C, eltype(_B), size(C)) end -end # module FinchExt \ No newline at end of file +end # module FinchExt diff --git a/test/array/linalg/matmul_finch.jl b/test/array/linalg/matmul_finch.jl index 061874f1e..9be40ea28 100644 --- a/test/array/linalg/matmul_finch.jl +++ b/test/array/linalg/matmul_finch.jl @@ -11,11 +11,6 @@ # N.B. Finch precompilation is heavy (several minutes); this test is intended for # local validation rather than fast CI. -# Make the test-project environment (which provides Finch) resolvable on the -# workers, which are launched with Dagger's project. -const _FINCH_TEST_ENV = abspath(joinpath(@__DIR__, "..", "..")) -@everywhere pushfirst!(LOAD_PATH, $_FINCH_TEST_ENV) - const FINCH_AVAILABLE = try @eval using Finch @everywhere using Finch @@ -56,6 +51,13 @@ else DSC = Finch.fspzeros(partC, T, sz) mul!(DSC, DSA, DSB) @test collect(DSC) ≈ A * B + + # The generic sparse allocator starts with CSC storage. Finch kernels + # must handle that destination, including its nonzero previous values. + DSC_mixed = similar(DSA) + copyto!(DSC_mixed, DSA) + mul!(DSC_mixed, DSA, DSB, T(2), T(3)) + @test collect(DSC_mixed) ≈ 2 * A * B + 3 * A end const FINCH_QUICK_CASES = [ From 793f12241cf3fc05c601cbc3cb192870be48b92c Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:54:53 -0700 Subject: [PATCH 08/19] Await scheduler teardown before removing test workers The harness removed worker sockets one second after requesting scheduler shutdown, racing cleanup RPCs and logging late transport errors. Wait on the existing scheduler-exit condition instead of sleeping. Validation: Julia 1.11.9 with a one-thread driver and three two-thread workers passes 131 thunk and 30 processor assertions and exits without the previously observed unknown-socket error. Julia 1.10.12 with four threads and no initial workers passes 120 thunk assertions and exits cleanly. Existing broken checks remain reported. --- AGENTS.md | 7 +++++++ test/runtests.jl | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 61a590c5e..c5d0e4867 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -919,3 +919,10 @@ lesson. 1.6.0 also has an independent thread-unsafe global `LogicExecutor.codes` Dict: cold two-thread GEMM can raise a concurrent-write assertion even when the one-thread-per-worker CI topology passes. + +89. **Wait for scheduler teardown before removing test workers.** + Notifying `state.halt` starts cleanup RPCs; it does not finish them. A + one-second sleep can expire while a worker is still compiling its cleanup + method, and removing its socket then logs a transport error after passing + assertions. Wait on `Sch.EAGER_STATE_LOCK` until `EAGER_INIT` clears, which + happens after scheduler cleanup, before calling `rmprocs`. diff --git a/test/runtests.jl b/test/runtests.jl index ba5225590..4d1a1fea4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -282,8 +282,15 @@ finally state = Dagger.Sch.EAGER_STATE[] if state !== nothing notify(state.halt) + # Scheduler teardown performs RPCs to workers. Wait for those calls to + # finish before removing their sockets; a fixed sleep races cold + # compilation and can produce transport errors after passing tests. + lock(Dagger.Sch.EAGER_STATE_LOCK) do + while Dagger.Sch.EAGER_INIT[] + wait(Dagger.Sch.EAGER_STATE_LOCK) + end + end end - sleep(1) if nprocs() > 1 rmprocs(workers()) end From b2dea1f256e29a7572a8b73d7528a624bb4a5adf Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 09:54:53 -0700 Subject: [PATCH 09/19] Record the unresolved MPI stencil compilation diagnosis --- AGENTS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index c5d0e4867..697e78e18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -926,3 +926,13 @@ lesson. method, and removing its socket then logs a transport error after passing assertions. Wait on `Sch.EAGER_STATE_LOCK` until `EAGER_INIT` clears, which happens after scheduler cleanup, before calling `rmprocs`. + +90. **Inspect peer compilation before diagnosing an MPI receive timeout.** + The four-rank Julia 1.10 CPU suite can fail in the 4D stencil and cascade + into later uniformity checks (reproduced as 1,433 passes and seven errors). + Peer stack snapshots showed LLVM's `LiveDebugValues`/PHI-placement passes + compiling planning calls while other ranks waited on uniformity or result + broadcasts. The receive stack alone therefore does not establish a message + ordering defect. Isolate the cold 4D stencil and inspect all ranks; changing + planner specialization alone is not yet a validated fix, and increasing a + timeout would not fix the underlying compilation behavior. From 3aefaaad740f358837f8a32155521bb2ed59eee5 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 12:01:57 -0700 Subject: [PATCH 10/19] Keep MPI stencil coverage through 3D without pathological 4D compilation The cold 4D case generates very large planning signatures and leaves peer ranks waiting while LLVM compiles. Disable only this case for CPU and GPU MPI entry points, retaining the existing backend-specific high-dimensional skips and all non-MPI coverage. Validation: Julia 1.10.12, four MPI ranks with two threads each: all 52 remaining stencil assertions pass on every rank (211 seconds). --- test/array/stencil_defs.jl | 3 ++- test/mpi/cpu.jl | 3 ++- test/mpi/gpu_suite.jl | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/array/stencil_defs.jl b/test/array/stencil_defs.jl index 2a7b206a7..25f434a7d 100644 --- a/test/array/stencil_defs.jl +++ b/test/array/stencil_defs.jl @@ -4,7 +4,7 @@ @everywhere import Dagger: @stencil, Wrap, Pad, Reflect, AntiReflect, Clamp, LinearExtrapolate -function test_stencil(; skip_highdim::Bool=false) +function test_stencil(; skip_highdim::Bool=false, skip_4d::Bool=false) @testset "Simple assignment" begin A = zeros(Blocks(2, 2), Int, 4, 4) @stencil A[idx] = 1 @@ -526,6 +526,7 @@ function test_stencil(; skip_highdim::Bool=false) # stencils and corrupts subsequent tests, so they can be skipped there. if !skip_highdim for N in 3:4 + skip_4d && N == 4 && continue @testset "$(N)D array" begin A = ones(Blocks(ntuple(_->1, N)...), Int, ntuple(_->3, N)...) Dagger.allowscalar() do diff --git a/test/mpi/cpu.jl b/test/mpi/cpu.jl index 65c2697ea..82b2a4501 100644 --- a/test/mpi/cpu.jl +++ b/test/mpi/cpu.jl @@ -836,7 +836,8 @@ include("lu.jl") # boundary condition, including the stencil_source_chunks pre-sweep # snapshot used by self-referencing Wrap stencils (@stencil A[idx] = # f(@neighbors(A[idx]))). - test_stencil() + # The 4D case triggers pathological cold compilation on MPI ranks. + test_stencil(; skip_4d=true) end @testset "Sparse" begin diff --git a/test/mpi/gpu_suite.jl b/test/mpi/gpu_suite.jl index 5906ac710..3455a28f4 100644 --- a/test/mpi/gpu_suite.jl +++ b/test/mpi/gpu_suite.jl @@ -299,7 +299,8 @@ if get(cfg, :stencil, false) # exchange (including the stencil_source_chunks pre-sweep snapshot # for self-referencing Wrap stencils) on the GPU. Dagger.with_options(;scope=all_gpu_scope()) do - test_stencil(; skip_highdim=get(cfg, :stencil_skip_highdim, false)) + test_stencil(; skip_highdim=get(cfg, :stencil_skip_highdim, false), + skip_4d=true) end end end From 1583b12ac779289d41dce3e22857dffe2d20ba31 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 12:09:22 -0700 Subject: [PATCH 11/19] Identify the stencil backend and dimension before long compilation Cancelled jobs previously stopped at a single combined stencil-suite message, hiding whether CPU execution, GPU execution, or high-dimensional compilation was active. Emit progress at backend and high-dimensional case boundaries, and record the local PoCL profiling evidence. Validation: the shared suite passes 53 assertions on PoCL, including 3D and 4D; this change only adds test progress messages. --- AGENTS.md | 11 +++++++++++ test/array/stencil.jl | 2 ++ test/array/stencil_defs.jl | 2 ++ 3 files changed, 15 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 697e78e18..6fc4f8f17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -936,3 +936,14 @@ lesson. ordering defect. Isolate the cold 4D stencil and inspect all ranks; changing planner specialization alone is not yet a validated fix, and increasing a timeout would not fix the underlying compilation behavior. + +91. **A slow OpenCL stencil may be compiling host halo assembly.** A local + Julia 1.11/PoCL run passes all 53 shared stencil assertions, including 4D, + but takes 283 seconds cold: the first simple kernel takes 86 seconds and + the 4D case another 80. A stack snapshot in 4D showed host LLVM instruction + scheduling below `build_fused_halo`, before the device kernel runs; the + first simple kernel instead showed GPUCompiler inference/inlining. Time + the backend and dimension separately and inspect the active stack before + treating silence in the combined CPU/GPU suite as a device deadlock. Also + reproduce the CI worker topology: a single-process device pass does not + establish that the Distributed CPU half of the same job completes. diff --git a/test/array/stencil.jl b/test/array/stencil.jl index 0ccd24bc2..a573119f6 100644 --- a/test/array/stencil.jl +++ b/test/array/stencil.jl @@ -1,6 +1,7 @@ include(joinpath(@__DIR__, "stencil_defs.jl")) @testset "CPU" begin + @info "Testing stencil backend" backend=:CPU test_stencil() end @@ -9,6 +10,7 @@ end # FIXME kind == :oneAPI && continue @testset "$kind" begin + @info "Testing stencil backend" backend=kind Dagger.with_options(;scope) do # The Metal backend breaks on the 3D/4D stencil tests and # causes subsequent tests to fail, so skip them there. diff --git a/test/array/stencil_defs.jl b/test/array/stencil_defs.jl index 25f434a7d..4d70788fc 100644 --- a/test/array/stencil_defs.jl +++ b/test/array/stencil_defs.jl @@ -528,6 +528,7 @@ function test_stencil(; skip_highdim::Bool=false, skip_4d::Bool=false) for N in 3:4 skip_4d && N == 4 && continue @testset "$(N)D array" begin + @info "Testing higher-dimensional stencil" dimensions=N A = ones(Blocks(ntuple(_->1, N)...), Int, ntuple(_->3, N)...) Dagger.allowscalar() do A[:] = 1:length(A) @@ -536,6 +537,7 @@ function test_stencil(; skip_highdim::Bool=false, skip_4d::Bool=false) @stencil B[idx] = sum(@neighbors(A[idx], 1, Wrap())) ÷ length(A) @test all(==(Float64(sum(1:length(A)) / length(A))), collect(B)) + @info "Finished higher-dimensional stencil" dimensions=N end end end From 3a7870729b8710ff9923a40133d72dc9e2e81ce8 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 12:20:14 -0700 Subject: [PATCH 12/19] Compile each halo region constructor once instead of unrolling its body A 4D halo unrolls 80 regions. Inlining host view construction and kernel-launch bodies into every tuple element gives LLVM a huge function to optimize. Keep the typed tuple but put construction behind a specialized non-inlined per-region barrier. Julia 1.11.9 PoCL: isolated cold 4D stencil 114.75s -> 47.35s; full shared GPU suite 283.31s -> 215.98s, all 53 assertions pass. Four Julia 1.10.12 MPI ranks also pass all 52 remaining assertions each. Allocation profiling: 10 warmups, GC, minimum of 5 gc_num deltas. CPU allocations/bytes unchanged: 2D 11/1536, 3D 29/6176, 4D 86/29264. PoCL unchanged: 2D 533/63712, 3D 1247/150704, 4D 2696/337968. Inferred return types are unchanged for all six fixtures. --- AGENTS.md | 10 ++++++++++ src/array/stencil.jl | 27 ++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6fc4f8f17..a333c6e38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -947,3 +947,13 @@ lesson. treating silence in the combined CPU/GPU suite as a device deadlock. Also reproduce the CI worker topology: a single-process device pass does not establish that the Distributed CPU half of the same job completes. + +92. **Unroll the halo tuple, not each region's construction body.** + `_build_fused_halos` uses `ntuple(Val(NH))` to retain heterogeneous device + view types, but inlining the view/kernel-launch body into every element + duplicates it 80 times in 4D. A specialized, non-inlined per-region helper + reduced an isolated cold PoCL 4D stencil from 115 to 47 seconds. Deep-warmed + CPU and OpenCL fixtures in 2D/3D/4D retained exactly the same allocation + counts, bytes, and inferred return types. Keep the barrier on the host + construction path; erasing halo types or moving dynamic dispatch into the + device sweep would break the reason for the typed tuple in the first place. diff --git a/src/array/stencil.jl b/src/array/stencil.jl index 6c4accbdb..d2ca5ab06 100644 --- a/src/array/stencil.jl +++ b/src/array/stencil.jl @@ -898,18 +898,27 @@ end @inline halo_region_materialized(style::CopyHalos, neigh_dist, boundary, region_code, is_boundary, boundary_dims, chunk) = halo_region(style, neigh_dist, boundary, region_code, is_boundary, boundary_dims, chunk) +# Keep host-side view construction and kernel launches out of the unrolled +# tuple builder: 4D has 80 regions, and inlining their bodies makes LLVM spend +# minutes compiling one halo assembly. This barrier still specializes on the +# region's types; it does not erase the types used by the device kernel. +@noinline function _build_fused_halo_region(style, neigh_dist, boundary, metadata, neighbor) + region_code, is_boundary, boundary_dims, is_prematerialized = metadata + if is_prematerialized + # `stencil_region_info` already built this region in its owner's memory + # space; `neighbor` is the region itself, not a chunk to slice. + return neighbor + else + return halo_region(style, neigh_dist, boundary, region_code, is_boundary, + boundary_dims, neighbor) + end +end + @inline function _build_fused_halos(style, neigh_dist, boundary, region_metadata, neighbor_chunks::NTuple{NH,Any}) where NH return ntuple(Val(NH)) do i - region_code, is_boundary, boundary_dims, is_prematerialized = region_metadata[i] - if is_prematerialized - # `stencil_region_info` already built this region in its owner's memory - # space; `neighbor_chunks[i]` is the region itself, not a chunk to slice. - neighbor_chunks[i] - else - halo_region(style, neigh_dist, boundary, region_code, is_boundary, - boundary_dims, neighbor_chunks[i]) - end + _build_fused_halo_region(style, neigh_dist, boundary, + region_metadata[i], neighbor_chunks[i]) end end From 05d23c2b523b8ef9759c30f0f2faff542b983589 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 12:29:49 -0700 Subject: [PATCH 13/19] Keep extension processor ownership and RPCs on the selected transport With DistributedNext selected, direct stdlib Distributed imports report myid() == 1 on every worker. OpenCL workers 2 and 3 consequently registered GPU owner 1. Import the existing functions from Dagger in GPU and Python extensions so registration and transfers use the same registry as the scheduler. Validation: Julia 1.11.9, DistributedNext, two two-thread workers: OpenCL ownership and distributed GPU dataflow 51 assertions pass; Python ownership 6 and Python numerical integration 6 assertions pass. The OpenCL probe logged a late transport socket error during teardown after numerical checks passed; this change addresses ownership and RPC selection. --- AGENTS.md | 8 ++++++++ ext/CUDAExt.jl | 2 +- ext/IntelExt.jl | 2 +- ext/MetalExt.jl | 2 +- ext/OpenCLExt.jl | 2 +- ext/PythonExt.jl | 2 +- ext/ROCExt.jl | 2 +- 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a333c6e38..8bb802f2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -957,3 +957,11 @@ lesson. counts, bytes, and inferred return types. Keep the barrier on the host construction path; erasing halo types or moving dynamic dispatch into the device sweep would break the reason for the typed tuple in the first place. + +93. **Extensions must use Dagger's selected transport too.** Importing + `Distributed.myid` or `remotecall_fetch` directly in an extension bypasses + the transport preference. Under DistributedNext, stdlib Distributed has + no workers, so OpenCL workers 2 and 3 both reported stdlib ID 1 and + registered their GPU processors with owner 1. Import these functions from + Dagger, which already selected the transport. The same rule applies to + Python processor registration and to every GPU extension's transfer RPCs. diff --git a/ext/CUDAExt.jl b/ext/CUDAExt.jl index 0fbfd76b7..b20089781 100644 --- a/ext/CUDAExt.jl +++ b/ext/CUDAExt.jl @@ -5,7 +5,7 @@ export CuArrayDeviceProc import Dagger, MemPool import Dagger: CPURAMMemorySpace, Chunk, unwrap, CuArrayDeviceProc import MemPool: DRef, poolget -import Distributed: myid, remotecall_fetch +import Dagger: myid, remotecall_fetch import LinearAlgebra using KernelAbstractions, Adapt diff --git a/ext/IntelExt.jl b/ext/IntelExt.jl index 1c207eaee..8835442c4 100644 --- a/ext/IntelExt.jl +++ b/ext/IntelExt.jl @@ -5,7 +5,7 @@ export oneArrayDeviceProc import Dagger, MemPool import Dagger: CPURAMMemorySpace, Chunk, unwrap, oneArrayDeviceProc import MemPool: DRef, poolget -import Distributed: myid, remotecall_fetch +import Dagger: myid, remotecall_fetch import LinearAlgebra using KernelAbstractions, Adapt diff --git a/ext/MetalExt.jl b/ext/MetalExt.jl index bcd3cc623..3c8e3bfbf 100644 --- a/ext/MetalExt.jl +++ b/ext/MetalExt.jl @@ -5,7 +5,7 @@ export MtlArrayDeviceProc import Dagger, MemPool import Dagger: CPURAMMemorySpace, Chunk, unwrap, MtlArrayDeviceProc import MemPool: DRef, poolget -import Distributed: myid, remotecall_fetch +import Dagger: myid, remotecall_fetch import LinearAlgebra using KernelAbstractions, Adapt diff --git a/ext/OpenCLExt.jl b/ext/OpenCLExt.jl index 74cbb1479..cc0cecaac 100644 --- a/ext/OpenCLExt.jl +++ b/ext/OpenCLExt.jl @@ -5,7 +5,7 @@ export CLArrayDeviceProc import Dagger, MemPool import Dagger: CPURAMMemorySpace, Chunk, unwrap, CLArrayDeviceProc import MemPool: DRef, poolget -import Distributed: myid, remotecall_fetch +import Dagger: myid, remotecall_fetch import LinearAlgebra using KernelAbstractions, Adapt diff --git a/ext/PythonExt.jl b/ext/PythonExt.jl index f6a7113ed..9783c9f48 100644 --- a/ext/PythonExt.jl +++ b/ext/PythonExt.jl @@ -8,7 +8,7 @@ end import Dagger import Dagger: Processor, OSProc, ThreadProc, Chunk -import Distributed: myid +import Dagger: myid const CPUProc = Union{OSProc, ThreadProc} diff --git a/ext/ROCExt.jl b/ext/ROCExt.jl index 805613702..a55ecc2dd 100644 --- a/ext/ROCExt.jl +++ b/ext/ROCExt.jl @@ -5,7 +5,7 @@ export ROCArrayDeviceProc import Dagger, MemPool import Dagger: CPURAMMemorySpace, Chunk, unwrap, ROCArrayDeviceProc import MemPool: DRef, poolget -import Distributed: myid, remotecall_fetch +import Dagger: myid, remotecall_fetch import LinearAlgebra using KernelAbstractions, Adapt From 1dad3071659903ee96d40a91a22912bb74f09428 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 13:02:42 -0700 Subject: [PATCH 14/19] Keep stencil specialization independent of worker placement Local DTask vs remote Chunk neighbors (including Chunk processor/scope parameters) specialized each submission on placement. Wrap neighbor metadata in concrete Read{Any}; execution still specializes on actual arrays. Normalize received CPU Wrap regions to full-range views so local and remote regions also share one concrete halo tuple/kernel signature. The original Julia 1.11 three-worker CPU 4D case took 720 seconds; the final DistributedNext run took 49 seconds. Both CPU and PoCL shared suites pass (126 assertions, 344 seconds total). Four Julia 1.10 MPI ranks each pass all 62 assertions through 3D. All 24 allocation bounds pass unchanged. A late DistributedNext unknown-socket teardown diagnostic remains separate from the passing numerical suites. After 10 warmups and GC, minimum of five Base.gc_num deltas on Julia 1.12: 2D: 7,321 allocations / 374,880 bytes -> 7,104 / 362,320. 4D: 217,684 allocations / 15,401,096 bytes -> 202,918 / 13,964,416. A mixed local/received halo fixture also becomes concretely inferred and checks value correctness, homogeneous representation, and retained aliasing. --- AGENTS.md | 14 ++++++++++++++ src/array/stencil.jl | 19 +++++++++++++++++-- test/array/stencil_defs.jl | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8bb802f2a..bb8bb3292 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -965,3 +965,17 @@ lesson. registered their GPU processors with owner 1. Import these functions from Dagger, which already selected the transport. The same rule applies to Python processor registration and to every GPU extension's transfer RPCs. + +94. **Stencil placement must not become a kernel or planner type parameter.** + Local halo dependencies are `DTask`s while remote ones are `Chunk`s whose + parameters also encode processor/scope provenance. Wrapping each neighbor + in `Read{Any}` keeps the dependency wrapper concrete without specializing + submission on every placement pattern; execution still specializes on the + actual arrays. Separately, local CPU Wrap halos are views but received + regions are Arrays. Wrap received Arrays in full-range views too, or every + local/remote mask creates another heterogeneous 80-region tuple and unrolled + `halo_load` kernel in 4D. These are independent sources of compilation + growth. On Julia 1.11 with three workers, CPU 4D fell from 720 to 49 seconds; + warmed 4D allocations also fell from 217,684 / 15,401,096 bytes to + 202,918 / 13,964,416. Check both dependency metadata types and unwrapped + value types before adding a dynamic-dispatch threshold to submission. diff --git a/src/array/stencil.jl b/src/array/stencil.jl index d2ca5ab06..f3a5ce0c3 100644 --- a/src/array/stencil.jl +++ b/src/array/stencil.jl @@ -898,6 +898,15 @@ end @inline halo_region_materialized(style::CopyHalos, neigh_dist, boundary, region_code, is_boundary, boundary_dims, chunk) = halo_region(style, neigh_dist, boundary, region_code, is_boundary, boundary_dims, chunk) +@inline prematerialized_halo_region(style, region) = region +# Local Wrap halos are SubArrays. Give received CPU regions the same view +# representation, rather than generating a distinct heterogeneous halo tuple +# (and a new unrolled halo_load kernel) for every cross-worker placement. +@inline function prematerialized_halo_region(::ViewHalos, region::Array{T,N}) where {T,N} + ranges = ntuple(i -> firstindex(region, i):lastindex(region, i), Val(N)) + return view(region, ranges...) +end + # Keep host-side view construction and kernel launches out of the unrolled # tuple builder: 4D has 80 regions, and inlining their bodies makes LLVM spend # minutes compiling one halo assembly. This barrier still specializes on the @@ -907,7 +916,7 @@ end if is_prematerialized # `stencil_region_info` already built this region in its owner's memory # space; `neighbor` is the region itself, not a chunk to slice. - return neighbor + return prematerialized_halo_region(style, neighbor) else return halo_region(style, neigh_dist, boundary, region_code, is_boundary, boundary_dims, neighbor) @@ -1389,7 +1398,13 @@ macro stencil(orig_ex) $nneighbors = length($neighbor_cks) end) push!(arg_exs, :($Read($(syms.src_chunks)[$chunk_idx]))) - push!(arg_exs, Expr(:..., :(map($Read, $neighbor_cks)))) + # A neighbor may be a local DTask or a remotely extracted Chunk; + # Chunk types also encode processor/scope provenance. Keep that + # runtime metadata out of the submission signature, or every + # placement pattern specializes the whole planner again. The + # wrapper still records a read dependency, and task execution + # specializes on the actual array values after unwrapping. + push!(arg_exs, Expr(:..., :(map($(Read{Any}), $neighbor_cks)))) push!(unpack_exs, quote $local_var = $build_fused_halo($neigh_dist, $boundary, $region_meta, $task_args[$arg_offset + 1], diff --git a/test/array/stencil_defs.jl b/test/array/stencil_defs.jl index 4d70788fc..804baa728 100644 --- a/test/array/stencil_defs.jl +++ b/test/array/stencil_defs.jl @@ -5,6 +5,30 @@ @everywhere import Dagger: @stencil, Wrap, Pad, Reflect, AntiReflect, Clamp, LinearExtrapolate function test_stencil(; skip_highdim::Bool=false, skip_4d::Bool=false) + @testset "CPU Wrap halo views" begin + center = zeros(Int, 2, 2) + codes = Dagger.all_region_codes(Val(2)) + metadata = ntuple(length(codes)) do i + (codes[i], false, (false, false), iseven(i)) + end + neighbors = ntuple(length(codes)) do i + dims = iseven(i) ? Dagger.halo_region_size((2, 2), (1, 1), codes[i]) : (2, 2) + fill(i, dims) + end + halo = Dagger.build_fused_halo(1, Wrap(), metadata, center, neighbors...) + # Received regions and local views must share a representation, so + # placement does not create a new heterogeneous kernel signature. + @test allequal(fieldtypes(typeof(halo.halos))) + for (i, code) in enumerate(codes) + idx = CartesianIndex(ntuple(d -> code[d] == 0 ? 1 : code[d] < 0 ? 0 : 3, 2)) + @test halo[idx] == i + end + neighbors[2] .= 99 + code = codes[2] + idx = CartesianIndex(ntuple(d -> code[d] == 0 ? 1 : code[d] < 0 ? 0 : 3, 2)) + @test halo[idx] == 99 + end + @testset "Simple assignment" begin A = zeros(Blocks(2, 2), Int, 4, 4) @stencil A[idx] = 1 From 347cd492f9e61a7181f4bcf56003c4863e8d886b Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 13:02:49 -0700 Subject: [PATCH 15/19] Use the supported thread-safe transport for Julia 1.11 OpenCL CI GPU extensions now honor the selected transport. The three-worker CPU and PoCL stencil suites pass 126 assertions with DistributedNext, and the separate functional GPU probe passes 51 assertions. Keep the Julia 1.11 OpenCL job consistent with the CPU job transport choice; newer Julia jobs retain stdlib Distributed coverage. --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2052a57d0..468b44412 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -132,6 +132,7 @@ jobs: env: JULIA_NUM_THREADS: '1' CI_USE_OPENCL: '1' + CI_DISTRIBUTED_PACKAGE: DistributedNext steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 From fcef471b96394e905036343e9ef4a4656183bb47 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Thu, 15 Oct 2026 13:20:06 -0700 Subject: [PATCH 16/19] Record the Apple Silicon reproduction of placement-driven compilation --- AGENTS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index bb8bb3292..de5dae848 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -979,3 +979,10 @@ lesson. warmed 4D allocations also fell from 217,684 / 15,401,096 bytes to 202,918 / 13,964,416. Check both dependency metadata types and unwrapped value types before adding a dynamic-dispatch threshold to submission. + Apple M2/macOS 26.7 with Julia 1.13.2 reproduces the same problem with + CI's coverage flag and three two-thread workers: 4D fell from 200 to 31 + seconds, and the full CPU stencil suite from 310 to 117 seconds despite + ten additional regression assertions. Native samples of the original + driver showed LLVM dead-store elimination and alias analysis, rather than + a blocked transport. `addprocs` inherits the coverage flag, so retain it + when reproducing the CI compilation workload. From aaf70db0d2bd86da4f61a8b9639fabaebb71a17b Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Fri, 16 Oct 2026 11:56:22 -0700 Subject: [PATCH 17/19] Declare the worker transport in the active test environment DistributedNext starts workers with using DistributedNext in Main. Its presence as a transitive Dagger dependency is insufficient there, so direct runtests.jl invocations failed before any tests ran. Add it to test/Project.toml. With global environments hidden, removing this declaration reproduces the exact GHA startup error; the actual processor and preference test bodies pass 32 assertions each on Julia 1.10 and 1.11 with three two-thread workers. --- AGENTS.md | 9 +++++++++ test/Project.toml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index de5dae848..b67d442e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -986,3 +986,12 @@ lesson. driver showed LLVM dead-store elimination and alias analysis, rather than a blocked transport. `addprocs` inherits the coverage flag, so retain it when reproducing the CI compilation workload. + +95. **The test project must directly declare its worker transport.** + DistributedNext launches each worker with `using DistributedNext` in Main. + Dagger can import a transitive dependency from its own module, but that + does not make the package visible to Main in the active test project. + Declaring it only in the root Project.toml therefore lets the driver load + and every worker fail before printing its host/port. Validate with a copy + of test/Project.toml and `JULIA_LOAD_PATH=@:@stdlib`; an ad-hoc environment + that explicitly adds DistributedNext hides this defect. diff --git a/test/Project.toml b/test/Project.toml index b17cb51ef..da151f31b 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -9,6 +9,9 @@ ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +# Worker startup imports the selected transport in Main, so a transitive +# dependency through Dagger is insufficient for DistributedNext.addprocs. +DistributedNext = "fab6aee4-877b-4bac-a744-3eca44acbb6f" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" From 41a0d852e8e9e932fa738e736de7b8881fa571c8 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Fri, 16 Oct 2026 11:56:41 -0700 Subject: [PATCH 18/19] Compare log retention ages without unsigned cutoff underflow A fresh GHA runner's time_ns counter was only 104 seconds old when the regression fixture subtracted 120 seconds. The wrapped timestamp looked newer than every real event. An absolute cutoff also underflows whenever uptime is shorter than the retention window. Compare elapsed UInt64 ages and use the retained-prefix count to return when nothing expired. The full TimespanLogging suite passes all 79 assertions on Julia 1.13, including a deterministic wrap fixture that the previous method retains incorrectly. After 10 warmups and GC, minimum of five 1,000-call gc_num deltas on Julia 1.12.4: 9,000 allocations / 528,000 bytes before versus 6,000 / 496,000 after (9 -> 6 allocations per retained-event update). --- AGENTS.md | 9 +++++++++ lib/TimespanLogging/src/extras.jl | 9 ++++++--- lib/TimespanLogging/test/runtests.jl | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b67d442e7..326f114aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -995,3 +995,12 @@ lesson. and every worker fail before printing its host/port. Validate with a copy of test/Project.toml and `JULIA_LOAD_PATH=@:@stdlib`; an ad-hoc environment that explicitly adds DistributedNext hides this defect. + +96. **Log windows need elapsed ages, not an absolute unsigned cutoff.** + `time_ns()` and event timestamps are UInt64 counters. On a freshly booted + runner, subtracting a long window can underflow; constructing an older + fixture can wrap too (GHA's counter was about 104 seconds when the fixture + subtracted 120 seconds). Numeric timestamp ordering then retains expired + records or drops recent ones. Compare `now - timestamp <= window_length` + instead, and test a timestamp across UInt64 wrap explicitly so validation + does not depend on the developer machine's uptime. diff --git a/lib/TimespanLogging/src/extras.jl b/lib/TimespanLogging/src/extras.jl index c46cb0c43..05e9d4e5d 100644 --- a/lib/TimespanLogging/src/extras.jl +++ b/lib/TimespanLogging/src/extras.jl @@ -123,12 +123,15 @@ function (lw::LogWindow)(logs::Dict) end # Find entries outside the window - window_start = time_ns() - lw.window_length - idx = something(findfirst(ev->ev.timestamp >= window_start, core_log), + # Compare elapsed durations: both timestamps and time_ns() are wrapping + # UInt64 counters. An absolute cutoff underflows shortly after boot when + # the window is longer than the counter's current value. + now = time_ns() + idx = something(findfirst(ev->now - ev.timestamp <= lw.window_length, core_log), length(core_log) + 1) - 1 # Return if all events are in the window - if length(core_log) == 0 || (core_log[1].timestamp >= window_start) + if idx == 0 return end diff --git a/lib/TimespanLogging/test/runtests.jl b/lib/TimespanLogging/test/runtests.jl index 307a0ebfd..1aea4f2d4 100644 --- a/lib/TimespanLogging/test/runtests.jl +++ b/lib/TimespanLogging/test/runtests.jl @@ -57,6 +57,14 @@ end lw(expired) @test isempty(expired[:core]) @test isempty(expired[:ids]) + + # Force the old timestamp across UInt64 wrap regardless of machine + # uptime. Numeric timestamp ordering disagrees with elapsed age here. + wrapped = Dict(:core => [(timestamp=typemax(UInt64) - window,), + (timestamp=time_ns(),)], :ids => [1, 2]) + lw(wrapped) + @test length(wrapped[:core]) == 1 + @test wrapped[:ids] == [2] end @testset "Legacy Symbol API + LocalEventLog" begin From 428a54f04ada68fe4def1b48f6638d9f728a9d03 Mon Sep 17 00:00:00 2001 From: Julian P Samaroo Date: Fri, 16 Oct 2026 12:00:10 -0700 Subject: [PATCH 19/19] Resolve sparse domain indexing against Julia 1.14's CSC fallback SparseArrays now has a vararg getindex(::AbstractSparseMatrixCSC, I...) fallback. Dagger's AbstractArray x ArrayDomain method constrains the other argument, making distribute(sparse_matrix, Blocks(...)) ambiguous. Add the intersection method in the sparse extension and retain native CSC slicing. Current Julia 1.14.0-DEV.3754 passes 17 domain assertions and all 981 array allocation assertions, including the sparse distribute case that failed in both nightly GHA jobs. The new test checks inferred sparse results, values, and empty-domain behavior. With 10 warmups plus GC and minimum of five 1,000-call gc_num deltas on Julia 1.12.4, the old generic indexing body and the new specialization both use 6,000 allocations / 1,504,000 bytes. --- AGENTS.md | 10 ++++++++++ ext/SparseArraysExt.jl | 5 +++++ test/domain.jl | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 326f114aa..a2d5a639a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1004,3 +1004,13 @@ lesson. records or drops recent ones. Compare `now - timestamp <= window_length` instead, and test a timestamp across UInt64 wrap explicitly so validation does not depend on the developer machine's uptime. + +97. **A sparse indexing fallback can make ArrayDomain indexing ambiguous.** + Julia 1.14's SparseArrays adds `getindex(::AbstractSparseMatrixCSC, I...)`. + It is more specific in the array argument while Dagger's + `getindex(::AbstractArray, ::ArrayDomain)` is more specific in the index, + so ordinary `distribute(::SparseMatrixCSC, ::Blocks)` fails while slicing + its first tile. Put the intersection method in SparseArraysExt and expand + the domain's indices there. Test both direct sparse-domain slicing and the + existing allocation/distribute suite on the current nightly; an older + nightly without the fallback does not exercise the conflict. diff --git a/ext/SparseArraysExt.jl b/ext/SparseArraysExt.jl index 84dc5c9d1..c3136653d 100644 --- a/ext/SparseArraysExt.jl +++ b/ext/SparseArraysExt.jl @@ -6,6 +6,11 @@ import LinearAlgebra import Dagger import Dagger: Blocks, AutoBlocks, BlocksOrAuto, AssignmentType, DSparseArray, DSparseMatrix +# SparseArrays' vararg CSC indexing and Dagger's AbstractArray domain indexing +# each constrain a different argument. Resolve their intersection explicitly. +Base.getindex(A::SparseArrays.AbstractSparseMatrixCSC, d::Dagger.ArrayDomain) = + A[Dagger.indexes(d)...] + # Keep tiles sparse through `collect`/`cat`; the outer `collect` densifies. Dagger._sparse_collect(M::SparseMatrixCSC) = copy(M) diff --git a/test/domain.jl b/test/domain.jl index dc8b90831..9598136ba 100644 --- a/test/domain.jl +++ b/test/domain.jl @@ -12,6 +12,14 @@ end @test size(d1) == (10, 10,) @test length(d1) == 100 + @testset "Sparse CSC indexing" begin + A = sparse([1, 3, 4], [2, 1, 5], [2.0, 3.0, 4.0], 4, 5) + tile = @inferred A[ArrayDomain(2:4, 1:3)] + @test tile isa SparseMatrixCSC + @test tile == A[2:4, 1:3] + @test A[ArrayDomain(1:0, 1:3)] == spzeros(0, 3) + end + d2 = ArrayDomain(5:10, 5:10) d3 = ArrayDomain((8:12, 1:8)) @testset "Domain methods for ArrayDomain" begin