diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 919e9d7..aaa188a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -142,7 +142,12 @@ jobs: OMJL_COVERAGE_TIMEOUT: ${{ inputs.timeout || '600' }} # One thread per worker: OMLibraryTesting runs no parallel workloads. JULIA_NUM_THREADS: "1" - run: julia --color=yes --project=. scripts/run_full_coverage.jl + # A model whose worker passes its share of 11 GB fails with its memory (the limit is + # per worker: OMJL_COVERAGE_MEMORY_LIMIT_GB); past the runner's 16 GB the runner shut + # down and the shard's results were lost (MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24). + run: | + export OMJL_COVERAGE_MEMORY_LIMIT_GB=$(awk -v w="$OMJL_COVERAGE_WORKERS" 'BEGIN { printf "%.1f", 11 / w }') + julia --color=yes --project=. scripts/run_full_coverage.jl - name: Upload the shard's results if: ${{ !cancelled() }} diff --git a/scripts/run_full_coverage.jl b/scripts/run_full_coverage.jl index 72d39b8..cfc1bf5 100644 --- a/scripts/run_full_coverage.jl +++ b/scripts/run_full_coverage.jl @@ -20,6 +20,9 @@ Environment variables: model goes through every phase (what really stops each one; as docs/COVERAGE.md). OMJL_COVERAGE_SHARD "k/n": only the k-th of every n models (run_coverage's `shard`), for a run split over n jobs; scripts/merge_coverage.jl joins their results (default "1/1"). + OMJL_COVERAGE_MEMORY_LIMIT_GB A worker over this many GB while it runs a model is killed and the model + fails with its memory (run_on_worker; unset or 0: no limit). + OMJL_COVERAGE_RECYCLE_GB A worker over this many GB is restarted before its next model (unset or 0: never). =# using Dates diff --git a/src/runner.jl b/src/runner.jl index 98943e1..e2f2fb0 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -550,6 +550,10 @@ end Run all phases for a model on the worker process with an escalating timeout: 1. After `timeout` seconds: send SIGINT via `interrupt(pid)` 2. After `timeout + grace_period` seconds: kill the worker (`kill_worker!`: SIGKILL, then `rmprocs`) + +and a memory limit (`memoryLimitGB`): a worker over it while the model runs is killed, and the +model fails with its memory, instead of the machine running out (a CI runner's 16 GB: the runner +shut down and its shard's results were lost, MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24). """ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, phases_to_run::Vector{Phase}; @@ -595,6 +599,9 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, t0 = time() interrupted = Ref(false) killed = Ref(false) + #= the worker's memory when the limit killed it; 0 while it did not =# + over_memory = Ref(0.0) + memory_limit = memoryLimitGB() _future_pending() = !istaskdone(fetcher) @@ -618,8 +625,20 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, # Poll instead of blocking on fetch so Ctrl-C can be delivered between iterations. # Stop waiting once the worker is killed: kill_worker! only kills workers this # module spawned, so the fetcher of an adopted worker may never finish. + # The worker's memory is read every fourth poll (2 s). + polls = 0 while _future_pending() && !killed[] sleep(0.5) + polls += 1 + if memory_limit > 0 && polls % 4 == 0 && mgr.os_pid !== nothing + gb = workerMemoryGB(mgr.os_pid) + if gb > memory_limit && _future_pending() + over_memory[] = gb + killed[] = true + @warn " $(spec.name): worker memory $(round(gb, digits=1)) GB, over $(memory_limit) GB, killing..." + try; kill_worker!(mgr); catch; end + end + end end raw = killed[] && _future_pending() ? ProcessExitedException(pid) : fetch(fetcher) raw isa Exception && throw(raw) @@ -650,6 +669,16 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec, @warn " $(spec.name): INTERRUPTED by user" kill_worker!(mgr) rethrow() + elseif over_memory[] > 0 + msg = "Memory $(round(over_memory[], digits=1)) GB after $(round(elapsed, digits=1))s " * + "(limit: $(memory_limit) GB) [worker killed]" + mgr.pid = nothing + mgr.ready = false + @warn " $(spec.name): MEMORY: $msg" + phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1] + phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)] + return ModelResult(spec, phases, BROKEN, + Dates.format(Dates.now(), "yyyy-mm-dd HH:MM")) elseif interrupted[] || killed[] msg = "Timeout after $(round(elapsed, digits=1))s (limit: $(round(timeout, digits=0))s)" if killed[] @@ -743,6 +772,14 @@ function recycleLargeWorker!(mgr::WorkerManager)::Nothing return nothing end +""" + memoryLimitGB() -> Float64 + +The memory in gigabytes (`workerMemoryGB`) a worker may reach while it runs a model, +`OMJL_COVERAGE_MEMORY_LIMIT_GB` (unset or 0: no limit); over it `run_on_worker` kills the worker. +""" +memoryLimitGB()::Float64 = max(something(tryparse(Float64, get(ENV, "OMJL_COVERAGE_MEMORY_LIMIT_GB", "0")), 0.0), 0.0) + """ workerMemoryGB(os_pid) -> Float64 diff --git a/test/runnerTests.jl b/test/runnerTests.jl index 6c42486..1b40164 100644 --- a/test/runnerTests.jl +++ b/test/runnerTests.jl @@ -52,6 +52,42 @@ end end end +#= A worker over OMJL_COVERAGE_MEMORY_LIMIT_GB while it runs a model must be killed and the model + fail with its memory: past a CI runner's 16 GB the runner shut down and its shard's results were + lost (MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24). Here the worker's _run_model_phases fills 10 MB + buffers until 4 GB, under a limit 0.5 GB above the worker's memory at the start. =# +@testset "run_on_worker kills a worker over the memory limit" begin + spec = OLT.ModelSpec("Large.Model", "Large_Model", "Large", 1.0, OLT.SIMULATE, + Dict{String, Float64}(), 0.0, 0.0, "", Dict{String, String}(), "", + Set{OLT.Phase}()) + mgr = OLT.WorkerManager() + try + pid = OLT.ensure_worker!(mgr) + Distributed.remotecall_eval(OLT, pid, quote + foreach(Base.delete_method, collect(methods(_run_model_phases))) + _run_model_phases(args...) = let buffers = Vector{UInt8}[] + for _ in 1:400 + push!(buffers, ones(UInt8, 10^7)) + sleep(0.05) + end + (Tuple{Int, Bool, Float64, Union{Nothing, String}}[], 0) + end + end) + limit = round(OLT.workerMemoryGB(mgr.os_pid) + 0.5; digits = 2) + elapsed = withenv("OMJL_COVERAGE_MEMORY_LIMIT_GB" => string(limit)) do + @elapsed global memoryResult = OLT.run_on_worker(mgr, spec, [OLT.SIMULATE]; timeout = 120.0) + end + @test memoryResult.highest == OLT.BROKEN + @test startswith(something(memoryResult.phases[1].error, ""), "Memory") + @test endswith(memoryResult.phases[1].error, "[worker killed]") + @test elapsed < 20.0 + @test mgr.pid === nothing + @test withenv(OLT.memoryLimitGB, "OMJL_COVERAGE_MEMORY_LIMIT_GB" => nothing) == 0.0 # unset: no limit + finally + OLT.cleanup!(mgr) + end +end + #= A worker that cannot load the packages (a precompile of changed sources failed) must fail its model, not the run: the error left run_coverage, and every result of the run was lost. Here the worker starts in an empty project with a load path of only that project and the