Coverage: a worker over OMJL_COVERAGE_MEMORY_LIMIT_GB is killed, its model fails

.github/workflows/coverage.yml

@@ -142,7 +142,12 @@ jobs:
           OMJL_COVERAGE_TIMEOUT: ${{ inputs.timeout || '600' }}
           # One thread per worker: OMLibraryTesting runs no parallel workloads.
           JULIA_NUM_THREADS: "1"
-        run: julia --color=yes --project=. scripts/run_full_coverage.jl
+        # A model whose worker passes its share of 11 GB fails with its memory (the limit is
+        # per worker: OMJL_COVERAGE_MEMORY_LIMIT_GB); past the runner's 16 GB the runner shut
+        # down and the shard's results were lost (MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24).
+        run: |
+          export OMJL_COVERAGE_MEMORY_LIMIT_GB=$(awk -v w="$OMJL_COVERAGE_WORKERS" 'BEGIN { printf "%.1f", 11 / w }')
+          julia --color=yes --project=. scripts/run_full_coverage.jl
 
       - name: Upload the shard's results
         if: ${{ !cancelled() }}

scripts/run_full_coverage.jl

@@ -20,6 +20,9 @@ Environment variables:
                             model goes through every phase (what really stops each one; as docs/COVERAGE.md).
     OMJL_COVERAGE_SHARD     "k/n": only the k-th of every n models (run_coverage's `shard`), for a run split
                             over n jobs; scripts/merge_coverage.jl joins their results (default "1/1").
+    OMJL_COVERAGE_MEMORY_LIMIT_GB  A worker over this many GB while it runs a model is killed and the model
+                            fails with its memory (run_on_worker; unset or 0: no limit).
+    OMJL_COVERAGE_RECYCLE_GB  A worker over this many GB is restarted before its next model (unset or 0: never).
 =#
 
 using Dates

src/runner.jl

@@ -550,6 +550,10 @@ end
 Run all phases for a model on the worker process with an escalating timeout:
 1. After `timeout` seconds: send SIGINT via `interrupt(pid)`
 2. After `timeout + grace_period` seconds: kill the worker (`kill_worker!`: SIGKILL, then `rmprocs`)
+
+and a memory limit (`memoryLimitGB`): a worker over it while the model runs is killed, and the
+model fails with its memory, instead of the machine running out (a CI runner's 16 GB: the runner
+shut down and its shard's results were lost, MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24).
 """
 function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
                        phases_to_run::Vector{Phase};
@@ -595,6 +599,9 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
     t0 = time()
     interrupted = Ref(false)
     killed = Ref(false)
+    #= the worker's memory when the limit killed it; 0 while it did not =#
+    over_memory = Ref(0.0)
+    memory_limit = memoryLimitGB()
 
     _future_pending() = !istaskdone(fetcher)
 
@@ -618,8 +625,20 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
         # Poll instead of blocking on fetch so Ctrl-C can be delivered between iterations.
         # Stop waiting once the worker is killed: kill_worker! only kills workers this
         # module spawned, so the fetcher of an adopted worker may never finish.
+        # The worker's memory is read every fourth poll (2 s).
+        polls = 0
         while _future_pending() && !killed[]
             sleep(0.5)
+            polls += 1
+            if memory_limit > 0 && polls % 4 == 0 && mgr.os_pid !== nothing
+                gb = workerMemoryGB(mgr.os_pid)
+                if gb > memory_limit && _future_pending()
+                    over_memory[] = gb
+                    killed[] = true
+                    @warn "  $(spec.name): worker memory $(round(gb, digits=1)) GB, over $(memory_limit) GB, killing..."
+                    try; kill_worker!(mgr); catch; end
+                end
+            end
         end
         raw = killed[] && _future_pending() ? ProcessExitedException(pid) : fetch(fetcher)
         raw isa Exception && throw(raw)
@@ -650,6 +669,16 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
             @warn "    $(spec.name): INTERRUPTED by user"
             kill_worker!(mgr)
             rethrow()
+        elseif over_memory[] > 0
+            msg = "Memory $(round(over_memory[], digits=1)) GB after $(round(elapsed, digits=1))s " *
+                  "(limit: $(memory_limit) GB) [worker killed]"
+            mgr.pid = nothing
+            mgr.ready = false
+            @warn "    $(spec.name): MEMORY: $msg"
+            phase = isempty(phases_to_run) ? BROKEN : phases_to_run[1]
+            phases = PhaseResult[PhaseResult(phase, false, elapsed, msg)]
+            return ModelResult(spec, phases, BROKEN,
+                              Dates.format(Dates.now(), "yyyy-mm-dd HH:MM"))
         elseif interrupted[] || killed[]
             msg = "Timeout after $(round(elapsed, digits=1))s (limit: $(round(timeout, digits=0))s)"
             if killed[]
@@ -743,6 +772,14 @@ function recycleLargeWorker!(mgr::WorkerManager)::Nothing
     return nothing
 end
 
+"""
+    memoryLimitGB() -> Float64
+
+The memory in gigabytes (`workerMemoryGB`) a worker may reach while it runs a model,
+`OMJL_COVERAGE_MEMORY_LIMIT_GB` (unset or 0: no limit); over it `run_on_worker` kills the worker.
+"""
+memoryLimitGB()::Float64 = max(something(tryparse(Float64, get(ENV, "OMJL_COVERAGE_MEMORY_LIMIT_GB", "0")), 0.0), 0.0)
+
 """
     workerMemoryGB(os_pid) -> Float64
 

test/runnerTests.jl

@@ -52,6 +52,42 @@ end
     end
 end
 
+#= A worker over OMJL_COVERAGE_MEMORY_LIMIT_GB while it runs a model must be killed and the model
+   fail with its memory: past a CI runner's 16 GB the runner shut down and its shard's results were
+   lost (MSL 3.2.3 Rectifier6pulseFFT, 2026-10-24). Here the worker's _run_model_phases fills 10 MB
+   buffers until 4 GB, under a limit 0.5 GB above the worker's memory at the start. =#
+@testset "run_on_worker kills a worker over the memory limit" begin
+    spec = OLT.ModelSpec("Large.Model", "Large_Model", "Large", 1.0, OLT.SIMULATE,
+                         Dict{String, Float64}(), 0.0, 0.0, "", Dict{String, String}(), "",
+                         Set{OLT.Phase}())
+    mgr = OLT.WorkerManager()
+    try
+        pid = OLT.ensure_worker!(mgr)
+        Distributed.remotecall_eval(OLT, pid, quote
+            foreach(Base.delete_method, collect(methods(_run_model_phases)))
+            _run_model_phases(args...) = let buffers = Vector{UInt8}[]
+                for _ in 1:400
+                    push!(buffers, ones(UInt8, 10^7))
+                    sleep(0.05)
+                end
+                (Tuple{Int, Bool, Float64, Union{Nothing, String}}[], 0)
+            end
+        end)
+        limit = round(OLT.workerMemoryGB(mgr.os_pid) + 0.5; digits = 2)
+        elapsed = withenv("OMJL_COVERAGE_MEMORY_LIMIT_GB" => string(limit)) do
+            @elapsed global memoryResult = OLT.run_on_worker(mgr, spec, [OLT.SIMULATE]; timeout = 120.0)
+        end
+        @test memoryResult.highest == OLT.BROKEN
+        @test startswith(something(memoryResult.phases[1].error, ""), "Memory")
+        @test endswith(memoryResult.phases[1].error, "[worker killed]")
+        @test elapsed < 20.0
+        @test mgr.pid === nothing
+        @test withenv(OLT.memoryLimitGB, "OMJL_COVERAGE_MEMORY_LIMIT_GB" => nothing) == 0.0   # unset: no limit
+    finally
+        OLT.cleanup!(mgr)
+    end
+end
+
 #= A worker that cannot load the packages (a precompile of changed sources failed) must fail
    its model, not the run: the error left run_coverage, and every result of the run was lost.
    Here the worker starts in an empty project with a load path of only that project and the