From 0e8ffdc72fce157ef5dbd72134b6171367bee6ce Mon Sep 17 00:00:00 2001
From: Daniel Han <23090290+danielhanchen@users.noreply.github.com>
Date: Thu, 8 Oct 2026 03:49:33 -0700
Subject: [PATCH] ggml-cuda: drop the CUDA graph cache count cap, as upstream
 does

The shape keyed graph cache added a 64 entry LRU cap on top of the
existing time based sweep, to bound a workload with many distinct shapes.
Tensor split needs about 2 * n_layers + 1 graphs per device for a single
shape, one per segment between the cross-device reductions, so every
model past ~31 layers exceeds 64. The LRU then evicts graphs that are
still in use and every token re-captures instead of replaying, which is
as slow as running with GGML_CUDA_DISABLE_GRAPHS=1.

Upstream tried the same cap (64, then 128) in ggml-org/llama.cpp#21611
and dropped it in review for this reason, keeping only the sweep that
evicts graphs unused for 10 s. Do the same here. The shape key stays, so
an alternating speculative verify batch keeps one graph per shape.

Refs unslothai/unsloth#12468
---
 ggml/src/ggml-cuda/common.cuh | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index fdb75a916fd1..72b6b16a319f 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -1459,11 +1459,12 @@ struct ggml_backend_cuda_context {
     // Map from graph key to cuda_graph - allows multiple graphs per context when the
     // computation is split across CPU/GPU (e.g., with --n-cpu-moe), and when the same
     // split is called with different tensor shapes (e.g. a speculative verify batch)
-    std::unordered_map<uint64_t, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;
-
     // a cuda graph instance is only valid for the shapes it captured, so a caller that
-    // alternates shapes needs one instance per shape to stay on the graph path
-    static const size_t max_cuda_graphs = 64;
+    // alternates shapes needs one instance per shape to stay on the graph path. the map is
+    // bounded only by the time based sweep below, as upstream does: tensor split mode alone
+    // needs about 2 * n_layers + 1 graphs per device and shape, so any count cap small enough
+    // to matter evicts graphs that are still in use and re-captures them on every token
+    std::unordered_map<uint64_t, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;
 
     int64_t last_graph_eviction_sweep = 0;
 
@@ -1484,16 +1485,6 @@ struct ggml_backend_cuda_context {
 
         auto it = cuda_graphs.find(graph_key);
         if (it == cuda_graphs.end()) {
-            // a workload with many distinct shapes must not grow this without bound
-            while (cuda_graphs.size() >= max_cuda_graphs) {
-                auto lru = cuda_graphs.begin();
-                for (auto c = cuda_graphs.begin(); c != cuda_graphs.end(); ++c) {
-                    if (c->second->last_used_time < lru->second->last_used_time) {
-                        lru = c;
-                    }
-                }
-                cuda_graphs.erase(lru);
-            }
             it = cuda_graphs.emplace(graph_key, std::make_unique<ggml_cuda_graph>()).first;
         }
         it->second->last_used_time = time_now;