Merge remote-tracking branch 'origin/main' into bsh/dynamic_pd_chunk

# Conflicts:
#	lightllm/server/router/model_infer/mode_backend/base_backend.py

Author: shihaobai

docs/CN/source/tutorial/api_server_args.rst

@@ -267,8 +267,6 @@ PD 分离模式参数
 输出约束参数
 ------------
 
-.. option:: --token_healing_mode
-
 .. option:: --output_constraint_mode
 
     设置输出约束后端，可选值：

docs/CN/source/tutorial/multi_level_cache_deployment.rst

@@ -37,11 +37,26 @@ LightLLM 的多级缓存系统采用分层设计:
 
 **工作原理:**
 
-1. 目前的机制是会将GPU cache中的数据原模原样备份一份到CPU cache中，并非只存储GPU cache放不下的内容
+1. 缓存放置行为由 ``--cache_placement_strategy`` 控制，可选择兼容旧行为的逐层备份或自适应分层放置
 2. L1、L2、L3 cache都基于LRU淘汰策略进行数据管理
 3. 为了避免L3缓存频繁写盘，可通过LIGHTLLM_DISK_CACHE_PROMPT_LIMIT_LENGTH环境变量控制写入的最小长度阈值，如果设为0，则所有L2数据都会写入L3缓存
 4. 查询时，会先查询L1找出命中的最长前缀，再去L2查询以继续增加最长匹配前缀，最后再去L3查询剩余部分
 
+缓存放置策略
+~~~~~~~~~~~~
+
+``--cache_placement_strategy`` 用于选择请求完成后的 KV Cache 放置策略，可选值如下：
+
+- ``adaptive``（默认）：冷启动阶段先收集 128 个请求，快速生成 GPU 与低层缓存路径之间的首个长度分界点；之后保留最近 512 个请求的滑动窗口，每 36 个放置步更新一次分界点。短请求放入 GPU，长请求放入 CPU；开启 Disk 时，长请求沿 CPU → Disk 路径异步落盘。由于 Disk 必须经过 CPU，计算比例时低层有效容量取 CPU 与 Disk 容量的较大值，而不是二者之和。默认只使用物理 GPU token 容量的 80% 进行放置估算，为运行态请求预留容量。首次小窗口尚未填满、没有可用分界点时，使用 ``legacy`` 行为完成冷启动。
+- ``legacy``：兼容原有的逐层备份行为。请求始终保留在 GPU cache，同时复制到所有已开启的下级缓存；开启 CPU cache 时写入 GPU 和 CPU，同时开启 Disk cache 时写入 GPU、CPU 和 Disk。
+
+未开启 ``--enable_cpu_cache`` 时，该参数不会改变运行行为，所有完成请求都只写入 GPU cache。
+
+.. note::
+
+   Disk cache 是通过 CPU cache 异步写入的，因此使用 Disk cache 时仍需同时开启 ``--enable_cpu_cache``。
+   ``LIGHTLLM_DISK_CACHE_PROMPT_LIMIT_LENGTH`` 的最小写盘长度限制对两种策略都生效。
+
 **适用场景:**
 
 - 超长文本处理 (如百万 token 级别的上下文)
@@ -69,7 +84,8 @@ LightLLM 的多级缓存系统采用分层设计:
         --mem_fraction 0.88 \
         --enable_cpu_cache \
         --cpu_cache_storage_size 400 \
-        --cpu_cache_token_page_size 64
+        --cpu_cache_token_page_size 64 \
+        --cache_placement_strategy adaptive
 
 **参数说明:**
 
@@ -99,6 +115,9 @@ CPU 缓存参数
   - 较大的页大小 (如 256) 适合大批量数据迁移,提高传输效率
   - 该值需要权衡内存利用率和传输开销
 
+- ``--cache_placement_strategy adaptive``: **缓存放置策略**，默认使用自适应分层；如需保持原有的 GPU + CPU 逐层备份行为，设置为 ``legacy``
+- ``LIGHTLLM_CACHE_PLACEMENT_GPU_CAPACITY_RATIO=0.8``: **GPU 容量估算比例**，adaptive 默认使用物理 GPU token 容量的 ``0.8`` 进行放置估算，合法范围为 ``(0, 1]``
+
 **性能优化建议:**
 
 1. **使用 Hugepages**: 执行如下命令并设置环境变量LIGHTLLM_HUGE_PAGE_ENABLE可启用大页模式，启用大页内存可以显著提升服务启动速度，如果觉得服务启动太久可以开启大页模式加速，注意大页模式会长期占据内存空间
@@ -132,6 +151,7 @@ CPU 缓存参数
         --enable_cpu_cache \
         --cpu_cache_storage_size 400 \
         --cpu_cache_token_page_size 256 \
+        --cache_placement_strategy adaptive \
         --enable_disk_cache \
         --disk_cache_storage_size 1000 \
         --disk_cache_dir /mnt/ssd/disk_cache_dir
@@ -157,6 +177,12 @@ CPU 缓存参数
   - 确保目录具有足够的读写权限和磁盘空间
   - 注意使用磁盘缓存时, 保证使用的SSD硬盘是长寿命的硬盘, 否则可能会快速消耗其使用寿命。
 
+如需使用兼容旧行为的三级逐层备份策略，将启动参数改为：
+
+.. code-block:: bash
+
+    --cache_placement_strategy legacy
+
 相关文档
 --------
 

docs/CN/source/tutorial/reasoning_parser.rst

@@ -175,6 +175,20 @@ GPT-OSS
     // 答案块
     {"choices": [{"delta": {"content": "答案片段"}}]}
 
+启用 ``--reasoning_parser`` 后，Chat Completions 的 usage 会返回生成的推理
+token 数量（不包含推理分隔符）：
+
+.. code-block:: json
+
+    {
+        "usage": {
+            "completion_tokens": 128,
+            "completion_tokens_details": {"reasoning_tokens": 96}
+        }
+    }
+
+Responses API 会在 ``usage.output_tokens_details.reasoning_tokens`` 中返回相同计数。
+
 高级功能
 --------
 

docs/EN/source/tutorial/api_server_args.rst

@@ -268,8 +268,6 @@ Scheduling Parameters
 Output Constraint Parameters
 ----------------------------
 
-.. option:: --token_healing_mode
-
 .. option:: --output_constraint_mode
 
     Set the output constraint backend, optional values:

docs/EN/source/tutorial/multi_level_cache_deployment.rst

@@ -37,11 +37,26 @@ LightLLM's multi-level cache system adopts a hierarchical design:
 
 **Working Principle:**
 
-1. The current mechanism creates an exact backup copy of GPU cache data in CPU cache, not just storing content that doesn't fit in GPU cache
+1. Cache placement is controlled by ``--cache_placement_strategy``, which selects either legacy cascading copies or adaptive tier placement
 2. L1, L2, and L3 caches all use LRU eviction strategy for data management
 3. To avoid frequent disk writes in L3 cache, you can use the LIGHTLLM_DISK_CACHE_PROMPT_LIMIT_LENGTH environment variable to control the minimum length threshold for writes. If set to 0, all L2 data will be written to L3 cache
 4. During queries, L1 is checked first to find the longest matching prefix, then L2 is queried to continue extending the longest matching prefix, and finally L3 is queried for the remaining part
 
+Cache Placement Strategies
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``--cache_placement_strategy`` controls where completed-request KV Cache is stored:
+
+- ``adaptive`` (default): During cold start, it first collects 128 requests to quickly establish an initial boundary between GPU and the lower-tier cache path. It then keeps a sliding window of the latest 512 requests and updates the boundary every 36 placement steps. Short requests stay on GPU; long requests move to CPU, or follow the asynchronous CPU → Disk path when Disk is enabled. Because Disk must pass through CPU, the effective lower-tier capacity is the larger of CPU and Disk capacity rather than their sum. By default, only 80% of physical GPU token capacity is used for placement estimation, leaving capacity for running requests. Until the initial small window is full and a boundary is available, placement uses ``legacy`` behavior.
+- ``legacy``: Preserves the previous cascading-copy behavior. A request always remains in GPU cache and is also copied to every enabled lower tier. Enabling CPU cache stores it in GPU and CPU; enabling Disk cache as well stores it in GPU, CPU, and Disk.
+
+When ``--enable_cpu_cache`` is not enabled, this option does not change runtime behavior and every completed request uses GPU cache only.
+
+.. note::
+
+   Disk cache is populated asynchronously through CPU cache, so ``--enable_cpu_cache`` must also be enabled when using Disk cache.
+   The ``LIGHTLLM_DISK_CACHE_PROMPT_LIMIT_LENGTH`` minimum write threshold applies to both strategies.
+
 **Applicable Scenarios:**
 
 - Ultra-long text processing (e.g., million-token level context)
@@ -69,7 +84,8 @@ Suitable for most scenarios, significantly increasing cache capacity while maint
         --mem_fraction 0.88 \
         --enable_cpu_cache \
         --cpu_cache_storage_size 400 \
-        --cpu_cache_token_page_size 64
+        --cpu_cache_token_page_size 64 \
+        --cache_placement_strategy adaptive
 
 **Parameter Description:**
 
@@ -99,6 +115,9 @@ CPU Cache Parameters
   - Larger page sizes (e.g., 256) are suitable for bulk data migration, improving transfer efficiency
   - This value needs to balance memory utilization and transfer overhead
 
+- ``--cache_placement_strategy adaptive``: **Cache placement strategy**. Adaptive tier placement is the default; use ``legacy`` to retain the previous GPU + CPU cascading-copy behavior
+- ``LIGHTLLM_CACHE_PLACEMENT_GPU_CAPACITY_RATIO=0.8``: Ratio of physical GPU token capacity used by adaptive placement estimation. The default is ``0.8`` and the valid range is ``(0, 1]``
+
 **Performance Optimization Suggestions:**
 
 1. **Using Hugepages**: Execute the following commands and set the LIGHTLLM_HUGE_PAGE_ENABLE environment variable to enable huge page mode. Enabling huge page memory can significantly improve service startup speed. If you find the service takes too long to start, you can enable huge page mode for acceleration. Note that huge page mode will occupy memory space for the long term
@@ -132,6 +151,7 @@ Suitable for ultra-long text or extremely high-concurrency scenarios, providing
         --enable_cpu_cache \
         --cpu_cache_storage_size 400 \
         --cpu_cache_token_page_size 256 \
+        --cache_placement_strategy adaptive \
         --enable_disk_cache \
         --disk_cache_storage_size 1000 \
         --disk_cache_dir /mnt/ssd/disk_cache_dir
@@ -156,6 +176,12 @@ In addition to the two-level cache, add the following parameters:
   - Strongly recommended to use SSD/NVMe storage, avoid using HDD (performance difference can be 10-100x)
   - Ensure the directory has sufficient read/write permissions and disk space
 
+To use the legacy three-level cascading-copy behavior, change the strategy argument to:
+
+.. code-block:: bash
+
+    --cache_placement_strategy legacy
+
 Related Documentation
 ---------------------
 

docs/EN/source/tutorial/reasoning_parser.rst

@@ -175,6 +175,21 @@ Response Format
     // Answer chunk
     {"choices": [{"delta": {"content": "Answer fragment"}}]}
 
+When ``--reasoning_parser`` is enabled, Chat Completions usage reports the
+number of generated reasoning tokens (excluding reasoning delimiters):
+
+.. code-block:: json
+
+    {
+        "usage": {
+            "completion_tokens": 128,
+            "completion_tokens_details": {"reasoning_tokens": 96}
+        }
+    }
+
+The Responses API exposes the same count as
+``usage.output_tokens_details.reasoning_tokens``.
+
 Advanced Features
 -----------------
 

lightllm/common/basemodel/basemodel.py

@@ -82,11 +82,7 @@ def __init__(self, kvargs):
         self.finetune_config = kvargs.get("finetune_config", None)
         self.max_req_num = kvargs.get("max_req_num", 1000)
         self.max_seq_length = kvargs.get("max_seq_length", 1024 * 5)
-        # is_token_healing 和 return_all_prompt_logics 是有排斥关系的两个模式，只能单独有一个生效
-        # 主要是在prefill阶段返回多少个token的用于后续处理相关。
-        self.is_token_healing = kvargs.get("is_token_healing", False)
         self.return_all_prompt_logics = kvargs.get("return_all_prompt_logics", False)
-        assert not (self.is_token_healing and self.return_all_prompt_logics), "can not be true in same time"
         self.data_type = get_llm_data_type()
         self.graph_max_batch_size = kvargs.get("graph_max_batch_size", 16)
         self.graph_max_batch_size = (
@@ -387,7 +383,6 @@ def _create_inferstate(self, model_input: ModelInput, microbatch_index: int = 0)
         infer_state.hidden_collector = self.hidden_collector_prototype.new_instance()
         infer_state.input_ids = model_input.input_ids
         infer_state.is_prefill = model_input.is_prefill
-        infer_state.is_token_healing = self.is_token_healing
         infer_state.return_all_prompt_logics = self.return_all_prompt_logics
         infer_state.batch_size = model_input.batch_size
         infer_state.total_token_num = model_input.total_token_num

lightllm/common/basemodel/infer_struct.py

@@ -51,7 +51,6 @@ def __init__(self):
 
         self.mem_index: torch.Tensor = None
 
-        self.is_token_healing: bool = False
         self.return_all_prompt_logics: bool = False
         # 在开启 return_all_prompt_logics 模式时，保存整个 prefill 阶段每一个
         # token 位置的 logits，供后续回传 prompt logprobs 信息使用。

lightllm/models/llama/layer_infer/post_layer_infer.py

@@ -23,22 +23,6 @@ def _norm(self, input, infer_state, layer_weight: LlamaPreAndPostLayerWeight) ->
 
     def _slice_get_last_input(self, input_embdings: torch.Tensor, infer_state: LlamaInferStateInfo):
         embed_dim_ = input_embdings.shape[1]
-        if infer_state.is_prefill and infer_state.is_token_healing:
-            batch_size = infer_state.batch_size
-            b_seq_len_numpy = (infer_state.b_seq_len - infer_state.b_ready_cache_len).detach().cpu().numpy()
-            select_index = []
-            start_index = 0
-            select_token_num = 0
-            for cur_len in b_seq_len_numpy:
-                select_index.append(start_index + cur_len - 1)
-                start_index += cur_len
-                select_token_num += 1
-
-            last_index = torch.tensor(select_index, dtype=torch.long, device="cpu").cuda(non_blocking=True)
-            last_input = self.alloc_tensor((select_token_num, embed_dim_), dtype=input_embdings.dtype, device="cuda")
-            last_input[:, :] = input_embdings[last_index, :]
-            return last_input, select_token_num
-
         if infer_state.is_prefill:
             # logits 始终只取每个请求最后一个位置的 hidden state，用于正常采样。
             batch_size = infer_state.batch_size

lightllm/server/api_cli.py

@@ -348,11 +348,10 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         "--short_prefill_token_threshold",
         type=int,
         default=None,
-        help="""Enable long/short prefill scheduling and set the maximum remaining tokens of a short request.
+        help="""Enable short prefill request priority scheduling.
         The remaining tokens are calculated after prefix-cache matching. Disabled by default.""",
     )
     parser.add_argument("--diverse_mode", action="store_true", help="diversity generation mode")
-    parser.add_argument("--token_healing_mode", action="store_true", help="code model infer mode")
 
     parser.add_argument(
         "--output_constraint_mode",
@@ -827,6 +826,16 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         default=256,
         help="""The token page size of cpu cache""",
     )
+    parser.add_argument(
+        "--cache_placement_strategy",
+        type=str,
+        choices=["adaptive", "legacy"],
+        default="adaptive",
+        help="""Cache placement strategy used when CPU cache is enabled.
+        adaptive: place requests between GPU and the lower-tier cache path based on recent input lengths and capacity;
+        Disk placement is offloaded through CPU cache, so CPU and Disk capacities are not additive.
+        legacy: retain GPU cache and also copy each request to every enabled lower cache level.""",
+    )
     parser.add_argument("--enable_disk_cache", action="store_true", help="""enable disk cache to store kv cache.""")
     parser.add_argument(
         "--disk_cache_storage_size", type=float, default=10, help="""The capacity of disk cache. GB used."""