From 8834429cf08a63855fdf0420c35c72fd565088de Mon Sep 17 00:00:00 2001 From: sufubao Date: Tue, 1 Sep 2026 16:32:55 +0800 Subject: [PATCH 1/3] feat(api): cap request output tokens --- docs/CN/source/tutorial/api_server_args.rst | 5 +++++ docs/EN/source/tutorial/api_server_args.rst | 5 +++++ lightllm/server/api_cli.py | 8 ++++++++ lightllm/server/api_models.py | 4 ++-- lightllm/server/api_start.py | 2 ++ lightllm/server/core/objs/sampling_params.py | 6 +++++- lightllm/server/core/objs/start_args_type.py | 1 + .../test_api/test_max_request_output_tokens.py | 18 ++++++++++++++++++ 8 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 test/test_api/test_max_request_output_tokens.py diff --git a/docs/CN/source/tutorial/api_server_args.rst b/docs/CN/source/tutorial/api_server_args.rst index fbb63d09f0..30da249490 100644 --- a/docs/CN/source/tutorial/api_server_args.rst +++ b/docs/CN/source/tutorial/api_server_args.rst @@ -145,6 +145,11 @@ PD 分离模式参数 新批次的最大 token 数量，控制预填充批次大小以防止 OOM +.. option:: --max-request-output-tokens + + 单请求输出 token 的默认值和硬上限，默认为 ``65536``。 + 请求中更大的 ``max_new_tokens`` 或 ``max_tokens`` 会被截断到该值。 + .. option:: --running_max_req_size 同时进行前向推理的最大请求数量，默认为 ``1000`` diff --git a/docs/EN/source/tutorial/api_server_args.rst b/docs/EN/source/tutorial/api_server_args.rst index 69edf50a86..b067212018 100644 --- a/docs/EN/source/tutorial/api_server_args.rst +++ b/docs/EN/source/tutorial/api_server_args.rst @@ -147,6 +147,11 @@ Memory and Batch Processing Parameters Maximum token count for new batches, controls prefill batch size to prevent OOM +.. option:: --max-request-output-tokens + + Default and hard limit for output tokens per request, default is ``65536``. + Requests with a larger ``max_new_tokens`` or ``max_tokens`` value are capped at this limit. + .. option:: --running_max_req_size Maximum number of requests for simultaneous forward inference, default is ``1000`` diff --git a/lightllm/server/api_cli.py b/lightllm/server/api_cli.py index 60b5fad4e8..d500d9dd73 100644 --- a/lightllm/server/api_cli.py +++ b/lightllm/server/api_cli.py @@ -174,6 +174,14 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: default=None, help="max tokens num for new cat batch, it control prefill batch size to Preventing OOM", ) + parser.add_argument( + "--max-request-output-tokens", + "--max_request_output_tokens", + dest="max_request_output_tokens", + type=int, + default=65536, + help="default and hard limit for the number of output tokens per request", + ) parser.add_argument( "--eos_id", nargs="+", type=int, default=None, help="eos stop token id, if None, will load from config.json" ) diff --git a/lightllm/server/api_models.py b/lightllm/server/api_models.py index bfb19ff0eb..50259fa17b 100644 --- a/lightllm/server/api_models.py +++ b/lightllm/server/api_models.py @@ -123,7 +123,7 @@ class CompletionRequest(BaseModel): prompt: Union[str, List[str], List[int], List[List[int]]] suffix: Optional[str] = None max_tokens: Optional[int] = Field( - default=65536, deprecated="max_tokens is deprecated, please use max_completion_tokens instead" + default=None, deprecated="max_tokens is deprecated, please use max_completion_tokens instead" ) max_completion_tokens: Optional[int] = None temperature: Optional[float] = 1.0 @@ -199,7 +199,7 @@ class ChatCompletionRequest(BaseModel): stream_options: Optional[StreamOptions] = None stop: Optional[Union[str, List[str]]] = None max_tokens: Optional[int] = Field( - default=65536, deprecated="max_tokens is deprecated, please use max_completion_tokens instead" + default=None, deprecated="max_tokens is deprecated, please use max_completion_tokens instead" ) max_completion_tokens: Optional[int] = None presence_penalty: Optional[float] = 0.0 diff --git a/lightllm/server/api_start.py b/lightllm/server/api_start.py index 37fe837ad1..3f7a5d764f 100644 --- a/lightllm/server/api_start.py +++ b/lightllm/server/api_start.py @@ -120,6 +120,8 @@ def _launch_subprocesses(args: StartArgs): assert ( args.mem_fraction > 0 and args.mem_fraction < 1 ), f"Invalid mem_fraction {args.mem_fraction}, The expected value is between 0 and 1." + if args.max_request_output_tokens < 1: + raise ValueError("max_request_output_tokens must be a positive integer.") if args.graph_max_len_in_batch == 0: args.graph_max_len_in_batch = args.max_req_total_len diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index 8e31c50624..7d0a0a80d0 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -333,7 +333,11 @@ def init(self, tokenizer, **kwargs): self.top_k = kwargs.get("top_k", SamplingParams._top_k) self.ignore_eos = kwargs.get("ignore_eos", False) self.image_max_patch_num = kwargs.get("image_max_patch_num", -1) - self.max_new_tokens = kwargs.get("max_new_tokens", 65535) + try: + max_request_output_tokens = getattr(get_env_start_args(), "max_request_output_tokens", 65536) + except KeyError: # SamplingParams is also used without a running server in libraries/tests. + max_request_output_tokens = 65536 + self.max_new_tokens = min(kwargs.get("max_new_tokens", max_request_output_tokens), max_request_output_tokens) self.min_new_tokens = kwargs.get("min_new_tokens", 1) self.input_penalty = kwargs.get("input_penalty", DEFAULT_INPUT_PENALTY) self.group_request_id = kwargs.get("group_request_id", -1) diff --git a/lightllm/server/core/objs/start_args_type.py b/lightllm/server/core/objs/start_args_type.py index a9aef608bd..4811c24538 100644 --- a/lightllm/server/core/objs/start_args_type.py +++ b/lightllm/server/core/objs/start_args_type.py @@ -40,6 +40,7 @@ class StartArgs: max_total_token_num: Optional[int] = field(default=None) mem_fraction: float = field(default=0.8) batch_max_tokens: Optional[int] = field(default=None) + max_request_output_tokens: int = field(default=65536) eos_id: Optional[List[int]] = field(default=None) tool_call_parser: Optional[str] = field( default=None, diff --git a/test/test_api/test_max_request_output_tokens.py b/test/test_api/test_max_request_output_tokens.py new file mode 100644 index 0000000000..f64ebbe9dc --- /dev/null +++ b/test/test_api/test_max_request_output_tokens.py @@ -0,0 +1,18 @@ +from types import SimpleNamespace + +import lightllm.server.core.objs.sampling_params as sampling_params_module +from lightllm.server.core.objs.sampling_params import SamplingParams + + +def test_max_request_output_tokens_is_default_and_hard_limit(monkeypatch): + monkeypatch.setattr( + sampling_params_module, + "get_env_start_args", + lambda: SimpleNamespace(max_request_output_tokens=1024), + ) + + for requested, expected in ((None, 1024), (256, 256), (2048, 1024)): + params = SamplingParams() + kwargs = {} if requested is None else {"max_new_tokens": requested} + params.init(None, **kwargs) + assert params.max_new_tokens == expected From 755c76d30142785f1ea23e4f6c59be845d4ec165 Mon Sep 17 00:00:00 2001 From: sufubao Date: Tue, 1 Sep 2026 16:57:16 +0800 Subject: [PATCH 2/3] refactor(api): access output token limit directly --- lightllm/server/core/objs/sampling_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index 7d0a0a80d0..ad01137497 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -334,7 +334,7 @@ def init(self, tokenizer, **kwargs): self.ignore_eos = kwargs.get("ignore_eos", False) self.image_max_patch_num = kwargs.get("image_max_patch_num", -1) try: - max_request_output_tokens = getattr(get_env_start_args(), "max_request_output_tokens", 65536) + max_request_output_tokens = get_env_start_args().max_request_output_tokens except KeyError: # SamplingParams is also used without a running server in libraries/tests. max_request_output_tokens = 65536 self.max_new_tokens = min(kwargs.get("max_new_tokens", max_request_output_tokens), max_request_output_tokens) From 13271d4cc0312c9c9e9661624e26277647adc0f0 Mon Sep 17 00:00:00 2001 From: sufubao Date: Tue, 1 Sep 2026 16:58:13 +0800 Subject: [PATCH 3/3] test(api): provide server args to sampling params --- lightllm/server/core/objs/sampling_params.py | 5 +---- test/test_api/test_seed_validation.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lightllm/server/core/objs/sampling_params.py b/lightllm/server/core/objs/sampling_params.py index ad01137497..479ede5fdf 100644 --- a/lightllm/server/core/objs/sampling_params.py +++ b/lightllm/server/core/objs/sampling_params.py @@ -333,10 +333,7 @@ def init(self, tokenizer, **kwargs): self.top_k = kwargs.get("top_k", SamplingParams._top_k) self.ignore_eos = kwargs.get("ignore_eos", False) self.image_max_patch_num = kwargs.get("image_max_patch_num", -1) - try: - max_request_output_tokens = get_env_start_args().max_request_output_tokens - except KeyError: # SamplingParams is also used without a running server in libraries/tests. - max_request_output_tokens = 65536 + max_request_output_tokens = get_env_start_args().max_request_output_tokens self.max_new_tokens = min(kwargs.get("max_new_tokens", max_request_output_tokens), max_request_output_tokens) self.min_new_tokens = kwargs.get("min_new_tokens", 1) self.input_penalty = kwargs.get("input_penalty", DEFAULT_INPUT_PENALTY) diff --git a/test/test_api/test_seed_validation.py b/test/test_api/test_seed_validation.py index 1168c5ac53..bad296986b 100644 --- a/test/test_api/test_seed_validation.py +++ b/test/test_api/test_seed_validation.py @@ -1,11 +1,23 @@ +from types import SimpleNamespace + import pytest from pydantic import ValidationError +import lightllm.server.core.objs.sampling_params as sampling_params_module from lightllm.server.api_models import ChatCompletionRequest, CompletionRequest, MAX_SEED from lightllm.server.core.objs.py_sampling_params import SamplingParams as PySamplingParams from lightllm.server.core.objs.sampling_params import SamplingParams +@pytest.fixture(autouse=True) +def mock_start_args(monkeypatch): + monkeypatch.setattr( + sampling_params_module, + "get_env_start_args", + lambda: SimpleNamespace(max_request_output_tokens=65536), + ) + + @pytest.mark.parametrize( ("request_type", "request_data"), [