From c5f5eb0883a5d20974134d60c2e9bdfed980150c Mon Sep 17 00:00:00 2001 From: WANDY666 <1060304770@qq.com> Date: Mon, 24 Aug 2026 16:37:27 +0800 Subject: [PATCH] perf(pd): omit unused output logprob metadata --- lightllm/server/api_openai.py | 5 +++++ lightllm/server/httpserver/pd_loop.py | 4 ++++ lightllm/server/httpserver_for_pd_master/manager.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/lightllm/server/api_openai.py b/lightllm/server/api_openai.py index e89ec5c5a..1ef9f9de3 100644 --- a/lightllm/server/api_openai.py +++ b/lightllm/server/api_openai.py @@ -349,6 +349,9 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req sampling_params = SamplingParams() sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict) + # The chat API does not expose output-token logprobs. PD nodes use this + # transport-only marker to avoid forwarding unused per-token metadata. + sampling_params.return_output_logprobs = False sampling_params.verify() results_generator = g_objs.httpserver_manager.generate( @@ -843,6 +846,7 @@ async def completions_impl(request: CompletionRequest, raw_request: Request) -> sampling_params = SamplingParams() sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict) + sampling_params.return_output_logprobs = request.logprobs is not None sampling_params.verify() # v1/completions does not support multimodal inputs, so we use an empty MultimodalParams @@ -880,6 +884,7 @@ async def process_single_prompt(prompt: Union[str, List[int]]): if len(prompts) > 1: individual_sampling_params = SamplingParams() individual_sampling_params.init(tokenizer=g_objs.httpserver_manager.tokenizer, **sampling_params_dict) + individual_sampling_params.return_output_logprobs = request.logprobs is not None individual_sampling_params.verify() else: individual_sampling_params = sampling_params diff --git a/lightllm/server/httpserver/pd_loop.py b/lightllm/server/httpserver/pd_loop.py index ba843db0b..f2c7d4b9b 100644 --- a/lightllm/server/httpserver/pd_loop.py +++ b/lightllm/server/httpserver/pd_loop.py @@ -234,6 +234,7 @@ async def _pd_process_generate( pd_upload_websocket: ClientConnection, pd_event: asyncio.Event, ): + return_output_logprobs = getattr(sampling_params, "return_output_logprobs", True) try: async for sub_req_id, request_output, metadata, finish_status in manager.generate( prompt=prompt, @@ -244,6 +245,9 @@ async def _pd_process_generate( pd_event=pd_event, ): metadata["node_mode"] = manager.args.run_mode + if not return_output_logprobs: + for key in ("id", "logprob", "cumlogprob", "special", "logprobs"): + metadata.pop(key, None) await forwarding_queue.put((sub_req_id, request_output, metadata, finish_status)) except PDPrefillNodeStopGenToken as e: logger.info(f"pd prefill node stop gen token for group_request_id {e.group_request_id}") diff --git a/lightllm/server/httpserver_for_pd_master/manager.py b/lightllm/server/httpserver_for_pd_master/manager.py index 0dfaca6cd..c636caded 100644 --- a/lightllm/server/httpserver_for_pd_master/manager.py +++ b/lightllm/server/httpserver_for_pd_master/manager.py @@ -160,7 +160,9 @@ async def _generate( self, prompt_ids=fake_prompt_ids, sampling_params=sampling_params ) + return_output_logprobs = getattr(sampling_params, "return_output_logprobs", True) origin_sampling_params = SamplingParams.from_buffer_copy(sampling_params) + origin_sampling_params.return_output_logprobs = return_output_logprobs origin_group_request_id = self.id_gen.generate_id() # Record one user request even when it is expanded into multiple independent @@ -174,6 +176,7 @@ async def _generate( generators = [] for choice_index in range(choice_count): choice_sampling_params = SamplingParams.from_buffer_copy(origin_sampling_params) + choice_sampling_params.return_output_logprobs = return_output_logprobs choice_sampling_params.n = 1 choice_sampling_params.best_of = 1 generators.append( @@ -224,6 +227,7 @@ async def _generate_one( for iter_index, block_max_new_tokens in enumerate(max_new_tokens_list): sampling_params = SamplingParams.from_buffer_copy(origin_sampling_params) + sampling_params.return_output_logprobs = getattr(origin_sampling_params, "return_output_logprobs", True) block_group_request_id = self.id_gen.generate_id() sampling_params.group_request_id = block_group_request_id logger.info(f"pd log gen sub req id {block_group_request_id} for main req id {origin_request_id}")