From ff0987a14192a0166ea20ec97a453573bb72b3fe Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Tue, 11 Aug 2026 11:23:37 -0700 Subject: [PATCH 1/2] Fix guided decoding for Harmony parser Apply structural guidance only after Harmony enters the final channel. Preserve unconstrained reasoning while keeping JSON and grammar evaluation scoped to final content. Signed-off-by: Dongfeng Yu --- tensorrt_llm/evaluate/interface.py | 44 +++++ tensorrt_llm/evaluate/json_mode_eval.py | 45 ++++- tensorrt_llm/llmapi/llm.py | 21 +++ tensorrt_llm/llmapi/reasoning_parser.py | 133 +++++++++++++- tensorrt_llm/serve/openai_protocol.py | 63 +------ .../evaluate/test_generation_content.py | 88 +++++++++ tests/unittest/llmapi/test_sampling_params.py | 172 +++++++++++++++++- 7 files changed, 502 insertions(+), 64 deletions(-) create mode 100644 tests/unittest/evaluate/test_generation_content.py diff --git a/tensorrt_llm/evaluate/interface.py b/tensorrt_llm/evaluate/interface.py index c27ba88d6327..b411df8edee3 100644 --- a/tensorrt_llm/evaluate/interface.py +++ b/tensorrt_llm/evaluate/interface.py @@ -26,6 +26,8 @@ import tensorrt_llm.profiler as profiler from ..llmapi import RequestOutput +from ..llmapi.reasoning_parser import (HARMONY_REASONING_PARSER, + ReasoningParserFactory) from ..logger import logger from ..sampling_params import SamplingParams @@ -66,6 +68,48 @@ def get_model_context(llm: Any) -> tuple[str, str]: return str(model_dir), str(model_type) +def extract_final_content_from_generation( + output: RequestOutput, + *, + reasoning_parser: Optional[str] = None, +) -> str: + """Return the scoreable final-answer text from a raw generation. + + Handles plain JSON text, Harmony token transcripts, and normal reasoning + parser formats like `...{...}` for eval-only scoring. + """ + # Eval scoring has historically used the first completion choice. + text = output.outputs[0].text + + if reasoning_parser is None: + # Plain models keep the raw output; never guess a parser from text. + return text + + if reasoning_parser.lower() == HARMONY_REASONING_PARSER: + # Harmony preserves final-channel boundaries in token ids, not text. + try: + from tensorrt_llm.serve.harmony_adapter import get_harmony_adapter + + parsed = get_harmony_adapter().harmony_output_to_openai( + output.outputs[0].token_ids) + content = parsed.get("content") + if not parsed.get("_harmony_parsing_failed") and isinstance( + content, str): + return content + except (ImportError, AttributeError, TypeError, RuntimeError, + ValueError): + return text + return text + + try: + # Normal reasoning formats expose final content through their parser. + parser = ReasoningParserFactory.create_reasoning_parser( + reasoning_parser) + return parser.parse(text).content or text + except (AttributeError, TypeError, ValueError): + return text + + class Evaluator(ABC): def __init__(self, diff --git a/tensorrt_llm/evaluate/json_mode_eval.py b/tensorrt_llm/evaluate/json_mode_eval.py index ce35d474b333..2a34c4303899 100644 --- a/tensorrt_llm/evaluate/json_mode_eval.py +++ b/tensorrt_llm/evaluate/json_mode_eval.py @@ -14,7 +14,7 @@ # limitations under the License. import json import os -from typing import Iterable, List, Optional +from typing import Iterable, List, Optional, Union import click import datasets @@ -23,9 +23,31 @@ from .. import LLM as PyTorchLLM from ..llmapi import RequestOutput +from ..llmapi.reasoning_parser import resolve_guided_decoding_reasoning_parser from ..logger import logger from ..sampling_params import GuidedDecodingParams, SamplingParams -from .interface import Evaluator +from .interface import Evaluator, extract_final_content_from_generation + +JsonValue = Union[dict[str, object], list[object], str, int, float, bool, None] + + +def _load_json_from_generation( + output: RequestOutput, + reasoning_parser: Optional[str] = None) -> JsonValue: + text = output.outputs[0].text + try: + # Plain/non-reasoning evals already return the constrained JSON as raw + # text, so keep the existing fast path and semantics unchanged. + return json.loads(text) + except json.JSONDecodeError as original_error: + # Reasoning models may return a raw transcript such as Harmony channels + # or `...{...}`. Score the final answer content when it + # can be extracted; otherwise preserve the original JSON failure. + final_content = extract_final_content_from_generation( + output, reasoning_parser=reasoning_parser) + if final_content == text: + raise original_error + return json.loads(final_content) class JsonModeEval(Evaluator): @@ -44,6 +66,7 @@ def __init__(self, apply_chat_template=apply_chat_template, system_prompt=system_prompt, output_dir=output_dir) + self._reasoning_parser: Optional[str] = None if dataset_path is None: dataset_path = "NousResearch/json-mode-eval" self.data = datasets.load_dataset(dataset_path, @@ -55,6 +78,21 @@ def __init__(self, else: self.num_samples = min(num_samples, self.data.num_rows) + def evaluate(self, + llm: PyTorchLLM, + sampling_params: Optional[SamplingParams] = None, + streaming: bool = False) -> float: + # Resolve once from the LLM rather than guessing Harmony from every + # failed JSON sample. Plain models therefore keep strict raw-text + # scoring, while GPT-OSS uses its known final-channel token framing. + model_type = getattr(getattr(llm, "_hf_model_config", None), + "model_type", None) + self._reasoning_parser = resolve_guided_decoding_reasoning_parser( + getattr(getattr(llm, "args", None), "reasoning_parser", None), + model_type, + ) + return super().evaluate(llm, sampling_params, streaming) + def generate_samples(self) -> Iterable[tuple]: for i, sample in enumerate(self.data): if i >= self.num_samples: @@ -75,7 +113,8 @@ def compute_score(self, outputs: List[RequestOutput], references: List[str], all_corrections, all_grammar_corrections = [], [] for output, ref, schema in zip(outputs, references, schemas): try: - output_json = json.loads(output.outputs[0].text) + output_json = _load_json_from_generation( + output, self._reasoning_parser) jsonschema.validate(output_json, json.loads(schema)) except (json.JSONDecodeError, jsonschema.ValidationError): all_corrections.append(False) diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 825e26fb3455..98e4d5bc9792 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -45,6 +45,9 @@ from .llm_utils import (CachedModelLoader, KvCacheRetentionConfig, LlmBuildStats, ModelLoader) from .mpi_session import MpiPoolSession, external_mpi_comm_available +from .reasoning_parser import ( + adapt_guided_decoding_params_for_reasoning_parser, + resolve_raw_guided_decoding_reasoning_parser) from .thinking_budget import add_thinking_budget_logits_processor from .tokenizer import TokenizerBase # TODO[chunweiy]: move the following symbols back to utils scope, and remove the following import @@ -1390,6 +1393,24 @@ def _prepare_sampling_params( self._generation_config) self._configure_bart_decoder_prefix(sampling_params) self._add_whisper_suppress_tokens_logits_processor(sampling_params) + if sampling_params.guided_decoding is not None: + reasoning_format_for_guided_decoding = ( + resolve_raw_guided_decoding_reasoning_parser( + self.args.reasoning_parser, + getattr(self._hf_model_config, "model_type", None), + self.args.guided_decoding_backend, + )) + if reasoning_format_for_guided_decoding is not None: + # SamplingParams carries a caller-provided content + # constraint, but not how this model frames reasoning and + # final output. Add that model-aware framing here so + # xgrammar applies the guide only to Harmony's final + # channel. Other formats preserve the original guide. + sampling_params.guided_decoding = ( + adapt_guided_decoding_params_for_reasoning_parser( + sampling_params.guided_decoding, + reasoning_format_for_guided_decoding, + )) add_thinking_budget_logits_processor( sampling_params, reasoning_parser=self.args.reasoning_parser, diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index 4f847325748d..4500855b93b5 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -20,7 +20,8 @@ from pathlib import Path from typing import Any, ClassVar, Optional, Type -from tensorrt_llm import logger +from tensorrt_llm.logger import logger +from tensorrt_llm.sampling_params import GuidedDecodingParams @dataclass @@ -32,6 +33,15 @@ class ReasoningParserResult: # Enough of the rendered prompt's tail to hold a prefilled marker and any # trailing whitespace, without copying a prompt that may be very long. _PROMPT_TAIL_CHARS = 64 +HARMONY_REASONING_PARSER = "gpt_oss" +HARMONY_FINAL_CHANNEL_TRIGGER = "<|start|>assistant<|channel|>final<|message|>" + +# Unlike normal reasoning parsers, Harmony is selected automatically by the +# serving stack from the model type. Raw LLM requests need the same default +# when they arrive with pre-built SamplingParams. +_DEFAULT_GUIDED_DECODING_REASONING_PARSER_BY_MODEL_TYPE = { + "gpt_oss": HARMONY_REASONING_PARSER, +} def register_reasoning_parser(*keys: str, **default_kwargs): @@ -196,6 +206,127 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: return ReasoningParserResult(content=delta_text) +def resolve_guided_decoding_reasoning_parser( + reasoning_parser: Optional[str], + model_type: Optional[str], +) -> Optional[str]: + """Resolve the reasoning format used to scope a request's guide. + + An explicitly configured parser always wins. The model-type dispatch only + supplies formats that serving already selects implicitly, currently the + GPT-OSS Harmony protocol. Other raw LLM models therefore retain their + pre-existing guided-decoding behavior unless a parser was configured. + """ + if reasoning_parser is not None: + return reasoning_parser + return _DEFAULT_GUIDED_DECODING_REASONING_PARSER_BY_MODEL_TYPE.get( + model_type) + + +def resolve_raw_guided_decoding_reasoning_parser( + reasoning_parser: Optional[str], + model_type: Optional[str], + guided_decoding_backend: Optional[str], +) -> Optional[str]: + """Resolve the reasoning format that raw LLM requests should adapt. + + Raw LLM only needs new final-content scoping for Harmony, and structural + tags are supported by xgrammar only. Explicit normal parsers and + llguidance retain their pre-existing raw guided-decoding behavior. + """ + resolved_parser = resolve_guided_decoding_reasoning_parser( + reasoning_parser, model_type) + if (resolved_parser is not None + and resolved_parser.lower() == HARMONY_REASONING_PARSER + and guided_decoding_backend == "xgrammar"): + return HARMONY_REASONING_PARSER + return None + + +def _normalize_json_schema_for_structural_tag(json_schema: Any) -> Any: + """Convert supported schema representations to structural-tag JSON.""" + if hasattr(json_schema, "model_json_schema"): + json_schema = json_schema.model_json_schema() + if isinstance(json_schema, str): + json_schema = json.loads(json_schema) + return json_schema + + +def _guided_decoding_content( + guided_decoding_params: GuidedDecodingParams) -> Optional[dict]: + """Translate an ordinary guide into structural-tag content.""" + if guided_decoding_params.json is not None: + json_schema = _normalize_json_schema_for_structural_tag( + guided_decoding_params.json) + return {"type": "json_schema", "json_schema": json_schema} + if guided_decoding_params.json_object: + return {"type": "json_schema", "json_schema": {"type": "object"}} + if guided_decoding_params.regex is not None: + return {"type": "regex", "pattern": guided_decoding_params.regex} + if guided_decoding_params.grammar is not None: + return {"type": "grammar", "grammar": guided_decoding_params.grammar} + return None + + +def adapt_guided_decoding_params_for_reasoning_parser( + guided_decoding_params: Optional[GuidedDecodingParams], + reasoning_parser: Optional[str], +) -> Optional[GuidedDecodingParams]: + """Scope a guide to final content while leaving reasoning unconstrained. + + Normal reasoning formats use a reasoning-tag sequence. Harmony instead + activates the guide when the assistant's final channel begins. Existing + structural-tag guides are already fully specified and remain untouched. + """ + if guided_decoding_params is None or reasoning_parser is None: + return guided_decoding_params + if guided_decoding_params.structural_tag is not None: + return guided_decoding_params + + content = _guided_decoding_content(guided_decoding_params) + if content is None: + return guided_decoding_params + + if reasoning_parser.lower() == HARMONY_REASONING_PARSER: + stag_format = { + "type": + "triggered_tags", + "triggers": [HARMONY_FINAL_CHANNEL_TRIGGER], + "tags": [{ + "begin": HARMONY_FINAL_CHANNEL_TRIGGER, + "content": content, + "end": "", + }], + "stop_after_first": + True, + } + else: + parser = ReasoningParserFactory.create_reasoning_parser( + reasoning_parser) + stag_format = { + "type": + "sequence", + "elements": [ + { + "type": "tag", + "begin": parser.reasoning_start, + "content": { + "type": "any_text" + }, + "end": parser.reasoning_end, + }, + content, + ], + } + + structural_tag = { + "type": "structural_tag", + "format": stag_format, + } + return GuidedDecodingParams( + structural_tag=json.dumps(structural_tag, separators=(",", ":"))) + + @register_reasoning_parser("deepseek-r1", reasoning_at_start=True) @register_reasoning_parser("qwen3") # Qwen3.5 (and forced-thinking Qwen3 variants) use a chat template that diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index c9405ccbfa39..701c383d8326 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -42,7 +42,8 @@ from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams from tensorrt_llm.llmapi import (DisaggScheduleStyle, GuidedDecodingParams, SamplingParams) -from tensorrt_llm.llmapi.reasoning_parser import ReasoningParserFactory +from tensorrt_llm.llmapi.reasoning_parser import \ + adapt_guided_decoding_params_for_reasoning_parser from tensorrt_llm.sampling_params import (check_logprobs_limit, validate_thinking_token_budget) from tensorrt_llm.scheduling_params import AgentHierarchy @@ -422,64 +423,8 @@ def _response_format_to_guided_decoding_params( else: raise ValueError(f"Unsupported response format: {response_format.type}") - if guided_decoding_params is None or reasoning_parser is None: - return guided_decoding_params - - if guided_decoding_params.structural_tag is not None: - return guided_decoding_params - - # Adapt guided_decoding_params for reasoning parser - if guided_decoding_params.json is not None: - content = { - "type": "json_schema", - "json_schema": guided_decoding_params.json - } - elif guided_decoding_params.json_object: - content = {"type": "json_schema", "json_schema": {"type": "object"}} - elif guided_decoding_params.regex is not None: - content = {"type": "regex", "pattern": guided_decoding_params.regex} - elif guided_decoding_params.grammar is not None: - content = {"type": "grammar", "grammar": guided_decoding_params.grammar} - - if reasoning_parser == "gpt_oss": - # Trigger user constraint by final channel - stag_format = { - "type": - "triggered_tags", - "triggers": ["<|start|>assistant<|channel|>final<|message|>"], - "tags": [ - { - "begin": "<|start|>assistant<|channel|>final<|message|>", - "content": content, - "end": "", - }, - ], - "stop_after_first": - True, - } - else: - # Force thinking and then trigger user constraint - parser = ReasoningParserFactory.create_reasoning_parser( - reasoning_parser) - stag_format = { - "type": - "sequence", - "elements": [ - { - "type": "tag", - "begin": parser.reasoning_start, - "content": { - "type": "any_text" - }, - "end": parser.reasoning_end, - }, - content, - ], - } - - stag_format = ResponseFormat(type="structural_tag", format=stag_format) - return GuidedDecodingParams(structural_tag=stag_format.model_dump_json( - by_alias=True, exclude_none=True)) + return adapt_guided_decoding_params_for_reasoning_parser( + guided_decoding_params, reasoning_parser) def _response_format_text_config_to_guided_decoding_params( diff --git a/tests/unittest/evaluate/test_generation_content.py b/tests/unittest/evaluate/test_generation_content.py new file mode 100644 index 000000000000..5da815222267 --- /dev/null +++ b/tests/unittest/evaluate/test_generation_content.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import types +from types import SimpleNamespace + +from tensorrt_llm.evaluate.interface import extract_final_content_from_generation +from tensorrt_llm.llmapi.reasoning_parser import HARMONY_REASONING_PARSER + + +def _request_output(text: str, token_ids: list[int] | None = None): + completion = SimpleNamespace(text=text, token_ids=token_ids or []) + return SimpleNamespace(outputs=[completion]) + + +def test_extract_final_content_does_not_guess_reasoning_parser(): + text = 'scratch{"answer": "final"}' + output = _request_output(text) + + assert extract_final_content_from_generation(output) == text + + +def test_extract_final_content_uses_explicit_reasoning_parser(): + output = _request_output('scratch{"answer": "final"}') + + assert ( + extract_final_content_from_generation(output, reasoning_parser="qwen3") + == '{"answer": "final"}' + ) + + +def test_extract_final_content_non_harmony_parser_ignores_harmony_tokens(monkeypatch): + harmony_module = types.ModuleType("tensorrt_llm.serve.harmony_adapter") + + def _unexpected_adapter_lookup(): + raise AssertionError("Harmony adapter should not be used for qwen3") + + harmony_module.get_harmony_adapter = _unexpected_adapter_lookup + monkeypatch.setitem(sys.modules, "tensorrt_llm.serve.harmony_adapter", harmony_module) + output = _request_output('scratch{"answer": "final"}', token_ids=[1, 2, 3]) + + assert ( + extract_final_content_from_generation(output, reasoning_parser="qwen3") + == '{"answer": "final"}' + ) + + +def test_extract_final_content_does_not_guess_harmony_from_tokens(monkeypatch): + harmony_module = types.ModuleType("tensorrt_llm.serve.harmony_adapter") + + def _unexpected_adapter_lookup(): + raise AssertionError("Harmony must be selected from model context") + + harmony_module.get_harmony_adapter = _unexpected_adapter_lookup + monkeypatch.setitem(sys.modules, "tensorrt_llm.serve.harmony_adapter", harmony_module) + output = _request_output("not json", token_ids=[1, 2, 3]) + + assert extract_final_content_from_generation(output) == "not json" + + +def test_extract_final_content_uses_harmony_tokens(monkeypatch): + harmony_module = types.ModuleType("tensorrt_llm.serve.harmony_adapter") + + class _FakeHarmonyAdapter: + def harmony_output_to_openai(self, token_ids): + assert token_ids == [1, 2, 3] + return {"content": '{"answer": "harmony"}'} + + harmony_module.get_harmony_adapter = lambda: _FakeHarmonyAdapter() + monkeypatch.setitem(sys.modules, "tensorrt_llm.serve.harmony_adapter", harmony_module) + output = _request_output("raw harmony transcript", token_ids=[1, 2, 3]) + + assert ( + extract_final_content_from_generation(output, reasoning_parser=HARMONY_REASONING_PARSER) + == '{"answer": "harmony"}' + ) diff --git a/tests/unittest/llmapi/test_sampling_params.py b/tests/unittest/llmapi/test_sampling_params.py index 889a8abcee9a..61e2afa5a9f7 100644 --- a/tests/unittest/llmapi/test_sampling_params.py +++ b/tests/unittest/llmapi/test_sampling_params.py @@ -20,11 +20,23 @@ import torch from tensorrt_llm.llmapi.llm import BaseLLM +from tensorrt_llm.llmapi.reasoning_parser import ( + HARMONY_FINAL_CHANNEL_TRIGGER, + HARMONY_REASONING_PARSER, + adapt_guided_decoding_params_for_reasoning_parser, + resolve_guided_decoding_reasoning_parser, + resolve_raw_guided_decoding_reasoning_parser, +) from tensorrt_llm.llmapi.thinking_budget import ( ThinkingBudgetLogitsProcessor, add_thinking_budget_logits_processor, ) -from tensorrt_llm.sampling_params import MAX_TOP_LOGPROBS, SamplingParams, check_logprobs_limit +from tensorrt_llm.sampling_params import ( + MAX_TOP_LOGPROBS, + GuidedDecodingParams, + SamplingParams, + check_logprobs_limit, +) from tensorrt_llm.serve.openai_protocol import ( ChatCompletionRequest, CompletionRequest, @@ -201,6 +213,164 @@ def test_chat_top_logprobs_request_limit(): ) +def test_harmony_guided_decoding_triggers_on_final_channel(): + guided_decoding = GuidedDecodingParams(json={"type": "object"}) + + adapted = adapt_guided_decoding_params_for_reasoning_parser( + guided_decoding, HARMONY_REASONING_PARSER + ) + + assert adapted is not guided_decoding + assert adapted.structural_tag is not None + stag = json.loads(adapted.structural_tag) + assert stag["type"] == "structural_tag" + + fmt = stag["format"] + assert fmt["type"] == "triggered_tags" + assert fmt["triggers"] == [HARMONY_FINAL_CHANNEL_TRIGGER] + assert fmt["stop_after_first"] is True + + tag = fmt["tags"][0] + assert tag["begin"] == HARMONY_FINAL_CHANNEL_TRIGGER + assert tag["end"] == "" + assert tag["content"] == { + "type": "json_schema", + "json_schema": {"type": "object"}, + } + + +def test_harmony_guided_decoding_accepts_json_schema_string(): + json_schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + } + guided_decoding = GuidedDecodingParams(json=json.dumps(json_schema)) + + adapted = adapt_guided_decoding_params_for_reasoning_parser( + guided_decoding, HARMONY_REASONING_PARSER + ) + + stag = json.loads(adapted.structural_tag) + content = stag["format"]["tags"][0]["content"] + assert content == { + "type": "json_schema", + "json_schema": json_schema, + } + + +def test_guided_decoding_preserves_top_level_schema_property(): + json_schema = { + "type": "object", + "schema": {"type": "string"}, + } + + adapted = adapt_guided_decoding_params_for_reasoning_parser( + GuidedDecodingParams(json=json_schema), HARMONY_REASONING_PARSER + ) + + stag = json.loads(adapted.structural_tag) + assert stag["format"]["tags"][0]["content"]["json_schema"] == json_schema + + +@pytest.mark.parametrize( + ("reasoning_parser", "model_type", "expected"), + [ + (None, "gpt_oss", HARMONY_REASONING_PARSER), + (None, "llama", None), + ("qwen3", "llama", "qwen3"), + ], +) +def test_resolve_guided_decoding_reasoning_parser(reasoning_parser, model_type, expected): + assert resolve_guided_decoding_reasoning_parser(reasoning_parser, model_type) == expected + + +@pytest.mark.parametrize( + ("reasoning_parser", "model_type", "guided_backend", "expected"), + [ + (None, "gpt_oss", "xgrammar", HARMONY_REASONING_PARSER), + (HARMONY_REASONING_PARSER, "llama", "xgrammar", HARMONY_REASONING_PARSER), + (None, "gpt_oss", "llguidance", None), + ("qwen3", "qwen3", "xgrammar", None), + ("gemma4", "gemma4", "xgrammar", None), + ], +) +def test_resolve_raw_guided_decoding_reasoning_parser( + reasoning_parser, model_type, guided_backend, expected +): + assert ( + resolve_raw_guided_decoding_reasoning_parser(reasoning_parser, model_type, guided_backend) + == expected + ) + + +@pytest.mark.parametrize( + ("reasoning_parser", "model_type", "guided_backend"), + [ + (None, "gpt_oss", "llguidance"), + ("qwen3", "qwen3", "xgrammar"), + ("gemma4", "gemma4", "xgrammar"), + ], +) +def test_raw_llm_preserves_guides_outside_harmony_xgrammar( + reasoning_parser, model_type, guided_backend +): + guided_decoding = GuidedDecodingParams(json_object=True) + resolved_parser = resolve_raw_guided_decoding_reasoning_parser( + reasoning_parser, model_type, guided_backend + ) + + # These combinations worked without raw final-content adaptation before + # bug 6284101; keep the caller's ordinary guide unchanged. + assert ( + adapt_guided_decoding_params_for_reasoning_parser(guided_decoding, resolved_parser) + is guided_decoding + ) + + +def test_plain_model_guided_decoding_is_unchanged(): + guided_decoding = GuidedDecodingParams(json_object=True) + reasoning_parser = resolve_guided_decoding_reasoning_parser(None, "llama") + + assert ( + adapt_guided_decoding_params_for_reasoning_parser(guided_decoding, reasoning_parser) + is guided_decoding + ) + + +def test_reasoning_parser_guided_decoding_uses_sequence_for_normal_parser(): + guided_decoding = GuidedDecodingParams(json_object=True) + + adapted = adapt_guided_decoding_params_for_reasoning_parser(guided_decoding, "qwen3") + + stag = json.loads(adapted.structural_tag) + assert stag["type"] == "structural_tag" + + fmt = stag["format"] + assert fmt["type"] == "sequence" + reasoning_element, content_element = fmt["elements"] + assert reasoning_element == { + "type": "tag", + "begin": "", + "content": {"type": "any_text"}, + "end": "", + } + assert content_element == { + "type": "json_schema", + "json_schema": {"type": "object"}, + } + + +def test_existing_structural_tag_guided_decoding_is_unchanged(): + guided_decoding = GuidedDecodingParams( + structural_tag='{"type":"structural_tag","format":{"type":"any_text"}}' + ) + + assert ( + adapt_guided_decoding_params_for_reasoning_parser(guided_decoding, HARMONY_REASONING_PARSER) + is guided_decoding + ) + + def test_chat_template_request_override_respects_runtime_policy(): request = ChatCompletionRequest( model="test", From 18e774894693c97a8ebd9b3662d0b43b3521b738 Mon Sep 17 00:00:00 2001 From: Dongfeng Yu Date: Tue, 11 Aug 2026 18:00:38 -0700 Subject: [PATCH 2/2] Enable Harmony regression coverage in CI Signed-off-by: Dongfeng Yu --- tests/integration/test_lists/test-db/l0_cpu.yml | 1 + tests/integration/test_lists/waives.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 1a03cf840fbc..2532426ad983 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -39,6 +39,7 @@ l0_cpu: - unittest/executor/test_event_loop_error_broadcast.py - unittest/executor/test_stats_serializer.py - unittest/executor/test_spec_dec_perf_metrics.py + - unittest/evaluate/test_generation_content.py - unittest/inputs - unittest/llmapi/apps/test_chat_utils.py - unittest/llmapi/apps/test_harmony_channel_validation.py diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 13c717be49ba..60737dd94af8 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -56,7 +56,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backe accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_auto_dtype SKIP (https://nvbugs/6561677) accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm SKIP (https://nvbugs/6581063) accuracy/test_llm_api_pytorch.py::TestGLM52::test_nvfp4[tp_size=8-ep_size=8] SKIP (https://nvbugs/6507108) -accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_guided_decoding_4gpus[one_model] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/6026676) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-fp8] SKIP (https://nvbugs/5651865)