From 54c8991885c5c6c4cf7cb8b7b7fa0f47397b61d1 Mon Sep 17 00:00:00 2001 From: pos-ei-don Date: Sat, 29 Aug 2026 18:53:54 +0200 Subject: [PATCH] fix(sglang): force reasoning when the template prefills the think tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3-style chat templates append the opening tag to the *prompt* when thinking is enabled. The model therefore never generates it and emits only the reasoning text plus the closing . sglang's ReasoningParser keys off the opening tag: in_reasoning = self._in_reasoning or self.think_start_token in text if not in_reasoning: return StreamingParseResult(normal_text=text) so with such a template the entire completion — reasoning and answer, the raw in between — is returned as content and reasoning_content stays empty, no matter how reasoning_parser is configured. sglang's own OpenAI server handles this via force_reasoning = (self.template_manager.force_reasoning or self._get_reasoning_from_request(request)) This backend has no template manager, so derive the same signal from the rendered prompt: if it ends with the detector's think_start_token, the tag was prefilled and the parser is constructed with force_reasoning=True. Structured decoding is the exception, and it matters: a grammar applies from the first token, so the model cannot emit the closing tag even though the template opened the block. The whole completion is schema output and belongs in content — forcing there files it as reasoning and returns an empty answer. Measured against a JSON-schema code audit: 10107 characters of "reasoning", zero content. sglang's own server keeps the two apart for the same reason; its grammar backend owns the reasoning prefix when a reasoning parser is configured. force_reasoning is only passed when it is meant to be True, so detector defaults (DeepSeek-R1 already defaults to True) are untouched, and a prompt without a prefilled tag behaves exactly as before — which matters, because forcing unconditionally makes an answer generated with thinking off disappear into reasoning_content. The construction is factored into _new_reasoning_parser() so the streaming and non-streaming paths, which previously built the parser separately, cannot drift apart. Signed-off-by: pos-ei-don <1822533+pos-ei-don@users.noreply.github.com> --- backend/python/sglang/backend.py | 86 ++++++++++++++++++++++++-------- backend/python/sglang/test.py | 52 +++++++++++++++++++ 2 files changed, 118 insertions(+), 20 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index 76d99a726aaf..84c122cc0574 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -374,10 +374,67 @@ def _build_prompt(self, request) -> str: messages_dicts, tokenize=False, add_generation_prompt=True, ) - def _make_parsers(self, request): + def _new_reasoning_parser(self, stream_reasoning: bool, prompt: str = "", + grammar_constrained: bool = False): + """Build a ReasoningParser for one request, or None. + + Reasoning templates come in two flavours. Some let the model emit the + opening tag, others put it into the *prompt* — Qwen3's template appends + ```` when thinking is on, so the completion starts straight in + the reasoning block and only the closing ```` ever shows up. + sglang's detector keys off the opening tag, so in that second case it + classifies the whole completion as normal content and + ``reasoning_content`` stays empty. + + sglang's own OpenAI server covers this with + ``template_manager.force_reasoning``; this backend has no template + manager, so it derives the same signal from the rendered prompt. + ``force_reasoning`` is only passed when we mean True, leaving detector + defaults (e.g. DeepSeek-R1's built-in True) untouched. + + ``grammar_constrained`` suppresses the prefill heuristic. A structured + decoding constraint applies from the first token, so the model cannot + emit the closing tag even though the template opened the block: the + whole completion is schema output and belongs in ``content``. Forcing + there files the answer as reasoning and leaves content empty. sglang's + own server keeps the two apart for the same reason — its grammar + backend owns the reasoning prefix when a reasoning parser is set. + """ + if grammar_constrained: + prompt = "" + + if not (HAS_REASONING_PARSERS and self.reasoning_parser_name): + return None + + kwargs = { + "model_type": self.reasoning_parser_name, + "stream_reasoning": stream_reasoning, + } + try: + parser = ReasoningParser(**kwargs) + except Exception as e: + print(f"ReasoningParser init failed: {e!r}", file=sys.stderr) + return None + + start = getattr(getattr(parser, "detector", None), "think_start_token", None) + if start and prompt and prompt.rstrip().endswith(start): + try: + parser = ReasoningParser(force_reasoning=True, **kwargs) + except TypeError: + # sglang without the force_reasoning kwarg: keep the default + # parser rather than failing the request. + pass + except Exception as e: + print( + f"ReasoningParser(force_reasoning=True) failed: {e!r}", + file=sys.stderr, + ) + + return parser + + def _make_parsers(self, request, prompt: str = ""): """Construct fresh per-request parser instances (stateful).""" tool_parser = None - reasoning_parser = None if HAS_TOOL_PARSERS and self.tool_parser_name and request.Tools: try: @@ -389,14 +446,9 @@ def _make_parsers(self, request): except Exception as e: print(f"FunctionCallParser init failed: {e!r}", file=sys.stderr) - if HAS_REASONING_PARSERS and self.reasoning_parser_name: - try: - reasoning_parser = ReasoningParser( - model_type=self.reasoning_parser_name, - stream_reasoning=True, - ) - except Exception as e: - print(f"ReasoningParser init failed: {e!r}", file=sys.stderr) + reasoning_parser = self._new_reasoning_parser( + True, prompt, bool(getattr(request, "Grammar", "")), + ) return tool_parser, reasoning_parser @@ -404,7 +456,7 @@ async def _predict(self, request, context, streaming: bool = False): sampling_params = self._build_sampling_params(request) prompt = self._build_prompt(request) - tool_parser, reasoning_parser = self._make_parsers(request) + tool_parser, reasoning_parser = self._make_parsers(request, prompt) image_data = list(request.Images) if request.Images else None video_data = list(request.Videos) if request.Videos else None @@ -500,15 +552,9 @@ async def _predict(self, request, context, streaming: bool = False): final_tool_calls: List[backend_pb2.ToolCallDelta] = [] if not streaming: - final_reasoning_parser = None - if HAS_REASONING_PARSERS and self.reasoning_parser_name: - try: - final_reasoning_parser = ReasoningParser( - model_type=self.reasoning_parser_name, - stream_reasoning=False, - ) - except Exception: - final_reasoning_parser = None + final_reasoning_parser = self._new_reasoning_parser( + False, prompt, bool(getattr(request, "Grammar", "")), + ) if final_reasoning_parser is not None: try: diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index c50ed577f941..b64ee503200c 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -128,6 +128,58 @@ def kwargs_for(metadata): self.assertNotIn("enable_thinking", kwargs_for({})) self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + def test_reasoning_parser_forced_when_template_prefills_think_tag(self): + """Qwen3's template puts ```` in the prompt, so the completion + never contains it. Without force_reasoning the detector treats the whole + completion as normal text and reasoning_content stays empty.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + # What the model actually emits when the prompt ends in "". + completion = "adding two and two4" + + forced = servicer._new_reasoning_parser(False, prompt="user: hi\n\n") + reasoning, content = forced.parse_non_stream(completion) + self.assertEqual(reasoning, "adding two and two") + self.assertEqual(content, "4") + + # No prefilled tag in the prompt: detector default, unchanged behaviour. + unforced = servicer._new_reasoning_parser(False, prompt="user: hi\n") + reasoning, content = unforced.parse_non_stream(completion) + self.assertFalse(reasoning) + self.assertEqual(content, completion) + + def test_reasoning_parser_not_forced_when_thinking_is_off(self): + """Thinking off means no ```` in the prompt either, so the answer + must not be swallowed into reasoning_content.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + parser = servicer._new_reasoning_parser(False, prompt="user: primes?\n") + reasoning, content = parser.parse_non_stream("2,3,5,7,11") + self.assertFalse(reasoning) + self.assertEqual(content, "2,3,5,7,11") + + def test_grammar_constrained_output_is_not_forced_into_reasoning(self): + """Structured decoding applies from the first token, so the model cannot + emit the closing tag even though the template opened the block. The whole + completion is schema output and must stay in content.""" + servicer = self._servicer() + servicer.reasoning_parser_name = "qwen3" + + schema_out = '{"findings": [{"line": 42, "issue": "off-by-one"}]}' + parser = servicer._new_reasoning_parser( + False, prompt="audit this\n\n", grammar_constrained=True, + ) + reasoning, content = parser.parse_non_stream(schema_out) + self.assertFalse(reasoning) + self.assertEqual(content, schema_out) + + def test_reasoning_parser_absent_without_configured_parser(self): + servicer = self._servicer() + servicer.reasoning_parser_name = None + self.assertIsNone(servicer._new_reasoning_parser(False, prompt="")) + def test_explicit_zero_temperature_is_preserved(self): """Temperature=0 is valid greedy decoding, not an unset value.""" from types import SimpleNamespace