From 83d20bc5be30f9167ff174ee66d2eda3910b1666 Mon Sep 17 00:00:00 2001 From: hubert Date: Wed, 5 Aug 2026 07:22:32 +0000 Subject: [PATCH 1/2] fix: encode message content with split_special_tokens=True MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add_special_tokens=False only suppresses BOS/EOS; a special token spelled out inside caller-supplied text is still matched and collapsed to its reserved id, so message content can synthesize control tokens. Observed: a coding-agent trace whose tool output documents a path-template DSL using <...> for required entity values ("{type|bold}") emits the image placeholder id. On a VLM the model compares placeholder count against vision features before masked_scatter, so one stray token in a text-only row packed alongside image rows gives image_tokens=N+1 vs image_features=N and kills the job. On text-only models the same injection is silent. Covers both encode paths: the 14 renderer _encode methods and attribute_text_segments in base.py (segment attribution). Scaffold is unaffected — chat markers, / and thinking tags are emitted by resolved token id, not by encoding a marker string. Co-Authored-By: Claude Opus 5 (1M context) --- renderers/base.py | 1 + renderers/deepseek_v3.py | 4 +++- renderers/glm45.py | 4 +++- renderers/glm5.py | 4 +++- renderers/gpt_oss.py | 4 +++- renderers/hy3.py | 4 +++- renderers/kimi_k2.py | 4 +++- renderers/kimi_k25.py | 4 +++- renderers/laguna_xs2.py | 4 +++- renderers/llama_3.py | 4 +++- renderers/minimax_m2.py | 4 +++- renderers/nemotron3.py | 4 +++- renderers/qwen3.py | 4 +++- renderers/qwen35.py | 4 +++- renderers/qwen3_vl.py | 4 +++- 15 files changed, 43 insertions(+), 14 deletions(-) diff --git a/renderers/base.py b/renderers/base.py index e8c9f228..54b028b9 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -1867,6 +1867,7 @@ def attribute_text_segments( encoding = offset_tokenizer( full_text, add_special_tokens=False, + split_special_tokens=True, return_offsets_mapping=True, ) token_ids = list(encoding["input_ids"]) diff --git a/renderers/deepseek_v3.py b/renderers/deepseek_v3.py index a00f1f20..29ec2e3d 100644 --- a/renderers/deepseek_v3.py +++ b/renderers/deepseek_v3.py @@ -110,7 +110,9 @@ def _get_special_token(self, name: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Public API diff --git a/renderers/glm45.py b/renderers/glm45.py index bfc5f09c..8a1262ab 100644 --- a/renderers/glm45.py +++ b/renderers/glm45.py @@ -91,7 +91,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/glm5.py b/renderers/glm5.py index 4f34d98f..2594b4ee 100644 --- a/renderers/glm5.py +++ b/renderers/glm5.py @@ -108,7 +108,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/gpt_oss.py b/renderers/gpt_oss.py index 6165ed09..8e4c21c4 100644 --- a/renderers/gpt_oss.py +++ b/renderers/gpt_oss.py @@ -172,7 +172,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _prefix_content_mask( self, diff --git a/renderers/hy3.py b/renderers/hy3.py index 7eaef656..7d97b35a 100644 --- a/renderers/hy3.py +++ b/renderers/hy3.py @@ -150,7 +150,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/kimi_k2.py b/renderers/kimi_k2.py index 73376003..d14452b5 100644 --- a/renderers/kimi_k2.py +++ b/renderers/kimi_k2.py @@ -78,7 +78,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _ensure_system_message( self, messages: list[Message] diff --git a/renderers/kimi_k25.py b/renderers/kimi_k25.py index 48ea426d..7b050ee0 100644 --- a/renderers/kimi_k25.py +++ b/renderers/kimi_k25.py @@ -721,7 +721,9 @@ def _try_token_id(self, token: str) -> int | None: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Core render diff --git a/renderers/laguna_xs2.py b/renderers/laguna_xs2.py index bd174f42..984e4dbf 100644 --- a/renderers/laguna_xs2.py +++ b/renderers/laguna_xs2.py @@ -138,7 +138,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Content | None) -> str: diff --git a/renderers/llama_3.py b/renderers/llama_3.py index d18d8c87..40f33819 100644 --- a/renderers/llama_3.py +++ b/renderers/llama_3.py @@ -130,7 +130,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _content_str(content: Any) -> str: diff --git a/renderers/minimax_m2.py b/renderers/minimax_m2.py index a7f0bc70..a3fd5563 100644 --- a/renderers/minimax_m2.py +++ b/renderers/minimax_m2.py @@ -85,7 +85,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/nemotron3.py b/renderers/nemotron3.py index 5cc76c91..3cf4c7c6 100644 --- a/renderers/nemotron3.py +++ b/renderers/nemotron3.py @@ -172,7 +172,9 @@ def _token_id(self, token: str, *, optional: bool = False) -> int | None: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Content rendering diff --git a/renderers/qwen3.py b/renderers/qwen3.py index d85d161d..9dd84128 100644 --- a/renderers/qwen3.py +++ b/renderers/qwen3.py @@ -88,7 +88,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _query_boundary_text(content) -> str: diff --git a/renderers/qwen35.py b/renderers/qwen35.py index 52de8867..4f34ccbb 100644 --- a/renderers/qwen35.py +++ b/renderers/qwen35.py @@ -239,7 +239,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Content rendering (mirrors the render_content Jinja macro) diff --git a/renderers/qwen3_vl.py b/renderers/qwen3_vl.py index 97072d21..572f1927 100644 --- a/renderers/qwen3_vl.py +++ b/renderers/qwen3_vl.py @@ -370,7 +370,9 @@ def mm_token_type_id_map(self) -> dict[int, int]: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _get_processor(self): if self._processor is not None: From 23a702930cf8ad001cd6dc5ab874801345ad9de7 Mon Sep 17 00:00:00 2001 From: hubert Date: Wed, 5 Aug 2026 07:37:18 +0000 Subject: [PATCH 2/2] fix: exclude GLM-4.5-Air from split_special_tokens GLM's template appends /nothink into user content and relies on that string collapsing to special token 151360; users may also type it themselves. Applying split_special_tokens broke 7 parity shapes (enable_thinking=False). Reading control out of content is upstream GLM's design and this library's contract is byte-parity with apply_chat_template, so glm45 keeps the old behavior at all three encode points. attribute_text_segments gains a split_special_tokens parameter (default True) for the opt-out. Co-Authored-By: Claude Opus 5 (1M context) --- renderers/base.py | 3 ++- renderers/glm45.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/renderers/base.py b/renderers/base.py index 54b028b9..0c5afc35 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -1819,6 +1819,7 @@ def attribute_text_segments( segments: "list[tuple[str, bool]]", *, overlap_is_content: bool = False, + split_special_tokens: bool = True, ) -> "list[tuple[int, bool]]": """Tokenize concatenated segments as a single BPE pass and return ``(token_id, is_content)`` pairs. @@ -1867,7 +1868,7 @@ def attribute_text_segments( encoding = offset_tokenizer( full_text, add_special_tokens=False, - split_special_tokens=True, + split_special_tokens=split_special_tokens, return_offsets_mapping=True, ) token_ids = list(encoding["input_ids"]) diff --git a/renderers/glm45.py b/renderers/glm45.py index 8a1262ab..d93811fa 100644 --- a/renderers/glm45.py +++ b/renderers/glm45.py @@ -91,9 +91,7 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode( - text, add_special_tokens=False, split_special_tokens=True - ) + return self._tokenizer.encode(text, add_special_tokens=False) @staticmethod def _visible_text(content: Any) -> str: @@ -160,8 +158,10 @@ def emit_text_segments( same way as the chat template, but attributed separately" without splitting the encode call (which could shift BPE merges at the boundary).""" + # split_special_tokens=False: GLM's template reads /nothink out of + # user content, so content must keep matching special tokens here. for tok_id, is_content in attribute_text_segments( - self._tokenizer, segments + self._tokenizer, segments, split_special_tokens=False ): tokens.append(tok_id) indices.append(msg_idx) @@ -383,8 +383,10 @@ def emit_text_segments( *, is_sampled: bool = False, ) -> None: + # split_special_tokens=False: GLM's template reads /nothink out of + # user content, so content must keep matching special tokens here. for tok_id, is_content in attribute_text_segments( - self._tokenizer, segments + self._tokenizer, segments, split_special_tokens=False ): ext.append(tok_id) ext_indices.append(msg_idx)