diff --git a/renderers/base.py b/renderers/base.py index e8c9f22..0c5afc3 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -1819,6 +1819,7 @@ def attribute_text_segments( segments: "list[tuple[str, bool]]", *, overlap_is_content: bool = False, + split_special_tokens: bool = True, ) -> "list[tuple[int, bool]]": """Tokenize concatenated segments as a single BPE pass and return ``(token_id, is_content)`` pairs. @@ -1867,6 +1868,7 @@ def attribute_text_segments( encoding = offset_tokenizer( full_text, add_special_tokens=False, + split_special_tokens=split_special_tokens, return_offsets_mapping=True, ) token_ids = list(encoding["input_ids"]) diff --git a/renderers/deepseek_v3.py b/renderers/deepseek_v3.py index a00f1f2..29ec2e3 100644 --- a/renderers/deepseek_v3.py +++ b/renderers/deepseek_v3.py @@ -110,7 +110,9 @@ def _get_special_token(self, name: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Public API diff --git a/renderers/glm45.py b/renderers/glm45.py index bfc5f09..d93811f 100644 --- a/renderers/glm45.py +++ b/renderers/glm45.py @@ -158,8 +158,10 @@ def emit_text_segments( same way as the chat template, but attributed separately" without splitting the encode call (which could shift BPE merges at the boundary).""" + # split_special_tokens=False: GLM's template reads /nothink out of + # user content, so content must keep matching special tokens here. for tok_id, is_content in attribute_text_segments( - self._tokenizer, segments + self._tokenizer, segments, split_special_tokens=False ): tokens.append(tok_id) indices.append(msg_idx) @@ -381,8 +383,10 @@ def emit_text_segments( *, is_sampled: bool = False, ) -> None: + # split_special_tokens=False: GLM's template reads /nothink out of + # user content, so content must keep matching special tokens here. for tok_id, is_content in attribute_text_segments( - self._tokenizer, segments + self._tokenizer, segments, split_special_tokens=False ): ext.append(tok_id) ext_indices.append(msg_idx) diff --git a/renderers/glm5.py b/renderers/glm5.py index 4f34d98..2594b4e 100644 --- a/renderers/glm5.py +++ b/renderers/glm5.py @@ -108,7 +108,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/gpt_oss.py b/renderers/gpt_oss.py index 6165ed0..8e4c21c 100644 --- a/renderers/gpt_oss.py +++ b/renderers/gpt_oss.py @@ -172,7 +172,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _prefix_content_mask( self, diff --git a/renderers/hy3.py b/renderers/hy3.py index 7eaef65..7d97b35 100644 --- a/renderers/hy3.py +++ b/renderers/hy3.py @@ -150,7 +150,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/kimi_k2.py b/renderers/kimi_k2.py index 7337600..d14452b 100644 --- a/renderers/kimi_k2.py +++ b/renderers/kimi_k2.py @@ -78,7 +78,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _ensure_system_message( self, messages: list[Message] diff --git a/renderers/kimi_k25.py b/renderers/kimi_k25.py index 48ea426..7b050ee 100644 --- a/renderers/kimi_k25.py +++ b/renderers/kimi_k25.py @@ -721,7 +721,9 @@ def _try_token_id(self, token: str) -> int | None: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Core render diff --git a/renderers/laguna_xs2.py b/renderers/laguna_xs2.py index bd174f4..984e4db 100644 --- a/renderers/laguna_xs2.py +++ b/renderers/laguna_xs2.py @@ -138,7 +138,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Content | None) -> str: diff --git a/renderers/llama_3.py b/renderers/llama_3.py index d18d8c8..40f3381 100644 --- a/renderers/llama_3.py +++ b/renderers/llama_3.py @@ -130,7 +130,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _content_str(content: Any) -> str: diff --git a/renderers/minimax_m2.py b/renderers/minimax_m2.py index a7f0bc7..a3fd556 100644 --- a/renderers/minimax_m2.py +++ b/renderers/minimax_m2.py @@ -85,7 +85,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _visible_text(content: Any) -> str: diff --git a/renderers/nemotron3.py b/renderers/nemotron3.py index 5cc76c9..3cf4c7c 100644 --- a/renderers/nemotron3.py +++ b/renderers/nemotron3.py @@ -172,7 +172,9 @@ def _token_id(self, token: str, *, optional: bool = False) -> int | None: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Content rendering diff --git a/renderers/qwen3.py b/renderers/qwen3.py index d85d161..9dd8412 100644 --- a/renderers/qwen3.py +++ b/renderers/qwen3.py @@ -88,7 +88,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) @staticmethod def _query_boundary_text(content) -> str: diff --git a/renderers/qwen35.py b/renderers/qwen35.py index 52de886..4f34ccb 100644 --- a/renderers/qwen35.py +++ b/renderers/qwen35.py @@ -239,7 +239,9 @@ def _token_id(self, token: str) -> int: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) # ------------------------------------------------------------------ # Content rendering (mirrors the render_content Jinja macro) diff --git a/renderers/qwen3_vl.py b/renderers/qwen3_vl.py index 97072d2..572f192 100644 --- a/renderers/qwen3_vl.py +++ b/renderers/qwen3_vl.py @@ -370,7 +370,9 @@ def mm_token_type_id_map(self) -> dict[int, int]: def _encode(self, text: str) -> list[int]: if not text: return [] - return self._tokenizer.encode(text, add_special_tokens=False) + return self._tokenizer.encode( + text, add_special_tokens=False, split_special_tokens=True + ) def _get_processor(self): if self._processor is not None: