From 3bd9f6e927e82f8965ae9bd22a62025848b5d891 Mon Sep 17 00:00:00 2001
From: hallerite <git@hallerite.com>
Date: Sat, 8 Aug 2026 13:37:50 -0700
Subject: [PATCH 1/2] Make Transformers optional

---
 README.md                        |  23 ++++-
 pyproject.toml                   |  12 ++-
 renderers/__init__.py            |  15 ++-
 renderers/base.py                | 155 ++++++++++++++++++++++++-------
 renderers/client.py              |   5 +-
 renderers/deepseek_v3.py         |   4 +-
 renderers/default.py             |  11 ++-
 renderers/glm45.py               |   4 +-
 renderers/glm5.py                |   4 +-
 renderers/gpt_oss.py             |   4 +-
 renderers/hy3.py                 |   4 +-
 renderers/kimi_k2.py             |   4 +-
 renderers/kimi_k25.py            |  29 +++++-
 renderers/laguna_xs2.py          |   8 +-
 renderers/llama_3.py             |   4 +-
 renderers/minimax_m2.py          |   4 +-
 renderers/nemotron3.py           |   4 +-
 renderers/prime_qwen3.py         |   6 +-
 renderers/qwen3.py               |   4 +-
 renderers/qwen35.py              |  14 ++-
 renderers/qwen3_vl.py            |  14 ++-
 renderers/tokenizer.py           | 130 ++++++++++++++++++++++++++
 tests/test_renderer_config.py    |  10 +-
 tests/test_tokenizers_backend.py | 127 +++++++++++++++++++++++++
 uv.lock                          | 109 ++++++++++++----------
 25 files changed, 573 insertions(+), 135 deletions(-)
 create mode 100644 renderers/tokenizer.py
 create mode 100644 tests/test_tokenizers_backend.py

diff --git a/README.md b/README.md
index 346f963b..eb234030 100644
--- a/README.md
+++ b/README.md
@@ -10,13 +10,21 @@ Standalone on PyPI, and portable across training and inference stacks (transform
 uv add renderers
 ```
 
+The core install uses the standalone Rust `tokenizers` package. Add the
+optional Hugging Face integration only when you need an unknown model's
+`apply_chat_template`, a custom remote-code tokenizer such as Kimi's, or
+automatic multimodal processor loading:
+
+```bash
+uv add 'renderers[hf]'
+```
+
 ## At a glance
 
 ```python
-from transformers import AutoTokenizer
-from renderers import create_renderer
+from renderers import create_renderer, load_tokenizer
 
-tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")
+tok = load_tokenizer("Qwen/Qwen3-8B")
 r = create_renderer(tok)                            # → Qwen3Renderer (auto-resolved)
 
 prompt_ids = r.render_ids(
@@ -40,7 +48,7 @@ next_prompt_ids = r.bridge_to_next_turn(
 )
 ```
 
-Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `laguna-xs.2`, `laguna-xs-2.1`, `laguna-m.1`, `nemotron-3`, `nemotron-3-ultra`, `llama-3`, `gpt-oss`, `hy3`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper.
+Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `laguna-xs.2`, `laguna-xs-2.1`, `laguna-m.1`, `nemotron-3`, `nemotron-3-ultra`, `llama-3`, `gpt-oss`, `hy3`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper that requires `renderers[hf]` (or a compatible caller-supplied tokenizer).
 
 ## API
 
@@ -86,7 +94,12 @@ with pool.checkout() as r:
     ids = r.render_ids(messages)
 ```
 
-Each slot owns its own tokenizer copy. Construction fans out across a thread pool so a 32-slot pool doesn't serially eat ~10–15s of `from_pretrained` calls at startup.
+Each slot owns its own tokenizer copy. Core installs load `tokenizer.json` with
+the standalone `tokenizers` backend. If `renderers[hf]` is installed,
+`load_tokenizer(..., backend="auto")` preserves the Hugging Face wrapper for
+backward compatibility; pass `backend="tokenizers"` to select the lean backend
+explicitly. Construction fans out across a thread pool so a 32-slot pool
+doesn't serially eat ~10–15s of `from_pretrained` calls at startup.
 
 ## Why use a renderer
 
diff --git a/pyproject.toml b/pyproject.toml
index dc753970..61f23da3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,7 +19,10 @@ dependencies = [
     "openai>=1.108.1",
     "tiktoken",
     "jinja2",
-    "transformers>=4.50.0",
+    # Transformers 5.14.1 caps tokenizers at 0.23.0; 0.23.1 exceeds that cap,
+    # so 0.22.2 is the newest published compatible Rust backend. This keeps
+    # ``renderers[hf]`` resolvable.
+    "tokenizers>=0.22.2,<0.23",
     # Used by GptOssRenderer to render and parse harmony tokens. Vendoring
     # OpenAI's reference implementation keeps us byte-identical with vLLM
     # (which also uses it) and saves us mirroring a 330-line Jinja template.
@@ -38,6 +41,12 @@ dependencies = [
     "prime-pydantic-config>=0.3.0.dev83",
 ]
 
+[project.optional-dependencies]
+# Opaque HF chat templates, custom remote-code tokenizers, model-config
+# probing, and multimodal AutoProcessor loading. Model-specific renderers use
+# the standalone Rust ``tokenizers`` backend and do not need this extra.
+hf = ["transformers>=5.14.1,<6"]
+
 [tool.hatch.version]
 source = "vcs"
 # Tags look like ``renderers-v0.1.8`` (prefix matches the publish.yml
@@ -85,6 +94,7 @@ dev = [
     "ruff",
     "torch>=2.11.0",
     "torchvision>=0.26.0",
+    "transformers>=5.14.1,<6",
     "ty>=0.0.1a29,<0.0.22",
 ]
 
diff --git a/renderers/__init__.py b/renderers/__init__.py
index 4e8f964b..fdc91182 100644
--- a/renderers/__init__.py
+++ b/renderers/__init__.py
@@ -36,6 +36,7 @@
     create_renderer_pool,
     extract_message_tool_names,
     is_multimodal,
+    load_tokenizer,
     reject_assistant_in_extension,
     trim_to_turn_close,
 )
@@ -68,13 +69,15 @@
     Qwen3VLRendererConfig,
     RendererConfig,
 )
+from renderers.tokenizer import (
+    ChatTemplateTokenizerLike,
+    TokenizerLike,
+    TokenizersTokenizer,
+)
 
 # Concrete renderer classes are lazy-loaded so that consumers needing
 # only the config layer (``RendererConfig`` discriminated union) don't
-# pay the ``transformers`` import cost. Each renderer module does
-# ``from transformers.tokenization_utils import PreTrainedTokenizer``
-# at module level, so eager imports here would drag ``transformers``
-# into every downstream ``import renderers``. ``__getattr__`` (PEP 562)
+# pay renderer-import costs. ``__getattr__`` (PEP 562)
 # resolves the names on first attribute access, so ``from renderers
 # import DefaultRenderer`` and ``renderers.DefaultRenderer`` both work
 # transparently. ``create_renderer`` doesn't depend on these eager
@@ -124,6 +127,7 @@ def __dir__() -> list[str]:
 __all__ = [
     "AutoRendererConfig",
     "BaseRendererConfig",
+    "ChatTemplateTokenizerLike",
     "Content",
     "ContentPart",
     "DeepSeekR1Renderer",
@@ -192,6 +196,8 @@ def __dir__() -> list[str]:
     "ToolCallFunction",
     "ToolCallParseStatus",
     "ToolSpec",
+    "TokenizerLike",
+    "TokenizersTokenizer",
     "VideoPart",
     "__version__",
     "attribute_text_segments",
@@ -202,6 +208,7 @@ def __dir__() -> list[str]:
     "create_renderer_pool",
     "extract_message_tool_names",
     "is_multimodal",
+    "load_tokenizer",
     "reject_assistant_in_extension",
     "trim_to_turn_close",
 ]
diff --git a/renderers/base.py b/renderers/base.py
index 84c52477..ff186cb0 100644
--- a/renderers/base.py
+++ b/renderers/base.py
@@ -17,6 +17,8 @@
     runtime_checkable,
 )
 
+from renderers.tokenizer import TokenizerLike, TokenizersTokenizer
+
 if TYPE_CHECKING:
     from renderers.configs import (
         AutoRendererConfig,
@@ -1128,7 +1130,7 @@ def bridge_to_next_turn(self, *args: Any, **kwargs: Any) -> "RenderedTokens | No
 }
 
 
-def _model_has_vision_config(model_name: str) -> bool:
+def _model_has_vision_config(model_name: str) -> bool | None:
     """Return True if the HF config for ``model_name`` declares vision inputs.
 
     Used by ``create_renderer`` to fail loudly on VLMs that miss the
@@ -1138,12 +1140,18 @@ def _model_has_vision_config(model_name: str) -> bool:
     match what the trainer reconstructs — a class of bug the renderer
     abstraction exists to prevent.
 
-    Returns False on any AutoConfig failure (offline, gated, missing) so
-    a flaky HF probe never blocks a legitimate text-only fine-tune.
+    Returns ``None`` when the optional ``transformers`` integration is not
+    installed. Returns False on other AutoConfig failures (offline, gated,
+    missing) so a flaky HF probe never blocks a legitimate text-only
+    fine-tune.
     """
     try:
         from transformers import AutoConfig
-
+    except ModuleNotFoundError as exc:
+        if exc.name == "transformers":
+            return None
+        raise
+    try:
         cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
     except Exception:
         return False
@@ -1192,11 +1200,22 @@ def _tokenizer_source_for(model_name_or_path: str) -> str:
     return TOKENIZER_SOURCE_OVERRIDES.get(model_name_or_path, model_name_or_path)
 
 
-def _tokenizer_load_kwargs(model_name_or_path: str) -> dict[str, Any]:
-    revision = TRUSTED_REVISIONS.get(model_name_or_path)
+def _tokenizer_load_kwargs(
+    model_name_or_path: str, *, revision: str | None = None
+) -> dict[str, Any]:
+    trusted_revision = TRUSTED_REVISIONS.get(model_name_or_path)
+    if trusted_revision is not None:
+        if revision is not None and revision != trusted_revision:
+            raise ValueError(
+                f"{model_name_or_path!r} executes trusted remote tokenizer code "
+                f"only at reviewed revision {trusted_revision}; received "
+                f"revision={revision!r}."
+            )
+        return {"trust_remote_code": True, "revision": trusted_revision}
+    kwargs: dict[str, Any] = {"trust_remote_code": False}
     if revision is not None:
-        return {"trust_remote_code": True, "revision": revision}
-    return {"trust_remote_code": False}
+        kwargs["revision"] = revision
+    return kwargs
 
 
 def _preserve_requested_tokenizer_name(
@@ -1280,16 +1299,60 @@ def _load_tokenizer_via_auto(model_name_or_path: str, **kwargs) -> Any:
             type(exc).__name__,
             str(exc)[:160],
         )
-        return tok
+    return tok
+
+
+def _load_transformers_tokenizer(model_name_or_path: str, **kwargs: Any) -> Any:
+    try:
+        import transformers  # noqa: F401
+    except ModuleNotFoundError as exc:
+        if exc.name != "transformers":
+            raise
+        raise ImportError(
+            "This tokenizer requires the optional Hugging Face integration. "
+            "Install it with `pip install 'renderers[hf]'`."
+        ) from exc
+    return _load_tokenizer_via_auto(model_name_or_path, **kwargs)
 
 
-def load_tokenizer(model_name_or_path: str):
+def _load_tokenizers_tokenizer(
+    model_name_or_path: str, *, revision: str | None
+) -> TokenizersTokenizer:
+    try:
+        return TokenizersTokenizer.from_pretrained(
+            model_name_or_path,
+            revision=revision,
+        )
+    except Exception as exc:
+        raise RuntimeError(
+            f"The standalone tokenizers backend could not load "
+            f"{model_name_or_path!r}. The repository may not publish a "
+            "self-contained tokenizer.json (custom tokenizers such as Kimi "
+            "do not). Install `renderers[hf]` and use "
+            "backend='transformers' for that model."
+        ) from exc
+
+
+def load_tokenizer(
+    model_name_or_path: str,
+    *,
+    revision: str | None = None,
+    backend: Literal["auto", "tokenizers", "transformers"] = "auto",
+) -> TokenizerLike:
     """Load a tokenizer with the renderers-package security policy.
 
-    Default ``trust_remote_code=False``. Models listed in
-    ``TRUSTED_REVISIONS`` (Moonshot Kimi-K2 family) load with
-    ``trust_remote_code=True`` AND a pinned ``revision=<sha>`` so
-    transformers only executes the reviewed commit's tokenizer Python.
+    The default ``backend="auto"`` preserves the Hugging Face tokenizer when
+    the optional ``transformers`` integration is installed, and otherwise
+    loads ``tokenizer.json`` through the standalone Rust ``tokenizers``
+    package. Select a backend explicitly when environment-independent wrapper
+    behaviour matters.
+
+    Default ``trust_remote_code=False`` on the Transformers path. Models
+    listed in ``TRUSTED_REVISIONS`` (Moonshot Kimi-K2 family) load with
+    ``trust_remote_code=True`` AND a pinned ``revision=<sha>`` so Transformers
+    only executes the reviewed commit's tokenizer Python. Those custom
+    tokenizers do not publish ``tokenizer.json`` and therefore require the
+    ``renderers[hf]`` extra when loaded by model ID.
 
     ``AutoTokenizer.from_pretrained`` eagerly builds the model config to
     resolve the tokenizer class. If that construction raises on a
@@ -1303,9 +1366,32 @@ def load_tokenizer(model_name_or_path: str):
     ``unsloth`` mirrors instead, then restore ``tokenizer.name_or_path`` to
     the requested Meta ID so auto-resolution still selects ``Llama3Renderer``.
     """
+    if backend not in {"auto", "tokenizers", "transformers"}:
+        raise ValueError(
+            "backend must be one of 'auto', 'tokenizers', or 'transformers'."
+        )
+
     load_name_or_path = _tokenizer_source_for(model_name_or_path)
-    kwargs = _tokenizer_load_kwargs(load_name_or_path)
-    tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs)
+    kwargs = _tokenizer_load_kwargs(load_name_or_path, revision=revision)
+    if backend == "tokenizers":
+        tok = _load_tokenizers_tokenizer(
+            load_name_or_path,
+            revision=kwargs.get("revision"),
+        )
+    elif backend == "transformers":
+        tok = _load_transformers_tokenizer(load_name_or_path, **kwargs)
+    else:
+        try:
+            import transformers  # noqa: F401
+        except ModuleNotFoundError as exc:
+            if exc.name != "transformers":
+                raise
+            tok = _load_tokenizers_tokenizer(
+                load_name_or_path,
+                revision=kwargs.get("revision"),
+            )
+        else:
+            tok = _load_transformers_tokenizer(load_name_or_path, **kwargs)
     return _preserve_requested_tokenizer_name(
         tok,
         requested_name_or_path=model_name_or_path,
@@ -1389,9 +1475,9 @@ def create_renderer_pool(
     Every slot in the pool shares the same config; to run a different
     config, build a different pool.
 
-    Tokenizers load via ``load_tokenizer`` — see its docstring for the
-    ``trust_remote_code`` policy (default off; Moonshot Kimi-K2 family
-    opts in with a pinned ``revision``).
+    Tokenizers load via ``load_tokenizer``. Core installs use the standalone
+    Rust ``tokenizers`` backend; installs with the ``hf`` extra preserve the
+    Hugging Face wrapper and its custom-tokenizer support.
     """
 
     def factory() -> Renderer:
@@ -1532,7 +1618,8 @@ def _resolve_auto_config(
     # Catch this at the renderer-selection seam — well before any
     # rollout — so the failure mode is "config error at startup," not
     # "mysterious KL divergence after 100 steps."
-    if model_name in MULTIMODAL_MODELS or _model_has_vision_config(model_name):
+    vision_probe = _model_has_vision_config(model_name)
+    if model_name in MULTIMODAL_MODELS or vision_probe is True:
         supported_vlms = sorted(MULTIMODAL_MODELS)
         raise ValueError(
             f"No multimodal renderer registered for {model_name!r}, and "
@@ -1541,6 +1628,14 @@ def _resolve_auto_config(
             f"{supported_vlms}), or pass an explicit typed renderer "
             f"config if you know what you're doing."
         )
+    if vision_probe is None:
+        raise ValueError(
+            f"No renderer registered for {model_name!r}, and the optional "
+            "Transformers integration is not installed to determine whether "
+            "this unknown model is multimodal. Pass an explicit "
+            "DefaultRendererConfig with a tokenizer that implements "
+            "apply_chat_template, or install `renderers[hf]`."
+        )
 
     # Text-only fall back to default (apply_chat_template). For fine-tunes
     # with customized chat templates this is the *correct* choice, so we
@@ -1805,8 +1900,8 @@ def _get_offset_tokenizer(tokenizer):
     back to its source segment via the fast tokenizer's
     ``offset_mapping`` (see :func:`attribute_text_segments`). The
     contract: every BYO tokenizer must be a fast tokenizer with offset
-    support. Tokenizers loaded via :func:`load_tokenizer` are
-    ``PreTrainedTokenizerFast`` instances that satisfy this trivially.
+    support. Tokenizers loaded via :func:`load_tokenizer` satisfy this
+    structurally, whether backed by ``tokenizers`` or Transformers.
     """
     try:
         tokenizer("a", add_special_tokens=False, return_offsets_mapping=True)
@@ -1815,8 +1910,8 @@ def _get_offset_tokenizer(tokenizer):
             "Hand-coded renderers require a fast tokenizer with "
             "``return_offsets_mapping=True`` support for body/scaffold "
             "attribution. Pass a tokenizer loaded via "
-            "``renderers.base.load_tokenizer``, or any "
-            "``transformers.PreTrainedTokenizerFast`` instance."
+            "``renderers.base.load_tokenizer``, or any structurally "
+            "compatible tokenizer exposing offsets."
         ) from exc
     return tokenizer
 
@@ -1854,13 +1949,11 @@ def attribute_text_segments(
     every body byte inside the ``is_content=True`` run at the cost of a
     few adjacent wrap bytes.
 
-    Requires a HuggingFace fast tokenizer with offset tracking. Every
-    model in ``MODEL_RENDERER_MAP`` ships one, so the offset lookup
-    always succeeds for tokenizers obtained via :func:`load_tokenizer`.
-    BYO tokenizers must be a ``PreTrainedTokenizerFast`` (or anything
-    else exposing ``return_offsets_mapping=True``); slow tokenizers
-    aren't supported — BPE drift at the wrap/body boundary would
-    defeat the whole point.
+    Requires a tokenizer with offset tracking. The standalone
+    :class:`~renderers.tokenizer.TokenizersTokenizer`, Hugging Face fast
+    tokenizers, and any structurally compatible BYO tokenizer satisfy this
+    contract. Slow or offsetless tokenizers aren't supported — BPE drift at
+    the wrap/body boundary would defeat the whole point.
 
     Empty input or empty joined text returns an empty list.
     """
diff --git a/renderers/client.py b/renderers/client.py
index 5196f4a3..55291a73 100644
--- a/renderers/client.py
+++ b/renderers/client.py
@@ -474,8 +474,9 @@ def _build_qwen_vl_features(
     except ImportError as exc:
         raise RuntimeError(
             "Multimodal generate via /inference/v1/generate requires `vllm` "
-            "and `torch` to encode the features payload. Install vLLM in this "
-            "environment, or pre-build features upstream."
+            "and the `renderers[hf]` integration to encode the features "
+            "payload. Install them in this environment, or pre-build features "
+            "upstream."
         ) from exc
 
     out: dict[str, Any] = {
diff --git a/renderers/deepseek_v3.py b/renderers/deepseek_v3.py
index a00f1f20..fd222e19 100644
--- a/renderers/deepseek_v3.py
+++ b/renderers/deepseek_v3.py
@@ -14,7 +14,7 @@
 
 import json
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -63,7 +63,7 @@ class DeepSeekV3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: DeepSeekV3RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/default.py b/renderers/default.py
index 785a5375..226d3bff 100644
--- a/renderers/default.py
+++ b/renderers/default.py
@@ -11,7 +11,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import ChatTemplateTokenizerLike
 
 from renderers.base import (
     Message,
@@ -92,9 +92,16 @@ class DefaultRenderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: ChatTemplateTokenizerLike,
         config: DefaultRendererConfig | None = None,
     ):
+        if not callable(getattr(tokenizer, "apply_chat_template", None)):
+            raise TypeError(
+                "DefaultRenderer requires a tokenizer implementing "
+                "apply_chat_template. Install `renderers[hf]` and load with "
+                "backend='transformers', or use a registered model-specific "
+                "renderer with the standalone tokenizers backend."
+            )
         cfg = config or DefaultRendererConfig()
         if cfg.thinking_retention is not None:
             raise ValueError(
diff --git a/renderers/glm45.py b/renderers/glm45.py
index bfc5f09c..6b7ec324 100644
--- a/renderers/glm45.py
+++ b/renderers/glm45.py
@@ -13,7 +13,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -55,7 +55,7 @@ class GLM45Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: GLM45RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/glm5.py b/renderers/glm5.py
index 4f34d98f..5bfb0199 100644
--- a/renderers/glm5.py
+++ b/renderers/glm5.py
@@ -14,7 +14,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -64,7 +64,7 @@ class GLM5Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: GLM5RendererConfig | GLM51RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/gpt_oss.py b/renderers/gpt_oss.py
index 6165ed09..c8c6dddf 100644
--- a/renderers/gpt_oss.py
+++ b/renderers/gpt_oss.py
@@ -49,7 +49,7 @@
     ToolDescription,
     load_harmony_encoding,
 )
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -123,7 +123,7 @@ class GptOssRenderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: GptOssRendererConfig | None = None,
     ):
         """Initialise the renderer.
diff --git a/renderers/hy3.py b/renderers/hy3.py
index 7eaef656..7a6ee41c 100644
--- a/renderers/hy3.py
+++ b/renderers/hy3.py
@@ -27,7 +27,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -74,7 +74,7 @@ class Hy3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Hy3RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/kimi_k2.py b/renderers/kimi_k2.py
index 73376003..0fe03c3c 100644
--- a/renderers/kimi_k2.py
+++ b/renderers/kimi_k2.py
@@ -16,7 +16,7 @@
 
 import json
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -47,7 +47,7 @@ class KimiK2Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: KimiK2RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/kimi_k25.py b/renderers/kimi_k25.py
index 48ea426d..042c8828 100644
--- a/renderers/kimi_k25.py
+++ b/renderers/kimi_k25.py
@@ -25,7 +25,7 @@
 import re
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -592,7 +592,7 @@ class KimiK25Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: KimiK25RendererConfig | None = None,
         *,
         processor: Any = None,
@@ -655,7 +655,15 @@ def mm_token_type_id_map(self) -> dict[int, int]:
     def _get_processor(self):
         if self._processor is not None:
             return self._processor
-        from transformers import AutoProcessor
+        try:
+            from transformers import AutoProcessor
+        except ModuleNotFoundError as exc:
+            if exc.name != "transformers":
+                raise
+            raise ImportError(
+                "KimiK25Renderer image processing requires the optional HF "
+                "integration. Install `renderers[hf]`, or pass a processor."
+            ) from exc
 
         name = getattr(self._tokenizer, "name_or_path", None)
         if not name:
@@ -669,7 +677,20 @@ def _get_processor(self):
         # trust_remote_code=True. Callers using ``create_renderer_pool`` go
         # through ``load_tokenizer`` which already pins the revision; for
         # auto-load here, we delegate to AutoProcessor with the same flag.
-        self._processor = AutoProcessor.from_pretrained(name, trust_remote_code=True)
+        from renderers.base import TRUSTED_REVISIONS
+
+        revision = TRUSTED_REVISIONS.get(name)
+        if revision is None:
+            raise RuntimeError(
+                f"Kimi processor auto-loading is allowed only for reviewed "
+                f"model revisions; {name!r} is not allow-listed. Pass an "
+                "already-loaded processor explicitly."
+            )
+        self._processor = AutoProcessor.from_pretrained(
+            name,
+            trust_remote_code=True,
+            revision=revision,
+        )
         return self._processor
 
     def _process_image(self, part: dict[str, Any]):
diff --git a/renderers/laguna_xs2.py b/renderers/laguna_xs2.py
index 8ef3469b..f2b2093f 100644
--- a/renderers/laguna_xs2.py
+++ b/renderers/laguna_xs2.py
@@ -47,7 +47,7 @@
 
 import json
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Content,
@@ -115,7 +115,7 @@
 class LagunaXS2Renderer:
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: (
             LagunaXS2RendererConfig
             | LagunaM1RendererConfig
@@ -647,7 +647,7 @@ class LagunaM1Renderer(LagunaXS2Renderer):
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: LagunaM1RendererConfig | None = None,
     ):
         super().__init__(tokenizer, config or LagunaM1RendererConfig())
@@ -687,7 +687,7 @@ class LagunaXS21Renderer(LagunaXS2Renderer):
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: LagunaXS21RendererConfig | None = None,
     ):
         super().__init__(tokenizer, config or LagunaXS21RendererConfig())
diff --git a/renderers/llama_3.py b/renderers/llama_3.py
index d18d8c87..6a452f43 100644
--- a/renderers/llama_3.py
+++ b/renderers/llama_3.py
@@ -41,7 +41,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -93,7 +93,7 @@ class Llama3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Llama3RendererConfig | None = None,
     ):
         # ``thinking_retention`` is accepted but a no-op: Llama-3 ships no
diff --git a/renderers/minimax_m2.py b/renderers/minimax_m2.py
index a7f0bc70..e271e08c 100644
--- a/renderers/minimax_m2.py
+++ b/renderers/minimax_m2.py
@@ -14,7 +14,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -57,7 +57,7 @@ class MiniMaxM2Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: MiniMaxM2RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/nemotron3.py b/renderers/nemotron3.py
index 5cc76c91..bc2ddb50 100644
--- a/renderers/nemotron3.py
+++ b/renderers/nemotron3.py
@@ -17,7 +17,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -112,7 +112,7 @@ class Nemotron3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Nemotron3RendererConfig | Nemotron3UltraRendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/prime_qwen3.py b/renderers/prime_qwen3.py
index ba639a09..b5253ccf 100644
--- a/renderers/prime_qwen3.py
+++ b/renderers/prime_qwen3.py
@@ -6,7 +6,7 @@
 from collections.abc import Mapping, Sequence
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -120,7 +120,7 @@ def _tool_definition(tool: ToolSpec) -> str:
 
 
 class _TokenBuilder:
-    def __init__(self, tokenizer: PreTrainedTokenizer):
+    def __init__(self, tokenizer: TokenizerLike):
         self.tokenizer = tokenizer
         self.token_ids: list[int] = []
         self.message_indices: list[int] = []
@@ -192,7 +192,7 @@ class PrimeQwen3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: PrimeQwen3RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/qwen3.py b/renderers/qwen3.py
index d85d161d..72a8d362 100644
--- a/renderers/qwen3.py
+++ b/renderers/qwen3.py
@@ -20,7 +20,7 @@
 
 import json
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -59,7 +59,7 @@ class Qwen3Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Qwen3RendererConfig | None = None,
     ):
         self._tokenizer = tokenizer
diff --git a/renderers/qwen35.py b/renderers/qwen35.py
index 52de8867..f6533f15 100644
--- a/renderers/qwen35.py
+++ b/renderers/qwen35.py
@@ -25,7 +25,7 @@
 import json
 from typing import Any
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -123,7 +123,7 @@ class Qwen35Renderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Qwen35RendererConfig | None = None,
         *,
         processor: Any = None,
@@ -182,7 +182,15 @@ def mm_token_type_id_map(self) -> dict[int, int]:
     def _get_processor(self):
         if self._processor is not None:
             return self._processor
-        from transformers import AutoProcessor
+        try:
+            from transformers import AutoProcessor
+        except ModuleNotFoundError as exc:
+            if exc.name != "transformers":
+                raise
+            raise ImportError(
+                "Qwen35Renderer image/video processing requires the optional "
+                "HF integration. Install `renderers[hf]`, or pass a processor."
+            ) from exc
 
         name = getattr(self._tokenizer, "name_or_path", None)
         if not name:
diff --git a/renderers/qwen3_vl.py b/renderers/qwen3_vl.py
index 97072d21..83d14787 100644
--- a/renderers/qwen3_vl.py
+++ b/renderers/qwen3_vl.py
@@ -33,7 +33,7 @@
 from typing import Any
 from urllib.parse import urlparse
 
-from transformers.tokenization_utils import PreTrainedTokenizer
+from renderers.tokenizer import TokenizerLike
 
 from renderers.base import (
     Message,
@@ -311,7 +311,7 @@ class Qwen3VLRenderer:
 
     def __init__(
         self,
-        tokenizer: PreTrainedTokenizer,
+        tokenizer: TokenizerLike,
         config: Qwen3VLRendererConfig | None = None,
         *,
         processor: Any = None,
@@ -375,7 +375,15 @@ def _encode(self, text: str) -> list[int]:
     def _get_processor(self):
         if self._processor is not None:
             return self._processor
-        from transformers import AutoProcessor
+        try:
+            from transformers import AutoProcessor
+        except ModuleNotFoundError as exc:
+            if exc.name != "transformers":
+                raise
+            raise ImportError(
+                "Qwen3VLRenderer image/video processing requires the optional "
+                "HF integration. Install `renderers[hf]`, or pass a processor."
+            ) from exc
 
         name = getattr(self._tokenizer, "name_or_path", None)
         if not name:
diff --git a/renderers/tokenizer.py b/renderers/tokenizer.py
new file mode 100644
index 00000000..4aae1654
--- /dev/null
+++ b/renderers/tokenizer.py
@@ -0,0 +1,130 @@
+"""Tokenizer contracts and the lightweight ``tokenizers`` adapter.
+
+The renderer core only needs four operations: encode, decode, special-token
+lookup, and encoding with character offsets. Keeping that surface structural
+lets callers pass Hugging Face tokenizers, engine-owned tokenizers, or the
+Rust-backed :class:`tokenizers.Tokenizer` without importing ``transformers``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Protocol, TypedDict
+
+from tokenizers import Tokenizer
+
+
+class TokenizerEncoding(TypedDict, total=False):
+    input_ids: list[int]
+    offset_mapping: list[tuple[int, int]]
+
+
+class TokenizerLike(Protocol):
+    """Minimal tokenizer surface used by model-specific renderers."""
+
+    name_or_path: str
+    unk_token_id: int | None
+
+    def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]: ...
+
+    def decode(
+        self, ids: list[int], *, skip_special_tokens: bool = False
+    ) -> str: ...
+
+    def convert_tokens_to_ids(self, token: str) -> int | None: ...
+
+    def __call__(
+        self,
+        text: str,
+        *,
+        add_special_tokens: bool = False,
+        return_offsets_mapping: bool = False,
+        **kwargs: Any,
+    ) -> TokenizerEncoding: ...
+
+
+class ChatTemplateTokenizerLike(TokenizerLike, Protocol):
+    """Extra Hugging Face surface required by ``DefaultRenderer``."""
+
+    eos_token_id: int | None
+    all_special_tokens: list[str]
+
+    def apply_chat_template(self, *args: Any, **kwargs: Any) -> Any: ...
+
+
+class TokenizersTokenizer:
+    """HF-shaped adapter around the standalone Rust ``tokenizers`` package.
+
+    ``tokenizers.Tokenizer.encode`` returns an ``Encoding`` object, while the
+    established renderer contract expects a list of IDs and a mapping-shaped
+    offset result. This adapter performs only that shape conversion; it does
+    not implement chat templates or multimodal processing.
+    """
+
+    def __init__(self, tokenizer: Tokenizer, *, name_or_path: str):
+        self._tokenizer = tokenizer
+        self.name_or_path = name_or_path
+        unk_token = getattr(tokenizer.model, "unk_token", None)
+        self.unk_token_id = (
+            tokenizer.token_to_id(unk_token) if isinstance(unk_token, str) else None
+        )
+
+    @classmethod
+    def from_pretrained(
+        cls, model_name_or_path: str, *, revision: str | None = None
+    ) -> "TokenizersTokenizer":
+        path = Path(model_name_or_path)
+        if path.is_dir():
+            tokenizer_path = path / "tokenizer.json"
+            if not tokenizer_path.is_file():
+                raise FileNotFoundError(
+                    f"No tokenizer.json found in local tokenizer directory {path}."
+                )
+            tokenizer = Tokenizer.from_file(str(tokenizer_path))
+        elif path.is_file():
+            tokenizer = Tokenizer.from_file(str(path))
+        else:
+            tokenizer = Tokenizer.from_pretrained(
+                model_name_or_path,
+                revision=revision or "main",
+            )
+        return cls(tokenizer, name_or_path=model_name_or_path)
+
+    def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]:
+        return list(
+            self._tokenizer.encode(
+                text,
+                add_special_tokens=add_special_tokens,
+            ).ids
+        )
+
+    def decode(
+        self, ids: list[int], *, skip_special_tokens: bool = False
+    ) -> str:
+        return self._tokenizer.decode(
+            ids,
+            skip_special_tokens=skip_special_tokens,
+        )
+
+    def convert_tokens_to_ids(self, token: str) -> int | None:
+        return self._tokenizer.token_to_id(token)
+
+    def __call__(
+        self,
+        text: str,
+        *,
+        add_special_tokens: bool = False,
+        return_offsets_mapping: bool = False,
+        **kwargs: Any,
+    ) -> TokenizerEncoding:
+        if kwargs:
+            unexpected = ", ".join(sorted(kwargs))
+            raise TypeError(f"Unsupported tokenizer arguments: {unexpected}")
+        encoding = self._tokenizer.encode(
+            text,
+            add_special_tokens=add_special_tokens,
+        )
+        result: TokenizerEncoding = {"input_ids": list(encoding.ids)}
+        if return_offsets_mapping:
+            result["offset_mapping"] = list(encoding.offsets)
+        return result
diff --git a/tests/test_renderer_config.py b/tests/test_renderer_config.py
index a35f2709..3ea97a77 100644
--- a/tests/test_renderer_config.py
+++ b/tests/test_renderer_config.py
@@ -207,7 +207,10 @@ def __init__(self, tokenizer, config):
 def test_create_renderer_default_argument_is_auto():
     """Passing no config is equivalent to passing ``AutoRendererConfig()``
     — short form for the common case."""
-    tok = SimpleNamespace(name_or_path="")  # no MODEL_RENDERER_MAP entry
+    tok = SimpleNamespace(
+        name_or_path="",
+        apply_chat_template=lambda *args, **kwargs: [],
+    )  # no MODEL_RENDERER_MAP entry
     renderer = create_renderer(tok)
     # Falls through to DefaultRenderer when no match and no vision config.
     assert renderer.__class__.__name__ == "DefaultRenderer"
@@ -302,7 +305,10 @@ def test_thinking_retention_consistent_pairs_are_accepted(config_cls, kwargs):
 
 def test_default_renderer_rejects_explicit_retention():
     """Opaque apply_chat_template fallback cannot implement bridge policy."""
-    tok = SimpleNamespace(name_or_path="")
+    tok = SimpleNamespace(
+        name_or_path="",
+        apply_chat_template=lambda *args, **kwargs: [],
+    )
     create_renderer(tok, DefaultRendererConfig())
 
     for retention in ("tool_cycle", "all"):
diff --git a/tests/test_tokenizers_backend.py b/tests/test_tokenizers_backend.py
new file mode 100644
index 00000000..dd3be6d3
--- /dev/null
+++ b/tests/test_tokenizers_backend.py
@@ -0,0 +1,127 @@
+"""Tests for the Transformers-free tokenizer path."""
+
+from __future__ import annotations
+
+import builtins
+import subprocess
+import sys
+
+import pytest
+from tokenizers import Tokenizer, models, pre_tokenizers
+
+from renderers import DefaultRendererConfig, create_renderer
+from renderers.base import attribute_text_segments, load_tokenizer
+from renderers.tokenizer import TokenizersTokenizer
+
+
+def _write_word_tokenizer(path):
+    tokenizer = Tokenizer(
+        models.WordLevel(
+            {"[UNK]": 0, "hello": 1, "world": 2},
+            unk_token="[UNK]",
+        )
+    )
+    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
+    tokenizer.save(str(path / "tokenizer.json"))
+
+
+def test_tokenizers_backend_adapts_ids_decode_and_offsets(tmp_path):
+    _write_word_tokenizer(tmp_path)
+
+    tokenizer = load_tokenizer(str(tmp_path), backend="tokenizers")
+
+    assert isinstance(tokenizer, TokenizersTokenizer)
+    assert tokenizer.name_or_path == str(tmp_path)
+    assert tokenizer.unk_token_id == 0
+    assert tokenizer.convert_tokens_to_ids("hello") == 1
+    assert tokenizer.convert_tokens_to_ids("missing") is None
+    assert tokenizer.encode("hello world", add_special_tokens=False) == [1, 2]
+    assert tokenizer.decode([1, 2], skip_special_tokens=False) == "hello world"
+    assert tokenizer(
+        "hello world",
+        add_special_tokens=False,
+        return_offsets_mapping=True,
+    ) == {
+        "input_ids": [1, 2],
+        "offset_mapping": [(0, 5), (6, 11)],
+    }
+
+
+def test_tokenizers_backend_drives_segment_attribution(tmp_path):
+    _write_word_tokenizer(tmp_path)
+    tokenizer = load_tokenizer(str(tmp_path), backend="tokenizers")
+
+    assert attribute_text_segments(
+        tokenizer,
+        [("hello ", False), ("world", True)],
+    ) == [(1, False), (2, True)]
+
+
+def test_auto_backend_uses_tokenizers_when_transformers_is_absent(
+    tmp_path, monkeypatch
+):
+    _write_word_tokenizer(tmp_path)
+    real_import = builtins.__import__
+
+    def blocked_import(name, *args, **kwargs):
+        if name == "transformers" or name.startswith("transformers."):
+            raise ModuleNotFoundError("blocked for test", name="transformers")
+        return real_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", blocked_import)
+    tokenizer = load_tokenizer(str(tmp_path))
+    assert isinstance(tokenizer, TokenizersTokenizer)
+
+
+def test_default_renderer_rejects_tokenizer_without_chat_templates(tmp_path):
+    _write_word_tokenizer(tmp_path)
+    tokenizer = load_tokenizer(str(tmp_path), backend="tokenizers")
+
+    with pytest.raises(TypeError, match=r"apply_chat_template.*renderers\[hf\]"):
+        create_renderer(tokenizer, DefaultRendererConfig())
+
+
+def test_qwen_transformers_and_tokenizers_ids_and_offsets_match():
+    hf = load_tokenizer("Qwen/Qwen3-0.6B", backend="transformers")
+    rust = load_tokenizer("Qwen/Qwen3-0.6B", backend="tokenizers")
+
+    samples = [
+        "hello world",
+        "hello 👋🏽 café 中文",
+        '<|im_start|>assistant\n{"name":"f","arguments":{}}',
+    ]
+    for sample in samples:
+        assert hf.encode(sample, add_special_tokens=False) == rust.encode(
+            sample,
+            add_special_tokens=False,
+        )
+        hf_encoding = hf(
+            sample,
+            add_special_tokens=False,
+            return_offsets_mapping=True,
+        )
+        rust_encoding = rust(
+            sample,
+            add_special_tokens=False,
+            return_offsets_mapping=True,
+        )
+        assert list(hf_encoding["input_ids"]) == rust_encoding["input_ids"]
+        assert list(hf_encoding["offset_mapping"]) == rust_encoding["offset_mapping"]
+
+
+def test_every_renderer_module_imports_without_transformers():
+    script = """
+import builtins
+
+real_import = builtins.__import__
+def blocked_import(name, *args, **kwargs):
+    if name == "transformers" or name.startswith("transformers."):
+        raise ModuleNotFoundError("blocked for core-install smoke test", name="transformers")
+    return real_import(name, *args, **kwargs)
+
+builtins.__import__ = blocked_import
+import renderers
+for renderer_name in renderers._LAZY_RENDERERS:
+    getattr(renderers, renderer_name)
+"""
+    subprocess.run([sys.executable, "-c", script], check=True)
diff --git a/uv.lock b/uv.lock
index 62b41d63..c8b6f651 100644
--- a/uv.lock
+++ b/uv.lock
@@ -173,7 +173,7 @@ name = "cuda-bindings"
 version = "13.2.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "cuda-pathfinder" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/1a/fe/7351d7e586a8b4c9f89731bfe4cf0148223e8f9903ff09571f78b3fb0682/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556", size = 5744254, upload-time = "2026-03-11T00:12:29.798Z" },
@@ -204,43 +204,43 @@ wheels = [
 
 [package.optional-dependencies]
 cublas = [
-    { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cudart = [
-    { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cufft = [
-    { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cufile = [
-    { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cupti = [
-    { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 curand = [
-    { name = "nvidia-curand", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cusolver = [
-    { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 cusparse = [
-    { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
+    { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 nvjitlink = [
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 nvrtc = [
-    { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 nvtx = [
-    { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+    { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" },
 ]
 
 [[package]]
@@ -266,7 +266,7 @@ name = "exceptiongroup"
 version = "1.3.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+    { name = "typing-extensions" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
 wheels = [
@@ -748,7 +748,7 @@ name = "nvidia-cublas"
 version = "13.1.1.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "nvidia-cuda-nvrtc" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
@@ -787,7 +787,7 @@ name = "nvidia-cudnn-cu13"
 version = "9.20.0.48"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "nvidia-cublas" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" },
@@ -799,7 +799,7 @@ name = "nvidia-cufft"
 version = "12.0.0.61"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "nvidia-nvjitlink" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -829,9 +829,9 @@ name = "nvidia-cusolver"
 version = "12.0.4.66"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
-    { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "nvidia-cublas" },
+    { name = "nvidia-cusparse" },
+    { name = "nvidia-nvjitlink" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -843,7 +843,7 @@ name = "nvidia-cusparse"
 version = "12.6.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
+    { name = "nvidia-nvjitlink" },
 ]
 wheels = [
     { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -1345,6 +1345,11 @@ dependencies = [
     { name = "openai-harmony" },
     { name = "prime-pydantic-config" },
     { name = "tiktoken" },
+    { name = "tokenizers" },
+]
+
+[package.optional-dependencies]
+hf = [
     { name = "transformers" },
 ]
 
@@ -1357,6 +1362,7 @@ dev = [
     { name = "ruff" },
     { name = "torch" },
     { name = "torchvision" },
+    { name = "transformers" },
     { name = "ty" },
 ]
 
@@ -1368,8 +1374,10 @@ requires-dist = [
     { name = "openai-harmony", specifier = ">=0.0.4" },
     { name = "prime-pydantic-config", specifier = ">=0.3.0.dev83" },
     { name = "tiktoken" },
-    { name = "transformers", specifier = ">=4.50.0" },
+    { name = "tokenizers", specifier = ">=0.22.2,<0.23" },
+    { name = "transformers", marker = "extra == 'hf'", specifier = ">=5.14.1,<6" },
 ]
+provides-extras = ["hf"]
 
 [package.metadata.requires-dev]
 dev = [
@@ -1380,6 +1388,7 @@ dev = [
     { name = "ruff" },
     { name = "torch", specifier = ">=2.11.0" },
     { name = "torchvision", specifier = ">=0.26.0" },
+    { name = "transformers", specifier = ">=5.14.1,<6" },
     { name = "ty", specifier = ">=0.0.1a29,<0.0.22" },
 ]
 
@@ -1438,28 +1447,26 @@ wheels = [
 
 [[package]]
 name = "safetensors"
-version = "0.7.0"
+version = "0.8.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" },
-    { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" },
-    { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" },
-    { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" },
-    { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" },
-    { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" },
-    { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/6a/4d08d89a6fcbe905c5ae68b8b34f0791850882fc19782d0d02c65abbdf3b/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737", size = 492430, upload-time = "2025-11-19T15:18:11.884Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/29/59ed8152b30f72c42d00d241e58eaca558ae9dbfa5695206e2e0f54c7063/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd", size = 503977, upload-time = "2025-11-19T15:18:17.523Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/0b/4811bfec67fa260e791369b16dab105e4bae82686120554cc484064e22b4/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2", size = 623890, upload-time = "2025-11-19T15:18:22.666Z" },
-    { url = "https://files.pythonhosted.org/packages/58/5b/632a58724221ef03d78ab65062e82a1010e1bef8e8e0b9d7c6d7b8044841/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3", size = 531885, upload-time = "2025-11-19T15:18:27.146Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" },
+    { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" },
+    { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" },
+    { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" },
+    { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" },
+    { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" },
+    { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" },
+    { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" },
+    { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" },
+    { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" },
 ]
 
 [[package]]
@@ -1697,7 +1704,7 @@ wheels = [
 
 [[package]]
 name = "transformers"
-version = "5.6.2"
+version = "5.14.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "huggingface-hub" },
@@ -1711,9 +1718,9 @@ dependencies = [
     { name = "tqdm" },
     { name = "typer" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/a4/e9/c6c80a07690142a7d05444271f47b9f3c8aac7dea01d52e1137ee480ad78/transformers-5.6.2.tar.gz", hash = "sha256:e657134c3e5a6bc00a3c35f4e2674bb51adfcd89898495b788a18552bac2b91a", size = 8311867, upload-time = "2026-04-23T18:33:29.332Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5d/95/0b0218149b0d6f14df35f5b8f676fa83df4f19ed253c3cc447107ef86eca/transformers-5.6.2-py3-none-any.whl", hash = "sha256:f8d3a1bb96778fed9b8aabfd0dd6e19843e4b0f2bb6b59f32b8a92051b0f348f", size = 10364898, upload-time = "2026-04-23T18:33:26.081Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" },
 ]
 
 [[package]]

From 385a889a5c3c93f706313f4e14b115ca65eea61e Mon Sep 17 00:00:00 2001
From: hallerite <git@hallerite.com>
Date: Sat, 8 Aug 2026 13:40:59 -0700
Subject: [PATCH 2/2] Format tokenizer adapter

---
 renderers/tokenizer.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/renderers/tokenizer.py b/renderers/tokenizer.py
index 4aae1654..975c738f 100644
--- a/renderers/tokenizer.py
+++ b/renderers/tokenizer.py
@@ -27,9 +27,7 @@ class TokenizerLike(Protocol):
 
     def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]: ...
 
-    def decode(
-        self, ids: list[int], *, skip_special_tokens: bool = False
-    ) -> str: ...
+    def decode(self, ids: list[int], *, skip_special_tokens: bool = False) -> str: ...
 
     def convert_tokens_to_ids(self, token: str) -> int | None: ...
 
@@ -98,9 +96,7 @@ def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]:
             ).ids
         )
 
-    def decode(
-        self, ids: list[int], *, skip_special_tokens: bool = False
-    ) -> str:
+    def decode(self, ids: list[int], *, skip_special_tokens: bool = False) -> str:
         return self._tokenizer.decode(
             ids,
             skip_special_tokens=skip_special_tokens,