From 5530dd883e5d72657e220d0a9b2e0c5dcae49c3a Mon Sep 17 00:00:00 2001
From: SearchSavior <etatelbaum.psc@gmail.com>
Date: Sun, 5 Jul 2026 16:56:22 -0400
Subject: [PATCH] patch check for qwen35 dense/moe to apply tokenizer for all
 qwen35/qwen36 models

---
 src/engine/ov_genai/vlm.py           | 24 +++++++++++++++++++++---
 src/server/utils/resolve_vlm_type.py | 11 +++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/engine/ov_genai/vlm.py b/src/engine/ov_genai/vlm.py
index cce7af8..aff294f 100644
--- a/src/engine/ov_genai/vlm.py
+++ b/src/engine/ov_genai/vlm.py
@@ -17,7 +17,7 @@
 
 from src.server.models.ov_genai import OVGenAI_GenConfig
 from src.server.utils.chat import flatten_message_content
-from src.server.utils.resolve_vlm_type import resolve_vlm_vision_token
+from src.server.utils.resolve_vlm_type import is_qwen3_5_architecture, resolve_vlm_vision_token
 from src.server.model_registry import ModelRegistry
 from src.server.models.registration import ModelLoadConfig
 from src.engine.ov_genai.streamers import ChunkStreamer
@@ -33,6 +33,7 @@ def __init__(self, load_config: ModelLoadConfig):
         self.load_config = load_config
         self._active_request_id: Optional[str] = None
         self._active_streamer: Optional[ChunkStreamer] = None
+        self._default_chat_template_kwargs: dict = {}
 
     def _vision_token_for_index(self, index: int) -> str:
         """
@@ -47,7 +48,7 @@ def _vision_token_for_index(self, index: int) -> str:
     def prepare_inputs(self,
         messages: List[Dict[str, Any]],
         tools: Optional[List[Dict[str, Any]]] = None,
-        chat_template_kwargs: dict = {},
+        chat_template_kwargs: dict = {}
     ) -> Tuple[str, List[ov.Tensor]]:
         """
         Parse a messages list and prepare text prompt + image tensors for VLM inference.
@@ -115,7 +116,7 @@ def prepare_inputs(self,
             tokenize=False,
             tools=tools,
             add_generation_prompt=True,
-            **chat_template_kwargs,
+            **{**self._default_chat_template_kwargs, **chat_template_kwargs},
         )
 
         # Step 3: Convert images to OpenVINO Tensors
@@ -284,6 +285,9 @@ def load_model(self, loader: ModelLoadConfig):
     
             self.vision_token = resolve_vlm_vision_token(loader.model_path)
 
+            # Auto-detect Qwen3.5 architecture and inject enable_thinking
+            self._detect_chat_template_defaults(loader)
+
             logger.info(f"{loader.model_name} loaded successfully")
 
         except Exception as e:
@@ -312,6 +316,20 @@ async def unload_model(self, registry: ModelRegistry, model_name: str) -> bool:
         logger.info(f"[{self.load_config.model_name}] unloaded successfully")
         return removed
         
+    def _detect_chat_template_defaults(self, loader: ModelLoadConfig) -> None:
+        """Read config.json and set default chat_template_kwargs for known architectures."""
+        import json
+        config_path = os.path.join(loader.model_path, "config.json")
+        try:
+            with open(config_path, "r") as f:
+                config = json.load(f)
+            architectures = config.get("architectures", [])
+            if isinstance(architectures, list) and is_qwen3_5_architecture(architectures):
+                self._default_chat_template_kwargs = {"enable_thinking": True}
+                logger.info(f"{loader.model_name}: detected Qwen3.5 architecture, enabling thinking")
+        except Exception as e:
+            logger.debug(f"{loader.model_name}: could not detect architecture defaults: {e}")
+
     def create_generation_config(self, config: OVGenAI_GenConfig) -> GenerationConfig:
         """
         Converts the config received by the API to the OpenVino-compatible config.
diff --git a/src/server/utils/resolve_vlm_type.py b/src/server/utils/resolve_vlm_type.py
index 7592b2f..bf82dd2 100644
--- a/src/server/utils/resolve_vlm_type.py
+++ b/src/server/utils/resolve_vlm_type.py
@@ -3,6 +3,12 @@
 from typing import Any, Iterable
 
 
+QWEN3_5_ARCHITECTURES: set[str] = {
+    "Qwen3_5ForConditionalGeneration",
+    "Qwen3_5MoeForConditionalGeneration",
+}
+
+
 ARCHITECTURE_VISION_TOKENS = {
     "Gemma4ForConditionalGeneration": "<|image><|image|><image|>",
     "Gemma3ForConditionalGeneration": "<start_of_image>",
@@ -23,6 +29,11 @@
 }
 
 
+def is_qwen3_5_architecture(architectures: list[str]) -> bool:
+    """Check if any architecture in the list is a Qwen3.5 variant."""
+    return bool(QWEN3_5_ARCHITECTURES & set(architectures))
+
+
 def _architecture_values(config: Any) -> Iterable[str]:
     if not isinstance(config, dict):
         return ()