diff --git a/docs/source/Instruction/Supported-models-and-datasets.md b/docs/source/Instruction/Supported-models-and-datasets.md index a9660ecd87..0e29612378 100644 --- a/docs/source/Instruction/Supported-models-and-datasets.md +++ b/docs/source/Instruction/Supported-models-and-datasets.md @@ -493,6 +493,7 @@ |[moonshotai/Kimi-K2-Instruct](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct](https://huggingface.co/moonshotai/Kimi-K2-Instruct)| |[moonshotai/Kimi-K2-Instruct-0905](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct-0905)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct-0905](https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905)| |[moonshotai/Kimi-K2-Thinking](https://modelscope.cn/models/moonshotai/Kimi-K2-Thinking)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Thinking](https://huggingface.co/moonshotai/Kimi-K2-Thinking)| +|[TeleAI/TeleChat3-105B-A4.7B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-105B-A4.7B-Thinking)|deepseek_v3|telechat3|telechat3|transformers>=4.46.3|✘|-|-| |[deepseek-ai/DeepSeek-V3.1-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Base)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Base](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Base)| |[deepseek-ai/DeepSeek-V3.1](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1](https://huggingface.co/deepseek-ai/DeepSeek-V3.1)| |[deepseek-ai/DeepSeek-V3.1-Terminus](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Terminus)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Terminus](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus)| @@ -547,6 +548,8 @@ |[TeleAI/TeleChat2-7B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-7B-32K)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-7B-32K](https://huggingface.co/Tele-AI/TeleChat2-7B-32K)| |[TeleAI/TeleChat2-35B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-35B-32K)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-35B-32K](https://huggingface.co/Tele-AI/TeleChat2-35B-32K)| |[TeleAI/TeleChat2-35B-Nov](https://modelscope.cn/models/TeleAI/TeleChat2-35B-Nov)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-35B-Nov](https://huggingface.co/Tele-AI/TeleChat2-35B-Nov)| +|[TeleAI/TeleChat3-36B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-36B-Thinking)|telechat3|telechat3|telechat3|transformers>=4.53.2|✘|-|[Tele-AI/TeleChat3-36B-Thinking](https://huggingface.co/Tele-AI/TeleChat3-36B-Thinking)| +|[TeleAI/TeleChat3-Coder-36B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-Coder-36B-Thinking)|telechat3|telechat3_coder|telechat3_coder|transformers>=4.53.2|✘|coding|[Tele-AI/TeleChat3-Coder-36B-Thinking](https://huggingface.co/Tele-AI/TeleChat3-Coder-36B-Thinking)| |[AI-ModelScope/Mistral-7B-Instruct-v0.1](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.1)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1)| |[AI-ModelScope/Mistral-7B-Instruct-v0.2](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.2)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)| |[LLM-Research/Mistral-7B-Instruct-v0.3](https://modelscope.cn/models/LLM-Research/Mistral-7B-Instruct-v0.3)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3)| diff --git a/docs/source_en/Instruction/Supported-models-and-datasets.md b/docs/source_en/Instruction/Supported-models-and-datasets.md index e7a6a3a951..670b13b79d 100644 --- a/docs/source_en/Instruction/Supported-models-and-datasets.md +++ b/docs/source_en/Instruction/Supported-models-and-datasets.md @@ -494,6 +494,7 @@ The table below introduces the models integrated with ms-swift: |[moonshotai/Kimi-K2-Instruct](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct](https://huggingface.co/moonshotai/Kimi-K2-Instruct)| |[moonshotai/Kimi-K2-Instruct-0905](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct-0905)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct-0905](https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905)| |[moonshotai/Kimi-K2-Thinking](https://modelscope.cn/models/moonshotai/Kimi-K2-Thinking)|deepseek_v3|kimi_k2||transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Thinking](https://huggingface.co/moonshotai/Kimi-K2-Thinking)| +|[TeleAI/TeleChat3-105B-A4.7B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-105B-A4.7B-Thinking)|deepseek_v3|telechat3|telechat3|transformers>=4.46.3|✘|-|-| |[deepseek-ai/DeepSeek-V3.1-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Base)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Base](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Base)| |[deepseek-ai/DeepSeek-V3.1](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1](https://huggingface.co/deepseek-ai/DeepSeek-V3.1)| |[deepseek-ai/DeepSeek-V3.1-Terminus](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Terminus)|deepseek_v3|deepseek_v3_1|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Terminus](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus)| @@ -548,6 +549,8 @@ The table below introduces the models integrated with ms-swift: |[TeleAI/TeleChat2-7B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-7B-32K)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-7B-32K](https://huggingface.co/Tele-AI/TeleChat2-7B-32K)| |[TeleAI/TeleChat2-35B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-35B-32K)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-35B-32K](https://huggingface.co/Tele-AI/TeleChat2-35B-32K)| |[TeleAI/TeleChat2-35B-Nov](https://modelscope.cn/models/TeleAI/TeleChat2-35B-Nov)|telechat2|telechat2||-|✘|-|[Tele-AI/TeleChat2-35B-Nov](https://huggingface.co/Tele-AI/TeleChat2-35B-Nov)| +|[TeleAI/TeleChat3-36B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-36B-Thinking)|telechat3|telechat3|telechat3|transformers>=4.53.2|✘|-|[Tele-AI/TeleChat3-36B-Thinking](https://huggingface.co/Tele-AI/TeleChat3-36B-Thinking)| +|[TeleAI/TeleChat3-Coder-36B-Thinking](https://modelscope.cn/models/TeleAI/TeleChat3-Coder-36B-Thinking)|telechat3|telechat3_coder|telechat3_coder|transformers>=4.53.2|✘|coding|[Tele-AI/TeleChat3-Coder-36B-Thinking](https://huggingface.co/Tele-AI/TeleChat3-Coder-36B-Thinking)| |[AI-ModelScope/Mistral-7B-Instruct-v0.1](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.1)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1)| |[AI-ModelScope/Mistral-7B-Instruct-v0.2](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.2)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)| |[LLM-Research/Mistral-7B-Instruct-v0.3](https://modelscope.cn/models/LLM-Research/Mistral-7B-Instruct-v0.3)|mistral|llama||transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3)| diff --git a/swift/agent_template/base.py b/swift/agent_template/base.py index 6ec8df4527..39f57dd988 100644 --- a/swift/agent_template/base.py +++ b/swift/agent_template/base.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union from swift.infer_engine import Function -from swift.template import Prompt, split_str_parts_by +from swift.template import Prompt, Tool, split_str_parts_by @dataclass @@ -168,6 +168,10 @@ def _add_tool_call_prefix(self, tool_content: str, pre_message=None) -> str: """ return tool_content + def get_toolcall_with_tools(self, response: str, tools: Optional[List[Tool]] = None) -> List[Function]: + """Parse tool calls with request tools; legacy parsers ignore the schemas.""" + return self.get_toolcall(response) + @staticmethod def _get_tool_name(tool): return tool.get('name_for_model') or tool.get('name') diff --git a/swift/agent_template/mapping.py b/swift/agent_template/mapping.py index ddecd0044b..823372292c 100644 --- a/swift/agent_template/mapping.py +++ b/swift/agent_template/mapping.py @@ -17,6 +17,7 @@ from .qwen3_coder import Qwen3_5AgentTemplate, Qwen3CoderAgentTemplate from .react import ReactEnAgentTemplate, ReactZnAgentTemplate from .seed_oss import SeedAgentTemplate +from .telechat3 import TeleChat3AgentTemplate, TeleChat3CoderAgentTemplate from .toolbench import ToolBenchAgentTemplate from .youtu import YoutuAgentTemplate @@ -50,6 +51,8 @@ 'minimax_m2': MinimaxM2AgentTemplate, 'minimax_m3': MinimaxM3AgentTemplate, 'seed_oss': SeedAgentTemplate, + 'telechat3': TeleChat3AgentTemplate, + 'telechat3_coder': TeleChat3CoderAgentTemplate, # ref: https://modelscope.cn/models/google/gemma-4-12B-it 'gemma4': Gemma4AgentTemplate, # extra diff --git a/swift/agent_template/telechat3.py b/swift/agent_template/telechat3.py new file mode 100644 index 0000000000..483172cbbb --- /dev/null +++ b/swift/agent_template/telechat3.py @@ -0,0 +1,152 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import json +import re +from typing import List, Optional, Tuple, Union + +from swift.infer_engine import Function +from swift.template import Prompt, Tool +from .base import BaseAgentTemplate + + +class TeleChat3AgentTemplate(BaseAgentTemplate): + + @staticmethod + def _dump_tool(tool) -> str: + return json.dumps(tool, ensure_ascii=False) + + @staticmethod + def _dump_arg_value(value) -> str: + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + + def get_toolcall(self, response: str) -> List[Function]: + functions = [] + for tool_call in re.findall(r'\s*(.*?)\s*', response, re.DOTALL): + tool_call = self._parse_json(tool_call.strip()) + if isinstance(tool_call, dict) and 'name' in tool_call: + functions.append(Function(name=tool_call['name'], arguments=tool_call.get('arguments') or {})) + return functions + + def _format_tools(self, tools: List[Union[str, dict]], system: Optional[str] = None, user_message=None) -> str: + tool_descs = '\n'.join(self._dump_tool(tool) for tool in tools) + tool_instruction = ('\n\n# 可用工具\n' + '你可以调用标签中包含的一个或多个工具来辅助你回答问题,以下是可用工具详情：\n' + '\n' + f'{tool_descs}\n' + '\n\n' + '# 调用方法\n' + '你需要遵循工具的要求，使用json格式返回工具名称及参数，并用包含。下方是一个调用模板：\n' + '\n' + '{"name": , "arguments": }\n' + '\n') + return (system or '') + tool_instruction + + def _format_tool_calls(self, tool_call_messages) -> str: + tool_calls = [] + for message in tool_call_messages: + tool_call = self._parse_tool_call(message['content']) + tool_calls.append(f'\n{json.dumps(tool_call, ensure_ascii=False)}\n') + return '\n'.join(tool_calls) + + @staticmethod + def _to_prompt(content: Union[str, Prompt]) -> Prompt: + return content if isinstance(content, list) else [content] + + def _format_tool_responses(self, assistant_content: Union[str, Prompt], tool_messages) -> Tuple[Prompt, Prompt]: + res: Prompt = [] + for i, tool_message in enumerate(tool_messages): + tool_content = tool_message['content'] + if i == 0: + res.append('<_user>\n') + else: + res.append('\n\n') + res.append(tool_content) + res.append('\n') + res.append('<_bot>') + return self._to_prompt(assistant_content) + ['<_end>\n'], res + + +class TeleChat3CoderAgentTemplate(TeleChat3AgentTemplate): + + def _add_tool_call_prefix(self, tool_content: str, pre_message=None) -> str: + if pre_message is None or pre_message['role'] != 'assistant': + return '' + tool_content + return tool_content + + def get_toolcall(self, response: str) -> List[Function]: + return self.get_toolcall_with_tools(response) + + def get_toolcall_with_tools(self, response: str, tools: Optional[List[Tool]] = None) -> List[Function]: + functions = [] + for block in re.findall(r'(.*?)', response, re.DOTALL): + name, args = self._parse_coder_tool_call(block, tools) + if name: + functions.append(Function(name=name, arguments=args)) + return functions + + def _parse_coder_tool_call(self, block: str, tools: Optional[List[Tool]] = None): + name = re.split(r'', block, maxsplit=1)[0].strip() + args = {} + pattern = re.compile(r'(.*?)\s*(.*?)', re.DOTALL) + for key, value in pattern.findall(block): + key = key.strip() + value = value.strip() + if self._is_string_arg(name, key, tools): + args[key] = value + else: + parsed_value = self._parse_json(value) + args[key] = parsed_value if parsed_value is not None or value in {'null', 'None'} else value + return name, args + + def _is_string_arg(self, tool_name, arg_name, tools): + # Match the model's official parser: only an exact schema type of "string" keeps the raw value. + for tool in tools or []: + if isinstance(tool, str): + tool = self._parse_json(tool) + if not isinstance(tool, dict): + continue + tool = self.unwrap_tool(tool) + if self._get_tool_name(tool) != tool_name: + continue + parameters = self._parse_json(tool.get('parameters') or {}) + if not isinstance(parameters, dict): + return False + properties = parameters.get('properties') or {} + if not isinstance(properties, dict): + return False + arg_schema = properties.get(arg_name) or {} + return isinstance(arg_schema, dict) and arg_schema.get('type') == 'string' + return False + + def _format_tools(self, tools: List[Union[str, dict]], system: Optional[str] = None, user_message=None) -> str: + tool_descs = '\n'.join(self._dump_tool(tool) for tool in tools) + tool_instruction = ( + '\n# Tools\n\n' + 'You may call one or more functions to assist with the user query.\n\n' + 'You are provided with function signatures within XML tags:\n' + '\n' + f'{tool_descs}\n' + '\n\n' + 'For each function call, output the function name and arguments within the following XML format:\n' + '{function-name}{param-key-1}{param-value-1}' + '{param-key-2}{param-value-2}...') + return (system or '') + tool_instruction + + def _format_tool_calls(self, tool_call_messages) -> str: + tool_calls = [] + for message in tool_call_messages: + tool_call = self._parse_tool_call(message['content']) + arguments = tool_call.get('arguments') or {} + parts = [f'{tool_call["name"]}'] + for key, value in arguments.items(): + parts.append(f'{key}{self._dump_arg_value(value)}') + parts.append('') + tool_calls.append(''.join(parts)) + return ''.join(tool_calls) + + def _format_tool_responses(self, assistant_content: Union[str, Prompt], tool_messages) -> Tuple[Prompt, Prompt]: + res: Prompt = ['<_observation>'] + for tool_message in tool_messages: + tool_content = tool_message['content'] + res += ['', tool_content, ''] + res.append('<_bot>') + return self._to_prompt(assistant_content) + ['<_end>'], res diff --git a/swift/infer_engine/grpo_vllm_engine.py b/swift/infer_engine/grpo_vllm_engine.py index 2c9254e3d7..9bffb42c9e 100644 --- a/swift/infer_engine/grpo_vllm_engine.py +++ b/swift/infer_engine/grpo_vllm_engine.py @@ -110,7 +110,7 @@ def _create_chat_completion_response(self, result, inputs, request_config, reque output.token_ids = list(output.token_ids) response = self.template.decode_generate_ids(output.token_ids, template_inputs=inputs['template_inputs']) logprobs = self._get_logprobs(output.logprobs, output.token_ids, request_config.top_logprobs) - toolcall = self._get_toolcall(response) + toolcall = self._get_toolcall(response, inputs['template_inputs'].tools) token_ids = output.token_ids if request_config.return_details else None choice = ChatCompletionResponseChoice( diff --git a/swift/infer_engine/infer_engine.py b/swift/infer_engine/infer_engine.py index b6a2e932eb..cdcadea9a7 100644 --- a/swift/infer_engine/infer_engine.py +++ b/swift/infer_engine/infer_engine.py @@ -9,7 +9,7 @@ from swift.metrics import Metric from swift.model import get_ckpt_dir -from swift.template import Template, get_template +from swift.template import Template, Tool, get_template from swift.utils import Processor, ProcessorMixin, get_logger from .base import BaseInferEngine from .protocol import (ChatCompletionMessageToolCall, ChatCompletionResponse, ChatCompletionStreamResponse, @@ -187,9 +187,11 @@ def infer(self, use_tqdm = not request_config.stream and len(infer_requests) > 1 return self._batch_infer_stream(tasks, request_config.stream, use_tqdm, metrics) - def _get_toolcall(self, response: str) -> Optional[List[ChatCompletionMessageToolCall]]: + def _get_toolcall(self, + response: str, + tools: Optional[List[Tool]] = None) -> Optional[List[ChatCompletionMessageToolCall]]: try: - functions = self.template.agent_template.get_toolcall(response) + functions = self.template.agent_template.get_toolcall_with_tools(response, tools) except Exception: functions = None if functions: diff --git a/swift/infer_engine/lmdeploy_engine.py b/swift/infer_engine/lmdeploy_engine.py index 61a9409e7b..db1276b4cb 100644 --- a/swift/infer_engine/lmdeploy_engine.py +++ b/swift/infer_engine/lmdeploy_engine.py @@ -226,7 +226,8 @@ async def _infer_stream_async( toolcall = None if is_finished: toolcall = self._get_toolcall( - self.template.decode_generate_ids(output.token_ids, template_inputs=inputs['template_inputs'])) + self.template.decode_generate_ids(output.token_ids, template_inputs=inputs['template_inputs']), + inputs['template_inputs'].tools) finish_reason = self._get_finish_reason(generation_config.max_new_tokens, output.num_token, output.status.name == 'FINISH') choices = [ @@ -265,7 +266,7 @@ async def _infer_full_async( logprobs = self._get_logprobs(output.logprobs, output.token_ids, request_config.top_logprobs) usage_info = self._get_usage_info(len(inputs['input_ids']), output.num_token) - toolcall = self._get_toolcall(response) + toolcall = self._get_toolcall(response, inputs['template_inputs'].tools) finish_reason = self._get_finish_reason(generation_config.max_new_tokens, output.num_token, output.status.name == 'FINISH') token_ids = output.token_ids if request_config.return_details else None diff --git a/swift/infer_engine/sglang_engine.py b/swift/infer_engine/sglang_engine.py index 15045e9881..71bdc6697c 100644 --- a/swift/infer_engine/sglang_engine.py +++ b/swift/infer_engine/sglang_engine.py @@ -186,7 +186,7 @@ def _create_chat_completion_response(self, output, inputs, return_details: bool meta_info = output['meta_info'] usage_info = self._get_usage_info(meta_info['prompt_tokens'], meta_info['completion_tokens']) response = self.template.decode_generate_ids(output['output_ids'], template_inputs=inputs['template_inputs']) - toolcall = self._get_toolcall(response) + toolcall = self._get_toolcall(response, inputs['template_inputs'].tools) token_ids = output['output_ids'] if return_details else None choice = ChatCompletionResponseChoice( index=0, @@ -290,7 +290,8 @@ def _create_chat_completion_stream_response(self, output, infer_streamer) -> Opt if is_finished: finish_reason = finish_reason['type'] toolcall = self._get_toolcall( - self.template.decode_generate_ids(output['output_ids'], **infer_streamer.decode_kwargs)) + self.template.decode_generate_ids(output['output_ids'], **infer_streamer.decode_kwargs), + infer_streamer.decode_kwargs['template_inputs'].tools) meta_info = output['meta_info'] usage_info = self._get_usage_info(meta_info['prompt_tokens'], meta_info['completion_tokens']) # TODO: logprobs diff --git a/swift/infer_engine/transformers_engine.py b/swift/infer_engine/transformers_engine.py index 98c356789d..e4f406ef7a 100644 --- a/swift/infer_engine/transformers_engine.py +++ b/swift/infer_engine/transformers_engine.py @@ -310,7 +310,8 @@ def _model_generate(**kwargs): toolcall = None if is_finished[i]: toolcall = self._get_toolcall( - self.template.decode_generate_ids(generate_ids, template_inputs=template_inputs[i])) + self.template.decode_generate_ids(generate_ids, template_inputs=template_inputs[i]), + template_inputs[i].tools) finish_reason = self._get_finish_reason(generation_config.max_new_tokens, usage_info.completion_tokens, is_finished[i]) @@ -436,7 +437,7 @@ def _infer_full(self, inputs: Dict[str, Any], *, generation_config: GenerationCo usage_info = self._update_usage_info(usage_info, len(generate_ids)) response = self.template.decode_generate_ids(generate_ids, template_inputs=template_inputs[i]) finish_reason = self._get_finish_reason(generation_config.max_new_tokens, len(generate_ids), True) - toolcall = self._get_toolcall(response) + toolcall = self._get_toolcall(response, template_inputs[i].tools) token_ids = generate_ids if request_config.return_details else None choices.append( ChatCompletionResponseChoice( diff --git a/swift/infer_engine/vllm_engine.py b/swift/infer_engine/vllm_engine.py index 03e5784f1b..b2b4a84350 100644 --- a/swift/infer_engine/vllm_engine.py +++ b/swift/infer_engine/vllm_engine.py @@ -641,7 +641,8 @@ def _create_chat_completion_stream_response(self, result, request_config, reques toolcall = None if output.is_finished: toolcall = self._get_toolcall( - self.template.decode_generate_ids(output.token_ids, **infer_streamers[i].decode_kwargs)) + self.template.decode_generate_ids(output.token_ids, **infer_streamers[i].decode_kwargs), + infer_streamers[i].decode_kwargs['template_inputs'].tools) choice = ChatCompletionResponseStreamChoice( index=i, @@ -711,7 +712,8 @@ def _create_chat_completion_response( content = response logprobs = self._get_logprobs(output.logprobs, output.token_ids, request_config.top_logprobs) - toolcall = self._get_toolcall(content) # Use content instead of response for tool calls + # Use content instead of response for tool calls. + toolcall = self._get_toolcall(content, inputs['template_inputs'].tools) token_ids = output.token_ids if request_config.return_details else None choice = ChatCompletionResponseChoice( index=output.index, diff --git a/swift/model/constant.py b/swift/model/constant.py index 5568f488e6..71b8649dc2 100644 --- a/swift/model/constant.py +++ b/swift/model/constant.py @@ -62,6 +62,7 @@ class LLMModelType: telechat = 'telechat' telechat2 = 'telechat2' + telechat3 = 'telechat3' mistral = 'mistral' devstral = 'devstral' diff --git a/swift/model/models/deepseek.py b/swift/model/models/deepseek.py index 3bcd40c8c9..267f164ed4 100644 --- a/swift/model/models/deepseek.py +++ b/swift/model/models/deepseek.py @@ -114,6 +114,11 @@ def get_model(self, model_dir: str, *args, **kwargs) -> PreTrainedModel: Model('moonshotai/Kimi-K2-Instruct-0905', 'moonshotai/Kimi-K2-Instruct-0905'), Model('moonshotai/Kimi-K2-Thinking', 'moonshotai/Kimi-K2-Thinking'), ], TemplateType.kimi_k2), + ModelGroup([ + Model('TeleAI/TeleChat3-105B-A4.7B-Thinking'), + ], + TemplateType.telechat3, + requires=['transformers>=4.46.3']), ModelGroup([ Model('deepseek-ai/DeepSeek-V3.1-Base', 'deepseek-ai/DeepSeek-V3.1-Base'), Model('deepseek-ai/DeepSeek-V3.1', 'deepseek-ai/DeepSeek-V3.1'), diff --git a/swift/model/models/telechat.py b/swift/model/models/telechat.py index 73217bf06e..ee4d4b6949 100644 --- a/swift/model/models/telechat.py +++ b/swift/model/models/telechat.py @@ -58,3 +58,22 @@ def get_model(self, model_dir: str, config, processor, **kwargs) -> PreTrainedMo model_arch=ModelArch.telechat, architectures=['TeleChat2ForCausalLM'], )) + +register_model( + ModelMeta( + LLMModelType.telechat3, + [ + ModelGroup([ + Model('TeleAI/TeleChat3-36B-Thinking', 'Tele-AI/TeleChat3-36B-Thinking'), + ]), + ModelGroup([ + Model('TeleAI/TeleChat3-Coder-36B-Thinking', 'Tele-AI/TeleChat3-Coder-36B-Thinking'), + ], + template=TemplateType.telechat3_coder, + tags=['coding']), + ], + template=TemplateType.telechat3, + model_arch=ModelArch.llama, + architectures=['TeleChat3ForCausalLM'], + requires=['transformers>=4.53.2'], + )) diff --git a/swift/template/constant.py b/swift/template/constant.py index a77a0f04e3..0ac9cf7eb1 100644 --- a/swift/template/constant.py +++ b/swift/template/constant.py @@ -79,6 +79,8 @@ class LLMTemplateType: minicpm5 = 'minicpm5' telechat = 'telechat' telechat2 = 'telechat2' + telechat3 = 'telechat3' + telechat3_coder = 'telechat3_coder' codefuse = 'codefuse' codefuse_codellama = 'codefuse_codellama' diff --git a/swift/template/templates/llm.py b/swift/template/templates/llm.py index deaec030de..195ec4b09f 100644 --- a/swift/template/templates/llm.py +++ b/swift/template/templates/llm.py @@ -1,4 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +import json from dataclasses import dataclass, field from datetime import datetime from typing import Optional @@ -8,7 +9,7 @@ from ..constant import LLMTemplateType, MLLMTemplateType from ..register import TemplateMeta, register_template from ..template_inputs import StdTemplateInputs -from ..utils import Prompt +from ..utils import Prompt, get_last_user_round from .llama import Llama3_2TemplateMeta from .qwen import Qwen2VLTemplate, QwenTemplateMeta from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta @@ -197,6 +198,303 @@ class TeleChatTemplateMeta(TemplateMeta): telechat_system = '你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。' register_template(TeleChatTemplateMeta(LLMTemplateType.telechat2, default_system=telechat_system)) + +class TeleChat3Template(Template): + + class _TruthyEmptySystem(str): + + def __bool__(self): + return True + + def _get_system(self, inputs): + system = super()._get_system(inputs) + if system == '' and not inputs.tools: + system = self._TruthyEmptySystem('') + return system + + def _get_response_prefix(self, inputs=None): + response_prefix = None if inputs is None else inputs.chat_template_kwargs.get('response_prefix') + if response_prefix is None: + response_prefix = self.response_prefix + if response_prefix is not None: + return response_prefix + if not self.use_chat_template: + return '' + return self.template_meta.thinking_prefix + + @staticmethod + def _to_content_segments(content): + if isinstance(content, list) and (not content or not isinstance(content[0], int)): + return content.copy() + return [content] + + @classmethod + def _concat_content(cls, pre_content, content): + if isinstance(pre_content, str) and isinstance(content, str): + return pre_content + content + return cls._to_content_segments(pre_content) + cls._to_content_segments(content) + + @staticmethod + def _expand_metadata(value, num_segments, key): + if isinstance(value, list): + if len(value) > num_segments: + raise ValueError(f'{key} has {len(value)} values for {num_segments} content segments.') + return value + [None] * (num_segments - len(value)) + return [value] * num_segments + + def _merge_assistant_messages(self, pre_message, message): + pre_segments = self._to_content_segments(pre_message['content']) + cur_segments = self._to_content_segments(message['content']) + pre_message['content'] = pre_segments + cur_segments + for key in ['loss', 'loss_scale']: + if key not in pre_message and key not in message: + continue + pre_values = self._expand_metadata(pre_message.get(key), len(pre_segments), key) + cur_values = self._expand_metadata(message.get(key), len(cur_segments), key) + pre_message[key] = pre_values + cur_values + + def _extend_assistant_metadata(self, message, old_num_segments): + new_num_segments = len(self._to_content_segments(message['content'])) + if new_num_segments <= old_num_segments: + return + for key in ['loss', 'loss_scale']: + if key not in message: + continue + values = message[key] + if not isinstance(values, list): + continue + values = self._expand_metadata(values, old_num_segments, key) + fill_value = values[-1] if values else None + message[key] = values + [fill_value] * (new_num_segments - old_num_segments) + + def _merge_natural_messages(self, inputs): + messages = inputs.messages + i = 1 + while i < len(messages): + pre_message, message = messages[i - 1], messages[i] + role = message['role'] + if pre_message['role'] != role or role not in {'assistant', 'user'}: + i += 1 + continue + pre_content = pre_message.get('content') + content = message.get('content') + pre_content = '' if pre_content is None else pre_content + content = '' if content is None else content + if role == 'assistant': + for key in ['loss', 'loss_scale']: + if key not in pre_message and key not in message: + continue + if key not in pre_message or key not in message or pre_message[key] != message[key]: + raise ValueError( + f'TeleChat3 cannot merge consecutive assistant messages with different `{key}` values. ' + 'Merge the messages before encoding or use the same value.') + pre_message['content'] = self._concat_content(pre_content, content) + tool_calls = list(pre_message.get('tool_calls') or []) + list(message.get('tool_calls') or []) + if tool_calls: + pre_message['tool_calls'] = tool_calls + pre_reasoning = pre_message.get('reasoning_content') + reasoning = message.get('reasoning_content') + if isinstance(reasoning, str): + pre_message['reasoning_content'] = (pre_reasoning or '') + reasoning + else: + pre_message['content'] = self._concat_content(pre_content, content) + messages.pop(i) + + def _prepare_assistant_thinking(self, inputs): + for message in inputs.messages: + if message['role'] != 'assistant': + continue + content = message['content'] + if isinstance(content, list) and (not content or not isinstance(content[0], int)): + for i, value in enumerate(content): + if isinstance(value, str): + content[i] = value.split('')[-1].lstrip('\n') + elif isinstance(content, str): + message['content'] = content.split('')[-1].lstrip('\n') + + def _preprocess_tool_call_jinja(self, inputs): + messages = inputs.messages + i = 0 + while i < len(messages): + if messages[i]['role'] != 'tool_call': + i += 1 + continue + i_start = i + while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool_call': + i += 1 + tool_calls = [] + for message in messages[i_start:i + 1]: + tool_call = self.agent_template._parse_tool_call(message['content']) + tool_calls.append({'type': 'function', 'function': tool_call}) + if i_start > 0 and messages[i_start - 1]['role'] == 'assistant': + assistant_message = messages[i_start - 1] + if assistant_message.get('content') is None: + assistant_message['content'] = '' + assistant_message['tool_calls'] = list(assistant_message.get('tool_calls') or []) + tool_calls + messages[i_start:i + 1] = [] + i = i_start + else: + messages[i_start:i + 1] = [{'role': 'assistant', 'content': '', 'tool_calls': tool_calls}] + i = i_start + 1 + + def _preprocess_structured_tool_calls_swift(self, inputs): + messages = inputs.messages + i = 0 + while i < len(messages): + message = messages[i] + tool_calls = message.pop('tool_calls', None) if message['role'] == 'assistant' else None + if not tool_calls: + i += 1 + continue + tool_call_messages = [] + for tool_call in tool_calls: + function = tool_call.get('function', tool_call) + arguments = function.get('arguments') or {} + tool_call_message = { + 'role': 'tool_call', + 'content': json.dumps({ + 'name': function['name'], + 'arguments': arguments + }, ensure_ascii=False) + } + for key in ['loss', 'loss_scale']: + value = message.get(key) + if key in message: + tool_call_message[key] = value[-1] if isinstance(value, list) and value else value + tool_call_messages.append(tool_call_message) + messages[i + 1:i + 1] = tool_call_messages + i += len(tool_call_messages) + 1 + + def _normalize_structured_tool_calls(self, inputs): + for message in inputs.messages: + if message['role'] != 'assistant': + continue + for tool_call in message.get('tool_calls') or []: + function = tool_call.get('function', tool_call) + arguments = function.get('arguments') + if isinstance(arguments, str): + parsed_arguments = self.agent_template._parse_json(arguments) + if parsed_arguments is not None: + function['arguments'] = parsed_arguments + + def _swift_prepare_inputs(self, inputs): + # Normalize natural assistant text before tool calls are expanded into strings. + self._normalize_structured_tool_calls(inputs) + self._merge_natural_messages(inputs) + if self.template_backend == 'jinja': + self._preprocess_tool_call_jinja(inputs) + return + self._prepare_assistant_thinking(inputs) + self._preprocess_structured_tool_calls_swift(inputs) + self._preprocess_tool_call(inputs) + messages = inputs.messages + if len(messages) < 2: + return + i = 1 + while i < len(messages): + pre_message, message = messages[i - 1], messages[i] + pre_role, pre_content = pre_message['role'], pre_message['content'] + role, content = message['role'], message['content'] + if pre_role == 'assistant' and role == 'tool' and self.template_backend == 'swift': + i_start = i + while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool': + i += 1 + old_num_segments = len(self._to_content_segments(pre_content)) + pre_message['content'], tool_content = self.agent_template._format_tool_responses( + pre_content, messages[i_start:i + 1]) + self._extend_assistant_metadata(pre_message, old_num_segments) + messages[i_start:i + 1] = [{'role': 'tool', 'content': tool_content}] + i = i_start + 1 + elif pre_role == 'assistant' and role == 'assistant' or pre_role == 'user' and role == 'user': + if self.template_backend == 'swift' and pre_role == 'assistant': + self._merge_assistant_messages(pre_message, message) + else: + pre_message['content'] = self._concat_content(pre_content, content) + messages.pop(i) + else: + i += 1 + + def _remove_history_thinking(self, inputs) -> None: + # Every assistant turn is already normalized against the model Jinja above. + pass + + +@dataclass +class TeleChat3TemplateMeta(TemplateMeta): + template_cls: type = TeleChat3Template + prefix: Prompt = field(default_factory=lambda: ['<_system>']) + prompt: Prompt = field(default_factory=lambda: ['<_user>{{QUERY}}<_bot>']) + chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<_end>\n']) + suffix: Prompt = field(default_factory=lambda: ['<_end>\n']) + system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<_system>{{SYSTEM}}\n']) + is_thinking: bool = True + thinking_prefix: str = '\n' + agent_template: Optional[str] = 'telechat3' + + +register_template(TeleChat3TemplateMeta(LLMTemplateType.telechat3)) + + +class TeleChat3CoderTemplate(TeleChat3Template): + + def __init__(self, *args, **kwargs): + if kwargs.get('enable_thinking') is None: + kwargs['enable_thinking'] = True + super().__init__(*args, **kwargs) + + def _prepare_assistant_thinking(self, inputs): + messages = inputs.messages + last_user_round = get_last_user_round(messages) + clear_thinking_defined = ('clear_thinking' in self.chat_template_kwargs + or 'clear_thinking' in inputs.chat_template_kwargs) + clear_thinking = inputs.chat_template_kwargs.get('clear_thinking', + self.chat_template_kwargs.get('clear_thinking')) + preserve_all = clear_thinking_defined and not clear_thinking + for i, message in enumerate(messages): + if message['role'] != 'assistant': + continue + preserve_reasoning = preserve_all or i > last_user_round + content = message['content'] + reasoning_content = message.pop('reasoning_content', None) + if isinstance(content, list) and (not content or not isinstance(content[0], int)): + for j, value in enumerate(content): + if isinstance(value, str): + reasoning = reasoning_content if j == 0 else None + content[j] = self._normalize_current_thinking(value, reasoning, preserve_reasoning) + elif isinstance(content, str): + message['content'] = self._normalize_current_thinking(content, reasoning_content, preserve_reasoning) + + @staticmethod + def _normalize_current_thinking(content: str, + reasoning_content: Optional[str] = None, + preserve_reasoning: bool = True) -> str: + if not isinstance(reasoning_content, str): + reasoning_content = '' + if '' in content: + reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') + content = content.split('')[-1].lstrip('\n') + has_reasoning = bool(reasoning_content) + reasoning_content = reasoning_content.strip() + content = content.strip() + if preserve_reasoning and has_reasoning: + return f'\n{reasoning_content}\n{content}' + return f'{content}' + + +@dataclass +class TeleChat3CoderTemplateMeta(TeleChat3TemplateMeta): + template_cls: type = TeleChat3CoderTemplate + chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<_end>']) + suffix: Prompt = field(default_factory=lambda: ['<_end>']) + system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<_system>{{SYSTEM}}']) + thinking_prefix: str = '' + non_thinking_prefix: str = '' + history_thinking_prefix: str = '' + agent_template: Optional[str] = 'telechat3_coder' + + +register_template(TeleChat3CoderTemplateMeta(LLMTemplateType.telechat3_coder)) + DBRX_SYSTEM = ( 'You are DBRX, created by Databricks. You were last updated in December 2023. ' 'You answer questions based on information available up to that point.\n' diff --git a/tests/test_align/test_template/test_agent.py b/tests/test_align/test_template/test_agent.py index 367341c107..b0969f0732 100644 --- a/tests/test_align/test_template/test_agent.py +++ b/tests/test_align/test_template/test_agent.py @@ -1,8 +1,12 @@ +import json import os +import pytest +from types import SimpleNamespace os.environ['SWIFT_DEBUG'] = '1' os.environ['CUDA_VISIBLE_DEVICES'] = '0,1,2,3' os.environ['ASCEND_RT_VISIBLE_DEVICES'] = '0,1,2,3' + system = 'You are a helpful assistant.' tools = [{ @@ -61,6 +65,8 @@ def _infer(engine, num_tools: int = 1, agent_tools=None, tool_messages=None, query=None): + from swift import InferRequest, RequestConfig + if agent_tools is None: agent_tools = tools if tool_messages is None: @@ -657,6 +663,477 @@ def test_deepseek_v4(): assert template.safe_decode(encoded['input_ids']) == expected_input_ids +telechat3_tools = [{ + 'type': 'function', + 'function': { + 'name': 'get_weather', + 'description': 'Get weather', + 'parameters': { + 'type': 'object', + 'properties': { + 'city': { + 'type': 'string' + }, + 'unit': { + 'type': 'string' + }, + }, + 'required': ['city'] + } + } +}] + +telechat3_tool_call = '{"name": "get_weather", "arguments": {"city": "Beijing", "unit": "celsius"}}' +telechat3_tool_call2 = '{"name": "get_weather", "arguments": {"city": "Shanghai", "unit": "fahrenheit"}}' + + +def _assert_template_backend_equal(template, data): + template.template_backend = 'swift' + template.set_mode('train') + encoded = template.encode(data) + template.template_backend = 'jinja' + encoded2 = template.encode(data) + assert encoded['input_ids'] == encoded2['input_ids'] + return encoded + + +def _assert_generation_backend_equal(template, data): + template.set_mode('transformers') + template.template_backend = 'swift' + encoded = template.encode(data) + template.template_backend = 'jinja' + encoded2 = template.encode(data) + assert encoded['input_ids'] == encoded2['input_ids'] + return template.safe_decode(encoded['input_ids']) + + +def _assert_telechat3_agent_template(model_id: str, template_type: str): + from swift import get_processor, get_template + from swift.model import get_matched_model_meta + + model_meta = get_matched_model_meta(model_id) + assert model_meta.model_type == 'telechat3' + assert model_meta.template == template_type + tokenizer = get_processor(model_id) + template = get_template(tokenizer) + assert template.template_meta.template_type == template_type + + for role in ['user', 'assistant']: + messages = [{'role': role, 'content': [101, 102]}, {'role': role, 'content': 'tail'}] + template._merge_natural_messages(SimpleNamespace(messages=messages)) + assert messages == [{'role': role, 'content': [[101, 102], 'tail']}] + + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'system', + 'content': 'You are a helpful assistant.' + }, { + 'role': 'user', + 'content': 'weather?' + }, { + 'role': 'tool_call', + 'content': telechat3_tool_call + }, { + 'role': 'tool_response', + 'content': '{"temperature": 22}' + }, { + 'role': 'assistant', + 'content': 'sunny' + }] + } + _assert_template_backend_equal(template, data) + + data['messages'].insert(2, {'role': 'assistant', 'content': 'I will check.'}) + _assert_template_backend_equal(template, data) + + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'system', + 'content': 'You are a helpful assistant.' + }, { + 'role': 'user', + 'content': 'weather in two cities?' + }, { + 'role': 'tool_call', + 'content': telechat3_tool_call + }, { + 'role': 'tool_call', + 'content': telechat3_tool_call2 + }, { + 'role': 'tool_response', + 'content': '{"temperature": 22}' + }, { + 'role': 'tool_response', + 'content': '{"temperature": 28}' + }, { + 'role': 'assistant', + 'content': 'Beijing is 22 and Shanghai is 28.' + }] + } + _assert_template_backend_equal(template, data) + + thinking_arg = 'beforeafter' + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'user', + 'content': 'echo the text' + }, { + 'role': 'tool_call', + 'content': json.dumps({ + 'name': 'get_weather', + 'arguments': { + 'city': thinking_arg + } + }) + }] + } + encoded = _assert_template_backend_equal(template, data) + assert thinking_arg in template.safe_decode(encoded['input_ids']) + + structured_tool_call = { + 'type': 'function', + 'function': { + 'name': 'get_weather', + 'arguments': { + 'city': thinking_arg + } + } + } + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'user', + 'content': 'echo the text' + }, { + 'role': 'assistant', + 'content': '', + 'tool_calls': [structured_tool_call] + }] + } + encoded = _assert_template_backend_equal(template, data) + assert thinking_arg in template.safe_decode(encoded['input_ids']) + + structured_tool_call['function']['arguments'] = json.dumps({'city': thinking_arg}, separators=(',', ':')) + encoded = _assert_template_backend_equal(template, data) + assert thinking_arg in template.safe_decode(encoded['input_ids']) + + for key, pre_value, tool_value in [('loss', False, True), ('loss_scale', 0.2, 0.8)]: + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'user', + 'content': 'weather?' + }, { + 'role': 'assistant', + 'content': 'I will check.', + key: pre_value + }, { + 'role': 'tool_call', + 'content': telechat3_tool_call, + key: tool_value + }, { + 'role': 'tool_response', + 'content': '{"temperature": 22}' + }, { + 'role': 'assistant', + 'content': 'sunny' + }] + } + template.template_backend = 'swift' + template.set_mode('train') + is_binary_loss_scale = template.is_binary_loss_scale + if key == 'loss_scale': + template.is_binary_loss_scale = False + encoded = template.encode(data) + template.is_binary_loss_scale = is_binary_loss_scale + assert len(encoded['input_ids']) == len(encoded['labels']) + if key == 'loss_scale': + assert len(encoded['input_ids']) == len(encoded['loss_scale']) + assert pre_value in encoded['loss_scale'] + assert tool_value in encoded['loss_scale'] + else: + labels = template.safe_decode(encoded['labels']) + assert 'I will check.' not in labels + assert 'Beijing' in labels + return template + + +def test_telechat3(): + from swift import agent_template_map + from swift.model import get_matched_model_meta + + template = _assert_telechat3_agent_template('TeleAI/TeleChat3-36B-Thinking', 'telechat3') + model_meta = get_matched_model_meta('TeleAI/TeleChat3-105B-A4.7B-Thinking') + assert model_meta.model_type == 'deepseek_v3' + assert model_meta.template == 'telechat3' + + data = { + 'messages': [{ + 'role': 'system', + 'content': '' + }, { + 'role': 'user', + 'content': 'hi' + }, { + 'role': 'assistant', + 'content': 'answer' + }] + } + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == '<_system>\n<_user>hi<_bot>answer<_end>\n' + + data = { + 'messages': [{ + 'role': 'user', + 'content': 'question' + }, { + 'role': 'assistant', + 'content': '\nplan\n\nanswer' + }] + } + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == '<_system><_user>question<_bot>answer<_end>\n' + data['chat_template_kwargs'] = {'preserve_thinking': True} + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == '<_system><_user>question<_bot>answer<_end>\n' + + data = {'messages': [{'role': 'user', 'content': 'q'}, {'role': 'assistant', 'content': ' answer\n'}]} + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == '<_system><_user>q<_bot> answer\n<_end>\n' + + data = {'chat_template_kwargs': {'enable_thinking': False}, 'messages': [{'role': 'user', 'content': 'q'}]} + assert _assert_generation_backend_equal(template, data) == '<_system><_user>q<_bot>\n' + + agent_template = agent_template_map['telechat3']() + functions = agent_template.get_toolcall(f'\n{telechat3_tool_call}\n\n' + f'\n{telechat3_tool_call2}\n') + assert len(functions) == 2 + assert functions[0].arguments == '{"city": "Beijing", "unit": "celsius"}' + assert functions[1].arguments == '{"city": "Shanghai", "unit": "fahrenheit"}' + + +def test_telechat3_infer(): + from swift import TransformersEngine + + engine = TransformersEngine('TeleAI/TeleChat3-36B-Thinking') + engine.template.template_backend = 'jinja' + messages = _infer( + engine, + num_tools=2, + agent_tools=telechat3_tools, + query='Use the get_weather tool to get the weather in Beijing. Return only a tool call.') + assert messages[-1]['content'] + + +def test_telechat3_coder(): + from swift import agent_template_map + + template = _assert_telechat3_agent_template('TeleAI/TeleChat3-Coder-36B-Thinking', 'telechat3_coder') + + data = {'messages': [{'role': 'user', 'content': 'hi'}, {'role': 'assistant', 'content': 'answer'}]} + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == '<_system><_user>hi<_bot>answer<_end>' + + data = { + 'messages': [{ + 'role': 'user', + 'content': 'hi' + }, { + 'role': 'assistant', + 'content': 'plananswer' + }, { + 'role': 'user', + 'content': 'again' + }, { + 'role': 'assistant', + 'content': 'done' + }] + } + encoded = _assert_template_backend_equal(template, data) + expected_input = '<_system><_user>hi<_bot>answer<_end><_user>again<_bot>done<_end>' + assert template.safe_decode(encoded['input_ids']) == expected_input + data['chat_template_kwargs'] = {'preserve_thinking': True} + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == expected_input + + for key, first_value, second_value in [('loss', False, True), ('loss_scale', 0.2, 0.8)]: + data = { + 'messages': [{ + 'role': 'user', + 'content': 'q' + }, { + 'role': 'assistant', + 'content': 'left', + key: first_value + }, { + 'role': 'assistant', + 'content': 'right', + key: second_value + }] + } + with pytest.raises(ValueError, match=rf'different `{key}` values'): + template.encode(data) + + data = { + 'messages': [{ + 'role': 'user', + 'content': 'question' + }, { + 'role': 'assistant', + 'content': '\nplan\n\nanswer' + }] + } + encoded = _assert_template_backend_equal(template, data) + expected_input = '<_system><_user>question<_bot>\nplan\nanswer<_end>' + assert template.safe_decode(encoded['input_ids']) == expected_input + + cases = [ + ({ + 'messages': [{ + 'role': 'user', + 'content': 'q' + }, { + 'role': 'assistant', + 'content': 'answer', + 'reasoning_content': 'plan' + }] + }, '<_system><_user>q<_bot>\nplan\nanswer<_end>'), + ({ + 'messages': [{ + 'role': 'user', + 'content': 'q' + }, { + 'role': 'assistant', + 'content': '\n\nanswer' + }] + }, '<_system><_user>q<_bot>answer<_end>'), + ({ + 'chat_template_kwargs': { + 'clear_thinking': False + }, + 'messages': [{ + 'role': 'user', + 'content': 'q' + }, { + 'role': 'assistant', + 'content': 'answer', + 'reasoning_content': ' ' + }] + }, '<_system><_user>q<_bot>\n\nanswer<_end>'), + ({ + 'messages': [{ + 'role': 'user', + 'content': 'q' + }, { + 'role': 'assistant', + 'content': 'prefix' + }, { + 'role': 'assistant', + 'content': 'xanswer' + }] + }, '<_system><_user>q<_bot>\nx\nanswer<_end>'), + ] + for data, expected_input in cases: + encoded = _assert_template_backend_equal(template, data) + assert template.safe_decode(encoded['input_ids']) == expected_input + + data = { + 'chat_template_kwargs': { + 'clear_thinking': False + }, + 'messages': [{ + 'role': 'user', + 'content': 'q1' + }, { + 'role': 'assistant', + 'content': 'p1a1' + }, { + 'role': 'user', + 'content': 'q2' + }, { + 'role': 'assistant', + 'content': 'p2a2' + }] + } + encoded = _assert_template_backend_equal(template, data) + expected_input = ('<_system><_user>q1<_bot>\np1\na1<_end>' + '<_user>q2<_bot>\np2\na2<_end>') + assert template.safe_decode(encoded['input_ids']) == expected_input + + data = {'chat_template_kwargs': {'enable_thinking': False}, 'messages': [{'role': 'user', 'content': 'q'}]} + assert _assert_generation_backend_equal(template, data) == '<_system><_user>q<_bot>' + + data = { + 'tools': + telechat3_tools, + 'messages': [{ + 'role': 'user', + 'content': 'q1' + }, { + 'role': 'tool_call', + 'content': telechat3_tool_call + }, { + 'role': 'tool_response', + 'content': '{"temperature": 22}' + }, { + 'role': 'assistant', + 'content': 'done' + }, { + 'role': 'user', + 'content': 'q2' + }] + } + rendered = _assert_generation_backend_equal(template, data) + assert '<_user>q1<_bot>' in rendered + + agent_template = agent_template_map['telechat3_coder']() + functions = agent_template.get_toolcall('get_weather' + 'cityBeijing' + 'get_weather' + 'cityShanghai' + '') + assert len(functions) == 2 + assert functions[0].arguments == '{"city": "Beijing"}' + assert functions[1].arguments == '{"city": "Shanghai"}' + response = ('get_weather' + ' city \n 22 ' + 'unit null' + '') + functions = agent_template.get_toolcall(response) + assert len(functions) == 1 + assert functions[0].arguments == '{"city": 22, "unit": null}' + functions = agent_template.get_toolcall_with_tools(response, telechat3_tools) + assert len(functions) == 1 + assert functions[0].arguments == '{"city": "22", "unit": "null"}' + + from swift.infer_engine.infer_engine import InferEngine + engine = SimpleNamespace(template=SimpleNamespace(agent_template=agent_template)) + tool_calls = InferEngine._get_toolcall(engine, response, telechat3_tools) + assert len(tool_calls) == 1 + assert tool_calls[0].function.arguments == '{"city": "22", "unit": "null"}' + + +def test_telechat3_coder_infer(): + from swift import TransformersEngine + + engine = TransformersEngine('TeleAI/TeleChat3-Coder-36B-Thinking', template_type='telechat3_coder') + engine.template.template_backend = 'jinja' + messages = _infer( + engine, + num_tools=2, + agent_tools=telechat3_tools, + query='Use the get_weather tool to get the weather in Beijing. Return only a tool call.') + assert messages[-1]['content'] + + def test_seed_oss(): engine = TransformersEngine('ByteDance-Seed/Seed-OSS-36B-Instruct', load_model=False)