From f555f2545e8f9bac95abc8a91247bef2e1699c4e Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 24 Jun 2026 14:20:31 -0700 Subject: [PATCH 1/4] Read tool/reasoning tokens from GenAI model API instead of catalog metadata --- .../generative/chat/chat_session.cc | 45 ++++++------------- .../generative/genai_model_instance.cc | 9 ++++ .../generative/genai_model_instance.h | 5 +++ 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index eb76ef94d..b0f9040f3 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -117,51 +117,34 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const tool_ctx.tool_call_start = get_param(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); tool_ctx.tool_call_end = get_param(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); - // Fall back to model info properties if not specified in the request - const auto& info = CatalogModel().Info(); - - // Check if the model supports tool calling - const auto* tool_calling_val = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT); - if (tool_calling_val && *tool_calling_val == 1) { - tool_ctx.supports_tool_calling = true; - } - + // Fall back to GenAI model API (reads genai_config.json + model-family fallback map) if (tool_ctx.tool_call_start.empty()) { - const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); - if (val) { - tool_ctx.tool_call_start = *val; - } + tool_ctx.tool_call_start = Model().GetGenerationTag("tool_call_start"); } - if (tool_ctx.tool_call_end.empty()) { - const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); - if (val) { - tool_ctx.tool_call_end = *val; - } + tool_ctx.tool_call_end = Model().GetGenerationTag("tool_call_end"); } - // Check if the model supports chain-of-thought reasoning - const auto* reasoning_val = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT); - if (reasoning_val && *reasoning_val == 1) { - tool_ctx.supports_reasoning = true; + // Non-empty tool_call_start implies the model supports tool calling + if (!tool_ctx.tool_call_start.empty()) { + tool_ctx.supports_tool_calling = true; } // Read reasoning marker tokens — same pattern as tool_call tokens tool_ctx.reasoning_start = get_param(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); tool_ctx.reasoning_end = get_param(FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + // Fall back to GenAI model API for reasoning tokens if (tool_ctx.reasoning_start.empty()) { - const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); - if (val) { - tool_ctx.reasoning_start = *val; - } + tool_ctx.reasoning_start = Model().GetGenerationTag("reasoning_start"); } - if (tool_ctx.reasoning_end.empty()) { - const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); - if (val) { - tool_ctx.reasoning_end = *val; - } + tool_ctx.reasoning_end = Model().GetGenerationTag("reasoning_end"); + } + + // Non-empty reasoning_start implies the model supports reasoning + if (!tool_ctx.reasoning_start.empty()) { + tool_ctx.supports_reasoning = true; } // Accumulate tool definitions from the session. diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 8f307c88a..b3936106f 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -111,6 +111,15 @@ OgaModel& GenAIModelInstance::GetOgaModel() { return *oga_model_; } +std::string GenAIModelInstance::GetGenerationTag(const char* tag_name) const { + if (!oga_model_) { + return {}; + } + OgaString tag = oga_model_->GetGenerationTag(tag_name); + const char* p = tag; + return p ? std::string(p) : std::string(); +} + OgaTokenizer& GenAIModelInstance::GetOgaTokenizer() { if (!tokenizer_) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "OGA tokenizer is null"); diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 912f10ba1..b592a8965 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -36,6 +36,11 @@ class GenAIModelInstance { ExecutionProvider EP() const { return ep_; } bool IsMultiModal() const; + /// Query a generation tag from the GenAI model (reads genai_config.json with model-family fallback). + /// Returns the tag value or empty string if not found. Supported keys: + /// "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". + std::string GetGenerationTag(const char* tag_name) const; + /// Access the underlying OGA objects (for future chat generation work). OgaModel& GetOgaModel(); OgaTokenizer& GetOgaTokenizer(); From 42c3fc5943098febdc041a70c6cb49f0175ad1d3 Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Fri, 26 Jun 2026 11:34:26 -0700 Subject: [PATCH 2/4] Refactor: enrich ModelInfo at load time instead of reading GenAI in BuildToolCallContext --- .../generative/chat/chat_session.cc | 45 +++++++++++++------ .../generative/genai_model_instance.cc | 4 +- .../generative/genai_model_instance.h | 4 +- sdk_v2/cpp/src/model.cc | 31 +++++++++++++ 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index b0f9040f3..eb76ef94d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -117,34 +117,51 @@ ToolCallContext ChatSession::BuildToolCallContext(const Request& request) const tool_ctx.tool_call_start = get_param(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); tool_ctx.tool_call_end = get_param(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); - // Fall back to GenAI model API (reads genai_config.json + model-family fallback map) + // Fall back to model info properties if not specified in the request + const auto& info = CatalogModel().Info(); + + // Check if the model supports tool calling + const auto* tool_calling_val = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT); + if (tool_calling_val && *tool_calling_val == 1) { + tool_ctx.supports_tool_calling = true; + } + if (tool_ctx.tool_call_start.empty()) { - tool_ctx.tool_call_start = Model().GetGenerationTag("tool_call_start"); + const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + if (val) { + tool_ctx.tool_call_start = *val; + } } + if (tool_ctx.tool_call_end.empty()) { - tool_ctx.tool_call_end = Model().GetGenerationTag("tool_call_end"); + const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + if (val) { + tool_ctx.tool_call_end = *val; + } } - // Non-empty tool_call_start implies the model supports tool calling - if (!tool_ctx.tool_call_start.empty()) { - tool_ctx.supports_tool_calling = true; + // Check if the model supports chain-of-thought reasoning + const auto* reasoning_val = info.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT); + if (reasoning_val && *reasoning_val == 1) { + tool_ctx.supports_reasoning = true; } // Read reasoning marker tokens — same pattern as tool_call tokens tool_ctx.reasoning_start = get_param(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); tool_ctx.reasoning_end = get_param(FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); - // Fall back to GenAI model API for reasoning tokens if (tool_ctx.reasoning_start.empty()) { - tool_ctx.reasoning_start = Model().GetGenerationTag("reasoning_start"); - } - if (tool_ctx.reasoning_end.empty()) { - tool_ctx.reasoning_end = Model().GetGenerationTag("reasoning_end"); + const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + if (val) { + tool_ctx.reasoning_start = *val; + } } - // Non-empty reasoning_start implies the model supports reasoning - if (!tool_ctx.reasoning_start.empty()) { - tool_ctx.supports_reasoning = true; + if (tool_ctx.reasoning_end.empty()) { + const auto* val = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + if (val) { + tool_ctx.reasoning_end = *val; + } } // Accumulate tool definitions from the session. diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index b3936106f..7b1d9c21e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -111,11 +111,11 @@ OgaModel& GenAIModelInstance::GetOgaModel() { return *oga_model_; } -std::string GenAIModelInstance::GetGenerationTag(const char* tag_name) const { +std::string GenAIModelInstance::GetTag(const char* tag_name) const { if (!oga_model_) { return {}; } - OgaString tag = oga_model_->GetGenerationTag(tag_name); + OgaString tag = oga_model_->GetTag(tag_name); const char* p = tag; return p ? std::string(p) : std::string(); } diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index b592a8965..3d62ce065 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -36,10 +36,10 @@ class GenAIModelInstance { ExecutionProvider EP() const { return ep_; } bool IsMultiModal() const; - /// Query a generation tag from the GenAI model (reads genai_config.json with model-family fallback). + /// Query a tag from the GenAI model (reads genai_config.json with model-family fallback). /// Returns the tag value or empty string if not found. Supported keys: /// "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - std::string GetGenerationTag(const char* tag_name) const; + std::string GetTag(const char* tag_name) const; /// Access the underlying OGA objects (for future chat generation work). OgaModel& GetOgaModel(); diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index d7f2b0c7a..08db20273 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -218,6 +218,37 @@ void Model::Load(ExecutionProvider ep) { if (result.status == ModelLoadManager::LoadStatus::kModelNotFound) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model not found at path: " + local_path_); } + + // Enrich ModelInfo with metadata from the loaded GenAI model (genai_config.json + fallback map). + // This makes tool/reasoning tags available via ModelInfo regardless of whether they came from + // catalog metadata, so downstream code doesn't need to know about multiple metadata sources. + if (result.model && result.status == ModelLoadManager::LoadStatus::kSuccess) { + auto enrich = [&](const char* tag_name, const char* prop_key) { + std::string val = result.model->GetTag(tag_name); + if (!val.empty() && !info_.GetPropertyStr(prop_key)) { + info_.string_properties[prop_key] = std::move(val); + } + }; + + enrich("tool_call_start", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + enrich("tool_call_end", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + enrich("reasoning_start", FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + enrich("reasoning_end", FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + + // Infer support flags from the presence of start tokens + if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT)) { + const auto* tc = info_.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + if (tc && !tc->empty()) { + info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = 1; + } + } + if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT)) { + const auto* rs = info_.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + if (rs && !rs->empty()) { + info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT] = 1; + } + } + } } void Model::Unload() { From bd73d9846384c100674f9a48e5fda5add4b1b72b Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Tue, 30 Jun 2026 17:28:16 -0700 Subject: [PATCH 3/4] Optimize tool/reasoning detection with token ID comparison in decode loop --- .../generative/chat/onnx_chat_generator.cc | 32 +++++++++++++++- .../generative/genai_model_instance.cc | 37 ++++++++++++++----- .../generative/genai_model_instance.h | 21 +++++++++-- sdk_v2/cpp/src/model.cc | 23 ++++++------ 4 files changed, 87 insertions(+), 26 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 7b80bf430..86c600ce4 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -84,7 +84,37 @@ std::string OnnxChatGenerator::Decode() { int32_t token_id = next_tokens[0]; - // Decode through the normal tokenizer stream + // Fast path: use tag token IDs for efficient special-token detection. + // When tag IDs are configured (>= 0), we detect tool-call and reasoning tokens + // with a simple integer comparison, avoiding the expensive double-decode + string search. + const auto& tag_info = model_.GetTagInfo(); + bool has_tag_ids = (tag_info.tool_call_start_id >= 0 || tag_info.tool_call_end_id >= 0 || + tag_info.reasoning_start_id >= 0 || tag_info.reasoning_end_id >= 0); + + if (has_tag_ids) { + if (token_id == tag_info.tool_call_start_id) { + stream_->Decode(token_id); // keep normal stream in sync + return tag_info.tool_call_start_str; + } + if (token_id == tag_info.tool_call_end_id) { + stream_->Decode(token_id); + return tag_info.tool_call_end_str; + } + if (token_id == tag_info.reasoning_start_id) { + stream_->Decode(token_id); + return tag_info.reasoning_start_str; + } + if (token_id == tag_info.reasoning_end_id) { + stream_->Decode(token_id); + return tag_info.reasoning_end_str; + } + + // Not a tag token — decode normally (no special stream needed) + const char* token_text = stream_->Decode(token_id); + return token_text ? std::string(token_text) : ""; + } + + // Slow path: no tag IDs configured — fall back to double-decode + string matching. const char* token_text = stream_->Decode(token_id); // Also decode through the special-token stream to detect tool call and think tokens. diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 7b1d9c21e..774a56118 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -111,15 +111,6 @@ OgaModel& GenAIModelInstance::GetOgaModel() { return *oga_model_; } -std::string GenAIModelInstance::GetTag(const char* tag_name) const { - if (!oga_model_) { - return {}; - } - OgaString tag = oga_model_->GetTag(tag_name); - const char* p = tag; - return p ? std::string(p) : std::string(); -} - OgaTokenizer& GenAIModelInstance::GetOgaTokenizer() { if (!tokenizer_) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "OGA tokenizer is null"); @@ -149,4 +140,32 @@ const std::vector& GenAIModelInstance::GetEosTokenIds() { return eos_token_ids_; } +const GenAIModelInstance::TagInfo& GenAIModelInstance::GetTagInfo() { + std::call_once(tag_info_init_flag_, [this]() { + if (!oga_model_) return; + + // Get tag IDs from GenAI (config first, then fallback-encoded via tokenizer vocab lookup) + tag_info_.tool_call_start_id = oga_model_->GetTagId("tool_call_start"); + tag_info_.tool_call_end_id = oga_model_->GetTagId("tool_call_end"); + tag_info_.reasoning_start_id = oga_model_->GetTagId("reasoning_start"); + tag_info_.reasoning_end_id = oga_model_->GetTagId("reasoning_end"); + + // Decode each valid ID once through the special tokenizer to get the string. + // Uses tokenizer_with_special_ so that special token text (e.g., "") is produced. + auto decode_id = [this](int32_t id) -> std::string { + if (id < 0 || !tokenizer_with_special_) return {}; + OgaString text = tokenizer_with_special_->Decode(&id, 1); + const char* p = text; + return p ? std::string(p) : std::string(); + }; + + tag_info_.tool_call_start_str = decode_id(tag_info_.tool_call_start_id); + tag_info_.tool_call_end_str = decode_id(tag_info_.tool_call_end_id); + tag_info_.reasoning_start_str = decode_id(tag_info_.reasoning_start_id); + tag_info_.reasoning_end_str = decode_id(tag_info_.reasoning_end_id); + }); + + return tag_info_; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 3d62ce065..8bfff5aaf 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -36,10 +36,21 @@ class GenAIModelInstance { ExecutionProvider EP() const { return ep_; } bool IsMultiModal() const; - /// Query a tag from the GenAI model (reads genai_config.json with model-family fallback). - /// Returns the tag value or empty string if not found. Supported keys: - /// "tool_call_start", "tool_call_end", "reasoning_start", "reasoning_end". - std::string GetTag(const char* tag_name) const; + /// Cached tag token IDs and their decoded strings for efficient detection. + /// IDs are used for integer comparison in the decode loop (fast path). + /// Strings are used by ToolCallContext/Accumulator for text-based processing. + /// Populated once at first access via OgaModel::GetTagId + tokenizer decode. + struct TagInfo { + int32_t tool_call_start_id{-1}; + int32_t tool_call_end_id{-1}; + int32_t reasoning_start_id{-1}; + int32_t reasoning_end_id{-1}; + std::string tool_call_start_str; + std::string tool_call_end_str; + std::string reasoning_start_str; + std::string reasoning_end_str; + }; + const TagInfo& GetTagInfo(); /// Access the underlying OGA objects (for future chat generation work). OgaModel& GetOgaModel(); @@ -81,6 +92,8 @@ class GenAIModelInstance { std::unique_ptr processor_; // nullptr if not multimodal std::vector eos_token_ids_; // cached; populated on first GetEosTokenIds() call std::once_flag eos_token_ids_init_flag_; + TagInfo tag_info_; // cached; populated on first GetTagInfo() call + std::once_flag tag_info_init_flag_; std::chrono::steady_clock::time_point last_activity_; mutable std::atomic session_ref_count_{0}; }; diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 08db20273..6d12b70b5 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -223,28 +223,27 @@ void Model::Load(ExecutionProvider ep) { // This makes tool/reasoning tags available via ModelInfo regardless of whether they came from // catalog metadata, so downstream code doesn't need to know about multiple metadata sources. if (result.model && result.status == ModelLoadManager::LoadStatus::kSuccess) { - auto enrich = [&](const char* tag_name, const char* prop_key) { - std::string val = result.model->GetTag(tag_name); + const auto& tag_info = result.model->GetTagInfo(); + + auto enrich = [&](const std::string& val, const char* prop_key) { if (!val.empty() && !info_.GetPropertyStr(prop_key)) { - info_.string_properties[prop_key] = std::move(val); + info_.string_properties[prop_key] = val; } }; - enrich("tool_call_start", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); - enrich("tool_call_end", FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); - enrich("reasoning_start", FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); - enrich("reasoning_end", FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + enrich(tag_info.tool_call_start_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + enrich(tag_info.tool_call_end_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + enrich(tag_info.reasoning_start_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + enrich(tag_info.reasoning_end_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); - // Infer support flags from the presence of start tokens + // Infer support flags from the presence of valid tag IDs if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT)) { - const auto* tc = info_.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); - if (tc && !tc->empty()) { + if (tag_info.tool_call_start_id >= 0) { info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = 1; } } if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT)) { - const auto* rs = info_.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); - if (rs && !rs->empty()) { + if (tag_info.reasoning_start_id >= 0) { info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT] = 1; } } From 77886a51cb0803fa6363331181222a765e2ff36c Mon Sep 17 00:00:00 2001 From: Sayan Shaw Date: Wed, 8 Jul 2026 13:39:43 -0700 Subject: [PATCH 4/4] Refactor: adopt bot/eot/bor/eor naming, use Tokenizer getters --- .../generative/chat/onnx_chat_generator.cc | 20 +++++++++--------- .../generative/genai_model_instance.cc | 20 +++++++++--------- .../generative/genai_model_instance.h | 21 +++++++++++-------- sdk_v2/cpp/src/model.cc | 12 +++++------ 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 86c600ce4..6a43afe40 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -88,25 +88,25 @@ std::string OnnxChatGenerator::Decode() { // When tag IDs are configured (>= 0), we detect tool-call and reasoning tokens // with a simple integer comparison, avoiding the expensive double-decode + string search. const auto& tag_info = model_.GetTagInfo(); - bool has_tag_ids = (tag_info.tool_call_start_id >= 0 || tag_info.tool_call_end_id >= 0 || - tag_info.reasoning_start_id >= 0 || tag_info.reasoning_end_id >= 0); + bool has_tag_ids = (tag_info.bot_id >= 0 || tag_info.eot_id >= 0 || + tag_info.bor_id >= 0 || tag_info.eor_id >= 0); if (has_tag_ids) { - if (token_id == tag_info.tool_call_start_id) { + if (token_id == tag_info.bot_id) { stream_->Decode(token_id); // keep normal stream in sync - return tag_info.tool_call_start_str; + return tag_info.bot_str; } - if (token_id == tag_info.tool_call_end_id) { + if (token_id == tag_info.eot_id) { stream_->Decode(token_id); - return tag_info.tool_call_end_str; + return tag_info.eot_str; } - if (token_id == tag_info.reasoning_start_id) { + if (token_id == tag_info.bor_id) { stream_->Decode(token_id); - return tag_info.reasoning_start_str; + return tag_info.bor_str; } - if (token_id == tag_info.reasoning_end_id) { + if (token_id == tag_info.eor_id) { stream_->Decode(token_id); - return tag_info.reasoning_end_str; + return tag_info.eor_str; } // Not a tag token — decode normally (no special stream needed) diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 774a56118..485abbe6b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -142,13 +142,13 @@ const std::vector& GenAIModelInstance::GetEosTokenIds() { const GenAIModelInstance::TagInfo& GenAIModelInstance::GetTagInfo() { std::call_once(tag_info_init_flag_, [this]() { - if (!oga_model_) return; + if (!tokenizer_) return; - // Get tag IDs from GenAI (config first, then fallback-encoded via tokenizer vocab lookup) - tag_info_.tool_call_start_id = oga_model_->GetTagId("tool_call_start"); - tag_info_.tool_call_end_id = oga_model_->GetTagId("tool_call_end"); - tag_info_.reasoning_start_id = oga_model_->GetTagId("reasoning_start"); - tag_info_.reasoning_end_id = oga_model_->GetTagId("reasoning_end"); + // Get tag IDs from the tokenizer (reads from config, with fallback vocab lookup) + tag_info_.bot_id = tokenizer_->GetBotTokenId(); + tag_info_.eot_id = tokenizer_->GetEotTokenId(); + tag_info_.bor_id = tokenizer_->GetBorTokenId(); + tag_info_.eor_id = tokenizer_->GetEorTokenId(); // Decode each valid ID once through the special tokenizer to get the string. // Uses tokenizer_with_special_ so that special token text (e.g., "") is produced. @@ -159,10 +159,10 @@ const GenAIModelInstance::TagInfo& GenAIModelInstance::GetTagInfo() { return p ? std::string(p) : std::string(); }; - tag_info_.tool_call_start_str = decode_id(tag_info_.tool_call_start_id); - tag_info_.tool_call_end_str = decode_id(tag_info_.tool_call_end_id); - tag_info_.reasoning_start_str = decode_id(tag_info_.reasoning_start_id); - tag_info_.reasoning_end_str = decode_id(tag_info_.reasoning_end_id); + tag_info_.bot_str = decode_id(tag_info_.bot_id); + tag_info_.eot_str = decode_id(tag_info_.eot_id); + tag_info_.bor_str = decode_id(tag_info_.bor_id); + tag_info_.eor_str = decode_id(tag_info_.eor_id); }); return tag_info_; diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 8bfff5aaf..8a6a78b3b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -39,16 +39,19 @@ class GenAIModelInstance { /// Cached tag token IDs and their decoded strings for efficient detection. /// IDs are used for integer comparison in the decode loop (fast path). /// Strings are used by ToolCallContext/Accumulator for text-based processing. - /// Populated once at first access via OgaModel::GetTagId + tokenizer decode. + /// Populated once at first access via OgaTokenizer getters + tokenizer decode. + /// Naming follows bos/eos/pad convention: + /// bot = beginning of tool (call), eot = end of tool (call) + /// bor = beginning of reasoning, eor = end of reasoning struct TagInfo { - int32_t tool_call_start_id{-1}; - int32_t tool_call_end_id{-1}; - int32_t reasoning_start_id{-1}; - int32_t reasoning_end_id{-1}; - std::string tool_call_start_str; - std::string tool_call_end_str; - std::string reasoning_start_str; - std::string reasoning_end_str; + int32_t bot_id{-1}; + int32_t eot_id{-1}; + int32_t bor_id{-1}; + int32_t eor_id{-1}; + std::string bot_str; + std::string eot_str; + std::string bor_str; + std::string eor_str; }; const TagInfo& GetTagInfo(); diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 6d12b70b5..aba4f76f1 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -231,19 +231,19 @@ void Model::Load(ExecutionProvider ep) { } }; - enrich(tag_info.tool_call_start_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); - enrich(tag_info.tool_call_end_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); - enrich(tag_info.reasoning_start_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); - enrich(tag_info.reasoning_end_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + enrich(tag_info.bot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + enrich(tag_info.eot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + enrich(tag_info.bor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + enrich(tag_info.eor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); // Infer support flags from the presence of valid tag IDs if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT)) { - if (tag_info.tool_call_start_id >= 0) { + if (tag_info.bot_id >= 0) { info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = 1; } } if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT)) { - if (tag_info.reasoning_start_id >= 0) { + if (tag_info.bor_id >= 0) { info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT] = 1; } }