diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 7b80bf43..6a43afe4 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -84,7 +84,37 @@ std::string OnnxChatGenerator::Decode() { int32_t token_id = next_tokens[0]; - // Decode through the normal tokenizer stream + // Fast path: use tag token IDs for efficient special-token detection. + // When tag IDs are configured (>= 0), we detect tool-call and reasoning tokens + // with a simple integer comparison, avoiding the expensive double-decode + string search. + const auto& tag_info = model_.GetTagInfo(); + bool has_tag_ids = (tag_info.bot_id >= 0 || tag_info.eot_id >= 0 || + tag_info.bor_id >= 0 || tag_info.eor_id >= 0); + + if (has_tag_ids) { + if (token_id == tag_info.bot_id) { + stream_->Decode(token_id); // keep normal stream in sync + return tag_info.bot_str; + } + if (token_id == tag_info.eot_id) { + stream_->Decode(token_id); + return tag_info.eot_str; + } + if (token_id == tag_info.bor_id) { + stream_->Decode(token_id); + return tag_info.bor_str; + } + if (token_id == tag_info.eor_id) { + stream_->Decode(token_id); + return tag_info.eor_str; + } + + // Not a tag token — decode normally (no special stream needed) + const char* token_text = stream_->Decode(token_id); + return token_text ? std::string(token_text) : ""; + } + + // Slow path: no tag IDs configured — fall back to double-decode + string matching. const char* token_text = stream_->Decode(token_id); // Also decode through the special-token stream to detect tool call and think tokens. diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 8f307c88..485abbe6 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -140,4 +140,32 @@ const std::vector& GenAIModelInstance::GetEosTokenIds() { return eos_token_ids_; } +const GenAIModelInstance::TagInfo& GenAIModelInstance::GetTagInfo() { + std::call_once(tag_info_init_flag_, [this]() { + if (!tokenizer_) return; + + // Get tag IDs from the tokenizer (reads from config, with fallback vocab lookup) + tag_info_.bot_id = tokenizer_->GetBotTokenId(); + tag_info_.eot_id = tokenizer_->GetEotTokenId(); + tag_info_.bor_id = tokenizer_->GetBorTokenId(); + tag_info_.eor_id = tokenizer_->GetEorTokenId(); + + // Decode each valid ID once through the special tokenizer to get the string. + // Uses tokenizer_with_special_ so that special token text (e.g., "") is produced. + auto decode_id = [this](int32_t id) -> std::string { + if (id < 0 || !tokenizer_with_special_) return {}; + OgaString text = tokenizer_with_special_->Decode(&id, 1); + const char* p = text; + return p ? std::string(p) : std::string(); + }; + + tag_info_.bot_str = decode_id(tag_info_.bot_id); + tag_info_.eot_str = decode_id(tag_info_.eot_id); + tag_info_.bor_str = decode_id(tag_info_.bor_id); + tag_info_.eor_str = decode_id(tag_info_.eor_id); + }); + + return tag_info_; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h index 912f10ba..8a6a78b3 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h @@ -36,6 +36,25 @@ class GenAIModelInstance { ExecutionProvider EP() const { return ep_; } bool IsMultiModal() const; + /// Cached tag token IDs and their decoded strings for efficient detection. + /// IDs are used for integer comparison in the decode loop (fast path). + /// Strings are used by ToolCallContext/Accumulator for text-based processing. + /// Populated once at first access via OgaTokenizer getters + tokenizer decode. + /// Naming follows bos/eos/pad convention: + /// bot = beginning of tool (call), eot = end of tool (call) + /// bor = beginning of reasoning, eor = end of reasoning + struct TagInfo { + int32_t bot_id{-1}; + int32_t eot_id{-1}; + int32_t bor_id{-1}; + int32_t eor_id{-1}; + std::string bot_str; + std::string eot_str; + std::string bor_str; + std::string eor_str; + }; + const TagInfo& GetTagInfo(); + /// Access the underlying OGA objects (for future chat generation work). OgaModel& GetOgaModel(); OgaTokenizer& GetOgaTokenizer(); @@ -76,6 +95,8 @@ class GenAIModelInstance { std::unique_ptr processor_; // nullptr if not multimodal std::vector eos_token_ids_; // cached; populated on first GetEosTokenIds() call std::once_flag eos_token_ids_init_flag_; + TagInfo tag_info_; // cached; populated on first GetTagInfo() call + std::once_flag tag_info_init_flag_; std::chrono::steady_clock::time_point last_activity_; mutable std::atomic session_ref_count_{0}; }; diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 1f06201c..1b8dc7ed 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -316,6 +316,36 @@ void Model::Load(ExecutionProvider ep) { if (result.status == ModelLoadManager::LoadStatus::kModelNotFound) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model not found at path: " + local_path_); } + + // Enrich ModelInfo with metadata from the loaded GenAI model (genai_config.json + fallback map). + // This makes tool/reasoning tags available via ModelInfo regardless of whether they came from + // catalog metadata, so downstream code doesn't need to know about multiple metadata sources. + if (result.model && result.status == ModelLoadManager::LoadStatus::kSuccess) { + const auto& tag_info = result.model->GetTagInfo(); + + auto enrich = [&](const std::string& val, const char* prop_key) { + if (!val.empty() && !info_.GetPropertyStr(prop_key)) { + info_.string_properties[prop_key] = val; + } + }; + + enrich(tag_info.bot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_START_STR); + enrich(tag_info.eot_str, FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR); + enrich(tag_info.bor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR); + enrich(tag_info.eor_str, FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR); + + // Infer support flags from the presence of valid tag IDs + if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT)) { + if (tag_info.bot_id >= 0) { + info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT] = 1; + } + } + if (!info_.GetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT)) { + if (tag_info.bor_id >= 0) { + info_.int_properties[FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT] = 1; + } + } + } } void Model::Unload() {