[feat](inverted-index) Add IK tokenizers to custom analyzers

be/src/exprs/function/function_search.cpp

@@ -307,6 +307,29 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param(
         const IndexExecContext* index_exec_ctx,
         const std::unordered_map<std::string, int>& field_name_to_column_id,
         const std::shared_ptr<IndexQueryContext>& index_query_context) const {
+    // VSearchExpr enters here directly, outside IFunction::execute() and its exception boundary,
+    // so this is where a failure inside the search, such as an analyzer whose first token
+    // stream throws, has to become a Status.
+    try {
+        return evaluate_inverted_index_with_search_param_unguarded(
+                search_param, data_type_with_names, std::move(iterators), num_rows, bitmap_result,
+                enable_cache, index_exec_ctx, field_name_to_column_id, index_query_context);
+    } catch (const CLuceneError& e) {
+        return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>("search failed: {}",
+                                                                       e.what());
+    } catch (const Exception& e) {
+        return e.to_status();
+    }
+}
+
+Status FunctionSearch::evaluate_inverted_index_with_search_param_unguarded(
+        const TSearchParam& search_param,
+        const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
+        std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
+        InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+        const IndexExecContext* index_exec_ctx,
+        const std::unordered_map<std::string, int>& field_name_to_column_id,
+        const std::shared_ptr<IndexQueryContext>& index_query_context) const {
     const bool is_nested_query = search_param.root.clause_type == "NESTED";
     if (is_nested_query && !is_nested_group_search_supported()) {
         return Status::NotSupported(

be/src/exprs/function/function_search.h

@@ -93,6 +93,16 @@ class FunctionSearch : public IFunction {
             const std::unordered_map<std::string, int>& field_name_to_column_id,
             const std::shared_ptr<IndexQueryContext>& index_query_context = nullptr) const;
 
+    // The body of the overload above; it may throw, so only that overload calls it.
+    Status evaluate_inverted_index_with_search_param_unguarded(
+            const TSearchParam& search_param,
+            const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
+            std::unordered_map<std::string, IndexIterator*> iterators, uint32_t num_rows,
+            InvertedIndexResultBitmap& bitmap_result, bool enable_cache,
+            const IndexExecContext* index_exec_ctx,
+            const std::unordered_map<std::string, int>& field_name_to_column_id,
+            const std::shared_ptr<IndexQueryContext>& index_query_context) const;
+
     // Public methods for testing
     enum class ClauseTypeCategory {
         NON_TOKENIZED, // TERM, PREFIX, WILDCARD, REGEXP, RANGE, LIST - no tokenization, use EQUAL_QUERY

be/src/exprs/function/match.cpp

@@ -62,14 +62,7 @@ Status FunctionMatchBase::evaluate_inverted_index(
     }
     const std::string& function_name = get_name();
 
-    if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION ||
-        function_name == MATCH_PHRASE_EDGE_FUNCTION) {
-        auto reader = iter->get_reader(InvertedIndexReaderType::FULLTEXT);
-        if (reader && !segment_v2::IndexReaderHelper::is_support_phrase(reader)) {
-            return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
-                    "phrase queries require setting support_phrase = true");
-        }
-    }
+    // support_phrase is checked once the analyzer has selected the reader that runs the query.
     Field param_value;
     arguments[0].column->get(0, param_value);
     if (param_value.is_null()) {

be/src/exprs/function/variant_inverted_index_search.cpp

@@ -65,7 +65,7 @@ void add_search_binding_diagnostic(const std::shared_ptr<IndexQueryContext>& con
     }
 }
 
-InvertedIndexAnalyzerCtxSPtr build_analyzer_context(
+InvertedIndexAnalyzerCtxSPtr build_analyzer_context_unsafe(
         const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
     InvertedIndexAnalyzerConfig config;
     config.analyzer_name = get_analyzer_name_from_properties(properties);
@@ -90,6 +90,20 @@ InvertedIndexAnalyzerCtxSPtr build_analyzer_context(
 
 } // namespace
 
+// Replayed components can collide across policy families, so building the provider throws.
+Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
+        const std::map<std::string, std::string>& properties, const std::string& analyzer_key) {
+    try {
+        return build_analyzer_context_unsafe(properties, analyzer_key);
+    } catch (const CLuceneError& error) {
+        return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+                "Build search analyzer failed: {}", error.what()));
+    } catch (const Exception& error) {
+        return ResultError(Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
+                "Build search analyzer failed: {}", error.what()));
+    }
+}
+
 FieldReaderResolver::FieldReaderResolver(
         const std::unordered_map<std::string, IndexFieldNameAndTypePair>& data_type_with_names,
         const std::unordered_map<std::string, IndexIterator*>& iterators,
@@ -415,8 +429,12 @@ Status FieldReaderResolver::resolve_with_analyzer_context(const std::string& fie
         return Status::OK();
     }
 
-    binding->analyzer_context =
-            build_analyzer_context(binding->index_properties, binding->analyzer_key);
+    auto built_context =
+            build_search_analyzer_context(binding->index_properties, binding->analyzer_key);
+    if (!built_context.has_value()) {
+        return built_context.error();
+    }
+    binding->analyzer_context = std::move(built_context.value());
     _cache.at(binding->binding_key).analyzer_context = binding->analyzer_context;
     return Status::OK();
 }

be/src/exprs/function/variant_inverted_index_search.h

@@ -102,6 +102,11 @@ struct FieldReaderBinding {
     }
 };
 
+// Build the analyzer context a SEARCH binding executes with, converting a failure to build the
+// analyzer provider into a Status instead of letting the exception escape a Status-returning caller.
+Result<InvertedIndexAnalyzerCtxSPtr> build_search_analyzer_context(
+        const std::map<std::string, std::string>& properties, const std::string& analyzer_key);
+
 class FieldReaderResolver {
 public:
     FieldReaderResolver(

be/src/exprs/vmatch_predicate.cpp

@@ -58,8 +58,10 @@ namespace doris {
 using namespace doris::segment_v2;
 
 VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
-    const auto resolved = AnalyzerConfigParser::parse(node.match_predicate.analyzer_name,
-                                                      node.match_predicate.parser_type);
+    const auto resolved = AnalyzerConfigParser::parse(
+            node.match_predicate.analyzer_name, node.match_predicate.parser_type,
+            node.match_predicate.parser_mode, node.match_predicate.parser_lowercase,
+            node.match_predicate.char_filter_map);
 
     InvertedIndexAnalyzerConfig config;
     config.analyzer_name = resolved.provider_name;
@@ -81,9 +83,29 @@ VMatchPredicate::VMatchPredicate(const TExprNode& node) : VExpr(node) {
     _analyzer_ctx->parser_type = resolved.parser_type;
 
     if (_analyzer_ctx->requires_analysis()) {
-        _analyzer_provider =
-                inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(&config);
+        std::string bound_name;
+        std::string legacy_name;
+        _analyzer_provider = inverted_index::InvertedIndexAnalyzer::create_analyzer_provider(
+                &config, &bound_name, &legacy_name);
         _analyzer = _analyzer_provider->get_analyzer();
+        if (!legacy_name.empty() && legacy_name != bound_name) {
+            _analyzer_ctx->legacy_analyzer_key =
+                    AnalyzerConfigParser::parse(legacy_name, node.match_predicate.parser_type,
+                                                node.match_predicate.parser_mode,
+                                                node.match_predicate.parser_lowercase,
+                                                node.match_predicate.char_filter_map)
+                            .analyzer_key;
+        }
+        if (bound_name != resolved.provider_name) {
+            // Reader selection and query tokenization must use the same policy binding.
+            _analyzer_ctx->analyzer_key =
+                    AnalyzerConfigParser::parse(bound_name, node.match_predicate.parser_type,
+                                                node.match_predicate.parser_mode,
+                                                node.match_predicate.parser_lowercase,
+                                                node.match_predicate.char_filter_map)
+                            .analyzer_key;
+            _analyzer_ctx->analyzer_name = std::move(bound_name);
+        }
     }
 
     _analyzer_ctx->char_filter_map = std::move(config.char_filter_map);