diff --git a/crates/tantivy_ffi/src/reader.rs b/crates/tantivy_ffi/src/reader.rs index f43e313fe..e94c59e50 100644 --- a/crates/tantivy_ffi/src/reader.rs +++ b/crates/tantivy_ffi/src/reader.rs @@ -21,8 +21,9 @@ //! 1 MATCH_ALL — tokenize query, BooleanQuery (Must) //! 2 MATCH_ANY — tokenize query, BooleanQuery (Should) //! 3 PHRASE — tokenize query, PhraseQuery -//! 4 PREFIX — RegexQuery `.*` (no tokenization, mirrors lucene-fts) -//! 5 WILDCARD — RegexQuery from glob pattern (`*` → `.*`, `?` → `.`, others escaped) +//! 4 PREFIX — original + case-normalized RegexQuery `.*` (no tokenization) +//! 5 WILDCARD — original + case-normalized RegexQuery from glob pattern +//! (`*` → `.*`, `?` → `.`, others escaped) //! //! For paimon-java compatibility, row_id is stored as an explicit u64 field //! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id @@ -37,7 +38,9 @@ use std::path::Path; use croaring::{Portable, Treemap}; use tantivy::collector::{Collector, SegmentCollector}; use tantivy::columnar::Column; -use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery}; +use tantivy::query::{ + BooleanQuery, DisjunctionMaxQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery, +}; use tantivy::schema::{Field, IndexRecordOption}; use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal, SegmentReader, Term}; @@ -46,7 +49,7 @@ use crate::buffer::PaimonTantivyBuffer; use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks}; use crate::error::{set_last_error, PaimonTantivyStatus}; use crate::handle::{borrow_handle_mut, free_handle, into_handle}; -use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode}; +use crate::tokenizer::{normalize_case, PaimonJiebaTokenizer, TokenizeMode}; use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME}; /// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync @@ -229,22 +232,44 @@ impl PaimonTantivyReader { if query.is_empty() { return Err("prefix query is empty".into()); } - // Mirror lucene-fts: don't tokenize prefix; match indexed term bytes - // starting with the given prefix verbatim. - let pattern = format!("{}.*", regex_escape(query)); - RegexQuery::from_pattern(&pattern, self.text_field) - .map(|q| Box::new(q) as Box) - .map_err(|e| format!("RegexQuery from prefix {query:?}: {e}")) + // Mirror lucene-fts: don't tokenize the prefix. Retain the original form for mixed + // ASCII/CJK indexed terms, and add the normalized form for pure ASCII terms. + let normalized_query = normalize_case(query); + let create_query = |value: &str| -> Result, String> { + let pattern = format!("{}.*", regex_escape(value)); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from prefix {value:?}: {e}")) + }; + let original = create_query(query)?; + if normalized_query == query { + return Ok(original); + } + let normalized = create_query(&normalized_query)?; + Ok(Box::new(DisjunctionMaxQuery::new(vec![ + original, normalized, + ]))) } fn build_wildcard_query(&self, query: &str) -> Result, String> { if query.is_empty() { return Err("wildcard query is empty".into()); } - let pattern = wildcard_to_regex(query); - RegexQuery::from_pattern(&pattern, self.text_field) - .map(|q| Box::new(q) as Box) - .map_err(|e| format!("RegexQuery from wildcard {query:?} (pattern {pattern}): {e}")) + let normalized_query = normalize_wildcard_query(query); + let create_query = |value: &str| -> Result, String> { + let pattern = wildcard_to_regex(value); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from wildcard {value:?} (pattern {pattern}): {e}")) + }; + let original = create_query(query)?; + if normalized_query == query { + return Ok(original); + } + let normalized = create_query(&normalized_query)?; + Ok(Box::new(DisjunctionMaxQuery::new(vec![ + original, normalized, + ]))) } fn build_query(&self, search_type: SearchType, query: &str) -> Result, String> { @@ -471,6 +496,23 @@ fn regex_escape(input: &str) -> String { out } +/// Normalize each literal segment independently while preserving wildcard +/// operators. This matches lucene-fts, where `*` and `?` are not analyzed. +fn normalize_wildcard_query(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut segment_start = 0; + for (index, ch) in input.char_indices() { + if ch != '*' && ch != '?' { + continue; + } + out.push_str(&normalize_case(&input[segment_start..index])); + out.push(ch); + segment_start = index + ch.len_utf8(); + } + out.push_str(&normalize_case(&input[segment_start..])); + out +} + /// Translate a glob-style wildcard ('*' = any, '?' = single char) into a /// regex pattern, escaping all other regex metacharacters. fn wildcard_to_regex(input: &str) -> String { @@ -1024,6 +1066,14 @@ mod tests { assert_eq!(ids, vec![0u64]); } + #[test] + fn prefix_normalizes_ascii_alphanumeric_case() { + let bytes = build(&["This is a test document", "another document"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Prefix, "THIS").unwrap(); + assert_eq!(ids, vec![0u64]); + } + #[test] fn wildcard_with_star() { let bytes = build(&["unordered", "ordered", "border"]); @@ -1032,6 +1082,34 @@ mod tests { assert_eq!(ids, vec![0u64, 1, 2]); } + #[test] + fn wildcard_normalizes_ascii_alphanumeric_segments() { + let bytes = build(&["This is a test document", "another document"]); + let r = open(&bytes); + assert_eq!( + r.search_all(SearchType::Wildcard, "*THIS*").unwrap(), + vec![0u64] + ); + assert_eq!( + r.search_all(SearchType::Wildcard, "*?HIS*").unwrap(), + vec![0u64] + ); + } + + #[test] + fn prefix_and_wildcard_preserve_mixed_ascii_cjk_terms() { + let bytes = build(&["B超检查", "T恤"]); + let r = open(&bytes); + assert_eq!( + r.search_all(SearchType::Prefix, "B").unwrap(), + vec![0u64] + ); + assert_eq!( + r.search_all(SearchType::Wildcard, "*T*").unwrap(), + vec![1u64] + ); + } + #[test] fn empty_query_for_match_returns_query_parse_error() { let bytes = build(&["hello"]); @@ -1048,6 +1126,13 @@ mod tests { assert_eq!(wildcard_to_regex("*a*"), ".*a.*"); } + #[test] + fn wildcard_normalization_preserves_non_alphanumeric_segments() { + assert_eq!(normalize_wildcard_query("*?HIS*"), "*?his*"); + assert_eq!(normalize_wildcard_query("*THIS_IS*"), "*THIS_IS*"); + assert_eq!(normalize_wildcard_query("*机器*"), "*机器*"); + } + // ----- limit + pre_filter + scoring (row_id-based) ----- #[test] diff --git a/crates/tantivy_ffi/src/tokenizer.rs b/crates/tantivy_ffi/src/tokenizer.rs index b4b88fed7..8f4e83542 100644 --- a/crates/tantivy_ffi/src/tokenizer.rs +++ b/crates/tantivy_ffi/src/tokenizer.rs @@ -120,11 +120,7 @@ impl PaimonJiebaTokenizer { let start = piece.as_ptr() as usize - text_start; let end = start + piece.len(); // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior) - let token_text = if is_ascii_alnum(piece) { - piece.to_ascii_lowercase() - } else { - piece.to_string() - }; + let token_text = normalize_case(piece); out.push((start, end, token_text)); } out @@ -135,6 +131,14 @@ fn is_ascii_alnum(s: &str) -> bool { !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric()) } +pub(crate) fn normalize_case(s: &str) -> String { + if is_ascii_alnum(s) { + s.to_ascii_lowercase() + } else { + s.to_string() + } +} + fn load_jieba(dict_dir: &Path) -> Result { let main_dict = dict_dir.join("jieba.dict.utf8"); let mut jieba = if main_dict.exists() { diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 95b511c9f..ca0464daa 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -71,8 +71,15 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie - ``MATCH_ALL``: All terms in the query must be present (AND semantics). - ``MATCH_ANY``: Any term in the query can match (OR semantics). - ``PHRASE``: Matches the exact sequence of words (with proximity). -- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). -- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). +- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). The + query is not tokenized. The original prefix is retained, and a pure ASCII alphanumeric prefix + is also matched using the lowercase case-normalization applied to pure ASCII terms at indexing + time. This preserves matches for mixed terms such as ``B超`` while allowing ``THIS`` to match + terms indexed as ``this...``. +- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). The query + is not tokenized, and wildcard operators are preserved. Both the original pattern and an + alternative with each ASCII alphanumeric fragment lowercased are matched, covering pure ASCII + and mixed ASCII/non-ASCII terms. **Special Configuration:** diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h index 5a608eff1..97a02a7ff 100644 --- a/include/paimon/predicate/full_text_search.h +++ b/include/paimon/predicate/full_text_search.h @@ -77,10 +77,14 @@ struct PAIMON_EXPORT FullTextSearch { /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed. /// /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner). - /// Only the prefix part is considered; analysis will not be applied. + /// The query is not tokenized or filtered for stop words. The original prefix is retained, + /// and a prefix consisting entirely of ASCII letters and digits is also matched using the + /// lowercase case-normalization applied to pure ASCII terms at indexing time. /// /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e"). - /// Not passed through analyzer — matched directly against indexed terms. + /// The query is not tokenized or filtered for stop words. The wildcard operators are + /// preserved. Both the original pattern and an alternative with each ASCII alphanumeric + /// fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms. /// /// @note Analyzer consistency between indexing and querying is critical for correctness. std::string query; diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index 558536927..f93cfb91c 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -83,6 +83,21 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb } } +void JiebaTokenizer::NormalizeCase(std::string* term) { + bool is_alphanumeric = true; + for (const auto& c : *term) { + if (!std::isalnum(static_cast(c))) { + is_alphanumeric = false; + break; + } + } + if (is_alphanumeric && !term->empty()) { + std::transform(term->begin(), term->end(), term->begin(), [](char ch) { + return static_cast(std::tolower(static_cast(ch))); + }); + } +} + void JiebaTokenizer::Normalize(const std::unordered_set& stop_words, std::vector* input_ptr, std::vector* output_ptr) { @@ -98,19 +113,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words if (stop_words.find(term) != stop_words.end()) { continue; } - // to lower case - bool is_alphanumeric = true; - for (const auto& c : term) { - if (!std::isalnum(static_cast(c))) { - is_alphanumeric = false; - break; - } - } - if (is_alphanumeric && !term.empty()) { - std::transform(term.begin(), term.end(), term.begin(), [](char ch) { - return static_cast(std::tolower(static_cast(ch))); - }); - } + NormalizeCase(&term); output.emplace_back(term.data(), term.length()); } } diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index 5f6da9a4e..9008287fc 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -53,6 +53,8 @@ class JiebaTokenizer : public Lucene::Tokenizer { static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, const std::string& str, std::vector* terms_ptr); + static void NormalizeCase(std::string* term); + // In-place converts each string in `input` to lowercase to avoid data copying. static void Normalize(const std::unordered_set& stop_words, std::vector* input, std::vector* output); diff --git a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp index e26b72943..6262c175a 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp @@ -107,6 +107,24 @@ TEST_P(JiebaAnalyzerTest, TestNormalize) { ASSERT_EQ(expected, results); } +TEST(JiebaTokenizerTest, TestNormalizeCase) { + std::string alphanumeric_term = "THIS123"; + JiebaTokenizer::NormalizeCase(&alphanumeric_term); + ASSERT_EQ(alphanumeric_term, "this123"); + + std::string mixed_ascii_cjk_term = "B超"; + JiebaTokenizer::NormalizeCase(&mixed_ascii_cjk_term); + ASSERT_EQ(mixed_ascii_cjk_term, "B超"); + + std::string term_with_underscore = "THIS_IS"; + JiebaTokenizer::NormalizeCase(&term_with_underscore); + ASSERT_EQ(term_with_underscore, "THIS_IS"); + + std::string chinese_term = "机器"; + JiebaTokenizer::NormalizeCase(&chinese_term); + ASSERT_EQ(chinese_term, "机器"); +} + INSTANTIATE_TEST_SUITE_P(ReadBufferSize, JiebaAnalyzerTest, ::testing::ValuesIn(std::vector({2, 5, 10, 100}))); diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 442796ad9..975541132 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -16,6 +16,7 @@ #include "paimon/global_index/lucene/lucene_global_index_reader.h" #include "arrow/c/bridge.h" +#include "lucene++/DisjunctionMaxQuery.h" #include "lucene++/FileUtils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" @@ -116,6 +117,25 @@ std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::stri return wterms; } +std::string LuceneGlobalIndexReader::NormalizeWildcardQuery(const std::string& query) { + std::string normalized_query; + normalized_query.reserve(query.size()); + size_t term_begin = 0; + for (size_t i = 0; i <= query.size(); i++) { + if (i != query.size() && query[i] != '*' && query[i] != '?') { + continue; + } + std::string term = query.substr(term_begin, i - term_begin); + JiebaTokenizer::NormalizeCase(&term); + normalized_query.append(term); + if (i != query.size()) { + normalized_query.push_back(query[i]); + } + term_begin = i + 1; + } + return normalized_query; +} + Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL || @@ -153,15 +173,43 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery( Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX); - return Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + auto create_query = [this](const std::string& query) -> Lucene::QueryPtr { + return Lucene::newLucene( + Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query))); + }; + std::string normalized_query = full_text_search->query; + JiebaTokenizer::NormalizeCase(&normalized_query); + Lucene::QueryPtr query = create_query(full_text_search->query); + if (normalized_query == full_text_search->query) { + return query; + } + + // Preserve the original query for mixed ASCII/CJK indexed terms such as "B超", whose + // complete token is not lowercased by the index analyzer. The normalized alternative still + // matches pure ASCII terms. DisjunctionMax avoids double-counting scores if both match. + auto disjunction = Lucene::newLucene(0.0); + disjunction->add(query); + disjunction->add(create_query(normalized_query)); + return disjunction; } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery( const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD); - return Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + auto create_query = [this](const std::string& query) -> Lucene::QueryPtr { + return Lucene::newLucene( + Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query))); + }; + std::string normalized_query = NormalizeWildcardQuery(full_text_search->query); + Lucene::QueryPtr query = create_query(full_text_search->query); + if (normalized_query == full_text_search->query) { + return query; + } + + auto disjunction = Lucene::newLucene(0.0); + disjunction->add(query); + disjunction->add(create_query(normalized_query)); + return disjunction; } Result> LuceneGlobalIndexReader::SearchWithLimit( diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index 29c28f13a..a95efa8d4 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -126,6 +126,8 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { std::vector TokenizeQuery(const std::string& query) const; + static std::string NormalizeWildcardQuery(const std::string& query); + std::shared_ptr CreateAllResult() const { return nullptr; } diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index 97f99dec4..8443783b2 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -234,6 +234,14 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/std::nullopt))); CheckResult(result, {3l}); } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } // test wildcard query { ASSERT_OK_AND_ASSIGN(auto result, @@ -251,6 +259,22 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/std::nullopt))); CheckResult(result, {3l}); } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*?HIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {1l, 0l}); + } // test filter { ASSERT_OK_AND_ASSIGN(auto result, @@ -439,6 +463,43 @@ TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) { } } +TEST_P(LuceneGlobalIndexTest, TestMixedAsciiCjkPrefixAndWildcard) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + + std::map options = { + {"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.read.buffer-size", std::to_string(GetParam())}, + {"lucene-fts.jieba.tokenize-mode", "query"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["B超检查"], + ["T恤"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(test_root_dir->Str(), data_type_, options, + array, Range(0, 1), tmp_dir->Str())); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateGlobalIndexReader(test_root_dir->Str(), data_type_, options, meta)); + auto lucene_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(lucene_reader); + + ASSERT_OK_AND_ASSIGN(auto prefix_result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/10, "B", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + CheckResult(prefix_result, {0l}); + + ASSERT_OK_AND_ASSIGN(auto wildcard_result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/10, "*T*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(wildcard_result, {1l}); +} + TEST_F(LuceneGlobalIndexTest, TestInvalidWithoutTmpDir) { auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir); diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp index e51ee60b3..c787ad722 100644 --- a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp @@ -18,13 +18,11 @@ * EQUIVALENCE: a parametric corpus × query battery that compares lucene-fts * and tantivy-fulltext result *sets* (doc_id only — not score order, not score * values). Coverage targets: - * - English bag-of-words: MATCH_ALL / MATCH_ANY / PHRASE + * - English bag-of-words: MATCH_ALL / MATCH_ANY / PHRASE / PREFIX / WILDCARD * - Chinese (jieba "query" mode): MATCH_ALL / MATCH_ANY / PHRASE * - Pre_filter intersection (no scoring) - * PREFIX and WILDCARD are NOT compared as required-equal: tantivy's RegexQuery - * walks byte-level term dictionary, lucene's PrefixQuery/WildcardQuery walks - * its own; edge cases (empty input, anchors, multi-byte UTF-8) diverge by - * design. + * PREFIX and WILDCARD equivalence covers ASCII token patterns, including case + * normalization. Engine-specific regex edge cases are outside this test. * * BENCHMARK: build a 200-doc index per backend and time write + 100 queries. * Prints to stderr; never fails on perf — guarding against perf regressions @@ -222,22 +220,30 @@ TEST_F(TantivyEquivalenceTest, EnglishBagOfWordsBattery) { struct Case { std::string query; FullTextSearch::SearchType type; + std::set expected_ids; }; std::vector cases = { - {"alpha", FullTextSearch::SearchType::MATCH_ALL}, - {"alpha", FullTextSearch::SearchType::MATCH_ANY}, - {"alpha beta", FullTextSearch::SearchType::MATCH_ALL}, - {"alpha beta", FullTextSearch::SearchType::MATCH_ANY}, - {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL}, - {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY}, - {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL}, - {"alpha beta gamma", FullTextSearch::SearchType::PHRASE}, - {"beta gamma delta", FullTextSearch::SearchType::PHRASE}, - {"delta epsilon", FullTextSearch::SearchType::PHRASE}, + {"alpha", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY, {}}, + {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL, {}}, + {"alpha beta gamma", FullTextSearch::SearchType::PHRASE, {}}, + {"beta gamma delta", FullTextSearch::SearchType::PHRASE, {}}, + {"delta epsilon", FullTextSearch::SearchType::PHRASE, {}}, + {"ALP", FullTextSearch::SearchType::PREFIX, {0, 1, 4, 6, 9}}, + {"*ALPHA*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}}, + {"*ALP?A*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}}, }; for (const auto& c : cases) { auto [l, t] = RunPair(pair, c.query, c.type); ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type); + if (!c.expected_ids.empty()) { + ASSERT_EQ(l, c.expected_ids) << "unexpected matches: query=" << c.query + << " type=" << static_cast(c.type); + } } } @@ -277,6 +283,32 @@ TEST_F(TantivyEquivalenceTest, ChineseQueryModeBattery) { } } +TEST_F(TantivyEquivalenceTest, MixedAsciiCjkPrefixAndWildcard) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["B超检查"], + ["T恤"] + ])") + .ValueOrDie(); + std::map lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}}; + std::map topts = { + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto pair = WriteAndOpenBoth(data_type, array, lopts, topts); + + { + auto [l, t] = RunPair(pair, "B", FullTextSearch::SearchType::PREFIX); + ASSERT_EQ(l, (std::set{0})); + ASSERT_EQ(t, l); + } + { + auto [l, t] = RunPair(pair, "*T*", FullTextSearch::SearchType::WILDCARD); + ASSERT_EQ(l, (std::set{1})); + ASSERT_EQ(t, l); + } +} + TEST_F(TantivyEquivalenceTest, PreFilterIntersectionEquivalent) { auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index c0a06f57b..9fe5835c0 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -2368,6 +2368,22 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) { /*pre_filter=*/std::nullopt))); ASSERT_TRUE(index_result->ToString().find("row ids: {3}") != std::string::npos); } + { + ASSERT_OK_AND_ASSIGN(auto index_result, + index_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos); + } + { + ASSERT_OK_AND_ASSIGN(auto index_result, + index_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos); + } } TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {