From bd44cb1d9f7023e220685b00fd0eb5f0b41a802e Mon Sep 17 00:00:00 2001 From: Mryange Date: Wed, 29 Jul 2026 18:50:14 +0800 Subject: [PATCH] [fix](function) Preserve trailing zero bytes in string distances ### What problem does this PR solve? Issue Number: close #xxx Related PR: #63291 Problem Summary: String distance functions trimmed trailing zero bytes from every input during execution. CHAR padding is already removed at the storage read boundary, so this work was redundant for CHAR values and incorrectly discarded legitimate trailing zero bytes from STRING and VARCHAR values. Use the complete ColumnString values for Hamming, Levenshtein, and Damerau-Levenshtein distance calculations. ### Release note Fix string distance functions to preserve trailing zero bytes. ### Check List (For Author) - Test: Not run; awaiting author review before compilation and test validation - Behavior changed: Yes; trailing zero bytes now participate in string distance calculations - Does this need documentation: No --- .../function/function_hamming_distance.cpp | 33 +++++++++---------- .../exprs/function/function_levenshtein.cpp | 12 +++---- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/be/src/exprs/function/function_hamming_distance.cpp b/be/src/exprs/function/function_hamming_distance.cpp index 230b02c32d6b81..ddbbd00b610270 100644 --- a/be/src/exprs/function/function_hamming_distance.cpp +++ b/be/src/exprs/function/function_hamming_distance.cpp @@ -81,12 +81,11 @@ class FunctionHammingDistance : public IFunction { if (!has_nullable) { if (left_const) { - RETURN_IF_ERROR(scalar_vector(left_str_col->get_data_at(0).trim_tail_padding_zero(), - *right_str_col, res_data)); + RETURN_IF_ERROR( + scalar_vector(left_str_col->get_data_at(0), *right_str_col, res_data)); } else if (right_const) { - RETURN_IF_ERROR(vector_scalar( - *left_str_col, right_str_col->get_data_at(0).trim_tail_padding_zero(), - res_data)); + RETURN_IF_ERROR( + vector_scalar(*left_str_col, right_str_col->get_data_at(0), res_data)); } else { RETURN_IF_ERROR(vector_vector(*left_str_col, *right_str_col, res_data)); } @@ -104,7 +103,7 @@ class FunctionHammingDistance : public IFunction { return Status::OK(); } - const auto left = left_str_col->get_data_at(0).trim_tail_padding_zero(); + const auto left = left_str_col->get_data_at(0); RETURN_IF_ERROR(scalar_vector_nullable(left, *right_str_col, right_null_map, res_data, null_map)); } else if (right_const) { @@ -115,9 +114,8 @@ class FunctionHammingDistance : public IFunction { return Status::OK(); } - RETURN_IF_ERROR(vector_scalar_nullable( - *left_str_col, right_str_col->get_data_at(0).trim_tail_padding_zero(), - left_null_map, res_data, null_map)); + RETURN_IF_ERROR(vector_scalar_nullable(*left_str_col, right_str_col->get_data_at(0), + left_null_map, res_data, null_map)); } else { for (size_t i = 0; i < input_rows_count; ++i) { const bool left_is_null = left_null_map && (*left_null_map)[i]; @@ -128,9 +126,8 @@ class FunctionHammingDistance : public IFunction { continue; } - RETURN_IF_ERROR(hamming_distance( - left_str_col->get_data_at(i).trim_tail_padding_zero(), - right_str_col->get_data_at(i).trim_tail_padding_zero(), res_data[i], i)); + RETURN_IF_ERROR(hamming_distance(left_str_col->get_data_at(i), + right_str_col->get_data_at(i), res_data[i], i)); } } @@ -149,8 +146,8 @@ class FunctionHammingDistance : public IFunction { std::vector left_offsets; std::vector right_offsets; for (size_t i = 0; i < size; ++i) { - const auto left = lcol.get_data_at(i).trim_tail_padding_zero(); - const auto right = rcol.get_data_at(i).trim_tail_padding_zero(); + const auto left = lcol.get_data_at(i); + const auto right = rcol.get_data_at(i); RETURN_IF_ERROR(hamming_distance_with_offsets( left, left_offsets, false, simd::VStringFunctions::is_ascii(left), right, right_offsets, false, simd::VStringFunctions::is_ascii(right), res[i], i)); @@ -167,7 +164,7 @@ class FunctionHammingDistance : public IFunction { simd::VStringFunctions::get_utf8_char_offsets(rdata, right_offsets); std::vector left_offsets; for (size_t i = 0; i < size; ++i) { - const auto left = lcol.get_data_at(i).trim_tail_padding_zero(); + const auto left = lcol.get_data_at(i); RETURN_IF_ERROR(hamming_distance_with_offsets( left, left_offsets, false, simd::VStringFunctions::is_ascii(left), rdata, right_offsets, true, right_ascii, res[i], i)); @@ -184,7 +181,7 @@ class FunctionHammingDistance : public IFunction { simd::VStringFunctions::get_utf8_char_offsets(ldata, left_offsets); std::vector right_offsets; for (size_t i = 0; i < size; ++i) { - const auto right = rcol.get_data_at(i).trim_tail_padding_zero(); + const auto right = rcol.get_data_at(i); RETURN_IF_ERROR(hamming_distance_with_offsets( ldata, left_offsets, true, left_ascii, right, right_offsets, false, simd::VStringFunctions::is_ascii(right), res[i], i)); @@ -208,7 +205,7 @@ class FunctionHammingDistance : public IFunction { continue; } - const auto left = lcol.get_data_at(i).trim_tail_padding_zero(); + const auto left = lcol.get_data_at(i); RETURN_IF_ERROR(hamming_distance_with_offsets( left, left_offsets, false, simd::VStringFunctions::is_ascii(left), rdata, right_offsets, true, right_ascii, res[i], i)); @@ -232,7 +229,7 @@ class FunctionHammingDistance : public IFunction { continue; } - const auto right = rcol.get_data_at(i).trim_tail_padding_zero(); + const auto right = rcol.get_data_at(i); RETURN_IF_ERROR(hamming_distance_with_offsets( ldata, left_offsets, true, left_ascii, right, right_offsets, false, simd::VStringFunctions::is_ascii(right), res[i], i)); diff --git a/be/src/exprs/function/function_levenshtein.cpp b/be/src/exprs/function/function_levenshtein.cpp index b4ee35e99763e7..06f20eca1130f4 100644 --- a/be/src/exprs/function/function_levenshtein.cpp +++ b/be/src/exprs/function/function_levenshtein.cpp @@ -50,8 +50,7 @@ static StringRef string_ref_at(const ColumnString::Chars& data, const ColumnString::Offsets& offsets, size_t i) { DCHECK_LT(i, offsets.size()); const auto previous_offset = i == 0 ? 0 : offsets[i - 1]; - return StringRef(data.data() + previous_offset, offsets[i] - previous_offset) - .trim_tail_padding_zero(); + return StringRef(data.data() + previous_offset, offsets[i] - previous_offset); } static void get_utf8_char_offsets(const StringRef& ref, Utf8Offsets& offsets) { @@ -390,15 +389,14 @@ struct StringDistanceImplBase { ResultPaddedPODArray& res) { const size_t size = offsets.size(); res.resize(size); - const auto constant_ref = constant.trim_tail_padding_zero(); - const bool constant_ascii = simd::VStringFunctions::is_ascii(constant_ref); + const bool constant_ascii = simd::VStringFunctions::is_ascii(constant); Utf8Offsets constant_offsets; - get_utf8_char_offsets(constant_ref, constant_offsets); + get_utf8_char_offsets(constant, constant_offsets); Utf8Offsets value_offsets; for (size_t i = 0; i < size; ++i) { RETURN_IF_ERROR(distance_with_const_offsets(string_ref_at(data, offsets, i), - value_offsets, constant_ref, - constant_offsets, constant_ascii, res[i])); + value_offsets, constant, constant_offsets, + constant_ascii, res[i])); } return Status::OK(); }