From c3834ee16af015c5d20332dacb77da6d000a9a65 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Mon, 7 Sep 2026 19:02:31 +0800 Subject: [PATCH] [feature](function) Support nanosecond precision for TIMEV2 Issue Number: None Related PR: None Problem Summary: Commit 23c83f48bceae6578b55dcfe9cae476b9c15fc8a added TIMESTAMP_NS with 9 fractional digits, but TIMEV2 was capped at scale 6 and time functions converted values through integral microseconds. This truncated submicrosecond results such as TIMEDIFF(TIMESTAMP_NS, TIMESTAMP_NS). Extend TIMEV2 to scale 9 while retaining its existing double microsecond-unit representation, and preserve exact nanoseconds in differences, conversions, formatting, constant folding, external TIME readers, arithmetic, and binary query results. TIMEV2 supports scales up to 9, and time results such as TIMEDIFF on TIMESTAMP_NS preserve nanosecond precision. - Test: Unit Test and Regression Test - ASAN BE and FE builds passed - 26 targeted BE unit tests passed - 108 targeted FE unit tests passed - Relevant timestamp_ns, TIME function, scalar function, binary protocol, and documentation regression suites passed - Behavior changed: Yes (TIMEV2 supports nanosecond precision) - Does this need documentation: No --- be/src/core/data_type/data_type_time.h | 2 +- .../data_type_serde/data_type_time_serde.cpp | 89 ++++--- be/src/core/value/time_value.h | 91 +++++-- be/src/exprs/function/cast/cast_to_date.h | 63 +---- be/src/exprs/function/cast/cast_to_string.h | 2 +- .../exprs/function/cast/cast_to_time_impl.hpp | 106 ++++---- .../function/cast/cast_to_timestamp_ns.h | 28 ++- be/src/exprs/function/date_time_transforms.h | 5 +- .../function_date_or_datetime_computation.h | 84 +++++-- .../function/function_other_types_to_date.cpp | 13 +- be/src/format/parquet/schema_desc.cpp | 8 +- .../format_v2/parquet/native_schema_desc.cpp | 7 +- be/src/util/date_func.cpp | 61 +++-- .../data_type/data_type_datetime_v2_test.cpp | 5 +- .../data_type_serde_datetime_v2_test.cpp | 16 ++ .../data_type_serde_decoded_values_test.cpp | 9 + .../data_type_serde_parquet_test.cpp | 15 ++ .../function/cast/cast_timestamp_ns_test.cpp | 16 ++ .../exprs/function/cast/cast_to_time_test.cpp | 10 + .../function/function_timestamp_ns_test.cpp | 35 ++- .../format/parquet/parquet_thrift_test.cpp | 45 ++++ .../format_v2/parquet/parquet_schema_test.cpp | 45 ++++ be/test/runtime/time_value_test.cpp | 10 + .../apache/doris/analysis/TimeV2Literal.java | 49 +++- .../apache/doris/mysql/MysqlSerializer.java | 24 +- .../executable/DateTimeArithmetic.java | 16 +- .../DateTimeExtractAndTransform.java | 25 +- .../functions/scalar/CurrentTime.java | 5 +- .../functions/scalar/TimeDiff.java | 3 +- .../expressions/functions/scalar/UtcTime.java | 4 +- .../literal/TimeStampNsLiteral.java | 7 +- .../expressions/literal/TimeV2Literal.java | 234 +++++++++--------- .../apache/doris/nereids/types/DataType.java | 23 +- .../doris/nereids/types/DateTimeV2Type.java | 2 +- .../doris/nereids/types/TimeV2Type.java | 9 +- .../doris/nereids/util/TypeCoercionUtils.java | 9 +- .../mysql/MysqlSerializerDateTimeTest.java | 17 ++ .../TimestampNsFunctionSignatureTest.java | 3 +- .../literal/TimeStampNsLiteralTest.java | 11 +- .../literal/TimeV2LiteralTest.java | 27 ++ .../org/apache/doris/catalog/ScalarType.java | 1 + .../data/correctness/test_time_function.out | 16 +- .../test_timestamp_ns_binary_output.out | 3 + .../test_timestamp_ns_expressions.out | 6 +- .../test_timestamp_ns_functions.out | 23 +- .../sql-functions/doc_date_functions_test.out | 10 +- .../nereids_function_p0/scalar_function/T.out | 192 +++++++------- .../test_timestamp_ns_binary_output.groovy | 11 + .../test_timestamp_ns_functions.groovy | 14 ++ 49 files changed, 981 insertions(+), 528 deletions(-) diff --git a/be/src/core/data_type/data_type_time.h b/be/src/core/data_type/data_type_time.h index 1a0e083a4ee1c2..f62a50597f2d46 100644 --- a/be/src/core/data_type/data_type_time.h +++ b/be/src/core/data_type/data_type_time.h @@ -40,7 +40,7 @@ class IColumn; class DataTypeTimeV2 final : public DataTypeNumberBase { public: - static constexpr UInt32 MAX_SCALE = 6; + static constexpr UInt32 MAX_SCALE = 9; DataTypeTimeV2(int scale = 0) : _scale(scale) { if (UNLIKELY(scale > static_cast(MAX_SCALE))) { diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 566bd3982b4dda..1745c0f1fed1ae 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include "common/config.h" #include "core/data_type/data_type_decimal.h" @@ -30,6 +32,7 @@ #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" +#include "exec/common/int_exp.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" #include "util/unaligned.h" @@ -37,40 +40,40 @@ namespace doris { namespace { -TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64_t row) { - int64_t micros = 0; +TimeValue::TimeType time_from_nanoseconds(int64_t nanoseconds, int scale) { + const int effective_scale = std::max(scale, static_cast(TimeValue::MICROS_SCALE)); + const int64_t divisor = common::exp10_i64(TimeValue::NANOS_SCALE - effective_scale); + return TimeValue::from_nanoseconds(nanoseconds / divisor * divisor); +} + +TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64_t row, int scale) { if (view.value_kind == DecodedValueKind::INT32) { const auto* values = reinterpret_cast(view.values); - micros = static_cast(values[row]) * 1000; - } else { - const auto* values = reinterpret_cast(view.values); - micros = values[row]; - if (view.time_unit == DecodedTimeUnit::MILLIS) { - micros *= 1000; - } else if (view.time_unit == DecodedTimeUnit::NANOS) { - micros /= 1000; - } + return static_cast(values[row]) * 1000; } - const bool negative = micros < 0; - const int64_t abs_micros = std::abs(micros); - return TimeValue::make_time( - abs_micros / TimeValue::ONE_HOUR_MICROSECONDS, - (abs_micros % TimeValue::ONE_HOUR_MICROSECONDS) / TimeValue::ONE_MINUTE_MICROSECONDS, - (abs_micros % TimeValue::ONE_MINUTE_MICROSECONDS) / TimeValue::ONE_SECOND_MICROSECONDS, - abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); + const auto* values = reinterpret_cast(view.values); + if (view.time_unit == DecodedTimeUnit::MILLIS) { + return static_cast(values[row]) * 1000; + } + if (view.time_unit == DecodedTimeUnit::NANOS) { + return time_from_nanoseconds(values[row], scale); + } + return static_cast(values[row]); } class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { public: - TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + TimeV2ParquetConsumer(IColumn& column, int scale, const ParquetDecodeContext& context, ParquetMaterializationState* state = nullptr) : _data(assert_cast(column).get_data()), + _scale(scale), _context(context), _state(state) {} - TimeV2ParquetConsumer(ColumnTimeV2::Container& data, const ParquetDecodeContext& context, + TimeV2ParquetConsumer(ColumnTimeV2::Container& data, int scale, + const ParquetDecodeContext& context, ParquetMaterializationState* state = nullptr) - : _data(data), _context(context), _state(state) {} + : _data(data), _scale(scale), _context(context), _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { const size_t old_size = _data.size(); @@ -106,21 +109,20 @@ class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { return Status::DataQualityError( "Parquet TIME value {} is outside the one-day domain", raw_value); } - int64_t micros = raw_value; + TimeValue::TimeType time = raw_value; if (_context.time_unit == ParquetTimeUnit::MILLIS) { - micros *= 1000; + time *= 1000; } else if (_context.time_unit == ParquetTimeUnit::NANOS) { - micros /= 1000; + time = time_from_nanoseconds(raw_value, _scale); } - // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields - // and immediately recombining them is an identity operation with several divisions. - _data[old_size + row] = static_cast(micros); + _data[old_size + row] = time; } return Status::OK(); } private: ColumnTimeV2::Container& _data; + const int _scale; const ParquetDecodeContext& _context; ParquetMaterializationState* _state; }; @@ -134,11 +136,12 @@ class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { class TimeV2PredicateParquetConsumer final : public ParquetFixedValueConsumer { public: - TimeV2PredicateParquetConsumer(const ParquetDecodeContext& context, bool enable_strict_mode, - ParquetLogicalValueConsumer& consumer, + TimeV2PredicateParquetConsumer(const ParquetDecodeContext& context, int scale, + bool enable_strict_mode, ParquetLogicalValueConsumer& consumer, ColumnTimeV2::Container& logical_values, IColumn::Filter& conversion_nulls) : _context(context), + _scale(scale), _enable_strict_mode(enable_strict_mode), _consumer(consumer), _logical_values(logical_values), @@ -151,7 +154,7 @@ class TimeV2PredicateParquetConsumer final : public ParquetFixedValueConsumer { ParquetMaterializationState state; state.enable_strict_mode = _enable_strict_mode; state.conversion_failure_null_map = &_conversion_nulls; - TimeV2ParquetConsumer converter(_logical_values, _context, &state); + TimeV2ParquetConsumer converter(_logical_values, _scale, _context, &state); RETURN_IF_ERROR(converter.consume(values, num_values, value_width)); return _consumer.consume(reinterpret_cast(_logical_values.data()), num_values, sizeof(TimeValue::TimeType), _conversion_nulls.data()); @@ -159,6 +162,7 @@ class TimeV2PredicateParquetConsumer final : public ParquetFixedValueConsumer { private: const ParquetDecodeContext& _context; + const int _scale; bool _enable_strict_mode; ParquetLogicalValueConsumer& _consumer; ColumnTimeV2::Container& _logical_values; @@ -173,7 +177,14 @@ Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, const FormatOptions& options) const { const auto& data = assert_cast(column).get_data(); const auto col_index = index_check_const(row_idx, col_const); - if (UNLIKELY(0 != result.push_timev2(data[col_index], _scale))) { + int push_result; + if (std::cmp_greater(_scale, TimeValue::MICROS_SCALE)) { + const auto value = TimeValue::to_string(data[col_index], _scale); + push_result = result.push_string(value.data(), value.size()); + } else { + push_result = result.push_timev2(data[col_index], _scale); + } + if (UNLIKELY(push_result != 0)) { return Status::InternalError("pack mysql buffer failed."); } return Status::OK(); @@ -365,8 +376,10 @@ Status DataTypeTimeV2SerDe::read_column_from_arrow(IColumn& column, const arrow: "Arrow Time64 value is outside the time-of-day range: row={}, value={}", row, value); } - const int64_t micros = type->unit() == arrow::TimeUnit::NANO ? value / 1000 : value; - data.emplace_back(static_cast(micros)); + const auto time = type->unit() == arrow::TimeUnit::NANO + ? time_from_nanoseconds(value, _scale) + : static_cast(value); + data.emplace_back(time); } return Status::OK(); } @@ -391,14 +404,14 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, data.push_back(TimeValue::TimeType()); continue; } - data.push_back(read_time_decoded_value(view, row)); + data.push_back(read_time_decoded_value(view, row, _scale)); } return Status::OK(); } Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { - TimeV2ParquetConsumer consumer(column, context); + TimeV2ParquetConsumer consumer(column, _scale, context); RejectTimeV2BinaryConsumer binary_consumer; return source.decode_dictionary(consumer, binary_consumer); } @@ -412,14 +425,14 @@ Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec context.logical_type != ParquetLogicalType::TIME) { return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); } - TimeV2ParquetConsumer consumer(column, context, &state); + TimeV2ParquetConsumer consumer(column, _scale, context, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); - TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, _scale, context, &state); RejectTimeV2BinaryConsumer binary_consumer; const Status dictionary_status = source.decode_dictionary(dictionary_consumer, binary_consumer); @@ -445,7 +458,7 @@ Status DataTypeTimeV2SerDe::read_parquet_raw_predicate( if (!supports_parquet_raw_predicate(context)) { return Status::NotSupported("Unsupported Parquet raw predicate conversion for TIMEV2"); } - TimeV2PredicateParquetConsumer predicate_consumer(context, enable_strict_mode, consumer, + TimeV2PredicateParquetConsumer predicate_consumer(context, _scale, enable_strict_mode, consumer, _parquet_predicate_values, _parquet_predicate_nulls); return source.decode_fixed_values(num_values, predicate_consumer); diff --git a/be/src/core/value/time_value.h b/be/src/core/value/time_value.h index 292151fb95e2e2..1dcdfab9df8588 100644 --- a/be/src/core/value/time_value.h +++ b/be/src/core/value/time_value.h @@ -39,9 +39,13 @@ class TimeValue { constexpr static int64_t ONE_SECOND_MICROSECONDS = 1000000; constexpr static int64_t ONE_MINUTE_MICROSECONDS = 60 * ONE_SECOND_MICROSECONDS; constexpr static int64_t ONE_HOUR_MICROSECONDS = 60 * ONE_MINUTE_MICROSECONDS; + constexpr static int64_t NANOS_PER_MICROSECOND = 1000; + constexpr static int64_t ONE_SECOND_NANOSECONDS = + ONE_SECOND_MICROSECONDS * NANOS_PER_MICROSECOND; constexpr static int64_t ONE_MINUTE_SECONDS = 60; constexpr static int64_t ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS; constexpr static uint32_t MICROS_SCALE = 6; + constexpr static uint32_t NANOS_SCALE = 9; constexpr static int64_t MAX_TIME = 838 * ONE_HOUR_MICROSECONDS + 59 * ONE_MINUTE_MICROSECONDS + 59 * ONE_SECOND_MICROSECONDS; // 838:59:59.000000 @@ -50,13 +54,13 @@ class TimeValue { using ColumnTimeV2 = typename PrimitiveTypeTraits::ColumnType; #include "common/compile_check_avoid_begin.h" - static int64_t round_time(TimeType value, uint32_t scale) { - int64_t time = value; - DCHECK(scale <= MICROS_SCALE); - int64_t factor = std::pow(10, 6 - scale); - int64_t roundedValue = (time >= 0) ? (time + factor / 2) / factor * factor - : (time - factor / 2) / factor * factor; - return roundedValue; + static TimeType round_time(TimeType value, uint32_t scale) { + DCHECK(scale <= NANOS_SCALE); + int64_t time = to_nanoseconds(value); + int64_t factor = std::pow(10, NANOS_SCALE - scale); + int64_t rounded_value = (time >= 0) ? (time + factor / 2) / factor * factor + : (time - factor / 2) / factor * factor; + return from_nanoseconds(rounded_value); } // Construct time based on hour/minute/second/microsecond @@ -80,6 +84,27 @@ class TimeValue { return static_cast(negative ? -value : value); } + // Construct time based on hour/minute/second/nanosecond. + template + static TimeType make_time_from_nanoseconds(int64_t hour, int64_t minute, int64_t second, + int64_t nanosecond = 0, bool negative = false) { + if constexpr (CHECK) { + if (std::abs(hour) > 838 || std::abs(minute) >= 60 || std::abs(second) >= 60 || + std::abs(nanosecond) >= ONE_SECOND_NANOSECONDS) [[unlikely]] { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Invalid time value: hour={}, minute={}, second={}, nanosecond={}", + hour, minute, second, nanosecond); + } + } + DCHECK(hour >= 0 && minute >= 0 && second >= 0 && nanosecond >= 0) + << "Hour, minute, second and nanosecond must be non-negative but got " << hour + << ":" << minute << ":" << second << "." << nanosecond; + const int64_t value = ((hour * ONE_HOUR_SECONDS) + (minute * ONE_MINUTE_SECONDS) + second) * + ONE_SECOND_NANOSECONDS + + nanosecond; + return from_nanoseconds(negative ? -value : value); + } + // if time is negative, ms should be negative too. in existing scenario, we ensure microsecond's bound by caller. static TimeType init_microsecond(TimeType time, int32_t microsecond) { DCHECK(std::signbit(time) == std::signbit(microsecond) || !time || !microsecond) @@ -88,6 +113,14 @@ class TimeValue { return static_cast(time + microsecond); } + + // If time is negative, nanosecond should be negative too. Callers validate its bound. + static TimeType init_nanosecond(TimeType time, int64_t nanosecond) { + DCHECK(std::signbit(time) == std::signbit(nanosecond) || !time || !nanosecond) + << "Time and nanosecond must have the same sign but got " << time << " and " + << nanosecond; + return time + static_cast(nanosecond) / NANOS_PER_MICROSECOND; + } #include "common/compile_check_avoid_end.h" // in existing scenario, we ensure microsecond's bound by caller. @@ -107,25 +140,45 @@ class TimeValue { /// Return the hour/minute/second part of the time, ignoring the sign static int32_t hour(TimeType time) { - return (int32_t)std::abs( - static_cast(limit_with_bound(time) / ONE_HOUR_MICROSECONDS)); + return static_cast( + std::abs(to_nanoseconds(time) / (ONE_HOUR_MICROSECONDS * NANOS_PER_MICROSECOND))); } static int32_t minute(TimeType time) { - return (int32_t)std::abs( - (static_cast(limit_with_bound(time)) % ONE_HOUR_MICROSECONDS) / - ONE_MINUTE_MICROSECONDS); + return static_cast( + std::abs((to_nanoseconds(time) % (ONE_HOUR_MICROSECONDS * NANOS_PER_MICROSECOND)) / + (ONE_MINUTE_MICROSECONDS * NANOS_PER_MICROSECOND))); } static int32_t second(TimeType time) { - return (int32_t)std::abs( - (static_cast(limit_with_bound(time)) / ONE_SECOND_MICROSECONDS) % - ONE_MINUTE_SECONDS); + return static_cast( + std::abs((to_nanoseconds(time) / ONE_SECOND_NANOSECONDS) % ONE_MINUTE_SECONDS)); } - static int32_t microsecond(TimeType time) { - return (int32_t)std::abs(static_cast(limit_with_bound(time)) % - ONE_SECOND_MICROSECONDS); + static int32_t microsecond(TimeType time) { return nanosecond(time) / NANOS_PER_MICROSECOND; } + + static int32_t nanosecond(TimeType time) { + return static_cast(std::abs(to_nanoseconds(time) % ONE_SECOND_NANOSECONDS)); + } + + static int64_t to_nanoseconds(TimeType time) { + return std::llround(limit_with_bound(time) * NANOS_PER_MICROSECOND); + } + + static TimeType from_nanoseconds(int64_t nanoseconds) { + return static_cast(nanoseconds) / NANOS_PER_MICROSECOND; + } + + static TimeType from_nanoseconds_with_limit(__int128 nanoseconds) { + constexpr __int128 max_time_nanoseconds = + static_cast<__int128>(MAX_TIME) * NANOS_PER_MICROSECOND; + if (nanoseconds > max_time_nanoseconds) { + return MAX_TIME; + } + if (nanoseconds < -max_time_nanoseconds) { + return -MAX_TIME; + } + return from_nanoseconds(static_cast(nanoseconds)); } static int8_t sign(TimeType time) { return (time < 0) ? -1 : 1; } @@ -167,7 +220,7 @@ class TimeValue { return DatetimeValueUtil::to_format_string_without_check( format, len, to, max_valid_length, 0, 0, 0, TimeValue::hour(time), TimeValue::minute(time), TimeValue::second(time), TimeValue::microsecond(time), - nanosecond); + nanosecond < 0 ? TimeValue::nanosecond(time) : nanosecond); } }; } // namespace doris diff --git a/be/src/exprs/function/cast/cast_to_date.h b/be/src/exprs/function/cast/cast_to_date.h index 903ef17083aa6b..f8e84197ec691c 100644 --- a/be/src/exprs/function/cast/cast_to_date.h +++ b/be/src/exprs/function/cast/cast_to_date.h @@ -165,31 +165,14 @@ class CastToImpl : public CastToBase { } else { static_assert(IsTimeV2Type); const auto to_scale = block.get_by_position(result).type->get_scale(); - const int64_t seconds = source.time_part_to_seconds(); - uint32_t hour = static_cast(seconds / 3600); - uint32_t minute = static_cast(seconds / 60 % 60); - uint32_t second = static_cast(seconds % 60); - uint32_t nanoseconds = source.nanosecond(); - const auto divisor = static_cast(common::exp10_i64(9 - to_scale)); - const uint32_t remainder = nanoseconds % divisor; - nanoseconds = nanoseconds / divisor * divisor; - if (remainder >= divisor / 2) { + int64_t nanoseconds = source.time_part_to_nanosecond(); + const int64_t divisor = common::exp10_i64(9 - to_scale); + const int64_t remainder = nanoseconds % divisor; + nanoseconds -= remainder; + if (remainder * 2 >= divisor) { nanoseconds += divisor; } - uint32_t microseconds = nanoseconds / TimeStampNsValue::NANOS_PER_MICROSECOND; - if (microseconds >= TimeValue::ONE_SECOND_MICROSECONDS) { - microseconds = 0; - if (++second == 60) { - second = 0; - if (++minute == 60) { - minute = 0; - ++hour; - } - } - } - col_to->get_data()[i] = - ((hour * 60 + minute) * 60 + second) * TimeValue::ONE_SECOND_MICROSECONDS + - microseconds; + col_to->get_data()[i] = TimeValue::from_nanoseconds(nanoseconds); } } @@ -309,7 +292,7 @@ class CastToImpl : public CastToBase { } else if constexpr (IsTimeV2Type && IsDateTimeV2Type) { const auto* type = assert_cast( block.get_by_position(arguments[0]).type.get()); - auto scale = type->get_scale(); + auto scale = std::min(type->get_scale(), TimeValue::MICROS_SCALE); DateV2Value dtmv2; dtmv2.from_unixtime(context->state()->timestamp_ms() / 1000, @@ -385,35 +368,9 @@ class CastToImpl : public CastToBase { // nothing to do, just copy col_to->get_data()[i] = col_from->get_data()[i]; } else { - double time = col_from->get_data()[i]; - auto sign = TimeValue::sign(time); - time = std::abs(time); - // e.g. scale reduce to 4, means we need to round the last 2 digits - // 999956: 56 > 100/2, then round up to 1000000 - uint32_t microseconds = TimeValue::microsecond(time); - auto divisor = (uint32_t)common::exp10_i64(6 - to_scale); - uint32_t remainder = microseconds % divisor; - - if (remainder >= divisor / 2) { // need to round up - // do rounding up - uint32_t rounded_microseconds = ((microseconds / divisor) + 1) * divisor; - // need carry on - if (rounded_microseconds >= TimeValue::ONE_SECOND_MICROSECONDS) { - DCHECK(rounded_microseconds == TimeValue::ONE_SECOND_MICROSECONDS); - time = ((int64_t)time / TimeValue::ONE_SECOND_MICROSECONDS + 1) * - TimeValue::ONE_SECOND_MICROSECONDS; - - // the input data must be valid, so max to '838:59:59.0'. this value won't carry on - // to second. - DCHECK(TimeValue::valid(time)) << col_from->get_data()[i]; - } else { - time = TimeValue::reset_microsecond(time, rounded_microseconds); - } - } else { - // truncate - time = TimeValue::reset_microsecond(time, microseconds / divisor * divisor); - } - col_to->get_data()[i] = sign * time; + const auto time = TimeValue::round_time(col_from->get_data()[i], to_scale); + DCHECK(TimeValue::valid(time)) << col_from->get_data()[i]; + col_to->get_data()[i] = time; } } else if constexpr (IsDateTimeV2Type && IsTimeV2Type) { // from Datetime to Time diff --git a/be/src/exprs/function/cast/cast_to_string.h b/be/src/exprs/function/cast/cast_to_string.h index 3e4e188b7ccf90..1a1f82dba1da79 100644 --- a/be/src/exprs/function/cast/cast_to_string.h +++ b/be/src/exprs/function/cast/cast_to_string.h @@ -160,7 +160,7 @@ template <> constexpr size_t CastToString::string_length = sizeof("YYYY-MM-DD HH:MM:SS.ssssss") - 1; template <> -constexpr size_t CastToString::string_length = sizeof("-838:59:59.999999") - 1; +constexpr size_t CastToString::string_length = sizeof("-838:59:59.999999999") - 1; template <> constexpr size_t CastToString::string_length = sizeof("255.255 .255.255") - 1; template <> diff --git a/be/src/exprs/function/cast/cast_to_time_impl.hpp b/be/src/exprs/function/cast/cast_to_time_impl.hpp index 475c91d8bd0486..c6952c382daac8 100644 --- a/be/src/exprs/function/cast/cast_to_time_impl.hpp +++ b/be/src/exprs/function/cast/cast_to_time_impl.hpp @@ -20,6 +20,7 @@ #include #include +#include #include "core/data_type/data_type_decimal.h" // IWYU pragma: keep #include "core/data_type/primitive_type.h" @@ -34,11 +35,10 @@ namespace doris { // NOLINTBEGIN(readability-function-cognitive-complexity) template -[[nodiscard]] [[maybe_unused]] static bool init_microsecond(int64_t frac_input, - uint32_t frac_length, - TimeValue::TimeType& val, - uint32_t target_scale, - CastParameters& params) { +[[nodiscard]] [[maybe_unused]] static bool init_nanosecond(int64_t frac_input, uint32_t frac_length, + TimeValue::TimeType& val, + uint32_t target_scale, + CastParameters& params) { constexpr bool IsStrict = is_datelike_parse_strict(ParseMode); if (frac_length > 0) { int sign = 1; @@ -54,14 +54,14 @@ template ? (uint32_t)(frac_input / common::exp10_i64(frac_length - target_scale)) : (uint32_t)(frac_input * common::exp10_i64(target_scale - frac_length)); - if (frac_length > target_scale) { // to_scale is up to 6 + if (frac_length > target_scale) { // round off to at most `to_scale` digits auto digit_next = (uint32_t)(frac_input / common::exp10_i64(frac_length - target_scale - 1)) % 10; if (digit_next >= 5) { in_scale_part++; - DCHECK(in_scale_part <= 1000000); - if (in_scale_part == common::exp10_i32(target_scale)) { + DCHECK(in_scale_part <= TimeValue::ONE_SECOND_NANOSECONDS); + if (std::cmp_equal(in_scale_part, common::exp10_i32(target_scale))) { // overflow, round up to next second val += sign * TimeValue::ONE_SECOND_MICROSECONDS; SET_PARAMS_RET_FALSE_IFN(TimeValue::valid(val), @@ -70,8 +70,8 @@ template } } } - val = TimeValue::init_microsecond( - val, sign * in_scale_part * common::exp10_i32(6 - (int)target_scale)); + val = TimeValue::init_nanosecond( + val, sign * in_scale_part * common::exp10_i32(9 - (int)target_scale)); } return true; } @@ -127,10 +127,12 @@ struct CastToTimeV2 { } #include "common/compile_check_avoid_begin.h" - int ms_part_7 = (float_value - (double)int_part) * common::exp10_i32(7); + int64_t ns_part_10 = + (float_value - (double)int_part) * common::exp10_i64(TimeValue::NANOS_SCALE + 1); #include "common/compile_check_avoid_end.h" - if (!init_microsecond(ms_part_7, 7, val, to_scale, params)) { - return false; // status set in init_microsecond + if (!init_nanosecond(ns_part_10, TimeValue::NANOS_SCALE + 1, val, to_scale, + params)) { + return false; // status set in init_nanosecond } return true; } @@ -163,9 +165,9 @@ struct CastToTimeV2 { return false; } - if (!init_microsecond((int64_t)frac_part, (uint32_t)decimal_scale, res, to_scale, - params)) { - return false; // status set in init_microsecond + if (!init_nanosecond((int64_t)frac_part, (uint32_t)decimal_scale, res, to_scale, + params)) { + return false; // status set in init_nanosecond } return true; } @@ -233,14 +235,14 @@ inline bool CastToTimeV2::from_integer(T input, TimeValue::TimeType& val, CastPa /**