diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index a25d3400c4799..c2aaf18992838 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -14185,9 +14185,11 @@ def timestamp_diff(unit: str, start: "ColumnOrName", end: "ColumnOrName") -> Col unit : literal string This indicates the units of the difference between the given timestamps. Supported options are (case insensitive): "YEAR", "QUARTER", "MONTH", "WEEK", - "DAY", "HOUR", "MINUTE", "SECOND", "MILLISECOND" and "MICROSECOND". + "DAY", "HOUR", "MINUTE", "SECOND", "MILLISECOND", "MICROSECOND" and "NANOSECOND". start : :class:`~pyspark.sql.Column` or column name A timestamp which the expression subtracts from `endTimestamp`. + Nanosecond-precision timestamps (TIMESTAMP_NTZ/LTZ(p), p in [7, 9]) are accepted, and + their sub-microsecond fraction participates in the truncated difference for every unit. end : :class:`~pyspark.sql.Column` or column name A timestamp from which the expression subtracts `startTimestamp`. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 722265ac1c47e..6bc0a1af9f621 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -5140,8 +5140,11 @@ case class TimestampAdd( - "SECOND" - "MILLISECOND" - "MICROSECOND" + - "NANOSECOND" * startTimestamp - A timestamp which the expression subtracts from `endTimestamp`. * endTimestamp - A timestamp from which the expression subtracts `startTimestamp`. + Nanosecond-precision timestamps (TIMESTAMP_NTZ/LTZ(p), p in [7, 9]) are accepted, and their + sub-microsecond fraction participates in the truncated difference for every unit. """, examples = """ Examples: @@ -5153,6 +5156,8 @@ case class TimestampAdd( -10 > SELECT _FUNC_(YEAR, timestamp'2000-01-01 01:02:03.123456', timestamp'2010-01-01 01:02:03.123456'); 10 + > SELECT _FUNC_(NANOSECOND, timestamp'2022-02-11 20:30:00', timestamp'2022-02-11 20:30:00.000001'); + 1000 """, group = "datetime_funcs", since = "3.3.0") @@ -5179,27 +5184,73 @@ case class TimestampDiff( override def left: Expression = startTimestamp override def right: Expression = endTimestamp - override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, TimestampType) + // Micro-precision NTZ operands keep coercing to TIMESTAMP (LTZ), preserving the pre-nanos + // timestampdiff semantics for TIMESTAMP_NTZ; only the new nanosecond-precision types are accepted + // natively (there is no prior behavior to preserve for them). + override def inputTypes: Seq[AbstractDataType] = + Seq( + TypeCollection(TimestampType, AnyTimestampNanoType), + TypeCollection(TimestampType, AnyTimestampNanoType)) override def dataType: DataType = LongType + // A nanosecond-precision operand is carried as a TimestampNanosVal object rather than a primitive + // microsecond Long; when either operand is nanos, the difference is computed at full nanosecond + // resolution so the sub-microsecond fraction participates in the truncated unit count. + @transient private lazy val isNanos: Boolean = + startTimestamp.dataType.isInstanceOf[AnyTimestampNanoType] || + endTimestamp.dataType.isInstanceOf[AnyTimestampNanoType] + override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = copy(timeZoneId = Option(timeZoneId)) @transient private lazy val zoneIdInEval: ZoneId = zoneIdForType(endTimestamp.dataType) - override def nullSafeEval(startMicros: Any, endMicros: Any): Any = { - DateTimeUtils.timestampDiff( - unit, - startMicros.asInstanceOf[Long], - endMicros.asInstanceOf[Long], - zoneIdInEval) + // For the nanosecond carrier the child value is a boxed TimestampNanosVal; the microsecond + // timestamp types are already boxed Longs whose sub-microsecond remainder is zero. + private def epochMicrosOf(value: Any): Long = value match { + case v: TimestampNanosVal => v.epochMicros + case n => n.asInstanceOf[Long] + } + private def fractionOf(value: Any): Int = value match { + case v: TimestampNanosVal => v.nanosWithinMicro.toInt + case _ => 0 + } + + override def nullSafeEval(start: Any, end: Any): Any = { + if (isNanos) { + DateTimeUtils.timestampDiffNanos( + unit, epochMicrosOf(start), fractionOf(start), epochMicrosOf(end), fractionOf(end), + zoneIdInEval) + } else { + DateTimeUtils.timestampDiff( + unit, start.asInstanceOf[Long], end.asInstanceOf[Long], zoneIdInEval) + } } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") val zid = ctx.addReferenceObj("zoneId", zoneIdInEval, classOf[ZoneId].getName) - defineCodeGen(ctx, ev, (s, e) => - s"""$dtu.timestampDiff("$unit", $s, $e, $zid)""") + if (isNanos) { + // The nanosecond carrier exposes epochMicros / nanosWithinMicro as public fields; the + // microsecond types are primitive longs with a zero fraction. + def microsCode(e: Expression): String => String = e.dataType match { + case _: AnyTimestampNanoType => c => s"$c.epochMicros" + case _ => c => c + } + def fractionCode(e: Expression): String => String = e.dataType match { + case _: AnyTimestampNanoType => c => s"$c.nanosWithinMicro" + case _ => _ => "0" + } + val sM = microsCode(startTimestamp) + val sF = fractionCode(startTimestamp) + val eM = microsCode(endTimestamp) + val eF = fractionCode(endTimestamp) + defineCodeGen(ctx, ev, (s, e) => + s"""$dtu.timestampDiffNanos("$unit", ${sM(s)}, ${sF(s)}, ${eM(e)}, ${eF(e)}, $zid)""") + } else { + defineCodeGen(ctx, ev, (s, e) => + s"""$dtu.timestampDiff("$unit", $s, $e, $zid)""") + } } override def prettyName: String = "timestampdiff" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index f67d43c6502d1..2f2c2c8defafd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -1047,6 +1047,7 @@ object DateTimeUtils extends SparkDateTimeUtils { } private val timestampDiffMap = Map[String, (Temporal, Temporal) => Long]( + "NANOSECOND" -> ChronoUnit.NANOS.between, "MICROSECOND" -> ChronoUnit.MICROS.between, "MILLISECOND" -> ChronoUnit.MILLIS.between, "SECOND" -> ChronoUnit.SECONDS.between, @@ -1074,12 +1075,62 @@ object DateTimeUtils extends SparkDateTimeUtils { if (timestampDiffMap.contains(unitInUpperCase)) { val startLocalTs = getLocalDateTime(startTs, zoneId) val endLocalTs = getLocalDateTime(endTs, zoneId) - timestampDiffMap(unitInUpperCase)(startLocalTs, endLocalTs) + try { + timestampDiffMap(unitInUpperCase)(startLocalTs, endLocalTs) + } catch { + // NANOSECOND is the only unit whose count overflows a 64-bit long in the supported + // ~292-year range; surface DATETIME_OVERFLOW instead of a raw ArithmeticException. + case _: ArithmeticException => + throw QueryExecutionErrors.timestampDiffOverflowError(unit) + } } else { throw QueryExecutionErrors.invalidDatetimeUnitError("TIMESTAMPDIFF", unit) } } + /** + * Gets the difference between two nanosecond-precision timestamps, expressed in whole `unit`s + * (truncated toward zero), honoring the sub-microsecond fraction of each operand. + * + * Each operand is given as its `epochMicros` plus a `nanosWithinMicro` fraction in [0, 999]. The + * fraction is folded into a nanosecond-precision `LocalDateTime` before the difference is taken, + * so a fraction of up to a microsecond can move the truncated result across a unit boundary for + * every unit (not only NANOSECOND). A microsecond operand simply passes a zero fraction. The + * `NANOSECOND` unit is added to the shared unit map, so it is accepted here and by the + * microsecond-only [[timestampDiff]] (where both fractions are zero). + * + * @param unit The unit in which to express the difference. + * @param startMicros `epochMicros` of the timestamp subtracted from `end`. + * @param startFraction `nanosWithinMicro` in [0, 999] of the start timestamp. + * @param endMicros `epochMicros` of the timestamp from which `start` is subtracted. + * @param endFraction `nanosWithinMicro` in [0, 999] of the end timestamp. + * @param zoneId The time zone ID at which the operation is performed. + * @return The truncated difference in the requested unit. + */ + def timestampDiffNanos( + unit: String, + startMicros: Long, + startFraction: Int, + endMicros: Long, + endFraction: Int, + zoneId: ZoneId): Long = { + val unitInUpperCase = unit.toUpperCase(Locale.ROOT) + timestampDiffMap.get(unitInUpperCase) match { + case Some(diff) => + val startLocalTs = getLocalDateTime(startMicros, zoneId).plusNanos(startFraction.toLong) + val endLocalTs = getLocalDateTime(endMicros, zoneId).plusNanos(endFraction.toLong) + try { + diff(startLocalTs, endLocalTs) + } catch { + // The NANOSECOND count can overflow a 64-bit long past ~292 years. + case _: ArithmeticException => + throw QueryExecutionErrors.timestampDiffOverflowError(unit) + } + case None => + throw QueryExecutionErrors.invalidDatetimeUnitError("TIMESTAMPDIFF", unit) + } + } + /** * Converts separate time fields in a long that represents nanoseconds since the start of * the day diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 46eb4aaa816a1..2d5535de1c530 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -2731,6 +2731,15 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE summary = "") } + def timestampDiffOverflowError(unit: String): ArithmeticException = { + new SparkArithmeticException( + errorClass = "DATETIME_OVERFLOW", + messageParameters = Map( + "operation" -> s"get the number of $unit between the two timestamps"), + context = Array.empty, + summary = "") + } + def calendarIntervalArrowNanosOverflowError( interval: CalendarInterval): SparkArithmeticException = { new SparkArithmeticException( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index de3773f145003..5d77d5c199d29 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -3011,6 +3011,65 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } } + test("SPARK-57833: timestampdiff over nanosecond-precision timestamps") { + val sec = 1000000L // microseconds per second + def ntz(micros: Long, frac: Int, p: Int = 9): Literal = + Literal(TimestampNanosVal.fromParts(micros, frac.toShort), TimestampNTZNanosType(p)) + def ltz(micros: Long, frac: Int, p: Int = 9): Literal = + Literal(TimestampNanosVal.fromParts(micros, frac.toShort), TimestampLTZNanosType(p)) + + // NANOSECOND unit reports the exact sub-microsecond difference (within a microsecond, across + // microseconds, and negative). + checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 100), ntz(0, 900)), 800L) + checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 900), ntz(2, 100)), 1200L) + checkEvaluation(TimestampDiff("NANOSECOND", ntz(2, 100), ntz(0, 900)), -1200L) + + // The sub-microsecond fraction participates in the truncated count for coarser units too: a + // start fraction larger than the end fraction means a whole second/microsecond has NOT elapsed. + // SECOND: 1s + 100ns - 900ns = 0.9999992s -> 0 (the micros-only count would wrongly be 1). + checkEvaluation(TimestampDiff("SECOND", ntz(0, 900), ntz(sec, 100)), 0L) + // 1s + 900ns - 100ns = 1.0000008s -> 1. + checkEvaluation(TimestampDiff("SECOND", ntz(0, 100), ntz(sec, 900)), 1L) + // MICROSECOND: 2100ns - 900ns = 1200ns -> 1 (the micros-only count would wrongly be 2). + checkEvaluation(TimestampDiff("MICROSECOND", ntz(0, 900), ntz(2, 100)), 1L) + + // Precision 7 (100ns step) and 8 (10ns step) fractions. + checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 100, 7), ntz(0, 300, 7)), 200L) + checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 110, 8), ntz(0, 200, 8)), 90L) + + // LTZ (zone-aware), exact whole minute with equal fractions. + checkEvaluation( + TimestampDiff("MINUTE", ltz(0, 500), ltz(60 * sec, 500), Some("UTC")), 1L) + + // Mixed operands: a microsecond TIMESTAMP_LTZ start (zero fraction) and a nanosecond LTZ end. + checkEvaluation( + TimestampDiff("SECOND", Literal(0L, TimestampType), ltz(sec, 500), Some("UTC")), 1L) + + // NANOSECOND between two microsecond timestamps is well-defined (fractions are zero): the + // difference is a whole number of microseconds times 1000. + checkEvaluation( + TimestampDiff("NANOSECOND", Literal(0L, TimestampType), Literal(1L, TimestampType)), 1000L) + + // Null propagation on either operand. + checkEvaluation( + TimestampDiff("SECOND", Literal.create(null, TimestampNTZNanosType(9)), ntz(sec, 0)), null) + checkEvaluation( + TimestampDiff("NANOSECOND", ntz(0, 1), Literal.create(null, TimestampNTZNanosType(9))), null) + + // A NANOSECOND difference wider than ~292 years overflows a 64-bit nanosecond count. It is + // surfaced as DATETIME_OVERFLOW (matching the timestampadd side) rather than a raw + // ArithmeticException, on both the microsecond and the nanosecond-carrier code paths. + checkErrorInExpression[SparkArithmeticException]( + TimestampDiff("NANOSECOND", + Literal(-5000000000000000L, TimestampType), Literal(5000000000000000L, TimestampType)), + condition = "DATETIME_OVERFLOW", + parameters = Map("operation" -> "get the number of NANOSECOND between the two timestamps")) + checkErrorInExpression[SparkArithmeticException]( + TimestampDiff("NANOSECOND", ntz(-5000000000000000L, 0), ntz(5000000000000000L, 0)), + condition = "DATETIME_OVERFLOW", + parameters = Map("operation" -> "get the number of NANOSECOND between the two timestamps")) + } + /** * Helper method to create a DATE literal from a string in date format. */ diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out index 297b91331d0bf..600c1721bc286 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out @@ -1698,3 +1698,17 @@ Project [timestampadd(NANOSECOND, cast(900 as bigint), 2019-12-31 16:00:00.00000 SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_LTZ '2020-01-01 00:00:00 UTC') -- !query analysis [Analyzer test output redacted due to nondeterminism] + + +-- !query +SELECT timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC') +-- !query analysis +Project [timestampdiff(NANOSECOND, 2019-12-31 16:00:00.0000001, 2019-12-31 16:00:00.0000009, Some(America/Los_Angeles)) AS timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2019-12-31 16:00:00.000000100', TIMESTAMP_LTZ '2019-12-31 16:00:00.000000900')#xL] ++- OneRowRelation + + +-- !query +SELECT timestampdiff(SECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:01.000000100 UTC') +-- !query analysis +Project [timestampdiff(SECOND, 2019-12-31 16:00:00.0000009, 2019-12-31 16:00:01.0000001, Some(America/Los_Angeles)) AS timestampdiff(SECOND, TIMESTAMP_LTZ '2019-12-31 16:00:00.000000900', TIMESTAMP_LTZ '2019-12-31 16:00:01.000000100')#xL] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out index a69d06a5b6ccb..b98c8712d16e4 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out @@ -1667,3 +1667,24 @@ SELECT timestampadd(NANOSECOND, 50, '2020-01-01 00:00:00.0000001' :: timestamp_n -- !query analysis Project [timestampadd(NANOSECOND, cast(50 as bigint), cast(2020-01-01 00:00:00.0000001 as timestamp_ntz(7)), Some(America/Los_Angeles)) AS timestampadd(NANOSECOND, 50, CAST(2020-01-01 00:00:00.0000001 AS TIMESTAMP_NTZ(7)))#x] +- OneRowRelation + + +-- !query +SELECT timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900') +-- !query analysis +Project [timestampdiff(NANOSECOND, 2020-01-01 00:00:00.0000001, 2020-01-01 00:00:00.0000009, Some(America/Los_Angeles)) AS timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900')#xL] ++- OneRowRelation + + +-- !query +SELECT timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100') +-- !query analysis +Project [timestampdiff(SECOND, 2020-01-01 00:00:00.0000009, 2020-01-01 00:00:01.0000001, Some(America/Los_Angeles)) AS timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100')#xL] ++- OneRowRelation + + +-- !query +SELECT timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100') +-- !query analysis +Project [timestampdiff(MICROSECOND, 2020-01-01 00:00:00.0000009, 2020-01-01 00:00:00.0000021, Some(America/Los_Angeles)) AS timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100')#xL] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql index f1aa0a45803e4..d5e3edf2d86fe 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql @@ -592,3 +592,8 @@ SELECT timestampadd(NANOSECOND, -300, TIMESTAMP_LTZ '2020-01-01 00:00:00.0000001 SELECT timestampadd(NANOSECOND, 900, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000200 UTC'); -- NANOSECOND is rejected on a microsecond-precision timestamp (nanoseconds are unrepresentable). SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_LTZ '2020-01-01 00:00:00 UTC'); + +-- SPARK-57833: timestampdiff over TIMESTAMP_LTZ(p). NANOSECOND reports the exact difference; the +-- fraction also tips coarser units (SECOND is 0 here because only 0.9999992s elapsed). +SELECT timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC'); +SELECT timestampdiff(SECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:01.000000100 UTC'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql index f634cfb0dba91..ec1682f3f819c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql @@ -585,3 +585,10 @@ SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_NTZ '2020-01-01 00:00:00'); -- .0000001 value lands on .0000002, and a sub-step +50ns is truncated back to .0000001. SELECT timestampadd(NANOSECOND, 150, '2020-01-01 00:00:00.0000001' :: timestamp_ntz(7)); SELECT timestampadd(NANOSECOND, 50, '2020-01-01 00:00:00.0000001' :: timestamp_ntz(7)); + +-- SPARK-57833: timestampdiff over TIMESTAMP_NTZ(p). NANOSECOND reports the exact sub-microsecond +-- difference, and the fraction participates in the truncated count for coarser units too: SECOND +-- between .000000900 and the next second's .000000100 is 0 (only 0.9999992s elapsed). +SELECT timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900'); +SELECT timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100'); +SELECT timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp.sql index 6f8270f0c9758..6ccf056c2dc25 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp.sql @@ -207,9 +207,9 @@ select timestampdiff(SECOND, date'2022-02-15', timestamp'2022-02-14 23:59:59'); select timestampdiff('MINUTE', timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 02:00:03'); select timestampdiff('YEAR', date'2022-02-15', date'2023-02-15'); --- NANOSECOND is a valid datetimeUnit keyword (timestampadd accepts it for nanosecond-precision --- timestamps) but timestampdiff does not support it, so it is rejected at runtime, matching other --- add-only units such as DAYOFYEAR. +-- NANOSECOND is supported by both timestampadd (for nanosecond-precision inputs) and timestampdiff. +-- Between two microsecond timestamps the sub-microsecond fractions are zero, so the difference is a +-- whole number of microseconds times 1000 (here one second = 1000000000 ns). select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04'); select timediff(QUARTER, timestamp'2023-08-10 01:02:03', timestamp'2022-01-14 01:02:03'); diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out index aa877a43a8590..76d94d5133eca 100644 --- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out @@ -2779,18 +2779,9 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04') -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkIllegalArgumentException -{ - "errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT", - "sqlState" : "22023", - "messageParameters" : { - "functionName" : "`TIMESTAMPDIFF`", - "invalidValue" : "'NANOSECOND'", - "parameter" : "`unit`" - } -} +1000000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/timestamp.sql.out index 09fd5add2e0a9..88ec350fd3645 100644 --- a/sql/core/src/test/resources/sql-tests/results/nonansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/nonansi/timestamp.sql.out @@ -1502,18 +1502,9 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04') -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkIllegalArgumentException -{ - "errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT", - "sqlState" : "22023", - "messageParameters" : { - "functionName" : "`TIMESTAMPDIFF`", - "invalidValue" : "'NANOSECOND'", - "parameter" : "`unit`" - } -} +1000000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out index 101ddf43099e9..fad2e842c61f0 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out @@ -1847,3 +1847,19 @@ org.apache.spark.SparkIllegalArgumentException "parameter" : "`unit`" } } + + +-- !query +SELECT timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC') +-- !query schema +struct +-- !query output +800 + + +-- !query +SELECT timestampdiff(SECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:01.000000100 UTC') +-- !query schema +struct +-- !query output +0 diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out index ed77e6e460b1b..88bc7d0337e3a 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out @@ -1737,3 +1737,27 @@ SELECT timestampadd(NANOSECOND, 50, '2020-01-01 00:00:00.0000001' :: timestamp_n struct -- !query output 2020-01-01 00:00:00.0000001 + + +-- !query +SELECT timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900') +-- !query schema +struct +-- !query output +800 + + +-- !query +SELECT timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100') +-- !query schema +struct +-- !query output +0 + + +-- !query +SELECT timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100') +-- !query schema +struct +-- !query output +1 diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp.sql.out index 574b795950373..78c66ef712f9e 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp.sql.out @@ -1506,18 +1506,9 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04') -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkIllegalArgumentException -{ - "errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT", - "sqlState" : "22023", - "messageParameters" : { - "functionName" : "`TIMESTAMPDIFF`", - "invalidValue" : "'NANOSECOND'", - "parameter" : "`unit`" - } -} +1000000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index 6b15e6911c855..811012cffc471 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -1502,18 +1502,9 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04') -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkIllegalArgumentException -{ - "errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT", - "sqlState" : "22023", - "messageParameters" : { - "functionName" : "`TIMESTAMPDIFF`", - "invalidValue" : "'NANOSECOND'", - "parameter" : "`unit`" - } -} +1000000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out index 687d608212bb8..0dfca719fb11e 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out @@ -1481,18 +1481,9 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04') -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkIllegalArgumentException -{ - "errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT", - "sqlState" : "22023", - "messageParameters" : { - "functionName" : "`TIMESTAMPDIFF`", - "invalidValue" : "'NANOSECOND'", - "parameter" : "`unit`" - } -} +1000000000 -- !query