Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion python/pyspark/sql/functions/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -14185,9 +14185,11 @@ def timestamp_diff(unit: str, start: "ColumnOrName", end: "ColumnOrName") -> Col
unit : literal string
This indicates the units of the difference between the given timestamps.
Supported options are (case insensitive): "YEAR", "QUARTER", "MONTH", "WEEK",
"DAY", "HOUR", "MINUTE", "SECOND", "MILLISECOND" and "MICROSECOND".
"DAY", "HOUR", "MINUTE", "SECOND", "MILLISECOND", "MICROSECOND" and "NANOSECOND".
start : :class:`~pyspark.sql.Column` or column name
A timestamp which the expression subtracts from `endTimestamp`.
Nanosecond-precision timestamps (TIMESTAMP_NTZ/LTZ(p), p in [7, 9]) are accepted, and
their sub-microsecond fraction participates in the truncated difference for every unit.
end : :class:`~pyspark.sql.Column` or column name
A timestamp from which the expression subtracts `startTimestamp`.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5140,8 +5140,11 @@ case class TimestampAdd(
- "SECOND"
- "MILLISECOND"
- "MICROSECOND"
- "NANOSECOND"
* startTimestamp - A timestamp which the expression subtracts from `endTimestamp`.
* endTimestamp - A timestamp from which the expression subtracts `startTimestamp`.
Nanosecond-precision timestamps (TIMESTAMP_NTZ/LTZ(p), p in [7, 9]) are accepted, and their
sub-microsecond fraction participates in the truncated difference for every unit.
""",
examples = """
Examples:
Expand All @@ -5153,6 +5156,8 @@ case class TimestampAdd(
-10
> SELECT _FUNC_(YEAR, timestamp'2000-01-01 01:02:03.123456', timestamp'2010-01-01 01:02:03.123456');
10
> SELECT _FUNC_(NANOSECOND, timestamp'2022-02-11 20:30:00', timestamp'2022-02-11 20:30:00.000001');
1000
""",
group = "datetime_funcs",
since = "3.3.0")
Expand All @@ -5179,27 +5184,73 @@ case class TimestampDiff(
override def left: Expression = startTimestamp
override def right: Expression = endTimestamp

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType, TimestampType)
// Micro-precision NTZ operands keep coercing to TIMESTAMP (LTZ), preserving the pre-nanos
// timestampdiff semantics for TIMESTAMP_NTZ; only the new nanosecond-precision types are accepted
// natively (there is no prior behavior to preserve for them).
override def inputTypes: Seq[AbstractDataType] =
Seq(
TypeCollection(TimestampType, AnyTimestampNanoType),
TypeCollection(TimestampType, AnyTimestampNanoType))
override def dataType: DataType = LongType

// A nanosecond-precision operand is carried as a TimestampNanosVal object rather than a primitive
// microsecond Long; when either operand is nanos, the difference is computed at full nanosecond
// resolution so the sub-microsecond fraction participates in the truncated unit count.
@transient private lazy val isNanos: Boolean =
startTimestamp.dataType.isInstanceOf[AnyTimestampNanoType] ||
endTimestamp.dataType.isInstanceOf[AnyTimestampNanoType]

override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
copy(timeZoneId = Option(timeZoneId))

@transient private lazy val zoneIdInEval: ZoneId = zoneIdForType(endTimestamp.dataType)

override def nullSafeEval(startMicros: Any, endMicros: Any): Any = {
DateTimeUtils.timestampDiff(
unit,
startMicros.asInstanceOf[Long],
endMicros.asInstanceOf[Long],
zoneIdInEval)
// For the nanosecond carrier the child value is a boxed TimestampNanosVal; the microsecond
// timestamp types are already boxed Longs whose sub-microsecond remainder is zero.
private def epochMicrosOf(value: Any): Long = value match {
case v: TimestampNanosVal => v.epochMicros
case n => n.asInstanceOf[Long]
}
private def fractionOf(value: Any): Int = value match {
case v: TimestampNanosVal => v.nanosWithinMicro.toInt
case _ => 0
}

override def nullSafeEval(start: Any, end: Any): Any = {
if (isNanos) {
DateTimeUtils.timestampDiffNanos(
unit, epochMicrosOf(start), fractionOf(start), epochMicrosOf(end), fractionOf(end),
zoneIdInEval)
} else {
DateTimeUtils.timestampDiff(
unit, start.asInstanceOf[Long], end.asInstanceOf[Long], zoneIdInEval)
}
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
val zid = ctx.addReferenceObj("zoneId", zoneIdInEval, classOf[ZoneId].getName)
defineCodeGen(ctx, ev, (s, e) =>
s"""$dtu.timestampDiff("$unit", $s, $e, $zid)""")
if (isNanos) {
// The nanosecond carrier exposes epochMicros / nanosWithinMicro as public fields; the
// microsecond types are primitive longs with a zero fraction.
def microsCode(e: Expression): String => String = e.dataType match {
case _: AnyTimestampNanoType => c => s"$c.epochMicros"
case _ => c => c
}
def fractionCode(e: Expression): String => String = e.dataType match {
case _: AnyTimestampNanoType => c => s"$c.nanosWithinMicro"
case _ => _ => "0"
}
val sM = microsCode(startTimestamp)
val sF = fractionCode(startTimestamp)
val eM = microsCode(endTimestamp)
val eF = fractionCode(endTimestamp)
defineCodeGen(ctx, ev, (s, e) =>
s"""$dtu.timestampDiffNanos("$unit", ${sM(s)}, ${sF(s)}, ${eM(e)}, ${eF(e)}, $zid)""")
} else {
defineCodeGen(ctx, ev, (s, e) =>
s"""$dtu.timestampDiff("$unit", $s, $e, $zid)""")
}
}

override def prettyName: String = "timestampdiff"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1047,6 +1047,7 @@ object DateTimeUtils extends SparkDateTimeUtils {
}

private val timestampDiffMap = Map[String, (Temporal, Temporal) => Long](
"NANOSECOND" -> ChronoUnit.NANOS.between,
"MICROSECOND" -> ChronoUnit.MICROS.between,
"MILLISECOND" -> ChronoUnit.MILLIS.between,
"SECOND" -> ChronoUnit.SECONDS.between,
Expand Down Expand Up @@ -1074,12 +1075,62 @@ object DateTimeUtils extends SparkDateTimeUtils {
if (timestampDiffMap.contains(unitInUpperCase)) {
val startLocalTs = getLocalDateTime(startTs, zoneId)
val endLocalTs = getLocalDateTime(endTs, zoneId)
timestampDiffMap(unitInUpperCase)(startLocalTs, endLocalTs)
try {
timestampDiffMap(unitInUpperCase)(startLocalTs, endLocalTs)
} catch {
// NANOSECOND is the only unit whose count overflows a 64-bit long in the supported
// ~292-year range; surface DATETIME_OVERFLOW instead of a raw ArithmeticException.
case _: ArithmeticException =>
throw QueryExecutionErrors.timestampDiffOverflowError(unit)
}
} else {
throw QueryExecutionErrors.invalidDatetimeUnitError("TIMESTAMPDIFF", unit)
}
}

/**
* Gets the difference between two nanosecond-precision timestamps, expressed in whole `unit`s
* (truncated toward zero), honoring the sub-microsecond fraction of each operand.
*
* Each operand is given as its `epochMicros` plus a `nanosWithinMicro` fraction in [0, 999]. The
* fraction is folded into a nanosecond-precision `LocalDateTime` before the difference is taken,
* so a fraction of up to a microsecond can move the truncated result across a unit boundary for
* every unit (not only NANOSECOND). A microsecond operand simply passes a zero fraction. The
* `NANOSECOND` unit is added to the shared unit map, so it is accepted here and by the
* microsecond-only [[timestampDiff]] (where both fractions are zero).
*
* @param unit The unit in which to express the difference.
* @param startMicros `epochMicros` of the timestamp subtracted from `end`.
* @param startFraction `nanosWithinMicro` in [0, 999] of the start timestamp.
* @param endMicros `epochMicros` of the timestamp from which `start` is subtracted.
* @param endFraction `nanosWithinMicro` in [0, 999] of the end timestamp.
* @param zoneId The time zone ID at which the operation is performed.
* @return The truncated difference in the requested unit.
*/
def timestampDiffNanos(
unit: String,
startMicros: Long,
startFraction: Int,
endMicros: Long,
endFraction: Int,
zoneId: ZoneId): Long = {
val unitInUpperCase = unit.toUpperCase(Locale.ROOT)
timestampDiffMap.get(unitInUpperCase) match {
case Some(diff) =>
val startLocalTs = getLocalDateTime(startMicros, zoneId).plusNanos(startFraction.toLong)
val endLocalTs = getLocalDateTime(endMicros, zoneId).plusNanos(endFraction.toLong)
try {
diff(startLocalTs, endLocalTs)
} catch {
// The NANOSECOND count can overflow a 64-bit long past ~292 years.
case _: ArithmeticException =>
throw QueryExecutionErrors.timestampDiffOverflowError(unit)
}
case None =>
throw QueryExecutionErrors.invalidDatetimeUnitError("TIMESTAMPDIFF", unit)
}
}

/**
* Converts separate time fields in a long that represents nanoseconds since the start of
* the day
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2731,6 +2731,15 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
summary = "")
}

def timestampDiffOverflowError(unit: String): ArithmeticException = {
new SparkArithmeticException(
errorClass = "DATETIME_OVERFLOW",
messageParameters = Map(
"operation" -> s"get the number of $unit between the two timestamps"),
context = Array.empty,
summary = "")
}

def calendarIntervalArrowNanosOverflowError(
interval: CalendarInterval): SparkArithmeticException = {
new SparkArithmeticException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3011,6 +3011,65 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
}

test("SPARK-57833: timestampdiff over nanosecond-precision timestamps") {
val sec = 1000000L // microseconds per second
def ntz(micros: Long, frac: Int, p: Int = 9): Literal =
Literal(TimestampNanosVal.fromParts(micros, frac.toShort), TimestampNTZNanosType(p))
def ltz(micros: Long, frac: Int, p: Int = 9): Literal =
Literal(TimestampNanosVal.fromParts(micros, frac.toShort), TimestampLTZNanosType(p))

// NANOSECOND unit reports the exact sub-microsecond difference (within a microsecond, across
// microseconds, and negative).
checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 100), ntz(0, 900)), 800L)
checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 900), ntz(2, 100)), 1200L)
checkEvaluation(TimestampDiff("NANOSECOND", ntz(2, 100), ntz(0, 900)), -1200L)

// The sub-microsecond fraction participates in the truncated count for coarser units too: a
// start fraction larger than the end fraction means a whole second/microsecond has NOT elapsed.
// SECOND: 1s + 100ns - 900ns = 0.9999992s -> 0 (the micros-only count would wrongly be 1).
checkEvaluation(TimestampDiff("SECOND", ntz(0, 900), ntz(sec, 100)), 0L)
// 1s + 900ns - 100ns = 1.0000008s -> 1.
checkEvaluation(TimestampDiff("SECOND", ntz(0, 100), ntz(sec, 900)), 1L)
// MICROSECOND: 2100ns - 900ns = 1200ns -> 1 (the micros-only count would wrongly be 2).
checkEvaluation(TimestampDiff("MICROSECOND", ntz(0, 900), ntz(2, 100)), 1L)

// Precision 7 (100ns step) and 8 (10ns step) fractions.
checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 100, 7), ntz(0, 300, 7)), 200L)
checkEvaluation(TimestampDiff("NANOSECOND", ntz(0, 110, 8), ntz(0, 200, 8)), 90L)

// LTZ (zone-aware), exact whole minute with equal fractions.
checkEvaluation(
TimestampDiff("MINUTE", ltz(0, 500), ltz(60 * sec, 500), Some("UTC")), 1L)

// Mixed operands: a microsecond TIMESTAMP_LTZ start (zero fraction) and a nanosecond LTZ end.
checkEvaluation(
TimestampDiff("SECOND", Literal(0L, TimestampType), ltz(sec, 500), Some("UTC")), 1L)

// NANOSECOND between two microsecond timestamps is well-defined (fractions are zero): the
// difference is a whole number of microseconds times 1000.
checkEvaluation(
TimestampDiff("NANOSECOND", Literal(0L, TimestampType), Literal(1L, TimestampType)), 1000L)

// Null propagation on either operand.
checkEvaluation(
TimestampDiff("SECOND", Literal.create(null, TimestampNTZNanosType(9)), ntz(sec, 0)), null)
checkEvaluation(
TimestampDiff("NANOSECOND", ntz(0, 1), Literal.create(null, TimestampNTZNanosType(9))), null)

// A NANOSECOND difference wider than ~292 years overflows a 64-bit nanosecond count. It is
// surfaced as DATETIME_OVERFLOW (matching the timestampadd side) rather than a raw
// ArithmeticException, on both the microsecond and the nanosecond-carrier code paths.
checkErrorInExpression[SparkArithmeticException](
TimestampDiff("NANOSECOND",
Literal(-5000000000000000L, TimestampType), Literal(5000000000000000L, TimestampType)),
condition = "DATETIME_OVERFLOW",
parameters = Map("operation" -> "get the number of NANOSECOND between the two timestamps"))
checkErrorInExpression[SparkArithmeticException](
TimestampDiff("NANOSECOND", ntz(-5000000000000000L, 0), ntz(5000000000000000L, 0)),
condition = "DATETIME_OVERFLOW",
parameters = Map("operation" -> "get the number of NANOSECOND between the two timestamps"))
}

/**
* Helper method to create a DATE literal from a string in date format.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1698,3 +1698,17 @@ Project [timestampadd(NANOSECOND, cast(900 as bigint), 2019-12-31 16:00:00.00000
SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_LTZ '2020-01-01 00:00:00 UTC')
-- !query analysis
[Analyzer test output redacted due to nondeterminism]


-- !query
SELECT timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC')
-- !query analysis
Project [timestampdiff(NANOSECOND, 2019-12-31 16:00:00.0000001, 2019-12-31 16:00:00.0000009, Some(America/Los_Angeles)) AS timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2019-12-31 16:00:00.000000100', TIMESTAMP_LTZ '2019-12-31 16:00:00.000000900')#xL]
+- OneRowRelation


-- !query
SELECT timestampdiff(SECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:01.000000100 UTC')
-- !query analysis
Project [timestampdiff(SECOND, 2019-12-31 16:00:00.0000009, 2019-12-31 16:00:01.0000001, Some(America/Los_Angeles)) AS timestampdiff(SECOND, TIMESTAMP_LTZ '2019-12-31 16:00:00.000000900', TIMESTAMP_LTZ '2019-12-31 16:00:01.000000100')#xL]
+- OneRowRelation
Original file line number Diff line number Diff line change
Expand Up @@ -1667,3 +1667,24 @@ SELECT timestampadd(NANOSECOND, 50, '2020-01-01 00:00:00.0000001' :: timestamp_n
-- !query analysis
Project [timestampadd(NANOSECOND, cast(50 as bigint), cast(2020-01-01 00:00:00.0000001 as timestamp_ntz(7)), Some(America/Los_Angeles)) AS timestampadd(NANOSECOND, 50, CAST(2020-01-01 00:00:00.0000001 AS TIMESTAMP_NTZ(7)))#x]
+- OneRowRelation


-- !query
SELECT timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900')
-- !query analysis
Project [timestampdiff(NANOSECOND, 2020-01-01 00:00:00.0000001, 2020-01-01 00:00:00.0000009, Some(America/Los_Angeles)) AS timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900')#xL]
+- OneRowRelation


-- !query
SELECT timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100')
-- !query analysis
Project [timestampdiff(SECOND, 2020-01-01 00:00:00.0000009, 2020-01-01 00:00:01.0000001, Some(America/Los_Angeles)) AS timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100')#xL]
+- OneRowRelation


-- !query
SELECT timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100')
-- !query analysis
Project [timestampdiff(MICROSECOND, 2020-01-01 00:00:00.0000009, 2020-01-01 00:00:00.0000021, Some(America/Los_Angeles)) AS timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100')#xL]
+- OneRowRelation
Original file line number Diff line number Diff line change
Expand Up @@ -592,3 +592,8 @@ SELECT timestampadd(NANOSECOND, -300, TIMESTAMP_LTZ '2020-01-01 00:00:00.0000001
SELECT timestampadd(NANOSECOND, 900, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000200 UTC');
-- NANOSECOND is rejected on a microsecond-precision timestamp (nanoseconds are unrepresentable).
SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_LTZ '2020-01-01 00:00:00 UTC');

-- SPARK-57833: timestampdiff over TIMESTAMP_LTZ(p). NANOSECOND reports the exact difference; the
-- fraction also tips coarser units (SECOND is 0 here because only 0.9999992s elapsed).
SELECT timestampdiff(NANOSECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC');
SELECT timestampdiff(SECOND, TIMESTAMP_LTZ '2020-01-01 00:00:00.000000900 UTC', TIMESTAMP_LTZ '2020-01-01 00:00:01.000000100 UTC');
Original file line number Diff line number Diff line change
Expand Up @@ -585,3 +585,10 @@ SELECT timestampadd(NANOSECOND, 1, TIMESTAMP_NTZ '2020-01-01 00:00:00');
-- .0000001 value lands on .0000002, and a sub-step +50ns is truncated back to .0000001.
SELECT timestampadd(NANOSECOND, 150, '2020-01-01 00:00:00.0000001' :: timestamp_ntz(7));
SELECT timestampadd(NANOSECOND, 50, '2020-01-01 00:00:00.0000001' :: timestamp_ntz(7));

-- SPARK-57833: timestampdiff over TIMESTAMP_NTZ(p). NANOSECOND reports the exact sub-microsecond
-- difference, and the fraction participates in the truncated count for coarser units too: SECOND
-- between .000000900 and the next second's .000000100 is 0 (only 0.9999992s elapsed).
SELECT timestampdiff(NANOSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100', TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900');
SELECT timestampdiff(SECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:01.000000100');
SELECT timestampdiff(MICROSECOND, TIMESTAMP_NTZ '2020-01-01 00:00:00.000000900', TIMESTAMP_NTZ '2020-01-01 00:00:00.000002100');
6 changes: 3 additions & 3 deletions sql/core/src/test/resources/sql-tests/inputs/timestamp.sql
Original file line number Diff line number Diff line change
Expand Up @@ -207,9 +207,9 @@ select timestampdiff(SECOND, date'2022-02-15', timestamp'2022-02-14 23:59:59');
select timestampdiff('MINUTE', timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 02:00:03');
select timestampdiff('YEAR', date'2022-02-15', date'2023-02-15');

-- NANOSECOND is a valid datetimeUnit keyword (timestampadd accepts it for nanosecond-precision
-- timestamps) but timestampdiff does not support it, so it is rejected at runtime, matching other
-- add-only units such as DAYOFYEAR.
-- NANOSECOND is supported by both timestampadd (for nanosecond-precision inputs) and timestampdiff.
-- Between two microsecond timestamps the sub-microsecond fractions are zero, so the difference is a
-- whole number of microseconds times 1000 (here one second = 1000000000 ns).
select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04');

select timediff(QUARTER, timestamp'2023-08-10 01:02:03', timestamp'2022-01-14 01:02:03');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2779,18 +2779,9 @@ org.apache.spark.sql.catalyst.parser.ParseException
-- !query
select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04')
-- !query schema
struct<>
struct<timestampdiff(NANOSECOND, TIMESTAMP '2022-02-14 01:02:03', TIMESTAMP '2022-02-14 01:02:04'):bigint>
-- !query output
org.apache.spark.SparkIllegalArgumentException
{
"errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
"sqlState" : "22023",
"messageParameters" : {
"functionName" : "`TIMESTAMPDIFF`",
"invalidValue" : "'NANOSECOND'",
"parameter" : "`unit`"
}
}
1000000000


-- !query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1502,18 +1502,9 @@ org.apache.spark.sql.catalyst.parser.ParseException
-- !query
select timestampdiff(NANOSECOND, timestamp'2022-02-14 01:02:03', timestamp'2022-02-14 01:02:04')
-- !query schema
struct<>
struct<timestampdiff(NANOSECOND, TIMESTAMP '2022-02-14 01:02:03', TIMESTAMP '2022-02-14 01:02:04'):bigint>
-- !query output
org.apache.spark.SparkIllegalArgumentException
{
"errorClass" : "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
"sqlState" : "22023",
"messageParameters" : {
"functionName" : "`TIMESTAMPDIFF`",
"invalidValue" : "'NANOSECOND'",
"parameter" : "`unit`"
}
}
1000000000


-- !query
Expand Down
Loading