From bcc24a8f6310133f14f922919d677d991bfbdc65 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 18 Aug 2026 21:12:19 +0200 Subject: [PATCH 01/22] Add Iceberg tests for nanosecond timestamp pruning Identity partition pruning on `timestamp_ns` over-prunes matching files. Min/max pruning also returns empty results when Iceberg `timestamp` (`DateTime64(6)`) lower/upper bounds are stored as nanoseconds, which is the customer `toDateTime64(..., 6)` case. Co-authored-by: Cursor --- .../helpers/iceberg_timestamp_ns.py | 444 ++++++++++++++++++ .../test_pruning_datetime64_nanoseconds.py | 244 ++++++++++ .../test_pruning_datetime64_nanoseconds.py | 181 +++++++ 3 files changed, 869 insertions(+) create mode 100644 tests/integration/helpers/iceberg_timestamp_ns.py create mode 100644 tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_pruning_datetime64_nanoseconds.py diff --git a/tests/integration/helpers/iceberg_timestamp_ns.py b/tests/integration/helpers/iceberg_timestamp_ns.py new file mode 100644 index 0000000000000..37b9e365fa82d --- /dev/null +++ b/tests/integration/helpers/iceberg_timestamp_ns.py @@ -0,0 +1,444 @@ +"""Helpers to turn Iceberg `long` nanosecond columns into `timestamp_ns`. + +Spark 3.5 + Iceberg 1.8.1 and pyiceberg 0.11 cannot write Iceberg v3 +`timestamp_ns`. Tests write epoch nanoseconds as `long`, then call +`patch_iceberg_table_long_to_timestamp_ns` before ClickHouse reads the table. +""" + +import json +import struct +from pathlib import Path + +import avro.datafile +import avro.io +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +# Epoch nanoseconds and the matching UTC wall time. Values differ in the last 3 +# digits so they collapse together if the reader/writer silently truncates to +# microseconds. +ROWS = [ + (1704067200123456789, 1, "2024-01-01 00:00:00.123456789"), + (1710504000000000001, 2, "2024-03-15 12:00:00.000000001"), + (1719791999999999999, 3, "2024-06-30 23:59:59.999999999"), + (1767205845111111111, 4, "2025-12-31 18:30:45.111111111"), +] + +NS_COLUMN_NAMES = ("ts", "value") + +PRUNE_OFF = { + "use_iceberg_partition_pruning": 0, + "input_format_parquet_bloom_filter_push_down": 0, + "input_format_parquet_filter_push_down": 0, +} +PRUNE_ON = { + "use_iceberg_partition_pruning": 1, + "input_format_parquet_bloom_filter_push_down": 0, + "input_format_parquet_filter_push_down": 0, +} + + +def expected_all_ids_and_nanos(): + return "".join(f"{row_id}\t{ts_ns}\t{ts_ns}\n" for ts_ns, row_id, _utc in ROWS) + + +def select_ids_and_nanos(table): + return ( + f"SELECT id, toUnixTimestamp64Nano(ts), toUnixTimestamp64Nano(value) " + f"FROM {table} ORDER BY id" + ) + + +def from_unix_timestamp64_nano(ns): + return f"fromUnixTimestamp64Nano({ns})" + + +def to_datetime64_utc(ns): + for ts_ns, _row_id, utc in ROWS: + if ts_ns == ns: + return datetime64_literal(utc, scale=9, timezone="UTC") + raise KeyError(ns) + + +def datetime64_literal(utc, scale, timezone="UTC"): + whole, frac = utc.split(".") + frac = frac[:scale].ljust(scale, "0") + value = f"{whole}.{frac}" if scale else whole + if timezone is None: + return f"toDateTime64('{value}', {scale})" + return f"toDateTime64('{value}', {scale}, '{timezone}')" + + +# Both constructors must keep nanosecond precision in the pushed-down predicate. +FILTER_LITERALS = [ + from_unix_timestamp64_nano, + to_datetime64_utc, +] +FILTER_LITERAL_IDS = ["fromUnixTimestamp64Nano", "toDateTime64"] + + +def check_ns_pruning(node, table, column, pruned_files, make_literal): + """Equality / range pruning for a timestamp_ns column, using one SQL literal style.""" + assert node.query(select_ids_and_nanos(table), settings=PRUNE_OFF) == expected_all_ids_and_nanos() + + eq_ns, eq_id, _eq_utc = ROWS[1] + eq_lit = make_literal(eq_ns) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} = {eq_lit} ORDER BY id") + == 3 + ) + assert ( + node.query( + f"SELECT id FROM {table} WHERE {column} = {eq_lit} ORDER BY id", + settings=PRUNE_ON, + ) + == f"{eq_id}\n" + ) + + le_ns = ROWS[1][0] + le_lit = make_literal(le_ns) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} <= {le_lit} ORDER BY id") + == 2 + ) + assert node.query( + f"SELECT id FROM {table} WHERE {column} <= {le_lit} ORDER BY id", + settings=PRUNE_ON, + ) == "1\n2\n" + + ge_ns = ROWS[2][0] + ge_lit = make_literal(ge_ns) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} >= {ge_lit} ORDER BY id") + == 2 + ) + assert node.query( + f"SELECT id FROM {table} WHERE {column} >= {ge_lit} ORDER BY id", + settings=PRUNE_ON, + ) == "3\n4\n" + + assert pruned_files(select_ids_and_nanos(table)) == 0 + + +def check_ns_pruning_datetime64_microseconds(node, table, column, pruned_files, timezone="UTC"): + """Min/max (or partition) pruning with a DateTime64(6) predicate on timestamp_ns. + + Same SQL shape as the customer query (`WHERE col <= toDateTime64(..., 6)`), but + against a well-typed Iceberg `timestamp_ns` column (`DateTime64(9)`). KeyCondition + already rescales DateTime64(6) vs DateTime64(9) bounds, so this currently passes. + """ + assert node.query(select_ids_and_nanos(table), settings=PRUNE_OFF) == expected_all_ids_and_nanos() + + # Truncating row 2 to microseconds is 1ns before the stored value, so only row 1 matches. + le_before_row2 = datetime64_literal(ROWS[1][2], scale=6, timezone=timezone) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} <= {le_before_row2} ORDER BY id") + == 3 + ) + assert node.query( + f"SELECT id FROM {table} WHERE {column} <= {le_before_row2} ORDER BY id", + settings=PRUNE_ON, + ) == "1\n" + + # One microsecond after row 2 includes rows 1 and 2. Same shape as the customer query. + le_after_row2 = datetime64_literal("2024-03-15 12:00:00.000001000", scale=6, timezone=timezone) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} <= {le_after_row2} ORDER BY id") + == 2 + ) + assert node.query( + f"SELECT id FROM {table} WHERE {column} <= {le_after_row2} ORDER BY id", + settings=PRUNE_ON, + ) == "1\n2\n" + + ge_row3 = datetime64_literal(ROWS[2][2], scale=6, timezone=timezone) + assert ( + pruned_files(f"SELECT id FROM {table} WHERE {column} >= {ge_row3} ORDER BY id") + == 2 + ) + assert node.query( + f"SELECT id FROM {table} WHERE {column} >= {ge_row3} ORDER BY id", + settings=PRUNE_ON, + ) == "3\n4\n" + + +def check_minmax_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): + """Customer min/max over-prune: Iceberg `timestamp` with nanosecond min/max bytes. + + OTEL `time_unix_nano` tables are often declared as Iceberg `timestamp` + (`DateTime64(6)`) while 8-byte lower/upper bounds stay in nanoseconds. + ClickHouse then deserializes those ns bytes as microseconds (year 2299), + so `WHERE col <= toDateTime64('2026-...', 6)` prunes every file while the + parquet scan (microseconds) with `use_iceberg_partition_pruning = 0` is correct. + """ + type_name = node.query( + f"SELECT toTypeName({column}) FROM {table} LIMIT 1", + settings=PRUNE_OFF, + ) + assert "DateTime64(6" in type_name, type_name + assert "DateTime64(9" not in type_name, type_name + + ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) + assert ids == "1\n2\n3\n4\n" + # Parquet is timestamp[us] (correct 2024 wall times). Iceberg lower/upper bounds + # are still the original nanosecond int64 bytes, so DateTime64(6) min/max + # interprets them as year 2299 and over-prunes. + micros = node.query( + f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", + settings=PRUNE_OFF, + ).strip() + assert micros == "1704067200123456", micros + + # Same predicate shape as the customer query: scale-6 literal near a file's + # nanosecond upper bound. Rows 1 and 2 must survive; pruning must not drop them. + le_after_row2 = datetime64_literal( + "2024-03-15 12:00:00.000001000", scale=6, timezone=None + ) + query = f"SELECT id FROM {table} WHERE {column} <= {le_after_row2} ORDER BY id" + # One data file covering all four timestamps; the predicate overlaps it, so + # min/max pruning must keep the file. Over-pruning currently returns 0 rows. + assert pruned_files(query) == 0 + assert node.query(query, settings=PRUNE_ON) == "1\n2\n" + + count_query = f"SELECT count() FROM {table} WHERE {column} <= {le_after_row2}" + assert pruned_files(count_query) == 0 + assert node.query(count_query, settings=PRUNE_ON) == "2\n" + + +def _scale_bound_bytes(raw, factor=1000): + if not isinstance(raw, (bytes, bytearray)) or len(raw) != 8: + return raw + value = struct.unpack(" 0, f"no timestamp min/max bounds scaled under {table_location}" + + +def upgrade_long_to_iceberg_timestamp(obj, iceberg_type="timestamp_ns"): + if isinstance(obj, dict): + if obj.get("name") in NS_COLUMN_NAMES and obj.get("type") == "long": + obj["type"] = iceberg_type + for value in obj.values(): + upgrade_long_to_iceberg_timestamp(value, iceberg_type) + elif isinstance(obj, list): + for value in obj: + upgrade_long_to_iceberg_timestamp(value, iceberg_type) + + +def upgrade_long_to_timestamp_ns(obj): + upgrade_long_to_iceberg_timestamp(obj, "timestamp_ns") + + +def _deep_replace(obj, old, new): + if isinstance(obj, str): + return obj.replace(old, new) + if isinstance(obj, dict): + return {k: _deep_replace(v, old, new) for k, v in obj.items()} + if isinstance(obj, list): + return [_deep_replace(v, old, new) for v in obj] + return obj + + +def rewrite_parquet_int64_to_timestamp(path: Path, unit="ns"): + # Read the file itself; pq.read_table() infers Hive partitions from the + # parent directory name (`ts=/`) and then fails to merge that + # dictionary field with the int64 column in the file. + table = pq.ParquetFile(path).read() + columns = {} + scale = {"ns": 1, "us": 1_000, "ms": 1_000_000, "s": 1_000_000_000}[unit] + for name in table.column_names: + column = table[name] + if name in NS_COLUMN_NAMES and pa.types.is_int64(column.type): + if scale == 1: + columns[name] = column.cast(pa.timestamp("ns")) + else: + columns[name] = pc.divide(column, scale).cast(pa.timestamp(unit)) + else: + columns[name] = column + pq.write_table(pa.table(columns), path) + rewritten = pq.ParquetFile(path).schema_arrow + for name in NS_COLUMN_NAMES: + if name in rewritten.names: + field_type = rewritten.field(name).type + assert pa.types.is_timestamp(field_type), (path, name, field_type) + assert field_type.unit == unit, (path, name, field_type) + + +def rewrite_parquet_int64_to_timestamp_ns(path: Path): + rewrite_parquet_int64_to_timestamp(path, unit="ns") + + +def _meta_text(value): + if isinstance(value, (bytes, bytearray)): + return value.decode("utf-8") + return value + + +def rewrite_avro_long_to_timestamp_ns( + src: Path, dst: Path, old=None, new=None, iceberg_type="timestamp_ns" +): + with open(src, "rb") as handle: + reader = avro.datafile.DataFileReader(handle, avro.io.DatumReader()) + schema = reader.datum_reader.writers_schema + codec = reader.codec + meta = dict(reader.meta) + if old is not None and new is not None: + records = [_deep_replace(record, old, new) for record in reader] + else: + records = list(reader) + reader.close() + + with open(dst, "wb") as handle: + writer = avro.datafile.DataFileWriter(handle, avro.io.DatumWriter(), schema, codec=codec) + for key, value in meta.items(): + key_str = key.decode("utf-8") if isinstance(key, (bytes, bytearray)) else key + if key_str.startswith("avro."): + continue + text = _meta_text(value) + if key_str in ("schema", "iceberg.schema"): + iceberg_schema = json.loads(text) + upgrade_long_to_iceberg_timestamp(iceberg_schema, iceberg_type) + text = json.dumps(iceberg_schema, separators=(",", ":")) + elif old is not None and new is not None: + text = text.replace(old, new) + writer.set_meta(key_str, text.encode("utf-8")) + for record in records: + writer.append(record) + writer.close() + + +def patch_iceberg_table_long_to_timestamp_ns( + table_location: Path, + old_prefix=None, + new_prefix=None, + iceberg_type="timestamp_ns", + parquet_unit="ns", +): + """Rewrite Parquet timestamps and Iceberg metadata to `iceberg_type`. + + Manifest lower/upper bounds stay the original 8-byte little-endian nanosecond + int64 values. For the customer min/max over-prune, use Iceberg `timestamp` + (`DateTime64(6)`) with `parquet_unit='us'` so the scan is correct while + bounds are still nanoseconds (year 2299 when decoded as microseconds). + """ + for parquet_file in table_location.rglob("*.parquet"): + rewrite_parquet_int64_to_timestamp(parquet_file, unit=parquet_unit) + + meta_dir = table_location / "metadata" + for metadata_file in meta_dir.glob("*.metadata.json"): + text = metadata_file.read_text() + if old_prefix is not None and new_prefix is not None: + text = text.replace(old_prefix, new_prefix) + meta = json.loads(text) + upgrade_long_to_iceberg_timestamp(meta, iceberg_type) + metadata_file.write_text(json.dumps(meta, separators=(",", ":"))) + + for avro_file in list(meta_dir.glob("*.avro")): + tmp = avro_file.with_suffix(".avro.tmp") + rewrite_avro_long_to_timestamp_ns( + avro_file, tmp, old_prefix, new_prefix, iceberg_type=iceberg_type + ) + tmp.replace(avro_file) diff --git a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py new file mode 100644 index 0000000000000..e703492ec35fd --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Iceberg identity-partition and min/max pruning for timestamp_ns (DateTime64(9)). + +pyiceberg 0.11 cannot write Iceberg v3 (`timestamp_ns`), and the REST catalog +image used here also rejects that type. The table is therefore written as +Iceberg v2 with `long` columns holding epoch nanoseconds, then patched to +`timestamp_ns` and Parquet `timestamp[ns]` before ClickHouse reads it. +""" + +import shutil +import tempfile +import uuid +from pathlib import Path + +import pyarrow as pa +import pytest +from pyiceberg.catalog.sql import SqlCatalog +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import NestedField, Schema +from pyiceberg.transforms import IdentityTransform +from pyiceberg.types import IntegerType, LongType, TimestampType + +from helpers.iceberg_timestamp_ns import ( + FILTER_LITERAL_IDS, + FILTER_LITERALS, + PRUNE_OFF, + PRUNE_ON, + ROWS, + check_minmax_ns_bounds_on_timestamp_schema, + check_ns_pruning, + check_ns_pruning_datetime64_microseconds, + patch_iceberg_table_long_to_timestamp_ns, + patch_iceberg_timestamp_bounds_us_to_ns, +) +from helpers.iceberg_utils import check_validity_and_get_prunned_files_general + + +def _ns_schema(): + return Schema( + NestedField(field_id=1, name="ts", field_type=LongType(), required=False), + NestedField(field_id=2, name="value", field_type=LongType(), required=False), + NestedField(field_id=3, name="id", field_type=IntegerType(), required=False), + ) + + +def _arrow_row(ts_ns, row_id): + return pa.Table.from_pydict( + { + "ts": pa.array([ts_ns], type=pa.int64()), + "value": pa.array([ts_ns], type=pa.int64()), + "id": pa.array([row_id], type=pa.int32()), + } + ) + + +def _timestamp_schema(): + return Schema( + NestedField(field_id=1, name="ts", field_type=TimestampType(), required=False), + NestedField(field_id=2, name="value", field_type=TimestampType(), required=False), + NestedField(field_id=3, name="id", field_type=IntegerType(), required=False), + ) + + +def _arrow_timestamp_row(ts_ns, row_id): + ts_us = pa.array([ts_ns // 1000], type=pa.timestamp("us")) + return pa.Table.from_pydict( + { + "ts": ts_us, + "value": ts_us, + "id": pa.array([row_id], type=pa.int32()), + } + ) + + +def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): + """Iceberg v2 `timestamp` (microseconds) whose manifest min/max bytes are nanoseconds.""" + work = Path(tempfile.mkdtemp(prefix="iceberg_ns_bounds_")) + try: + catalog = SqlCatalog( + "ns_bounds", + **{ + "uri": f"sqlite:///{work}/catalog.db", + "warehouse": f"file://{work}/warehouse", + }, + ) + catalog.create_namespace("default") + identifier = f"default.{table_name}" + table = catalog.create_table( + identifier=identifier, + schema=_timestamp_schema(), + partition_spec=PartitionSpec(), + ) + table.append( + pa.concat_tables( + [_arrow_timestamp_row(ts_ns, row_id) for ts_ns, row_id, _utc in ROWS] + ) + ) + table = catalog.load_table(identifier) + + old_prefix = table.location() + table_location = Path(old_prefix.removeprefix("file://")) + s3_key = f"iceberg_ns_bounds_{uuid.uuid4().hex}" + new_prefix = f"s3://{started_cluster.minio_bucket}/{s3_key}" + + patch_iceberg_timestamp_bounds_us_to_ns(table_location, old_prefix, new_prefix) + + uploader = started_cluster.default_s3_uploader + for path in table_location.rglob("*"): + if path.is_file(): + rel = path.relative_to(table_location).as_posix() + uploader.upload_file(str(path), f"{s3_key}/{rel}") + + return f"{s3_key}/" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def _create_v3_table( + started_cluster, table_name, partition_spec, iceberg_type="timestamp_ns", parquet_unit="ns" +): + work = Path(tempfile.mkdtemp(prefix="iceberg_ns_pruning_")) + try: + catalog = SqlCatalog( + "ns_pruning", + **{ + "uri": f"sqlite:///{work}/catalog.db", + "warehouse": f"file://{work}/warehouse", + }, + ) + catalog.create_namespace("default") + identifier = f"default.{table_name}" + table = catalog.create_table( + identifier=identifier, + schema=_ns_schema(), + partition_spec=partition_spec, + ) + for ts_ns, row_id, _utc in ROWS: + table.append(_arrow_row(ts_ns, row_id)) + table = catalog.load_table(identifier) + + old_prefix = table.location() + table_location = Path(old_prefix.removeprefix("file://")) + s3_key = f"iceberg_ns_pruning_{uuid.uuid4().hex}" + new_prefix = f"s3://{started_cluster.minio_bucket}/{s3_key}" + + patch_iceberg_table_long_to_timestamp_ns( + table_location, + old_prefix, + new_prefix, + iceberg_type=iceberg_type, + parquet_unit=parquet_unit, + ) + + uploader = started_cluster.default_s3_uploader + for path in table_location.rglob("*"): + if path.is_file(): + rel = path.relative_to(table_location).as_posix() + uploader.upload_file(str(path), f"{s3_key}/{rel}") + + return f"{s3_key}/" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def _iceberg_table(s3_filename): + return ( + f"icebergS3(s3, filename = '{s3_filename}', format=Parquet, " + f"url = 'http://minio1:9001/root/')" + ) + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_partition_pruning_datetime64_nanoseconds(started_cluster_iceberg_no_spark, make_literal): + """Identity partition pruning on Iceberg timestamp_ns must not drop matching rows.""" + node = started_cluster_iceberg_no_spark.instances["node1"] + partition_spec = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="ts") + ) + s3_filename = _create_v3_table(started_cluster_iceberg_no_spark, "part_ns", partition_spec) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, "part_ns", PRUNE_OFF, PRUNE_ON, "IcebergPartitionPrunedFiles", select_expression + ) + + check_ns_pruning(node, table, "ts", pruned_files, make_literal) + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_minmax_pruning_datetime64_nanoseconds(started_cluster_iceberg_no_spark, make_literal): + """Min/max file pruning on Iceberg timestamp_ns must not drop matching rows.""" + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_v3_table(started_cluster_iceberg_no_spark, "minmax_ns", PartitionSpec()) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, "minmax_ns", PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_ns_pruning(node, table, "value", pruned_files, make_literal) + + +@pytest.mark.parametrize("timezone", [None, "UTC"], ids=["toDateTime64_6", "toDateTime64_6_UTC"]) +def test_minmax_pruning_datetime64_nanoseconds_microsecond_predicate( + started_cluster_iceberg_no_spark, timezone +): + """DateTime64(6) predicate vs timestamp_ns min/max (customer over-prune).""" + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_v3_table(started_cluster_iceberg_no_spark, "minmax_ns_us", PartitionSpec()) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, "minmax_ns_us", PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_ns_pruning_datetime64_microseconds(node, table, "value", pruned_files, timezone=timezone) + + +def test_minmax_pruning_ns_bounds_on_timestamp_schema(started_cluster_iceberg_no_spark): + """Iceberg `timestamp` (`DateTime64(6)`) with nanosecond min/max bytes must not over-prune. + + Customer: `WHERE time_unix_nano <= toDateTime64('2026-08-11 20:02:07.897356', 6)` + returns 0 with min/max pruning and the correct row count without it. + """ + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_timestamp_table_with_ns_bounds( + started_cluster_iceberg_no_spark, "minmax_ns_on_ts" + ) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, + "minmax_ns_on_ts", + PRUNE_OFF, + PRUNE_ON, + "IcebergMinMaxIndexPrunedFiles", + select_expression, + ) + + check_minmax_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_with_spark/test_pruning_datetime64_nanoseconds.py new file mode 100644 index 0000000000000..229d0b4b239b8 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_pruning_datetime64_nanoseconds.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Spark-written Iceberg identity-partition and min/max pruning for timestamp_ns. + +Spark 3.5 + Iceberg 1.8.1 (the integration-test runtime) cannot create or write +Iceberg `timestamp_ns`: Spark TIMESTAMP is microsecond-only, and Iceberg Spark +maps it to `timestamp` / `timestamptz`. This test therefore writes epoch +nanoseconds as BIGINT via Spark, patches the table to `timestamp_ns`, then +checks ClickHouse pruning. +""" + +from pathlib import Path + +import pytest + +from helpers.iceberg_timestamp_ns import ( + FILTER_LITERAL_IDS, + FILTER_LITERALS, + PRUNE_OFF, + PRUNE_ON, + ROWS, + check_minmax_ns_bounds_on_timestamp_schema, + check_ns_pruning, + check_ns_pruning_datetime64_microseconds, + patch_iceberg_table_long_to_timestamp_ns, + patch_iceberg_timestamp_bounds_us_to_ns, +) +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + default_upload_directory, + get_creation_expression, + get_uuid_str, +) + +WAREHOUSE_PREFIX = "/var/lib/clickhouse/user_files/iceberg_data/default" + + +def _create_spark_ns_table( + spark, + started_cluster, + table_name, + partitioned, + iceberg_type="timestamp_ns", + parquet_unit="ns", +): + partition_clause = "PARTITIONED BY (identity(ts))" if partitioned else "" + spark.sql( + f""" + CREATE TABLE {table_name} ( + ts BIGINT, + value BIGINT, + id INT + ) + USING iceberg + {partition_clause} + OPTIONS('format-version'='2') + """ + ) + for ts_ns, row_id, _utc in ROWS: + spark.sql( + f"INSERT INTO {table_name} VALUES ({ts_ns}, {ts_ns}, {row_id})" + ) + + table_location = Path(f"{WAREHOUSE_PREFIX}/{table_name}") + patch_iceberg_table_long_to_timestamp_ns( + table_location, iceberg_type=iceberg_type, parquet_unit=parquet_unit + ) + default_upload_directory( + started_cluster, + "s3", + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +def _create_spark_timestamp_table_with_ns_bounds(spark, started_cluster, table_name): + """Spark Iceberg `TIMESTAMP` (microseconds) whose manifest min/max bytes are nanoseconds.""" + spark.sql( + f""" + CREATE TABLE {table_name} ( + ts TIMESTAMP, + value TIMESTAMP, + id INT + ) + USING iceberg + OPTIONS('format-version'='2') + """ + ) + values = ", ".join( + f"(TIMESTAMP '{utc.split('.')[0]}.{utc.split('.')[1][:6]}', " + f"TIMESTAMP '{utc.split('.')[0]}.{utc.split('.')[1][:6]}', {row_id})" + for _ts_ns, row_id, utc in ROWS + ) + spark.sql(f"INSERT INTO {table_name} VALUES {values}") + + table_location = Path(f"{WAREHOUSE_PREFIX}/{table_name}") + patch_iceberg_timestamp_bounds_us_to_ns(table_location) + default_upload_directory( + started_cluster, + "s3", + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_spark_partition_pruning_datetime64_nanoseconds(started_cluster_iceberg_with_spark, make_literal): + """Identity partition pruning on Spark-written Iceberg timestamp_ns must not drop matching rows.""" + node = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_spark_part_ns_" + get_uuid_str() + _create_spark_ns_table(spark, started_cluster_iceberg_with_spark, table_name, partitioned=True) + table = get_creation_expression( + "s3", table_name, started_cluster_iceberg_with_spark, table_function=True + ) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, table_name, PRUNE_OFF, PRUNE_ON, "IcebergPartitionPrunedFiles", select_expression + ) + + check_ns_pruning(node, table, "ts", pruned_files, make_literal) + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_spark_minmax_pruning_datetime64_nanoseconds(started_cluster_iceberg_with_spark, make_literal): + """Min/max file pruning on Spark-written Iceberg timestamp_ns must not drop matching rows.""" + node = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_spark_minmax_ns_" + get_uuid_str() + _create_spark_ns_table(spark, started_cluster_iceberg_with_spark, table_name, partitioned=False) + table = get_creation_expression( + "s3", table_name, started_cluster_iceberg_with_spark, table_function=True + ) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, table_name, PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_ns_pruning(node, table, "value", pruned_files, make_literal) + + +@pytest.mark.parametrize("timezone", [None, "UTC"], ids=["toDateTime64_6", "toDateTime64_6_UTC"]) +def test_spark_minmax_pruning_datetime64_nanoseconds_microsecond_predicate( + started_cluster_iceberg_with_spark, timezone +): + """DateTime64(6) predicate vs Spark-written timestamp_ns min/max (customer over-prune).""" + node = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_spark_minmax_ns_us_" + get_uuid_str() + _create_spark_ns_table(spark, started_cluster_iceberg_with_spark, table_name, partitioned=False) + table = get_creation_expression( + "s3", table_name, started_cluster_iceberg_with_spark, table_function=True + ) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, table_name, PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_ns_pruning_datetime64_microseconds(node, table, "value", pruned_files, timezone=timezone) + + +def test_spark_minmax_pruning_ns_bounds_on_timestamp_schema(started_cluster_iceberg_with_spark): + """Nanosecond parquet/bounds with Iceberg `timestamp` (DateTime64(6)) must not over-prune.""" + node = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_spark_minmax_ns_on_ts_" + get_uuid_str() + _create_spark_timestamp_table_with_ns_bounds( + spark, started_cluster_iceberg_with_spark, table_name + ) + table = get_creation_expression( + "s3", table_name, started_cluster_iceberg_with_spark, table_function=True + ) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, table_name, PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_minmax_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) From 93583fc014c059b8ccb655cb70cff0a258fbd572 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 18 Aug 2026 21:48:30 +0200 Subject: [PATCH 02/22] Fix Iceberg over-pruning of nanosecond timestamps Identity partition values for `timestamp_ns` arrive as Avro Int64, and some writers store nanosecond min/max bytes on Iceberg `timestamp` (`DateTime64(6)`). Both were compared at the wrong scale and dropped matching files. Write `DateTime64(9)` as `timestamp_ns` so ClickHouse does not recreate that layout. Co-authored-by: Cursor --- .../table-engines/integrations/iceberg.mdx | 2 + .../Iceberg/IcebergFieldParseHelpers.cpp | 64 +++++++++++ .../DataLakes/Iceberg/IcebergWrites.cpp | 22 +++- .../Iceberg/ManifestFilesPruning.cpp | 36 +++++- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 18 ++- .../test.py | 2 +- .../test_writes_datetime64_nanoseconds.py | 108 ++++++++++++++++++ 7 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index 7a05dab165b5d..b3e359177334a 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -98,6 +98,8 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `fixed(N)` | `FixedString(N)` | | | `decimal(P, S)` | `Decimal(P, S)` | | +Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and `DateTime64` with an explicit timezone and scale 9 as `timestamptz_ns`. `DateTime` and `DateTime64` with scale 6 or less are stored as `timestamp`. + ### Complex types {#complex-types} | Iceberg type | ClickHouse type | diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 89e8a513560ba..34fb554631f3b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,66 @@ extern const int BAD_ARGUMENTS; namespace Iceberg { +namespace +{ + +/// ~year 2286 as DateTime64(6) microseconds. Spec-correct Iceberg `timestamp` bounds +/// for realistic dates stay below this; nanosecond ticks for dates after ~1970-04 +/// sit well above it (2024 is ~1.7e18). +constexpr Int64 microseconds_implausible_threshold = 10'000'000'000'000'000LL; +constexpr Int64 nanoseconds_per_microsecond = 1000; + +bool magnitudeExceeds(Int64 value, Int64 threshold) +{ + return value > threshold || value < -threshold; +} + +/// Convert ns ticks to us ticks without shrinking a [lower, upper] range. +Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) +{ + Int64 microseconds = nanoseconds / nanoseconds_per_microsecond; + if (nanoseconds % nanoseconds_per_microsecond == 0) + return microseconds; + + /// C++ integer division truncates toward zero. + if (lower_bound) + { + if (nanoseconds < 0) + --microseconds; + } + else if (nanoseconds > 0) + { + ++microseconds; + } + return microseconds; +} + +/// Iceberg timestamps are 8-byte little-endian (spec Appendix D). `insertData` +/// memcpy's native DateTime64 ticks, which is correct when the bound scale +/// matches the column. Some writers store nanosecond stats on Iceberg +/// `timestamp` (`DateTime64(6)`); treating those as microseconds yields dates +/// around year 2299 and over-prunes. Rescale those bounds; if they are still +/// implausible, fail open (`nullopt`) so min/max pruning is skipped. +std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) +{ + if (str.size() != sizeof(Int64)) + return std::nullopt; + + Int64 unscaled = unalignedLoadLittleEndian(str.data()); + const UInt32 scale = getDecimalScale(type); + + if (scale == 6 && magnitudeExceeds(unscaled, microseconds_implausible_threshold)) + { + unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); + if (magnitudeExceeds(unscaled, microseconds_implausible_threshold)) + return std::nullopt; + } + + return DecimalField(DateTime64(unscaled), scale); +} + +} + Int64 fieldToInt64(const Field & value, std::string_view context, std::string_view arg_name) { if (value.getType() == Field::Types::Int64) @@ -200,6 +261,9 @@ std::optional deserializeFieldFromBinaryRepr(const std::string & str, DataTypePtr expected_type, bool lower_bound, bool compensate_rounding) { auto non_nullable_type = removeNullable(expected_type); + if (WhichDataType(non_nullable_type).isDateTime64()) + return deserializeDateTime64FromBinaryRepr(str, *non_nullable_type, lower_bound); + auto column = non_nullable_type->createColumn(); if (WhichDataType(non_nullable_type).isDecimal()) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index f279cfa20528e..d96312030178e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -291,6 +292,25 @@ Int64 getTimeValueInMicroseconds(const Field & field, DataTypePtr type) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected Time or Time64, got {}", type->getName()); } +Int64 getDateTime64ValueForIcebergBounds(const Field & field, DataTypePtr type) +{ + if (type->isNullable()) + return getDateTime64ValueForIcebergBounds(field, assert_cast(type.get())->getNestedType()); + + if (!WhichDataType(type).isDateTime64()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected DateTime64, got {}", type->getName()); + + const auto scale = getDecimalScale(*type); + const auto value = field.safeGet().getValue().value; + if (scale == 9) + return value; + if (scale > 6) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + + /// Iceberg `timestamp` bounds are microseconds. DateTime64(s) for s < 6 stores coarser ticks. + return value * DataTypeDateTime64::getScaleMultiplier(6 - scale).value; +} + std::vector dumpFieldToBytes(const Field & field, DataTypePtr type) { switch (type->getTypeId()) @@ -326,7 +346,7 @@ std::vector dumpFieldToBytes(const Field & field, DataTypePtr type) case TypeIndex::Time64: return dumpValue(getTimeValueInMicroseconds(field, type)); case TypeIndex::DateTime64: - return dumpValue(field.safeGet().getValue().value); + return dumpValue(getDateTime64ValueForIcebergBounds(field, type)); case TypeIndex::String: { auto value = field.safeGet(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index 12ea9d941c91b..3a960114fdaad 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -34,6 +34,37 @@ namespace DB::ErrorCodes extern const int LOGICAL_ERROR; } +namespace +{ + +/// Avro does not map timestamp-nanos to DateTime64, so identity partition values for +/// Iceberg `timestamp_ns` / `timestamptz_ns` (and similarly `Time64`) arrive as Int64. +/// `KeyCondition` compares DateTime64/Time64 predicates as DecimalField with a scale, +/// so wrap the raw ticks before `mayBeTrueInRange`. Calendar transforms stay Int64. +void wrapIntegerPartitionValuesAsDateTime64(Row & partition_values, const DataTypes & data_types) +{ + const size_t size = std::min(partition_values.size(), data_types.size()); + for (size_t i = 0; i < size; ++i) + { + auto & field = partition_values[i]; + if (field.isNull() || field.getType() != Field::Types::Int64) + continue; + + auto type = removeNullable(data_types[i]); + WhichDataType which(type); + if (which.isDateTime64()) + { + field = DecimalField(DateTime64(field.safeGet()), getDecimalScale(*type)); + } + else if (which.isTime64()) + { + field = DecimalField(Time64(field.safeGet()), getDecimalScale(*type)); + } + } +} + +} + namespace DB::Iceberg { @@ -377,7 +408,10 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( if (partition_key_condition.has_value()) { - std::vector index_value(partition_value.begin(), partition_value.end()); + Row wrapped_partition_value = partition_value; + wrapIntegerPartitionValuesAsDateTime64(wrapped_partition_value, partition_key->data_types); + + std::vector index_value(wrapped_partition_value.begin(), wrapped_partition_value.end()); for (size_t i = 0; i < index_value.size(); ++i) { auto & field = index_value[i]; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 1512d060a03c9..3dcff43e0b030 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -653,8 +653,24 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite case TypeIndex::Date32: return {"date", true}; case TypeIndex::DateTime: - case TypeIndex::DateTime64: return {"timestamp", true}; + case TypeIndex::DateTime64: + { + /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds (v3). + /// Mapping all DateTime64 to `timestamp` while dumping unscaled ticks made + /// ClickHouse-written DateTime64(9) recreate ns-bounds-on-us-schema over-pruning. + const auto scale = getDecimalScale(*type); + if (scale == 9) + { + auto date_time64 = std::static_pointer_cast(type); + if (date_time64->hasExplicitTimeZone()) + return {Iceberg::f_timestamptz_ns, true}; + return {Iceberg::f_timestamp_ns, true}; + } + if (scale <= 6) + return {Iceberg::f_timestamp, true}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + } case TypeIndex::Time: return {"time", true}; case TypeIndex::Time64: diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index f3c73bb21b81a..64c6aa7dad341 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -219,7 +219,7 @@ def test_export_part_all_iceberg_types(cluster): float f32 Float32 double f64 Float64 date event_dt Date - timestamp ts DateTime64(6) (DateTime / DateTime64 both → "timestamp") + timestamp ts DateTime64(6) (DateTime / DateTime64(scale<=6) → "timestamp"; DateTime64(9) → "timestamp_ns") string name String uuid uid_val UUID diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py new file mode 100644 index 0000000000000..8790d2b4af87c --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""ClickHouse Iceberg writes of DateTime64(9) must use timestamp_ns, not timestamp. + +Mapping DateTime64(9) to Iceberg `timestamp` while dumping unscaled nanosecond +min/max ticks recreates the customer over-prune (schema microseconds, stats +nanoseconds). +""" + +import pytest + +from helpers.iceberg_timestamp_ns import ( + FILTER_LITERAL_IDS, + FILTER_LITERALS, + PRUNE_OFF, + PRUNE_ON, + ROWS, + check_ns_pruning, +) +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + create_iceberg_table, + get_uuid_str, +) + + +def _insert_ns_rows(instance, table_name): + for ts_ns, row_id, _utc in ROWS: + instance.query( + f"INSERT INTO {table_name} VALUES " + f"(fromUnixTimestamp64Nano({ts_ns}), fromUnixTimestamp64Nano({ts_ns}), {row_id})" + ) + + +def _assert_written_as_datetime64_9(instance, table_name, timezone=None): + describe = instance.query(f"DESCRIBE TABLE {table_name}") + assert "DateTime64(9" in describe, describe + assert "DateTime64(6" not in describe, describe + if timezone == "UTC": + assert "UTC" in describe, describe + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_writes_minmax_pruning_datetime64_nanoseconds( + started_cluster_iceberg_no_spark, make_literal +): + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_minmax_ns_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + "(ts DateTime64(9), value DateTime64(9), id Int32)", + format_version=3, + ) + _assert_written_as_datetime64_9(instance, table_name) + _insert_ns_rows(instance, table_name) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + instance, table_name, PRUNE_OFF, PRUNE_ON, "IcebergMinMaxIndexPrunedFiles", select_expression + ) + + check_ns_pruning(instance, table_name, "value", pruned_files, make_literal) + + +@pytest.mark.parametrize("make_literal", FILTER_LITERALS, ids=FILTER_LITERAL_IDS) +def test_writes_partition_pruning_datetime64_nanoseconds( + started_cluster_iceberg_no_spark, make_literal +): + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_part_ns_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + "(ts DateTime64(9), value DateTime64(9), id Int32)", + format_version=3, + partition_by="ts", + ) + _assert_written_as_datetime64_9(instance, table_name) + _insert_ns_rows(instance, table_name) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + instance, table_name, PRUNE_OFF, PRUNE_ON, "IcebergPartitionPrunedFiles", select_expression + ) + + check_ns_pruning(instance, table_name, "ts", pruned_files, make_literal) + + +def test_writes_datetime64_nanoseconds_with_timezone(started_cluster_iceberg_no_spark): + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_ns_tz_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + "(ts DateTime64(9, 'UTC'), id Int32)", + format_version=3, + ) + _assert_written_as_datetime64_9(instance, table_name, timezone="UTC") + instance.query( + f"INSERT INTO {table_name} VALUES (fromUnixTimestamp64Nano({ROWS[0][0]}), 1)" + ) + assert instance.query(f"SELECT id FROM {table_name}") == "1\n" From 192d658f608657345dc2594f51e6a6a3bcca44dc Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 18 Aug 2026 22:37:35 +0200 Subject: [PATCH 03/22] Do not treat far-future Iceberg timestamp bounds as nanoseconds `1e16` microseconds is ~year 2286, so spec-correct Spark `9999-12-31` and `DateTime64` max `2299-12-31` bounds were divided by 1000 and over-pruned. Convert only `|ticks| > 1e18` (true nanosecond band); fail open in between. Co-authored-by: Cursor --- .../Iceberg/IcebergFieldParseHelpers.cpp | 28 ++-- .../helpers/iceberg_timestamp_ns.py | 151 ++++++++++++++++++ .../test_pruning_datetime64_nanoseconds.py | 55 ++++++- 3 files changed, 218 insertions(+), 16 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 34fb554631f3b..08127a5887c8a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -28,10 +28,12 @@ namespace Iceberg namespace { -/// ~year 2286 as DateTime64(6) microseconds. Spec-correct Iceberg `timestamp` bounds -/// for realistic dates stay below this; nanosecond ticks for dates after ~1970-04 -/// sit well above it (2024 is ~1.7e18). -constexpr Int64 microseconds_implausible_threshold = 10'000'000'000'000'000LL; +/// 1e16 µs is ~year 2286. That is not an Iceberg `timestamp` max: Spark sentinels +/// such as `9999-12-31` (`~2.53e17` µs) and ClickHouse `DateTime64` max +/// `2299-12-31` (`~1.04e16` µs) are spec-correct microseconds above this. +/// 1e18 ns is ~year 2001; customer 2026 ns bounds are `~1.76e18`. +constexpr Int64 microseconds_ambiguous_threshold = 10'000'000'000'000'000LL; +constexpr Int64 nanoseconds_lower_threshold = 1'000'000'000'000'000'000LL; constexpr Int64 nanoseconds_per_microsecond = 1000; bool magnitudeExceeds(Int64 value, Int64 threshold) @@ -59,12 +61,11 @@ Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) return microseconds; } -/// Iceberg timestamps are 8-byte little-endian (spec Appendix D). `insertData` -/// memcpy's native DateTime64 ticks, which is correct when the bound scale -/// matches the column. Some writers store nanosecond stats on Iceberg -/// `timestamp` (`DateTime64(6)`); treating those as microseconds yields dates -/// around year 2299 and over-prunes. Rescale those bounds; if they are still -/// implausible, fail open (`nullopt`) so min/max pruning is skipped. +/// Iceberg timestamps are 8-byte little-endian (spec Appendix D). +/// Some writers store nanosecond stats on Iceberg `timestamp` (`DateTime64(6)`). +/// Convert only in a true nanosecond band (`|ticks| > 1e18`). Between 1e16 and +/// 1e18 the value may be far-future microseconds (do not convert; fail open so +/// min/max pruning is skipped). `|ticks| <= 1e16` is kept as microseconds. std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) { if (str.size() != sizeof(Int64)) @@ -73,11 +74,12 @@ std::optional deserializeDateTime64FromBinaryRepr(const std::string & str Int64 unscaled = unalignedLoadLittleEndian(str.data()); const UInt32 scale = getDecimalScale(type); - if (scale == 6 && magnitudeExceeds(unscaled, microseconds_implausible_threshold)) + if (scale == 6 && magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) { - unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); - if (magnitudeExceeds(unscaled, microseconds_implausible_threshold)) + if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) return std::nullopt; + + unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); } return DecimalField(DateTime64(unscaled), scale); diff --git a/tests/integration/helpers/iceberg_timestamp_ns.py b/tests/integration/helpers/iceberg_timestamp_ns.py index 37b9e365fa82d..351b3cc9eeb1c 100644 --- a/tests/integration/helpers/iceberg_timestamp_ns.py +++ b/tests/integration/helpers/iceberg_timestamp_ns.py @@ -7,6 +7,7 @@ import json import struct +from datetime import datetime, timezone from pathlib import Path import avro.datafile @@ -206,6 +207,52 @@ def check_minmax_ns_bounds_on_timestamp_schema(node, table, column, pruned_files assert node.query(count_query, settings=PRUNE_ON) == "2\n" +def _utc_datetime_to_micros(dt): + """Integer microseconds from Unix epoch. Do not use float `timestamp` * 1e6.""" + delta = dt - datetime(1970, 1, 1, tzinfo=timezone.utc) + return delta.days * 86_400_000_000 + delta.seconds * 1_000_000 + delta.microseconds + + +# Spec-correct Iceberg `timestamp` microseconds that sit in the ambiguous band +# (1e16, 1e18]: converting them as nanoseconds over-prunes. +SPARK_TIMESTAMP_SENTINEL_US = _utc_datetime_to_micros( + datetime(9999, 12, 31, 23, 59, 59, 999999, tzinfo=timezone.utc) +) +DATETIME64_MAX_US = _utc_datetime_to_micros( + datetime(2299, 12, 31, 23, 59, 59, 999999, tzinfo=timezone.utc) +) + + +def check_minmax_far_future_us_upper_bound(node, table, column, pruned_files): + """Iceberg `timestamp` with a far-future microsecond *upper* bound must not over-prune. + + Lower bound stays a real 2024 microsecond value. If the decoder treats the + upper bound as nanoseconds, max becomes ~1978, the [min, max] range inverts, + and `WHERE col <= toDateTime64('2024-03-15 ...', 6)` drops the file. + """ + type_name = node.query( + f"SELECT toTypeName({column}) FROM {table} LIMIT 1", + settings=PRUNE_OFF, + ) + assert "DateTime64(6" in type_name, type_name + assert "DateTime64(9" not in type_name, type_name + + ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) + assert ids == "1\n2\n3\n4\n" + micros = node.query( + f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", + settings=PRUNE_OFF, + ).strip() + assert micros == "1704067200123456", micros + + le_after_row2 = datetime64_literal( + "2024-03-15 12:00:00.000001000", scale=6, timezone=None + ) + query = f"SELECT id FROM {table} WHERE {column} <= {le_after_row2} ORDER BY id" + assert pruned_files(query) == 0 + assert node.query(query, settings=PRUNE_ON) == "1\n2\n" + + def _scale_bound_bytes(raw, factor=1000): if not isinstance(raw, (bytes, bytearray)) or len(raw) != 8: return raw @@ -317,6 +364,110 @@ def patch_iceberg_timestamp_bounds_us_to_ns( assert scaled_total > 0, f"no timestamp min/max bounds scaled under {table_location}" +def _set_bound_bytes(raw, value): + if not isinstance(raw, (bytes, bytearray)) or len(raw) != 8: + return raw + return struct.pack(" 0, f"no timestamp upper bounds patched under {table_location}" + + def upgrade_long_to_iceberg_timestamp(obj, iceberg_type="timestamp_ns"): if isinstance(obj, dict): if obj.get("name") in NS_COLUMN_NAMES and obj.get("type") == "long": diff --git a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py index e703492ec35fd..e30843636eec2 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py @@ -21,16 +21,20 @@ from pyiceberg.types import IntegerType, LongType, TimestampType from helpers.iceberg_timestamp_ns import ( + DATETIME64_MAX_US, FILTER_LITERAL_IDS, FILTER_LITERALS, PRUNE_OFF, PRUNE_ON, ROWS, + SPARK_TIMESTAMP_SENTINEL_US, + check_minmax_far_future_us_upper_bound, check_minmax_ns_bounds_on_timestamp_schema, check_ns_pruning, check_ns_pruning_datetime64_microseconds, patch_iceberg_table_long_to_timestamp_ns, patch_iceberg_timestamp_bounds_us_to_ns, + patch_iceberg_timestamp_upper_bounds, ) from helpers.iceberg_utils import check_validity_and_get_prunned_files_general @@ -72,8 +76,8 @@ def _arrow_timestamp_row(ts_ns, row_id): ) -def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): - """Iceberg v2 `timestamp` (microseconds) whose manifest min/max bytes are nanoseconds.""" +def _create_timestamp_table(started_cluster, table_name, patch): + """Iceberg v2 `timestamp` (microseconds parquet) with a custom manifest-bound patch.""" work = Path(tempfile.mkdtemp(prefix="iceberg_ns_bounds_")) try: catalog = SqlCatalog( @@ -102,7 +106,7 @@ def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): s3_key = f"iceberg_ns_bounds_{uuid.uuid4().hex}" new_prefix = f"s3://{started_cluster.minio_bucket}/{s3_key}" - patch_iceberg_timestamp_bounds_us_to_ns(table_location, old_prefix, new_prefix) + patch(table_location, old_prefix, new_prefix) uploader = started_cluster.default_s3_uploader for path in table_location.rglob("*"): @@ -115,6 +119,23 @@ def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): shutil.rmtree(work, ignore_errors=True) +def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): + """Iceberg v2 `timestamp` (microseconds) whose manifest min/max bytes are nanoseconds.""" + return _create_timestamp_table( + started_cluster, table_name, patch_iceberg_timestamp_bounds_us_to_ns + ) + + +def _create_timestamp_table_with_upper_bound_us(started_cluster, table_name, micros): + """Iceberg v2 `timestamp` whose upper bounds are spec-correct far-future microseconds.""" + def patch(table_location, old_prefix, new_prefix): + patch_iceberg_timestamp_upper_bounds( + table_location, micros, old_prefix, new_prefix + ) + + return _create_timestamp_table(started_cluster, table_name, patch) + + def _create_v3_table( started_cluster, table_name, partition_spec, iceberg_type="timestamp_ns", parquet_unit="ns" ): @@ -242,3 +263,31 @@ def pruned_files(select_expression): ) check_minmax_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) + + +@pytest.mark.parametrize( + "upper_bound_us", + [SPARK_TIMESTAMP_SENTINEL_US, DATETIME64_MAX_US], + ids=["spark_9999_12_31", "datetime64_2299_12_31"], +) +def test_minmax_pruning_far_future_us_upper_bound_on_timestamp_schema( + started_cluster_iceberg_no_spark, upper_bound_us +): + """Spec-correct far-future Iceberg `timestamp` upper bounds must not be treated as ns.""" + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_timestamp_table_with_upper_bound_us( + started_cluster_iceberg_no_spark, "minmax_far_future_us", upper_bound_us + ) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, + "minmax_far_future_us", + PRUNE_OFF, + PRUNE_ON, + "IcebergMinMaxIndexPrunedFiles", + select_expression, + ) + + check_minmax_far_future_us_upper_bound(node, table, "value", pruned_files) From df59db712c38942e4703f20c00b8672a9679a5ac Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 18 Aug 2026 22:49:54 +0200 Subject: [PATCH 04/22] Reject DateTime64(9) on Iceberg format versions below 3 `timestamp_ns` is v3-only. Mapping `DateTime64(9)` without checking `iceberg_format_version` wrote invalid v2 metadata (default is 2). Throw instead of emitting v3 types into v1/v2 tables. Co-authored-by: Cursor --- .../table-engines/integrations/iceberg.mdx | 2 +- .../DataLakes/Iceberg/Compaction.cpp | 4 +- .../DataLakes/Iceberg/MetadataGenerator.cpp | 9 +- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 26 +++-- .../ObjectStorage/DataLakes/Iceberg/Utils.h | 5 +- .../test_writes_datetime64_nanoseconds.py | 107 ++++++++++++++++++ 6 files changed, 136 insertions(+), 17 deletions(-) diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index b3e359177334a..9e73b43a07713 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -98,7 +98,7 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `fixed(N)` | `FixedString(N)` | | | `decimal(P, S)` | `Decimal(P, S)` | | -Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and `DateTime64` with an explicit timezone and scale 9 as `timestamptz_ns`. `DateTime` and `DateTime64` with scale 6 or less are stored as `timestamp`. +Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and `DateTime64` with an explicit timezone and scale 9 as `timestamptz_ns`, but only when `iceberg_format_version` is 3 or higher. `DateTime` and `DateTime64` with scale 6 or less are stored as `timestamp`. Creating or altering an Iceberg table with `DateTime64` scale 9 on format version 1 or 2 raises an exception. ### Complex types {#complex-types} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index d643e99c52a19..a2f275c9cd049 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -1063,7 +1063,9 @@ static void writeMetadataFiles( auto log = getLogger("IcebergCompaction"); ColumnsDescription columns_description = ColumnsDescription::fromNamesAndTypes(sample_block_->getNamesAndTypes()); - auto [metadata_object, metadata_object_str] = createEmptyMetadataFile(table_path, columns_description, nullptr, nullptr, context); + const auto format_version = plan.initial_metadata_object->getValue(Iceberg::f_format_version); + auto [metadata_object, metadata_object_str] = createEmptyMetadataFile( + table_path, columns_description, nullptr, nullptr, context, format_version); auto current_schema_id = metadata_object->getValue(Iceberg::f_current_schema_id); Poco::JSON::Object::Ptr current_schema; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 06452f64b2ac5..ba5e71171a75d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -820,9 +820,9 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da /// last-column-id must record so every added child gets a globally unique id. Int32 new_field_id = last_column_id + 1; Int32 iter = new_field_id; - auto new_type = Iceberg::getIcebergType(type, iter); + auto format_version = metadata_object->getValue(Iceberg::f_format_version); + auto new_type = Iceberg::getIcebergType(type, iter, format_version); metadata_object->set(Iceberg::f_last_column_id, iter); - Poco::JSON::Object::Ptr new_field = new Poco::JSON::Object; new_field->set(Iceberg::f_id, new_field_id); new_field->set(Iceberg::f_name, column_name); @@ -872,6 +872,7 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, auto current_schema = getCurrentSchema(); auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); + auto format_version = metadata_object->getValue(Iceberg::f_format_version); auto schema_fields = current_schema->getArray(Iceberg::f_fields); /// `AFTER` the column itself keeps its position, as in `ColumnsDescription::modifyColumnOrder`. @@ -880,7 +881,7 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (type) { - auto new_type = Iceberg::getIcebergType(type, last_column_id); + auto new_type = Iceberg::getIcebergType(type, last_column_id, format_version); for (UInt32 i = 0; i < schema_fields->size(); ++i) { @@ -944,7 +945,7 @@ bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, if (type_changed) { - auto new_type = Iceberg::getIcebergType(type, last_column_id); + auto new_type = Iceberg::getIcebergType(type, last_column_id, format_version); target_field->set(Iceberg::f_type, new_type.first); target_field->set(Iceberg::f_required, new_type.second); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 3dcff43e0b030..8119687302b5d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -626,7 +626,7 @@ void checkUnknownTypeAllowed(const String & column_name, const DataTypePtr & typ } /// Returns type and required -std::pair getIcebergType(DataTypePtr type, Int32 & iter) +std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version) { switch (type->getTypeId()) { @@ -656,12 +656,18 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite return {"timestamp", true}; case TypeIndex::DateTime64: { - /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds (v3). - /// Mapping all DateTime64 to `timestamp` while dumping unscaled ticks made - /// ClickHouse-written DateTime64(9) recreate ns-bounds-on-us-schema over-pruning. + /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds and is + /// allowed only in format version 3. Writing it into v1/v2 metadata is invalid. const auto scale = getDecimalScale(*type); if (scale == 9) { + if (format_version < 3) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Iceberg type {} requires format version 3 or higher, got version {}", + type->getName(), + format_version); + auto date_time64 = std::static_pointer_cast(type); if (date_time64->hasExplicitTimeZone()) return {Iceberg::f_timestamptz_ns, true}; @@ -710,7 +716,7 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite field->set(Iceberg::f_name, type_tuple->getNameByPosition(iter_names)); /// Recurse on the element itself: `getNormalizedType` would rename a nested /// tuple's elements to "1", "2", ... (`DataTypeTuple::getNormalizedType`). - auto child_type = getIcebergType(element, iter); + auto child_type = getIcebergType(element, iter, format_version); field->set(Iceberg::f_required, child_type.second); field->set(Iceberg::f_type, child_type.first); fields->add(field); @@ -726,7 +732,7 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite field->set(Iceberg::f_type, "list"); field->set(Iceberg::f_element_id, ++iter); - auto child_type = getIcebergType(type_array->getNestedType(), iter); + auto child_type = getIcebergType(type_array->getNestedType(), iter, format_version); field->set(Iceberg::f_required, false); field->set(Iceberg::f_element, child_type.first); field->set(Iceberg::f_element_required, child_type.second); @@ -741,8 +747,8 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite field->set(Iceberg::f_key_id, ++iter); field->set(Iceberg::f_value_id, ++iter); - field->set(Iceberg::f_key, getIcebergType(type_map->getKeyType(), iter).first); - auto value_type = getIcebergType(type_map->getValueType(), iter); + field->set(Iceberg::f_key, getIcebergType(type_map->getKeyType(), iter, format_version).first); + auto value_type = getIcebergType(type_map->getValueType(), iter, format_version); field->set(Iceberg::f_value, value_type.first); field->set(Iceberg::f_value_required, value_type.second); return {field, true}; @@ -750,7 +756,7 @@ std::pair getIcebergType(DataTypePtr type, Int32 & ite case TypeIndex::Nullable: { auto type_nullable = std::static_pointer_cast(type); - return {getIcebergType(type_nullable->getNestedType(), iter).first, false}; + return {getIcebergType(type_nullable->getNestedType(), iter, format_version).first, false}; } case TypeIndex::Variant: { @@ -1244,7 +1250,7 @@ std::pair createEmptyMetadataFile( Poco::JSON::Object::Ptr field = new Poco::JSON::Object; field->set(Iceberg::f_id, ++iter_for_initial_columns); field->set(Iceberg::f_name, column.name); - auto type = getIcebergType(column.type, iter); + auto type = getIcebergType(column.type, iter, format_version); field->set(Iceberg::f_required, type.second); field->set(Iceberg::f_type, type.first); column_name_to_source_id[column.name] = iter_for_initial_columns; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index 434ff0e8618e4..14bb5339a0d7a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -93,7 +93,10 @@ Poco::JSON::Object::Ptr getMetadataJSONObject( const std::optional & table_uuid); -std::pair getIcebergType(DataTypePtr type, Int32 & iter); +/// Maps a ClickHouse type to an Iceberg JSON type. `timestamp_ns` / `timestamptz_ns` +/// require `format_version` >= 3; lower versions throw instead of writing v3 types +/// into v1/v2 metadata. +std::pair getIcebergType(DataTypePtr type, Int32 & iter, UInt64 format_version); /// Throws if `type` contains `Nothing` at any depth (the Iceberg `unknown` type) and /// `format_version` is below 3, the first format version that defines `unknown`. diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py index 8790d2b4af87c..29591c3339400 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py @@ -19,6 +19,7 @@ from helpers.iceberg_utils import ( check_validity_and_get_prunned_files_general, create_iceberg_table, + get_creation_expression, get_uuid_str, ) @@ -106,3 +107,109 @@ def test_writes_datetime64_nanoseconds_with_timezone(started_cluster_iceberg_no_ f"INSERT INTO {table_name} VALUES (fromUnixTimestamp64Nano({ROWS[0][0]}), 1)" ) assert instance.query(f"SELECT id FROM {table_name}") == "1\n" + + +def _assert_timestamp_ns_requires_v3(error): + assert "BAD_ARGUMENTS" in error, error + assert "LOGICAL_ERROR" not in error, error + assert "format version 3" in error, error + + +@pytest.mark.parametrize("format_version", [1, 2]) +@pytest.mark.parametrize( + "schema", + [ + "(ts DateTime64(9), id Int32)", + "(ts DateTime64(9, 'UTC'), id Int32)", + "(ts Array(DateTime64(9)), id Int32)", + ], + ids=["timestamp_ns", "timestamptz_ns", "nested"], +) +def test_writes_datetime64_nanoseconds_rejected_on_v1_v2( + started_cluster_iceberg_no_spark, format_version, schema +): + """DateTime64(9) is Iceberg v3 timestamp_ns; v1/v2 (default is 2) must not emit it.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_ns_v2_reject_" + get_uuid_str() + error = instance.query_and_get_error( + get_creation_expression( + "s3", + table_name, + started_cluster_iceberg_no_spark, + schema, + format_version, + ) + ) + _assert_timestamp_ns_requires_v3(error) + + +def test_writes_datetime64_nanoseconds_rejected_on_default_format_version( + started_cluster_iceberg_no_spark, +): + """CREATE without iceberg_format_version uses default 2 and must reject DateTime64(9).""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_ns_default_ver_" + get_uuid_str() + bucket = started_cluster_iceberg_no_spark.minio_bucket + error = instance.query_and_get_error( + f""" + DROP TABLE IF EXISTS {table_name}; + CREATE TABLE {table_name} (ts DateTime64(9), id Int32) + ENGINE = IcebergS3(s3, filename = 'var/lib/clickhouse/user_files/iceberg_data/default/{table_name}/', format=Parquet, url = 'http://minio1:9001/{bucket}/'); + """ + ) + _assert_timestamp_ns_requires_v3(error) + + +def test_writes_datetime64_nanoseconds_alter_rejected_on_v2(started_cluster_iceberg_no_spark): + """ALTER ADD / MODIFY must not write timestamp_ns into existing v2 metadata.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_ns_alter_v2_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + "(ts DateTime64(6), id Int32)", + format_version=2, + ) + alter_settings = {"allow_insert_into_iceberg": 1} + + add_error = instance.query_and_get_error( + f"ALTER TABLE {table_name} ADD COLUMN ts_ns Nullable(DateTime64(9))", + settings=alter_settings, + ) + _assert_timestamp_ns_requires_v3(add_error) + + modify_error = instance.query_and_get_error( + f"ALTER TABLE {table_name} MODIFY COLUMN ts DateTime64(9)", + settings=alter_settings, + ) + _assert_timestamp_ns_requires_v3(modify_error) + + describe = instance.query(f"DESCRIBE TABLE {table_name}") + assert "DateTime64(6)" in describe, describe + assert "DateTime64(9" not in describe, describe + + +def test_writes_datetime64_nanoseconds_alter_allowed_on_v3(started_cluster_iceberg_no_spark): + """ALTER ADD Nullable(DateTime64(9)) is valid on Iceberg v3.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_ns_alter_v3_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + "(id Int32)", + format_version=3, + ) + instance.query( + f"ALTER TABLE {table_name} ADD COLUMN ts_ns Nullable(DateTime64(9))", + settings={"allow_insert_into_iceberg": 1}, + ) + describe = instance.query(f"DESCRIBE TABLE {table_name}") + assert "DateTime64(9" in describe, describe + instance.query( + f"INSERT INTO {table_name} VALUES (1, fromUnixTimestamp64Nano({ROWS[0][0]}))" + ) + assert instance.query(f"SELECT id FROM {table_name}") == "1\n" From bd10d114f451394255cacaef6e8676e38e6a9764 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 4 Sep 2026 11:39:22 +0200 Subject: [PATCH 05/22] Fix build --- .../DataLakes/Iceberg/MetadataGenerator.cpp | 6 ++-- .../tests/gtest_iceberg_type_mapping.cpp | 36 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index ba5e71171a75d..2ac34519693e1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -352,7 +352,8 @@ bool MetadataGenerator::isAddColumnApplied(const String & column_name, DataTypeP return false; Int32 unused_field_id = metadata_object->getValue(Iceberg::f_last_column_id); - auto expected_type = Iceberg::getIcebergType(type, unused_field_id); + auto format_version = metadata_object->getValue(Iceberg::f_format_version); + auto expected_type = Iceberg::getIcebergType(type, unused_field_id, format_version); auto fields = current_schema->getArray(Iceberg::f_fields); for (UInt32 i = 0; i < fields->size(); ++i) @@ -420,7 +421,8 @@ bool MetadataGenerator::isModifyColumnApplied(const String & column_name, DataTy if (type) { Int32 unused_field_id = metadata_object->getValue(Iceberg::f_last_column_id); - auto expected_type = Iceberg::getIcebergType(type, unused_field_id); + auto format_version = metadata_object->getValue(Iceberg::f_format_version); + auto expected_type = Iceberg::getIcebergType(type, unused_field_id, format_version); if (field->getValue(Iceberg::f_required) != expected_type.second || !icebergTypesEqualIgnoringIds(field->get(Iceberg::f_type), expected_type.first)) return false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp index 0b51a126abc8a..3168cdbf90922 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_type_mapping.cpp @@ -24,7 +24,8 @@ TEST(IcebergTypeMapping, BoolMapsToBoolean) { auto bool_type = DataTypeFactory::instance().get("Bool"); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(bool_type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(bool_type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "boolean"); EXPECT_TRUE(required); @@ -34,7 +35,8 @@ TEST(IcebergTypeMapping, NullableBoolMapsToBoolean) { auto bool_type = makeNullable(DataTypeFactory::instance().get("Bool")); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(bool_type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(bool_type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "boolean"); EXPECT_FALSE(required); @@ -44,7 +46,8 @@ TEST(IcebergTypeMapping, UInt8MapsToInt) { auto type = std::make_shared(); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "int"); EXPECT_TRUE(required); @@ -54,7 +57,8 @@ TEST(IcebergTypeMapping, Int8MapsToInt) { auto type = std::make_shared(); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "int"); } @@ -63,7 +67,8 @@ TEST(IcebergTypeMapping, UInt16MapsToInt) { auto type = std::make_shared(); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "int"); } @@ -72,7 +77,8 @@ TEST(IcebergTypeMapping, Int16MapsToInt) { auto type = std::make_shared(); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "int"); } @@ -81,7 +87,8 @@ TEST(IcebergTypeMapping, Decimal32MapsToDecimal) { auto type = std::make_shared>(9, 2); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "decimal(9, 2)"); EXPECT_TRUE(required); @@ -91,7 +98,8 @@ TEST(IcebergTypeMapping, Decimal64MapsToDecimal) { auto type = std::make_shared>(18, 5); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "decimal(18, 5)"); } @@ -100,7 +108,8 @@ TEST(IcebergTypeMapping, Decimal128MapsToDecimal) { auto type = std::make_shared>(38, 10); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "decimal(38, 10)"); } @@ -109,7 +118,8 @@ TEST(IcebergTypeMapping, NullableDecimalMapsToDecimalNotRequired) { auto type = makeNullable(std::make_shared>(7, 3)); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "decimal(7, 3)"); EXPECT_FALSE(required); @@ -126,9 +136,10 @@ constexpr UInt32 iceberg_max_decimal_precision = 38; void expectIcebergTypeRejected(const DataTypePtr & type) { Int32 iter = 0; + UInt64 format_version = 3; try { - auto [iceberg_type, required] = getIcebergType(type, iter); + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); FAIL() << type->getName() << " has no Iceberg representation but was mapped to " << iceberg_type.toString(); } @@ -146,7 +157,8 @@ TEST(IcebergTypeMapping, EveryDecimalPrecisionWithinSpecLimitMaps) { auto type = createDecimal(precision, 1); Int32 iter = 0; - auto [iceberg_type, required] = getIcebergType(type, iter); + UInt64 format_version = 3; + auto [iceberg_type, required] = getIcebergType(type, iter, format_version); ASSERT_TRUE(iceberg_type.isString()) << "precision " << precision; EXPECT_EQ(iceberg_type.extract(), "decimal(" + std::to_string(precision) + ", 1)"); EXPECT_TRUE(required); From 21600ea31062b00fb91a4e0dc1ef66bddbcf31a8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 1 Oct 2026 16:16:38 +0200 Subject: [PATCH 06/22] Fix Iceberg inserts into tables partitioned by `DateTime64(9)` `getAvroType` required `timestamp-micros` after the #109764 backport, so identity-partitioned `timestamp_ns` writes failed. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 8 +++- .../test_writes_datetime64_nanoseconds.py | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 8119687302b5d..d9fe54843110e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -797,12 +797,16 @@ Poco::Dynamic::Var getAvroType(DataTypePtr type, Int32 field_id) } case TypeIndex::DateTime64: { - if (getDecimalScale(*type) != 6) + /// Iceberg `timestamp` is microseconds; `timestamp_ns` is nanoseconds (v3). + /// contrib Avro has no `timestamp-nanos` enum, so it stores the logical type + /// as `NONE` and the value as `long`; other engines still see the annotation. + const auto scale = getDecimalScale(*type); + if (scale != 6 && scale != 9) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); Poco::JSON::Object::Ptr timestamp_type = new Poco::JSON::Object; timestamp_type->set("type", "long"); - timestamp_type->set("logicalType", "timestamp-micros"); + timestamp_type->set("logicalType", scale == 9 ? "timestamp-nanos" : "timestamp-micros"); timestamp_type->set("adjust-to-utc", assert_cast(*type).hasExplicitTimeZone()); return timestamp_type; } diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py index 29591c3339400..3b399f0e7eb96 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_datetime64_nanoseconds.py @@ -109,6 +109,47 @@ def test_writes_datetime64_nanoseconds_with_timezone(started_cluster_iceberg_no_ assert instance.query(f"SELECT id FROM {table_name}") == "1\n" +@pytest.mark.parametrize( + "schema, timezone", + [ + ("(ts DateTime64(9), id Int32)", None), + ("(ts DateTime64(9, 'UTC'), id Int32)", "UTC"), + ], + ids=["timestamp_ns", "timestamptz_ns"], +) +def test_writes_identity_partitioned_datetime64_nanoseconds_insert( + started_cluster_iceberg_no_spark, schema, timezone +): + """INSERT into a v3 Iceberg table partitioned by DateTime64(9) must succeed. + + Manifest partition fields go through `getAvroType`. Scale 6 is encoded as + Avro `timestamp-micros`; scale 9 must still be writable as `timestamp_ns` + (not rejected as an unsupported Iceberg type). + """ + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_writes_part_ns_insert_" + get_uuid_str() + create_iceberg_table( + "s3", + instance, + table_name, + started_cluster_iceberg_no_spark, + schema, + format_version=3, + partition_by="ts", + ) + _assert_written_as_datetime64_9(instance, table_name, timezone=timezone) + + ts_ns, row_id, _utc = ROWS[0] + instance.query( + f"INSERT INTO {table_name} VALUES (fromUnixTimestamp64Nano({ts_ns}), {row_id})" + ) + assert instance.query(f"SELECT id FROM {table_name}") == f"{row_id}\n" + assert ( + instance.query(f"SELECT toUnixTimestamp64Nano(ts) FROM {table_name}") + == f"{ts_ns}\n" + ) + + def _assert_timestamp_ns_requires_v3(error): assert "BAD_ARGUMENTS" in error, error assert "LOGICAL_ERROR" not in error, error From aac47f281ced81ffd545c10b312702e6f80bd312 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 7 Oct 2026 10:48:48 +0200 Subject: [PATCH 07/22] Honor `iceberg_timezone_for_timestamptz` for Iceberg `timestamptz_ns` `getSimpleType` hardcoded UTC for nanosecond timestamptz while microsecond `timestamptz` already used the setting. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp | 5 ++++- .../Iceberg/tests/gtest_iceberg_schema_processor.cpp | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 1ca873d245aee..fb98377d0ed05 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -613,7 +613,10 @@ DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name_arg, if (type_name == f_timestamp_ns) return std::make_shared(9); if (type_name == f_timestamptz_ns) - return std::make_shared(9, "UTC"); + { + std::string timezone = context_->getSettingsRef()[Setting::iceberg_timezone_for_timestamptz]; + return std::make_shared(9, timezone); + } if (type_name == f_string || type_name == f_binary) return std::make_shared(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index dd12fe651392a..77908fb3e0c08 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -94,6 +94,11 @@ TEST(IcebergSchemaProcessor, GetSimpleTypeTimestamptzNs) { auto type = IcebergSchemaProcessor::getSimpleType("timestamptz_ns", getContext().context); EXPECT_EQ(type->getName(), "DateTime64(9, 'UTC')"); + + auto context = DB::Context::createCopy(getContext().context); + context->setSetting("iceberg_timezone_for_timestamptz", String("Europe/Berlin")); + type = IcebergSchemaProcessor::getSimpleType("timestamptz_ns", context); + EXPECT_EQ(type->getName(), "DateTime64(9, 'Europe/Berlin')"); } TEST(IcebergSchemaProcessor, GetSimpleTypeString) From cb707f2a85838cbcf9dd3929b87a5856f44cc358 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 7 Oct 2026 11:53:52 +0200 Subject: [PATCH 08/22] Write Iceberg `timestamp_ns` Parquet as not UTC-adjusted Default Parquet still marks every `DateTime64` as `isAdjustedToUTC`. Iceberg writes set the flag from the column timezone so `timestamp_ns` matches `TIMESTAMP(NANOS, false)`. Co-authored-by: Cursor --- src/Formats/FormatSettings.h | 4 ++++ .../Formats/Impl/Parquet/PrepareForWrite.cpp | 8 ++++--- src/Processors/Formats/Impl/Parquet/Write.h | 1 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 1 + .../tests/gtest_write_parquet_page_index.cpp | 23 +++++++++++++++++++ .../DataLakes/Iceberg/Compaction.cpp | 6 ++++- .../DataLakes/Iceberg/MultipleFileWriter.cpp | 12 +++++----- .../DataLakes/Iceberg/Mutations.cpp | 8 +++++-- 8 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index a5d6118c2fc8f..529d763929b7b 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -402,6 +402,10 @@ struct FormatSettings bool output_datetime_as_uint32 = false; bool output_date_as_uint16 = false; bool output_enum_as_byte_array = false; + /// When false (default), DateTime64 is always written as isAdjustedToUTC=true because + /// ClickHouse ticks are UTC. Iceberg sets this so timestamp(_ns) is false and + /// timestamptz(_ns) is true, matching TIMESTAMP(*, hasExplicitTimeZone()). + bool output_timestamp_adjusted_to_utc_from_timezone = false; bool parallel_encoding = true; bool write_page_index = false; diff --git a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp index 2faa5a7d49860..ad23da3dc50d6 100644 --- a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp +++ b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp @@ -447,9 +447,11 @@ void preparePrimitiveColumn(ColumnPtr column, DataTypePtr type, const std::strin } parq::TimestampType tt; - /// (Shouldn't we check the DateTime64's timezone parameter here? No, the actual number - /// in DateTime64 column is always in UTC, regardless of the timezone parameter.) - tt.__set_isAdjustedToUTC(true); + /// ClickHouse ticks are always UTC. By default keep isAdjustedToUTC=true. Iceberg + /// sets output_timestamp_adjusted_to_utc_from_timezone so timestamp(_ns) is + /// TIMESTAMP(*, false) and timestamptz(_ns) is TIMESTAMP(*, true). + tt.__set_isAdjustedToUTC( + options.output_timestamp_adjusted_to_utc_from_timezone ? dt.hasExplicitTimeZone() : true); tt.__set_unit(unit); parq::LogicalType t; t.__set_TIMESTAMP(tt); diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index 8628e29561372..0eef9bc59d482 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -27,6 +27,7 @@ struct WriteOptions bool output_datetime_as_uint32 = false; bool output_date_as_uint16 = false; bool output_enum_as_byte_array = false; + bool output_timestamp_adjusted_to_utc_from_timezone = false; /// Note: the meaning of some compression methods here is different from /// wrapReadBufferWithCompressionMethod: diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 28c3c0816d54a..eed31e78a7a51 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -55,6 +55,7 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea options.output_datetime_as_uint32 = format_settings.parquet.output_datetime_as_uint32; options.output_date_as_uint16 = format_settings.parquet.output_date_as_uint16; options.output_enum_as_byte_array = format_settings.parquet.output_enum_as_byte_array; + options.output_timestamp_adjusted_to_utc_from_timezone = format_settings.parquet.output_timestamp_adjusted_to_utc_from_timezone; options.data_page_size = format_settings.parquet.data_page_size; options.write_batch_size = format_settings.parquet.write_batch_size; options.write_page_index = format_settings.parquet.write_page_index; diff --git a/src/Processors/tests/gtest_write_parquet_page_index.cpp b/src/Processors/tests/gtest_write_parquet_page_index.cpp index 506ee9b2732a1..17f4a29e6c355 100644 --- a/src/Processors/tests/gtest_write_parquet_page_index.cpp +++ b/src/Processors/tests/gtest_write_parquet_page_index.cpp @@ -8,6 +8,8 @@ # include # include # include +# include +# include # include # include # include @@ -509,5 +511,26 @@ TEST(Parquet, WriteParquetPageIndexOversizedStringStats) ASSERT_GT(offset_index_location.value().offset, 0); ASSERT_GT(offset_index_location.value().length, 0); } + +TEST(ParquetWrite, DateTime64AdjustedToUTCFromTimezone) +{ + auto ts = std::make_shared(9); + auto ts_tz = std::make_shared(9, "UTC"); + Block sample; + sample.insert({ts->createColumn(), ts, "ts"}); + sample.insert({ts_tz->createColumn(), ts_tz, "ts_tz"}); + + Parquet::WriteOptions default_options; + auto default_schema = Parquet::convertSchema(sample, default_options, std::nullopt); + ASSERT_EQ(default_schema.size(), 3); + EXPECT_TRUE(default_schema[1].logicalType.TIMESTAMP.isAdjustedToUTC); + EXPECT_TRUE(default_schema[2].logicalType.TIMESTAMP.isAdjustedToUTC); + + Parquet::WriteOptions iceberg_options; + iceberg_options.output_timestamp_adjusted_to_utc_from_timezone = true; + auto iceberg_schema = Parquet::convertSchema(sample, iceberg_options, std::nullopt); + EXPECT_FALSE(iceberg_schema[1].logicalType.TIMESTAMP.isAdjustedToUTC); + EXPECT_TRUE(iceberg_schema[2].logicalType.TIMESTAMP.isAdjustedToUTC); +} } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index a2f275c9cd049..e8bebbef824f7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -423,8 +423,12 @@ static void writeDataFiles( FormatFilterInfoPtr output_format_filter_info = std::make_shared(nullptr, context, column_mapper, nullptr, nullptr); + auto write_format_settings = format_settings; + if (!write_format_settings) + write_format_settings = getFormatSettings(context); + write_format_settings->parquet.output_timestamp_adjusted_to_utc_from_timezone = true; auto output_format = FormatFactory::instance().getOutputFormat( - write_format, *write_buffer, *sample_block, context, format_settings, output_format_filter_info); + write_format, *write_buffer, *sample_block, context, write_format_settings, output_format_filter_info); while (true) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp index d41fc24e41623..14a2c60f756d7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp @@ -239,12 +239,12 @@ void MultipleFileWriter::startNewFile() buffer = object_storage->writeObject( StoredObject(storage_path), WriteMode::Rewrite, std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); - if (format_settings) - { - format_settings->parquet.write_page_index = true; - format_settings->parquet.bloom_filter_push_down = true; - format_settings->parquet.filter_push_down = true; - } + if (!format_settings) + format_settings = getFormatSettings(context); + format_settings->parquet.write_page_index = true; + format_settings->parquet.bloom_filter_push_down = true; + format_settings->parquet.filter_push_down = true; + format_settings->parquet.output_timestamp_adjusted_to_utc_from_timezone = true; /// The ORC String/FixedString logical-type handling lives in ORCBlockOutputFormat, keyed on /// the column mapper, so it covers the compaction/mutation rewrite paths too. FormatFilterInfoPtr format_filter_info = std::make_shared(nullptr, context, column_mapper, nullptr, nullptr); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 677c2201fcdd2..8c11cd40620ee 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -224,6 +224,10 @@ static std::optional writeDataFiles( Poco::JSON::Object::Ptr data_schema) { chassert(commands.size() == 1); + auto iceberg_format_settings = format_settings; + if (!iceberg_format_settings) + iceberg_format_settings = getFormatSettings(context); + iceberg_format_settings->parquet.output_timestamp_adjusted_to_utc_from_timezone = true; DataFileWriteResultByPartitionKey delete_data_result; DataFileStatisticsByPartitionKey delete_data_statistics; std::unordered_map, ChunkPartitioner::PartitionKeyHasher> delete_data_write_buffers; @@ -294,7 +298,7 @@ static std::optional writeDataFiles( column_mapper->setIcebergStringPaths({String(IcebergPositionDeleteTransform::data_file_path_column_name)}); FormatFilterInfoPtr format_filter_info = std::make_shared(nullptr, context, column_mapper, nullptr, nullptr); auto output_format = FormatFactory::instance().getOutputFormat( - write_format, *write_buffer, delete_file_sample_block, context, format_settings, format_filter_info); + write_format, *write_buffer, delete_file_sample_block, context, iceberg_format_settings, format_filter_info); delete_data_write_buffers[partition_key] = std::move(write_buffer); delete_data_writers[partition_key] = std::move(output_format); @@ -398,7 +402,7 @@ static std::optional writeDataFiles( ColumnMapperPtr data_column_mapper = createColumnMapper(data_schema); FormatFilterInfoPtr data_format_filter_info = std::make_shared(nullptr, context, data_column_mapper, nullptr, nullptr); auto data_output_format = FormatFactory::instance().getOutputFormat( - write_format, *data_write_buffer, data_block, context, format_settings, data_format_filter_info); + write_format, *data_write_buffer, data_block, context, iceberg_format_settings, data_format_filter_info); update_data_write_buffers[partition_key] = std::move(data_write_buffer); it = update_data_writers.emplace(partition_key, std::move(data_output_format)).first; From 05aac0bb30e3fd9bb931dd616f34641af64d9e2a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 8 Oct 2026 10:22:29 +0200 Subject: [PATCH 09/22] Keep Iceberg `timestamp-nanos` in identity-partition Avro schemas contrib Avro `toJson` drops unknown logical types, so DataFileWriter was writing a bare `long`. Persist the original JSON in `avro.schema` instead. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index d96312030178e..8447bbd148491 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -766,8 +766,9 @@ void generateManifestFile( auto adapter = std::make_unique(buf); avro::DataFileWriter writer(std::move(adapter), schema); - /// avro-cpp's compiled schema loses the Iceberg field-id/element-id attributes; write the - /// original id-carrying JSON as the avro.schema header so external readers can plan a scan. + /// avro-cpp's compiled schema loses Iceberg field-id attributes, and `toJson` drops + /// unknown logical types such as `timestamp-nanos`. Write the original JSON as the + /// `avro.schema` header so external readers keep both. writer.setMetadata(Iceberg::f_avro_schema, schema_representation); writer.setMetadata(Iceberg::f_schema, json_representation); writer.setMetadata(Iceberg::f_schema_id, std::to_string(schema_object_to_write->getValue(Iceberg::f_schema_id))); From e728cfe99c33c47bac40f2bcc504bc84d5df9a7a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 18:09:09 +0200 Subject: [PATCH 10/22] Pass `format_version` to Iceberg `unknown` type unit tests `getIcebergType` now requires `format_version`; the two `Nothing` tests still used two arguments and failed debug builds. Co-authored-by: Cursor --- .../Iceberg/tests/gtest_iceberg_metadata_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp index 019926ea07faf..7453998c7bc80 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_metadata_generator.cpp @@ -796,7 +796,7 @@ TEST(IcebergMetadataGenerator, ModifyColumnWideningRecordsTheNewTypeInANewSchema TEST(IcebergMetadataGenerator, GetIcebergTypeNothingProducesUnknown) { Int32 iter = 0; - auto [iceberg_type, required] = Iceberg::getIcebergType(std::make_shared(), iter); + auto [iceberg_type, required] = Iceberg::getIcebergType(std::make_shared(), iter, /* format_version */ 3); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "unknown"); EXPECT_FALSE(required); @@ -806,7 +806,7 @@ TEST(IcebergMetadataGenerator, GetIcebergTypeNothingProducesUnknown) TEST(IcebergMetadataGenerator, GetIcebergTypeNullableNothingProducesUnknown) { Int32 iter = 0; - auto [iceberg_type, required] = Iceberg::getIcebergType(makeNullable(std::make_shared()), iter); + auto [iceberg_type, required] = Iceberg::getIcebergType(makeNullable(std::make_shared()), iter, /* format_version */ 3); ASSERT_TRUE(iceberg_type.isString()); EXPECT_EQ(iceberg_type.extract(), "unknown"); EXPECT_FALSE(required); From 856b6326585b1e78f3ee6d1ce51637b77da060ea Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 18:17:18 +0200 Subject: [PATCH 11/22] Fail open Iceberg min/max pruning when decoded bounds invert Mixed-scale `timestamp` stats can decode a near-epoch ns min as microseconds and a later ns max into a smaller value. An inverted range pruned the whole file; skip that column's hyperrectangle instead. Co-authored-by: Cursor --- src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp | 2 +- .../ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp index 876b7c900a194..a3f6898562874 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp @@ -157,7 +157,7 @@ DataFileMetaInfo::DataFileMetaInfo( { auto left = Iceberg::deserializeFieldFromBinaryRepr(left_str, type, true); auto right = Iceberg::deserializeFieldFromBinaryRepr(right_str, type, false); - if (left && right) + if (left && right && !DB::Range::less(*right, *left)) hyperrectangle = DB::Range(*left, true, *right, true); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 322bb68f6516b..43868071a14bc 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -485,6 +485,10 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) continue; } + /// Mixed-scale timestamp stats can decode to lower > upper; skip min/max for this column. + if (DB::Range::less(*right, *left)) + continue; + hyperrectangles.emplace(column_id, DB::Range(*left, true, *right, true)); } } From d9ed903bfee1a93abb61756d0aa354611435d46b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 12:48:07 +0200 Subject: [PATCH 12/22] Accept Int64 Iceberg `DateTime64(9)` partition values on manifest rewrite contrib Avro has no `timestamp-nanos`, so compaction reads identity partition keys as `Int64`. `getDateTime64ValueForIcebergBounds` required `Decimal64` and threw `BAD_GET` before the snapshot committed. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 8447bbd148491..019c24f05decf 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -300,6 +300,12 @@ Int64 getDateTime64ValueForIcebergBounds(const Field & field, DataTypePtr type) if (!WhichDataType(type).isDateTime64()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected DateTime64, got {}", type->getName()); + /// Manifest rewrite reads the Avro `long` back as Int64 (contrib Avro has no `timestamp-nanos`). + if (field.getType() == Field::Types::Int64) + return field.safeGet(); + if (field.getType() == Field::Types::UInt64) + return static_cast(field.safeGet()); + const auto scale = getDecimalScale(*type); const auto value = field.safeGet().getValue().value; if (scale == 9) From 5d9b88be0ec3ef11aa3a2ed76ca580e6a3b66fe0 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 14:11:02 +0200 Subject: [PATCH 13/22] Fail open Iceberg `timestamp` min/max for near-epoch nanosecond bounds `|ticks| <= 1e16` was kept as microseconds, so a 1-day-after-epoch ns min/max decoded as ~1972 and over-pruned. Skip that column's hyperrectangle below 2024-01-01 UTC so 2024+ pruning stays intact. Co-authored-by: Cursor --- .../Iceberg/IcebergFieldParseHelpers.cpp | 23 ++++-- .../gtest_iceberg_field_parse_helpers.cpp | 72 +++++++++++++++++++ .../helpers/iceberg_timestamp_ns.py | 34 +++++++++ .../test_pruning_datetime64_nanoseconds.py | 31 ++++++-- 4 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 08127a5887c8a..93a644c9a2eb8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -35,6 +35,13 @@ namespace constexpr Int64 microseconds_ambiguous_threshold = 10'000'000'000'000'000LL; constexpr Int64 nanoseconds_lower_threshold = 1'000'000'000'000'000'000LL; constexpr Int64 nanoseconds_per_microsecond = 1000; +/// 2024-01-01 00:00:00 UTC as microseconds. Below this, `|ticks|` is also a +/// plausible near-epoch nanosecond count (1.704e15 ns is ~19.7 days). Fail-open +/// that overlap so a 1-day-after-epoch ns min/max is not read as year ~1972. +/// Ticks from this instant through 1e16 stay microseconds so 2024+ Iceberg +/// `timestamp` pruning is unchanged. Residual: 19.7..115 days as ns still collide +/// with 2024..2286 as µs. +constexpr Int64 min_kept_microseconds = 1'704'067'200'000'000LL; bool magnitudeExceeds(Int64 value, Int64 threshold) { @@ -65,7 +72,9 @@ Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) /// Some writers store nanosecond stats on Iceberg `timestamp` (`DateTime64(6)`). /// Convert only in a true nanosecond band (`|ticks| > 1e18`). Between 1e16 and /// 1e18 the value may be far-future microseconds (do not convert; fail open so -/// min/max pruning is skipped). `|ticks| <= 1e16` is kept as microseconds. +/// min/max pruning is skipped). Below `min_kept_microseconds`, ticks may be +/// near-epoch nanoseconds (also fail open). `(min_kept_microseconds, 1e16]` is +/// kept as microseconds. std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) { if (str.size() != sizeof(Int64)) @@ -74,12 +83,18 @@ std::optional deserializeDateTime64FromBinaryRepr(const std::string & str Int64 unscaled = unalignedLoadLittleEndian(str.data()); const UInt32 scale = getDecimalScale(type); - if (scale == 6 && magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) + if (scale == 6) { - if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) + if (!magnitudeExceeds(unscaled, min_kept_microseconds)) return std::nullopt; - unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); + if (magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) + { + if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) + return std::nullopt; + + unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); + } } return DecimalField(DateTime64(unscaled), scale); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp new file mode 100644 index 0000000000000..7d6e0851c024c --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp @@ -0,0 +1,72 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace +{ + +std::string littleEndianInt64(Int64 value) +{ + std::string bytes(sizeof(Int64), '\0'); + unalignedStoreLittleEndian(bytes.data(), value); + return bytes; +} + +Int64 decimal64Ticks(const Field & field) +{ + return static_cast(field.safeGet>().getValue()); +} + +} + +TEST(IcebergFieldParseHelpers, NearEpochNanosecondDateTime64BoundsFailOpen) +{ + auto type = std::make_shared(6); + /// One day after the epoch as nanoseconds: kept as µs this is ~1972, not 1970-01-02. + constexpr Int64 one_day_ns = 86'400'000'000'000LL; + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_day_ns), type, true).has_value()); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_day_ns), type, false).has_value()); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(-one_day_ns), type, true).has_value()); +} + +TEST(IcebergFieldParseHelpers, ModernMicrosecondDateTime64BoundsAreKept) +{ + auto type = std::make_shared(6); + /// 2024-02-20 10:00:00 UTC as microseconds — spec-correct Iceberg `timestamp`. + constexpr Int64 us_2024_02_20 = 1'708'423'200'000'000LL; + auto lower = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, true); + auto upper = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, false); + ASSERT_TRUE(lower.has_value()); + ASSERT_TRUE(upper.has_value()); + EXPECT_EQ(decimal64Ticks(*lower), us_2024_02_20); + EXPECT_EQ(decimal64Ticks(*upper), us_2024_02_20); +} + +TEST(IcebergFieldParseHelpers, ProvenNanosecondDateTime64BoundsAreConverted) +{ + auto type = std::make_shared(6); + constexpr Int64 ns_2024_01_01 = 1'704'067'200'123'456'789LL; + auto field = deserializeFieldFromBinaryRepr(littleEndianInt64(ns_2024_01_01), type, true); + ASSERT_TRUE(field.has_value()); + EXPECT_EQ(decimal64Ticks(*field), ns_2024_01_01 / 1000); +} + +TEST(IcebergFieldParseHelpers, FarFutureMicrosecondDateTime64BoundsFailOpen) +{ + auto type = std::make_shared(6); + /// Spark 9999-12-31 sentinel sits in (1e16, 1e18]. + constexpr Int64 spark_9999_us = 253'402'300'799'999'999LL; + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(spark_9999_us), type, false).has_value()); +} + +#endif diff --git a/tests/integration/helpers/iceberg_timestamp_ns.py b/tests/integration/helpers/iceberg_timestamp_ns.py index 351b3cc9eeb1c..5e91221fc5955 100644 --- a/tests/integration/helpers/iceberg_timestamp_ns.py +++ b/tests/integration/helpers/iceberg_timestamp_ns.py @@ -223,6 +223,40 @@ def _utc_datetime_to_micros(dt): ) +# One day after the Unix epoch as nanoseconds. Kept as microseconds this is ~1972, +# so a predicate on 1970-01-02 over-prunes unless the decoder fails open. +NEAR_EPOCH_NS_ROWS = [ + (86_400_000_000_000, 1, "1970-01-02 00:00:00.000000000"), +] + + +def check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): + """Iceberg `timestamp` whose min/max bytes are near-epoch nanoseconds must not over-prune. + + One day after the epoch is `8.64e13` ns (`|ticks| <= 1e16`). The inverted-range + check does not fire when both bounds stay in that band. + """ + type_name = node.query( + f"SELECT toTypeName({column}) FROM {table} LIMIT 1", + settings=PRUNE_OFF, + ) + assert "DateTime64(6" in type_name, type_name + assert "DateTime64(9" not in type_name, type_name + + ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) + assert ids == "1\n" + micros = node.query( + f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", + settings=PRUNE_OFF, + ).strip() + assert micros == "86400000000", micros + + eq_epoch_day = datetime64_literal("1970-01-02 00:00:00.000000000", scale=6, timezone=None) + query = f"SELECT id FROM {table} WHERE {column} = {eq_epoch_day} ORDER BY id" + assert pruned_files(query) == 0 + assert node.query(query, settings=PRUNE_ON) == "1\n" + + def check_minmax_far_future_us_upper_bound(node, table, column, pruned_files): """Iceberg `timestamp` with a far-future microsecond *upper* bound must not over-prune. diff --git a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py index e30843636eec2..2c470a2eda21c 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py @@ -24,11 +24,13 @@ DATETIME64_MAX_US, FILTER_LITERAL_IDS, FILTER_LITERALS, + NEAR_EPOCH_NS_ROWS, PRUNE_OFF, PRUNE_ON, ROWS, SPARK_TIMESTAMP_SENTINEL_US, check_minmax_far_future_us_upper_bound, + check_minmax_near_epoch_ns_bounds_on_timestamp_schema, check_minmax_ns_bounds_on_timestamp_schema, check_ns_pruning, check_ns_pruning_datetime64_microseconds, @@ -76,7 +78,7 @@ def _arrow_timestamp_row(ts_ns, row_id): ) -def _create_timestamp_table(started_cluster, table_name, patch): +def _create_timestamp_table(started_cluster, table_name, patch, rows=ROWS): """Iceberg v2 `timestamp` (microseconds parquet) with a custom manifest-bound patch.""" work = Path(tempfile.mkdtemp(prefix="iceberg_ns_bounds_")) try: @@ -96,7 +98,7 @@ def _create_timestamp_table(started_cluster, table_name, patch): ) table.append( pa.concat_tables( - [_arrow_timestamp_row(ts_ns, row_id) for ts_ns, row_id, _utc in ROWS] + [_arrow_timestamp_row(ts_ns, row_id) for ts_ns, row_id, _utc in rows] ) ) table = catalog.load_table(identifier) @@ -119,10 +121,10 @@ def _create_timestamp_table(started_cluster, table_name, patch): shutil.rmtree(work, ignore_errors=True) -def _create_timestamp_table_with_ns_bounds(started_cluster, table_name): +def _create_timestamp_table_with_ns_bounds(started_cluster, table_name, rows=ROWS): """Iceberg v2 `timestamp` (microseconds) whose manifest min/max bytes are nanoseconds.""" return _create_timestamp_table( - started_cluster, table_name, patch_iceberg_timestamp_bounds_us_to_ns + started_cluster, table_name, patch_iceberg_timestamp_bounds_us_to_ns, rows=rows ) @@ -265,6 +267,27 @@ def pruned_files(select_expression): check_minmax_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) +def test_minmax_pruning_near_epoch_ns_bounds_on_timestamp_schema(started_cluster_iceberg_no_spark): + """Near-epoch ns min/max on Iceberg `timestamp` must not be decoded as microseconds.""" + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_timestamp_table_with_ns_bounds( + started_cluster_iceberg_no_spark, "minmax_near_epoch_ns", rows=NEAR_EPOCH_NS_ROWS + ) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, + "minmax_near_epoch_ns", + PRUNE_OFF, + PRUNE_ON, + "IcebergMinMaxIndexPrunedFiles", + select_expression, + ) + + check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) + + @pytest.mark.parametrize( "upper_bound_us", [SPARK_TIMESTAMP_SENTINEL_US, DATETIME64_MAX_US], From 6fe81c0eebee909dd751b6d81ceb08322c5049b1 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 14:30:00 +0200 Subject: [PATCH 14/22] Keep Iceberg `timestamp` min/max at the 2024-01-01 microsecond threshold `magnitudeExceeds` is strict, so `|ticks| == 2024-01-01 UTC` fail-opened while the comment said that instant is kept. Fail-open only below the threshold. Co-authored-by: Cursor --- .../DataLakes/Iceberg/IcebergFieldParseHelpers.cpp | 9 +++++++-- .../tests/gtest_iceberg_field_parse_helpers.cpp | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 93a644c9a2eb8..1e68382b41e0d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -48,6 +48,11 @@ bool magnitudeExceeds(Int64 value, Int64 threshold) return value > threshold || value < -threshold; } +bool magnitudeBelow(Int64 value, Int64 threshold) +{ + return value < threshold && value > -threshold; +} + /// Convert ns ticks to us ticks without shrinking a [lower, upper] range. Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) { @@ -73,7 +78,7 @@ Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) /// Convert only in a true nanosecond band (`|ticks| > 1e18`). Between 1e16 and /// 1e18 the value may be far-future microseconds (do not convert; fail open so /// min/max pruning is skipped). Below `min_kept_microseconds`, ticks may be -/// near-epoch nanoseconds (also fail open). `(min_kept_microseconds, 1e16]` is +/// near-epoch nanoseconds (also fail open). `[min_kept_microseconds, 1e16]` is /// kept as microseconds. std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) { @@ -85,7 +90,7 @@ std::optional deserializeDateTime64FromBinaryRepr(const std::string & str if (scale == 6) { - if (!magnitudeExceeds(unscaled, min_kept_microseconds)) + if (magnitudeBelow(unscaled, min_kept_microseconds)) return std::nullopt; if (magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp index 7d6e0851c024c..2687fd5687027 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp @@ -42,6 +42,16 @@ TEST(IcebergFieldParseHelpers, NearEpochNanosecondDateTime64BoundsFailOpen) TEST(IcebergFieldParseHelpers, ModernMicrosecondDateTime64BoundsAreKept) { auto type = std::make_shared(6); + /// 2024-01-01 00:00:00 UTC is the first kept microsecond bound (`min_kept_microseconds`). + constexpr Int64 us_2024_01_01 = 1'704'067'200'000'000LL; + auto at_threshold = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01), type, true); + ASSERT_TRUE(at_threshold.has_value()); + EXPECT_EQ(decimal64Ticks(*at_threshold), us_2024_01_01); + auto negative_threshold = deserializeFieldFromBinaryRepr(littleEndianInt64(-us_2024_01_01), type, true); + ASSERT_TRUE(negative_threshold.has_value()); + EXPECT_EQ(decimal64Ticks(*negative_threshold), -us_2024_01_01); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01 - 1), type, true).has_value()); + /// 2024-02-20 10:00:00 UTC as microseconds — spec-correct Iceberg `timestamp`. constexpr Int64 us_2024_02_20 = 1'708'423'200'000'000LL; auto lower = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, true); From 6696ff665a3cf603e0c6dc7cb4620f707362be02 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 14 Oct 2026 11:18:30 +0200 Subject: [PATCH 15/22] =?UTF-8?q?Fail=20open=20Iceberg=20`timestamp`=20min?= =?UTF-8?q?/max=20in=20the=20remaining=20ns/=C2=B5s=20overlap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[1.704e15, 1e16]` is both ~19.7–115 days as nanoseconds and 2024–2286 as microseconds. Convert only `|ticks| > 1e18`; skip min/max pruning otherwise because no integer test splits those scales. Co-authored-by: Cursor --- .../Iceberg/IcebergFieldParseHelpers.cpp | 34 +++---------------- .../gtest_iceberg_field_parse_helpers.cpp | 29 +++++++--------- .../helpers/iceberg_timestamp_ns.py | 33 ++++++++++++++++++ .../test_pruning_datetime64_nanoseconds.py | 23 +++++++++++++ 4 files changed, 73 insertions(+), 46 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 1e68382b41e0d..13bc6f15f7692 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -28,31 +28,15 @@ namespace Iceberg namespace { -/// 1e16 µs is ~year 2286. That is not an Iceberg `timestamp` max: Spark sentinels -/// such as `9999-12-31` (`~2.53e17` µs) and ClickHouse `DateTime64` max -/// `2299-12-31` (`~1.04e16` µs) are spec-correct microseconds above this. /// 1e18 ns is ~year 2001; customer 2026 ns bounds are `~1.76e18`. -constexpr Int64 microseconds_ambiguous_threshold = 10'000'000'000'000'000LL; constexpr Int64 nanoseconds_lower_threshold = 1'000'000'000'000'000'000LL; constexpr Int64 nanoseconds_per_microsecond = 1000; -/// 2024-01-01 00:00:00 UTC as microseconds. Below this, `|ticks|` is also a -/// plausible near-epoch nanosecond count (1.704e15 ns is ~19.7 days). Fail-open -/// that overlap so a 1-day-after-epoch ns min/max is not read as year ~1972. -/// Ticks from this instant through 1e16 stay microseconds so 2024+ Iceberg -/// `timestamp` pruning is unchanged. Residual: 19.7..115 days as ns still collide -/// with 2024..2286 as µs. -constexpr Int64 min_kept_microseconds = 1'704'067'200'000'000LL; bool magnitudeExceeds(Int64 value, Int64 threshold) { return value > threshold || value < -threshold; } -bool magnitudeBelow(Int64 value, Int64 threshold) -{ - return value < threshold && value > -threshold; -} - /// Convert ns ticks to us ticks without shrinking a [lower, upper] range. Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) { @@ -75,11 +59,9 @@ Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) /// Iceberg timestamps are 8-byte little-endian (spec Appendix D). /// Some writers store nanosecond stats on Iceberg `timestamp` (`DateTime64(6)`). -/// Convert only in a true nanosecond band (`|ticks| > 1e18`). Between 1e16 and -/// 1e18 the value may be far-future microseconds (do not convert; fail open so -/// min/max pruning is skipped). Below `min_kept_microseconds`, ticks may be -/// near-epoch nanoseconds (also fail open). `[min_kept_microseconds, 1e16]` is -/// kept as microseconds. +/// Convert only when `|ticks| > 1e18`. Otherwise fail open: `|ticks| <= 1e16` +/// collides near-epoch nanoseconds (~115 days) with 1970–2286 microseconds, and +/// `(1e16, 1e18]` collides Spark `9999-12-31` sentinels with 1970–2001 nanoseconds. std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) { if (str.size() != sizeof(Int64)) @@ -90,16 +72,10 @@ std::optional deserializeDateTime64FromBinaryRepr(const std::string & str if (scale == 6) { - if (magnitudeBelow(unscaled, min_kept_microseconds)) + if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) return std::nullopt; - if (magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) - { - if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) - return std::nullopt; - - unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); - } + unscaled = nanosecondsToMicrosecondsForBound(unscaled, lower_bound); } return DecimalField(DateTime64(unscaled), scale); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp index 2687fd5687027..069a01bed01c1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp @@ -39,27 +39,22 @@ TEST(IcebergFieldParseHelpers, NearEpochNanosecondDateTime64BoundsFailOpen) EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(-one_day_ns), type, true).has_value()); } -TEST(IcebergFieldParseHelpers, ModernMicrosecondDateTime64BoundsAreKept) +TEST(IcebergFieldParseHelpers, AmbiguousScaleDateTime64BoundsFailOpen) { auto type = std::make_shared(6); - /// 2024-01-01 00:00:00 UTC is the first kept microsecond bound (`min_kept_microseconds`). + /// 2024-01-01 UTC µs is also ~19.7 days as ns. No integer test splits those scales. constexpr Int64 us_2024_01_01 = 1'704'067'200'000'000LL; - auto at_threshold = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01), type, true); - ASSERT_TRUE(at_threshold.has_value()); - EXPECT_EQ(decimal64Ticks(*at_threshold), us_2024_01_01); - auto negative_threshold = deserializeFieldFromBinaryRepr(littleEndianInt64(-us_2024_01_01), type, true); - ASSERT_TRUE(negative_threshold.has_value()); - EXPECT_EQ(decimal64Ticks(*negative_threshold), -us_2024_01_01); - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01 - 1), type, true).has_value()); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01), type, true).has_value()); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(-us_2024_01_01), type, true).has_value()); - /// 2024-02-20 10:00:00 UTC as microseconds — spec-correct Iceberg `timestamp`. - constexpr Int64 us_2024_02_20 = 1'708'423'200'000'000LL; - auto lower = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, true); - auto upper = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, false); - ASSERT_TRUE(lower.has_value()); - ASSERT_TRUE(upper.has_value()); - EXPECT_EQ(decimal64Ticks(*lower), us_2024_02_20); - EXPECT_EQ(decimal64Ticks(*upper), us_2024_02_20); + /// 50 days after the epoch as ns (`4.32e15`) sits in the same band as 2024–2286 µs. + constexpr Int64 fifty_days_ns = 4'320'000'000'000'000LL; + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(fifty_days_ns), type, true).has_value()); + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(fifty_days_ns), type, false).has_value()); + + /// Band upper edge: 1e16 ns is ~1970-04-26, 1e16 µs is ~year 2286. + constexpr Int64 one_e16 = 10'000'000'000'000'000LL; + EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_e16), type, true).has_value()); } TEST(IcebergFieldParseHelpers, ProvenNanosecondDateTime64BoundsAreConverted) diff --git a/tests/integration/helpers/iceberg_timestamp_ns.py b/tests/integration/helpers/iceberg_timestamp_ns.py index 5e91221fc5955..3922f8d37d891 100644 --- a/tests/integration/helpers/iceberg_timestamp_ns.py +++ b/tests/integration/helpers/iceberg_timestamp_ns.py @@ -229,6 +229,12 @@ def _utc_datetime_to_micros(dt): (86_400_000_000_000, 1, "1970-01-02 00:00:00.000000000"), ] +# 50 days after the epoch as nanoseconds (`4.32e15`). Same integer as ~2106 µs, +# so a predicate on 1970-02-20 over-prunes unless the decoder fails open. +OVERLAP_BAND_NS_ROWS = [ + (4_320_000_000_000_000, 1, "1970-02-20 00:00:00.000000000"), +] + def check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): """Iceberg `timestamp` whose min/max bytes are near-epoch nanoseconds must not over-prune. @@ -257,6 +263,33 @@ def check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, column, p assert node.query(query, settings=PRUNE_ON) == "1\n" +def check_minmax_overlap_band_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): + """Iceberg `timestamp` whose min/max bytes are leftover-band nanoseconds must not over-prune. + + 50 days after the epoch is `4.32e15` ns, which is also a 2024–2286 microsecond + timestamp. The decoder cannot tell the scales apart and must fail open. + """ + type_name = node.query( + f"SELECT toTypeName({column}) FROM {table} LIMIT 1", + settings=PRUNE_OFF, + ) + assert "DateTime64(6" in type_name, type_name + assert "DateTime64(9" not in type_name, type_name + + ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) + assert ids == "1\n" + micros = node.query( + f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", + settings=PRUNE_OFF, + ).strip() + assert micros == "4320000000000", micros + + eq_day = datetime64_literal("1970-02-20 00:00:00.000000000", scale=6, timezone=None) + query = f"SELECT id FROM {table} WHERE {column} = {eq_day} ORDER BY id" + assert pruned_files(query) == 0 + assert node.query(query, settings=PRUNE_ON) == "1\n" + + def check_minmax_far_future_us_upper_bound(node, table, column, pruned_files): """Iceberg `timestamp` with a far-future microsecond *upper* bound must not over-prune. diff --git a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py index 2c470a2eda21c..5a78ef2a69005 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py @@ -25,6 +25,7 @@ FILTER_LITERAL_IDS, FILTER_LITERALS, NEAR_EPOCH_NS_ROWS, + OVERLAP_BAND_NS_ROWS, PRUNE_OFF, PRUNE_ON, ROWS, @@ -32,6 +33,7 @@ check_minmax_far_future_us_upper_bound, check_minmax_near_epoch_ns_bounds_on_timestamp_schema, check_minmax_ns_bounds_on_timestamp_schema, + check_minmax_overlap_band_ns_bounds_on_timestamp_schema, check_ns_pruning, check_ns_pruning_datetime64_microseconds, patch_iceberg_table_long_to_timestamp_ns, @@ -288,6 +290,27 @@ def pruned_files(select_expression): check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) +def test_minmax_pruning_overlap_band_ns_bounds_on_timestamp_schema(started_cluster_iceberg_no_spark): + """Ns min/max in `[1.704e15, 1e16]` on Iceberg `timestamp` must not be decoded as microseconds.""" + node = started_cluster_iceberg_no_spark.instances["node1"] + s3_filename = _create_timestamp_table_with_ns_bounds( + started_cluster_iceberg_no_spark, "minmax_overlap_band_ns", rows=OVERLAP_BAND_NS_ROWS + ) + table = _iceberg_table(s3_filename) + + def pruned_files(select_expression): + return check_validity_and_get_prunned_files_general( + node, + "minmax_overlap_band_ns", + PRUNE_OFF, + PRUNE_ON, + "IcebergMinMaxIndexPrunedFiles", + select_expression, + ) + + check_minmax_overlap_band_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) + + @pytest.mark.parametrize( "upper_bound_us", [SPARK_TIMESTAMP_SENTINEL_US, DATETIME64_MAX_US], From da866b680aa66fabeb39da4d6a41d518664d1117 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 19 Oct 2026 14:30:32 +0200 Subject: [PATCH 16/22] Document Iceberg `DateTime64(9)` writes as `timestamp_ns`. The engine reference still said every `DateTime64` maps to `timestamp` and that `timestamp_ns` partitions cannot be written. Co-authored-by: Cursor --- src/Storages/ObjectStorage/registerStorageObjectStorage.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index 5073a40917e49..0ba3c0200fed6 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -1247,6 +1247,8 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `fixed(N)` | `FixedString(N)` | | | `decimal(P, S)` | `Decimal(P, S)` | | +Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and `DateTime64` with an explicit timezone and scale 9 as `timestamptz_ns`, but only when `iceberg_format_version` is 3 or higher. `DateTime` and `DateTime64` with scale 6 or less are stored as `timestamp`. Creating or altering an Iceberg table with `DateTime64` scale 9 on format version 1 or 2 raises an exception. + ### Complex types {#complex-types} | Iceberg type | ClickHouse type | @@ -1260,8 +1262,8 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t The data type mappings above apply to reads. The following limitations apply when ClickHouse creates or evolves Iceberg schemas and writes data: - ClickHouse cannot create an Iceberg schema containing `Bool`, `Decimal`, `FixedString`, `Int8`, `UInt8`, `Int16`, or `UInt16`, or add or modify a column to one of these types. The operation fails with an unsupported-type exception. This does not prevent inserting `Bool` or `Decimal` values into existing Iceberg columns. -- When ClickHouse creates or evolves an Iceberg schema, it maps every `DateTime` and `DateTime64` column to Iceberg `timestamp`, which has microsecond precision and no timezone. ClickHouse cannot generate `timestamptz`, `timestamp_ns`, or `timestamptz_ns` schema types, so higher precision and timezone semantics are not represented in the Iceberg schema. -- ClickHouse cannot write to an Iceberg table that uses a `decimal`, `fixed`, `timestamp_ns`, or `timestamptz_ns` column as a direct partition field. The operation fails with an unsupported-type exception. +- When ClickHouse creates or evolves an Iceberg schema, `DateTime` and `DateTime64` with scale 6 or less map to Iceberg `timestamp`. `DateTime64(9)` maps to `timestamp_ns`, and `DateTime64(9)` with an explicit timezone maps to `timestamptz_ns`, but only when `iceberg_format_version` is 3 or higher. Format versions 1 and 2 reject `DateTime64` scale 9. Microsecond `timestamptz` is still not generated. +- ClickHouse cannot write to an Iceberg table that uses a `decimal` or `fixed` column as a direct partition field. The operation fails with an unsupported-type exception. - For data files containing types whose bounds ClickHouse cannot serialize, including `Bool` and `Decimal`, ClickHouse omits all lower and upper column bounds from the Iceberg manifest entry. Column sizes and null counts are still included, and the data remains correct, but readers cannot use manifest-level min-max pruning for those files. ## Schema evolution {#schema-evolution} From d79dfa71a9733df55ea95adc4edd14fab5932ef1 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 19 Oct 2026 17:03:02 +0200 Subject: [PATCH 17/22] Write Iceberg `timestamp_ns` Avro data files as `long`. Scale-9 `DateTime64` was serialized as decimal `bytes`, so an Avro data file disagreed with `timestamp_ns` table metadata. Co-authored-by: Cursor --- .../Formats/Impl/AvroRowOutputFormat.cpp | 13 +++-- .../05385_iceberg_avro_timestamp_ns.reference | 3 ++ .../05385_iceberg_avro_timestamp_ns.sh | 50 +++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.reference create mode 100755 tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index e113e8e9882eb..69ef074300c7a 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -211,11 +211,18 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF auto schema = avro::LongSchema(); const auto & provided_type = assert_cast(*data_type); - if (provided_type.getScale() == 3) + const auto scale = provided_type.getScale(); + /// Iceberg `timestamp_ns` and `timestamptz_ns` are Avro `long` values of nanosecond ticks. + /// contrib Avro has no `timestamp-nanos` logical type, so an Iceberg data file stores a + /// plain `long`. The table metadata still records `timestamp_ns`. Other scales stay a + /// decimal `bytes` value, which is what generic Avro output already writes. + const bool iceberg_timestamp_ns = scale == 9 && column_mapper; + + if (scale == 3) schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MILLIS)); - else if (provided_type.getScale() == 6) + else if (scale == 6) schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MICROS)); - else + else if (!iceberg_timestamp_ns) return createDecimalSchemaWithSerializeFn(data_type); return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) diff --git a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.reference b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.reference new file mode 100644 index 0000000000000..caa33f8652edf --- /dev/null +++ b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.reference @@ -0,0 +1,3 @@ +2020-01-02 03:04:05.123456789 +Int64 +Decimal diff --git a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh new file mode 100755 index 0000000000000..e69e3eb8dae05 --- /dev/null +++ b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# - no-fasttest: requires `IcebergLocal` (USE_AVRO). +# - no-replicated-database: `IcebergLocal` is non-replicated. + +# Iceberg `timestamp_ns` is an Avro `long`. A scale-9 `DateTime64` used to be written as +# decimal `bytes`, so the data file disagreed with the table metadata. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE_PATH="${CLICKHOUSE_USER_FILES_UNIQUE}/t_ice/" +PLAIN_AVRO="${CLICKHOUSE_USER_FILES_UNIQUE}/plain.avro" +rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}" +mkdir -p "${TABLE_PATH}" +trap 'rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE}"' EXIT + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query " +CREATE TABLE t (ts DateTime64(9)) +ENGINE = IcebergLocal('${TABLE_PATH}', 'Avro') +SETTINGS iceberg_format_version = 3 +" + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --async_insert=0 --query " +INSERT INTO t VALUES ('2020-01-02 03:04:05.123456789') +" + +${CLICKHOUSE_CLIENT} --query "SELECT ts FROM t" + +DATA_FILE=$(find "${TABLE_PATH}/data" -name 'data-*.avro' -type f | head -1) +if [ -z "${DATA_FILE}" ]; then + echo "Iceberg Avro data file not found" + exit 1 +fi +RELATIVE_DATA_FILE=${DATA_FILE#"${CLICKHOUSE_USER_FILES}/"} + +# The data file is a plain Avro `long`, which schema inference reads as `Int64`. +${CLICKHOUSE_CLIENT} --query "SELECT toTypeName(ts) FROM file('${RELATIVE_DATA_FILE}', 'Avro')" + +# Generic Avro output is unchanged: scale 9 stays a decimal. +RELATIVE_PLAIN_AVRO=${PLAIN_AVRO#"${CLICKHOUSE_USER_FILES}/"} +${CLICKHOUSE_CLIENT} --query " +INSERT INTO FUNCTION file('${RELATIVE_PLAIN_AVRO}', 'Avro') +SELECT toDateTime64('2020-01-02 03:04:05.123456789', 9) AS ts +" +${CLICKHOUSE_CLIENT} --query " +SELECT if(toTypeName(ts) LIKE 'Decimal%', 'Decimal', toTypeName(ts)) +FROM file('${RELATIVE_PLAIN_AVRO}', 'Avro') +" From eab752b9543fb9a5db44a7a586b2ddf9d622415e Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 19 Oct 2026 17:06:41 +0200 Subject: [PATCH 18/22] Drop the direct edit of the generated Iceberg reference page. The `DateTime64(9)` write mapping stays in the engine registration. The autogenerated docs page is updated by the documentation workflow. Co-authored-by: Cursor --- docs/reference/engines/table-engines/integrations/iceberg.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index 9e73b43a07713..7a05dab165b5d 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -98,8 +98,6 @@ The following table shows how Iceberg data types are mapped to ClickHouse data t | `fixed(N)` | `FixedString(N)` | | | `decimal(P, S)` | `Decimal(P, S)` | | -Writes use the inverse mapping: `DateTime64(9)` is stored as `timestamp_ns`, and `DateTime64` with an explicit timezone and scale 9 as `timestamptz_ns`, but only when `iceberg_format_version` is 3 or higher. `DateTime` and `DateTime64` with scale 6 or less are stored as `timestamp`. Creating or altering an Iceberg table with `DateTime64` scale 9 on format version 1 or 2 raises an exception. - ### Complex types {#complex-types} | Iceberg type | ClickHouse type | From 956936d783cd87aab746c704d1c948556f4dae6f Mon Sep 17 00:00:00 2001 From: blau-ai Date: Tue, 20 Oct 2026 16:26:29 +0000 Subject: [PATCH 19/22] Fix DateTime64 identity-partition read overflow and 05385 stderr getIdentityPartitionColumnValues passed the raw Avro `long` nanosecond ticks of a timestamp_ns / timestamptz_ns identity partition through convertFieldToTypeOrThrow, which treats an integer target of DateTime64 as a whole number of seconds and overflows while rescaling (DECIMAL_OVERFLOW on any read of such a table). Wrap the ticks into a DecimalField at the column scale directly, mirroring the existing pruning path (wrapIntegerPartitionValuesAsDateTime64). 05385_iceberg_avro_timestamp_ns created and read back a format-version-3 table; reading it logs a benign 'parsed using v2 specification' warning (v3 metadata has no top-level 'schema' field for the v1 parser), which lands on stderr and clickhouse-test turns into a failure. Suppress server logs in the test, as the other Iceberg .sh tests do. Co-Authored-By: Claude Opus 4.8 --- .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 22 ++++++++++++++++++- .../05385_iceberg_avro_timestamp_ns.sh | 4 ++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index d9fe54843110e..0a3774fb40baf 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -2121,7 +2121,27 @@ PartitionColumnValues getIdentityPartitionColumnValues( if (!name_and_type.has_value()) continue; - Field value = convertFieldToTypeOrThrow(partition_key_value[partition_field.tuple_index], *name_and_type->type); + const Field & raw_value = partition_key_value[partition_field.tuple_index]; + /// Avro has no `timestamp-nanos` logical type, so `timestamp_ns` / `timestamptz_ns` (and + /// similarly `Time64`) identity partition values arrive as a raw Int64 of ticks at the + /// column's scale. `convertFieldToType` would treat that integer as whole seconds and + /// overflow while rescaling, so wrap the ticks directly instead (mirrors the pruning path). + Field value; + if (!raw_value.isNull() && raw_value.getType() == Field::Types::Int64) + { + auto target_type = removeNullable(name_and_type->type); + WhichDataType which(target_type); + if (which.isDateTime64()) + value = DecimalField(DateTime64(raw_value.safeGet()), getDecimalScale(*target_type)); + else if (which.isTime64()) + value = DecimalField(Time64(raw_value.safeGet()), getDecimalScale(*target_type)); + else + value = convertFieldToTypeOrThrow(raw_value, *name_and_type->type); + } + else + { + value = convertFieldToTypeOrThrow(raw_value, *name_and_type->type); + } if (value.isNull()) continue; diff --git a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh index e69e3eb8dae05..0708f07210033 100755 --- a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh +++ b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh @@ -7,6 +7,10 @@ # decimal `bytes`, so the data file disagreed with the table metadata. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# Reading a ClickHouse-written format-version-3 table logs a benign "parsed using v2 +# specification" warning (v3 metadata has no top-level `schema` for the v1 parse to use). +# That warning would land on stderr, which clickhouse-test turns into a failure. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="none" # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh From 4d1dac23155bdc67f8a682009ebe80cc18c985ee Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:30:13 +0200 Subject: [PATCH 20/22] Restore Iceberg `timestamp` min/max pruning for microsecond bounds `deserializeDateTime64FromBinaryRepr` dropped every scale-6 bound with `|ticks| <= 1e18`, so ordinary Iceberg `timestamp` (`DateTime64(6)`) columns lost min/max pruning (`test_minmax_pruning`, `test_writes_statistics_by_minmax_pruning`). Read `|ticks| <= 1e16` as microseconds again, as the spec requires; keep failing open in `(1e16, 1e18]` and converting nanoseconds above `1e18`. Drop the near-epoch and overlap-band nanosecond-on-`timestamp` tests: those bounds are indistinguishable from spec-correct microseconds. https://github.com/Altinity/ClickHouse/pull/2610#issuecomment-6295808375 --- .../Iceberg/IcebergFieldParseHelpers.cpp | 11 +-- .../gtest_iceberg_field_parse_helpers.cpp | 39 +++++------ .../helpers/iceberg_timestamp_ns.py | 67 ------------------- .../test_pruning_datetime64_nanoseconds.py | 46 ------------- 4 files changed, 25 insertions(+), 138 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp index 13bc6f15f7692..e062057a99c6a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergFieldParseHelpers.cpp @@ -28,6 +28,9 @@ namespace Iceberg namespace { +/// 1e16 µs is ~year 2286. Spark sentinels such as `9999-12-31` (`~2.53e17` µs) +/// are spec-correct microseconds above this. +constexpr Int64 microseconds_ambiguous_threshold = 10'000'000'000'000'000LL; /// 1e18 ns is ~year 2001; customer 2026 ns bounds are `~1.76e18`. constexpr Int64 nanoseconds_lower_threshold = 1'000'000'000'000'000'000LL; constexpr Int64 nanoseconds_per_microsecond = 1000; @@ -59,9 +62,9 @@ Int64 nanosecondsToMicrosecondsForBound(Int64 nanoseconds, bool lower_bound) /// Iceberg timestamps are 8-byte little-endian (spec Appendix D). /// Some writers store nanosecond stats on Iceberg `timestamp` (`DateTime64(6)`). -/// Convert only when `|ticks| > 1e18`. Otherwise fail open: `|ticks| <= 1e16` -/// collides near-epoch nanoseconds (~115 days) with 1970–2286 microseconds, and -/// `(1e16, 1e18]` collides Spark `9999-12-31` sentinels with 1970–2001 nanoseconds. +/// `|ticks| <= 1e16` is kept as microseconds, as the spec requires. `|ticks| > 1e18` +/// is converted from nanoseconds. `(1e16, 1e18]` fails open: it collides Spark +/// `9999-12-31` sentinels with 1970–2001 nanoseconds. std::optional deserializeDateTime64FromBinaryRepr(const std::string & str, const IDataType & type, bool lower_bound) { if (str.size() != sizeof(Int64)) @@ -70,7 +73,7 @@ std::optional deserializeDateTime64FromBinaryRepr(const std::string & str Int64 unscaled = unalignedLoadLittleEndian(str.data()); const UInt32 scale = getDecimalScale(type); - if (scale == 6) + if (scale == 6 && magnitudeExceeds(unscaled, microseconds_ambiguous_threshold)) { if (!magnitudeExceeds(unscaled, nanoseconds_lower_threshold)) return std::nullopt; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp index 069a01bed01c1..5286d5217c3fa 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_field_parse_helpers.cpp @@ -29,32 +29,29 @@ Int64 decimal64Ticks(const Field & field) } -TEST(IcebergFieldParseHelpers, NearEpochNanosecondDateTime64BoundsFailOpen) +TEST(IcebergFieldParseHelpers, MicrosecondDateTime64BoundsAreKept) { auto type = std::make_shared(6); - /// One day after the epoch as nanoseconds: kept as µs this is ~1972, not 1970-01-02. - constexpr Int64 one_day_ns = 86'400'000'000'000LL; - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_day_ns), type, true).has_value()); - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_day_ns), type, false).has_value()); - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(-one_day_ns), type, true).has_value()); -} - -TEST(IcebergFieldParseHelpers, AmbiguousScaleDateTime64BoundsFailOpen) -{ - auto type = std::make_shared(6); - /// 2024-01-01 UTC µs is also ~19.7 days as ns. No integer test splits those scales. - constexpr Int64 us_2024_01_01 = 1'704'067'200'000'000LL; - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_01_01), type, true).has_value()); - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(-us_2024_01_01), type, true).has_value()); + /// 2024-02-20 10:00:00 UTC as microseconds — spec-correct Iceberg `timestamp`. + constexpr Int64 us_2024_02_20 = 1'708'423'200'000'000LL; + auto lower = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, true); + auto upper = deserializeFieldFromBinaryRepr(littleEndianInt64(us_2024_02_20), type, false); + ASSERT_TRUE(lower.has_value()); + ASSERT_TRUE(upper.has_value()); + EXPECT_EQ(decimal64Ticks(*lower), us_2024_02_20); + EXPECT_EQ(decimal64Ticks(*upper), us_2024_02_20); - /// 50 days after the epoch as ns (`4.32e15`) sits in the same band as 2024–2286 µs. - constexpr Int64 fifty_days_ns = 4'320'000'000'000'000LL; - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(fifty_days_ns), type, true).has_value()); - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(fifty_days_ns), type, false).has_value()); + /// 1960-01-01 00:00:00 UTC as microseconds. + constexpr Int64 us_1960_01_01 = -315'619'200'000'000LL; + auto negative = deserializeFieldFromBinaryRepr(littleEndianInt64(us_1960_01_01), type, true); + ASSERT_TRUE(negative.has_value()); + EXPECT_EQ(decimal64Ticks(*negative), us_1960_01_01); - /// Band upper edge: 1e16 ns is ~1970-04-26, 1e16 µs is ~year 2286. + /// Upper edge of the microsecond band (~year 2286). constexpr Int64 one_e16 = 10'000'000'000'000'000LL; - EXPECT_FALSE(deserializeFieldFromBinaryRepr(littleEndianInt64(one_e16), type, true).has_value()); + auto edge = deserializeFieldFromBinaryRepr(littleEndianInt64(one_e16), type, false); + ASSERT_TRUE(edge.has_value()); + EXPECT_EQ(decimal64Ticks(*edge), one_e16); } TEST(IcebergFieldParseHelpers, ProvenNanosecondDateTime64BoundsAreConverted) diff --git a/tests/integration/helpers/iceberg_timestamp_ns.py b/tests/integration/helpers/iceberg_timestamp_ns.py index 3922f8d37d891..351b3cc9eeb1c 100644 --- a/tests/integration/helpers/iceberg_timestamp_ns.py +++ b/tests/integration/helpers/iceberg_timestamp_ns.py @@ -223,73 +223,6 @@ def _utc_datetime_to_micros(dt): ) -# One day after the Unix epoch as nanoseconds. Kept as microseconds this is ~1972, -# so a predicate on 1970-01-02 over-prunes unless the decoder fails open. -NEAR_EPOCH_NS_ROWS = [ - (86_400_000_000_000, 1, "1970-01-02 00:00:00.000000000"), -] - -# 50 days after the epoch as nanoseconds (`4.32e15`). Same integer as ~2106 µs, -# so a predicate on 1970-02-20 over-prunes unless the decoder fails open. -OVERLAP_BAND_NS_ROWS = [ - (4_320_000_000_000_000, 1, "1970-02-20 00:00:00.000000000"), -] - - -def check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): - """Iceberg `timestamp` whose min/max bytes are near-epoch nanoseconds must not over-prune. - - One day after the epoch is `8.64e13` ns (`|ticks| <= 1e16`). The inverted-range - check does not fire when both bounds stay in that band. - """ - type_name = node.query( - f"SELECT toTypeName({column}) FROM {table} LIMIT 1", - settings=PRUNE_OFF, - ) - assert "DateTime64(6" in type_name, type_name - assert "DateTime64(9" not in type_name, type_name - - ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) - assert ids == "1\n" - micros = node.query( - f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", - settings=PRUNE_OFF, - ).strip() - assert micros == "86400000000", micros - - eq_epoch_day = datetime64_literal("1970-01-02 00:00:00.000000000", scale=6, timezone=None) - query = f"SELECT id FROM {table} WHERE {column} = {eq_epoch_day} ORDER BY id" - assert pruned_files(query) == 0 - assert node.query(query, settings=PRUNE_ON) == "1\n" - - -def check_minmax_overlap_band_ns_bounds_on_timestamp_schema(node, table, column, pruned_files): - """Iceberg `timestamp` whose min/max bytes are leftover-band nanoseconds must not over-prune. - - 50 days after the epoch is `4.32e15` ns, which is also a 2024–2286 microsecond - timestamp. The decoder cannot tell the scales apart and must fail open. - """ - type_name = node.query( - f"SELECT toTypeName({column}) FROM {table} LIMIT 1", - settings=PRUNE_OFF, - ) - assert "DateTime64(6" in type_name, type_name - assert "DateTime64(9" not in type_name, type_name - - ids = node.query(f"SELECT id FROM {table} ORDER BY id", settings=PRUNE_OFF) - assert ids == "1\n" - micros = node.query( - f"SELECT toUnixTimestamp64Micro({column}) FROM {table} WHERE id = 1", - settings=PRUNE_OFF, - ).strip() - assert micros == "4320000000000", micros - - eq_day = datetime64_literal("1970-02-20 00:00:00.000000000", scale=6, timezone=None) - query = f"SELECT id FROM {table} WHERE {column} = {eq_day} ORDER BY id" - assert pruned_files(query) == 0 - assert node.query(query, settings=PRUNE_ON) == "1\n" - - def check_minmax_far_future_us_upper_bound(node, table, column, pruned_files): """Iceberg `timestamp` with a far-future microsecond *upper* bound must not over-prune. diff --git a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py index 5a78ef2a69005..b6e62d970a4f5 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_pruning_datetime64_nanoseconds.py @@ -24,16 +24,12 @@ DATETIME64_MAX_US, FILTER_LITERAL_IDS, FILTER_LITERALS, - NEAR_EPOCH_NS_ROWS, - OVERLAP_BAND_NS_ROWS, PRUNE_OFF, PRUNE_ON, ROWS, SPARK_TIMESTAMP_SENTINEL_US, check_minmax_far_future_us_upper_bound, - check_minmax_near_epoch_ns_bounds_on_timestamp_schema, check_minmax_ns_bounds_on_timestamp_schema, - check_minmax_overlap_band_ns_bounds_on_timestamp_schema, check_ns_pruning, check_ns_pruning_datetime64_microseconds, patch_iceberg_table_long_to_timestamp_ns, @@ -269,48 +265,6 @@ def pruned_files(select_expression): check_minmax_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) -def test_minmax_pruning_near_epoch_ns_bounds_on_timestamp_schema(started_cluster_iceberg_no_spark): - """Near-epoch ns min/max on Iceberg `timestamp` must not be decoded as microseconds.""" - node = started_cluster_iceberg_no_spark.instances["node1"] - s3_filename = _create_timestamp_table_with_ns_bounds( - started_cluster_iceberg_no_spark, "minmax_near_epoch_ns", rows=NEAR_EPOCH_NS_ROWS - ) - table = _iceberg_table(s3_filename) - - def pruned_files(select_expression): - return check_validity_and_get_prunned_files_general( - node, - "minmax_near_epoch_ns", - PRUNE_OFF, - PRUNE_ON, - "IcebergMinMaxIndexPrunedFiles", - select_expression, - ) - - check_minmax_near_epoch_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) - - -def test_minmax_pruning_overlap_band_ns_bounds_on_timestamp_schema(started_cluster_iceberg_no_spark): - """Ns min/max in `[1.704e15, 1e16]` on Iceberg `timestamp` must not be decoded as microseconds.""" - node = started_cluster_iceberg_no_spark.instances["node1"] - s3_filename = _create_timestamp_table_with_ns_bounds( - started_cluster_iceberg_no_spark, "minmax_overlap_band_ns", rows=OVERLAP_BAND_NS_ROWS - ) - table = _iceberg_table(s3_filename) - - def pruned_files(select_expression): - return check_validity_and_get_prunned_files_general( - node, - "minmax_overlap_band_ns", - PRUNE_OFF, - PRUNE_ON, - "IcebergMinMaxIndexPrunedFiles", - select_expression, - ) - - check_minmax_overlap_band_ns_bounds_on_timestamp_schema(node, table, "value", pruned_files) - - @pytest.mark.parametrize( "upper_bound_us", [SPARK_TIMESTAMP_SENTINEL_US, DATETIME64_MAX_US], From 7555664f5d4a9d0e6fe7abc8e19d254dcd5da697 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:30:14 +0200 Subject: [PATCH 21/22] Use Iceberg format version 3 for the `DateTime64(9)` arm of 04652 `DateTime64(9)` maps to `timestamp_ns`, which requires format version 3, so `CREATE TABLE` at the default version 2 threw and broke the batched setup. The arm now round-trips full nanoseconds. https://github.com/Altinity/ClickHouse/pull/2610#issuecomment-6295808375 --- ...erg_mv_target_stale_schema_write.reference | 2 +- ...52_iceberg_mv_target_stale_schema_write.sh | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.reference b/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.reference index 50cd3438841f0..d062fd1931f87 100644 --- a/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.reference +++ b/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.reference @@ -1,7 +1,7 @@ datetime: 2020-01-02 03:04:05.000000 datetime64_3: 2020-01-02 03:04:05.123000 datetime64_0: 2020-01-02 03:04:05.000000 -datetime64_9: 2020-01-02 03:04:05.123456 +datetime64_9: 2020-01-02 03:04:05.123456789 array_datetime: ['2020-01-02 03:04:05.000000'] datetime64_6_ctl: 2020-01-02 03:04:05.123456 parquet_ctl: 2020-01-02 03:04:05.000000 diff --git a/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.sh b/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.sh index d404e2b4a0164..4c22ce9148939 100755 --- a/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.sh +++ b/tests/queries/0_stateless/04652_iceberg_mv_target_stale_schema_write.sh @@ -21,23 +21,24 @@ mkdir -p "${BASE_DIR}" CLIENT="${CLICKHOUSE_CLIENT} --session_timezone UTC --allow_insert_into_iceberg=1 --async_insert=0" DB="${CLICKHOUSE_DATABASE}" -# arm_name|column type|write format|inserted literal. datetime64_0 and datetime64_9 used to -# produce a file that could not be read back at all ("Cannot insert Avro decimal with scale N -# ... scale 6"). The last two arms are controls: correct both before and after the fix. -ARMS="datetime|DateTime|Avro|'2020-01-02 03:04:05' -datetime64_3|DateTime64(3)|Avro|'2020-01-02 03:04:05.123' -datetime64_0|DateTime64(0)|Avro|'2020-01-02 03:04:05' -datetime64_9|DateTime64(9)|Avro|'2020-01-02 03:04:05.123456789' -array_datetime|Array(DateTime)|Avro|[toDateTime('2020-01-02 03:04:05')] -datetime64_6_ctl|DateTime64(6)|Avro|'2020-01-02 03:04:05.123456' -parquet_ctl|DateTime|Parquet|'2020-01-02 03:04:05'" +# arm_name|column type|write format|Iceberg format version|inserted literal. datetime64_0 and +# datetime64_9 used to produce a file that could not be read back at all ("Cannot insert Avro +# decimal with scale N ... scale 6"). `DateTime64(9)` maps to `timestamp_ns`, which requires +# format version 3. The last two arms are controls: correct both before and after the fix. +ARMS="datetime|DateTime|Avro|2|'2020-01-02 03:04:05' +datetime64_3|DateTime64(3)|Avro|2|'2020-01-02 03:04:05.123' +datetime64_0|DateTime64(0)|Avro|2|'2020-01-02 03:04:05' +datetime64_9|DateTime64(9)|Avro|3|'2020-01-02 03:04:05.123456789' +array_datetime|Array(DateTime)|Avro|2|[toDateTime('2020-01-02 03:04:05')] +datetime64_6_ctl|DateTime64(6)|Avro|2|'2020-01-02 03:04:05.123456' +parquet_ctl|DateTime|Parquet|2|'2020-01-02 03:04:05'" ARM_SETUP="" ARM_DROP="" -while IFS='|' read -r arm col_type format value; do +while IFS='|' read -r arm col_type format version value; do ARM_SETUP="${ARM_SETUP} CREATE TABLE src_${arm}_${DB} (c0 ${col_type}) ENGINE = MergeTree ORDER BY tuple(); - CREATE TABLE tgt_${arm}_${DB} (c0 ${col_type}) ENGINE = IcebergLocal('${BASE_DIR}/${arm}/', '${format}'); + CREATE TABLE tgt_${arm}_${DB} (c0 ${col_type}) ENGINE = IcebergLocal('${BASE_DIR}/${arm}/', '${format}') SETTINGS iceberg_format_version = ${version}; CREATE MATERIALIZED VIEW mv_${arm}_${DB} TO tgt_${arm}_${DB} AS SELECT c0 FROM src_${arm}_${DB}; INSERT INTO src_${arm}_${DB} VALUES (${value});" ARM_DROP="${ARM_DROP} From 347bb563b9bef42983440ced3ba5f558e1107709 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 21 Oct 2026 11:46:38 +0200 Subject: [PATCH 22/22] Parse Iceberg v3 table schemas with the v2 method v3 keeps the `schemas` array and has no top-level `schema`. Trying the v1 parser first logged a false spec-violation warning on every ClickHouse-written v3 read, including `timestamp_ns`. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp | 4 +++- tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 17a2a27a726c2..1dee5de337d86 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -398,7 +398,9 @@ Int32 IcebergMetadata::parseTableSchema( { const auto format_version = metadata_object->getValue(f_format_version); - if (format_version == 2) + /// v2 and v3 (and later) store schemas in the `schemas` array. Only v1 has a top-level `schema`. + /// Using the v1 parser first for v3 looks like a spec violation and logs a false warning. + if (format_version >= 2) { auto [schema, current_schema_id] = parseTableSchemaV2Method(metadata_object); schema_processor.addIcebergTableSchema(schema, context_); diff --git a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh index 0708f07210033..e69e3eb8dae05 100755 --- a/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh +++ b/tests/queries/0_stateless/05385_iceberg_avro_timestamp_ns.sh @@ -7,10 +7,6 @@ # decimal `bytes`, so the data file disagreed with the table metadata. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# Reading a ClickHouse-written format-version-3 table logs a benign "parsed using v2 -# specification" warning (v3 metadata has no top-level `schema` for the v1 parse to use). -# That warning would land on stderr, which clickhouse-test turns into a failure. -CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="none" # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh