From cba62d6caad51925b3ec2caf46a034c7d109f645 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 4 Sep 2026 13:30:55 +0200 Subject: [PATCH 01/52] Add a regression test for global RIGHT JOIN on Distributed tables The expected result matches initiator-side join semantics; the test currently fails because shards emit extra unmatched right-side rows. Related: https://github.com/Altinity/ClickHouse/issues/1664 Co-authored-by: Cursor --- ...58_distributed_global_right_join.reference | 36 ++++++++++++ .../05058_distributed_global_right_join.sql | 55 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/queries/0_stateless/05058_distributed_global_right_join.reference create mode 100644 tests/queries/0_stateless/05058_distributed_global_right_join.sql diff --git a/tests/queries/0_stateless/05058_distributed_global_right_join.reference b/tests/queries/0_stateless/05058_distributed_global_right_join.reference new file mode 100644 index 0000000000000..3277112362bcd --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_global_right_join.reference @@ -0,0 +1,36 @@ +initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +distributed_product_mode +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +explicit_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C diff --git a/tests/queries/0_stateless/05058_distributed_global_right_join.sql b/tests/queries/0_stateless/05058_distributed_global_right_join.sql new file mode 100644 index 0000000000000..b43495a419e74 --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_global_right_join.sql @@ -0,0 +1,55 @@ +-- Tags: distributed + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05058; +DROP TABLE IF EXISTS right_local_05058; +DROP TABLE IF EXISTS left_distributed_05058; +DROP TABLE IF EXISTS right_distributed_05058; + +CREATE TABLE left_local_05058 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05058 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE left_distributed_05058 AS left_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), left_local_05058); + +CREATE TABLE right_distributed_05058 AS right_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05058); + +INSERT INTO left_local_05058 VALUES (1, 'a'), (2, 'b'), (4, 'd'); +INSERT INTO right_local_05058 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +-- The subqueries force the join to run on the initiator and define the correct result. +SELECT 'initiator'; +SELECT * +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN (SELECT * FROM right_distributed_05058) AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- The global join must produce the same rows and preserve the original column order. +SELECT 'distributed_product_mode'; +SELECT * +FROM left_distributed_05058 AS l +RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +SETTINGS distributed_product_mode = 'global' +FORMAT TSVWithNames; + +SELECT 'explicit_global'; +SELECT * +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +DROP TABLE left_distributed_05058; +DROP TABLE right_distributed_05058; +DROP TABLE left_local_05058; +DROP TABLE right_local_05058; From 3ec16e41cc603914946dc85bd40e9f4a92fff9ef Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 4 Sep 2026 14:03:46 +0200 Subject: [PATCH 02/52] Rewrite global RIGHT JOIN of Distributed tables as LEFT JOIN Broadcast the original left table and keep the preserved right side sharded, so unmatched right rows are not duplicated on every shard. Related: https://github.com/Altinity/ClickHouse/issues/1664 Co-authored-by: Cursor --- src/Planner/PlannerJoinTree.cpp | 48 ++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index dd4ef0a462a12..101b81a412d7a 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -104,6 +104,7 @@ namespace Setting extern const SettingsMap additional_table_filters; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; extern const SettingsBool async_socket_for_remote; + extern const SettingsDistributedProductMode distributed_product_mode; extern const SettingsBool empty_result_for_aggregation_by_empty_set; extern const SettingsBool enable_unaligned_array_join; extern const SettingsBool join_use_nulls; @@ -129,6 +130,7 @@ namespace Setting extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree; extern const SettingsUInt64 parallel_replicas_min_number_of_rows_per_replica; extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsBool prefer_global_in_and_join; extern const SettingsBool optimize_move_to_prewhere; extern const SettingsBool optimize_move_to_prewhere_if_final; extern const SettingsBool use_concurrency_control; @@ -155,6 +157,47 @@ namespace ErrorCodes namespace { +const StorageDistributed * getDistributedStorageFromTableExpression(const QueryTreeNodePtr & table_expression) +{ + const auto * table_node = table_expression->as(); + if (!table_node) + return nullptr; + + return typeid_cast(table_node->getStorage().get()); +} + +void tryRewriteGlobalRightJoinAsLeftJoin(QueryNode & query_node, const ContextPtr & context) +{ + auto * join_node = query_node.getJoinTree()->as(); + if (!join_node + || join_node->getKind() != JoinKind::Right + || join_node->getStrictness() != JoinStrictness::All + || !join_node->isOnJoinExpression()) + return; + + const auto & settings = context->getSettingsRef(); + const auto distributed_product_mode = settings[Setting::distributed_product_mode]; + const bool is_global = join_node->getLocality() == JoinLocality::Global + || distributed_product_mode == DistributedProductMode::GLOBAL + || (distributed_product_mode != DistributedProductMode::LOCAL && settings[Setting::prefer_global_in_and_join]); + if (!is_global) + return; + + const auto * left_storage = getDistributedStorageFromTableExpression(join_node->getLeftTableExpression()); + const auto * right_storage = getDistributedStorageFromTableExpression(join_node->getRightTableExpression()); + if (!left_storage || !right_storage || left_storage->getShardCount() < 2 || right_storage->getShardCount() < 2) + return; + + /** A `GLOBAL RIGHT JOIN` cannot run with the left table sharded and the right table broadcast. + * Every shard would independently emit unmatched rows from the complete right table. + * Swap the inputs before choosing the `Distributed` table that will execute the query, so the + * preserved side stays sharded and the original left table is broadcast instead. + * Projection nodes are already resolved and keep the user-visible column order unchanged. + */ + std::swap(join_node->getLeftTableExpression(), join_node->getRightTableExpression()); + join_node->setKind(JoinKind::Left); +} + /// Check if current user has privileges to SELECT columns from table /// Throws an exception if access to any column from `column_names` is not granted /// If `column_names` is empty, check access to any columns and return names of accessible columns @@ -1976,7 +2019,10 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, const ColumnIdentifierSet & outer_scope_columns, PlannerContextPtr & planner_context) { - const QueryTreeNodePtr & join_tree_node = query_node->as().getJoinTree(); + auto & query_node_typed = query_node->as(); + tryRewriteGlobalRightJoinAsLeftJoin(query_node_typed, planner_context->getQueryContext()); + + const QueryTreeNodePtr & join_tree_node = query_node_typed.getJoinTree(); auto table_expressions_stack = buildTableExpressionsStack(join_tree_node); size_t table_expressions_stack_size = table_expressions_stack.size(); bool is_single_table_expression = table_expressions_stack_size == 1; From 5ca17e59261261cb4e7f5234a4540e4d68b7014f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 4 Sep 2026 15:20:08 +0200 Subject: [PATCH 03/52] Skip wrapping IStorageCluster on GLOBAL INNER/LEFT joins After #94748 the planner always wrapped the left cluster source, so object_storage_cluster_join_mode='global' never saw the JOIN and ran on the initiator. Keep wrapping for allow/local and for RIGHT/FULL to avoid duplicated unmatched rows. Co-authored-by: Cursor --- src/Planner/PlannerJoinTree.cpp | 33 +++++++++++++------ tests/integration/test_s3_cluster/test.py | 39 +++++++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 101b81a412d7a..5d42f9a69662e 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -117,6 +117,7 @@ namespace Setting extern const SettingsUInt64 max_parser_depth; extern const SettingsUInt64 max_query_size; extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsObjectStorageClusterJoinMode object_storage_cluster_join_mode; extern const SettingsFloat max_streams_to_max_threads_ratio; extern const SettingsMaxThreads max_threads; extern const SettingsUInt64 max_threads_min_free_memory_per_thread; @@ -2042,6 +2043,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, size_t joins_count = 0; bool is_full_join = false; + bool is_right_join = false; bool is_global_join = false; bool is_right_join_with_remote_table = false; int first_join_pos = -1; @@ -2070,6 +2072,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, if (join_kind == JoinKind::Full) is_full_join = true; + if (join_kind == JoinKind::Right) + is_right_join = true; if (join_node.getLocality() == JoinLocality::Global) is_global_join = true; @@ -2140,21 +2144,32 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, */ auto left_table_expression = table_expressions_stack.front(); - /** If the leftmost table uses IStorageCluster (e.g., s3Cluster, hdfsCluster) - * and there are multiple tables (indicating a JOIN), we must wrap it in a subquery. - * This prevents IStorageCluster from receiving the full JOIN query, which it cannot handle. + /** If the leftmost table uses `IStorageCluster` (e.g., `s3Cluster`, `hdfsCluster`) + * and there are multiple tables (indicating a JOIN), we normally wrap it in a subquery. + * This prevents `IStorageCluster` from receiving the full JOIN query, which it cannot handle. * - * IStorageCluster is a simple storage that just forwards queries to remote nodes. - * Unlike StorageDistributed, it cannot decompose and handle JOINs across multiple tables, + * `IStorageCluster` is a simple storage that just forwards queries to remote nodes. + * Unlike `StorageDistributed`, it cannot decompose and handle JOINs across multiple tables, * because remote nodes don't have access to other tables in the JOIN. * - * StorageDistributed has sophisticated query planning logic to handle JOINs and should + * With `object_storage_cluster_join_mode = 'global'`, `IStorageCluster` must receive a + * supported full query so it can materialize the right side and send a `GLOBAL JOIN` to + * remote nodes. `RIGHT JOIN` and `FULL JOIN` stay on the initiator because broadcasting their + * preserved right side would make every shard emit the same unmatched rows. + * + * `StorageDistributed` has sophisticated query planning logic to handle JOINs and should * NOT be wrapped (wrapping breaks tests like 03577_server_constant_folding). */ bool should_wrap_left_table = false; - bool has_multiple_tables = table_expressions_stack.size() > 1; - - if (has_multiple_tables) + const bool has_multiple_tables = table_expressions_stack.size() > 1; + const bool use_global_join + = planner_context->getQueryContext()->getSettingsRef()[Setting::object_storage_cluster_join_mode] + == ObjectStorageClusterJoinMode::GLOBAL + && joins_count > 0 + && !is_right_join + && !is_full_join; + + if (has_multiple_tables && !use_global_join) { // Get the actual storage to check its type auto * table_node = left_table_expression->as(); diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index ff13f493571d4..6f241b0b1adb3 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -1078,6 +1078,45 @@ def test_remote_no_hedged(started_cluster): assert TSV(pure_s3) == TSV(s3_distributed) +def test_global_join_executes_on_shards(started_cluster): + node = started_cluster.instances["s0_0_0"] + + node.query("DROP TABLE IF EXISTS join_table SYNC") + node.query( + """ + CREATE TABLE join_table ( + id UInt32, + name String + ) ENGINE=Memory() + """ + ) + + query = f""" + SELECT t1.name, t2.name FROM + s3Cluster('cluster_simple', + 'http://minio1:9001/root/data/{{clickhouse,database}}/*', 'minio', '{minio_secret_key}', 'CSV', + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') AS t1 + JOIN join_table AS t2 ON t1.value = t2.id + """ + + global_pipeline = node.query( + f"EXPLAIN PIPELINE {query} SETTINGS object_storage_cluster_join_mode='global'" + ) + assert "JoiningTransform" not in global_pipeline + + allow_pipeline = node.query( + f"EXPLAIN PIPELINE {query} SETTINGS object_storage_cluster_join_mode='allow'" + ) + assert "JoiningTransform" in allow_pipeline + + for join_kind in ("RIGHT", "FULL"): + outer_join_query = query.replace("JOIN join_table", f"{join_kind} JOIN join_table") + outer_join_pipeline = node.query( + f"EXPLAIN PIPELINE {outer_join_query} SETTINGS object_storage_cluster_join_mode='global'" + ) + assert "JoiningTransform" in outer_join_pipeline + + @pytest.mark.parametrize("join_mode", ["local", "global"]) def test_joins(started_cluster, join_mode): node = started_cluster.instances["s0_0_0"] From 3df7433ba7243005eb6c91d010130a6e70a8d825 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 8 Sep 2026 14:41:09 +0200 Subject: [PATCH 04/52] Widen the global RIGHT JOIN rewrite of Distributed tables Port of the review fixes made for the `master` pull request https://github.com/ClickHouse/ClickHouse/pull/118611 `tryRewriteGlobalRightJoinAsLeftJoin` only recognised a narrow shape of the query it was meant to fix, so a `GLOBAL RIGHT JOIN` whose left table is a sharded `Distributed` table still emitted the rows preserved from the broadcast side once per shard in these cases: * `USING` and `NATURAL` joins were skipped, because only `ON` was accepted. A `USING` key stores its sides positionally, so the key pairs are now swapped together with the table expressions, and a key that does not hold a plain column per side leaves the query alone. * The rewrite required the right side to be a sharded `Distributed` table too. Only the left table fans the query out across shards, so the gate is now on the left table alone, and a local table, a one shard `Distributed` table, a subquery or a table function on the right is handled as well. `Any`, `Semi` and `Anti` join over to the swapped sides the same way `All` does and are accepted; `Asof` and `RightAny` do not mirror and stay out. * Only the topmost join was examined. Join trees are left deep, so the walk now descends to the join that reads the leftmost table. A `RIGHT` or `FULL` join that survives the rewrite, such as the one in `(A INNER C) RIGHT B`, cannot be swapped without moving a join into the right table expression. Those now read the leftmost table through a subquery, which keeps the join on the initiator, the only remaining way to emit the preserved rows once. `getDistributedStorageFromTableExpression` also missed `remote` and `clusterAllReplicas` on the left, because it only looked at `TableNode`. Co-authored-by: Cursor --- src/Planner/PlannerJoinTree.cpp | 121 +++++- ...58_distributed_global_right_join.reference | 384 ++++++++++++++++++ .../05058_distributed_global_right_join.sql | 331 +++++++++++++++ 3 files changed, 816 insertions(+), 20 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 5d42f9a69662e..46e71b6cc3c3a 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -161,41 +162,101 @@ namespace const StorageDistributed * getDistributedStorageFromTableExpression(const QueryTreeNodePtr & table_expression) { const auto * table_node = table_expression->as(); - if (!table_node) - return nullptr; + if (table_node) + return typeid_cast(table_node->getStorage().get()); + + const auto * table_function_node = table_expression->as(); + if (table_function_node) + return typeid_cast(table_function_node->getStorage().get()); + + return nullptr; +} - return typeid_cast(table_node->getStorage().get()); +bool isGlobalJoin(const JoinNode & join_node, const Settings & settings) +{ + const auto distributed_product_mode = settings[Setting::distributed_product_mode]; + return join_node.getLocality() == JoinLocality::Global + || distributed_product_mode == DistributedProductMode::GLOBAL + || (distributed_product_mode != DistributedProductMode::LOCAL && settings[Setting::prefer_global_in_and_join]); } void tryRewriteGlobalRightJoinAsLeftJoin(QueryNode & query_node, const ContextPtr & context) { + /** Join trees are left deep, so the join that reads the leftmost table is the deepest one, and it is + * the only one whose sides can be swapped without moving a join into the right table expression. + */ auto * join_node = query_node.getJoinTree()->as(); - if (!join_node - || join_node->getKind() != JoinKind::Right - || join_node->getStrictness() != JoinStrictness::All - || !join_node->isOnJoinExpression()) + while (join_node) + { + auto * deeper_join_node = join_node->getLeftTableExpression()->as(); + if (!deeper_join_node) + break; + join_node = deeper_join_node; + } + + if (!join_node || join_node->getKind() != JoinKind::Right || !join_node->hasJoinExpression()) return; - const auto & settings = context->getSettingsRef(); - const auto distributed_product_mode = settings[Setting::distributed_product_mode]; - const bool is_global = join_node->getLocality() == JoinLocality::Global - || distributed_product_mode == DistributedProductMode::GLOBAL - || (distributed_product_mode != DistributedProductMode::LOCAL && settings[Setting::prefer_global_in_and_join]); - if (!is_global) + /** These strictnesses mirror when both the table expressions and the kind are flipped. + * `Asof` does not: its last key is an inequality, and swapping the sides reverses its direction. + * `RightAny` does not either, because the strictness itself names the side to take a row from, + * and that name does not follow the tables across the swap. + */ + const auto strictness = join_node->getStrictness(); + if (strictness != JoinStrictness::All && strictness != JoinStrictness::Any + && strictness != JoinStrictness::Semi && strictness != JoinStrictness::Anti) return; + if (!isGlobalJoin(*join_node, context->getSettingsRef())) + return; + + /// Only the left table fans the query out across shards, so only its shard count decides whether + /// the rows of the preserved side get emitted more than once. What the right side is does not matter. const auto * left_storage = getDistributedStorageFromTableExpression(join_node->getLeftTableExpression()); - const auto * right_storage = getDistributedStorageFromTableExpression(join_node->getRightTableExpression()); - if (!left_storage || !right_storage || left_storage->getShardCount() < 2 || right_storage->getShardCount() < 2) + if (!left_storage || left_storage->getShardCount() < 2) return; - /** A `GLOBAL RIGHT JOIN` cannot run with the left table sharded and the right table broadcast. - * Every shard would independently emit unmatched rows from the complete right table. - * Swap the inputs before choosing the `Distributed` table that will execute the query, so the - * preserved side stays sharded and the original left table is broadcast instead. + /** A `JOIN USING` key records its sides positionally, the left one first. The join condition, the + * `USING (a AS b)` clause shipped to the shards and the key supertype all read that order, so the + * sides have to be swapped together with the table expressions. A key that does not hold a plain + * column per side is not swappable that way, so leave such a query alone. `NATURAL` needs no separate + * handling: the analyzer has already turned it into `USING` by now. + */ + std::vector using_key_sides; + if (join_node->isUsingJoinExpression()) + { + for (const auto & using_key : join_node->getJoinExpression()->as().getNodes()) + { + auto * using_column = using_key->as(); + if (!using_column || !using_column->hasExpression()) + return; + + auto * key_sides = using_column->getExpression()->as(); + if (!key_sides || key_sides->getNodes().size() != 2) + return; + + for (const auto & side : key_sides->getNodes()) + { + const auto * side_column = side->as(); + if (!side_column || side_column->hasExpression()) + return; + } + + using_key_sides.push_back(key_sides); + } + } + + /** A `GLOBAL RIGHT JOIN` cannot run with the left table sharded and the right side broadcast. + * Every shard would independently emit the rows of the complete right side that the kind preserves. + * Swap the inputs before choosing the table expression that will execute the query, so the preserved + * side moves out of the broadcast position. It then keeps running on the shards if it is a sharded + * `Distributed` table of its own, and falls back to the initiator otherwise, which is slower but is + * the only way to emit those rows once. * Projection nodes are already resolved and keep the user-visible column order unchanged. */ std::swap(join_node->getLeftTableExpression(), join_node->getRightTableExpression()); + for (auto * key_sides : using_key_sides) + std::swap(key_sides->getNodes()[0], key_sides->getNodes()[1]); join_node->setKind(JoinKind::Left); } @@ -2049,6 +2110,7 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, int first_join_pos = -1; int last_right_join_pos = -1; bool is_cross_join = false; + bool has_global_join_preserving_broadcast_rows = false; /// For each table, table function, query, union table expressions prepare before query plan build for (size_t i = 0; i < table_expressions_stack_size; ++i) { @@ -2078,6 +2140,13 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, if (join_node.getLocality() == JoinLocality::Global) is_global_join = true; + /// Rows of the right side are preserved by these kinds, and that side is broadcast whole to + /// every shard. `tryRewriteGlobalRightJoinAsLeftJoin` swaps the sides where it can, so a join + /// still standing here would emit those rows once per shard. + if ((join_kind == JoinKind::Right || join_kind == JoinKind::Full) + && isGlobalJoin(join_node, planner_context->getQueryContext()->getSettingsRef())) + has_global_join_preserving_broadcast_rows = true; + // save join positions for later check if (first_join_pos < 0 && (join_kind == JoinKind::Left || join_kind == JoinKind::Inner || join_kind == JoinKind::Right)) first_join_pos = static_cast(i); @@ -2158,7 +2227,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, * preserved right side would make every shard emit the same unmatched rows. * * `StorageDistributed` has sophisticated query planning logic to handle JOINs and should - * NOT be wrapped (wrapping breaks tests like 03577_server_constant_folding). + * NOT be wrapped (wrapping breaks tests like 03577_server_constant_folding), except for the + * join trees that stay wrong when the join runs per shard, handled below. */ bool should_wrap_left_table = false; const bool has_multiple_tables = table_expressions_stack.size() > 1; @@ -2181,6 +2251,17 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, // Only wrap if it's specifically IStorageCluster, not StorageDistributed or other remote storages should_wrap_left_table = (dynamic_cast(storage.get()) != nullptr); } + + /** Reading the leftmost table through a subquery keeps the join on the initiator instead of running + * it on every shard, which is the only way left to emit the preserved rows once. It costs the + * distributed execution of the join, and shard specific values such as `shardNum` stop varying, + * so do it only for the join trees that are wrong without it. + */ + if (!should_wrap_left_table && has_global_join_preserving_broadcast_rows) + { + const auto * left_storage = getDistributedStorageFromTableExpression(left_table_expression); + should_wrap_left_table = left_storage && left_storage->getShardCount() > 1; + } } auto left_table_expression_query_plan = buildQueryPlanForTableExpression( diff --git a/tests/queries/0_stateless/05058_distributed_global_right_join.reference b/tests/queries/0_stateless/05058_distributed_global_right_join.reference index 3277112362bcd..2e645c47b92be 100644 --- a/tests/queries/0_stateless/05058_distributed_global_right_join.reference +++ b/tests/queries/0_stateless/05058_distributed_global_right_join.reference @@ -34,3 +34,387 @@ k1 v1 k2 v2 2 b 2 B \N \N 3 C \N \N 3 C +using_initiator +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +3 \N C +3 \N C +using_global +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +3 \N C +3 \N C +using_distributed_product_mode +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +3 \N C +3 \N C +using_multiple_keys_global +k k2 v1 v2 +1 1 a A +1 1 a A +1 1 a A +1 1 a A +2 2 b B +2 2 b B +2 2 b B +2 2 b B +3 3 \N C +3 3 \N C +natural_initiator +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +3 \N C +3 \N C +natural_global +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +3 \N C +3 \N C +using_alias_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +3 \N 3 C +3 \N 3 C +using_alias_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +3 \N 3 C +3 \N 3 C +using_left_join_global +k v1 v2 +1 a A +1 a A +1 a A +1 a A +2 b B +2 b B +2 b B +2 b B +4 d \N +4 d \N +right_is_local_table_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +right_is_local_table_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +right_is_one_shard_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +right_is_one_shard_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +right_is_subquery_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +right_is_subquery_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +right_is_table_function_initiator +k1 v1 n +1 a 1 +1 a 1 +2 b 2 +2 b 2 +4 d 4 +4 d 4 +\N \N 0 +\N \N 3 +right_is_table_function_global +k1 v1 n +1 a 1 +1 a 1 +2 b 2 +2 b 2 +4 d 4 +4 d 4 +\N \N 0 +\N \N 3 +any_initiator +k2 v2 +1 A +2 B +3 C +any_global +k2 v2 +1 A +2 B +3 C +semi_initiator +k2 v2 +1 A +2 B +semi_global +k2 v2 +1 A +2 B +anti_initiator +k2 v2 +3 C +anti_global +k2 v2 +3 C +left_is_one_shard_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +left_is_one_shard_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +nested_right_then_inner_initiator +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +nested_right_then_inner_global +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +nested_right_then_right_initiator +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N \N \N 4 W +\N \N \N \N 4 W +nested_right_then_right_global +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N 3 C 3 Z +\N \N \N \N 4 W +\N \N \N \N 4 W +nested_inner_then_right_initiator +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C \N \N +\N \N 3 C \N \N +nested_inner_then_right_global +k1 v1 k2 v2 k3 v3 +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +1 a 1 A 1 X +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +2 b 2 B 2 Y +\N \N 3 C \N \N +\N \N 3 C \N \N +full_join_initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +4 d \N \N +4 d \N \N +\N \N 3 C +\N \N 3 C +full_join_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +4 d \N \N +4 d \N \N +\N \N 3 C +\N \N 3 C +shard_num_is_untouched_without_right_join +s +1 +2 diff --git a/tests/queries/0_stateless/05058_distributed_global_right_join.sql b/tests/queries/0_stateless/05058_distributed_global_right_join.sql index b43495a419e74..620d9931cbc84 100644 --- a/tests/queries/0_stateless/05058_distributed_global_right_join.sql +++ b/tests/queries/0_stateless/05058_distributed_global_right_join.sql @@ -49,6 +49,337 @@ GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 ORDER BY ALL FORMAT TSVWithNames; +-- The same rewrite must happen for the USING carrier, and for NATURAL, which the analyzer turns into USING. +DROP TABLE IF EXISTS shared_left_local_05058; +DROP TABLE IF EXISTS shared_right_local_05058; +DROP TABLE IF EXISTS shared_left_distributed_05058; +DROP TABLE IF EXISTS shared_right_distributed_05058; + +CREATE TABLE shared_left_local_05058 (k UInt32, k2 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k; + +CREATE TABLE shared_right_local_05058 (k UInt32, k2 Int64, v2 String) +ENGINE = MergeTree +ORDER BY k; + +CREATE TABLE shared_left_distributed_05058 AS shared_left_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), shared_left_local_05058); + +CREATE TABLE shared_right_distributed_05058 AS shared_right_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), shared_right_local_05058); + +INSERT INTO shared_left_local_05058 VALUES (1, 1, 'a'), (2, 2, 'b'), (4, 4, 'd'); +INSERT INTO shared_right_local_05058 VALUES (1, 1, 'A'), (2, 2, 'B'), (3, 3, 'C'); + +SELECT 'using_initiator'; +SELECT k, v1, v2 +FROM (SELECT * FROM shared_left_distributed_05058) AS l +RIGHT JOIN (SELECT * FROM shared_right_distributed_05058) AS r USING (k) +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'using_global'; +SELECT k, v1, v2 +FROM shared_left_distributed_05058 AS l +GLOBAL RIGHT JOIN shared_right_distributed_05058 AS r USING (k) +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'using_distributed_product_mode'; +SELECT k, v1, v2 +FROM shared_left_distributed_05058 AS l +RIGHT JOIN shared_right_distributed_05058 AS r USING (k) +ORDER BY ALL +SETTINGS distributed_product_mode = 'global' +FORMAT TSVWithNames; + +-- Two keys at once, the second one needing a common supertype. +SELECT 'using_multiple_keys_global'; +SELECT k, k2, v1, v2 +FROM shared_left_distributed_05058 AS l +GLOBAL RIGHT JOIN shared_right_distributed_05058 AS r USING (k, k2) +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'natural_initiator'; +SELECT * +FROM (SELECT k, v1 FROM shared_left_distributed_05058) AS l +NATURAL RIGHT JOIN (SELECT k, v2 FROM shared_right_distributed_05058) AS r +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'natural_global'; +SELECT * +FROM (SELECT k, v1 FROM shared_left_distributed_05058) AS l +GLOBAL NATURAL RIGHT JOIN (SELECT k, v2 FROM shared_right_distributed_05058) AS r +ORDER BY ALL +FORMAT TSVWithNames; + +-- USING (a AS b) takes the key from the left table as `a` and from the right one as `b`. +SELECT 'using_alias_initiator'; +SELECT * +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN (SELECT * FROM right_distributed_05058) AS r USING (k1 AS k2) +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'using_alias_global'; +SELECT * +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_distributed_05058 AS r USING (k1 AS k2) +ORDER BY ALL +FORMAT TSVWithNames; + +-- LEFT JOIN is not rewritten and must keep working. +SELECT 'using_left_join_global'; +SELECT k, v1, v2 +FROM shared_left_distributed_05058 AS l +GLOBAL LEFT JOIN shared_right_distributed_05058 AS r USING (k) +ORDER BY ALL +FORMAT TSVWithNames; + +-- Only the left table fans the query out, so the right side may be anything at all. +-- Every result below is paired with the same join forced onto the initiator, which defines the correct rows. +DROP TABLE IF EXISTS right_one_shard_05058; + +CREATE TABLE right_one_shard_05058 AS right_local_05058 +ENGINE = Distributed('test_shard_localhost', currentDatabase(), right_local_05058); + +SELECT 'right_is_local_table_initiator'; +SELECT k1, v1, k2, v2 +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_local_table_global'; +SELECT k1, v1, k2, v2 +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_one_shard_initiator'; +SELECT k1, v1, k2, v2 +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN right_one_shard_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_one_shard_global'; +SELECT k1, v1, k2, v2 +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_one_shard_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_subquery_initiator'; +SELECT k1, v1, k2, v2 +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN (SELECT * FROM right_distributed_05058) AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_subquery_global'; +SELECT k1, v1, k2, v2 +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN (SELECT * FROM right_distributed_05058) AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_table_function_initiator'; +SELECT k1, v1, n +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN (SELECT number AS n FROM numbers(5)) AS r ON l.k1 = r.n +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'right_is_table_function_global'; +SELECT k1, v1, n +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN (SELECT number AS n FROM numbers(5)) AS r ON l.k1 = r.n +ORDER BY ALL +FORMAT TSVWithNames; + +-- ANY, SEMI and ANTI mirror into their LEFT counterparts, so they are rewritten as well. +-- The duplicate key on the left makes ANY and ALL return a different number of rows. +DROP TABLE IF EXISTS dup_left_local_05058; +DROP TABLE IF EXISTS dup_left_distributed_05058; + +CREATE TABLE dup_left_local_05058 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE dup_left_distributed_05058 AS dup_left_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), dup_left_local_05058); + +INSERT INTO dup_left_local_05058 VALUES (1, 'a1'), (1, 'a2'), (2, 'b'), (4, 'd'); + +SELECT 'any_initiator'; +SELECT k2, v2 +FROM (SELECT * FROM dup_left_distributed_05058) AS l +RIGHT ANY JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'any_global'; +SELECT k2, v2 +FROM dup_left_distributed_05058 AS l +GLOBAL RIGHT ANY JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'semi_initiator'; +SELECT k2, v2 +FROM (SELECT * FROM dup_left_distributed_05058) AS l +RIGHT SEMI JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'semi_global'; +SELECT k2, v2 +FROM dup_left_distributed_05058 AS l +GLOBAL RIGHT SEMI JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'anti_initiator'; +SELECT k2, v2 +FROM (SELECT * FROM dup_left_distributed_05058) AS l +RIGHT ANTI JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'anti_global'; +SELECT k2, v2 +FROM dup_left_distributed_05058 AS l +GLOBAL RIGHT ANTI JOIN right_local_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- A single shard on the left never duplicates anything, so nothing is rewritten there. +DROP TABLE IF EXISTS left_one_shard_05058; + +CREATE TABLE left_one_shard_05058 AS left_local_05058 +ENGINE = Distributed('test_shard_localhost', currentDatabase(), left_local_05058); + +SELECT 'left_is_one_shard_initiator'; +SELECT k1, v1, k2, v2 +FROM (SELECT * FROM left_one_shard_05058) AS l +RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'left_is_one_shard_global'; +SELECT k1, v1, k2, v2 +FROM left_one_shard_05058 AS l +GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +DROP TABLE left_one_shard_05058; + +-- A RIGHT join anywhere in the tree emits the preserved rows once per shard, not only at the root. +-- The third table is a superset of the keys, otherwise the outer join filters out exactly the +-- duplicated rows and the test passes while the bug is still there. +DROP TABLE IF EXISTS third_local_05058; +DROP TABLE IF EXISTS third_distributed_05058; + +CREATE TABLE third_local_05058 (k3 UInt32, v3 String) +ENGINE = MergeTree +ORDER BY k3; + +CREATE TABLE third_distributed_05058 AS third_local_05058 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), third_local_05058); + +INSERT INTO third_local_05058 VALUES (1, 'X'), (2, 'Y'), (3, 'Z'), (4, 'W'); + +SELECT 'nested_right_then_inner_initiator'; +SELECT k1, v1, k2, v2, k3, v3 +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +INNER JOIN third_distributed_05058 AS t ON r.k2 = t.k3 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'nested_right_then_inner_global'; +SELECT k1, v1, k2, v2, k3, v3 +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +GLOBAL INNER JOIN third_distributed_05058 AS t ON r.k2 = t.k3 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'nested_right_then_right_initiator'; +SELECT k1, v1, k2, v2, k3, v3 +FROM (SELECT * FROM left_distributed_05058) AS l +RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +RIGHT JOIN third_distributed_05058 AS t ON r.k2 = t.k3 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'nested_right_then_right_global'; +SELECT k1, v1, k2, v2, k3, v3 +FROM left_distributed_05058 AS l +GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +GLOBAL RIGHT JOIN third_distributed_05058 AS t ON r.k2 = t.k3 +ORDER BY ALL +FORMAT TSVWithNames; + +-- Here the RIGHT join sits at the root and its left side is another join, so the sides cannot be +-- swapped. The join runs on the initiator instead. +SELECT 'nested_inner_then_right_initiator'; +SELECT k1, v1, k2, v2, k3, v3 +FROM (SELECT * FROM left_distributed_05058) AS l +INNER JOIN third_distributed_05058 AS t ON l.k1 = t.k3 +RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'nested_inner_then_right_global'; +SELECT k1, v1, k2, v2, k3, v3 +FROM left_distributed_05058 AS l +GLOBAL INNER JOIN third_distributed_05058 AS t ON l.k1 = t.k3 +GLOBAL RIGHT JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- FULL JOIN preserves the broadcast side too, and cannot be repaired by swapping. +SELECT 'full_join_initiator'; +SELECT k1, v1, k2, v2 +FROM (SELECT * FROM left_distributed_05058) AS l +FULL JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +SELECT 'full_join_global'; +SELECT k1, v1, k2, v2 +FROM left_distributed_05058 AS l +GLOBAL FULL JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- Trees without a RIGHT or FULL join keep running on the shards, so shardNum still varies there. +SELECT 'shard_num_is_untouched_without_right_join'; +SELECT DISTINCT shardNum() AS s +FROM left_distributed_05058 AS l +GLOBAL INNER JOIN right_distributed_05058 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +DROP TABLE third_distributed_05058; +DROP TABLE third_local_05058; + +DROP TABLE dup_left_distributed_05058; +DROP TABLE dup_left_local_05058; +DROP TABLE right_one_shard_05058; + +DROP TABLE shared_left_distributed_05058; +DROP TABLE shared_right_distributed_05058; +DROP TABLE shared_left_local_05058; +DROP TABLE shared_right_local_05058; + DROP TABLE left_distributed_05058; DROP TABLE right_distributed_05058; DROP TABLE left_local_05058; From be7d6edbeef1decbad24cea812982cf048dbdb89 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 9 Sep 2026 13:09:29 +0200 Subject: [PATCH 05/52] Look through wrapper storages when rewriting a global RIGHT JOIN of Distributed tables. An `Alias`, `Buffer`, `MaterializedView` or `StorageProxy` in front of a sharded `Distributed` table still fans the query out across shards, so unmatched right rows would otherwise be duplicated once per shard. Co-authored-by: Cursor --- src/Planner/PlannerJoinTree.cpp | 35 ++++++++++--- src/Storages/StorageBuffer.h | 3 +- ...tributed_global_right_join_alias.reference | 24 +++++++++ ...38_distributed_global_right_join_alias.sql | 52 +++++++++++++++++++ ...ributed_global_right_join_buffer.reference | 24 +++++++++ ...9_distributed_global_right_join_buffer.sql | 52 +++++++++++++++++++ 6 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05138_distributed_global_right_join_alias.reference create mode 100644 tests/queries/0_stateless/05138_distributed_global_right_join_alias.sql create mode 100644 tests/queries/0_stateless/05139_distributed_global_right_join_buffer.reference create mode 100644 tests/queries/0_stateless/05139_distributed_global_right_join_buffer.sql diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 46e71b6cc3c3a..031e3fd7a6ca0 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -25,11 +25,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -161,13 +164,33 @@ namespace const StorageDistributed * getDistributedStorageFromTableExpression(const QueryTreeNodePtr & table_expression) { - const auto * table_node = table_expression->as(); - if (table_node) - return typeid_cast(table_node->getStorage().get()); + StoragePtr storage; + if (const auto * table_node = table_expression->as()) + storage = table_node->getStorage(); + else if (const auto * table_function_node = table_expression->as()) + storage = table_function_node->getStorage(); + else + return nullptr; - const auto * table_function_node = table_expression->as(); - if (table_function_node) - return typeid_cast(table_function_node->getStorage().get()); + /// `Alias`, `MaterializedView`, `Buffer` and `StorageProxy` (for example `lazy_load_tables`) + /// forward `read` to a nested storage. If that nested storage is `Distributed`, the join still + /// fans out across shards, so look through the wrappers before deciding. + for (size_t i = 0; storage && i < 16; ++i) + { + if (const auto * distributed = typeid_cast(storage.get())) + return distributed; + + if (const auto * proxy = dynamic_cast(storage.get())) + storage = proxy->getNested(); + else if (const auto * alias = storage->as()) + storage = alias->tryGetTargetTable(); + else if (const auto * materialized_view = storage->as()) + storage = materialized_view->tryGetTargetTable(); + else if (const auto * buffer = storage->as()) + storage = buffer->getDestinationTable(); + else + break; + } return nullptr; } diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index aad37e3badfdc..b1120111ca9ed 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -88,6 +88,7 @@ friend class BufferSink; size_t max_block_size, size_t num_streams) override; bool isRemote() const override; + StoragePtr getDestinationTable() const; bool supportsParallelInsert() const override { return true; } @@ -200,8 +201,6 @@ friend class BufferSink; void backgroundFlush(); void reschedule(size_t min_delay); - StoragePtr getDestinationTable() const; - BackgroundSchedulePool & bg_pool; BackgroundSchedulePoolTaskHolder flush_handle; diff --git a/tests/queries/0_stateless/05138_distributed_global_right_join_alias.reference b/tests/queries/0_stateless/05138_distributed_global_right_join_alias.reference new file mode 100644 index 0000000000000..cd2e4ce4bb534 --- /dev/null +++ b/tests/queries/0_stateless/05138_distributed_global_right_join_alias.reference @@ -0,0 +1,24 @@ +initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +alias_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C diff --git a/tests/queries/0_stateless/05138_distributed_global_right_join_alias.sql b/tests/queries/0_stateless/05138_distributed_global_right_join_alias.sql new file mode 100644 index 0000000000000..ff92cdca3a16f --- /dev/null +++ b/tests/queries/0_stateless/05138_distributed_global_right_join_alias.sql @@ -0,0 +1,52 @@ +-- Tags: distributed + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05138; +DROP TABLE IF EXISTS right_local_05138; +DROP TABLE IF EXISTS left_distributed_05138; +DROP TABLE IF EXISTS right_distributed_05138; +DROP TABLE IF EXISTS left_alias_05138; + +CREATE TABLE left_local_05138 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05138 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE left_distributed_05138 AS left_local_05138 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), left_local_05138); + +CREATE TABLE right_distributed_05138 AS right_local_05138 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05138); + +CREATE TABLE left_alias_05138 +ENGINE = Alias('left_distributed_05138'); + +INSERT INTO left_local_05138 VALUES (1, 'a'), (2, 'b'), (4, 'd'); +INSERT INTO right_local_05138 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +SELECT 'initiator'; +SELECT * +FROM (SELECT * FROM left_alias_05138) AS l +RIGHT JOIN (SELECT * FROM right_distributed_05138) AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- An `Alias` of a sharded `Distributed` table still fans the query out across shards, +-- so the rewrite must look through it the same way as a bare `Distributed` table. +SELECT 'alias_global'; +SELECT * +FROM left_alias_05138 AS l +GLOBAL RIGHT JOIN right_distributed_05138 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +DROP TABLE left_alias_05138; +DROP TABLE left_distributed_05138; +DROP TABLE right_distributed_05138; +DROP TABLE left_local_05138; +DROP TABLE right_local_05138; diff --git a/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.reference b/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.reference new file mode 100644 index 0000000000000..514e988378d58 --- /dev/null +++ b/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.reference @@ -0,0 +1,24 @@ +initiator +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C +buffer_global +k1 v1 k2 v2 +1 a 1 A +1 a 1 A +1 a 1 A +1 a 1 A +2 b 2 B +2 b 2 B +2 b 2 B +2 b 2 B +\N \N 3 C +\N \N 3 C diff --git a/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.sql b/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.sql new file mode 100644 index 0000000000000..ccdc9a895b3ab --- /dev/null +++ b/tests/queries/0_stateless/05139_distributed_global_right_join_buffer.sql @@ -0,0 +1,52 @@ +-- Tags: distributed + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05139; +DROP TABLE IF EXISTS right_local_05139; +DROP TABLE IF EXISTS left_distributed_05139; +DROP TABLE IF EXISTS right_distributed_05139; +DROP TABLE IF EXISTS left_buffer_05139; + +CREATE TABLE left_local_05139 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05139 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE left_distributed_05139 AS left_local_05139 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), left_local_05139); + +CREATE TABLE right_distributed_05139 AS right_local_05139 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05139); + +CREATE TABLE left_buffer_05139 AS left_local_05139 +ENGINE = Buffer(currentDatabase(), left_distributed_05139, 1, 1, 1, 1, 1, 1, 1); + +INSERT INTO left_local_05139 VALUES (1, 'a'), (2, 'b'), (4, 'd'); +INSERT INTO right_local_05139 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +SELECT 'initiator'; +SELECT * +FROM (SELECT * FROM left_buffer_05139) AS l +RIGHT JOIN (SELECT * FROM right_distributed_05139) AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +-- A `Buffer` whose destination is a sharded `Distributed` table still fans the query out +-- across shards, so the rewrite must look through it the same way as a bare `Distributed` table. +SELECT 'buffer_global'; +SELECT * +FROM left_buffer_05139 AS l +GLOBAL RIGHT JOIN right_distributed_05139 AS r ON l.k1 = r.k2 +ORDER BY ALL +FORMAT TSVWithNames; + +DROP TABLE left_buffer_05139; +DROP TABLE left_distributed_05139; +DROP TABLE right_distributed_05139; +DROP TABLE left_local_05139; +DROP TABLE right_local_05139; From 4a73d6d660557e41b3d711ee507aac1896e7fc12 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 25 Sep 2026 14:43:40 +0200 Subject: [PATCH 06/52] add cas disk to existing backup tests Signed-off-by: Konstantin Morozov --- .../configs/backups_disk.xml | 1 + .../configs/cas_gc.xml | 10 ++ .../configs/cas_storage.xml | 24 +++++ .../test_backup_restore_new/test.py | 102 ++++++++++++++++-- .../test_cancel_backup.py | 68 +++++++++++- .../concurrency_helper.py | 39 ++++++- .../configs/cas_server_root_node1.xml | 9 ++ .../configs/cas_server_root_node2.xml | 9 ++ .../configs/cas_server_root_node3.xml | 9 ++ .../configs/cas_storage.xml | 23 ++++ .../test_backup_restore_on_cluster/test.py | 51 ++++++++- .../test_cancel_backup.py | 43 +++++--- .../test_concurrency.py | 46 +++++--- .../configs/disk_cas.xml | 41 +++++++ .../test_backup_restore_s3/test.py | 39 ++++++- .../configs/storage_config.xml | 16 +++ .../test.py | 23 ++++ .../test_database_backup/configs/backups.xml | 25 +++++ .../integration/test_database_backup/test.py | 23 ++-- 19 files changed, 546 insertions(+), 55 deletions(-) create mode 100644 tests/integration/test_backup_restore_new/configs/cas_gc.xml create mode 100644 tests/integration/test_backup_restore_new/configs/cas_storage.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node1.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node2.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node3.xml create mode 100644 tests/integration/test_backup_restore_on_cluster/configs/cas_storage.xml create mode 100644 tests/integration/test_backup_restore_s3/configs/disk_cas.xml diff --git a/tests/integration/test_backup_restore_new/configs/backups_disk.xml b/tests/integration/test_backup_restore_new/configs/backups_disk.xml index f7d666c65426e..1909aaacca257 100644 --- a/tests/integration/test_backup_restore_new/configs/backups_disk.xml +++ b/tests/integration/test_backup_restore_new/configs/backups_disk.xml @@ -9,6 +9,7 @@ backups + cas /backups/ diff --git a/tests/integration/test_backup_restore_new/configs/cas_gc.xml b/tests/integration/test_backup_restore_new/configs/cas_gc.xml new file mode 100644 index 0000000000000..f415af9d6ee63 --- /dev/null +++ b/tests/integration/test_backup_restore_new/configs/cas_gc.xml @@ -0,0 +1,10 @@ + + + + + 1 + 1 + + + + diff --git a/tests/integration/test_backup_restore_new/configs/cas_storage.xml b/tests/integration/test_backup_restore_new/configs/cas_storage.xml new file mode 100644 index 0000000000000..349ce11ab2152 --- /dev/null +++ b/tests/integration/test_backup_restore_new/configs/cas_storage.xml @@ -0,0 +1,24 @@ + + + + + object_storage + s3 + cas + itest-backup-restore-new + http://rustfs1:11121/test/backup_restore_new/ + clickhouse + clickhouse + + + + + + + cas + + + + + + diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index c300ca2b8a7a1..271f491b1eafa 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -18,9 +18,10 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance( "instance", - main_configs=["configs/backups_disk.xml"], + main_configs=["configs/backups_disk.xml", "configs/cas_storage.xml"], user_configs=["configs/zookeeper_retries.xml"], external_dirs=["/backups/"], + with_rustfs=True, ) instance_with_short_timeout = cluster.add_instance( "instance_with_short_timeout", @@ -30,11 +31,14 @@ ) -def create_and_fill_table(engine="MergeTree", n=100): +def create_and_fill_table(engine="MergeTree", n=100, storage_policy=None): if engine == "MergeTree": engine = "MergeTree ORDER BY y PARTITION BY x%10" instance.query("CREATE DATABASE test") - instance.query(f"CREATE TABLE test.table(x UInt32, y String) ENGINE={engine}") + create_query = f"CREATE TABLE test.table(x UInt32, y String) ENGINE={engine}" + if storage_policy is not None: + create_query += f" SETTINGS storage_policy = '{storage_policy}'" + instance.query(create_query) instance.query( f"INSERT INTO test.table SELECT number, toString(number) FROM numbers({n})" ) @@ -224,6 +228,33 @@ def test_restore_table(engine): assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" +@pytest.mark.parametrize("backup_disk", ["backups", "cas"]) +def test_restore_table_on_cas_disk(backup_disk): + backup_name = new_backup_name() + if backup_disk == "cas": + backup_name = backup_name.replace("Disk('backups', '", "Disk('cas', 'backups/") + create_and_fill_table(storage_policy="cas_policy") + + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == "cas_policy\n" + ) + instance.query(f"BACKUP TABLE test.table TO {backup_name}") + instance.query("DROP TABLE test.table") + instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + + assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == "cas_policy\n" + ) + assert ( + instance.query("CHECK TABLE test.table SETTINGS check_query_single_value_result = 1") + == "1\n" + ) + + @pytest.mark.parametrize( "engine", ["MergeTree", "Log", "TinyLog", "StripeLog", "Memory"] ) @@ -1504,8 +1535,9 @@ def test_system_users_async(): ) -def test_projection(): - create_and_fill_table(n=3) +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_projection(storage_policy): + create_and_fill_table(n=3, storage_policy=storage_policy) instance.query("ALTER TABLE test.table ADD PROJECTION prjmax (SELECT MAX(x))") instance.query("INSERT INTO test.table VALUES (100, 'a'), (101, 'b')") @@ -1555,6 +1587,18 @@ def test_projection(): == "2\n" ) + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == f"{storage_policy}\n" + ) + assert ( + instance.query( + "CHECK TABLE test.table SETTINGS check_query_single_value_result = 1" + ) + == "1\n" + ) + def test_restore_table_not_evaluate_table_defaults(): instance.query("CREATE DATABASE test") @@ -1950,8 +1994,10 @@ def verify_restore_info(): assert info.bytes_read == 0 -def test_mutation(): - create_and_fill_table(engine="MergeTree ORDER BY tuple()", n=5) +def check_mutation(storage_policy=None): + create_and_fill_table( + engine="MergeTree ORDER BY tuple()", n=5, storage_policy=storage_policy + ) instance.query( "INSERT INTO test.table SELECT number, toString(number) FROM numbers(5, 5)" @@ -1973,6 +2019,12 @@ def test_mutation(): instance.query("ALTER TABLE test.table UPDATE x=x+1 WHERE 1") instance.query("ALTER TABLE test.table UPDATE x=x+1 WHERE 1") + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == f"{storage_policy}\n" + ) + assert instance.query("SELECT count() FROM test.table") == "15\n" backup_name = new_backup_name() instance.query(f"BACKUP TABLE test.table TO {backup_name}") @@ -1986,6 +2038,20 @@ def test_mutation(): instance.query("DROP TABLE test.table") instance.query(f"RESTORE TABLE test.table FROM {backup_name}") + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == f"{storage_policy}\n" + ) + assert instance.query("SELECT count() FROM test.table") == "15\n" + + +def test_mutation(): + check_mutation() + + +def test_mutation_on_cas_disk(): + check_mutation(storage_policy="cas_policy") def test_tables_dependency(): @@ -2215,12 +2281,17 @@ def test_restore_table_with_checksum_data_file_name(engine): assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" -def test_incremental_backup_with_checksum_data_file_name(): +def check_incremental_backup_with_checksum_data_file_name(storage_policy=None): backup_name = new_backup_name() incremental_backup_name = new_backup_name() - create_and_fill_table() + create_and_fill_table(storage_policy=storage_policy) assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table'") + == f"{storage_policy}\n" + ) instance.query( f"BACKUP TABLE test.table TO {backup_name} SETTINGS data_file_name_generator='checksum'" ) @@ -2236,6 +2307,19 @@ def test_incremental_backup_with_checksum_data_file_name(): f"RESTORE TABLE test.table AS test.table2 FROM {incremental_backup_name}" ) assert instance.query("SELECT count(), sum(x) FROM test.table2") == "102\t5081\n" + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name = 'table2'") + == f"{storage_policy}\n" + ) + + +def test_incremental_backup_with_checksum_data_file_name(): + check_incremental_backup_with_checksum_data_file_name() + + +def test_incremental_backup_with_checksum_data_file_name_on_cas_disk(): + check_incremental_backup_with_checksum_data_file_name(storage_policy="cas_policy") def test_async_backup_restore_with_max_execution_time_zero(): diff --git a/tests/integration/test_backup_restore_new/test_cancel_backup.py b/tests/integration/test_backup_restore_new/test_cancel_backup.py index c6e378ca4bf6e..16a67db9318ee 100644 --- a/tests/integration/test_backup_restore_new/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_new/test_cancel_backup.py @@ -1,4 +1,5 @@ import re +import time import uuid import pytest @@ -17,6 +18,8 @@ "configs/backups_disk.xml", "configs/slow_backups.xml", "configs/shutdown_cancel_backups.xml", + "configs/cas_storage.xml", + "configs/cas_gc.xml", ] node = cluster.add_instance( @@ -24,8 +27,11 @@ main_configs=main_configs, external_dirs=["/backups/"], stay_alive=True, + with_rustfs=True, ) +CAS_POOL_PREFIXES = ["backup_restore_new/blobs/", "backup_restore_new/parts/"] + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -179,11 +185,57 @@ def cancel_restore(restore_id): assert kill_duration_ms < kill_duration_ms_threshold +def table_settings(storage_policy): + if storage_policy is None: + return "" + return f" SETTINGS storage_policy = '{storage_policy}'" + + +def count_cas_pool_objects(): + return sum( + len( + list( + cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, prefix, recursive=True + ) + ) + ) + for prefix in CAS_POOL_PREFIXES + ) + + +def wait_cas_pool_settled(): + previous = count_cas_pool_objects() + for _ in range(40): + time.sleep(3) + current = count_cas_pool_objects() + if current == previous: + return current + previous = current + return previous + + +def wait_cas_pool_reclaimed(baseline): + current = count_cas_pool_objects() + for _ in range(120): + if current <= baseline: + break + time.sleep(1) + current = count_cas_pool_objects() + assert ( + current <= baseline + ), f"CAS pool objects were not reclaimed: baseline={baseline}, current={current}" + + # Test that BACKUP and RESTORE operations can be cancelled with KILL QUERY. -def test_cancel_backup(): +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_cancel_backup(storage_policy): + cas_pool_baseline = wait_cas_pool_settled() if storage_policy == "cas_policy" else None + # We use partitioning so backups would contain more files. node.query( "CREATE TABLE tbl (x UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY x%20" + + table_settings(storage_policy) ) node.query("INSERT INTO tbl SELECT number FROM numbers(500)") @@ -209,11 +261,23 @@ def test_cancel_backup(): start_restore(restore_id, backup_id) wait_restore(restore_id) + assert node.query("SELECT count(), sum(x) FROM tbl") == "500\t124750\n" + assert ( + node.query("CHECK TABLE tbl SETTINGS check_query_single_value_result = 1") + == "1\n" + ) + + if cas_pool_baseline is not None: + node.query("DROP TABLE tbl SYNC") + wait_cas_pool_reclaimed(cas_pool_baseline) + # Test that shutdown cancels a running backup and doesn't wait until it finishes. -def test_shutdown_cancel_backup(): +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_shutdown_cancel_backup(storage_policy): node.query( "CREATE TABLE tbl (x UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY x%5" + + table_settings(storage_policy) ) node.query("INSERT INTO tbl SELECT number FROM numbers(500)") diff --git a/tests/integration/test_backup_restore_on_cluster/concurrency_helper.py b/tests/integration/test_backup_restore_on_cluster/concurrency_helper.py index f75600d112966..724796de06c6f 100644 --- a/tests/integration/test_backup_restore_on_cluster/concurrency_helper.py +++ b/tests/integration/test_backup_restore_on_cluster/concurrency_helper.py @@ -38,17 +38,50 @@ def generate_cluster_def(file: str, num_nodes: int) -> str: return str(path.absolute()) +def generate_cas_server_root_def(file: str, node_index: int) -> str: + path = ( + Path(__file__).parent + / f"_gen/cas_server_root_{Path(file).stem}_node{node_index}.xml" + ) + config = f""" + + + + itest-{Path(file).stem}-node{node_index} + + + +""" + if path.is_file() and path.read_text(encoding="utf-8") == config: + return str(path.absolute()) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(encoding="utf-8", data=config) + return str(path.absolute()) + + def add_nodes_to_cluster( cluster: ClickHouseCluster, num_nodes: int, main_configs: List[str], user_configs: List[str], + cas_file: str = None, **kwargs ) -> List[ClickHouseInstance]: + def node_main_configs(i): + if cas_file is None: + return main_configs + return main_configs + [ + "configs/cas_storage.xml", + generate_cas_server_root_def(cas_file, i), + ] + + if cas_file is not None: + kwargs["with_rustfs"] = True + nodes = [ cluster.add_instance( f"node{i}", - main_configs=main_configs, + main_configs=node_main_configs(i), user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": f"node{i}", "shard": "shard1"}, @@ -60,9 +93,11 @@ def add_nodes_to_cluster( return nodes -def create_test_table(node: ClickHouseInstance) -> None: +def create_test_table(node: ClickHouseInstance, storage_policy: str = None) -> None: + settings = f" SETTINGS storage_policy = '{storage_policy}'" if storage_policy else "" node.query( """CREATE TABLE tbl ON CLUSTER 'cluster' ( x UInt64 ) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}') ORDER BY tuple()""" + + settings ) diff --git a/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node1.xml b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node1.xml new file mode 100644 index 0000000000000..e240cf0308ae9 --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node1.xml @@ -0,0 +1,9 @@ + + + + + itest-backup-restore-on-cluster-node1 + + + + diff --git a/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node2.xml b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node2.xml new file mode 100644 index 0000000000000..fca0f5d5a472d --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node2.xml @@ -0,0 +1,9 @@ + + + + + itest-backup-restore-on-cluster-node2 + + + + diff --git a/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node3.xml b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node3.xml new file mode 100644 index 0000000000000..49b8f8620c78b --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/cas_server_root_node3.xml @@ -0,0 +1,9 @@ + + + + + itest-backup-restore-on-cluster-node3 + + + + diff --git a/tests/integration/test_backup_restore_on_cluster/configs/cas_storage.xml b/tests/integration/test_backup_restore_on_cluster/configs/cas_storage.xml new file mode 100644 index 0000000000000..5884c99c7f156 --- /dev/null +++ b/tests/integration/test_backup_restore_on_cluster/configs/cas_storage.xml @@ -0,0 +1,23 @@ + + + + + object_storage + s3 + cas + http://rustfs1:11121/test/backup_restore_on_cluster/ + clickhouse + clickhouse + + + + + + + cas + + + + + + diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 6e1e0de0ec60b..33ae40071a035 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -19,6 +19,8 @@ "configs/lesser_timeouts.xml", # Default timeouts are quite big (a few minutes), the tests don't need them to be that big. ] +cas_configs = ["configs/cas_storage.xml"] + user_configs = [ "configs/allow_database_types.xml", "configs/zookeeper_retries.xml", @@ -26,30 +28,33 @@ node1 = cluster.add_instance( "node1", - main_configs=main_configs, + main_configs=main_configs + cas_configs + ["configs/cas_server_root_node1.xml"], user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node1", "shard": "shard1"}, with_zookeeper=True, + with_rustfs=True, ) node2 = cluster.add_instance( "node2", - main_configs=main_configs, + main_configs=main_configs + cas_configs + ["configs/cas_server_root_node2.xml"], user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node2", "shard": "shard1"}, with_zookeeper=True, + with_rustfs=True, stay_alive=True, # Necessary for the "test_stop_other_host_while_backup" test ) node3 = cluster.add_instance( "node3", - main_configs=main_configs, + main_configs=main_configs + cas_configs + ["configs/cas_server_root_node3.xml"], user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node3", "shard": "shard1"}, with_zookeeper=True, + with_rustfs=True, ) @@ -125,6 +130,46 @@ def test_replicated_table(): ) +def test_replicated_table_on_cas_disk(): + expected = TSV([[1, "Don\\'t"], [2, "count"], [3, "your"], [4, "chickens"]]) + node1.query( + "CREATE TABLE tbl ON CLUSTER 'cluster' (" + "x UInt8, y String" + ") ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')" + "ORDER BY x SETTINGS storage_policy = 'cas_policy'" + ) + + node1.query("INSERT INTO tbl VALUES (1, 'Don''t')") + node2.query("INSERT INTO tbl VALUES (2, 'count')") + node1.query("INSERT INTO tbl SETTINGS async_insert=true VALUES (3, 'your')") + node2.query("INSERT INTO tbl SETTINGS async_insert=true VALUES (4, 'chickens')") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") + + assert node1.query("SELECT * FROM tbl ORDER BY x") == expected + assert node2.query("SELECT * FROM tbl ORDER BY x") == expected + for node in [node1, node2]: + assert ( + node.query("SELECT storage_policy FROM system.tables WHERE name = 'tbl'") + == "cas_policy\n" + ) + + backup_name = new_backup_name() + node1.query( + f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name} SETTINGS replica_num=1" + ) + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") + node2.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") + + assert node1.query("SELECT * FROM tbl ORDER BY x") == expected + assert node2.query("SELECT * FROM tbl ORDER BY x") == expected + for node in [node1, node2]: + assert ( + node.query("SELECT storage_policy FROM system.tables WHERE name = 'tbl'") + == "cas_policy\n" + ) + + def test_empty_replicated_table(): node1.query( "CREATE TABLE tbl ON CLUSTER 'cluster' (" diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py index dceb3e89744fe..1b0884e47e4ff 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py @@ -19,6 +19,7 @@ "configs/lesser_timeouts.xml", # Default timeouts are quite big (a few minutes), the tests don't need them to be that big. "configs/slow_backups.xml", "configs/shutdown_cancel_backups.xml", + "configs/cas_storage.xml", ] user_configs = [ @@ -27,21 +28,23 @@ node1 = cluster.add_instance( "node1", - main_configs=main_configs, + main_configs=main_configs + ["configs/cas_server_root_node1.xml"], user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node1", "shard": "shard1"}, with_zookeeper=True, + with_rustfs=True, stay_alive=True, # Necessary for "test_shutdown_cancel_backup" ) node2 = cluster.add_instance( "node2", - main_configs=main_configs, + main_configs=main_configs + ["configs/cas_server_root_node2.xml"], user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node2", "shard": "shard1"}, with_zookeeper=True, + with_rustfs=True, stay_alive=True, # Necessary for "test_shutdown_cancel_backup" ) @@ -79,7 +82,7 @@ def random_node(): # Makes table "tbl" and fill it with data. -def create_and_fill_table(node, num_parts=10, on_cluster=True): +def create_and_fill_table(node, num_parts=10, on_cluster=True, storage_policy=None): # We use partitioning to make sure there will be more files in a backup. partition_by_clause = " PARTITION BY x%" + str(num_parts) if num_parts > 1 else "" node.query( @@ -88,6 +91,7 @@ def create_and_fill_table(node, num_parts=10, on_cluster=True): + "(x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}') " + "ORDER BY tuple()" + partition_by_clause + + (f" SETTINGS storage_policy = '{storage_policy}'" if storage_policy else "") ) if num_parts > 0: node.query(f"INSERT INTO tbl SELECT number FROM numbers({num_parts})") @@ -406,18 +410,17 @@ def wait_for_backups_to_finish(): ) -__backup_id_of_successful_backup = None +__backup_id_of_successful_backup = {} # Generates a backup which will be used to test RESTORE. -def get_backup_id_of_successful_backup(): - global __backup_id_of_successful_backup - if __backup_id_of_successful_backup is None: - __backup_id_of_successful_backup = random_id() +def get_backup_id_of_successful_backup(storage_policy=None): + if storage_policy not in __backup_id_of_successful_backup: + __backup_id_of_successful_backup[storage_policy] = random_id() with NoTrashChecker() as no_trash_checker: print("Will make backup successfully") - backup_id = __backup_id_of_successful_backup - create_and_fill_table(random_node()) + backup_id = __backup_id_of_successful_backup[storage_policy] + create_and_fill_table(random_node(), storage_policy=storage_policy) initiator = random_node() print(f"Using {get_node_name(initiator)} as initiator") initiator.query( @@ -430,7 +433,7 @@ def get_backup_id_of_successful_backup(): # Dropping the table before restoring. node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") - return __backup_id_of_successful_backup + return __backup_id_of_successful_backup[storage_policy] # Actual tests @@ -489,9 +492,10 @@ def test_cancel_backup(): # Test that a RESTORE operation can be cancelled with KILL QUERY. -def test_cancel_restore(): +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_cancel_restore(storage_policy): # Make backup. - backup_id = get_backup_id_of_successful_backup() + backup_id = get_backup_id_of_successful_backup(storage_policy) # Cancel restoring. with NoTrashChecker() as no_trash_checker: @@ -556,6 +560,19 @@ def test_cancel_restore(): wait_status(initiator, "RESTORED", restore_id=restore_id) assert get_num_system_processes(nodes, restore_id=restore_id) == 0 + if storage_policy is not None: + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") + for node in nodes: + assert ( + node.query("SELECT storage_policy FROM system.tables WHERE name = 'tbl'") + == f"{storage_policy}\n" + ) + assert node.query("SELECT count(), sum(x) FROM tbl") == "10\t45\n" + assert ( + node.query("CHECK TABLE tbl SETTINGS check_query_single_value_result = 1") + == "1\n" + ) + # Test that shutdown cancels a running backup and doesn't wait until it finishes. def test_shutdown_cancels_backup(): diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py index 02431752f64b3..cc9b1ebe52f22 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py @@ -25,7 +25,9 @@ # No [Zoo]Keeper retries for tests with concurrency user_configs = ["configs/allow_database_types.xml"] -nodes = add_nodes_to_cluster(cluster, num_nodes, main_configs, user_configs, cpu_limit=12) +nodes = add_nodes_to_cluster( + cluster, num_nodes, main_configs, user_configs, cas_file=__file__, cpu_limit=12 +) node0 = nodes[0] @@ -51,8 +53,8 @@ def drop_after_test(): backup_id_counter = 0 -def create_and_fill_table() -> None: - create_test_table(node0) +def create_and_fill_table(storage_policy=None) -> None: + create_test_table(node0, storage_policy) for i, node in enumerate(nodes): node.query(f"INSERT INTO tbl VALUES ({i})") @@ -66,8 +68,23 @@ def new_backup_name(): expected_sum = num_nodes * (num_nodes - 1) // 2 -def test_replicated_table(): - create_and_fill_table() +def check_restored_table(storage_policy): + for node in nodes: + assert node.query("SELECT sum(x) FROM tbl") == TSV([expected_sum]) + if storage_policy is not None: + assert ( + node.query("SELECT storage_policy FROM system.tables WHERE name = 'tbl'") + == f"{storage_policy}\n" + ) + assert ( + node.query("CHECK TABLE tbl SETTINGS check_query_single_value_result = 1") + == "1\n" + ) + + +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_replicated_table(storage_policy): + create_and_fill_table(storage_policy) backup_name = new_backup_name() node0.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") @@ -76,15 +93,15 @@ def test_replicated_table(): node0.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node0.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") - for i in range(num_nodes): - assert nodes[i].query("SELECT sum(x) FROM tbl") == TSV([expected_sum]) + check_restored_table(storage_policy) num_concurrent_backups = 4 -def test_concurrent_backups_on_same_node(): - create_and_fill_table() +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_concurrent_backups_on_same_node(storage_policy): + create_and_fill_table(storage_policy) backup_names = [new_backup_name() for _ in range(num_concurrent_backups)] @@ -113,12 +130,12 @@ def test_concurrent_backups_on_same_node(): node0.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node0.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node0.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") - for i in range(num_nodes): - assert nodes[i].query("SELECT sum(x) FROM tbl") == TSV([expected_sum]) + check_restored_table(storage_policy) -def test_concurrent_backups_on_different_nodes(): - create_and_fill_table() +@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) +def test_concurrent_backups_on_different_nodes(storage_policy): + create_and_fill_table(storage_policy) assert num_concurrent_backups <= num_nodes backup_names = [new_backup_name() for _ in range(num_concurrent_backups)] @@ -150,8 +167,7 @@ def test_concurrent_backups_on_different_nodes(): nodes[i].query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") nodes[i].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_names[i]}") nodes[i].query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") - for j in range(num_nodes): - assert nodes[j].query("SELECT sum(x) FROM tbl") == TSV([expected_sum]) + check_restored_table(storage_policy) @pytest.mark.parametrize( diff --git a/tests/integration/test_backup_restore_s3/configs/disk_cas.xml b/tests/integration/test_backup_restore_s3/configs/disk_cas.xml new file mode 100644 index 0000000000000..83392e27c7e4e --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/disk_cas.xml @@ -0,0 +1,41 @@ + + + + + + object_storage + s3 + cas + 30 + 10000 + itest-backup-restore-s3 + http://rustfs1:11121/test/backup_restore_s3_cas/ + clickhouse + clickhouse + + + cache + disk_cas + /tmp/cas_cache/ + 1000000000 + 1 + + + + + +
+ disk_cas +
+ + + + +
+ disk_cas_cache +
+ + + + + diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 77e4e1c0dcbb6..f5d4b77b4aa21 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -143,6 +143,7 @@ def setup_cluster(request): "node", main_configs=[ "configs/disk_s3.xml", + "configs/disk_cas.xml", "configs/named_collection_s3_backups.xml", "configs/s3_settings.xml", "configs/blob_log.xml", @@ -154,6 +155,7 @@ def setup_cluster(request): "configs/zookeeper_retries.xml", ], with_minio=True, + with_rustfs=True, # The test compares some S3 events. We disable the remote DB disk, so it doesn't affect the comparing events. with_remote_database_disk=False, with_zookeeper=True, @@ -177,6 +179,14 @@ def cluster(setup_cluster): backup_id_counter = 0 +CAS_POLICIES = ["policy_cas", "policy_cas_cache"] + + +def s3_backup_destination(storage_policy, backup_name): + if storage_policy in CAS_POLICIES: + return f"S3('http://rustfs1:11121/test/backups/{backup_name}', 'clickhouse', 'clickhouse')" + return f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + def new_backup_name(): global backup_id_counter @@ -293,6 +303,8 @@ def check_system_tables(cluster, backup_query_id=None): ("disk_s3_plain", "ObjectStorage", "S3", "Plain"), ("disk_s3_plain_rewritable", "ObjectStorage", "S3", "PlainRewritable"), ("disk_s3_restricted_user", "ObjectStorage", "S3", "Local"), + ("disk_cas", "ObjectStorage", "S3", "CAS"), + ("disk_cas_cache", "ObjectStorage", "S3", "CAS"), ) assert len(expected_disks) == len(disks) for expected_disk in expected_disks: @@ -336,6 +348,16 @@ def check_system_tables(cluster, backup_query_id=None): "disk_s3_plain", id="from_local_to_s3_plain", ), + pytest.param( + "policy_cas", + "default", + id="from_cas_to_local", + ), + pytest.param( + "policy_cas", + "disk_s3", + id="from_cas_to_s3", + ), ], ) def test_backup_to_disk(cluster, storage_policy, to_disk): @@ -531,14 +553,22 @@ def test_backup_to_s3_multipart(cluster): "policy_s3", "policy_s3_other_bucket", "policy_s3_plain_rewritable", + "policy_cas", ], ) def test_backup_to_s3_native_copy(cluster, storage_policy): backup_name = new_backup_name() - backup_destination = f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + backup_destination = s3_backup_destination(storage_policy, backup_name) (backup_events, restore_events) = check_backup_and_restore( cluster, storage_policy, backup_destination ) + + if storage_policy in CAS_POLICIES: + # A CAS file can be a range inside a blob object, so the server-side copy may be ranged. + for events in (backup_events, restore_events): + assert events.get("S3CopyObject", 0) + events.get("S3UploadPartCopy", 0) > 0 + return + # single part upload assert backup_events["S3CopyObject"] > 0 assert restore_events["S3CopyObject"] > 0 @@ -703,13 +733,12 @@ def test_incremental_backup_append_table_def(cluster): (True, True, True), ], ) +@pytest.mark.parametrize("storage_policy", ["policy_s3_cache", "policy_cas_cache"]) def test_backup_with_fs_cache( - cluster, in_cache_initially, allow_backup_read_cache, allow_s3_native_copy + cluster, in_cache_initially, allow_backup_read_cache, allow_s3_native_copy, storage_policy ): - storage_policy = "policy_s3_cache" - backup_name = new_backup_name() - backup_destination = f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + backup_destination = s3_backup_destination(storage_policy, backup_name) insert_settings = { "enable_filesystem_cache_on_write_operations": int(in_cache_initially) diff --git a/tests/integration/test_backup_restore_storage_policy/configs/storage_config.xml b/tests/integration/test_backup_restore_storage_policy/configs/storage_config.xml index de9f68cbe8704..6c841d4612e50 100644 --- a/tests/integration/test_backup_restore_storage_policy/configs/storage_config.xml +++ b/tests/integration/test_backup_restore_storage_policy/configs/storage_config.xml @@ -9,6 +9,15 @@ local /var/lib/disks/two/ + + object_storage + s3 + cas + itest-backup-restore-storage-policy + http://rustfs1:11121/test/backup_restore_storage_policy/ + clickhouse + clickhouse + @@ -25,6 +34,13 @@ + + + + cas + + + diff --git a/tests/integration/test_backup_restore_storage_policy/test.py b/tests/integration/test_backup_restore_storage_policy/test.py index b2e907dcaaa7a..e45402408c6ac 100644 --- a/tests/integration/test_backup_restore_storage_policy/test.py +++ b/tests/integration/test_backup_restore_storage_policy/test.py @@ -9,6 +9,7 @@ instance = cluster.add_instance( "instance", main_configs=["configs/storage_config.xml"], + with_rustfs=True, ) @@ -42,6 +43,12 @@ def create_table_backup(backup_name, storage_policy=None): create_query += f" SETTINGS storage_policy = '{storage_policy}'" instance.query(create_query) instance.query("INSERT INTO test.table SELECT number FROM numbers(10)") + assert instance.query("SELECT count(), sum(x) FROM test.table") == "10\t45\n" + if storage_policy is not None: + assert ( + instance.query("SELECT storage_policy FROM system.tables WHERE name='table'") + == f"{storage_policy}\n" + ) instance.query(f"BACKUP TABLE test.table TO {backup_name}") instance.query("DROP TABLE test.table SYNC") @@ -63,6 +70,11 @@ def restore_table(backup_name, storage_policy=None): ("policy1", "policy2", "policy2"), ("policy1", "", "default"), ("policy1", None, "policy1"), + ("cas_policy", None, "cas_policy"), + ("cas_policy", "policy1", "policy1"), + (None, "cas_policy", "cas_policy"), + ("policy1", "cas_policy", "cas_policy"), + ("cas_policy", "cas_policy", "cas_policy"), ], ) def test_storage_policies(origin_policy, restore_policy, expected_policy): @@ -74,3 +86,14 @@ def test_storage_policies(origin_policy, restore_policy, expected_policy): instance.query("SELECT storage_policy FROM system.tables WHERE name='table'") == f"{expected_policy}\n" ) + assert instance.query("SELECT count(), sum(x) FROM test.table") == "10\t45\n" + assert ( + instance.query( + "SELECT sum(rows) FROM system.parts WHERE database = 'test' AND table = 'table' AND active" + ) + == "10\n" + ) + assert ( + instance.query("CHECK TABLE test.table SETTINGS check_query_single_value_result = 1") + == "1\n" + ) diff --git a/tests/integration/test_database_backup/configs/backups.xml b/tests/integration/test_database_backup/configs/backups.xml index 32bc9dce7eec6..d791dababff44 100644 --- a/tests/integration/test_database_backup/configs/backups.xml +++ b/tests/integration/test_database_backup/configs/backups.xml @@ -17,12 +17,37 @@ plain /local_plain/ + + s3_plain + http://rustfs1:11121/test/backup_disk_s3_plain_rustfs/ + clickhouse + clickhouse + + + object_storage + s3 + cas + itest-database-backup + http://rustfs1:11121/test/database_backup_cas/ + clickhouse + clickhouse + + + + + + cas + + + + backup_disk_local backup_disk_s3_plain backup_disk_object_storage_local_plain + backup_disk_s3_plain_rustfs /backups diff --git a/tests/integration/test_database_backup/test.py b/tests/integration/test_database_backup/test.py index 9dc7392d77972..9fcafffb4efb3 100644 --- a/tests/integration/test_database_backup/test.py +++ b/tests/integration/test_database_backup/test.py @@ -10,6 +10,7 @@ main_configs=["configs/backups.xml"], stay_alive=True, with_minio=True, + with_rustfs=True, ) @@ -31,6 +32,12 @@ def cleanup_backup_files(instance): for s3_object in s3_objects: minio.remove_object(cluster.minio_bucket, s3_object) + rustfs = cluster.rustfs_client + for s3_object in rustfs.list_objects( + cluster.rustfs_bucket, prefix="backup_disk_s3_plain_rustfs/", recursive=True + ): + rustfs.remove_object(cluster.rustfs_bucket, s3_object.object_name) + @pytest.mark.parametrize( "backup_destination", @@ -39,9 +46,11 @@ def cleanup_backup_files(instance): "Disk('backup_disk_local', 'test_database_backup')", "Disk('backup_disk_s3_plain', 'test_database_backup')", "Disk('backup_disk_object_storage_local_plain', 'test_database_backup')", + "Disk('backup_disk_s3_plain_rustfs', 'test_database_backup')", ], ) -def test_database_backup_database(backup_destination): +@pytest.mark.parametrize("storage_policy", ["default", "cas_policy"]) +def test_database_backup_database(backup_destination, storage_policy): cleanup_backup_files(instance) instance.query( @@ -51,13 +60,13 @@ def test_database_backup_database(backup_destination): CREATE DATABASE test_database; - CREATE TABLE test_database.test_table_1 (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + CREATE TABLE test_database.test_table_1 (id UInt64, value String) ENGINE=MergeTree ORDER BY id SETTINGS storage_policy = '{storage_policy}'; INSERT INTO test_database.test_table_1 VALUES (0, 'test_database.test_table_1'); - CREATE TABLE test_database.test_table_2 (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + CREATE TABLE test_database.test_table_2 (id UInt64, value String) ENGINE=MergeTree ORDER BY id SETTINGS storage_policy = '{storage_policy}'; INSERT INTO test_database.test_table_2 VALUES (0, 'test_database.test_table_2'); - CREATE TABLE test_database.test_table_3 (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + CREATE TABLE test_database.test_table_3 (id UInt64, value String) ENGINE=MergeTree ORDER BY id SETTINGS storage_policy = '{storage_policy}'; INSERT INTO test_database.test_table_3 VALUES (0, 'test_database.test_table_3'); BACKUP DATABASE test_database TO {backup_destination}; @@ -109,9 +118,11 @@ def test_database_backup_database(backup_destination): "Disk('backup_disk_local', 'test_table_backup')", "Disk('backup_disk_s3_plain', 'test_table_backup')", "Disk('backup_disk_object_storage_local_plain', 'test_table_backup')", + "Disk('backup_disk_s3_plain_rustfs', 'test_table_backup')", ], ) -def test_database_backup_table(backup_destination): +@pytest.mark.parametrize("storage_policy", ["default", "cas_policy"]) +def test_database_backup_table(backup_destination, storage_policy): cleanup_backup_files(instance) instance.query( @@ -121,7 +132,7 @@ def test_database_backup_table(backup_destination): CREATE DATABASE test_database; - CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id SETTINGS storage_policy = '{storage_policy}'; INSERT INTO test_database.test_table VALUES (0, 'test_database.test_table'); BACKUP TABLE test_database.test_table TO {backup_destination}; From 3300f85dc1cdfec5e4e5baf81e93cee951d69de0 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 25 Sep 2026 16:04:12 +0200 Subject: [PATCH 07/52] update test Signed-off-by: Konstantin Morozov --- .../test_backup_restore_on_cluster/test_cancel_backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py index 1b0884e47e4ff..653050259e2ef 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py @@ -363,6 +363,7 @@ def __exit__(self, type, value, traceback): + start_time + "') " + "AND NOT ((name == 'KEEPER_EXCEPTION') AND (last_error_message LIKE '%Fault injection%')) " + + "AND NOT ((name == 'S3_ERROR') AND (last_error_message LIKE '%This error is expected for S3 disk%')) " + "AND NOT (name == 'NO_ELEMENTS_IN_CONFIG')" ) errors = errors_query_result.splitlines() From 46c5fb69a04152d39c27ad6a569df9ec12954ca2 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 1 Oct 2026 17:12:59 +0200 Subject: [PATCH 08/52] Move PREWHERE onto the broadcast side of a global RIGHT JOIN After the join is rewritten as a LEFT JOIN, a `PREWHERE` bound to the original left table followed it into a `Memory` table and shards threw `ILLEGAL_PREWHERE`. Move the predicate into the broadcast subquery when its columns belong to that table, and skip the swap when `PREWHERE` has no column so the initiator wrapper keeps the filter. When wrapping a remote table that owns `PREWHERE`, move the full predicate onto the wrap and drop the initiator actions so nested `GLOBAL IN` sets are not materialized twice. Port of https://github.com/ClickHouse/ClickHouse/pull/119780 Co-authored-by: Cursor --- src/Analyzer/Utils.cpp | 49 ++++++++ src/Analyzer/Utils.h | 6 + src/Planner/PlannerJoinTree.cpp | 32 ++++- src/Planner/TableExpressionData.h | 6 + src/Storages/buildQueryTreeForShard.cpp | 38 ++++++ ...distributed_global_join_prewhere.reference | 37 ++++++ ...05194_distributed_global_join_prewhere.sql | 116 ++++++++++++++++++ ...global_join_prewhere_in_subquery.reference | 3 + ...buted_global_join_prewhere_in_subquery.sql | 42 +++++++ ...uted_remote_prewhere_in_subquery.reference | 1 + ...istributed_remote_prewhere_in_subquery.sql | 37 ++++++ 11 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05194_distributed_global_join_prewhere.reference create mode 100644 tests/queries/0_stateless/05194_distributed_global_join_prewhere.sql create mode 100644 tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.reference create mode 100644 tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.sql create mode 100644 tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.reference create mode 100644 tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.sql diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index fe5ddbdd6d7fd..a541c49199147 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -1052,6 +1052,55 @@ std::pair getExpressionSource(const QueryTreeNodePtr & n return {nullptr, false}; } +namespace +{ + +class CollectPrewhereTableExpressionVisitor : public ConstInDepthQueryTreeVisitor +{ +public: + const QueryTreeNodePtr & getTableExpression() const + { + return table_expression; + } + + void visitImpl(const QueryTreeNodePtr & node) + { + const auto * column_node = node->as(); + if (!column_node) + return; + + auto column_source = column_node->getColumnSourceOrNull(); + if (!column_source || (!column_source->as() && !column_source->as())) + return; + + if (!table_expression) + table_expression = std::move(column_source); + } + + static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node) + { + const auto child_type = child_node->getNodeType(); + return child_type != QueryTreeNodeType::QUERY + && child_type != QueryTreeNodeType::UNION + && child_type != QueryTreeNodeType::LAMBDA; + } + +private: + QueryTreeNodePtr table_expression; +}; + +} + +QueryTreeNodePtr getPrewhereTableExpression(const QueryTreeNodePtr & prewhere) +{ + if (!prewhere) + return {}; + + CollectPrewhereTableExpressionVisitor visitor; + visitor.visit(prewhere); + return visitor.getTableExpression(); +} + /** There are no limits on the maximum size of the result for the subquery. * Since the result of the query is not the result of the entire query. */ diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 1e1182c76649a..14990e2aa4400 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -162,6 +162,12 @@ void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const Stri /// Second element of pair is true if there is at most one source, false if there are multiple sources. std::pair getExpressionSource(const QueryTreeNodePtr & node); +/** Return the table or table function that a `PREWHERE` expression's columns come from, or nullptr + * when the expression has no such column (a constant `PREWHERE`). Nested `QUERY` / `UNION` / `LAMBDA` + * nodes are not visited, so a set subquery does not hide the outer column source. + */ +QueryTreeNodePtr getPrewhereTableExpression(const QueryTreeNodePtr & prewhere); + /// Update mutable context for subquery execution void updateContextForSubqueryExecution(ContextMutablePtr & mutable_context); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 6c80af15b999a..5e54593d74fe5 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -240,6 +240,13 @@ void tryRewriteGlobalRightJoinAsLeftJoin(QueryNode & query_node, const ContextPt if (!left_storage || left_storage->getShardCount() < 2) return; + /** A `PREWHERE` without a column is bound to the leftmost table. There is no column source that + * `buildQueryTreeForShard` could use to recognize that table after the swap, so leave this join + * unchanged and let the initiator-side wrapper preserve the filter. + */ + if (query_node.hasPrewhere() && !getPrewhereTableExpression(query_node.getPrewhere())) + return; + /** A `JOIN USING` key records its sides positionally, the left one first. The join condition, the * `USING (a AS b)` clause shipped to the shards and the key supertype all read that order, so the * sides have to be swapped together with the table expressions. A key that does not hold a plain @@ -1129,7 +1136,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres const SelectQueryOptions & select_query_options, PlannerContextPtr & planner_context, bool is_single_table_expression, - bool wrap_read_columns_in_subquery) + bool wrap_read_columns_in_subquery, + QueryTreeNodePtr & query_prewhere) { const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); @@ -1164,9 +1172,16 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres auto columns = table_expression_data.getColumns(); table_expression = buildSubqueryToReadColumnsFromTableExpression(columns, original_table_expression, query_context); + /** This table owns the outer `PREWHERE`. Move it onto the wrap and drop the initiator copy. + * `appendSetsFromActionsDAG` would otherwise keep the original `IN` sets as useful after the + * nested planner built a separate copy. + */ + const bool move_owned_prewhere = query_prewhere && table_expression_data.getPrewhereFilterActions(); + /// Wrap is planned as `SELECT cols FROM icebergCluster` with no JOIN. Copy left-only /// WHERE/PREWHERE so initiator file listing sees the same predicate as a single-table /// `icebergCluster` read. Same helper as `IStorageCluster::updateQueryWithJoinToSendIfNeeded`. + /// Skip `PREWHERE` when this table owns it: the full predicate is moved below. if (can_prefilter_wrapped_table) { auto copy_left_only = [&](const QueryTreeNodePtr & predicate) -> QueryTreeNodePtr @@ -1183,12 +1198,19 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (auto pred = copy_left_only(parent_query->getWhere())) wrap_query.getWhere() = std::move(pred); } - if (parent_query->hasPrewhere()) + if (!move_owned_prewhere && parent_query->hasPrewhere()) { if (auto pred = copy_left_only(parent_query->getPrewhere())) wrap_query.getPrewhere() = std::move(pred); } } + + if (move_owned_prewhere) + { + table_expression->as().getPrewhere() = query_prewhere->clone(); + query_prewhere = {}; + table_expression_data.resetPrewhereFilterActions(); + } } auto * table_node = table_expression->as(); @@ -2448,7 +2470,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, select_query_options, planner_context, is_single_table_expression, - should_wrap_left_table /*wrap_read_columns_in_subquery*/); + should_wrap_left_table /*wrap_read_columns_in_subquery*/, + query_node_typed.getPrewhere()); if (left_table_expression_query_plan.stage != QueryProcessingStage::FetchColumns) return left_table_expression_query_plan; @@ -2548,7 +2571,8 @@ JoinTreeQueryPlan buildJoinTreeQueryPlan(const QueryTreeNodePtr & query_node, select_query_options, planner_context, is_single_table_expression, - is_remote /*wrap_read_columns_in_subquery*/)); + is_remote /*wrap_read_columns_in_subquery*/, + query_node_typed.getPrewhere())); } } diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h index 29b765a27dc77..8f28a27c153aa 100644 --- a/src/Planner/TableExpressionData.h +++ b/src/Planner/TableExpressionData.h @@ -247,6 +247,12 @@ class TableExpressionData prewhere_filter_actions = std::move(prewhere_filter_actions_value); } + /// Drop initiator-side `PREWHERE` actions after the predicate has been moved onto a subquery. + void resetPrewhereFilterActions() + { + prewhere_filter_actions.reset(); + } + const std::optional & getFilterActions() const { return filter_actions; diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 9669cd40ad7e1..6155ffc965126 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -152,6 +152,7 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito struct InFunctionOrJoin { QueryTreeNodePtr query_node; + QueryTreeNodePtr parent_query_node; size_t subquery_depth = 0; }; @@ -180,6 +181,9 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito void enterImpl(QueryTreeNodePtr & node) { + if (node->getNodeType() == QueryTreeNodeType::QUERY) + query_node_stack.push_back(node); + auto * function_node = node->as(); auto * join_node = node->as(); CrossJoinNode * cross_join_node = find_cross_join ? node->as() : nullptr; @@ -190,6 +194,8 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito { InFunctionOrJoin in_function_or_join_entry; in_function_or_join_entry.query_node = node; + if (!query_node_stack.empty()) + in_function_or_join_entry.parent_query_node = query_node_stack.back(); in_function_or_join_entry.subquery_depth = getSubqueryDepth(); global_in_or_join_nodes.push_back(std::move(in_function_or_join_entry)); return; @@ -200,6 +206,8 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito { InFunctionOrJoin in_function_or_join_entry; in_function_or_join_entry.query_node = node; + if (!query_node_stack.empty()) + in_function_or_join_entry.parent_query_node = query_node_stack.back(); in_function_or_join_entry.subquery_depth = getSubqueryDepth(); in_function_or_join_stack.push_back(in_function_or_join_entry); return; @@ -213,6 +221,9 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito { if (!in_function_or_join_stack.empty() && node.get() == in_function_or_join_stack.back().query_node.get()) in_function_or_join_stack.pop_back(); + + if (node->getNodeType() == QueryTreeNodeType::QUERY) + query_node_stack.pop_back(); } private: @@ -284,6 +295,7 @@ class DistributedProductModeRewriteInJoinVisitor : public InDepthQueryTreeVisito } std::vector in_function_or_join_stack; + std::vector query_node_stack; std::unordered_map replacement_map; std::vector global_in_or_join_nodes; @@ -761,6 +773,14 @@ QueryTreeNodePtr buildQueryTreeForShard( for (const auto & global_in_or_join_node : global_in_or_join_nodes) { + /** `PREWHERE` is bound to the broadcast table and is moved into that subquery (below). + * Nested `GLOBAL IN` / `GLOBAL JOIN` nodes collected from the original `PREWHERE` then + * point at a tree that is no longer attached. Executing them here would materialize the + * nested subquery a second time, after `executeSubqueryNode` already ran the moved copy. + */ + if (!isNodePartOfTree(global_in_or_join_node.query_node.get(), query_tree_to_modify.get())) + continue; + if (auto * join_node = global_in_or_join_node.query_node->as()) { QueryTreeNodePtr join_table_expression; @@ -780,6 +800,24 @@ QueryTreeNodePtr buildQueryTreeForShard( auto subquery_node = getSubqueryFromTableExpression(join_table_expression, column_source_to_columns, planner_context->getQueryContext()); + auto * parent_query_node = global_in_or_join_node.parent_query_node + ? global_in_or_join_node.parent_query_node->as() + : nullptr; + if (parent_query_node && parent_query_node->hasPrewhere()) + { + auto prewhere_table_expression = getPrewhereTableExpression(parent_query_node->getPrewhere()); + + if (prewhere_table_expression && isNodePartOfTree(prewhere_table_expression.get(), join_table_expression.get())) + { + auto * subquery_query_node = subquery_node->as(); + if (!subquery_query_node || subquery_query_node->hasPrewhere()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot move PREWHERE into GLOBAL JOIN subquery"); + + subquery_query_node->getPrewhere() = parent_query_node->getPrewhere()->clone(); + parent_query_node->getPrewhere() = {}; + } + } + auto temporary_table_expression_node = executeSubqueryNode(subquery_node, planner_context->getMutableQueryContext(), global_in_or_join_node.subquery_depth); diff --git a/tests/queries/0_stateless/05194_distributed_global_join_prewhere.reference b/tests/queries/0_stateless/05194_distributed_global_join_prewhere.reference new file mode 100644 index 0000000000000..d051191c13c16 --- /dev/null +++ b/tests/queries/0_stateless/05194_distributed_global_join_prewhere.reference @@ -0,0 +1,37 @@ +without_prewhere +1 a 1 A 4 +2 b 2 B 4 +\N \N 3 C 2 +issue_prewhere +1 a 1 A 4 +2 b 2 B 4 +\N \N 3 C 2 +left_prewhere +1 a 1 A 4 +\N \N 2 B 2 +\N \N 3 C 2 +right_prewhere +1 a 1 A 4 +\N \N 3 C 2 +constant_prewhere +\N \N 1 A 2 +\N \N 2 B 2 +\N \N 3 C 2 +distributed_product_mode +1 a 1 A 4 +\N \N 2 B 2 +\N \N 3 C 2 +prewhere_with_set +1 a 1 A 4 +\N \N 2 B 2 +\N \N 3 C 2 +full_join +1 a 1 A 4 +\N \N 2 B 2 +\N \N 3 C 2 +one_shard +1 a 1 A 2 +\N \N 2 B 2 +\N \N 3 C 2 +remote_right_prewhere +1 a 1 A 2 diff --git a/tests/queries/0_stateless/05194_distributed_global_join_prewhere.sql b/tests/queries/0_stateless/05194_distributed_global_join_prewhere.sql new file mode 100644 index 0000000000000..a0f5c9b15b14b --- /dev/null +++ b/tests/queries/0_stateless/05194_distributed_global_join_prewhere.sql @@ -0,0 +1,116 @@ +-- Tags: distributed + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05194; +DROP TABLE IF EXISTS right_local_05194; +DROP TABLE IF EXISTS left_distributed_05194; +DROP TABLE IF EXISTS right_distributed_05194; +DROP TABLE IF EXISTS left_one_shard_05194; + +CREATE TABLE left_local_05194 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05194 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE left_distributed_05194 AS left_local_05194 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), left_local_05194); + +CREATE TABLE right_distributed_05194 AS right_local_05194 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05194); + +CREATE TABLE left_one_shard_05194 AS left_local_05194 +ENGINE = Distributed('test_shard_localhost', currentDatabase(), left_local_05194); + +INSERT INTO left_local_05194 VALUES (1, 'a'), (2, 'b'); +INSERT INTO right_local_05194 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +SELECT 'without_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +GROUP BY ALL +ORDER BY ALL; + +SELECT 'issue_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.v1 != '' +GROUP BY ALL +ORDER BY ALL; + +SELECT 'left_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.v1 = 'a' +GROUP BY ALL +ORDER BY ALL; + +SELECT 'right_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE r.v2 != 'B' +GROUP BY ALL +ORDER BY ALL; + +SELECT 'constant_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE 0 +GROUP BY ALL +ORDER BY ALL; + +SELECT 'distributed_product_mode'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.v1 = 'a' +GROUP BY ALL +ORDER BY ALL +SETTINGS distributed_product_mode = 'global'; + +SELECT 'prewhere_with_set'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.k1 IN (SELECT 1) +GROUP BY ALL +ORDER BY ALL; + +SELECT 'full_join'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05194 AS l +GLOBAL FULL JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.v1 = 'a' +GROUP BY ALL +ORDER BY ALL; + +SELECT 'one_shard'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_one_shard_05194 AS l +GLOBAL RIGHT JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE l.v1 = 'a' +GROUP BY ALL +ORDER BY ALL; + +SELECT 'remote_right_prewhere'; +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_local_05194 AS l +INNER JOIN right_distributed_05194 AS r ON l.k1 = r.k2 +PREWHERE r.v2 = 'A' +GROUP BY ALL +ORDER BY ALL; + +DROP TABLE left_one_shard_05194; +DROP TABLE left_distributed_05194; +DROP TABLE right_distributed_05194; +DROP TABLE left_local_05194; +DROP TABLE right_local_05194; diff --git a/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.reference b/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.reference new file mode 100644 index 0000000000000..8db38839e4e58 --- /dev/null +++ b/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.reference @@ -0,0 +1,3 @@ +1 a 1 A 4 +\N \N 2 B 2 +\N \N 3 C 2 diff --git a/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.sql b/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.sql new file mode 100644 index 0000000000000..22e429d29a49b --- /dev/null +++ b/tests/queries/0_stateless/05210_distributed_global_join_prewhere_in_subquery.sql @@ -0,0 +1,42 @@ +-- Tags: distributed +-- Nested GLOBAL IN inside PREWHERE must not be materialized twice after the +-- PREWHERE is moved into the GLOBAL JOIN broadcast subquery. +-- https://github.com/ClickHouse/ClickHouse/pull/119780 + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05210; +DROP TABLE IF EXISTS right_local_05210; +DROP TABLE IF EXISTS left_distributed_05210; +DROP TABLE IF EXISTS right_distributed_05210; + +CREATE TABLE left_local_05210 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05210 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE left_distributed_05210 AS left_local_05210 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), left_local_05210); + +CREATE TABLE right_distributed_05210 AS right_local_05210 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05210); + +INSERT INTO left_local_05210 VALUES (1, 'a'), (2, 'b'); +INSERT INTO right_local_05210 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_distributed_05210 AS l +GLOBAL RIGHT JOIN right_distributed_05210 AS r ON l.k1 = r.k2 +PREWHERE l.k1 IN (SELECT k2 FROM right_distributed_05210 WHERE k2 = 1) +GROUP BY ALL +ORDER BY ALL +SETTINGS distributed_product_mode = 'global'; + +DROP TABLE left_distributed_05210; +DROP TABLE right_distributed_05210; +DROP TABLE left_local_05210; +DROP TABLE right_local_05210; diff --git a/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.reference b/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.reference new file mode 100644 index 0000000000000..f3f5c6d974099 --- /dev/null +++ b/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.reference @@ -0,0 +1 @@ +1 a 1 A 2 diff --git a/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.sql b/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.sql new file mode 100644 index 0000000000000..6de1022188535 --- /dev/null +++ b/tests/queries/0_stateless/05211_distributed_remote_prewhere_in_subquery.sql @@ -0,0 +1,37 @@ +-- Tags: distributed +-- Wrapping a remote non-leftmost table must move `PREWHERE` (including `GLOBAL IN`) +-- onto that subquery exclusively, so the outer planner does not keep the +-- original not-ready `Set`. +-- https://github.com/ClickHouse/ClickHouse/pull/119780 + +SET enable_analyzer = 1; +SET join_use_nulls = 1; + +DROP TABLE IF EXISTS left_local_05211; +DROP TABLE IF EXISTS right_local_05211; +DROP TABLE IF EXISTS right_distributed_05211; + +CREATE TABLE left_local_05211 (k1 UInt32, v1 String) +ENGINE = MergeTree +ORDER BY k1; + +CREATE TABLE right_local_05211 (k2 UInt32, v2 String) +ENGINE = MergeTree +ORDER BY k2; + +CREATE TABLE right_distributed_05211 AS right_local_05211 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), right_local_05211); + +INSERT INTO left_local_05211 VALUES (1, 'a'), (2, 'b'); +INSERT INTO right_local_05211 VALUES (1, 'A'), (2, 'B'), (3, 'C'); + +SELECT l.k1, l.v1, r.k2, r.v2, count() +FROM left_local_05211 AS l +INNER JOIN right_distributed_05211 AS r ON l.k1 = r.k2 +PREWHERE r.k2 GLOBAL IN (SELECT k2 FROM right_distributed_05211 WHERE k2 = 1) +GROUP BY ALL +ORDER BY ALL; + +DROP TABLE right_distributed_05211; +DROP TABLE left_local_05211; +DROP TABLE right_local_05211; From ba2bac0897ef9737253aaad0c3b2852c19022767 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 14:01:27 +0200 Subject: [PATCH 09/52] CAS GC: batch the namespace janitor's deletes under a soft time budget The namespace janitor deleted a dropped table's dead `_log`/`_snap` keys one page per round, one `HEAD` + `DELETE` per key. `Gc::runNamespaceJanitor` now repeats `NamespaceJanitor::runOnePage` while a page deleted something, until a 20 s soft budget. A page deletes its dead `_log`/`_snap` keys as write-once batches split evenly across the GC I/O pool; `_ckpt` and `_files` keep the exact-token delete. After the last page of the stream the pass continues from its beginning, so debris on both sides of the cursor drains in one round. A smaller implementation of https://github.com/Altinity/ClickHouse/pull/2468. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- .../cas/architecture/garbage-collection.md | 33 +- docs/en/antalya/cas/configuration.md | 2 +- .../en/operations/system-tables/cas_gc_log.md | 2 +- src/Common/ProfileEvents.cpp | 2 +- src/Common/setThreadName.h | 1 + .../ContentAddressedSettings.cpp | 2 +- .../ContentAddressed/Formats/CasLayout.h | 22 +- .../ContentAddressed/Gc/CasGc.cpp | 101 +++++- .../ContentAddressed/Gc/CasGc.h | 16 +- .../Gc/CasNamespaceJanitor.cpp | 60 +++- .../ContentAddressed/Gc/CasNamespaceJanitor.h | 19 +- .../tests/gtest_cas_gc_frontier_gate.cpp | 19 +- src/Disks/tests/gtest_cas_gc_round_defer.cpp | 6 +- .../gtest_cas_namespace_janitor_batches.cpp | 334 ++++++++++++++++++ .../test_cas_janitor_drain/__init__.py | 0 .../configs/storage_conf.xml | 36 ++ .../test_cas_janitor_drain/test.py | 86 +++++ 17 files changed, 688 insertions(+), 53 deletions(-) create mode 100644 src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp create mode 100644 tests/integration/test_cas_janitor_drain/__init__.py create mode 100644 tests/integration/test_cas_janitor_drain/configs/storage_conf.xml create mode 100644 tests/integration/test_cas_janitor_drain/test.py diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 21953307f9d15..81f57d608ca71 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -57,7 +57,7 @@ follower or a deferred round execution returns before that commit. | 13 | `round_commit` | fold | Retention-prune old generations, then publish the single `gc/state` `CAS` that adopts the whole round | | 14 | `handoff_reclaim` | post-`CAS` | Reclaim a generation a ref moved off during this round, which the ordinary retention prune already skipped and will not revisit | | 15 | `manifest_deletes` | post-`CAS` | Delete manifest bodies whose owner-removal minus-one edge the `CAS` in phase 13 just adopted | -| 16 | `namespace_cleanup` | leader; suppressed on `DEFER` | One bounded page of the perpetual namespace janitor, reclaiming dead-life debris | +| 16 | `namespace_cleanup` | leader; suppressed on `DEFER` | Pages of the perpetual namespace janitor under a 20 s soft budget, reclaiming dead-life debris | | 17 | `ref_object_cleanup` | post-`CAS` | Prune ref logs and snapshots once both fold coverage and a live snapshot make them safe to delete | | 18 | `orphan_sweep` | post-`CAS` | Exact-token deletion for the [orphan-manifest sweep](/antalya/cas/architecture/manifests-and-refs#orphan-sweep), after phase 13 adopted each candidate's blob-source retirements and the cursor | @@ -518,23 +518,32 @@ then picked up by the orphan-manifest sweep (phase 18). ## Phase 16 — namespace cleanup {#phase-16-namespace-cleanup} -One bounded page of the perpetual namespace janitor: deletes the physical objects of namespace lives -no longer in the catalog (dead-life debris). +Pages of the perpetual namespace janitor: deletes the physical objects of namespace lives +no longer in the catalog (dead-life debris). The phase takes the next page while the previous one +deleted something and published its cursor, until a 20 s soft budget: no page starts after it, the +page in progress finishes. After the last page of `cas/ns/` the next one starts from its beginning, so +debris on both sides of the cursor drains in one round. A pool without debris costs one `LIST` per +round. - **Runs on:** fold path here; also on the deferred path right after phase 4 with `suppress_destructive` forced on -- **Reads:** the durable `janitor_cursor`; one `LIST` page (≤ 1000 keys) of `cas/ns/`; a fresh - ref-catalog snapshot; `gc/state` per fence re-check -- **Writes / deletes:** exact-token `DELETE` per dead-life `_log` / `_snap` / `_ckpt` / `_files` - object; one `CAS` on the maintenance state when the page is decided -- **Safety:** each delete is under a GC fence re-check (`lease.owner` / `lease.seq`) before it and - once at the end; the incarnation segment in every key makes an old life's objects structurally +- **Reads:** per page: the durable `janitor_cursor`; one `LIST` page (≤ 1000 keys) of `cas/ns/`; a + fresh ref-catalog snapshot; `gc/state` for the fence re-check +- **Writes / deletes:** per page: batch `DELETE`s of the dead-life `_log` / `_snap` objects, which + are write-once and need no token; the page's keys are split evenly across the GC I/O pool + (`cas_gc_io_concurrency`), at most `cas_gc_bulk_delete_chunk_keys` per request, and a storage + without a batch delete gets one request per key from the same jobs; exact-token `DELETE` per + dead-life `_ckpt` / `_files` object; one `CAS` on the maintenance state when the page is decided +- **Safety:** each request is under a GC fence re-check (`lease.owner` / `lease.seq`), read once per + page and checked again before the cursor is published; the incarnation segment in every key makes an old life's objects structurally unreachable from a reborn same-name namespace, so a missed key can only leak storage, never expose it -- **Fails the round if:** nothing — the whole page is wrapped in a catch-all ("namespace janitor - skipped this round") +- **Fails the round if:** nothing — the whole phase is wrapped in a catch-all ("namespace janitor + stopped this round"). A failed batch `DELETE` leaks its keys and the cursor still advances - **Observability:** phase row `namespace_cleanup`; metrics `janitor_pages`, `janitor_keys`, - `janitor_deleted`, `leaked` + `janitor_deleted` (keys of successful batch requests, absent ones included, plus exact removals), + `leaked`, `budget_exhausted`. Batch deletes run on the GC I/O pool, so the row's `ProfileEvents` + do not include their requests The cursor advances only when the whole page was decided under a held fence and an unambiguous catalog; under suppression it lists and classifies but deletes nothing and does not advance. diff --git a/docs/en/antalya/cas/configuration.md b/docs/en/antalya/cas/configuration.md index 17b1e3482768b..84be3753f7323 100644 --- a/docs/en/antalya/cas/configuration.md +++ b/docs/en/antalya/cas/configuration.md @@ -106,7 +106,7 @@ entirely before release. Treat this table as a snapshot of the current build, no | `cas_part_folder_cache_max_entry_bytes` | 16 MiB | Oversized part-folder views bypass retention above this size | | `cas_manifest_decode_cache_bytes` | 128 MiB | Manifest decode cache byte budget (`0` disables) | | `cas_gc_meta_pool_size` | `16` | Bounded pool size for GC per-hash freshness-meta writes | -| `cas_gc_io_concurrency` | `16` | Bounded pool size for GC object-storage requests that run in parallel: the fold's read-ahead (checkpoints, ref logs, manifests, zero-candidate HEADs), the orphan-manifest sweep planning reads, the `SYSTEM CAS GC REBUILD` read-ahead, and the `pending_deletes` blob `HEAD` + conditional `DELETE` fan-out. Not covered: meta writes (`cas_gc_meta_pool_size`) and all other GC requests, which run on the round thread. `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | +| `cas_gc_io_concurrency` | `16` | Bounded pool size for GC object-storage requests that run in parallel: the fold's read-ahead (checkpoints, ref logs, manifests, zero-candidate HEADs), the orphan-manifest sweep planning reads, the `SYSTEM CAS GC REBUILD` read-ahead, the `pending_deletes` blob `HEAD` + conditional `DELETE` fan-out, and the namespace janitor's batch deletes of dead ref logs and snapshots. Not covered: meta writes (`cas_gc_meta_pool_size`) and all other GC requests, which run on the round thread. `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | | `cas_attempt_timeout_ms` | `5000` | Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. Together with the connect cap it forms the attempt envelope (`cas_attempt_timeout_ms + 2 × cap`; the cap is `cas_attempt_timeout_ms` itself when the disk's `connect_timeout_ms` is `0`, else `min(connect_timeout_ms, cas_attempt_timeout_ms)`) that the lease arithmetic reserves: one TCP connect and one TLS handshake under the cap each, send/receive bounded per socket operation by `cas_attempt_timeout_ms`. With background renewal the cadence check requires `cas_mount_renew_period_ms + 2 × envelope + cas_lease_safety_margin_ms < cas_mount_lease_ttl_ms`, which puts an effective ceiling on the frozen connect cap: under the defaults (TTL 30000, period 10000, margin 2000) the envelope must stay under 9000, so a disk `connect_timeout_ms` of 2000 ms or more refuses to open writable — lower the connect timeout or raise the TTL if you hit this | | `cas_lease_safety_margin_ms` | `2000` | Startup-only margin validated against the mount lease TTL: the attempt envelope + `cas_lease_safety_margin_ms` must be strictly less than the mount lease TTL, and `cas_mount_renew_period_ms` + 2 × envelope + `cas_lease_safety_margin_ms` too, or the disk refuses to open writable | | `cas_unsafe_remount_no_delay` | `0` | Reclaim a mount slot that carries this server's own uuid at once after a hard restart, without observing the slot's token for the lease TTL. Unsafe whenever two processes can hold the same `server_uuid` (a copied uuid file, a stalled predecessor). After such a reclaim the predecessor can still start conditional writes until its own cutoff (`confirmed deadline − cas_lease_safety_margin_ms − 2 × envelope`) or until its next renewal meets the token guard, and a request it already sent may still materialize later. That is not a data hazard: ref-log keys carry `(writer_epoch, sequence)` and creates are conditional, so two writers can never commit different bodies to one key, and recovery's epoch seal settles any straggler (recovery fails closed after 64 successive seal-create attempts displaced by newly materializing old-epoch transactions). The exposure is availability, not data. Intended for test stands and deployments that guarantee one process per uuid | diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index 46fcd559cfb02..e8b5138bc85b3 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -86,7 +86,7 @@ The phases, in execution order: | `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | | `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | | `manifest_deletes` | Exact-token deletes of owner-removed manifest bodies, after their decrements were adopted. | one `DELETE` per body | -| `namespace_cleanup` | Run one bounded `cas/ns/` page across the stream and state subtrees for the perpetual dead-life janitor. This phase is physical reclamation, not a lifecycle gate. | one namespace-root page `LIST`, catalog cut, exact-token deletes | +| `namespace_cleanup` | Run `cas/ns/` pages across the stream and state subtrees for the perpetual dead-life janitor, under a 20 s soft budget; `budget_exhausted` is 1 when the budget stopped the phase. This phase is physical reclamation, not a lifecycle gate. | per page: one namespace-root page `LIST`, catalog cut, batch deletes of dead ref logs and snapshots, exact-token deletes of the rest | | `ref_object_cleanup` | Delete ref logs covered by both the durable fold cursor and a durable snapshot, plus superseded snapshots. | one `HEAD` + one `DELETE` per deletable object | | `orphan_sweep` | The budgeted, cursor-paced orphan part-manifest backstop. | budgeted `LIST` and deletes | diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index a9986b5aa4858..3ef8a1752aa09 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -927,7 +927,7 @@ The server successfully detected this situation and will download merged part fr M(CASGCRefWalkPlansBuilt, "Number of complete catalog-authoritative CAS ref walk plans constructed by ordinary GC and rebuild. A regular or rebuilding invocation that reaches the post-LIST catalog cut increments this exactly once, including a round that later defers.", ValueType::Number) \ M(CASGCUnmatchedAdoptedParentLives, "Number of adopted-parent CAS ref-life rows dropped because the post-LIST catalog cut has no matching physical life. Each occurrence is inert for planning and suppression and is logged with its exact physical life id; a persistent nonzero rate indicates old generation state is outliving catalog removal.", ValueType::Number) \ M(CASGCStuckRemovals, "Number of adopted CAS GC rounds that observed a Removing namespace at or beyond the diagnostic age threshold without terminal cleanup evidence. Incremented and warned every such round; diagnostic only, with no effect on folding, suppression, appends, or deletion.", ValueType::Number) \ - M(CASGCNamespaceCleanupLeaks, "Number of dead-life namespace objects whose reclamation the perpetual janitor could not confirm because HEAD or exact-delete failed. Each occurrence is logged with the exact key and remains leak-only: it neither suppresses destructive GC nor blocks catalog lifecycle progress.", ValueType::Number) \ + M(CASGCNamespaceCleanupLeaks, "Number of dead-life namespace objects whose reclamation the perpetual janitor could not confirm because HEAD, exact-delete or a batch delete failed. Each occurrence is logged with the key, or the first key of the batch, and remains leak-only: it neither suppresses destructive GC nor blocks catalog lifecycle progress.", ValueType::Number) \ M(CASDetachedWorkDrainTimeouts, "Counts CAS storage teardowns whose bounded wait for detached background work expired with work still in flight. The teardown proceeds, but for that teardown it could not be established that no tracked task still holds the pool. Expected to stay at zero.", ValueType::Number) \ M(CASEventDroppedContextExpired, "Number of CAS system-log events dropped because the storage's `Context` reference expired before delivery. A non-zero value indicates event production outlived the owning server context.", ValueType::Number) \ M(CASGCUnappliedFoldedTransactions, "Number of ref transactions a GC round folded and merged but whose blob deltas never reached a shard reducer. Always 0 on a healthy round; a nonzero value fails the round closed, because the round would otherwise advance its fold cursor past a transaction it never applied.", ValueType::Number) \ diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 8a0222dc96392..b34b48cc51716 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -35,6 +35,7 @@ namespace DB M(CAS_ANOMALY_DIAG, "CasAnomalyDiag") \ M(CAS_GC_HEARTBEAT, "CasGcHeartbeat") \ M(CAS_GC_REDELETE, "CasGcRedelete") \ + M(CAS_GC_JANITOR, "CasGcJanitor") \ M(CAS_GC_SCHEDULER, "CasGcSched") \ M(CAS_LEASE_RENEWER, "CasLeaseRenewer") \ M(CAS_REF_SNAPSHOT_PUBLISH, "CasRefSnapPub") \ diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp index 7331d2bad29a1..7c6b80c540fff 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp @@ -79,7 +79,7 @@ constexpr std::string_view CAS_KEY_PREFIX = "cas_"; DECLARE(UInt64, part_folder_cache_max_entry_bytes, 16ULL << 20, "Oversized part-folder views bypass retention above this size", 0) \ DECLARE(UInt64, manifest_decode_cache_bytes, 128ULL << 20, "Manifest DECODE cache byte budget (0 disables)", 0) \ DECLARE(UInt64, gc_meta_pool_size, 16, "Bounded pool size for GC per-hash freshness-meta writes", 0) \ - DECLARE(UInt64, gc_io_concurrency, 16, "Maximum number of threads in the GC I/O pool. Used for fold and rebuild read-ahead, orphan-manifest sweep planning reads, and pending_deletes HEAD plus conditional DELETE. Per-hash meta writes use gc_meta_pool_size; other GC requests run on the round thread. 1 disables parallel GC I/O", 0) \ + DECLARE(UInt64, gc_io_concurrency, 16, "Maximum number of threads in the GC I/O pool. Used for fold and rebuild read-ahead, orphan-manifest sweep planning reads, pending_deletes HEAD plus conditional DELETE, and namespace janitor batch deletes. Per-hash meta writes use gc_meta_pool_size; other GC requests run on the round thread. 1 disables parallel GC I/O", 0) \ DECLARE(UInt64, gc_bulk_delete_chunk_keys, 1000, "Keys per batch delete request in GC's write-once families (owner-removed manifest bodies, covered ref logs and snapshots); 1 to 1000", 0) \ DECLARE(UInt64, attempt_timeout_ms, 5000, "Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. With the connect cap it forms the attempt envelope the lease arithmetic reserves", 0) \ DECLARE(UInt64, lease_safety_margin_ms, 2000, "Startup-only margin validated against the mount lease TTL: attempt envelope + this must be strictly less than the TTL, and renew period + 2 × envelope + this too", 0) \ diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h index 5ab57bb7854b0..856b623b99b65 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h @@ -158,7 +158,7 @@ class Layout /// not try an uncompressed variant. String refLogKey(const NamespaceLifeId & ns_id, const RefTxnId & id) const { - return namespaceStreamPrefix(ns_id) + "_log/" + renderRefTxnId(id) + String(storedSuffix(FormatId::RefLog)); + return refStreamObjectKey(ns_id.incarnation, RefObjectKind::Log, id); } /// Writer-published table snapshot at `.../_snap/.zst`. The snapshot @@ -166,7 +166,7 @@ class Layout /// `X` reuses the `RefTxnId` of the last log it covers. String refSnapshotKey(const NamespaceLifeId & ns_id, const RefTxnId & id) const { - return namespaceStreamPrefix(ns_id) + "_snap/" + renderRefTxnId(id) + String(storedSuffix(FormatId::RefSnapshot)); + return refStreamObjectKey(ns_id.incarnation, RefObjectKind::Snap, id); } /// The write-once forms of `refLogKey` and `refSnapshotKey`: the same strings, typed as keys a @@ -181,6 +181,17 @@ class Layout return WriteOnceKey(refSnapshotKey(ns_id, id)); } + /// The write-once form of a listed `_log`/`_snap` key that `parseRefObjectKey` accepted. `std::nullopt` + /// when `listed_key` is not the canonical key of the parsed identity: a precondition-free delete must + /// name only keys this layout writes. + std::optional writeOnceStreamKey(const ParsedRefObjectKey & parsed, std::string_view listed_key) const + { + String canonical = refStreamObjectKey(parsed.life_id, parsed.kind, parsed.txn_id); + if (canonical != listed_key) + return std::nullopt; + return WriteOnceKey(std::move(canonical)); + } + /// The life's checkpoint object (spec INV-4) at `/cas/ns/state//_ckpt`. Unlike /// immutable stream objects it is mutable (token-CAS), carries no transaction id, and therefore lives /// in the point/path-addressed state tree rather than a `_log`/`_snap` directory -- @@ -486,6 +497,13 @@ class Layout /// Parses the one physical-id segment after the rest of a life-owned key identified its family. NamespaceLifePhysicalId namespaceLifePhysicalIdOf(std::string_view key, std::string_view segment) const; + String refStreamObjectKey(NamespaceLifePhysicalId life_id, RefObjectKind kind, const RefTxnId & id) const + { + const bool log = kind == RefObjectKind::Log; + return namespaceStreamRootPrefix() + renderIncarnation(life_id) + (log ? "/_log/" : "/_snap/") + renderRefTxnId(id) + + String(storedSuffix(log ? FormatId::RefLog : FormatId::RefSnapshot)); + } + /// Build ///. /// Throws BAD_ARGUMENTS if id is shorter than 2 characters. String shardedKey(const String & ns, const String & id) const diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index f871182c8edb4..c407cdf099885 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -85,6 +85,9 @@ namespace DB::Cas namespace { +/// Soft limit on the namespace janitor's phase: no page starts after it, the page in progress finishes. +constexpr uint64_t kJanitorBudgetMs = 20'000; + /// The `on_page_fetched` hook GC passes to every `forEachListedKey`/`recoverRefTable` /// call it owns (never passed by fsck/offline-repair callers of those shared helpers) -- one increment /// per physical LIST page, never per listed key. @@ -360,34 +363,106 @@ Gc::Gc(PoolPtr store_, UInt128 gc_id_, std::function now_ms_fn_, io_pool_refuse_at_for_test = store->poolConfig().gc_io_pool_refuse_at_for_test; } -void Gc::runNamespaceJanitorPage( +void Gc::runNamespaceJanitor( const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows) { GcPhaseTimer t(phase_sink, "namespace_cleanup"); t.metric("evidence_rows", cleanup_evidence_rows); NamespaceJanitorResult janitor_result; + bool budget_exhausted = false; try { CasRequests & requests = store->openRequests(); const Layout & layout = store->layout(); - NamespaceJanitor janitor(requests, layout, 1000); - /// ONE authority read per page, made here rather than from the predicate: the janitor's - /// operation samples its liveness before every request, and a page walks up to a thousand keys. - refreshAuthority(leased_state.lease.seq); - janitor_result = janitor.runOnePage(suppress_destructive, [this] { return authority_held; }); - for (const String & anomaly : janitor_result.anomalies) - LOG_WARNING(logger, "CAS namespace janitor: {}", anomaly); - if (janitor_result.leaked) - ProfileEvents::increment(ProfileEvents::CASGCNamespaceCleanupLeaks, janitor_result.leaked); + NamespaceJanitor janitor(requests, layout, 1000, + [this](CasOperation & op, const std::vector & keys, std::vector & anomalies) + { + return removeWriteOnceOnIoPool(op, keys, anomalies); + }); + const uint64_t deadline_ms = mono_ms_fn() + kJanitorBudgetMs; + while (true) + { + /// ONE authority read per page, made here rather than from the predicate: the janitor's + /// operation samples its liveness before every request, and a page walks up to a thousand keys. + refreshAuthority(leased_state.lease.seq); + const NamespaceJanitorResult page = janitor.runOnePage(suppress_destructive, [this] { return authority_held; }); + janitor_result.pages += page.pages; + janitor_result.keys += page.keys; + janitor_result.deleted += page.deleted; + janitor_result.leaked += page.leaked; + for (const String & anomaly : page.anomalies) + LOG_WARNING(logger, "CAS namespace janitor: {}", anomaly); + if (!page.more) + break; + if (mono_ms_fn() >= deadline_ms) + { + budget_exhausted = true; + break; + } + } } catch (const std::exception & e) { - LOG_WARNING(logger, "CAS namespace janitor skipped this round: {}", e.what()); + LOG_WARNING(logger, "CAS namespace janitor stopped this round: {}", e.what()); } + if (janitor_result.leaked) + ProfileEvents::increment(ProfileEvents::CASGCNamespaceCleanupLeaks, janitor_result.leaked); t.metric("janitor_pages", janitor_result.pages); t.metric("janitor_keys", janitor_result.keys); t.metric("janitor_deleted", janitor_result.deleted); t.metric("leaked", janitor_result.leaked); + t.metric("budget_exhausted", budget_exhausted ? 1 : 0); +} + +uint64_t Gc::removeWriteOnceOnIoPool( + CasOperation & op, const std::vector & keys, std::vector & anomalies) +{ + /// An even split, so a store without a batch delete sends its one-key requests from every pool thread. + const size_t concurrency = std::max(1, store->poolConfig().gc_io_concurrency); + const size_t chunk_keys = std::min( + std::clamp(store->poolConfig().gc_bulk_delete_chunk_keys, 1, kBulkDeleteMaxKeys), + (keys.size() + concurrency - 1) / concurrency); + std::vector> chunks; + for (size_t begin = 0; begin < keys.size(); begin += chunk_keys) + chunks.emplace_back(keys.begin() + begin, keys.begin() + std::min(keys.size(), begin + chunk_keys)); + + ThreadPoolCallbackRunnerLocal runner(*io_pool, ThreadName::CAS_GC_JANITOR); + std::vector::Task>> handles; + handles.reserve(chunks.size()); + SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); + for (size_t i = 0; i < chunks.size(); ++i) + { + if (io_pool_refuse_at_for_test == i) + { + io_pool_refuse_at_for_test.reset(); + throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS namespace janitor enqueue refusal at index {}", i); + } + /// The round thread waits for every job before it refreshes `authority_held` again. + handles.emplace_back(runner.enqueueAndGiveOwnership( + [this, chunk = &chunks[i], admitted_generation = op.generation()] + { + CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); + removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); + })); + } + ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); + + uint64_t leaked = 0; + for (size_t i = 0; i < handles.size(); ++i) + { + try + { + handles[i]->future.get(); + } + catch (...) + { + leaked += chunks[i].size(); + anomalies.push_back(fmt::format( + "leaked {} dead-life objects starting at '{}': batch delete failed: {}", + chunks[i].size(), chunks[i].front().str(), getCurrentExceptionMessage(false))); + } + } + return leaked; } uint64_t removeChunkWriteOnceOrOneByOne(CasOperation & op, const std::vector & chunk, const Retry & policy) @@ -820,7 +895,7 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al /// DEFER has no `FoldResult`, hence no complete global destructive verdict. The janitor still /// takes its bounded page and catalog cut, but suppression keeps both deletes and valid-page /// cursor progress at the same position for the bounded forced fold to retry. - runNamespaceJanitorPage(state, /*suppress_destructive=*/true, /*cleanup_evidence_rows=*/0); + runNamespaceJanitor(state, /*suppress_destructive=*/true, /*cleanup_evidence_rows=*/0); return report; /// no fold, no pre-CAS deletes, no gc/state CAS — sealed generation stays pinned } @@ -1331,7 +1406,7 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al uint64_t cleanup_evidence_rows = 0; for (const auto & [life_id, ref_life_state] : folded.fold_seal.ref_lives) cleanup_evidence_rows += ref_life_state.cleanup_evidence ? 1 : 0; - runNamespaceJanitorPage(state, suppress_destructive, cleanup_evidence_rows); + runNamespaceJanitor(state, suppress_destructive, cleanup_evidence_rows); /// PHASE 17/18 `ref_object_cleanup`. Emitted even when the whole pass is skipped (`trim_enabled` is /// a test seam, `suppressed` gates the deletes), because "this phase did nothing and why" is exactly /// what a reader of a round that reclaimed nothing needs to see. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 33f2bdcf09a80..8cdebbe98df6a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -548,13 +548,19 @@ class Gc /// It performs no physical LIST or delete. CatalogLifecycleReconcileResult drainCompletedRemoving(const GcState & leased_state); - /// Run exactly one independently paced physical namespace-maintenance page. The caller supplies + /// Run the independently paced physical namespace maintenance: pages from the persisted cursor while + /// each one deleted dead-life objects, under a soft time budget. The caller supplies /// the one round-wide destructive verdict when it exists; DEFER passes suppression because it has - /// no folded frontier verdict. This helper owns only janitor I/O and phase metrics, never lifecycle - /// transitions or the hot stream walk plan. - void runNamespaceJanitorPage( + /// no folded frontier verdict, which takes one page. This helper owns only janitor I/O and phase + /// metrics, never lifecycle transitions or the hot stream walk plan. + void runNamespaceJanitor( const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows); + /// The janitor's `RemoveWriteOnce`: splits `keys` evenly across the GC I/O pool and waits for every job. + /// A failed job leaks its keys. Throws when the pool refuses a job, after the scheduled ones finished. + uint64_t removeWriteOnceOnIoPool( + CasOperation & op, const std::vector & keys, std::vector & anomalies); + void reportStuckRemovals(const RefPlan & plan, uint64_t current_round); /// What one fold produced. The blob deltas are sealed @@ -1024,7 +1030,7 @@ class Gc std::unique_ptr meta_writer; /// The GC I/O pool: the fold's and rebuild's read-ahead, the orphan-manifest sweep planning reads, - /// and the `pending_deletes` fan-out, sized by `gc_io_concurrency`. A `unique_ptr` for the same reason as `meta_writer`: the size comes from + /// the `pending_deletes` fan-out and the namespace janitor's batch deletes, sized by `gc_io_concurrency`. A `unique_ptr` for the same reason as `meta_writer`: the size comes from /// `store->poolConfig()`, which may only be read after the constructor body has validated `store`. std::unique_ptr io_pool; std::optional io_pool_refuse_at_for_test; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp index f8c153ed0f20b..6857c39f49bf5 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include namespace DB::Cas { @@ -19,6 +21,37 @@ void throwOnRefusedOrGaveUp(WriteResult && result, std::string_view what) (void)orThrow(std::move(result), what); } +uint64_t removeOnCallerThread(CasOperation & op, const std::vector & keys, std::vector & anomalies) +{ + uint64_t leaked = 0; + for (size_t begin = 0; begin < keys.size(); begin += kBulkDeleteMaxKeys) + { + const std::vector batch( + keys.begin() + begin, keys.begin() + std::min(keys.size(), begin + kBulkDeleteMaxKeys)); + try + { + op.removeManyWriteOnce(batch, Retry::standard()); + } + catch (const std::exception & e) + { + leaked += batch.size(); + anomalies.push_back(fmt::format( + "leaked {} dead-life objects starting at '{}': batch delete failed: {}", + batch.size(), batch.front().str(), e.what())); + } + } + return leaked; +} + +} + +NamespaceJanitor::NamespaceJanitor( + CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_) + : requests(requests_) + , layout(layout_) + , page_budget(page_budget_) + , remove_write_once(remove_write_once_ ? std::move(remove_write_once_) : removeOnCallerThread) +{ } NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liveness liveness) @@ -70,16 +103,19 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven /// absent objects and token mismatches are final per-key outcomes and therefore do not by /// themselves prevent progress. bool page_decided = !ambiguous && !suppress_deletes; + std::vector dead_stream_keys; for (const ListedKey & listed : page.keys) { std::optional life_id; + std::optional stream_key; try { if (listed.key.starts_with(layout.namespaceStreamRootPrefix())) { - if (const auto parsed = layout.parseRefObjectKey(listed.key)) - life_id = parsed->life_id; + stream_key = layout.parseRefObjectKey(listed.key); + if (stream_key) + life_id = stream_key->life_id; } else if (listed.key.starts_with(layout.namespaceStateRootPrefix())) { @@ -103,6 +139,15 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven if (ambiguous || suppress_deletes || catalog_cut.life_index.resolve(*life_id)) continue; + if (stream_key) + { + if (std::optional write_once = layout.writeOnceStreamKey(*stream_key, listed.key)) + { + dead_stream_keys.push_back(std::move(*write_once)); + continue; + } + } + std::optional etag = listed.etag; if (!etag) { @@ -139,8 +184,15 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven } } + if (page_decided && !dead_stream_keys.empty() && op.admitted()) + { + const uint64_t leaked = remove_write_once(op, dead_stream_keys, result.anomalies); + result.leaked += leaked; + result.deleted += dead_stream_keys.size() - leaked; + } + /// Recheck even when the page had no dead candidate. A tenure that observes fence loss after LIST - /// or after the last exact delete must not publish progress. Loss after this check may still race + /// or after the last delete must not publish progress. Loss after this check may still race /// with the leak-only maintenance CAS; already completed exact deletes remain safe to repeat. if (page_decided && !op.admitted()) page_decided = false; @@ -153,6 +205,8 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven const WriteResult published = casGcMaintenanceState(op, layout, progress.etag, next, Retry::standard()); if (std::holds_alternative(published) || std::holds_alternative(published)) result.anomalies.push_back("cursor publication did not commit"); + else if (std::holds_alternative(published)) + result.more = result.deleted > 0 && !(cursor.empty() && page.next_cursor.empty()); } catch (const std::exception & e) { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h index d23d21402c73b..9a8d3e5a82ff9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h @@ -13,15 +13,24 @@ struct NamespaceJanitorResult uint64_t keys = 0; uint64_t deleted = 0; uint64_t leaked = 0; + /// The page deleted something, published its cursor and did not cover the whole stream by itself, so + /// the next page is worth taking now. After the last page of the stream the next one starts from its + /// beginning. + bool more = false; std::vector anomalies; }; -/// Runs one bounded, leak-only page over the physical namespace ownership tree. +/// Deletes one page's dead `_log`/`_snap` keys under the page's operation. Returns how many keys it could +/// not confirm deleted and adds one anomaly per failed request; throws only when nothing may be published. +using RemoveWriteOnce + = std::function & keys, std::vector & anomalies)>; + +/// Runs one bounded, leak-only page over the physical namespace ownership tree; `Gc` repeats it under a time budget. class NamespaceJanitor { public: - NamespaceJanitor(CasRequests & requests_, const Layout & layout_, size_t page_budget_) - : requests(requests_), layout(layout_), page_budget(page_budget_) {} + /// An empty `remove_write_once_` deletes on the caller's thread, one request per `kBulkDeleteMaxKeys` keys. + NamespaceJanitor(CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_ = {}); /// `liveness` is admitted once for the whole page (one `CasOperation` covers the read, the list, /// every delete and the cursor publication): a fact the fence cannot see, such as "this tenure @@ -31,12 +40,16 @@ class NamespaceJanitor /// that returns false ends whichever request was about to be sent: a read verb (the maintenance /// read, the list, a HEAD) throws out of this call, and a write verb (a delete, the cursor /// publication) reports it as `GaveUp` rather than sending anything. + /// + /// A dead life's canonical `_log`/`_snap` keys are write-once, so the page deletes them together through + /// `remove_write_once` with no per-key `HEAD` or token; `_ckpt` and `_files` keep the exact-token delete. NamespaceJanitorResult runOnePage(bool suppress_deletes, Liveness liveness); private: CasRequests & requests; const Layout & layout; size_t page_budget; + RemoveWriteOnce remove_write_once; }; } diff --git a/src/Disks/tests/gtest_cas_gc_frontier_gate.cpp b/src/Disks/tests/gtest_cas_gc_frontier_gate.cpp index d8c90fa7dc1cf..a35f8c50e6a97 100644 --- a/src/Disks/tests/gtest_cas_gc_frontier_gate.cpp +++ b/src/Disks/tests/gtest_cas_gc_frontier_gate.cpp @@ -241,7 +241,7 @@ class DrainRaceBackend final : public CountingBackend std::function after_read_hook; }; -class PostFoldUnreadableTerminalBackend final : public CountingBackend +class DeadCheckpointHeadFailureBackend final : public CountingBackend { public: /// Unhide the names the primitive overrides below would otherwise shadow. @@ -260,7 +260,7 @@ class PostFoldUnreadableTerminalBackend final : public CountingBackend std::optional head(const String & key, TransportAccess & access) override { if (!bypass_fault && key == unreadable_key) - throw std::runtime_error("injected post-fold terminal read failure for " + key); + throw std::runtime_error("injected dead checkpoint HEAD failure for " + key); return CountingBackend::head(key, access); } @@ -2821,10 +2821,11 @@ TEST(CASGCFrontierGate, CleanupEvidenceLeavesRemovedNamespaceCheckpointForJanito /// Once a terminal has folded, a later physical read failure is janitor debt, not lifecycle evidence /// loss. Removing this per-key leak handling would either make the signal disappear or let one dead -/// object prevent the janitor from considering the rest of its page. -TEST(CASGCFrontierGate, PostFoldUnreadableTerminalIsCountedWithoutSuppressingProgress) +/// object prevent the janitor from considering the rest of its page. The unreadable object is the dead +/// checkpoint: dead `_log`/`_snap` keys are deleted token-free and never read. +TEST(CASGCFrontierGate, PostFoldUnreadableDeadCheckpointIsCountedWithoutSuppressingProgress) { - auto backend = std::make_shared(); + auto backend = std::make_shared(); auto store = openPoolForTest(backend, /*gc_fold_max_defer_rounds=*/0); CasRequests requests = openRequestsForTest(backend); CasOperation op = requests.admit(); @@ -2876,10 +2877,11 @@ TEST(CASGCFrontierGate, PostFoldUnreadableTerminalIsCountedWithoutSuppressingPro dropRefTransition(*backend, layout, progressing, "victim", manifest); const String terminal_key = layout.refLogKey(removed_life, RefTxnId{1, 2}); + const String checkpoint_key = layout.refCkptKey(removed_life); const String later_dead_residue = layout.refLogKey(removed_life, RefTxnId{1, 3}); ASSERT_TRUE(std::holds_alternative( op.create(later_dead_residue, "dead residue after the folded terminal", Retry::once()))); - backend->makeUnreadable(terminal_key); + backend->makeUnreadable(checkpoint_key); std::map namespace_cleanup; const uint64_t leaks_before @@ -2900,7 +2902,8 @@ TEST(CASGCFrontierGate, PostFoldUnreadableTerminalIsCountedWithoutSuppressingPro EXPECT_EQ(report.manifests_deleted, 1u) << "the janitor leak cannot promote itself into pool-wide destructive suppression"; EXPECT_FALSE(op.head(layout.manifestKey(manifest_id), Retry::once()).has_value()); - EXPECT_TRUE(backend->existsIgnoringFault(terminal_key)); + EXPECT_TRUE(backend->existsIgnoringFault(checkpoint_key)); + EXPECT_FALSE(backend->existsIgnoringFault(terminal_key)); EXPECT_FALSE(backend->existsIgnoringFault(later_dead_residue)) << "one unreadable key cannot stop the perpetual janitor from deciding the rest of its page"; ASSERT_FALSE(namespace_cleanup.empty()); @@ -2909,7 +2912,7 @@ TEST(CASGCFrontierGate, PostFoldUnreadableTerminalIsCountedWithoutSuppressingPro ProfileEvents::global_counters[ProfileEvents::CASGCNamespaceCleanupLeaks].load() - leaks_before, 1u); const String captured = log_capture.captured(); - EXPECT_NE(captured.find(terminal_key), String::npos); + EXPECT_NE(captured.find(checkpoint_key), String::npos); EXPECT_NE(captured.find("leak"), String::npos); } diff --git a/src/Disks/tests/gtest_cas_gc_round_defer.cpp b/src/Disks/tests/gtest_cas_gc_round_defer.cpp index 43a6a909e86a9..4de654aa23f59 100644 --- a/src/Disks/tests/gtest_cas_gc_round_defer.cpp +++ b/src/Disks/tests/gtest_cas_gc_round_defer.cpp @@ -472,14 +472,14 @@ TEST(CASGCRoundDefer, DeferredRoundRetriesPartialJanitorPageAtForcedFoldWithoutP ASSERT_TRUE(folded.acquired_lease); ASSERT_FALSE(folded.deferred) << "gc_fold_max_defer_rounds=1 forces the round immediately following one DEFER to fold"; - EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 1u) - << "the authoritative fold must run the janitor exactly once, not once per call site"; + EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 2u) + << "one janitor run, not one per call site: the page that deletes, then one from the stream start"; const auto folded_cleanup = std::find_if(phases.begin(), phases.end(), [](const GcPhaseRecord & phase) { return phase.phase == "namespace_cleanup"; }); ASSERT_NE(folded_cleanup, phases.end()); - EXPECT_EQ(folded_cleanup->metrics.at("janitor_pages"), 1u); + EXPECT_EQ(folded_cleanup->metrics.at("janitor_pages"), 2u); EXPECT_GE(folded_cleanup->metrics.at("janitor_keys"), 1u); EXPECT_EQ(folded_cleanup->metrics.at("janitor_deleted"), 1u); EXPECT_EQ(static_cast(op.head(key_a, Retry::once()).has_value()) + static_cast(op.head(key_b, Retry::once()).has_value()), 0u) diff --git a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp new file mode 100644 index 0000000000000..7412215320fc5 --- /dev/null +++ b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp @@ -0,0 +1,334 @@ +#include "cas_test_helpers.h" +#include +#include +#include +#include + +#include + +/// Batch deletes of dead-life `_log`/`_snap` keys: one page through `NamespaceJanitor`, then whole rounds +/// through `Gc`, which adds the page loop, the time budget and the GC I/O pool. + +using namespace DB::Cas; +using namespace DB::Cas::tests; + +namespace DB::ErrorCodes +{ + extern const int NETWORK_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + +void createObj(Backend & backend, const String & key, const String & bytes) +{ + OperationForTest op(backend); + ASSERT_TRUE(std::holds_alternative((*op).create(key, bytes, Retry::once()))); +} + +bool present(Backend & backend, const String & key) +{ + OperationForTest op(backend); + return (*op).head(key, Retry::standard()).has_value(); +} + +size_t countPresent(Backend & backend, const std::vector & keys) +{ + size_t n = 0; + for (const String & key : keys) + n += present(backend, key) ? 1 : 0; + return n; +} + +NamespaceLifeId life(const char * name, uint64_t id) +{ + return NamespaceLifeId::fromCatalogEntry(RootNamespace{name}, DB::UInt128{id}); +} + +std::vector seedLogs(Backend & backend, const Layout & layout, const NamespaceLifeId & of, uint64_t count) +{ + std::vector keys; + for (uint64_t i = 1; i <= count; ++i) + { + keys.push_back(layout.refLogKey(of, RefTxnId{.writer_epoch = 1, .ref_sequence = i})); + createObj(backend, keys.back(), "log"); + } + return keys; +} + +GcMaintenanceReadResult readState(CasRequests & requests, const Layout & layout) +{ + auto op = requests.admit(); + return readGcMaintenanceState(op, layout); +} + +/// Every batch delete fails; single-key behaviour is untouched. +class FailingBulkBackend : public CountingBackend +{ +public: + void removeManyWriteOnce(const std::vector &, TransportAccess &) override + { + throw DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "bulk delete failed"); + } +}; + +/// The first batch delete goes through and flips `delete_done`, which a liveness predicate reads. +class FlipAfterBulkBackend : public CountingBackend +{ +public: + void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override + { + CountingBackend::removeManyWriteOnce(keys, access); + delete_done = true; + } + std::atomic delete_done{false}; +}; + +/// A store without a batch delete: more than one key in a request is `NOT_IMPLEMENTED`. +class NoBatchDeleteBackend : public CountingBackend +{ +public: + void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override + { + if (keys.size() > 1) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "no batch delete"); + ++single_key_requests; + CountingBackend::removeManyWriteOnce(keys, access); + } + std::atomic single_key_requests{0}; +}; + +/// Each `LIST` of the namespace root moves the clock, so a test can spend the janitor's budget in one page. +class ClockOnNamespaceListBackend : public CountingBackend +{ +public: + RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override + { + if (prefix.ends_with("/cas/ns/")) + clock_ms += step_ms; + return CountingBackend::list(prefix, cursor, limit, access); + } + std::atomic clock_ms{1}; + uint64_t step_ms = 0; +}; + +template +PoolPtr openPool(std::shared_ptr backend, std::optional refuse_at = std::nullopt) +{ + PoolConfig config{.pool_prefix = "p", .server_root_id = "test"}; + config.gc_fold_max_defer_rounds = 0; + config.gc_io_concurrency = 4; + config.gc_io_pool_refuse_at_for_test = refuse_at; + return Pool::open(std::move(backend), std::move(config)); +} + +/// One reclaiming round; returns the `namespace_cleanup` phase metrics. +std::map runRoundAndGetJanitorMetrics(Gc & gc) +{ + std::map metrics; + gc.setPhaseSink([&](const GcPhaseRecord & record) + { + if (record.phase == "namespace_cleanup") + metrics = record.metrics; + }); + const RoundReport report = runRegularRoundReclaiming(gc); + gc.setPhaseSink({}); + EXPECT_TRUE(report.acquired_lease); + return metrics; +} + +} + +TEST(CASNamespaceJanitorBatches, DeadStreamKeysGoInOneBatchAndStateKeysStayExact) +{ + auto backend = std::make_shared(); + CasRequests requests(backend, Fence::open()); + const Layout layout("p"); + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog({})); + const NamespaceLifeId dead = life("dead", 301); + std::vector stream = seedLogs(*backend, layout, dead, 3); + stream.push_back(layout.refSnapshotKey(dead, RefTxnId{.writer_epoch = 1, .ref_sequence = 2})); + createObj(*backend, stream.back(), "snap"); + const String ckpt = layout.refCkptKey(dead); + createObj(*backend, ckpt, "ckpt"); + backend->resetCounts(); + + const NamespaceJanitorResult result = NamespaceJanitor(requests, layout, 100).runOnePage(false, [] { return true; }); + + EXPECT_EQ(backend->bulkRemoveCalls(), 1u); + EXPECT_EQ(backend->headTotal(), 0u); + EXPECT_EQ(result.deleted, 5u); + EXPECT_EQ(result.leaked, 0u); + EXPECT_EQ(countPresent(*backend, stream), 0u); + EXPECT_FALSE(present(*backend, ckpt)); +} + +TEST(CASNamespaceJanitorBatches, LiveStreamKeysAreRetained) +{ + auto backend = std::make_shared(); + CasRequests requests(backend, Fence::open()); + const Layout layout("p"); + const CatalogEntry entry{.ns = RootNamespace{"live"}, .state = NsState::Live, .incarnation = DB::UInt128{302}}; + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog(RefCatalog{.entries = {entry}})); + const std::vector stream + = seedLogs(*backend, layout, NamespaceLifeId::fromCatalogEntry(entry.ns, entry.incarnation), 3); + backend->resetCounts(); + + const NamespaceJanitorResult result = NamespaceJanitor(requests, layout, 100).runOnePage(false, [] { return true; }); + + EXPECT_EQ(result.deleted, 0u); + EXPECT_EQ(backend->deleteTotal(), 0u); + EXPECT_EQ(backend->bulkRemoveCalls(), 0u); + EXPECT_EQ(countPresent(*backend, stream), 3u); +} + +TEST(CASNamespaceJanitorBatches, FailedBatchLeaksAndCursorAdvances) +{ + auto backend = std::make_shared(); + CasRequests requests(backend, Fence::open()); + const Layout layout("p"); + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog({})); + const std::vector stream = seedLogs(*backend, layout, life("dead", 303), 4); + + const NamespaceJanitorResult result = NamespaceJanitor(requests, layout, 2).runOnePage(false, [] { return true; }); + + EXPECT_EQ(result.deleted, 0u); + EXPECT_EQ(result.leaked, 2u); + EXPECT_EQ(countPresent(*backend, stream), 4u); + const auto state = readState(requests, layout); + ASSERT_EQ(state.status, GcMaintenanceReadStatus::Valid); + EXPECT_FALSE(state.state->janitor_cursor.empty()) << "a failed batch is leak-only: it must not pin the cursor"; +} + +TEST(CASNamespaceJanitorBatches, AuthorityLostAfterBatchHoldsCursor) +{ + auto backend = std::make_shared(); + CasRequests requests(backend, Fence::open()); + const Layout layout("p"); + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog({})); + const std::vector stream = seedLogs(*backend, layout, life("dead", 304), 4); + + const NamespaceJanitorResult result + = NamespaceJanitor(requests, layout, 2).runOnePage(false, [&] { return !backend->delete_done.load(); }); + + EXPECT_EQ(result.deleted, 2u); + EXPECT_EQ(countPresent(*backend, stream), 2u); + EXPECT_EQ(readState(requests, layout).status, GcMaintenanceReadStatus::Absent) + << "a tenure that lost authority must not publish progress"; +} + +TEST(CASNamespaceJanitorBatches, RoundDrainsEveryPageOfDebris) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 311), 2500); + backend->resetCounts(); + + Gc gc(store, DB::UInt128{312}); + const auto metrics = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(countPresent(*backend, stream), 0u); + /// Three pages of debris, then one empty page from the stream start ends the pass. + EXPECT_EQ(metrics.at("janitor_pages"), 4u); + EXPECT_EQ(metrics.at("janitor_deleted"), 2500u); + EXPECT_EQ(metrics.at("leaked"), 0u); + EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 4u); + /// Every page is split across the 4 pool threads. + EXPECT_EQ(backend->bulkRemoveCalls(), 12u); +} + +TEST(CASNamespaceJanitorBatches, RoundDrainsDebrisOnBothSidesOfTheCursor) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 320), 2500); + { + CasOperation op = store->openRequests().admit(); + const GcMaintenanceReadResult progress = readGcMaintenanceState(op, layout); + ASSERT_TRUE(std::holds_alternative(casGcMaintenanceState( + op, layout, progress.etag, GcMaintenanceState{.janitor_cursor = stream[1499]}, Retry::standard()))); + } + + Gc gc(store, DB::UInt128{321}); + const auto metrics = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(metrics.at("janitor_deleted"), 2500u); + EXPECT_EQ(countPresent(*backend, stream), 0u) << "the pass continues from the stream start after its last page"; +} + +TEST(CASNamespaceJanitorBatches, BudgetStopsThePassAfterThePageInProgress) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 313), 2500); + backend->step_ms = 30'000; + + Gc gc(store, DB::UInt128{314}, {}, [&] { return backend->clock_ms.load(); }); + const auto first = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(first.at("janitor_pages"), 1u); + EXPECT_EQ(first.at("janitor_deleted"), 1000u); + ASSERT_TRUE(first.contains("budget_exhausted")); + EXPECT_EQ(first.at("budget_exhausted"), 1u); + EXPECT_EQ(countPresent(*backend, stream), 1500u); + + backend->step_ms = 0; + const auto second = runRoundAndGetJanitorMetrics(gc); + EXPECT_EQ(second.at("budget_exhausted"), 0u); + EXPECT_EQ(countPresent(*backend, stream), 0u) << "the next round resumes from the published cursor"; +} + +TEST(CASNamespaceJanitorBatches, QuietPoolCostsOneListPerRound) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const RootNamespace live_namespace{"00/live@cas@"}; + fixture::admitLive(*backend, layout, live_namespace); + seedLogs(*backend, layout, fixture::fixtureLife(live_namespace), 3); + backend->resetCounts(); + + Gc gc(store, DB::UInt128{315}); + const auto metrics = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(metrics.at("janitor_pages"), 1u); + EXPECT_EQ(metrics.at("janitor_deleted"), 0u); + EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 1u); +} + +TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 316), 1200); + + Gc gc(store, DB::UInt128{317}); + const auto metrics = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(countPresent(*backend, stream), 0u); + EXPECT_EQ(metrics.at("janitor_deleted"), 1200u); + EXPECT_EQ(metrics.at("leaked"), 0u); + EXPECT_EQ(backend->single_key_requests.load(), 1200u); +} + +TEST(CASNamespaceJanitorBatches, RefusedEnqueueDeletesNothingAndTheNextRoundDrains) +{ + auto backend = std::make_shared(); + auto store = openPool(backend, /*refuse_at=*/0); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 318), 100); + + Gc gc(store, DB::UInt128{319}); + const auto refused = runRoundAndGetJanitorMetrics(gc); + EXPECT_EQ(refused.at("janitor_deleted"), 0u); + EXPECT_EQ(countPresent(*backend, stream), 100u); + + const auto drained = runRoundAndGetJanitorMetrics(gc); + EXPECT_EQ(drained.at("janitor_deleted"), 100u); + EXPECT_EQ(countPresent(*backend, stream), 0u); +} diff --git a/tests/integration/test_cas_janitor_drain/__init__.py b/tests/integration/test_cas_janitor_drain/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml b/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml new file mode 100644 index 0000000000000..28aedcb757036 --- /dev/null +++ b/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml @@ -0,0 +1,36 @@ + + + + + object_storage + s3 + cas + itest-janitor-batch + http://rustfs1:11121/test/cas_janitor_batch/ + clickhouse + clickhouse + 1 + 86400 + + 1 + + + object_storage + s3 + cas + itest-janitor-no-batch + http://rustfs1:11121/test/cas_janitor_no_batch/ + clickhouse + clickhouse + false + 1 + 86400 + 1 + + + +
cas_batch
 +
cas_no_batch
 + + + diff --git a/tests/integration/test_cas_janitor_drain/test.py b/tests/integration/test_cas_janitor_drain/test.py new file mode 100644 index 0000000000000..c5f54b44ad91a --- /dev/null +++ b/tests/integration/test_cas_janitor_drain/test.py @@ -0,0 +1,86 @@ +""" +A dropped table's ref stream drains in one deleting round of the namespace janitor on RustFS, on a store +with a batch delete and on one without. +""" +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +INSERTS = 2500 +MAX_ROUNDS = 3 + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def stream_keys(disk): + prefix = f"cas_janitor_{disk.removeprefix('cas_')}/cas/ns/stream/" + return [ + o.object_name + for o in cluster.rustfs_client.list_objects(cluster.rustfs_bucket, prefix, recursive=True) + if "/_log/" in o.object_name or "/_snap/" in o.object_name + ] + + +def seed_and_drop(node, disk): + node.query(f"DROP TABLE IF EXISTS t_{disk} SYNC") + node.query( + f"CREATE TABLE t_{disk} (k UInt64) ENGINE = MergeTree ORDER BY k " + f"SETTINGS storage_policy = '{disk}', parts_to_delay_insert = 100000, parts_to_throw_insert = 100000" + ) + node.query(f"SYSTEM STOP MERGES t_{disk}") + # One row per block gives one part per row without INSERTS client round trips. + node.query( + f"INSERT INTO t_{disk} SELECT number FROM numbers({INSERTS}) SETTINGS max_block_size = 1, " + "max_insert_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 0, " + "max_insert_threads = 1" + ) + assert len(stream_keys(disk)) >= INSERTS + node.query(f"DROP TABLE t_{disk} SYNC") + return len(stream_keys(disk)) + + +def janitor_rows(node, disk, since): + node.query("SYSTEM FLUSH LOGS") + columns = ["janitor_deleted", "janitor_pages", "leaked", "budget_exhausted"] + select = ", ".join(f"phase_metrics['{c}']" for c in columns) + out = node.query( + f"SELECT {select} FROM system.cas_gc_log WHERE event_type = 'Phase' AND phase = 'namespace_cleanup' " + f"AND disk_name = '{disk}' AND event_time_microseconds >= toDateTime64('{since}', 6) " + "ORDER BY event_time_microseconds FORMAT TSV" + ) + return [dict(zip(columns, map(int, line.split("\t")))) for line in out.strip().splitlines()] + + +@pytest.mark.parametrize("disk", ["cas_batch", "cas_no_batch"]) +def test_dropped_table_drains_in_one_round(disk): + node = cluster.instances["node"] + since = node.query("SELECT now64(6)").strip() + dead_keys = seed_and_drop(node, disk) + for _ in range(MAX_ROUNDS): + node.query(f"SYSTEM CAS GC RUN {disk}") + if not stream_keys(disk): + break + assert not stream_keys(disk), f"{disk}: dead ref stream keys left after {MAX_ROUNDS} rounds" + rows = janitor_rows(node, disk, since) + deleting = [r for r in rows if r["janitor_deleted"] > 0] + assert len(deleting) == 1, f"{disk}: expected one deleting round, got {rows}" + row = deleting[0] + assert row["janitor_deleted"] >= dead_keys, f"{disk}: the round did not drain all {dead_keys} dead keys: {rows}" + assert row["janitor_pages"] >= 3 + assert row["leaked"] == 0 + assert row["budget_exhausted"] == 0 From aba7e09d9c7a68412785285ed7de69cacb5fd299 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 14:19:30 +0200 Subject: [PATCH 10/52] CAS GC janitor: a refused delete job leaks instead of dropping the page A pool that refused a delete job threw out of `runOnePage` and lost that page's counters and anomalies. The unscheduled keys now count as leaked, like a failed request, and the page is published. Tests: a quiet pool larger than one page takes one `LIST`; a refusal after the first job; the split across the pool on a store without a batch delete; a dead and a reborn life on one page. Docs: the phase 16 cost table and `namespaces.md` described one page per round and a `HEAD` + `DELETE` per object. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- .../cas/architecture/garbage-collection.md | 12 +++-- .../en/antalya/cas/architecture/namespaces.md | 2 +- .../ContentAddressed/Gc/CasGc.cpp | 41 ++++++++++------ .../ContentAddressed/Gc/CasGc.h | 2 +- .../ContentAddressed/Gc/CasNamespaceJanitor.h | 2 +- .../gtest_cas_namespace_janitor_batches.cpp | 49 ++++++++++++------- 6 files changed, 70 insertions(+), 38 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 81f57d608ca71..de051237d3d30 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -522,8 +522,8 @@ Pages of the perpetual namespace janitor: deletes the physical objects of namesp no longer in the catalog (dead-life debris). The phase takes the next page while the previous one deleted something and published its cursor, until a 20 s soft budget: no page starts after it, the page in progress finishes. After the last page of `cas/ns/` the next one starts from its beginning, so -debris on both sides of the cursor drains in one round. A pool without debris costs one `LIST` per -round. +a pass that began mid-stream also reaches the debris before its cursor; the first page that deletes +nothing ends the pass. A pool without debris costs one `LIST` per round. - **Runs on:** fold path here; also on the deferred path right after phase 4 with `suppress_destructive` forced on @@ -939,13 +939,17 @@ backend without batch delete: the refused bulk call plus one `DELETE` per key). ### Phase 16 — namespace cleanup {#cost-phase-16} +Per page; the phase takes one page in a round without debris and more under its 20 s budget while +pages delete. + | Key | Operation | Requests | |---|---|---:| | `/gc/maintenance_state` | `GET` | 1 (durable `janitor_cursor`) | | `/cas/ns/` | `LIST` | one page | | `/cas/ref_catalog` | `GET` | 1 | -| `/gc/state` | `GET` | one per fence check | -| dead-life object | `DELETE` | one per object (plus one `HEAD` per object whose `LIST` entry carried no token) | +| `/gc/state` | `GET` | 1 (fence check) | +| dead-life `_log` / `_snap` objects | batch `DELETE` | up to `cas_gc_io_concurrency` requests for the page's keys; one per key on a storage without a batch delete | +| dead-life `_ckpt` / `_files` object | `DELETE` | one per object (plus one `HEAD` per object whose `LIST` entry carried no token) | | `/gc/maintenance_state` | `CAS` | 1 when the page is decided | ### Phase 17 — ref object cleanup {#cost-phase-17} diff --git a/docs/en/antalya/cas/architecture/namespaces.md b/docs/en/antalya/cas/architecture/namespaces.md index c38b65b0ce178..f3ba271ee4ad3 100644 --- a/docs/en/antalya/cas/architecture/namespaces.md +++ b/docs/en/antalya/cas/architecture/namespaces.md @@ -151,7 +151,7 @@ proven its ref history is fully drained. | Catalog row (`cas/ref_catalog` entry) | `GC` phase 2, `pre_fold_ref_drain` | The round after the fold that sealed cleanup evidence for this life | | Part manifest bodies | Ordinary owner-removal ([phase 15](/antalya/cas/architecture/garbage-collection#the-round)) for anything that had a committed or precommit binding, the [orphan-manifest sweep](/antalya/cas/architecture/manifests-and-refs#orphan-sweep) for anything that never got that far | As each owning ref is dropped by the removal transaction itself, independent of the catalog row | | Blob bodies | The ordinary condemn/graduate/delete pipeline | Whenever the manifests that named them stop being live, same as any other blob | -| Ref stream/state objects (`_log`, `_snap`, `_ckpt`, `_files`) under the dead `life_id` | The perpetual namespace janitor ([phase 16](/antalya/cas/architecture/garbage-collection#the-round)) | Best-effort, one bounded `LIST` page at a time, whenever it next lists a key whose `life_id` a fresh catalog cut no longer names — independent of, and not gated on, catalog-row deletion | +| Ref stream/state objects (`_log`, `_snap`, `_ckpt`, `_files`) under the dead `life_id` | The perpetual namespace janitor ([phase 16](/antalya/cas/architecture/garbage-collection#the-round)) | Best-effort, in bounded `LIST` pages under a per-round time budget, whenever it next lists a key whose `life_id` a fresh catalog cut no longer names — independent of, and not gated on, catalog-row deletion | The janitor is leak-only: it never fails a round, never blocks progress on an unreadable key, and a crash mid-page simply leaves debris for its next page. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index c407cdf099885..76a04e8d12e9a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -347,13 +347,14 @@ Gc::Gc(PoolPtr store_, UInt128 gc_id_, std::function now_ms_fn_, /// `store->poolConfig()` AFTER the null check above. meta_writer = std::make_unique( store, logger, static_cast(store->poolConfig().gc_meta_pool_size)); - /// The GC I/O pool (read-ahead and the `pending_deletes` fan-out), built here for the same reason. + /// The GC I/O pool (read-ahead, the `pending_deletes` fan-out and the namespace janitor's batch deletes), + /// built here for the same reason. /// The queue is UNBOUNDED because the hinting sites throttle themselves against /// `GcReadAhead::window`; a bounded queue would only move the throttle into /// `scheduleOrThrowOnError`, blocking the round thread instead of the hint loop that already knows /// how much it wants in flight. const size_t io_concurrency = std::max(1, store->poolConfig().gc_io_concurrency); - /// Keep `shutdown_on_exception` false: every task on this pool (read-ahead and re-delete) is submitted + /// Keep `shutdown_on_exception` false: every task on this pool (read-ahead, re-delete, janitor delete) is submitted /// through a callback runner that stores its exception in the task's future, and no caller invokes /// `wait` on this pool. io_pool = std::make_unique( @@ -430,24 +431,36 @@ uint64_t Gc::removeWriteOnceOnIoPool( std::vector::Task>> handles; handles.reserve(chunks.size()); SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); - for (size_t i = 0; i < chunks.size(); ++i) + uint64_t leaked = 0; + try { - if (io_pool_refuse_at_for_test == i) + for (size_t i = 0; i < chunks.size(); ++i) { - io_pool_refuse_at_for_test.reset(); - throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS namespace janitor enqueue refusal at index {}", i); - } - /// The round thread waits for every job before it refreshes `authority_held` again. - handles.emplace_back(runner.enqueueAndGiveOwnership( - [this, chunk = &chunks[i], admitted_generation = op.generation()] + if (io_pool_refuse_at_for_test == i) { - CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); - removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); - })); + io_pool_refuse_at_for_test.reset(); + throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS namespace janitor enqueue refusal at index {}", i); + } + /// The round thread waits for every job before it refreshes `authority_held` again. + handles.emplace_back(runner.enqueueAndGiveOwnership( + [this, chunk = &chunks[i], admitted_generation = op.generation()] + { + CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); + removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); + })); + } + } + catch (...) + { + /// Leak-only like a failed request: the keys stay for the cursor's next pass over this page. + for (size_t i = handles.size(); i < chunks.size(); ++i) + leaked += chunks[i].size(); + anomalies.push_back(fmt::format( + "leaked {} dead-life objects starting at '{}': delete job not scheduled: {}", + leaked, chunks[handles.size()].front().str(), getCurrentExceptionMessage(false))); } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); - uint64_t leaked = 0; for (size_t i = 0; i < handles.size(); ++i) { try diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 8cdebbe98df6a..5b157cb10927a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -557,7 +557,7 @@ class Gc const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows); /// The janitor's `RemoveWriteOnce`: splits `keys` evenly across the GC I/O pool and waits for every job. - /// A failed job leaks its keys. Throws when the pool refuses a job, after the scheduled ones finished. + /// A failed job leaks its keys, and so does a job the pool refused to schedule. uint64_t removeWriteOnceOnIoPool( CasOperation & op, const std::vector & keys, std::vector & anomalies); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h index 9a8d3e5a82ff9..d7093c11b3367 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h @@ -21,7 +21,7 @@ struct NamespaceJanitorResult }; /// Deletes one page's dead `_log`/`_snap` keys under the page's operation. Returns how many keys it could -/// not confirm deleted and adds one anomaly per failed request; throws only when nothing may be published. +/// not confirm deleted and adds an anomaly for each group of them. An exception leaves the page unpublished. using RemoveWriteOnce = std::function & keys, std::vector & anomalies)>; diff --git a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp index 7412215320fc5..bf19808ca3007 100644 --- a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp +++ b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp @@ -92,10 +92,14 @@ class NoBatchDeleteBackend : public CountingBackend void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override { if (keys.size() > 1) + { + ++refused_batches; throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "no batch delete"); + } ++single_key_requests; CountingBackend::removeManyWriteOnce(keys, access); } + std::atomic refused_batches{0}; std::atomic single_key_requests{0}; }; @@ -164,23 +168,24 @@ TEST(CASNamespaceJanitorBatches, DeadStreamKeysGoInOneBatchAndStateKeysStayExact EXPECT_FALSE(present(*backend, ckpt)); } -TEST(CASNamespaceJanitorBatches, LiveStreamKeysAreRetained) +TEST(CASNamespaceJanitorBatches, RebornLifeOnTheSamePageKeepsItsKeys) { auto backend = std::make_shared(); CasRequests requests(backend, Fence::open()); const Layout layout("p"); - const CatalogEntry entry{.ns = RootNamespace{"live"}, .state = NsState::Live, .incarnation = DB::UInt128{302}}; - createObj(*backend, layout.refCatalogKey(), encodeRefCatalog(RefCatalog{.entries = {entry}})); - const std::vector stream - = seedLogs(*backend, layout, NamespaceLifeId::fromCatalogEntry(entry.ns, entry.incarnation), 3); + const CatalogEntry reborn{.ns = RootNamespace{"t"}, .state = NsState::Live, .incarnation = DB::UInt128{302}}; + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog(RefCatalog{.entries = {reborn}})); + const std::vector dead_stream = seedLogs(*backend, layout, life("t", 301), 3); + const std::vector live_stream + = seedLogs(*backend, layout, NamespaceLifeId::fromCatalogEntry(reborn.ns, reborn.incarnation), 3); backend->resetCounts(); const NamespaceJanitorResult result = NamespaceJanitor(requests, layout, 100).runOnePage(false, [] { return true; }); - EXPECT_EQ(result.deleted, 0u); - EXPECT_EQ(backend->deleteTotal(), 0u); - EXPECT_EQ(backend->bulkRemoveCalls(), 0u); - EXPECT_EQ(countPresent(*backend, stream), 3u); + EXPECT_EQ(result.deleted, 3u); + EXPECT_EQ(backend->deleteTotal(), 3u); + EXPECT_EQ(countPresent(*backend, dead_stream), 0u); + EXPECT_EQ(countPresent(*backend, live_stream), 3u); } TEST(CASNamespaceJanitorBatches, FailedBatchLeaksAndCursorAdvances) @@ -284,20 +289,25 @@ TEST(CASNamespaceJanitorBatches, BudgetStopsThePassAfterThePageInProgress) TEST(CASNamespaceJanitorBatches, QuietPoolCostsOneListPerRound) { - auto backend = std::make_shared(); + /// More than one page of live keys, and a clock that spends the budget on the first `LIST`: a pass that + /// went on after a page without debris would stop on the budget instead. + auto backend = std::make_shared(); auto store = openPool(backend); const Layout & layout = store->layout(); const RootNamespace live_namespace{"00/live@cas@"}; fixture::admitLive(*backend, layout, live_namespace); - seedLogs(*backend, layout, fixture::fixtureLife(live_namespace), 3); + const std::vector stream = seedLogs(*backend, layout, fixture::fixtureLife(live_namespace), 1200); + backend->step_ms = 30'000; backend->resetCounts(); - Gc gc(store, DB::UInt128{315}); + Gc gc(store, DB::UInt128{315}, {}, [&] { return backend->clock_ms.load(); }); const auto metrics = runRoundAndGetJanitorMetrics(gc); EXPECT_EQ(metrics.at("janitor_pages"), 1u); EXPECT_EQ(metrics.at("janitor_deleted"), 0u); + EXPECT_EQ(metrics.at("budget_exhausted"), 0u); EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 1u); + EXPECT_EQ(countPresent(*backend, stream), 1200u); } TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) @@ -314,21 +324,26 @@ TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) EXPECT_EQ(metrics.at("janitor_deleted"), 1200u); EXPECT_EQ(metrics.at("leaked"), 0u); EXPECT_EQ(backend->single_key_requests.load(), 1200u); + /// Two pages, each split into one job per pool thread. + EXPECT_EQ(backend->refused_batches.load(), 8u); } -TEST(CASNamespaceJanitorBatches, RefusedEnqueueDeletesNothingAndTheNextRoundDrains) +TEST(CASNamespaceJanitorBatches, RefusedEnqueueLeaksTheUnscheduledJobsAndTheNextRoundDrains) { auto backend = std::make_shared(); - auto store = openPool(backend, /*refuse_at=*/0); + auto store = openPool(backend, /*refuse_at=*/1); const Layout & layout = store->layout(); const std::vector stream = seedLogs(*backend, layout, life("dead", 318), 100); + /// Four jobs of 25 keys: the first runs, the second is refused and takes the rest with it. Gc gc(store, DB::UInt128{319}); const auto refused = runRoundAndGetJanitorMetrics(gc); - EXPECT_EQ(refused.at("janitor_deleted"), 0u); - EXPECT_EQ(countPresent(*backend, stream), 100u); + EXPECT_EQ(refused.at("janitor_deleted"), 25u); + EXPECT_EQ(refused.at("leaked"), 75u); + EXPECT_EQ(countPresent(*backend, stream), 75u); const auto drained = runRoundAndGetJanitorMetrics(gc); - EXPECT_EQ(drained.at("janitor_deleted"), 100u); + EXPECT_EQ(drained.at("janitor_deleted"), 75u); + EXPECT_EQ(drained.at("leaked"), 0u); EXPECT_EQ(countPresent(*backend, stream), 0u); } From 2644a44739b3a6c583770bf4e23bf289f7cf2442 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 14:32:18 +0200 Subject: [PATCH 11/52] CAS GC janitor test: make the quiet-pool round a decided one The live namespace had no `_ckpt`, so the fold suppressed the round and the janitor page was never decided: the test passed with the `deleted > 0` clause of `more` removed. The namespace now has a checkpoint and 1200 `_files` keys, and the test asserts the published cursor. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- .../gtest_cas_namespace_janitor_batches.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp index bf19808ca3007..8072446304647 100644 --- a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp +++ b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp @@ -4,6 +4,7 @@ #include #include +#include #include /// Batch deletes of dead-life `_log`/`_snap` keys: one page through `NamespaceJanitor`, then whole rounds @@ -296,7 +297,16 @@ TEST(CASNamespaceJanitorBatches, QuietPoolCostsOneListPerRound) const Layout & layout = store->layout(); const RootNamespace live_namespace{"00/live@cas@"}; fixture::admitLive(*backend, layout, live_namespace); - const std::vector stream = seedLogs(*backend, layout, fixture::fixtureLife(live_namespace), 1200); + const NamespaceLifeId live = fixture::fixtureLife(live_namespace); + createObj(*backend, layout.refCkptKey(live), + encodeRefCkpt(RefCkpt{.life_epoch = std::optional{1}, + .checkpoint_snapshot_id = std::nullopt, .last_epoch_seal = std::nullopt})); + std::vector files; + for (size_t i = 0; i < 1200; ++i) + { + files.push_back(layout.namespaceFilesPrefix(live) + fmt::format("file-{:04}", i)); + createObj(*backend, files.back(), "file"); + } backend->step_ms = 30'000; backend->resetCounts(); @@ -307,7 +317,11 @@ TEST(CASNamespaceJanitorBatches, QuietPoolCostsOneListPerRound) EXPECT_EQ(metrics.at("janitor_deleted"), 0u); EXPECT_EQ(metrics.at("budget_exhausted"), 0u); EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 1u); - EXPECT_EQ(countPresent(*backend, stream), 1200u); + EXPECT_EQ(countPresent(*backend, files), 1200u); + /// The page was decided, not suppressed: its cursor is published. + const auto state = readState(store->openRequests(), layout); + ASSERT_EQ(state.status, GcMaintenanceReadStatus::Valid); + EXPECT_FALSE(state.state->janitor_cursor.empty()); } TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) From b7b813d3c0add3a9a869f07d265c21c4fe3c6c3e Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 14:51:27 +0200 Subject: [PATCH 12/52] CAS GC janitor: test `writeOnceStreamKey`, name the janitor in the batch-size setting `cas_gc_bulk_delete_chunk_keys` now also caps the namespace janitor's batch deletes; its description said only phases 15 and 17. The test pins that a listed key which is not the canonical key of its parsed identity gets no write-once form. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- docs/en/antalya/cas/configuration.md | 2 +- .../ContentAddressedSettings.cpp | 2 +- src/Disks/tests/gtest_cas_write_once_key.cpp | 22 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/en/antalya/cas/configuration.md b/docs/en/antalya/cas/configuration.md index 84be3753f7323..bdc7a3f2f6e2d 100644 --- a/docs/en/antalya/cas/configuration.md +++ b/docs/en/antalya/cas/configuration.md @@ -179,7 +179,7 @@ for the remaining caps, `0` means unbounded. |---|---|---|---| | `cas_manifest_sweep_list_budget_keys` | `1000` | `UInt64` | Orphan-manifest sweep `LIST` budget per round | | `cas_manifest_sweep_delete_budget_keys` | `100` | `UInt64` | Orphan-manifest sweep `DELETE` budget per round | -| `cas_gc_bulk_delete_chunk_keys` | `1000` | `1`–`1000` | Keys per batch delete request in GC's write-once families (owner-removed manifest bodies, covered ref logs and snapshots) | +| `cas_gc_bulk_delete_chunk_keys` | `1000` | `1`–`1000` | Most keys per batch delete request in GC's write-once families (owner-removed manifest bodies, covered ref logs and snapshots, the namespace janitor's dead ref logs and snapshots) | | `cas_gc_round_graduation_budget` | `5000` | `0` = unbounded | Blob-graduation (`condemned` → `delete_pending`) cohort cap per round | | `cas_gc_round_redelete_budget` | `5000` | `0` = unbounded | Exact-token re-delete cohort cap for prior `delete_pending` rows per round | | `cas_gc_round_sweep_namespace_budget` | `20` | `0` = unbounded | Distinct namespaces per orphan-manifest sweep page whose protection view may be built | diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp index 7c6b80c540fff..73f8380c668d5 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp @@ -80,7 +80,7 @@ constexpr std::string_view CAS_KEY_PREFIX = "cas_"; DECLARE(UInt64, manifest_decode_cache_bytes, 128ULL << 20, "Manifest DECODE cache byte budget (0 disables)", 0) \ DECLARE(UInt64, gc_meta_pool_size, 16, "Bounded pool size for GC per-hash freshness-meta writes", 0) \ DECLARE(UInt64, gc_io_concurrency, 16, "Maximum number of threads in the GC I/O pool. Used for fold and rebuild read-ahead, orphan-manifest sweep planning reads, pending_deletes HEAD plus conditional DELETE, and namespace janitor batch deletes. Per-hash meta writes use gc_meta_pool_size; other GC requests run on the round thread. 1 disables parallel GC I/O", 0) \ - DECLARE(UInt64, gc_bulk_delete_chunk_keys, 1000, "Keys per batch delete request in GC's write-once families (owner-removed manifest bodies, covered ref logs and snapshots); 1 to 1000", 0) \ + DECLARE(UInt64, gc_bulk_delete_chunk_keys, 1000, "Most keys per batch delete request in GC's write-once families (owner-removed manifest bodies, covered ref logs and snapshots, the namespace janitor's dead ref logs and snapshots); 1 to 1000", 0) \ DECLARE(UInt64, attempt_timeout_ms, 5000, "Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. With the connect cap it forms the attempt envelope the lease arithmetic reserves", 0) \ DECLARE(UInt64, lease_safety_margin_ms, 2000, "Startup-only margin validated against the mount lease TTL: attempt envelope + this must be strictly less than the TTL, and renew period + 2 × envelope + this too", 0) \ DECLARE(String, staging_backend, "local", "Blob staging backend (local | s3); s3 is opt-in", 0) \ diff --git a/src/Disks/tests/gtest_cas_write_once_key.cpp b/src/Disks/tests/gtest_cas_write_once_key.cpp index c05eb6b1e3457..d9fd69c6756dc 100644 --- a/src/Disks/tests/gtest_cas_write_once_key.cpp +++ b/src/Disks/tests/gtest_cas_write_once_key.cpp @@ -30,3 +30,25 @@ TEST(CASWriteOnceKey, FactoriesMintTheSameStringsAsThePlainKeyFunctions) EXPECT_TRUE(layout.parseManifestKey(layout.writeOnceManifestKey(manifest).str()).has_value()); EXPECT_TRUE(layout.parseRefObjectKey(layout.writeOnceRefLogKey(life, id).str()).has_value()); } + +TEST(CASWriteOnceKey, StreamKeyMintedFromAListedKeyMustBeThatKey) +{ + const Layout layout{"p"}; + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(RootNamespace{"test/aa@cas@"}, DB::UInt128(0x1234)); + const RefTxnId id{5, 7}; + + for (const String & listed : {layout.refLogKey(life, id), layout.refSnapshotKey(life, id)}) + { + const auto parsed = layout.parseRefObjectKey(listed); + ASSERT_TRUE(parsed.has_value()) << listed; + const auto minted = layout.writeOnceStreamKey(*parsed, listed); + ASSERT_TRUE(minted.has_value()) << listed; + EXPECT_EQ(minted->str(), listed); + } + + const auto parsed_log = layout.parseRefObjectKey(layout.refLogKey(life, id)); + ASSERT_TRUE(parsed_log.has_value()); + EXPECT_FALSE(layout.writeOnceStreamKey(*parsed_log, layout.refSnapshotKey(life, id)).has_value()); + EXPECT_FALSE(layout.writeOnceStreamKey(*parsed_log, layout.refLogKey(life, RefTxnId{5, 8})).has_value()); + EXPECT_FALSE(layout.writeOnceStreamKey(*parsed_log, "q" + layout.refLogKey(life, id).substr(1)).has_value()); +} From 2c757e5dcecf620474e55f81018243e35736507f Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 16:48:02 +0200 Subject: [PATCH 13/52] CAS GC janitor: keep the cursor on a later LIST failure, count requests - A failed `LIST` reset the persisted cursor on any page. It now resets only on the first page of a pass; later the cursor the previous page published stays, so one transient failure mid-drain does not send the next round back to the stream start. - `leaked` is zero for a page that did not publish its cursor: the page is listed again, so authority lost mid-page no longer inflates `CASGCNamespaceCleanupLeaks`. - New phase metric `delete_requests`: the delete jobs run on the GC I/O pool, so the phase row's `ProfileEvents` miss their requests. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- .../cas/architecture/garbage-collection.md | 10 +++- .../en/operations/system-tables/cas_gc_log.md | 2 +- .../ContentAddressed/Gc/CasGc.cpp | 36 ++++++----- .../ContentAddressed/Gc/CasGc.h | 4 +- .../Gc/CasNamespaceJanitor.cpp | 35 ++++++----- .../ContentAddressed/Gc/CasNamespaceJanitor.h | 18 ++++-- .../gtest_cas_namespace_janitor_batches.cpp | 59 +++++++++++++++++++ 7 files changed, 125 insertions(+), 39 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index de051237d3d30..539750f87ea1b 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -539,11 +539,15 @@ nothing ends the pass. A pool without debris costs one `LIST` per round. unreachable from a reborn same-name namespace, so a missed key can only leak storage, never expose it - **Fails the round if:** nothing — the whole phase is wrapped in a catch-all ("namespace janitor - stopped this round"). A failed batch `DELETE` leaks its keys and the cursor still advances + stopped this round"). A failed batch `DELETE` leaks its keys and the cursor still advances. A failed + `LIST` resets the cursor to the stream start only on the first page of the phase; on a later page + the cursor the previous page published stays - **Observability:** phase row `namespace_cleanup`; metrics `janitor_pages`, `janitor_keys`, `janitor_deleted` (keys of successful batch requests, absent ones included, plus exact removals), - `leaked`, `budget_exhausted`. Batch deletes run on the GC I/O pool, so the row's `ProfileEvents` - do not include their requests + `leaked` (failed keys of published pages; an unpublished page is listed again), `delete_requests` + (delete calls: one per exact-token delete and per batch, including a batch the storage refused + before its keys went one by one), `budget_exhausted`. Batch deletes run on the GC I/O pool, so the + row's `ProfileEvents` do not include their requests The cursor advances only when the whole page was decided under a held fence and an unambiguous catalog; under suppression it lists and classifies but deletes nothing and does not advance. diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index e8b5138bc85b3..53309724de68c 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -86,7 +86,7 @@ The phases, in execution order: | `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | | `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | | `manifest_deletes` | Exact-token deletes of owner-removed manifest bodies, after their decrements were adopted. | one `DELETE` per body | -| `namespace_cleanup` | Run `cas/ns/` pages across the stream and state subtrees for the perpetual dead-life janitor, under a 20 s soft budget; `budget_exhausted` is 1 when the budget stopped the phase. This phase is physical reclamation, not a lifecycle gate. | per page: one namespace-root page `LIST`, catalog cut, batch deletes of dead ref logs and snapshots, exact-token deletes of the rest | +| `namespace_cleanup` | Run `cas/ns/` pages across the stream and state subtrees for the perpetual dead-life janitor, under a 20 s soft budget; `budget_exhausted` is 1 when the budget stopped the phase, `delete_requests` counts the phase's delete calls, which run on the GC I/O pool and are missing from this row's `ProfileEvents`. This phase is physical reclamation, not a lifecycle gate. | per page: one namespace-root page `LIST`, catalog cut, batch deletes of dead ref logs and snapshots, exact-token deletes of the rest | | `ref_object_cleanup` | Delete ref logs covered by both the durable fold cursor and a durable snapshot, plus superseded snapshots. | one `HEAD` + one `DELETE` per deletable object | | `orphan_sweep` | The budgeted, cursor-paced orphan part-manifest backstop. | budgeted `LIST` and deletes | diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 76a04e8d12e9a..2debbc2787e76 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -376,9 +376,9 @@ void Gc::runNamespaceJanitor( CasRequests & requests = store->openRequests(); const Layout & layout = store->layout(); NamespaceJanitor janitor(requests, layout, 1000, - [this](CasOperation & op, const std::vector & keys, std::vector & anomalies) + [this](CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out) { - return removeWriteOnceOnIoPool(op, keys, anomalies); + removeWriteOnceOnIoPool(op, keys, out); }); const uint64_t deadline_ms = mono_ms_fn() + kJanitorBudgetMs; while (true) @@ -386,11 +386,13 @@ void Gc::runNamespaceJanitor( /// ONE authority read per page, made here rather than from the predicate: the janitor's /// operation samples its liveness before every request, and a page walks up to a thousand keys. refreshAuthority(leased_state.lease.seq); - const NamespaceJanitorResult page = janitor.runOnePage(suppress_destructive, [this] { return authority_held; }); + const NamespaceJanitorResult page = janitor.runOnePage( + suppress_destructive, [this] { return authority_held; }, /*first_page_of_pass=*/janitor_result.pages == 0); janitor_result.pages += page.pages; janitor_result.keys += page.keys; janitor_result.deleted += page.deleted; janitor_result.leaked += page.leaked; + janitor_result.delete_requests += page.delete_requests; for (const String & anomaly : page.anomalies) LOG_WARNING(logger, "CAS namespace janitor: {}", anomaly); if (!page.more) @@ -412,11 +414,11 @@ void Gc::runNamespaceJanitor( t.metric("janitor_keys", janitor_result.keys); t.metric("janitor_deleted", janitor_result.deleted); t.metric("leaked", janitor_result.leaked); + t.metric("delete_requests", janitor_result.delete_requests); t.metric("budget_exhausted", budget_exhausted ? 1 : 0); } -uint64_t Gc::removeWriteOnceOnIoPool( - CasOperation & op, const std::vector & keys, std::vector & anomalies) +void Gc::removeWriteOnceOnIoPool(CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out) { /// An even split, so a store without a batch delete sends its one-key requests from every pool thread. const size_t concurrency = std::max(1, store->poolConfig().gc_io_concurrency); @@ -427,11 +429,12 @@ uint64_t Gc::removeWriteOnceOnIoPool( for (size_t begin = 0; begin < keys.size(); begin += chunk_keys) chunks.emplace_back(keys.begin() + begin, keys.begin() + std::min(keys.size(), begin + chunk_keys)); + /// Written by the job of the same index, read after every job finished. + std::vector requests(chunks.size(), 0); ThreadPoolCallbackRunnerLocal runner(*io_pool, ThreadName::CAS_GC_JANITOR); std::vector::Task>> handles; handles.reserve(chunks.size()); SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); - uint64_t leaked = 0; try { for (size_t i = 0; i < chunks.size(); ++i) @@ -443,39 +446,44 @@ uint64_t Gc::removeWriteOnceOnIoPool( } /// The round thread waits for every job before it refreshes `authority_held` again. handles.emplace_back(runner.enqueueAndGiveOwnership( - [this, chunk = &chunks[i], admitted_generation = op.generation()] + [this, chunk = &chunks[i], chunk_requests = &requests[i], admitted_generation = op.generation()] { CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); - removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); + /// A job that fails made at least the one call. + *chunk_requests = 1; + *chunk_requests = removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); })); } } catch (...) { /// Leak-only like a failed request: the keys stay for the cursor's next pass over this page. + uint64_t unscheduled = 0; for (size_t i = handles.size(); i < chunks.size(); ++i) - leaked += chunks[i].size(); - anomalies.push_back(fmt::format( + unscheduled += chunks[i].size(); + out.leaked += unscheduled; + out.anomalies.push_back(fmt::format( "leaked {} dead-life objects starting at '{}': delete job not scheduled: {}", - leaked, chunks[handles.size()].front().str(), getCurrentExceptionMessage(false))); + unscheduled, chunks[handles.size()].front().str(), getCurrentExceptionMessage(false))); } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); for (size_t i = 0; i < handles.size(); ++i) { + out.delete_requests += requests[i]; try { handles[i]->future.get(); + out.deleted += chunks[i].size(); } catch (...) { - leaked += chunks[i].size(); - anomalies.push_back(fmt::format( + out.leaked += chunks[i].size(); + out.anomalies.push_back(fmt::format( "leaked {} dead-life objects starting at '{}': batch delete failed: {}", chunks[i].size(), chunks[i].front().str(), getCurrentExceptionMessage(false))); } } - return leaked; } uint64_t removeChunkWriteOnceOrOneByOne(CasOperation & op, const std::vector & chunk, const Retry & policy) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 5b157cb10927a..8e6c43a7c9864 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -249,6 +249,7 @@ struct RefScanSummary class RefPlan; class RoundInput; +struct NamespaceJanitorResult; RefPlan buildRefWalkPlan(RoundInput && round_input); namespace tests @@ -558,8 +559,7 @@ class Gc /// The janitor's `RemoveWriteOnce`: splits `keys` evenly across the GC I/O pool and waits for every job. /// A failed job leaks its keys, and so does a job the pool refused to schedule. - uint64_t removeWriteOnceOnIoPool( - CasOperation & op, const std::vector & keys, std::vector & anomalies); + void removeWriteOnceOnIoPool(CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out); void reportStuckRemovals(const RefPlan & plan, uint64_t current_round); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp index 6857c39f49bf5..dba6b927c97cf 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp @@ -21,26 +21,26 @@ void throwOnRefusedOrGaveUp(WriteResult && result, std::string_view what) (void)orThrow(std::move(result), what); } -uint64_t removeOnCallerThread(CasOperation & op, const std::vector & keys, std::vector & anomalies) +void removeOnCallerThread(CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out) { - uint64_t leaked = 0; for (size_t begin = 0; begin < keys.size(); begin += kBulkDeleteMaxKeys) { const std::vector batch( keys.begin() + begin, keys.begin() + std::min(keys.size(), begin + kBulkDeleteMaxKeys)); + ++out.delete_requests; try { op.removeManyWriteOnce(batch, Retry::standard()); + out.deleted += batch.size(); } catch (const std::exception & e) { - leaked += batch.size(); - anomalies.push_back(fmt::format( + out.leaked += batch.size(); + out.anomalies.push_back(fmt::format( "leaked {} dead-life objects starting at '{}': batch delete failed: {}", batch.size(), batch.front().str(), e.what())); } } - return leaked; } } @@ -54,7 +54,7 @@ NamespaceJanitor::NamespaceJanitor( { } -NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liveness liveness) +NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liveness liveness, bool first_page_of_pass) { NamespaceJanitorResult result; CasOperation op = requests.admit(std::move(liveness)); @@ -76,7 +76,8 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven } catch (...) { - (void)casGcMaintenanceState(op, layout, progress.etag, GcMaintenanceState{}, Retry::once()); + if (first_page_of_pass) + (void)casGcMaintenanceState(op, layout, progress.etag, GcMaintenanceState{}, Retry::once()); throw; } result.pages = 1; @@ -171,6 +172,7 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven page_decided = false; break; } + ++result.delete_requests; try { if (op.remove(listed.key, *etag, Retry::standard()) == Removal::Removed) @@ -185,11 +187,7 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven } if (page_decided && !dead_stream_keys.empty() && op.admitted()) - { - const uint64_t leaked = remove_write_once(op, dead_stream_keys, result.anomalies); - result.leaked += leaked; - result.deleted += dead_stream_keys.size() - leaked; - } + remove_write_once(op, dead_stream_keys, result); /// Recheck even when the page had no dead candidate. A tenure that observes fence loss after LIST /// or after the last delete must not publish progress. Loss after this check may still race @@ -197,22 +195,29 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven if (page_decided && !op.admitted()) page_decided = false; + bool published = false; if (page_decided) { const GcMaintenanceState next{.janitor_cursor = page.next_cursor}; try { - const WriteResult published = casGcMaintenanceState(op, layout, progress.etag, next, Retry::standard()); - if (std::holds_alternative(published) || std::holds_alternative(published)) + const WriteResult outcome = casGcMaintenanceState(op, layout, progress.etag, next, Retry::standard()); + if (std::holds_alternative(outcome) || std::holds_alternative(outcome)) result.anomalies.push_back("cursor publication did not commit"); - else if (std::holds_alternative(published)) + else if (std::holds_alternative(outcome)) + { + published = true; result.more = result.deleted > 0 && !(cursor.empty() && page.next_cursor.empty()); + } } catch (const std::exception & e) { result.anomalies.push_back("cursor publication failed: " + String(e.what())); } } + /// An unpublished page is listed again, so its failed keys are not leaked yet. + if (!published) + result.leaked = 0; return result; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h index d7093c11b3367..801d3967bf66a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h @@ -13,6 +13,9 @@ struct NamespaceJanitorResult uint64_t keys = 0; uint64_t deleted = 0; uint64_t leaked = 0; + /// Delete calls the page made: one per exact-token delete and per batch, including a batch the storage + /// refused before its keys went one by one. Reissues of a call are not counted. + uint64_t delete_requests = 0; /// The page deleted something, published its cursor and did not cover the whole stream by itself, so /// the next page is worth taking now. After the last page of the stream the next one starts from its /// beginning. @@ -20,10 +23,11 @@ struct NamespaceJanitorResult std::vector anomalies; }; -/// Deletes one page's dead `_log`/`_snap` keys under the page's operation. Returns how many keys it could -/// not confirm deleted and adds an anomaly for each group of them. An exception leaves the page unpublished. +/// Deletes one page's dead `_log`/`_snap` keys under the page's operation and adds to `out` the keys it +/// deleted, the keys it could not confirm deleted with an anomaly for each group of them, and its delete +/// calls. An exception leaves the page unpublished. using RemoveWriteOnce - = std::function & keys, std::vector & anomalies)>; + = std::function & keys, NamespaceJanitorResult & out)>; /// Runs one bounded, leak-only page over the physical namespace ownership tree; `Gc` repeats it under a time budget. class NamespaceJanitor @@ -43,7 +47,13 @@ class NamespaceJanitor /// /// A dead life's canonical `_log`/`_snap` keys are write-once, so the page deletes them together through /// `remove_write_once` with no per-key `HEAD` or token; `_ckpt` and `_files` keep the exact-token delete. - NamespaceJanitorResult runOnePage(bool suppress_deletes, Liveness liveness); + /// + /// A failed `LIST` resets the persisted cursor only when `first_page_of_pass`: that cursor comes from an + /// earlier round and the store may reject it forever, while the one a page of this pass just published + /// is fresh, so the failure is transient and the next round resumes from it. + /// + /// `leaked` is zero for a page that did not publish its cursor: the page is listed again. + NamespaceJanitorResult runOnePage(bool suppress_deletes, Liveness liveness, bool first_page_of_pass = true); private: CasRequests & requests; diff --git a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp index 8072446304647..ecb6accdd109e 100644 --- a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp +++ b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp @@ -6,6 +6,7 @@ #include #include +#include /// Batch deletes of dead-life `_log`/`_snap` keys: one page through `NamespaceJanitor`, then whole rounds /// through `Gc`, which adds the page loop, the time budget and the GC I/O pool. @@ -70,8 +71,24 @@ class FailingBulkBackend : public CountingBackend public: void removeManyWriteOnce(const std::vector &, TransportAccess &) override { + attempted = true; throw DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "bulk delete failed"); } + std::atomic attempted{false}; +}; + +/// Fails every `LIST` of the namespace root after the first `allowed_lists` ones. +class FailingNamespaceListBackend : public CountingBackend +{ +public: + RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override + { + if (prefix.ends_with("/cas/ns/") && lists++ >= allowed_lists) + throw DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "namespace LIST failed"); + return CountingBackend::list(prefix, cursor, limit, access); + } + std::atomic lists{0}; + std::atomic allowed_lists{std::numeric_limits::max()}; }; /// The first batch delete goes through and flips `delete_done`, which a liveness predicate reads. @@ -163,6 +180,7 @@ TEST(CASNamespaceJanitorBatches, DeadStreamKeysGoInOneBatchAndStateKeysStayExact EXPECT_EQ(backend->bulkRemoveCalls(), 1u); EXPECT_EQ(backend->headTotal(), 0u); + EXPECT_EQ(result.delete_requests, 2u) << "one batch and one exact-token delete"; EXPECT_EQ(result.deleted, 5u); EXPECT_EQ(result.leaked, 0u); EXPECT_EQ(countPresent(*backend, stream), 0u); @@ -207,6 +225,22 @@ TEST(CASNamespaceJanitorBatches, FailedBatchLeaksAndCursorAdvances) EXPECT_FALSE(state.state->janitor_cursor.empty()) << "a failed batch is leak-only: it must not pin the cursor"; } +TEST(CASNamespaceJanitorBatches, UnpublishedPageReportsNoLeaks) +{ + auto backend = std::make_shared(); + CasRequests requests(backend, Fence::open()); + const Layout layout("p"); + createObj(*backend, layout.refCatalogKey(), encodeRefCatalog({})); + seedLogs(*backend, layout, life("dead", 305), 4); + + const NamespaceJanitorResult result + = NamespaceJanitor(requests, layout, 2).runOnePage(false, [&] { return !backend->attempted.load(); }); + + EXPECT_EQ(readState(requests, layout).status, GcMaintenanceReadStatus::Absent); + EXPECT_EQ(result.leaked, 0u) << "the page is listed again, so nothing on it is leaked yet"; + EXPECT_FALSE(result.anomalies.empty()); +} + TEST(CASNamespaceJanitorBatches, AuthorityLostAfterBatchHoldsCursor) { auto backend = std::make_shared(); @@ -241,6 +275,7 @@ TEST(CASNamespaceJanitorBatches, RoundDrainsEveryPageOfDebris) EXPECT_EQ(metrics.at("janitor_deleted"), 2500u); EXPECT_EQ(metrics.at("leaked"), 0u); EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 4u); + EXPECT_EQ(metrics.at("delete_requests"), 12u); /// Every page is split across the 4 pool threads. EXPECT_EQ(backend->bulkRemoveCalls(), 12u); } @@ -265,6 +300,29 @@ TEST(CASNamespaceJanitorBatches, RoundDrainsDebrisOnBothSidesOfTheCursor) EXPECT_EQ(countPresent(*backend, stream), 0u) << "the pass continues from the stream start after its last page"; } +TEST(CASNamespaceJanitorBatches, ListFailureAfterTheFirstPageKeepsTheCursor) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 322), 2500); + backend->lists = 0; + backend->allowed_lists = 1; + + Gc gc(store, DB::UInt128{323}); + const auto failed = runRoundAndGetJanitorMetrics(gc); + EXPECT_EQ(failed.at("janitor_deleted"), 1000u); + EXPECT_EQ(countPresent(*backend, stream), 1500u); + const auto state = readState(store->openRequests(), layout); + ASSERT_EQ(state.status, GcMaintenanceReadStatus::Valid); + EXPECT_EQ(state.state->janitor_cursor, stream[999]) << "the cursor the first page published stays"; + + backend->allowed_lists = std::numeric_limits::max(); + const auto resumed = runRoundAndGetJanitorMetrics(gc); + EXPECT_EQ(resumed.at("janitor_deleted"), 1500u); + EXPECT_EQ(countPresent(*backend, stream), 0u); +} + TEST(CASNamespaceJanitorBatches, BudgetStopsThePassAfterThePageInProgress) { auto backend = std::make_shared(); @@ -340,6 +398,7 @@ TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) EXPECT_EQ(backend->single_key_requests.load(), 1200u); /// Two pages, each split into one job per pool thread. EXPECT_EQ(backend->refused_batches.load(), 8u); + EXPECT_EQ(metrics.at("delete_requests"), 1208u); } TEST(CASNamespaceJanitorBatches, RefusedEnqueueLeaksTheUnscheduledJobsAndTheNextRoundDrains) From a39bccb67de7eab42e108fa3c3aca1a376ea821a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 5 Oct 2026 16:52:31 +0200 Subject: [PATCH 14/52] CAS GC janitor: check `delete_requests` in the drain integration test A batch store drains in under a tenth as many delete calls as keys; a store without a batch delete makes at least one call per key. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- tests/integration/test_cas_janitor_drain/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_cas_janitor_drain/test.py b/tests/integration/test_cas_janitor_drain/test.py index c5f54b44ad91a..8526355320b2a 100644 --- a/tests/integration/test_cas_janitor_drain/test.py +++ b/tests/integration/test_cas_janitor_drain/test.py @@ -56,7 +56,7 @@ def seed_and_drop(node, disk): def janitor_rows(node, disk, since): node.query("SYSTEM FLUSH LOGS") - columns = ["janitor_deleted", "janitor_pages", "leaked", "budget_exhausted"] + columns = ["janitor_deleted", "janitor_pages", "leaked", "delete_requests", "budget_exhausted"] select = ", ".join(f"phase_metrics['{c}']" for c in columns) out = node.query( f"SELECT {select} FROM system.cas_gc_log WHERE event_type = 'Phase' AND phase = 'namespace_cleanup' " @@ -84,3 +84,7 @@ def test_dropped_table_drains_in_one_round(disk): assert row["janitor_pages"] >= 3 assert row["leaked"] == 0 assert row["budget_exhausted"] == 0 + if disk == "cas_batch": + assert row["delete_requests"] * 10 < dead_keys, f"{disk}: the dead keys did not go in batches: {rows}" + else: + assert row["delete_requests"] >= dead_keys, f"{disk}: expected one request per key: {rows}" From 804c619254d58f7699d08d3df546b2d6b830c22e Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 6 Oct 2026 13:31:26 +0200 Subject: [PATCH 15/52] Fix GROUP BY hostName() over s3Cluster and object_storage_cluster The old analyzer drops constant GROUP BY keys when distributed_depth is 0. Cluster table function reads never increased that depth, so workers folded hostName() and returned only the aggregate. https: //github.com/Altinity/ClickHouse/issues/2139 Co-authored-by: Cursor --- src/Storages/IStorageCluster.cpp | 1 + ...296_s3_cluster_hostname_group_by.reference | 3 ++ .../05296_s3_cluster_hostname_group_by.sql | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.reference create mode 100644 tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.sql diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 3b8867d16f4f3..aca542a595af1 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -955,6 +955,7 @@ ContextPtr ReadFromCluster::updateSettings(const Settings & settings) auto new_context = Context::createCopy(context); new_context->setSettings(new_settings); + new_context->increaseDistributedDepth(); return new_context; } diff --git a/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.reference b/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.reference new file mode 100644 index 0000000000000..456b57d5a4e9c --- /dev/null +++ b/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.reference @@ -0,0 +1,3 @@ +1 10 +1 10 +1 10 diff --git a/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.sql b/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.sql new file mode 100644 index 0000000000000..242e9b4b75646 --- /dev/null +++ b/tests/queries/0_stateless/05296_s3_cluster_hostname_group_by.sql @@ -0,0 +1,32 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: Depends on Minio +-- https://github.com/Altinity/ClickHouse/issues/2139 + +INSERT INTO FUNCTION s3('http://localhost:11111/test/05296_s3_cluster_hostname_group_by.tsv', 'TSV', 'x UInt32') +SELECT number FROM numbers(10) +SETTINGS s3_truncate_on_insert = 1; + +SELECT host != '', c FROM +( + SELECT hostName() AS host, count() AS c + FROM s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/05296_s3_cluster_hostname_group_by.tsv', 'TSV', 'x UInt32') + GROUP BY host +) +SETTINGS enable_analyzer = 0; + +SELECT host != '', c FROM +( + SELECT hostName() AS host, count() AS c + FROM s3('http://localhost:11111/test/05296_s3_cluster_hostname_group_by.tsv', 'TSV', 'x UInt32') + GROUP BY host + SETTINGS object_storage_cluster = 'test_cluster_two_shards_localhost' +) +SETTINGS enable_analyzer = 0; + +SELECT host != '', c FROM +( + SELECT hostName() AS host, count() AS c + FROM s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/05296_s3_cluster_hostname_group_by.tsv', 'TSV', 'x UInt32') + GROUP BY host +) +SETTINGS enable_analyzer = 1; From ea70a1fdffce79d0d26a413f0ea6fde6fae65f6f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 6 Oct 2026 14:03:54 +0200 Subject: [PATCH 16/52] Keep server-constant GROUP BY keys without raising distributed_depth Raising distributed_depth on cluster reads also changes parallel replicas, max_distributed_depth, and block marshalling. The old analyzer now keeps a key such as hostName() and still drops ordinary constants. https: //github.com/Altinity/ClickHouse/issues/2139 Co-authored-by: Cursor --- src/Interpreters/ExpressionAnalyzer.cpp | 36 +++++++++++++++++++++++-- src/Storages/IStorageCluster.cpp | 1 - 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 3c086d7e1f4ff..0a87f20f29112 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -151,6 +152,35 @@ bool allowEarlyConstantFolding(const ActionsDAG & actions, const Settings & sett return true; } +/// A folded server constant (`hostName()`, `tcpPort()`, ...) must stay a GROUP BY key. +/// Its value differs per shard. Dropping it makes the initiator look up a missing column. +bool containsServerConstant(const ActionsDAG::Node * node) +{ + std::stack stack; + if (node) + stack.push(node); + + while (!stack.empty()) + { + const auto * current = stack.top(); + stack.pop(); + + while (current && current->type == ActionsDAG::ActionType::ALIAS) + current = current->children.empty() ? nullptr : current->children.front(); + + if (!current) + continue; + + if (current->function_base && current->function_base->isServerConstant()) + return true; + + for (const auto * child : current->children) + stack.push(child); + } + + return false; +} + LoggerPtr getLogger() { return ::getLogger("ExpressionAnalyzer"); } } @@ -361,7 +391,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { /// Constant expressions have non-null column pointer at this stage. - if (node->column) + /// Server constants stay: they are different on each shard. + if (node->column && !containsServerConstant(node)) { select_query->group_by_with_constant_keys = true; @@ -415,7 +446,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { /// Constant expressions have non-null column pointer at this stage. - if (node->column) + /// Server constants stay: they are different on each shard. + if (node->column && !containsServerConstant(node)) { select_query->group_by_with_constant_keys = true; diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index aca542a595af1..3b8867d16f4f3 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -955,7 +955,6 @@ ContextPtr ReadFromCluster::updateSettings(const Settings & settings) auto new_context = Context::createCopy(context); new_context->setSettings(new_settings); - new_context->increaseDistributedDepth(); return new_context; } From b84beb33837b29460b0ea808f9c9936f4f33831b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 6 Oct 2026 14:55:41 +0200 Subject: [PATCH 17/52] Fix CAS backup test coverage --- .../configs/backups_disk.xml | 1 - .../test_backup_restore_new/test.py | 5 +-- .../test_cancel_backup.py | 44 ++++++------------- .../test_backup_restore_s3/test.py | 17 ++++--- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/tests/integration/test_backup_restore_new/configs/backups_disk.xml b/tests/integration/test_backup_restore_new/configs/backups_disk.xml index 1909aaacca257..f7d666c65426e 100644 --- a/tests/integration/test_backup_restore_new/configs/backups_disk.xml +++ b/tests/integration/test_backup_restore_new/configs/backups_disk.xml @@ -9,7 +9,6 @@ backups - cas /backups/ diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 271f491b1eafa..6c8a5124afca5 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -228,11 +228,8 @@ def test_restore_table(engine): assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" -@pytest.mark.parametrize("backup_disk", ["backups", "cas"]) -def test_restore_table_on_cas_disk(backup_disk): +def test_restore_table_on_cas_disk(): backup_name = new_backup_name() - if backup_disk == "cas": - backup_name = backup_name.replace("Disk('backups', '", "Disk('cas', 'backups/") create_and_fill_table(storage_policy="cas_policy") assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n" diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py index 653050259e2ef..a789295a61e1b 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py @@ -19,7 +19,6 @@ "configs/lesser_timeouts.xml", # Default timeouts are quite big (a few minutes), the tests don't need them to be that big. "configs/slow_backups.xml", "configs/shutdown_cancel_backups.xml", - "configs/cas_storage.xml", ] user_configs = [ @@ -28,23 +27,21 @@ node1 = cluster.add_instance( "node1", - main_configs=main_configs + ["configs/cas_server_root_node1.xml"], + main_configs=main_configs, user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node1", "shard": "shard1"}, with_zookeeper=True, - with_rustfs=True, stay_alive=True, # Necessary for "test_shutdown_cancel_backup" ) node2 = cluster.add_instance( "node2", - main_configs=main_configs + ["configs/cas_server_root_node2.xml"], + main_configs=main_configs, user_configs=user_configs, external_dirs=["/backups/"], macros={"replica": "node2", "shard": "shard1"}, with_zookeeper=True, - with_rustfs=True, stay_alive=True, # Necessary for "test_shutdown_cancel_backup" ) @@ -82,7 +79,7 @@ def random_node(): # Makes table "tbl" and fill it with data. -def create_and_fill_table(node, num_parts=10, on_cluster=True, storage_policy=None): +def create_and_fill_table(node, num_parts=10, on_cluster=True): # We use partitioning to make sure there will be more files in a backup. partition_by_clause = " PARTITION BY x%" + str(num_parts) if num_parts > 1 else "" node.query( @@ -91,7 +88,6 @@ def create_and_fill_table(node, num_parts=10, on_cluster=True, storage_policy=No + "(x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}') " + "ORDER BY tuple()" + partition_by_clause - + (f" SETTINGS storage_policy = '{storage_policy}'" if storage_policy else "") ) if num_parts > 0: node.query(f"INSERT INTO tbl SELECT number FROM numbers({num_parts})") @@ -411,17 +407,18 @@ def wait_for_backups_to_finish(): ) -__backup_id_of_successful_backup = {} +__backup_id_of_successful_backup = None # Generates a backup which will be used to test RESTORE. -def get_backup_id_of_successful_backup(storage_policy=None): - if storage_policy not in __backup_id_of_successful_backup: - __backup_id_of_successful_backup[storage_policy] = random_id() +def get_backup_id_of_successful_backup(): + global __backup_id_of_successful_backup + if __backup_id_of_successful_backup is None: + __backup_id_of_successful_backup = random_id() with NoTrashChecker() as no_trash_checker: print("Will make backup successfully") - backup_id = __backup_id_of_successful_backup[storage_policy] - create_and_fill_table(random_node(), storage_policy=storage_policy) + backup_id = __backup_id_of_successful_backup + create_and_fill_table(random_node()) initiator = random_node() print(f"Using {get_node_name(initiator)} as initiator") initiator.query( @@ -434,7 +431,7 @@ def get_backup_id_of_successful_backup(storage_policy=None): # Dropping the table before restoring. node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") - return __backup_id_of_successful_backup[storage_policy] + return __backup_id_of_successful_backup # Actual tests @@ -493,10 +490,9 @@ def test_cancel_backup(): # Test that a RESTORE operation can be cancelled with KILL QUERY. -@pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) -def test_cancel_restore(storage_policy): +def test_cancel_restore(): # Make backup. - backup_id = get_backup_id_of_successful_backup(storage_policy) + backup_id = get_backup_id_of_successful_backup() # Cancel restoring. with NoTrashChecker() as no_trash_checker: @@ -561,20 +557,6 @@ def test_cancel_restore(storage_policy): wait_status(initiator, "RESTORED", restore_id=restore_id) assert get_num_system_processes(nodes, restore_id=restore_id) == 0 - if storage_policy is not None: - node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") - for node in nodes: - assert ( - node.query("SELECT storage_policy FROM system.tables WHERE name = 'tbl'") - == f"{storage_policy}\n" - ) - assert node.query("SELECT count(), sum(x) FROM tbl") == "10\t45\n" - assert ( - node.query("CHECK TABLE tbl SETTINGS check_query_single_value_result = 1") - == "1\n" - ) - - # Test that shutdown cancels a running backup and doesn't wait until it finishes. def test_shutdown_cancels_backup(): wait_for_backups_to_finish() diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index f5d4b77b4aa21..3ecaeb73eb839 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -564,9 +564,16 @@ def test_backup_to_s3_native_copy(cluster, storage_policy): ) if storage_policy in CAS_POLICIES: - # A CAS file can be a range inside a blob object, so the server-side copy may be ranged. - for events in (backup_events, restore_events): - assert events.get("S3CopyObject", 0) + events.get("S3UploadPartCopy", 0) > 0 + assert ( + backup_events.get("S3CopyObject", 0) + + backup_events.get("S3UploadPartCopy", 0) + > 0 + ) + assert ( + restore_events.get("S3CopyObject", 0) + + restore_events.get("S3UploadPartCopy", 0) + == 0 + ) return # single part upload @@ -733,10 +740,10 @@ def test_incremental_backup_append_table_def(cluster): (True, True, True), ], ) -@pytest.mark.parametrize("storage_policy", ["policy_s3_cache", "policy_cas_cache"]) def test_backup_with_fs_cache( - cluster, in_cache_initially, allow_backup_read_cache, allow_s3_native_copy, storage_policy + cluster, in_cache_initially, allow_backup_read_cache, allow_s3_native_copy ): + storage_policy = "policy_s3_cache" backup_name = new_backup_name() backup_destination = s3_backup_destination(storage_policy, backup_name) From 498cec1948591f7c52b8bbbb5b2ddf1a8d52d86e Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Tue, 6 Oct 2026 15:29:09 +0200 Subject: [PATCH 18/52] test for diffrent endpoint Signed-off-by: Konstantin Morozov --- tests/integration/test_backup_restore_s3/test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 3ecaeb73eb839..564c62607fd5f 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -593,6 +593,20 @@ def test_backup_to_s3_native_copy(cluster, storage_policy): ) +def test_backup_from_cas_to_s3_different_endpoint(cluster): + backup_name = new_backup_name() + backup_destination = f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + backup_events, restore_events = check_backup_and_restore( + cluster, "policy_cas", backup_destination + ) + + for events in (backup_events, restore_events): + assert events.get("S3UploadPartCopy", 0) == 0 + assert events.get("S3CopyObject", 0) == 0 + assert events.get("S3GetObject", 0) > 0 + assert events.get("S3PutObject", 0) + events.get("S3UploadPart", 0) > 0 + + @pytest.mark.parametrize( "storage_policy", [ From 12460edf619b7fdaa6a57991cde20978ea3f64cf Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 6 Oct 2026 18:06:29 +0200 Subject: [PATCH 19/52] Drop __getScalar GROUP BY keys in the old analyzer __getScalar is marked as a server constant, but the initiator evaluates it and sends the same value to every shard. Keeping it made GROUP BY of a scalar subquery with Dynamic throw ILLEGAL_COLUMN. https: //github.com/Altinity/ClickHouse/pull/2490 Co-authored-by: Cursor --- src/Interpreters/ExpressionAnalyzer.cpp | 4 +++- .../05297_scalar_subquery_group_by_old_analyzer.reference | 2 ++ .../05297_scalar_subquery_group_by_old_analyzer.sql | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.reference create mode 100644 tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 0a87f20f29112..898b767eaf4a3 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -171,7 +171,9 @@ bool containsServerConstant(const ActionsDAG::Node * node) if (!current) continue; - if (current->function_base && current->function_base->isServerConstant()) + /// `__getScalar` is marked as a server constant, but the initiator evaluates it + /// and sends the same value to every shard, so it does not have to stay a key. + if (current->function_base && current->function_base->isServerConstant() && current->function_base->getName() != "__getScalar") return true; for (const auto * child : current->children) diff --git a/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.reference b/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.reference new file mode 100644 index 0000000000000..a5c8806279fa7 --- /dev/null +++ b/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.reference @@ -0,0 +1,2 @@ +3 +3 diff --git a/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.sql b/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.sql new file mode 100644 index 0000000000000..b495d5f16573b --- /dev/null +++ b/tests/queries/0_stateless/05297_scalar_subquery_group_by_old_analyzer.sql @@ -0,0 +1,7 @@ +-- A scalar subquery is evaluated once and sent to every shard as `__getScalar`. +-- It must not stay a GROUP BY key just because that function is marked as a server constant. +-- https://github.com/Altinity/ClickHouse/pull/2490 + +SELECT count() FROM numbers(3) GROUP BY (SELECT [1, 2]) SETTINGS enable_analyzer = 0; + +SELECT count() FROM numbers(3) GROUP BY (SELECT [CAST(1, 'Dynamic')]) SETTINGS enable_analyzer = 0; From f050971b17f599299aa29febf2ee59c8ef81fc12 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 7 Oct 2026 11:06:52 +0200 Subject: [PATCH 20/52] add test Signed-off-by: Konstantin Morozov --- .../test_backup_restore_new/test_cancel_backup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_backup_restore_new/test_cancel_backup.py b/tests/integration/test_backup_restore_new/test_cancel_backup.py index 16a67db9318ee..5491cd22e4862 100644 --- a/tests/integration/test_backup_restore_new/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_new/test_cancel_backup.py @@ -275,6 +275,8 @@ def test_cancel_backup(storage_policy): # Test that shutdown cancels a running backup and doesn't wait until it finishes. @pytest.mark.parametrize("storage_policy", [None, "cas_policy"]) def test_shutdown_cancel_backup(storage_policy): + cas_pool_baseline = wait_cas_pool_settled() if storage_policy == "cas_policy" else None + node.query( "CREATE TABLE tbl (x UInt64) ENGINE=MergeTree() ORDER BY tuple() PARTITION BY x%5" + table_settings(storage_policy) @@ -301,3 +303,6 @@ def test_shutdown_cancel_backup(storage_policy): f"RESTORE TABLE tbl FROM {get_backup_name(backup_id)}" ), ) + + if cas_pool_baseline is not None: + wait_cas_pool_reclaimed(cas_pool_baseline) From bbdbb2c7a026509a8b3f0ecf449500405b87b13d Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 7 Oct 2026 18:48:10 +0200 Subject: [PATCH 21/52] CAS GC janitor: cut delete jobs by the storage's batch limit, count leaks per key With `Gc::bulkDeleteChunkKeys` in the base, a page's dead keys go as one job per that many keys instead of an even split across the pool: 1000 keys are one `DeleteObjects` on S3, and a storage without a batch delete sends one request per key with no refused batch first. A job whose batch the storage refuses deletes its keys one by one and accounts each on its own, so a failure late in the chunk leaks only the keys after it. The unscheduled chunks after a refused enqueue are counted from an explicit `scheduled` index. Integration test: 1100 parts instead of 2500 (the INSERT timed out under MSan), an 1800 s INSERT timeout, and a disk with `objects_chunk_size_to_delete = 100`. Rename `remove_write_once` to `remove_write_once_keys`. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Mikhail Filimonov --- .../cas/architecture/garbage-collection.md | 8 +- .../ContentAddressed/Gc/CasGc.cpp | 124 ++++++++++++------ .../ContentAddressed/Gc/CasGc.h | 4 +- .../Gc/CasNamespaceJanitor.cpp | 6 +- .../ContentAddressed/Gc/CasNamespaceJanitor.h | 8 +- .../gtest_cas_namespace_janitor_batches.cpp | 65 ++++++--- .../configs/storage_conf.xml | 14 ++ .../test_cas_janitor_drain/test.py | 15 ++- 8 files changed, 170 insertions(+), 74 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 9db558adcaac5..24c90c6dd670e 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -531,9 +531,9 @@ nothing ends the pass. A pool without debris costs one `LIST` per round. - **Reads:** per page: the durable `janitor_cursor`; one `LIST` page (≤ 1000 keys) of `cas/ns/`; a fresh ref-catalog snapshot; `gc/state` for the fence re-check - **Writes / deletes:** per page: batch `DELETE`s of the dead-life `_log` / `_snap` objects, which - are write-once and need no token; the page's keys are split evenly across the GC I/O pool - (`cas_gc_io_concurrency`), at most `cas_gc_bulk_delete_chunk_keys` per request, and a storage - without a batch delete gets one request per key from the same jobs; exact-token `DELETE` per + are write-once and need no token; one request per `cas_gc_bulk_delete_chunk_keys` keys, capped by + the storage's batch-delete limit (`objects_chunk_size_to_delete` on S3; one key on a storage + without a batch delete), run as jobs on the GC I/O pool (`cas_gc_io_concurrency`); exact-token `DELETE` per dead-life `_ckpt` / `_files` object; one `CAS` on the maintenance state when the page is decided - **Safety:** each request is under a GC fence re-check (`lease.owner` / `lease.seq`), read once per page and checked again before the cursor is published; the incarnation segment in every key makes an old life's objects structurally @@ -962,7 +962,7 @@ pages delete. | `/cas/ns/` | `LIST` | one page | | `/cas/ref_catalog` | `GET` | 1 | | `/gc/state` | `GET` | 1 (fence check) | -| dead-life `_log` / `_snap` objects | batch `DELETE` | up to `cas_gc_io_concurrency` requests for the page's keys; one per key on a storage without a batch delete | +| dead-life `_log` / `_snap` objects | batch `DELETE` | one per `cas_gc_bulk_delete_chunk_keys` keys, capped by the storage's batch-delete limit; one per key on a storage without a batch delete | | dead-life `_ckpt` / `_files` object | `DELETE` | one per object (plus one `HEAD` per object whose `LIST` entry carried no token) | | `/gc/maintenance_state` | `CAS` | 1 when the page is decided | diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index b5e6ba636f6cd..58dfd377cca81 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -88,6 +88,48 @@ namespace /// Soft limit on the namespace janitor's phase: no page starts after it, the page in progress finishes. constexpr uint64_t kJanitorBudgetMs = 20'000; +/// What one janitor delete job did with its chunk. +struct WriteOnceJobResult +{ + uint64_t deleted = 0; + uint64_t leaked = 0; + uint64_t requests = 0; + String error; +}; + +/// Deletes `chunk` as one request. A storage that refuses the batch gets one request per key, each +/// accounted on its own, so a failure late in the chunk leaks only the keys it did not delete. +void removeChunkWriteOnceCountingKeys(CasOperation & op, const std::vector & chunk, WriteOnceJobResult & result) +{ + ++result.requests; + try + { + op.removeManyWriteOnce(chunk, Retry::standard()); + result.deleted += chunk.size(); + return; + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::NOT_IMPLEMENTED) + throw; + } + for (const WriteOnceKey & key : chunk) + { + ++result.requests; + try + { + op.removeManyWriteOnce({key}, Retry::standard()); + ++result.deleted; + } + catch (...) + { + ++result.leaked; + if (result.error.empty()) + result.error = getCurrentExceptionMessage(false); + } + } +} + /// The `on_page_fetched` hook GC passes to every `forEachListedKey`/`recoverRefTable` /// call it owns (never passed by fsck/offline-repair callers of those shared helpers) -- one increment /// per physical LIST page, never per listed key. @@ -420,69 +462,73 @@ void Gc::runNamespaceJanitor( void Gc::removeWriteOnceOnIoPool(CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out) { - /// An even split, so a store without a batch delete sends its one-key requests from every pool thread. - const size_t concurrency = std::max(1, store->poolConfig().gc_io_concurrency); - const size_t chunk_keys = std::min( - std::clamp(store->poolConfig().gc_bulk_delete_chunk_keys, 1, kBulkDeleteMaxKeys), - (keys.size() + concurrency - 1) / concurrency); + const size_t chunk_keys = bulkDeleteChunkKeys(); std::vector> chunks; for (size_t begin = 0; begin < keys.size(); begin += chunk_keys) chunks.emplace_back(keys.begin() + begin, keys.begin() + std::min(keys.size(), begin + chunk_keys)); - /// Written by the job of the same index, read after every job finished. - std::vector requests(chunks.size(), 0); + /// One per chunk, written by its job and read after every job finished. + std::vector results(chunks.size()); ThreadPoolCallbackRunnerLocal runner(*io_pool, ThreadName::CAS_GC_JANITOR); std::vector::Task>> handles; handles.reserve(chunks.size()); SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); - try + size_t scheduled = 0; + String refusal; + for (; scheduled < chunks.size(); ++scheduled) { - for (size_t i = 0; i < chunks.size(); ++i) + try { - if (io_pool_refuse_at_for_test == i) + if (io_pool_refuse_at_for_test == scheduled) { io_pool_refuse_at_for_test.reset(); - throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS namespace janitor enqueue refusal at index {}", i); + throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS namespace janitor enqueue refusal at index {}", scheduled); } /// The round thread waits for every job before it refreshes `authority_held` again. handles.emplace_back(runner.enqueueAndGiveOwnership( - [this, chunk = &chunks[i], chunk_requests = &requests[i], admitted_generation = op.generation()] + [this, chunk = &chunks[scheduled], result = &results[scheduled], admitted_generation = op.generation()] { - CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); - /// A job that fails made at least the one call. - *chunk_requests = 1; - *chunk_requests = removeChunkWriteOnceOrOneByOne(job_op, *chunk, Retry::standard()); + try + { + CasOperation job_op = store->openRequests().resume(admitted_generation, [this] { return authority_held; }); + removeChunkWriteOnceCountingKeys(job_op, *chunk, *result); + } + catch (...) + { + result->leaked = chunk->size() - result->deleted; + result->error = getCurrentExceptionMessage(false); + } })); } - } - catch (...) - { - /// Leak-only like a failed request: the keys stay for the cursor's next pass over this page. - uint64_t unscheduled = 0; - for (size_t i = handles.size(); i < chunks.size(); ++i) - unscheduled += chunks[i].size(); - out.leaked += unscheduled; - out.anomalies.push_back(fmt::format( - "leaked {} dead-life objects starting at '{}': delete job not scheduled: {}", - unscheduled, chunks[handles.size()].front().str(), getCurrentExceptionMessage(false))); + catch (...) + { + refusal = getCurrentExceptionMessage(false); + break; + } } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); - for (size_t i = 0; i < handles.size(); ++i) + for (size_t i = 0; i < scheduled; ++i) { - out.delete_requests += requests[i]; - try - { - handles[i]->future.get(); - out.deleted += chunks[i].size(); - } - catch (...) - { - out.leaked += chunks[i].size(); + handles[i]->future.get(); + out.deleted += results[i].deleted; + out.leaked += results[i].leaked; + out.delete_requests += results[i].requests; + if (results[i].leaked) out.anomalies.push_back(fmt::format( "leaked {} dead-life objects starting at '{}': batch delete failed: {}", - chunks[i].size(), chunks[i].front().str(), getCurrentExceptionMessage(false))); - } + results[i].leaked, chunks[i].front().str(), results[i].error)); + } + /// Leak-only like a failed request: the keys stay for the cursor's next pass over this page. + uint64_t unscheduled = 0; + for (size_t i = scheduled; i < chunks.size(); ++i) + unscheduled += chunks[i].size(); + if (unscheduled) + { + out.leaked += unscheduled; + out.anomalies.push_back(fmt::format( + "leaked {} dead-life objects starting at '{}': delete job not scheduled: {}", + unscheduled, chunks[scheduled].front().str(), refusal)); } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 523068caa4db5..316f413fbb664 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -561,8 +561,8 @@ class Gc void runNamespaceJanitor( const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows); - /// The janitor's `RemoveWriteOnce`: splits `keys` evenly across the GC I/O pool and waits for every job. - /// A failed job leaks its keys, and so does a job the pool refused to schedule. + /// The janitor's `RemoveWriteOnce`: one job per `bulkDeleteChunkKeys` keys on the GC I/O pool, waited for + /// before it returns. A failed job leaks the keys it did not delete, and so does a job the pool refused. void removeWriteOnceOnIoPool(CasOperation & op, const std::vector & keys, NamespaceJanitorResult & out); void reportStuckRemovals(const RefPlan & plan, uint64_t current_round); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp index dba6b927c97cf..0a8857f714799 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp @@ -46,11 +46,11 @@ void removeOnCallerThread(CasOperation & op, const std::vector & k } NamespaceJanitor::NamespaceJanitor( - CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_) + CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_keys_) : requests(requests_) , layout(layout_) , page_budget(page_budget_) - , remove_write_once(remove_write_once_ ? std::move(remove_write_once_) : removeOnCallerThread) + , remove_write_once_keys(remove_write_once_keys_ ? std::move(remove_write_once_keys_) : removeOnCallerThread) { } @@ -187,7 +187,7 @@ NamespaceJanitorResult NamespaceJanitor::runOnePage(bool suppress_deletes, Liven } if (page_decided && !dead_stream_keys.empty() && op.admitted()) - remove_write_once(op, dead_stream_keys, result); + remove_write_once_keys(op, dead_stream_keys, result); /// Recheck even when the page had no dead candidate. A tenure that observes fence loss after LIST /// or after the last delete must not publish progress. Loss after this check may still race diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h index 801d3967bf66a..417d2ba2b8f4a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h @@ -33,8 +33,8 @@ using RemoveWriteOnce class NamespaceJanitor { public: - /// An empty `remove_write_once_` deletes on the caller's thread, one request per `kBulkDeleteMaxKeys` keys. - NamespaceJanitor(CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_ = {}); + /// An empty `remove_write_once_keys_` deletes on the caller's thread, one request per `kBulkDeleteMaxKeys` keys. + NamespaceJanitor(CasRequests & requests_, const Layout & layout_, size_t page_budget_, RemoveWriteOnce remove_write_once_keys_ = {}); /// `liveness` is admitted once for the whole page (one `CasOperation` covers the read, the list, /// every delete and the cursor publication): a fact the fence cannot see, such as "this tenure @@ -46,7 +46,7 @@ class NamespaceJanitor /// publication) reports it as `GaveUp` rather than sending anything. /// /// A dead life's canonical `_log`/`_snap` keys are write-once, so the page deletes them together through - /// `remove_write_once` with no per-key `HEAD` or token; `_ckpt` and `_files` keep the exact-token delete. + /// `remove_write_once_keys` with no per-key `HEAD` or token; `_ckpt` and `_files` keep the exact-token delete. /// /// A failed `LIST` resets the persisted cursor only when `first_page_of_pass`: that cursor comes from an /// earlier round and the store may reject it forever, while the one a page of this pass just published @@ -59,7 +59,7 @@ class NamespaceJanitor CasRequests & requests; const Layout & layout; size_t page_budget; - RemoveWriteOnce remove_write_once; + RemoveWriteOnce remove_write_once_keys; }; } diff --git a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp index ecb6accdd109e..d9bf4c684c7b7 100644 --- a/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp +++ b/src/Disks/tests/gtest_cas_namespace_janitor_batches.cpp @@ -103,7 +103,8 @@ class FlipAfterBulkBackend : public CountingBackend std::atomic delete_done{false}; }; -/// A store without a batch delete: more than one key in a request is `NOT_IMPLEMENTED`. +/// A store that has not learned yet that it has no batch delete: it still advertises the CAS maximum, +/// refuses every batch of more than one key, and fails one chosen key. class NoBatchDeleteBackend : public CountingBackend { public: @@ -114,13 +115,23 @@ class NoBatchDeleteBackend : public CountingBackend ++refused_batches; throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "no batch delete"); } + if (keys.front().str() == failing_key) + throw DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "delete failed"); ++single_key_requests; CountingBackend::removeManyWriteOnce(keys, access); } + String failing_key; std::atomic refused_batches{0}; std::atomic single_key_requests{0}; }; +/// A store known to have no batch delete: its limit is one key. +class OneKeyLimitBackend : public CountingBackend +{ +public: + size_t bulkDeleteKeyLimit() const override { return 1; } +}; + /// Each `LIST` of the namespace root moves the clock, so a test can spend the janitor's budget in one page. class ClockOnNamespaceListBackend : public CountingBackend { @@ -275,9 +286,9 @@ TEST(CASNamespaceJanitorBatches, RoundDrainsEveryPageOfDebris) EXPECT_EQ(metrics.at("janitor_deleted"), 2500u); EXPECT_EQ(metrics.at("leaked"), 0u); EXPECT_EQ(backend->listCount(layout.namespaceRootPrefix()), 4u); - EXPECT_EQ(metrics.at("delete_requests"), 12u); - /// Every page is split across the 4 pool threads. - EXPECT_EQ(backend->bulkRemoveCalls(), 12u); + EXPECT_EQ(metrics.at("delete_requests"), 3u); + /// One request per page: the in-memory store's limit is the CAS maximum. + EXPECT_EQ(backend->bulkRemoveCalls(), 3u); } TEST(CASNamespaceJanitorBatches, RoundDrainsDebrisOnBothSidesOfTheCursor) @@ -382,12 +393,13 @@ TEST(CASNamespaceJanitorBatches, QuietPoolCostsOneListPerRound) EXPECT_FALSE(state.state->janitor_cursor.empty()); } -TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) +TEST(CASNamespaceJanitorBatches, StoreKnownToHaveNoBatchDeleteSendsOneRequestPerKey) { - auto backend = std::make_shared(); + auto backend = std::make_shared(); auto store = openPool(backend); const Layout & layout = store->layout(); const std::vector stream = seedLogs(*backend, layout, life("dead", 316), 1200); + backend->resetCounts(); Gc gc(store, DB::UInt128{317}); const auto metrics = runRoundAndGetJanitorMetrics(gc); @@ -395,28 +407,47 @@ TEST(CASNamespaceJanitorBatches, StoreWithoutBatchDeleteDrainsKeyByKeyOnThePool) EXPECT_EQ(countPresent(*backend, stream), 0u); EXPECT_EQ(metrics.at("janitor_deleted"), 1200u); EXPECT_EQ(metrics.at("leaked"), 0u); - EXPECT_EQ(backend->single_key_requests.load(), 1200u); - /// Two pages, each split into one job per pool thread. - EXPECT_EQ(backend->refused_batches.load(), 8u); - EXPECT_EQ(metrics.at("delete_requests"), 1208u); + EXPECT_EQ(backend->bulkRemoveCalls(), 1200u); + EXPECT_EQ(metrics.at("delete_requests"), 1200u); +} + +TEST(CASNamespaceJanitorBatches, RefusedBatchGoesKeyByKeyAndLeaksOnlyTheFailedKey) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const Layout & layout = store->layout(); + const std::vector stream = seedLogs(*backend, layout, life("dead", 324), 1200); + backend->failing_key = stream[1100]; + + Gc gc(store, DB::UInt128{325}); + const auto metrics = runRoundAndGetJanitorMetrics(gc); + + EXPECT_EQ(countPresent(*backend, stream), 1u); + EXPECT_TRUE(present(*backend, stream[1100])); + EXPECT_EQ(metrics.at("janitor_deleted"), 1199u); + EXPECT_EQ(metrics.at("leaked"), 1u) << "a failure late in the chunk leaks that key, not the whole chunk"; + EXPECT_EQ(backend->single_key_requests.load(), 1199u); + /// One refused batch per page, then one request per key. + EXPECT_EQ(backend->refused_batches.load(), 2u); + EXPECT_EQ(metrics.at("delete_requests"), 1202u); } TEST(CASNamespaceJanitorBatches, RefusedEnqueueLeaksTheUnscheduledJobsAndTheNextRoundDrains) { - auto backend = std::make_shared(); + auto backend = std::make_shared(); auto store = openPool(backend, /*refuse_at=*/1); const Layout & layout = store->layout(); - const std::vector stream = seedLogs(*backend, layout, life("dead", 318), 100); + const std::vector stream = seedLogs(*backend, layout, life("dead", 318), 4); - /// Four jobs of 25 keys: the first runs, the second is refused and takes the rest with it. + /// Four one-key jobs: the first runs, the second is refused and takes the rest with it. Gc gc(store, DB::UInt128{319}); const auto refused = runRoundAndGetJanitorMetrics(gc); - EXPECT_EQ(refused.at("janitor_deleted"), 25u); - EXPECT_EQ(refused.at("leaked"), 75u); - EXPECT_EQ(countPresent(*backend, stream), 75u); + EXPECT_EQ(refused.at("janitor_deleted"), 1u); + EXPECT_EQ(refused.at("leaked"), 3u); + EXPECT_EQ(countPresent(*backend, stream), 3u); const auto drained = runRoundAndGetJanitorMetrics(gc); - EXPECT_EQ(drained.at("janitor_deleted"), 75u); + EXPECT_EQ(drained.at("janitor_deleted"), 3u); EXPECT_EQ(drained.at("leaked"), 0u); EXPECT_EQ(countPresent(*backend, stream), 0u); } diff --git a/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml b/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml index 28aedcb757036..eefb958012864 100644 --- a/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml +++ b/tests/integration/test_cas_janitor_drain/configs/storage_conf.xml @@ -14,6 +14,19 @@ 1 + + object_storage + s3 + cas + itest-janitor-small-chunk + http://rustfs1:11121/test/cas_janitor_small_chunk/ + clickhouse + clickhouse + 100 + 1 + 86400 + 1 + object_storage s3 @@ -30,6 +43,7 @@
cas_batch
 +
cas_small_chunk
cas_no_batch
 diff --git a/tests/integration/test_cas_janitor_drain/test.py b/tests/integration/test_cas_janitor_drain/test.py index 8526355320b2a..1f7d69827b6b0 100644 --- a/tests/integration/test_cas_janitor_drain/test.py +++ b/tests/integration/test_cas_janitor_drain/test.py @@ -1,6 +1,6 @@ """ -A dropped table's ref stream drains in one deleting round of the namespace janitor on RustFS, on a store -with a batch delete and on one without. +A dropped table's ref stream drains in one deleting round of the namespace janitor on RustFS: on a store with a +batch delete, on one with a small `objects_chunk_size_to_delete`, and on one without a batch delete. """ import pytest @@ -8,7 +8,8 @@ cluster = ClickHouseCluster(__file__) -INSERTS = 2500 +# Enough for more than one janitor page; small enough for the INSERT to finish under MSan. +INSERTS = 1100 MAX_ROUNDS = 3 @@ -47,7 +48,8 @@ def seed_and_drop(node, disk): node.query( f"INSERT INTO t_{disk} SELECT number FROM numbers({INSERTS}) SETTINGS max_block_size = 1, " "max_insert_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 0, " - "max_insert_threads = 1" + "max_insert_threads = 1", + timeout=1800, ) assert len(stream_keys(disk)) >= INSERTS node.query(f"DROP TABLE t_{disk} SYNC") @@ -66,7 +68,7 @@ def janitor_rows(node, disk, since): return [dict(zip(columns, map(int, line.split("\t")))) for line in out.strip().splitlines()] -@pytest.mark.parametrize("disk", ["cas_batch", "cas_no_batch"]) +@pytest.mark.parametrize("disk", ["cas_batch", "cas_small_chunk", "cas_no_batch"]) def test_dropped_table_drains_in_one_round(disk): node = cluster.instances["node"] since = node.query("SELECT now64(6)").strip() @@ -86,5 +88,8 @@ def test_dropped_table_drains_in_one_round(disk): assert row["budget_exhausted"] == 0 if disk == "cas_batch": assert row["delete_requests"] * 10 < dead_keys, f"{disk}: the dead keys did not go in batches: {rows}" + elif disk == "cas_small_chunk": + # One request per 100 keys, plus at most one short request per page. + assert dead_keys // 100 <= row["delete_requests"] <= dead_keys // 100 + row["janitor_pages"], f"{disk}: {rows}" else: assert row["delete_requests"] >= dead_keys, f"{disk}: expected one request per key: {rows}" From 701266697fd355d4296ad9a8cc8c7d1248c51755 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 7 Oct 2026 14:50:38 -0300 Subject: [PATCH 22/52] backport --- docs/en/interfaces/formats/Parquet/Parquet.md | 1 + .../BuzzHouse/Generator/SessionSettings.cpp | 1 + src/Core/FormatFactorySettings.h | 3 ++ src/Core/SettingsChangesHistory.cpp | 5 +++ src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 2 + .../Formats/Impl/Parquet/ReadManager.cpp | 2 +- .../Formats/Impl/Parquet/Reader.cpp | 16 ++++++-- src/Processors/Formats/Impl/Parquet/Reader.h | 3 +- .../Impl/ParquetV3BlockInputFormat.cpp | 19 +++++----- tests/integration/test_storage_delta/test.py | 10 ++++- .../05058_parquet_footer_read_size.reference | 6 +++ .../05058_parquet_footer_read_size.sql | 37 +++++++++++++++++++ 13 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/05058_parquet_footer_read_size.reference create mode 100644 tests/queries/0_stateless/05058_parquet_footer_read_size.sql diff --git a/docs/en/interfaces/formats/Parquet/Parquet.md b/docs/en/interfaces/formats/Parquet/Parquet.md index db31fc0dab80e..8f8bd703adf4d 100644 --- a/docs/en/interfaces/formats/Parquet/Parquet.md +++ b/docs/en/interfaces/formats/Parquet/Parquet.md @@ -154,6 +154,7 @@ To exchange data with Hadoop, you can use the [`HDFS table engine`](/engines/tab | `input_format_parquet_preserve_order` | Avoid reordering rows when reading from Parquet files. Usually makes it much slower. | `0` | | `input_format_parquet_filter_push_down` | When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata. | `1` | | `input_format_parquet_bloom_filter_push_down` | When reading Parquet files, skip whole row groups based on the WHERE expressions and bloom filter in the Parquet metadata. | `0` | +| `input_format_parquet_footer_read_size` | Size in bytes of the initial tail read that fetches the Parquet footer when opening a file with reader v3. `0` (default) sizes the read adaptively to 1% of the file, clamped to [128 KiB, 2 MiB]. A non-zero value forces a fixed initial read size. | `0` | | `input_format_parquet_allow_missing_columns` | Allow missing columns while reading Parquet input formats | `1` | | `input_format_parquet_local_file_min_bytes_for_seek` | Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format | `8192` | | `input_format_parquet_enable_row_group_prefetch` | Enable row group prefetching during parquet parsing. Currently, only single-threaded parsing can prefetch. | `1` | diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index cdc597a907807..d58d8878c19fb 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -1765,6 +1765,7 @@ void loadFuzzerServerSettings(const FuzzConfig & fc) "input_format_max_block_size_bytes", "input_format_parquet_local_file_min_bytes_for_seek", "input_format_parquet_prefer_block_bytes", + "input_format_parquet_footer_read_size", "input_format_parquet_memory_low_watermark", "input_format_parquet_memory_high_watermark", "join_runtime_bloom_filter_bytes", diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index 70efdcb163ae9..24138c995b11c 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -191,6 +191,9 @@ When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE ex )", 0) \ DECLARE(Bool, input_format_parquet_bloom_filter_push_down, true, R"( When reading Parquet files, skip whole row groups based on the WHERE expressions and bloom filter in the Parquet metadata. +)", 0) \ + DECLARE(UInt64, input_format_parquet_footer_read_size, 0, R"( +Size (in bytes) of the initial tail read that fetches the Parquet footer (`FileMetaData`) when opening a file with reader v3. `0` (default) sizes the read adaptively to the file - 1% of the file size, clamped to `[128 KiB, 2 MiB]` - so it usually covers the whole footer in a single read even for wide files with hundreds of columns. Set a non-zero value to force a fixed initial read size instead of the adaptive one; this is useful for very high-latency object storage where reading a larger tail up front avoids a second round trip. The value is clamped to the file size. )", 0) \ DECLARE(Bool, input_format_parquet_enable_json_parsing, true, R"( When reading Parquet files, parse JSON columns as ClickHouse JSON Column. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index f093c4054cf68..44799b1c02f3b 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -39,6 +39,11 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.6.4.20001.altinityantalya", + { + {"input_format_parquet_footer_read_size", 65536, 0, "New setting to override the adaptive Parquet footer initial read size with a fixed number of bytes; 0 keeps the adaptive behavior. Before this setting the reader always read a fixed 64 KiB tail."}, + }); + addSettingsChanges(settings_changes_history, "26.6.2.20001.altinityantalya", { {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index d6c30363c0d2a..b6dbd487b8541 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -217,6 +217,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.preserve_order = settings[Setting::input_format_parquet_preserve_order]; format_settings.parquet.filter_push_down = settings[Setting::input_format_parquet_filter_push_down]; format_settings.parquet.bloom_filter_push_down = settings[Setting::input_format_parquet_bloom_filter_push_down]; + format_settings.parquet.footer_read_size = settings[Setting::input_format_parquet_footer_read_size]; format_settings.parquet.page_filter_push_down = settings[Setting::input_format_parquet_page_filter_push_down]; format_settings.parquet.use_offset_index = settings[Setting::input_format_parquet_use_offset_index]; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 745898c0c751b..f30f3b6211248 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -347,6 +347,8 @@ struct FormatSettings bool case_insensitive_column_matching = false; bool filter_push_down = true; bool bloom_filter_push_down = true; + /// `0` sizes the initial footer read adaptively (1% of the file, clamped to [128 KiB, 2 MiB]). + size_t footer_read_size = 0; bool page_filter_push_down = true; bool use_offset_index = true; diff --git a/src/Processors/Formats/Impl/Parquet/ReadManager.cpp b/src/Processors/Formats/Impl/Parquet/ReadManager.cpp index 3040d1c1df671..17bda9f374475 100644 --- a/src/Processors/Formats/Impl/Parquet/ReadManager.cpp +++ b/src/Processors/Formats/Impl/Parquet/ReadManager.cpp @@ -54,7 +54,7 @@ void ReadManager::init(FormatParserSharedResourcesPtr parser_shared_resources_, parser_shared_resources = parser_shared_resources_; if (reader.file_metadata.schema.empty()) - reader.file_metadata = Reader::readFileMetaData(reader.prefetcher); + reader.file_metadata = Reader::readFileMetaData(reader.prefetcher, reader.options.format.parquet.footer_read_size); if (buckets_to_read_) { diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 64e25a3015493..61839a51b60d3 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -191,7 +192,7 @@ void Reader::init(const ReadOptions & options_, const Block & sample_block_, For format_filter_info = format_filter_info_; } -parq::FileMetaData Reader::readFileMetaData(Prefetcher & prefetcher) +parq::FileMetaData Reader::readFileMetaData(Prefetcher & prefetcher, size_t footer_read_size) { /// Parquet file ends with: /// * serialized FileMetaData struct, @@ -202,9 +203,16 @@ parq::FileMetaData Reader::readFileMetaData(Prefetcher & prefetcher) if (file_size <= 8) throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet file too short: {} bytes", file_size); - /// Read the last 64 KiB in hopes that FileMetaData is smaller than that. - /// This is usually enough for files smaller than a few hundred MB. - size_t initial_read_size = std::min(file_size, 64ul << 10); + /// Read a tail sized to the file (1%, clamped to [128 KiB, 2 MiB]) so it usually covers the whole + /// footer - `FileMetaData`, and the column/offset index just before it - in one read. A non-zero + /// `footer_read_size` overrides this adaptive size with a fixed read size. + if (footer_read_size == 0) + { + footer_read_size = std::clamp(file_size / 100, 128ul << 10, 2ul << 20); + } + /// The read must cover at least the 8-byte trailer (metadata size + magic) so the offsets below + /// don't underflow; an explicit `footer_read_size` smaller than that is bumped up to 8. + size_t initial_read_size = std::min(file_size, std::max(footer_read_size, 8)); PODArray buf(initial_read_size); prefetcher.readSync(buf.data(), initial_read_size, file_size - initial_read_size); diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 09d523fe327ce..7fb69094d9f29 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -512,7 +512,8 @@ struct Reader void init(const ReadOptions & options_, const Block & sample_block_, FormatFilterInfoPtr format_filter_info_); - static parq::FileMetaData readFileMetaData(Prefetcher & prefetcher); + /// `footer_read_size` overrides the initial footer read size; 0 sizes it adaptively to the file. + static parq::FileMetaData readFileMetaData(Prefetcher & prefetcher, size_t footer_read_size); void prefilterAndInitRowGroups(const std::optional> & row_groups_to_read); void preparePrewhere(); diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 8436e55647f91..65387f4a900b0 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -15,9 +15,7 @@ #include #include #include -#include #include -#include namespace DB { @@ -113,11 +111,11 @@ parquet::format::FileMetaData ParquetV3BlockInputFormat::getFileMetadata(Parquet String etag = object_with_metadata->metadata->etag; ParquetMetadataCacheKey cache_key = ParquetMetadataCache::createKey(file_name, etag); return metadata_cache->getOrSetMetadata( - cache_key, [&]() { return Parquet::Reader::readFileMetaData(prefetcher); }); + cache_key, [&]() { return Parquet::Reader::readFileMetaData(prefetcher, read_options.format.parquet.footer_read_size); }); } else { - return Parquet::Reader::readFileMetaData(prefetcher); + return Parquet::Reader::readFileMetaData(prefetcher, read_options.format.parquet.footer_read_size); } } @@ -273,7 +271,7 @@ void NativeParquetSchemaReader::initializeIfNeeded() return; Parquet::Prefetcher prefetcher; prefetcher.init(&in, read_options, /*parser_shared_resources_=*/ nullptr); - file_metadata = Parquet::Reader::readFileMetaData(prefetcher); + file_metadata = Parquet::Reader::readFileMetaData(prefetcher, read_options.format.parquet.footer_read_size); initialized = true; } @@ -348,12 +346,13 @@ void registerParquetFileBucketInfo(std::unordered_map std::vector ParquetBucketSplitter::splitToBuckets(size_t bucket_size, ReadBuffer & buf, const FormatSettings & format_settings_) { - std::atomic is_stopped = false; - auto arrow_file = asArrowFile(buf, format_settings_, is_stopped, "Parquet", PARQUET_MAGIC_BYTES, /* avoid_buffering */ true, nullptr); - auto metadata = parquet::ReadMetaData(arrow_file); + Parquet::ReadOptions read_options = convertReadOptions(format_settings_); + Parquet::Prefetcher prefetcher; + prefetcher.init(&buf, read_options, /*parser_shared_resources_=*/ nullptr); + auto metadata = Parquet::Reader::readFileMetaData(prefetcher, read_options.format.parquet.footer_read_size); std::vector bucket_sizes; - for (int i = 0; i < metadata->num_row_groups(); ++i) - bucket_sizes.push_back(metadata->RowGroup(i)->total_byte_size()); + for (const auto & row_group : metadata.row_groups) + bucket_sizes.push_back(size_t(row_group.total_byte_size)); std::vector> buckets; size_t current_weight = 0; diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index 39684966f8013..f004cbedc9efa 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -2618,10 +2618,16 @@ def test_column_pruning(started_cluster): table_function = f"deltaLake('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/{result_file}/', 'minio', '{minio_secret_key}')" + # `input_format_parquet_footer_read_size` is pinned so that the number of bytes read stays a + # measure of column pruning only. With the adaptive default the initial footer read is at least + # 128 KiB, which is more than this (much smaller) file, so the whole file would be read and the + # numbers below would no longer say anything about which columns were fetched. + footer_read_size = 64 * 1024 + query_id = f"query_{TABLE_NAME}_1" sum = int( instance.query( - f"SELECT sum(id) FROM {table_function} SETTINGS allow_experimental_delta_kernel_rs=0, max_read_buffer_size_remote_fs=100, remote_read_min_bytes_for_seek=1", + f"SELECT sum(id) FROM {table_function} SETTINGS allow_experimental_delta_kernel_rs=0, max_read_buffer_size_remote_fs=100, remote_read_min_bytes_for_seek=1, input_format_parquet_footer_read_size={footer_read_size}", query_id=query_id, ) ) @@ -2637,7 +2643,7 @@ def test_column_pruning(started_cluster): query_id = f"query_{TABLE_NAME}_2" assert sum == int( instance.query( - f"SELECT sum(id) FROM {table_function} SETTINGS enable_filesystem_cache=0, max_read_buffer_size_remote_fs=100, remote_read_min_bytes_for_seek=1, use_parquet_metadata_cache=0", + f"SELECT sum(id) FROM {table_function} SETTINGS enable_filesystem_cache=0, max_read_buffer_size_remote_fs=100, remote_read_min_bytes_for_seek=1, use_parquet_metadata_cache=0, input_format_parquet_footer_read_size={footer_read_size}", query_id=query_id, ) ) diff --git a/tests/queries/0_stateless/05058_parquet_footer_read_size.reference b/tests/queries/0_stateless/05058_parquet_footer_read_size.reference new file mode 100644 index 0000000000000..3baa52177c35d --- /dev/null +++ b/tests/queries/0_stateless/05058_parquet_footer_read_size.reference @@ -0,0 +1,6 @@ +10000 49995000 0 9999 +10000 49995000 0 9999 +10000 49995000 0 9999 +10000 49995000 0 9999 +10000 49995000 0 9999 +65536 diff --git a/tests/queries/0_stateless/05058_parquet_footer_read_size.sql b/tests/queries/0_stateless/05058_parquet_footer_read_size.sql new file mode 100644 index 0000000000000..41c14681845bf --- /dev/null +++ b/tests/queries/0_stateless/05058_parquet_footer_read_size.sql @@ -0,0 +1,37 @@ +-- Tags: no-fasttest, no-random-settings +-- no-fasttest: needs the `Parquet` format, which is not built in fasttest. +-- no-random-settings: the last statement checks that `compatibility` = '26.6' restores 65536. +-- Tests `input_format_parquet_footer_read_size`: the v3 reader sizes the initial footer read +-- adaptively when the setting is 0 (default), and uses the explicit value otherwise. Every +-- value - the adaptive default, a huge fixed read that covers the whole footer in one go, and a +-- tiny fixed read that forces the second (metadata_size + 8 > initial_read_size) read path - must +-- produce identical, correct results. + +insert into function file(currentDatabase() || '_data_05058.parquet') + select number as x, toString(number) as s from numbers(10000) + settings engine_file_truncate_on_insert = 1; + +-- Adaptive (default 0): 1% clamped to [128 KiB, 2 MiB]. +select count(), sum(x), min(s), max(s) from file(currentDatabase() || '_data_05058.parquet', auto, 'x UInt64, s String') + settings input_format_parquet_footer_read_size = 0; + +-- Explicit large read: footer certainly fits in the initial read. +select count(), sum(x), min(s), max(s) from file(currentDatabase() || '_data_05058.parquet', auto, 'x UInt64, s String') + settings input_format_parquet_footer_read_size = 8388608; + +-- Explicit tiny read: forces the second read to fetch the rest of the footer. +select count(), sum(x), min(s), max(s) from file(currentDatabase() || '_data_05058.parquet', auto, 'x UInt64, s String') + settings input_format_parquet_footer_read_size = 16; + +-- Explicit value larger than the file: clamped to the file size, still correct. +select count(), sum(x), min(s), max(s) from file(currentDatabase() || '_data_05058.parquet', auto, 'x UInt64, s String') + settings input_format_parquet_footer_read_size = 1073741824; + +-- Explicit value below the 8-byte trailer: bumped up to 8, no out-of-bounds read. +select count(), sum(x), min(s), max(s) from file(currentDatabase() || '_data_05058.parquet', auto, 'x UInt64, s String') + settings input_format_parquet_footer_read_size = 1; + +-- Compatibility contract: the setting is new in 26.6.4 Antalya. `SET compatibility` to 26.6 must +-- restore the previous fixed 64 KiB footer read, not the new adaptive default (0). +set compatibility = '26.6'; +select getSetting('input_format_parquet_footer_read_size'); From 15351844b802d12f58511b4ddf45b6c6dc62fdff Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 8 Oct 2026 13:20:41 +0200 Subject: [PATCH 23/52] Document where a distributed JOIN runs and which table is sent A `GLOBAL` `INNER` or `LEFT` join sends the right table and reads the left table on the shards. A `GLOBAL` `RIGHT` join of a multi-shard `Distributed` table sends the left table instead, so unmatched rows of the preserved side are returned once. `FULL` joins and `object_storage_cluster_join_mode` `allow` and `local` run the join on the initiator. Co-authored-by: Cursor --- .../table-engines/special/distributed.md | 2 +- .../sql-reference/distribution-on-cluster.md | 14 ++- docs/en/sql-reference/operators/in.md | 2 +- .../sql-reference/statements/select/join.md | 88 ++++++++++++++++++- src/Core/Settings.cpp | 17 ++-- 5 files changed, 102 insertions(+), 21 deletions(-) diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index 9890c841eda45..9fef7adb0a1fe 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -217,7 +217,7 @@ When querying a `Distributed` table, `SELECT` queries are sent to all shards and When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#max_parallel_replicas). -To learn more about how distributed `in` and `global in` queries are processed, refer to [this](/sql-reference/operators/in#distributed-subqueries) documentation. +To learn more about how distributed `IN` and `GLOBAL IN` queries are processed, refer to [Distributed subqueries](/sql-reference/operators/in#distributed-subqueries). For where a `JOIN` runs and which table is sent to the shards, see [Distributed JOIN](/sql-reference/statements/select/join.md#distributed-join). ## Virtual columns {#virtual-columns} diff --git a/docs/en/sql-reference/distribution-on-cluster.md b/docs/en/sql-reference/distribution-on-cluster.md index 7405ed7906935..c1b9a4bdcfaae 100644 --- a/docs/en/sql-reference/distribution-on-cluster.md +++ b/docs/en/sql-reference/distribution-on-cluster.md @@ -1,6 +1,6 @@ -# Task distribution in *Cluster family functions +# Task distribution in *Cluster family functions {#task-distribution-in-cluster-family-functions} -## Task distribution algorithm +## Task distribution algorithm {#task-distribution-algorithm} Table functions such as `s3Cluster`, `azureBlobStorageCluster`, `hdsfCluster`, `icebergCluster`, and table engines like `S3`, `Azure`, `HDFS`, `Iceberg` with the setting `object_storage_cluster` distribute tasks across all cluster nodes or a subset limited by the `object_storage_max_nodes` setting. This setting limits the number of nodes involved in processing a distributed query, randomly selecting nodes for each query. @@ -12,14 +12,20 @@ For each file, one cluster node is selected as the primary node using a consiste This improves cache efficiency by minimizing data movement among nodes. -## `lock_object_storage_task_distribution_ms` setting +## `lock_object_storage_task_distribution_ms` setting {#lock-object-storage-task-distribution-ms} Each node begins processing files for which it is the primary node. After completing its assigned files, a node may take tasks from other nodes, either immediately or after waiting for `lock_object_storage_task_distribution_ms` milliseconds if the primary node does not request new files during that interval. The default value of `lock_object_storage_task_distribution_ms` is 500 milliseconds. This setting balances between caching efficiency and workload redistribution when nodes are imbalanced. The wait is a field of `ReadTaskResponse` and is sent only when every worker negotiated Antalya protocol version 2. If any worker negotiated a lower version, including an upstream node, the setting is ignored for that query and files are assigned immediately. -## `SYSTEM STOP SWARM MODE` command +## `SYSTEM STOP SWARM MODE` command {#system-stop-swarm-mode} If a node needs to shut down gracefully, the command `SYSTEM STOP SWARM MODE` prevents the node from receiving new tasks for *Cluster-family queries. The node finishes processing already assigned files before it can safely shut down without errors. Receiving new tasks can be resumed with the command `SYSTEM START SWARM MODE`. + +## Joins {#joins} + +[`object_storage_cluster_join_mode`](/operations/settings/settings#object_storage_cluster_join_mode) chooses where a `JOIN` of a `*Cluster` table function, or of a table with `object_storage_cluster` set, is executed, and which table is sent to the nodes. With `allow` and `local` the nodes read the left table and the join runs on the initiator. With `global`, an `INNER`, `LEFT`, or `CROSS` join runs on the nodes and the initiator sends the right table (every table after the first, for `CROSS JOIN`). A `RIGHT` or `FULL` join stays on the initiator. + +The same rules for a `Distributed` table, including which side of a `GLOBAL` `RIGHT` join is sent, are described in [Distributed JOIN](/sql-reference/statements/select/join.md#distributed-join). diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index 0f729ca7b854a..4bafb6d2be22d 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -172,7 +172,7 @@ When using the regular `IN`, the query is sent to remote servers, and each of th When using `GLOBAL IN` / `GLOBAL JOIN`, first all the subqueries are run for `GLOBAL IN` / `GLOBAL JOIN`, and the results are collected in temporary tables. Then the temporary tables are sent to each remote server, where the queries are run using this temporary data. -For `GLOBAL ... JOIN`, which side of the join is calculated as the subquery depends on the join kind: for `LEFT` and `INNER` joins, the right table is calculated; for `RIGHT` joins, the left table is calculated instead, since the right table is the preserved side and should be read from shards. +Which side of a `GLOBAL ... JOIN` is calculated on the initiator and sent to the nodes depends on the join kind, the number of shards, and whether the join runs on the shards or on the initiator. See [Distributed JOIN](../statements/select/join.md#distributed-join). For a non-distributed query, use the regular `IN` / `JOIN`. diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index 32fba41406d32..1cb12b0e3b3ad 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -389,12 +389,92 @@ SETTINGS max_block_size = 2; ## Distributed JOIN {#distributed-join} -There are two ways to execute a JOIN involving distributed tables: +A query whose left table is [`Distributed`](/engines/table-engines/special/distributed.md) is sent to the shards of its cluster. The same applies to a `*Cluster` table function (`s3Cluster`, `icebergCluster`, and the others) and to an object-storage table engine with [`object_storage_cluster`](/operations/settings/settings#object_storage_cluster) set (`S3`, `Iceberg`, and the others). An `Alias`, a `Buffer` table, or a `MaterializedView` in front of `Distributed` is treated as that `Distributed` table. -- When using a normal `JOIN`, the query is sent to remote servers. Subqueries are run on each of them in order to make the right table, and the join is performed with this table. In other words, the right table is formed on each server separately. -- When using `GLOBAL ... JOIN`, first the requestor server runs a subquery to calculate one side of the join and collects the result into a temporary table. This temporary table is then passed to each remote server, and queries are run on them using the temporary data that was transmitted. For `LEFT` and `INNER` joins, the right table is calculated as the subquery. For `RIGHT` joins, the left table is calculated instead, since the right table is the one being preserved and should be read from shards. +The join then runs in one of two places: -Be careful when using `GLOBAL`. For more information, see the [Distributed subqueries](/sql-reference/operators/in#distributed-subqueries) section. +- **On the shards.** Each node reads its part of the table that stays on the cluster and runs the join. The initiator is the server that received the query. For a `GLOBAL` join it reads the other side once, stores it in a temporary table, and sends that table to every node. For a join without `GLOBAL` it sends nothing: each node builds the right side from the query it receives. +- **On the initiator.** The nodes read the left table and return its rows. The initiator reads the right table and runs the join. No join side is sent to the nodes. + +`SELECT` column order stays the order written in the query, including when a `RIGHT` join is executed by reading the right table on the shards. + +### Which table is sent {#distributed-join-which-table-is-sent} + +| Where the join runs | Condition | Read on the nodes | Sent by the initiator | +| --- | --- | --- | --- | +| Shards | `JOIN` without `GLOBAL` | Left table. The right table is built on each node from the query as written | Nothing | +| Shards | `GLOBAL` `INNER`, `LEFT`, `LEFT SEMI`, `LEFT ANTI` | Left table | Right table | +| Shards | `GLOBAL` `RIGHT`, `RIGHT SEMI`, `RIGHT ANTI`, and the left table is `Distributed` over **two or more shards** | Right table | Left table | +| Shards | `GLOBAL` `RIGHT`, `RIGHT SEMI`, `RIGHT ANTI`, and the left table is `Distributed` over **one shard** | Left table | Right table | +| Shards | `CROSS JOIN` with [`object_storage_cluster_join_mode`](/operations/settings/settings#object_storage_cluster_join_mode) `global` | The first table | Every table after the first | +| Initiator | The cases in [Join on the initiator](#distributed-join-on-the-initiator) | Rows of the left table, returned to the initiator | Nothing | + +A join without `GLOBAL` sends nothing. Each shard builds the right side itself. That result is correct when the rows that have to match already live on the same shard, for example when both tables are sharded by the join key. [`distributed_product_mode`](#distributed-product-mode-join) applies when the right side is distributed as well. + +A join without the `GLOBAL` keyword is executed as `GLOBAL` when [`distributed_product_mode`](/operations/settings/settings#distributed_product_mode) is `global`. With [`prefer_global_in_and_join`](/operations/settings/settings#prefer_global_in_and_join) enabled, the join is executed as `GLOBAL` for every `distributed_product_mode` value except `local`. + +For a `GLOBAL` `INNER` or `LEFT` join the preserved rows are on the left, so the right table is the one that is sent: + +```sql +SELECT * +FROM left_distributed AS l +GLOBAL LEFT JOIN right_table AS r ON l.id = r.id +``` + +Each shard reads its own rows of `left_distributed` and joins them with one copy of `right_table`. + +For a `GLOBAL` `RIGHT` join the preserved rows are on the right. When `left_distributed` has two or more shards, the left table is the one that is sent, and each shard reads its own rows of the right table: + +```sql +SELECT * +FROM left_distributed AS l +GLOBAL RIGHT JOIN right_distributed AS r ON l.id = r.id +``` + +Unmatched rows of `right_distributed` are returned from the shard that stores them, once. Sending the right table instead would return each of those rows once per shard. + +`ALL`, `ANY`, `SEMI`, and `ANTI` use the same side as the join kind. `USING` and `NATURAL` do too. The right table of such a `RIGHT` join may be distributed, local, a subquery, or a table function. + +With one shard there is no second copy of the preserved side, so a `GLOBAL` `RIGHT` join sends the right table and reads the left table on that shard. + +### Join on the initiator {#distributed-join-on-the-initiator} + +The join runs on the initiator in these cases: + +- The left table is local or a subquery. A distributed table on the right is read by the initiator. +- [`object_storage_cluster_join_mode`](/operations/settings/settings#object_storage_cluster_join_mode) is `allow` (the default) or `local`. Nodes read the cluster table. The other table stays on the initiator. +- The join is `FULL` and the left table is `Distributed` over two or more shards, or the left table is an object-storage cluster table with `object_storage_cluster_join_mode = 'global'`. +- The join is `RIGHT`, `RIGHT SEMI`, or `RIGHT ANTI`, the left table has two or more shards, and the preserved side cannot stay on the shards. This is an `ASOF` join, `RIGHT ANY`, a `USING` key that is not a plain column on each side, a `PREWHERE` that names no column (`PREWHERE 0`), or a `RIGHT` / `FULL` join written above another join, such as `(A INNER JOIN C) RIGHT JOIN B`. A `RIGHT` or `FULL` join of an object-storage cluster table with `object_storage_cluster_join_mode = 'global'` stays on the initiator as well. + +In the last two cases a multi-shard left table is read to the initiator through a subquery, and the join runs there. Running it on every shard would emit each unmatched row of the preserved side once per shard. + +### `distributed_product_mode` {#distributed-product-mode-join} + +When the shard query joins two distributed tables and the join is not `GLOBAL`, [`distributed_product_mode`](/operations/settings/settings#distributed_product_mode) chooses what happens to the right table: + +| Value | Behavior | +| --- | --- | +| `deny` | The query is rejected with `DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED`. | +| `local` | On each shard the right `Distributed` table is replaced with its underlying local table. The join uses only the rows of the right table stored on that shard. | +| `global` | The join is executed as `GLOBAL`. The side from the table above is read on the initiator and sent. | +| `allow` | Each shard runs the right side as its own distributed query. | + +### `object_storage_cluster_join_mode` {#object-storage-cluster-join-mode} + +The setting applies when the left table is a `*Cluster` table function or a table engine with `object_storage_cluster` set. + +| Value | Where the join runs | What is sent | +| --- | --- | --- | +| `allow` | Initiator | Nothing. Nodes read the left table and return its rows. | +| `local` | Initiator | Nothing. Same execution as `allow`. | +| `global` | Nodes, for `INNER`, `LEFT`, and `CROSS` | The right table. For `CROSS JOIN`, every table after the first. | +| `global` | Initiator, for `RIGHT` and `FULL` | Nothing. A `RIGHT` or `FULL` join on every node would repeat unmatched rows of the preserved side. | + +### `PREWHERE` {#distributed-join-prewhere} + +`PREWHERE` is applied on the table it belongs to. When that table is the one sent to the nodes, the filter runs while the initiator builds the temporary table. When that table stays on the nodes, the filter runs there. A `PREWHERE` that names no column is attached to the leftmost table and keeps a `GLOBAL` `RIGHT` join on the initiator. + +`GLOBAL` copies one side of the join to every node. For a large side that is expensive, and the copy has to fit in memory on each node. See also [Distributed subqueries](/sql-reference/operators/in#distributed-subqueries). ## Implicit type conversion {#implicit-type-conversion} diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8f79261cec922..c3f0a39e28740 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -2129,20 +2129,15 @@ Possible values: - `allow` — Allows the use of these types of subqueries. )", IMPORTANT) \ DECLARE(ObjectStorageClusterJoinMode, object_storage_cluster_join_mode, ObjectStorageClusterJoinMode::ALLOW, R"( -Changes the behaviour of object storage cluster function or table. - -ClickHouse applies this setting when the query contains the product of object storage cluster function or table, i.e. when the query for a object storage cluster function or table contains a non-GLOBAL subquery for the object storage cluster function or table. - -Restrictions: - -- Only applied for JOIN subqueries. -- Only if the FROM section uses a object storage cluster function or table. +Where a [JOIN](../../sql-reference/statements/select/join.md#distributed-join) runs when its left table is a cluster table function (`s3Cluster`, `icebergCluster`, and the other `*Cluster` functions) or a table engine with [`object_storage_cluster`](#object_storage_cluster) set. Possible values: -- `local` — Replaces the database and table in the subquery with local ones for the destination server (shard), leaving the normal `IN`/`JOIN.` -- `global` — Replaces the `IN`/`JOIN` query with `GLOBAL IN`/`GLOBAL JOIN.` Right table executes first and is added to the secondary query as temporay table. -- `allow` — Default value. Allows the use of these types of subqueries. +- `allow` — Default. Nodes read the left table and return its rows. The join runs on the initiator. The right table is read there and is not sent to the nodes. +- `local` — The join runs on the initiator. Nodes read the left table and return its rows. The right table is read on the initiator and is not sent to the nodes. +- `global` — `INNER`, `LEFT`, and `CROSS` run on the nodes. The initiator reads the right table once and sends it to each node. For `CROSS JOIN`, every table after the first is sent. `RIGHT` and `FULL` stay on the initiator, so unmatched rows of the preserved side are returned once. + +See [Distributed JOIN](../../sql-reference/statements/select/join.md#object-storage-cluster-join-mode). )", 0) \ \ DECLARE(UInt64, max_concurrent_queries_for_all_users, 0, R"( From 7265c4bd0a30289bf7f0f647d242ce9b4bf20ccc Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Thu, 8 Oct 2026 15:57:45 -0300 Subject: [PATCH 24/52] Mark 02354_vector_search_rescoring_distance_in_select_list as broken on ARM sanitizer builds ARM sanitizer builds have been built without SimSIMD since #2341, so USearch converts float32 to bf16 with its own fallback. The fallback truncates while SimSIMD rounds, and the test gets 0.1875 instead of 0.203125 for id 7. It fails deterministically on Stateless tests (arm_asan_ubsan, azure, parallel). Signed-off-by: CarlosFelipeOR --- tests/broken_tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/broken_tests.yaml b/tests/broken_tests.yaml index f8f9043141d02..3e8cac0fc4dee 100644 --- a/tests/broken_tests.yaml +++ b/tests/broken_tests.yaml @@ -144,6 +144,15 @@ check_types: - arm - cpu-aarch64 +- name: 02354_vector_search_rescoring_distance_in_select_list + reason: 'KNOWN: https://github.com/Altinity/ClickHouse/issues/2340 - ARM sanitizer builds have no SimSIMD since #2341; USearch fallback truncates bf16 instead of rounding' + message: '\+7\s0\.1875' + regex: true + check_types: + - asan + - ubsan + - tsan + - msan - name: 01154_move_partition_long reason: INVESTIGATE - Unstable on debug message: 'curl: (28) Operation timed out' From ec8d3624fce01a7922a1ec238b28bf1e4ef705dc Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Thu, 8 Oct 2026 16:22:32 -0300 Subject: [PATCH 25/52] Add missing __init__.py to two Iceberg integration test dirs test_export_merge_tree_part_to_iceberg and test_iceberg_mixed_antalya_upstream were the only integration test directories without __init__.py. pytest imports both test.py files as the top-level module "test", so whenever they land in the same shard the second one fails collection with "import file mismatch" and never runs. This happens on every Integration tests (amd_tsan) run on antalya-26.6 since #2482 added the mixed test. Verified locally with praktika on "Integration tests (amd_tsan, 6/6)": without the files, 39 passed and 1 collection error; with them, 42 passed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../test_export_merge_tree_part_to_iceberg/__init__.py | 0 tests/integration/test_iceberg_mixed_antalya_upstream/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_export_merge_tree_part_to_iceberg/__init__.py create mode 100644 tests/integration/test_iceberg_mixed_antalya_upstream/__init__.py diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/__init__.py b/tests/integration/test_export_merge_tree_part_to_iceberg/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_iceberg_mixed_antalya_upstream/__init__.py b/tests/integration/test_iceberg_mixed_antalya_upstream/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d From c72ea99e99907c152cdb90f086cf9cd931b2aa39 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 8 Oct 2026 23:39:45 +0200 Subject: [PATCH 26/52] Added nullcheck for type before calling unknown type check --- .../ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 96e6e41462d35..a66fdb2ad8062 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -842,7 +842,8 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da bool MetadataGenerator::generateModifyColumnMetadata(const String & column_name, DataTypePtr type, ContextPtr context, bool first, const String & after_column) { - Iceberg::checkUnknownTypeAllowed(column_name, type, metadata_object->getValue(Iceberg::f_format_version)); + if (type) + Iceberg::checkUnknownTypeAllowed(column_name, type, metadata_object->getValue(Iceberg::f_format_version)); auto current_schema = getCurrentSchema(); auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); From f4cdab07f1182f6a11ed1c4b8d2ac00071d79b5e Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 09:38:38 +0200 Subject: [PATCH 27/52] Revert leftover changes in `test_backup_restore_on_cluster/test_cancel_backup.py` CAS is not used in this module since b84beb33837, so the `S3_ERROR` filter is not needed. Co-Authored-By: Claude Opus 5.5 --- .../test_backup_restore_on_cluster/test_cancel_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py index a789295a61e1b..dceb3e89744fe 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py @@ -359,7 +359,6 @@ def __exit__(self, type, value, traceback): + start_time + "') " + "AND NOT ((name == 'KEEPER_EXCEPTION') AND (last_error_message LIKE '%Fault injection%')) " - + "AND NOT ((name == 'S3_ERROR') AND (last_error_message LIKE '%This error is expected for S3 disk%')) " + "AND NOT (name == 'NO_ELEMENTS_IN_CONFIG')" ) errors = errors_query_result.splitlines() @@ -557,6 +556,7 @@ def test_cancel_restore(): wait_status(initiator, "RESTORED", restore_id=restore_id) assert get_num_system_processes(nodes, restore_id=restore_id) == 0 + # Test that shutdown cancels a running backup and doesn't wait until it finishes. def test_shutdown_cancels_backup(): wait_for_backups_to_finish() From cee39505bc5c269bf653224fb0cefe4da891b3fa Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 09:41:52 +0200 Subject: [PATCH 28/52] Remove unused `parts/` prefix from CAS pool check CAS has no `parts/` key family, so this prefix always counted zero objects. Co-Authored-By: Claude Opus 5.5 --- contrib/NuRaft | 2 +- contrib/StringZilla | 2 +- contrib/abseil-cpp | 2 +- contrib/aws-c-http | 2 +- contrib/azure | 2 +- contrib/boost | 2 +- contrib/chdig | 2 +- contrib/clickstack | 2 +- contrib/delta-kernel-rs | 2 +- contrib/fast_float | 2 +- contrib/google-cloud-cpp | 2 +- contrib/h3 | 2 +- contrib/jemalloc | 2 +- contrib/libarchive | 2 +- contrib/librdkafka | 2 +- contrib/libssh | 2 +- contrib/libxml2 | 2 +- contrib/llvm-project | 2 +- contrib/mariadb-connector-c | 2 +- contrib/mongo-c-driver | 2 +- contrib/msgpack-c | 2 +- contrib/nats-io | 2 +- contrib/openssl | 2 +- contrib/orc | 2 +- contrib/re2 | 2 +- contrib/replxx | 2 +- contrib/rust_vendor | 2 +- contrib/s2geometry | 2 +- contrib/silk | 2 +- contrib/simdjson | 2 +- contrib/sysroot | 2 +- contrib/usearch | 2 +- contrib/zstd | 2 +- tests/integration/test_backup_restore_new/test_cancel_backup.py | 2 +- 34 files changed, 34 insertions(+), 34 deletions(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index fff08ccaf1664..a0eda0069d2da 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit fff08ccaf16646941d91ab0d26b94c76c17e44d1 +Subproject commit a0eda0069d2dac0852c453cbe7b0285eda218479 diff --git a/contrib/StringZilla b/contrib/StringZilla index 56c8d90869c87..15c292ec71db5 160000 --- a/contrib/StringZilla +++ b/contrib/StringZilla @@ -1 +1 @@ -Subproject commit 56c8d90869c878a71dde292fa16872f661421643 +Subproject commit 15c292ec71db5c0e3d05f3bd693a6b5ec9429db8 diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 255c84dadd029..5650e9cf76d3b 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 255c84dadd029fd8ad25c5efb5933e47beaa00c7 +Subproject commit 5650e9cf76d3be4318d5fa3af38ee483ddfd5e4a diff --git a/contrib/aws-c-http b/contrib/aws-c-http index 8aefd899fc321..a9745ea9998f6 160000 --- a/contrib/aws-c-http +++ b/contrib/aws-c-http @@ -1 +1 @@ -Subproject commit 8aefd899fc3210bfd0e3fd414011a3cb708bf6e4 +Subproject commit a9745ea9998f679cd7456e7d23cc8820e38c97d4 diff --git a/contrib/azure b/contrib/azure index e1c250fe5dd43..20f8b46b87871 160000 --- a/contrib/azure +++ b/contrib/azure @@ -1 +1 @@ -Subproject commit e1c250fe5dd4306e3b7d19600788f4029c29786c +Subproject commit 20f8b46b878718b4536a88888a182f60f665465a diff --git a/contrib/boost b/contrib/boost index f05991ceab7d4..e24b1b25e3349 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit f05991ceab7d486b82251679cb1d54816cde9df0 +Subproject commit e24b1b25e3349236a3c5aac792e3294f62eda1f2 diff --git a/contrib/chdig b/contrib/chdig index 417e74751cef1..9302521212c89 160000 --- a/contrib/chdig +++ b/contrib/chdig @@ -1 +1 @@ -Subproject commit 417e74751cef15b1079ded06caee8c318163effd +Subproject commit 9302521212c898fa7d91df827136e74f3a760bb7 diff --git a/contrib/clickstack b/contrib/clickstack index 4aa7f5ed0369f..f5232dc36db0f 160000 --- a/contrib/clickstack +++ b/contrib/clickstack @@ -1 +1 @@ -Subproject commit 4aa7f5ed0369f46166d1fb56b1b8893f09d99793 +Subproject commit f5232dc36db0f151c6a19a770710fbeebfd9051b diff --git a/contrib/delta-kernel-rs b/contrib/delta-kernel-rs index 857565caf9ac6..88ff5a9474782 160000 --- a/contrib/delta-kernel-rs +++ b/contrib/delta-kernel-rs @@ -1 +1 @@ -Subproject commit 857565caf9ac61511c0e005ee3a32a30877a1a15 +Subproject commit 88ff5a947478215d285783b42aafe6f3d04aca58 diff --git a/contrib/fast_float b/contrib/fast_float index 01ce95dfe46ab..34164f547b7df 160000 --- a/contrib/fast_float +++ b/contrib/fast_float @@ -1 +1 @@ -Subproject commit 01ce95dfe46abccf3264fbccf9f5139ea9016cd2 +Subproject commit 34164f547b7df3f5d794ff67e9f885c36819ebfc diff --git a/contrib/google-cloud-cpp b/contrib/google-cloud-cpp index 83f30caadb861..499787a71646e 160000 --- a/contrib/google-cloud-cpp +++ b/contrib/google-cloud-cpp @@ -1 +1 @@ -Subproject commit 83f30caadb8613fb5c408d8c2fd545291596b53f +Subproject commit 499787a71646ee5f7d9a845aee79a1b0663c947d diff --git a/contrib/h3 b/contrib/h3 index 70d4c9e5007d7..e38f58ef05128 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit 70d4c9e5007d74d77714afa44d5a2e52606828bf +Subproject commit e38f58ef051280ebdd628d275a12cb667e8c4a3c diff --git a/contrib/jemalloc b/contrib/jemalloc index e6cea775d3162..5f864d8bd4c4f 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit e6cea775d316273b05581f6a7c3609be649eb754 +Subproject commit 5f864d8bd4c4fa7a99cb47c374d5c6465a160578 diff --git a/contrib/libarchive b/contrib/libarchive index 7219b0134d771..27cbc78271726 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 7219b0134d771dc4b51bf86b4d01761b87398b1b +Subproject commit 27cbc7827172698143e440801fc0ba39ccb4f1f5 diff --git a/contrib/librdkafka b/contrib/librdkafka index 6aa06cfec6ae3..a0f91df2d9eb9 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit 6aa06cfec6ae35efb6e6f895326ed0ec2ecc2a27 +Subproject commit a0f91df2d9eb989829ddc5d5358da48342b41d48 diff --git a/contrib/libssh b/contrib/libssh index 50313883f3a07..c0083428b0c7f 160000 --- a/contrib/libssh +++ b/contrib/libssh @@ -1 +1 @@ -Subproject commit 50313883f3a077458cde4ea95bf46bfeb0771b34 +Subproject commit c0083428b0c7f97f7b6a9c212d630c49b72a2d7a diff --git a/contrib/libxml2 b/contrib/libxml2 index c94eb0210183b..96498992efa48 160000 --- a/contrib/libxml2 +++ b/contrib/libxml2 @@ -1 +1 @@ -Subproject commit c94eb0210183b9d7cb43f8e7fddc6be55843ef49 +Subproject commit 96498992efa48d52b0e8b83058bd88dbdaf153c1 diff --git a/contrib/llvm-project b/contrib/llvm-project index 588b33887bd57..c7f5fc7904a38 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit 588b33887bd57ed2e5df6b15ea26d657e7ff0cef +Subproject commit c7f5fc7904a38ed3d1f6999fc615ad666905d1df diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 95d6264bb43d3..111ec1a5958cf 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 95d6264bb43d31f825cf2f85ce1928557457b62d +Subproject commit 111ec1a5958cf984f50a28b0fb82a4087918d677 diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index a26c8e251fec1..6b6b676bdbd46 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit a26c8e251fec147258e89b7e85e9adfa259fc42f +Subproject commit 6b6b676bdbd46fdb954ed1535893020ef91cce5b diff --git a/contrib/msgpack-c b/contrib/msgpack-c index 13a5894d9ffec..6b0d778ae059c 160000 --- a/contrib/msgpack-c +++ b/contrib/msgpack-c @@ -1 +1 @@ -Subproject commit 13a5894d9ffec2aeafab560d40a5d9e40d9c1cbe +Subproject commit 6b0d778ae059cfb77e26f2a011e99b17a8563c23 diff --git a/contrib/nats-io b/contrib/nats-io index cf441828d30fd..3e3a3f10572ed 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit cf441828d30fdd5de12d9da319e88d2586fdeeba +Subproject commit 3e3a3f10572ed9ff1b7ad17873881877cb62f301 diff --git a/contrib/openssl b/contrib/openssl index f3b2f0273ed59..614167b874c61 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit f3b2f0273ed59cef874db0d6943f8d7d1804bd31 +Subproject commit 614167b874c612973344b49f772079213b3606eb diff --git a/contrib/orc b/contrib/orc index 8ca5eeed390f5..6a2fe65eb16ce 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit 8ca5eeed390f57fe6d1d4f7f8d97da19de6c08cb +Subproject commit 6a2fe65eb16ce4c760964dca619260226c1b418e diff --git a/contrib/re2 b/contrib/re2 index 85dd7ad833a73..1f24555c97150 160000 --- a/contrib/re2 +++ b/contrib/re2 @@ -1 +1 @@ -Subproject commit 85dd7ad833a73095ecf3e3baea608ba051bbe2c7 +Subproject commit 1f24555c97150767d65aa77fde8ddd029d2bb97c diff --git a/contrib/replxx b/contrib/replxx index 3a55a1bbee3c4..c2de583a3cd41 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 3a55a1bbee3c4ccdbad38c0f1da4ce722e337807 +Subproject commit c2de583a3cd41f7b476cb625e954c191b6fcc448 diff --git a/contrib/rust_vendor b/contrib/rust_vendor index e7b13e76a6f7f..8f91308de3d1b 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit e7b13e76a6f7f9b6408d0695663539a89b010c4c +Subproject commit 8f91308de3d1b78a1fc3256a2e64bd91744c3a11 diff --git a/contrib/s2geometry b/contrib/s2geometry index a37aba69f14af..5409e159ec6e0 160000 --- a/contrib/s2geometry +++ b/contrib/s2geometry @@ -1 +1 @@ -Subproject commit a37aba69f14af676e9605cd9515c8aca6cef8342 +Subproject commit 5409e159ec6e0334022395ed33ed24e60c69b6ae diff --git a/contrib/silk b/contrib/silk index e7d5d331a284b..78aab9d603d43 160000 --- a/contrib/silk +++ b/contrib/silk @@ -1 +1 @@ -Subproject commit e7d5d331a284bd9d7608c41e1ce21303c5eeb72f +Subproject commit 78aab9d603d43247fa8f05f86c34ecdfe2ab4095 diff --git a/contrib/simdjson b/contrib/simdjson index 0a851a64cd984..e95c118160780 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit 0a851a64cd984e9e1a6cab93b6e773aa3f4dc30d +Subproject commit e95c118160780c2b808d86953938ce40b445d764 diff --git a/contrib/sysroot b/contrib/sysroot index bed292b837319..caf9c9f560e5a 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit bed292b8373193cd49005a617a0df741d9cbf52d +Subproject commit caf9c9f560e5a06295d961860ed717fe169e8cef diff --git a/contrib/usearch b/contrib/usearch index 1ae009a6aa573..6bd13d4c69a1b 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 1ae009a6aa573629c8871ddb97dd6e1a9c03dd5f +Subproject commit 6bd13d4c69a1baef1f668e652c3ca26028970f12 diff --git a/contrib/zstd b/contrib/zstd index 437081852c396..5b3e9e8fd2779 160000 --- a/contrib/zstd +++ b/contrib/zstd @@ -1 +1 @@ -Subproject commit 437081852c396eeb61edb7d47044d8cad911885e +Subproject commit 5b3e9e8fd2779260101a07e6cc7b34fd8764617f diff --git a/tests/integration/test_backup_restore_new/test_cancel_backup.py b/tests/integration/test_backup_restore_new/test_cancel_backup.py index 5491cd22e4862..0bcf43ebee785 100644 --- a/tests/integration/test_backup_restore_new/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_new/test_cancel_backup.py @@ -30,7 +30,7 @@ with_rustfs=True, ) -CAS_POOL_PREFIXES = ["backup_restore_new/blobs/", "backup_restore_new/parts/"] +CAS_POOL_PREFIXES = ["backup_restore_new/blobs/"] @pytest.fixture(scope="module", autouse=True) From c410f4be346ee1c2ce54fdd41e26dd1f7db3aa2d Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 09:42:02 +0200 Subject: [PATCH 29/52] Revert accidental `contrib` submodule changes Co-Authored-By: Claude Opus 5.5 --- contrib/NuRaft | 2 +- contrib/StringZilla | 2 +- contrib/abseil-cpp | 2 +- contrib/aws-c-http | 2 +- contrib/azure | 2 +- contrib/boost | 2 +- contrib/chdig | 2 +- contrib/clickstack | 2 +- contrib/delta-kernel-rs | 2 +- contrib/fast_float | 2 +- contrib/google-cloud-cpp | 2 +- contrib/h3 | 2 +- contrib/jemalloc | 2 +- contrib/libarchive | 2 +- contrib/librdkafka | 2 +- contrib/libssh | 2 +- contrib/libxml2 | 2 +- contrib/llvm-project | 2 +- contrib/mariadb-connector-c | 2 +- contrib/mongo-c-driver | 2 +- contrib/msgpack-c | 2 +- contrib/nats-io | 2 +- contrib/openssl | 2 +- contrib/orc | 2 +- contrib/re2 | 2 +- contrib/replxx | 2 +- contrib/rust_vendor | 2 +- contrib/s2geometry | 2 +- contrib/silk | 2 +- contrib/simdjson | 2 +- contrib/sysroot | 2 +- contrib/usearch | 2 +- contrib/zstd | 2 +- 33 files changed, 33 insertions(+), 33 deletions(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index a0eda0069d2da..fff08ccaf1664 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit a0eda0069d2dac0852c453cbe7b0285eda218479 +Subproject commit fff08ccaf16646941d91ab0d26b94c76c17e44d1 diff --git a/contrib/StringZilla b/contrib/StringZilla index 15c292ec71db5..56c8d90869c87 160000 --- a/contrib/StringZilla +++ b/contrib/StringZilla @@ -1 +1 @@ -Subproject commit 15c292ec71db5c0e3d05f3bd693a6b5ec9429db8 +Subproject commit 56c8d90869c878a71dde292fa16872f661421643 diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 5650e9cf76d3b..255c84dadd029 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 5650e9cf76d3be4318d5fa3af38ee483ddfd5e4a +Subproject commit 255c84dadd029fd8ad25c5efb5933e47beaa00c7 diff --git a/contrib/aws-c-http b/contrib/aws-c-http index a9745ea9998f6..8aefd899fc321 160000 --- a/contrib/aws-c-http +++ b/contrib/aws-c-http @@ -1 +1 @@ -Subproject commit a9745ea9998f679cd7456e7d23cc8820e38c97d4 +Subproject commit 8aefd899fc3210bfd0e3fd414011a3cb708bf6e4 diff --git a/contrib/azure b/contrib/azure index 20f8b46b87871..e1c250fe5dd43 160000 --- a/contrib/azure +++ b/contrib/azure @@ -1 +1 @@ -Subproject commit 20f8b46b878718b4536a88888a182f60f665465a +Subproject commit e1c250fe5dd4306e3b7d19600788f4029c29786c diff --git a/contrib/boost b/contrib/boost index e24b1b25e3349..f05991ceab7d4 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit e24b1b25e3349236a3c5aac792e3294f62eda1f2 +Subproject commit f05991ceab7d486b82251679cb1d54816cde9df0 diff --git a/contrib/chdig b/contrib/chdig index 9302521212c89..417e74751cef1 160000 --- a/contrib/chdig +++ b/contrib/chdig @@ -1 +1 @@ -Subproject commit 9302521212c898fa7d91df827136e74f3a760bb7 +Subproject commit 417e74751cef15b1079ded06caee8c318163effd diff --git a/contrib/clickstack b/contrib/clickstack index f5232dc36db0f..4aa7f5ed0369f 160000 --- a/contrib/clickstack +++ b/contrib/clickstack @@ -1 +1 @@ -Subproject commit f5232dc36db0f151c6a19a770710fbeebfd9051b +Subproject commit 4aa7f5ed0369f46166d1fb56b1b8893f09d99793 diff --git a/contrib/delta-kernel-rs b/contrib/delta-kernel-rs index 88ff5a9474782..857565caf9ac6 160000 --- a/contrib/delta-kernel-rs +++ b/contrib/delta-kernel-rs @@ -1 +1 @@ -Subproject commit 88ff5a947478215d285783b42aafe6f3d04aca58 +Subproject commit 857565caf9ac61511c0e005ee3a32a30877a1a15 diff --git a/contrib/fast_float b/contrib/fast_float index 34164f547b7df..01ce95dfe46ab 160000 --- a/contrib/fast_float +++ b/contrib/fast_float @@ -1 +1 @@ -Subproject commit 34164f547b7df3f5d794ff67e9f885c36819ebfc +Subproject commit 01ce95dfe46abccf3264fbccf9f5139ea9016cd2 diff --git a/contrib/google-cloud-cpp b/contrib/google-cloud-cpp index 499787a71646e..83f30caadb861 160000 --- a/contrib/google-cloud-cpp +++ b/contrib/google-cloud-cpp @@ -1 +1 @@ -Subproject commit 499787a71646ee5f7d9a845aee79a1b0663c947d +Subproject commit 83f30caadb8613fb5c408d8c2fd545291596b53f diff --git a/contrib/h3 b/contrib/h3 index e38f58ef05128..70d4c9e5007d7 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit e38f58ef051280ebdd628d275a12cb667e8c4a3c +Subproject commit 70d4c9e5007d74d77714afa44d5a2e52606828bf diff --git a/contrib/jemalloc b/contrib/jemalloc index 5f864d8bd4c4f..e6cea775d3162 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit 5f864d8bd4c4fa7a99cb47c374d5c6465a160578 +Subproject commit e6cea775d316273b05581f6a7c3609be649eb754 diff --git a/contrib/libarchive b/contrib/libarchive index 27cbc78271726..7219b0134d771 160000 --- a/contrib/libarchive +++ b/contrib/libarchive @@ -1 +1 @@ -Subproject commit 27cbc7827172698143e440801fc0ba39ccb4f1f5 +Subproject commit 7219b0134d771dc4b51bf86b4d01761b87398b1b diff --git a/contrib/librdkafka b/contrib/librdkafka index a0f91df2d9eb9..6aa06cfec6ae3 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit a0f91df2d9eb989829ddc5d5358da48342b41d48 +Subproject commit 6aa06cfec6ae35efb6e6f895326ed0ec2ecc2a27 diff --git a/contrib/libssh b/contrib/libssh index c0083428b0c7f..50313883f3a07 160000 --- a/contrib/libssh +++ b/contrib/libssh @@ -1 +1 @@ -Subproject commit c0083428b0c7f97f7b6a9c212d630c49b72a2d7a +Subproject commit 50313883f3a077458cde4ea95bf46bfeb0771b34 diff --git a/contrib/libxml2 b/contrib/libxml2 index 96498992efa48..c94eb0210183b 160000 --- a/contrib/libxml2 +++ b/contrib/libxml2 @@ -1 +1 @@ -Subproject commit 96498992efa48d52b0e8b83058bd88dbdaf153c1 +Subproject commit c94eb0210183b9d7cb43f8e7fddc6be55843ef49 diff --git a/contrib/llvm-project b/contrib/llvm-project index c7f5fc7904a38..588b33887bd57 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit c7f5fc7904a38ed3d1f6999fc615ad666905d1df +Subproject commit 588b33887bd57ed2e5df6b15ea26d657e7ff0cef diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 111ec1a5958cf..95d6264bb43d3 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 111ec1a5958cf984f50a28b0fb82a4087918d677 +Subproject commit 95d6264bb43d31f825cf2f85ce1928557457b62d diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 6b6b676bdbd46..a26c8e251fec1 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 6b6b676bdbd46fdb954ed1535893020ef91cce5b +Subproject commit a26c8e251fec147258e89b7e85e9adfa259fc42f diff --git a/contrib/msgpack-c b/contrib/msgpack-c index 6b0d778ae059c..13a5894d9ffec 160000 --- a/contrib/msgpack-c +++ b/contrib/msgpack-c @@ -1 +1 @@ -Subproject commit 6b0d778ae059cfb77e26f2a011e99b17a8563c23 +Subproject commit 13a5894d9ffec2aeafab560d40a5d9e40d9c1cbe diff --git a/contrib/nats-io b/contrib/nats-io index 3e3a3f10572ed..cf441828d30fd 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit 3e3a3f10572ed9ff1b7ad17873881877cb62f301 +Subproject commit cf441828d30fdd5de12d9da319e88d2586fdeeba diff --git a/contrib/openssl b/contrib/openssl index 614167b874c61..f3b2f0273ed59 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit 614167b874c612973344b49f772079213b3606eb +Subproject commit f3b2f0273ed59cef874db0d6943f8d7d1804bd31 diff --git a/contrib/orc b/contrib/orc index 6a2fe65eb16ce..8ca5eeed390f5 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit 6a2fe65eb16ce4c760964dca619260226c1b418e +Subproject commit 8ca5eeed390f57fe6d1d4f7f8d97da19de6c08cb diff --git a/contrib/re2 b/contrib/re2 index 1f24555c97150..85dd7ad833a73 160000 --- a/contrib/re2 +++ b/contrib/re2 @@ -1 +1 @@ -Subproject commit 1f24555c97150767d65aa77fde8ddd029d2bb97c +Subproject commit 85dd7ad833a73095ecf3e3baea608ba051bbe2c7 diff --git a/contrib/replxx b/contrib/replxx index c2de583a3cd41..3a55a1bbee3c4 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit c2de583a3cd41f7b476cb625e954c191b6fcc448 +Subproject commit 3a55a1bbee3c4ccdbad38c0f1da4ce722e337807 diff --git a/contrib/rust_vendor b/contrib/rust_vendor index 8f91308de3d1b..e7b13e76a6f7f 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit 8f91308de3d1b78a1fc3256a2e64bd91744c3a11 +Subproject commit e7b13e76a6f7f9b6408d0695663539a89b010c4c diff --git a/contrib/s2geometry b/contrib/s2geometry index 5409e159ec6e0..a37aba69f14af 160000 --- a/contrib/s2geometry +++ b/contrib/s2geometry @@ -1 +1 @@ -Subproject commit 5409e159ec6e0334022395ed33ed24e60c69b6ae +Subproject commit a37aba69f14af676e9605cd9515c8aca6cef8342 diff --git a/contrib/silk b/contrib/silk index 78aab9d603d43..e7d5d331a284b 160000 --- a/contrib/silk +++ b/contrib/silk @@ -1 +1 @@ -Subproject commit 78aab9d603d43247fa8f05f86c34ecdfe2ab4095 +Subproject commit e7d5d331a284bd9d7608c41e1ce21303c5eeb72f diff --git a/contrib/simdjson b/contrib/simdjson index e95c118160780..0a851a64cd984 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit e95c118160780c2b808d86953938ce40b445d764 +Subproject commit 0a851a64cd984e9e1a6cab93b6e773aa3f4dc30d diff --git a/contrib/sysroot b/contrib/sysroot index caf9c9f560e5a..bed292b837319 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit caf9c9f560e5a06295d961860ed717fe169e8cef +Subproject commit bed292b8373193cd49005a617a0df741d9cbf52d diff --git a/contrib/usearch b/contrib/usearch index 6bd13d4c69a1b..1ae009a6aa573 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 6bd13d4c69a1baef1f668e652c3ca26028970f12 +Subproject commit 1ae009a6aa573629c8871ddb97dd6e1a9c03dd5f diff --git a/contrib/zstd b/contrib/zstd index 5b3e9e8fd2779..437081852c396 160000 --- a/contrib/zstd +++ b/contrib/zstd @@ -1 +1 @@ -Subproject commit 5b3e9e8fd2779260101a07e6cc7b34fd8764617f +Subproject commit 437081852c396eeb61edb7d47044d8cad911885e From a97c46205bc301bb98ebb067b27b8506ff7b109f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 9 Oct 2026 15:22:38 +0200 Subject: [PATCH 30/52] Do not treat missing Iceberg column metrics as NULL `allow_experimental_iceberg_read_optimization` replaced nullable columns absent from partial manifest statistics with constant NULL. Record every column from the file schema so only columns added after the file was written are skipped. Related: https://github.com/Altinity/ClickHouse/issues/2525 Co-authored-by: Cursor --- .../DataLakes/IDataLakeMetadata.cpp | 14 + ...ptimization_partial_column_stats.reference | 6 + ..._read_optimization_partial_column_stats.sh | 272 ++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.reference create mode 100755 tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.sh diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp index e553907b4a041..1cada25d35628 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp @@ -164,6 +164,20 @@ DataFileMetaInfo::DataFileMetaInfo( columns_info[i_name->second] = {column.second.rows_count, column.second.nulls_count, hyperrectangle}; } + + /// Missing metrics do not mean the column is absent from the file. Record every column of the + /// file schema so the read optimization substitutes NULL only for columns added after the file + /// was written. `emplace` keeps the statistics already stored above. + if (schema_processor.hasClickhouseTableSchemaById(file_schema_id)) + { + const auto fields = schema_processor.getIcebergTableSchemaById(file_schema_id)->getArray(Iceberg::f_fields); + for (size_t i = 0; i < fields->size(); ++i) + { + const auto field_id = fields->getObject(static_cast(i))->getValue(Iceberg::f_id); + if (const auto table_field = schema_processor.tryGetFieldCharacteristics(table_schema_id, field_id)) + columns_info.emplace(table_field->getNameInStorage(), ColumnInfo{}); + } + } #else (void)schema_processor; (void)table_schema_id; diff --git a/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.reference b/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.reference new file mode 100644 index 0000000000000..b74a145edd416 --- /dev/null +++ b/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.reference @@ -0,0 +1,6 @@ +1 alpha +2 beta +3 gamma +1 alpha +2 beta +3 gamma diff --git a/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.sh b/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.sh new file mode 100755 index 0000000000000..168bafe8cfbee --- /dev/null +++ b/tests/queries/0_stateless/05300_iceberg_read_optimization_partial_column_stats.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: needs pyarrow to write the Parquet data file. +# +# Reproduces https://github.com/Altinity/ClickHouse/issues/2525 +# Iceberg column metrics are optional per column. A manifest that has statistics for +# some columns and none for others (Databricks UniForm keeps metrics for the first 32 +# columns only) must still be read from the data file. With +# `allow_experimental_iceberg_read_optimization` enabled, a nullable column missing +# from those maps was treated as absent and replaced with constant NULL. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE_PATH="${CLICKHOUSE_USER_FILES}/lakehouses/${CLICKHOUSE_DATABASE}_partial_column_stats" +rm -rf "${TABLE_PATH}" + +python3 - "${TABLE_PATH}" <<'PY' +import json +import os +import sys +import time +import uuid + +import pyarrow as pa +import pyarrow.parquet as pq + + +def zigzag(n, bits): + return (n << 1) ^ (n >> (bits - 1)) + + +def encode_varint(n): + out = bytearray() + while True: + byte = n & 0x7F + n >>= 7 + if n: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def encode_long(n): + return encode_varint(zigzag(n, 64)) + + +def encode_int(n): + return encode_varint(zigzag(n, 32)) + + +def encode_bytes(data): + return encode_long(len(data)) + data + + +def encode_string(text): + return encode_bytes(text.encode("utf-8")) + + +def encode_array(items): + if not items: + return encode_long(0) + return encode_long(len(items)) + b"".join(items) + encode_long(0) + + +def encode_map(pairs): + body = b"".join(encode_string(key) + encode_bytes(value) for key, value in pairs) + return encode_long(len(pairs)) + body + encode_long(0) + + +def encode_union_long(value): + # ["null", "long"], branch 1 is the long. + return encode_int(1) + encode_long(value) + + +def write_avro(path, schema, records, metadata): + sync = os.urandom(16) + header = [("avro.schema", json.dumps(schema).encode("utf-8")), ("avro.codec", b"null")] + header.extend((key, value if isinstance(value, bytes) else value.encode("utf-8")) for key, value in metadata.items()) + payload = b"".join(records) + block = encode_long(len(records)) + encode_long(len(payload)) + payload + sync + with open(path, "wb") as fh: + fh.write(b"Obj\x01" + encode_map(header) + sync + block) + + +def kv_array(name): + return { + "type": "array", + "items": { + "type": "record", + "name": name, + "fields": [{"name": "key", "type": "int"}, {"name": "value", "type": "long"}], + }, + } + + +MANIFEST_SCHEMA = { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int"}, + {"name": "snapshot_id", "type": ["null", "long"]}, + {"name": "sequence_number", "type": ["null", "long"]}, + {"name": "file_sequence_number", "type": ["null", "long"]}, + { + "name": "data_file", + "type": { + "type": "record", + "name": "r2", + "fields": [ + {"name": "content", "type": "int"}, + {"name": "file_path", "type": "string"}, + {"name": "file_format", "type": "string"}, + {"name": "partition", "type": {"type": "record", "name": "r102", "fields": []}}, + {"name": "record_count", "type": "long"}, + {"name": "file_size_in_bytes", "type": "long"}, + {"name": "column_sizes", "type": kv_array("k_cs")}, + {"name": "value_counts", "type": kv_array("k_vc")}, + {"name": "null_value_counts", "type": kv_array("k_nc")}, + ], + }, + }, + ], +} + +MANIFEST_LIST_SCHEMA = { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string"}, + {"name": "manifest_length", "type": "long"}, + {"name": "partition_spec_id", "type": "int"}, + {"name": "content", "type": "int"}, + {"name": "sequence_number", "type": "long"}, + {"name": "min_sequence_number", "type": "long"}, + {"name": "added_snapshot_id", "type": "long"}, + {"name": "added_files_count", "type": "int"}, + {"name": "existing_files_count", "type": "int"}, + {"name": "deleted_files_count", "type": "int"}, + {"name": "added_rows_count", "type": "long"}, + {"name": "existing_rows_count", "type": "long"}, + {"name": "deleted_rows_count", "type": "long"}, + ], +} + +root = sys.argv[1] +data_dir = os.path.join(root, "data") +meta_dir = os.path.join(root, "metadata") +os.makedirs(data_dir) +os.makedirs(meta_dir) + +data_path = os.path.join(data_dir, "00000-0-data.parquet") +arrow_schema = pa.schema([ + pa.field("id", pa.int32(), nullable=True, metadata={b"PARQUET:field_id": b"1"}), + pa.field("extra", pa.string(), nullable=True, metadata={b"PARQUET:field_id": b"2"}), +]) +pq.write_table( + pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int32()), + "extra": pa.array(["alpha", "beta", "gamma"], type=pa.string()), + }, + schema=arrow_schema, + ), + data_path, +) + +iceberg_schema = { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": False, "type": "int"}, + {"id": 2, "name": "extra", "required": False, "type": "string"}, + ], +} + +# Stats exist, but only for field id 1 (`id`). `extra` is in the file and in the +# table schema, and has no manifest metrics. Empty maps still count as "some stats". +entry = b"".join([ + encode_int(1), + encode_union_long(1), + encode_union_long(1), + encode_union_long(1), + encode_int(0), + encode_string(data_path), + encode_string("PARQUET"), + b"", + encode_long(3), + encode_long(os.path.getsize(data_path)), + encode_array([]), + encode_array([]), + encode_array([encode_int(1) + encode_long(0)]), +]) + +manifest_path = os.path.join(meta_dir, "00000-0-manifest.avro") +write_avro( + manifest_path, + MANIFEST_SCHEMA, + [entry], + {"schema": json.dumps(iceberg_schema), "partition-spec": "[]", "format-version": "2"}, +) + +mlist_path = os.path.join(meta_dir, "snap-1-0-manifest-list.avro") +mlist_entry = b"".join([ + encode_string(manifest_path), + encode_long(os.path.getsize(manifest_path)), + encode_int(0), + encode_int(0), + encode_long(1), + encode_long(1), + encode_long(1), + encode_int(1), + encode_int(0), + encode_int(0), + encode_long(3), + encode_long(0), + encode_long(0), +]) +write_avro(mlist_path, MANIFEST_LIST_SCHEMA, [mlist_entry], {"format-version": "2"}) + +now_ms = int(time.time() * 1000) +metadata = { + "format-version": 2, + "table-uuid": str(uuid.uuid4()), + "location": root, + "last-sequence-number": 1, + "last-updated-ms": now_ms, + "last-column-id": 2, + "current-schema-id": 0, + "schemas": [iceberg_schema], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "current-snapshot-id": 1, + "snapshots": [{ + "snapshot-id": 1, + "sequence-number": 1, + "timestamp-ms": now_ms, + "manifest-list": mlist_path, + "summary": {"operation": "append", "added-records": "3", "total-records": "3"}, + "schema-id": 0, + }], + "snapshot-log": [{"timestamp-ms": now_ms, "snapshot-id": 1}], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 1, "type": "branch"}}, +} +with open(os.path.join(meta_dir, "v1.metadata.json"), "w") as fh: + json.dump(metadata, fh) +with open(os.path.join(meta_dir, "version-hint.text"), "w") as fh: + fh.write("1\n") +PY + +${CLICKHOUSE_CLIENT} --query " + SELECT id, extra + FROM icebergLocal('${TABLE_PATH}') + ORDER BY id + SETTINGS allow_experimental_iceberg_read_optimization = 1 +" + +${CLICKHOUSE_CLIENT} --query " + SELECT id, extra + FROM icebergLocal('${TABLE_PATH}') + ORDER BY id + SETTINGS allow_experimental_iceberg_read_optimization = 0 +" + +rm -rf "${TABLE_PATH}" From e8fc6425da002e7e78b228002682169b90abe6c3 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:57:43 -0400 Subject: [PATCH 31/52] upgrade all packages when building alpine images --- docker/keeper/Dockerfile | 4 +++- docker/server/Dockerfile.alpine | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 59efba0f5998a..6d26ef8bb3c33 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -16,7 +16,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac -FROM alpine:3.21 +FROM alpine ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ @@ -89,6 +89,8 @@ RUN clickhouse-keeper --version \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +RUN apk update && apk upgrade --no-cache + # /var/lib/clickhouse is necessary due to the current default configuration for Keeper VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 3fea5228c1b5e..d698053de9ab4 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -107,8 +107,7 @@ RUN mkdir -p \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" -RUN apk update -RUN apk add --upgrade libssl3 libcrypto3 +RUN apk update && apk upgrade --no-cache VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 From 92a9e65835f7a28145b23d9265404a14452bff16 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 9 Oct 2026 17:02:35 +0200 Subject: [PATCH 32/52] Log how many Iceberg columns in a file have no metrics Partial manifest statistics are read from the data file. A debug message names that file and reports only the count of columns without metrics. Co-authored-by: Cursor --- .../ObjectStorage/DataLakes/IDataLakeMetadata.cpp | 12 ++++++++++-- .../ObjectStorage/DataLakes/IDataLakeMetadata.h | 3 ++- .../DataLakes/Iceberg/IcebergIterator.cpp | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp index 1cada25d35628..e642bfca2bcea 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp @@ -104,7 +104,8 @@ DataFileMetaInfo::DataFileMetaInfo( Int32 table_schema_id, Int32 file_schema_id, const std::unordered_map & columns_info_, - const std::unordered_map> & value_bounds_) + const std::unordered_map> & value_bounds_, + const String & file_path) { #if USE_AVRO std::vector column_ids; @@ -168,6 +169,7 @@ DataFileMetaInfo::DataFileMetaInfo( /// Missing metrics do not mean the column is absent from the file. Record every column of the /// file schema so the read optimization substitutes NULL only for columns added after the file /// was written. `emplace` keeps the statistics already stored above. + size_t columns_without_metrics = 0; if (schema_processor.hasClickhouseTableSchemaById(file_schema_id)) { const auto fields = schema_processor.getIcebergTableSchemaById(file_schema_id)->getArray(Iceberg::f_fields); @@ -175,15 +177,21 @@ DataFileMetaInfo::DataFileMetaInfo( { const auto field_id = fields->getObject(static_cast(i))->getValue(Iceberg::f_id); if (const auto table_field = schema_processor.tryGetFieldCharacteristics(table_schema_id, field_id)) - columns_info.emplace(table_field->getNameInStorage(), ColumnInfo{}); + { + if (columns_info.emplace(table_field->getNameInStorage(), ColumnInfo{}).second) + ++columns_without_metrics; + } } } + if (columns_without_metrics) + LOG_DEBUG(getLogger("DataFileMetaInfo"), "File {} has {} columns without Iceberg metrics", file_path, columns_without_metrics); #else (void)schema_processor; (void)table_schema_id; (void)file_schema_id; (void)columns_info_; (void)value_bounds_; + (void)file_path; #endif } diff --git a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h index 43ee6e5c4c535..f2c21c9dc8cf1 100644 --- a/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h @@ -75,7 +75,8 @@ class DataFileMetaInfo Int32 table_schema_id, Int32 file_schema_id, const std::unordered_map & columns_info_, - const std::unordered_map> & value_bounds_); + const std::unordered_map> & value_bounds_, + const String & file_path); void serialize(WriteBuffer & out) const; static DataFileMetaInfo deserialize(ReadBuffer & in); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 1fd5ceed10d6a..a3aecf79d7343 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -531,7 +531,8 @@ ObjectInfoPtr IcebergIterator::next(size_t) table_schema_id, /// current schema id to use current column names manifest_file_entry->resolved_schema_id, /// file's schema id to interpret value_bounds bytes manifest_file_entry->parsed_entry->columns_infos, - manifest_file_entry->parsed_entry->value_bounds)); + manifest_file_entry->parsed_entry->value_bounds, + data_file_path.serialize())); ProfileEvents::increment(ProfileEvents::IcebergMetadataReturnedObjectInfos); From 26d3a68f3e62bd9e55af0a6e243966e023efeaff Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 9 Oct 2026 17:43:48 +0200 Subject: [PATCH 33/52] Drop local server-constant GROUP BY keys in the old analyzer A local query has one server, so a folded server constant is an ordinary constant. Keeping it made GROUP BY of `hostName` cast to `Dynamic` throw `ILLEGAL_COLUMN`. Secondary queries still keep the key, because cluster table function workers also have `distributed_depth` 0. https: //github.com/Altinity/ClickHouse/pull/2490 Co-authored-by: Cursor --- src/Interpreters/ExpressionAnalyzer.cpp | 17 +++++++++++------ ...ver_constant_group_by_old_analyzer.reference | 3 +++ ...al_server_constant_group_by_old_analyzer.sql | 9 +++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.reference create mode 100644 tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 898b767eaf4a3..eaa673972a149 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -152,8 +152,8 @@ bool allowEarlyConstantFolding(const ActionsDAG & actions, const Settings & sett return true; } -/// A folded server constant (`hostName()`, `tcpPort()`, ...) must stay a GROUP BY key. -/// Its value differs per shard. Dropping it makes the initiator look up a missing column. +/// True when `node` folds a server constant such as `hostName` or `tcpPort`. +/// `__getScalar` is excluded: the initiator evaluates it and sends one value to every shard. bool containsServerConstant(const ActionsDAG::Node * node) { std::stack stack; @@ -357,6 +357,9 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) else group_by_kind = GroupByKind::ORDINARY; bool use_nulls = group_by_kind != GroupByKind::ORDINARY && getContext()->getSettingsRef()[Setting::group_by_use_nulls]; + /// Cluster table function workers leave `distributed_depth` at 0, same as a local query. + /// `SECONDARY_QUERY` is what distinguishes them: a server constant differs per shard only there. + const bool is_secondary_query = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column /// With set number, which is used as an additional key at the stage of merging aggregating data. @@ -393,8 +396,9 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { /// Constant expressions have non-null column pointer at this stage. - /// Server constants stay: they are different on each shard. - if (node->column && !containsServerConstant(node)) + /// A server constant stays only on a secondary query, where each shard has its own value. + /// A local query has one value, so the key is removed with other constants. + if (node->column && !(is_secondary_query && containsServerConstant(node))) { select_query->group_by_with_constant_keys = true; @@ -448,8 +452,9 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) if (getContext()->getClientInfo().distributed_depth == 0 || settings.distributed_group_by_no_merge > 0) { /// Constant expressions have non-null column pointer at this stage. - /// Server constants stay: they are different on each shard. - if (node->column && !containsServerConstant(node)) + /// A server constant stays only on a secondary query, where each shard has its own value. + /// A local query has one value, so the key is removed with other constants. + if (node->column && !(is_secondary_query && containsServerConstant(node))) { select_query->group_by_with_constant_keys = true; diff --git a/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.reference b/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.reference new file mode 100644 index 0000000000000..1f242fa6f0004 --- /dev/null +++ b/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.reference @@ -0,0 +1,3 @@ +3 +3 +3 diff --git a/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.sql b/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.sql new file mode 100644 index 0000000000000..8109ac535241d --- /dev/null +++ b/tests/queries/0_stateless/05298_local_server_constant_group_by_old_analyzer.sql @@ -0,0 +1,9 @@ +-- A local query has one server, so a folded server constant is an ordinary GROUP BY constant. +-- Keeping it makes `Dynamic` fail group-key validation. +-- https://github.com/Altinity/ClickHouse/pull/2490 + +SELECT count() FROM numbers(3) GROUP BY hostName() SETTINGS enable_analyzer = 0; + +SELECT count() FROM numbers(3) GROUP BY CAST(hostName(), 'Dynamic') SETTINGS enable_analyzer = 0; + +SELECT count() FROM numbers(3) GROUP BY [CAST(hostName(), 'Dynamic')] SETTINGS enable_analyzer = 0; From c2ef6adbc1174ff0eeba0289d1a12cf757f73d1d Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 9 Oct 2026 11:54:50 -0400 Subject: [PATCH 34/52] Add workflow to switch active bucket --- .github/workflows/deploy-static.yml | 26 +++++------ .../packaging/ansible/deploy-static.yml | 4 ++ .../roles/get_cloudfront_info/tasks/main.yml | 4 +- .../ansible/roles/publish_pkgs/tasks/main.yml | 45 +++++++++---------- 4 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 tests/ci/release/packaging/ansible/deploy-static.yml diff --git a/.github/workflows/deploy-static.yml b/.github/workflows/deploy-static.yml index 8fae7bdc4c8fb..0e52912f01384 100644 --- a/.github/workflows/deploy-static.yml +++ b/.github/workflows/deploy-static.yml @@ -7,14 +7,11 @@ on: description: 'The environment to deploy to. "staging" or "production"' required: true default: 'staging' - keep_current_bucket: + push_to_active: description: >- - Deploy onto whichever Blue/Green bucket is currently live, instead of - the normal release toggle. This workflow never touches packages, so - the other bucket is likely several releases stale - toggling to it - would make those stale packages live again. Leave this true unless - you specifically want to stage static changes on the inactive side - first. + If true, deploy static assets onto the currently live Blue/Green + prefix and do not rotate CloudFront. If false, deploy static assets + onto the inactive prefix and rotate CloudFront to it. required: true type: boolean default: true @@ -24,7 +21,10 @@ env: jobs: deploy-static: - runs-on: [altinity-style-checker-aarch64, altinity-on-demand] + runs-on: ${{ + inputs.release_environment == 'staging' && 'arc-runners-clickhouse-signer' || + inputs.release_environment == 'production' && 'arc-runners-clickhouse-signer-prod' + }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -55,15 +55,15 @@ jobs: - name: Run static deploy playbook run: | - echo "Deploying static site assets (installer.sh, landing page, CloudFront routing) to ${{ inputs.release_environment }}..." - echo "Keep current bucket (skip Blue/Green toggle): ${{ inputs.keep_current_bucket }}" - if ! ansible-playbook -i release/packaging/ansible/inventory/localhost.yml \ + echo "Deploying static site assets to ${{ inputs.release_environment }}..." + echo "push_to_active=${{ inputs.push_to_active }} (true=stay on live prefix, false=rotate Blue/Green)" + if ! ansible-playbook -i tests/ci/release/packaging/ansible/inventory/localhost.yml \ -e aws_region="$AWS_REGION" \ -e repo_dns_name="$REPO_DNS_NAME" \ -e repo_name="$REPO_NAME" \ -e repo_subtitle="$REPO_SUBTITLE" \ - -e push_to_active=${{ inputs.keep_current_bucket }} \ - release/packaging/ansible/deploy-static.yml; then + -e push_to_active=${{ inputs.push_to_active }} \ + tests/ci/release/packaging/ansible/deploy-static.yml; then echo "Ansible playbook failed" exit 1 fi diff --git a/tests/ci/release/packaging/ansible/deploy-static.yml b/tests/ci/release/packaging/ansible/deploy-static.yml new file mode 100644 index 0000000000000..cf2d39fa5903c --- /dev/null +++ b/tests/ci/release/packaging/ansible/deploy-static.yml @@ -0,0 +1,4 @@ +- hosts: localhost + roles: + - get_cloudfront_info + - publish_pkgs diff --git a/tests/ci/release/packaging/ansible/roles/get_cloudfront_info/tasks/main.yml b/tests/ci/release/packaging/ansible/roles/get_cloudfront_info/tasks/main.yml index b65d0cdad3855..d3886218d3b3a 100644 --- a/tests/ci/release/packaging/ansible/roles/get_cloudfront_info/tasks/main.yml +++ b/tests/ci/release/packaging/ansible/roles/get_cloudfront_info/tasks/main.yml @@ -15,12 +15,12 @@ - name: Decide on the target (blue/green) set_fact: new_cloudfront_origin_path: "{{ '/Green' if cloudfront_origin_path == '/Blue' else '/Blue' }}" - when: 'not (push_to_active|default(false))' + when: 'not (push_to_active|default(false)|bool)' - name: Update the active site, when push_to_active is defined set_fact: new_cloudfront_origin_path: "{{ cloudfront_origin_path }}" - when: 'push_to_active|default(false)' + when: 'push_to_active|default(false)|bool' - name: Calculate the target S3 path set_fact: diff --git a/tests/ci/release/packaging/ansible/roles/publish_pkgs/tasks/main.yml b/tests/ci/release/packaging/ansible/roles/publish_pkgs/tasks/main.yml index 627a77d627435..9c018a5edf77b 100644 --- a/tests/ci/release/packaging/ansible/roles/publish_pkgs/tasks/main.yml +++ b/tests/ci/release/packaging/ansible/roles/publish_pkgs/tasks/main.yml @@ -62,29 +62,26 @@ loop_control: label: "{{ item.path }}" -- name: Update CloudFront distribution - community.aws.cloudfront_distribution: - region: "{{ aws_region }}" - state: present - caller_reference: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.CallerReference }}" - origins: - - id: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Origins.Items[0].Id }}" - domain_name: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Origins.Items[0].DomainName }}" - origin_path: "{{ new_cloudfront_origin_path | default(omit) }}" - custom_origin_config: - origin_protocol_policy: "http-only" - aliases: - - "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Aliases.Items[0] }}" - viewer_certificate: - acm_certificate_arn: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.ViewerCertificate.ACMCertificateArn }}" - ssl_support_method: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.ViewerCertificate.SSLSupportMethod }}" - logging: - bucket: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Logging.Bucket }}" - enabled: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Logging.Enabled }}" - include_cookies: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Logging.IncludeCookies }}" - prefix: "{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.DistributionConfig.Logging.Prefix }}" - when: 'not (push_to_active|default(false))' - register: distribution +- name: Update CloudFront origin path + when: 'not (push_to_active|default(false)|bool)' + shell: | + set -euo pipefail + DIST_ID="{{ cloudfront_info.cloudfront[repo_dns_name].Distribution.Id }}" + ORIGIN_PATH="{{ new_cloudfront_origin_path }}" + CONFIG_JSON=/tmp/cf-origin-path-config.json + NEW_CONFIG_JSON=/tmp/cf-origin-path-new-config.json + trap 'rm -f "$CONFIG_JSON" "$NEW_CONFIG_JSON"' EXIT + aws cloudfront get-distribution-config --id "$DIST_ID" > "$CONFIG_JSON" + DIST_ETAG=$(jq -r '.ETag' "$CONFIG_JSON") + jq --arg path "$ORIGIN_PATH" \ + '.DistributionConfig | .Origins.Items[0].OriginPath = $path' \ + "$CONFIG_JSON" > "$NEW_CONFIG_JSON" + aws cloudfront update-distribution \ + --id "$DIST_ID" \ + --if-match "$DIST_ETAG" \ + --distribution-config "file://$NEW_CONFIG_JSON" + args: + executable: /bin/bash - name: Invalidate CloudFront cache community.aws.cloudfront_invalidation: @@ -94,5 +91,5 @@ register: invalidation - name: Wait for CloudFront invalidation to be complete - shell: "aws cloudfront wait invalidation-completed --distribution-id {{ distribution.id }} --id {{ invalidation.invalidation.id }}" + shell: "aws cloudfront wait invalidation-completed --distribution-id {{ cloudfront_info.cloudfront[repo_dns_name].Distribution.Id }} --id {{ invalidation.invalidation.id }}" From 9aad315f609a92a0b6121772e8ab113be07d1449 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 13:41:16 +0200 Subject: [PATCH 35/52] Fix cluster reads of Iceberg tables reached through Alias. DataLakeCatalog builds a fresh storage on every lookup, so object_storage_cluster file listing ran before the snapshot was loaded and threw "Metadata is not initialized". Co-authored-by: Cursor --- .../StorageObjectStorageSource.cpp | 2 + .../integration/test_database_iceberg/test.py | 62 +++++++ ...01_alias_iceberg_catalog_cluster.reference | 24 +++ .../05301_alias_iceberg_catalog_cluster.sh | 174 ++++++++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference create mode 100755 tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index ab23503dfc413..f86d0b4b2757b 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -411,6 +411,8 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( if (configuration->isDataLakeConfiguration() && (!storage_metadata || !storage_metadata->datalake_table_state.has_value())) { + configuration->lazyInitializeIfNeeded(object_storage, local_context); + if (auto state = configuration->getTableStateSnapshot(local_context)) { auto fixed_metadata = storage_metadata diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 5270a2b5a3bd1..c428f8e2bd6b8 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3088,3 +3088,65 @@ def test_catalog_listing_error_surfaces_in_system_tables(started_cluster): assert "table_x" in result node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") + + +def test_alias_to_datalake_catalog_object_storage_cluster(started_cluster): + """ + A DataLakeCatalog lookup builds a fresh storage. Query analysis initializes one + copy, then an Alias read looks the table up again. Cluster planning lists files + from that second copy in StorageObjectStorageCluster::getTaskIteratorExtension. + + Direct catalog reads and count() through the alias succeed. A distributed scan + through the alias must return the same rows. + """ + node = started_cluster.instances["node1"] + + test_ref = f"test_alias_osc_{uuid.uuid4().hex[:8]}" + namespace = f"{test_ref}_ns" + table_name = f"{test_ref}_t" + catalog_table = f"{CATALOG_NAME}.`{namespace}.{table_name}`" + alias_name = f"{test_ref}_alias" + table_settings = { + "allow_database_iceberg": 1, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + } + cluster_settings = { + **table_settings, + "object_storage_cluster": "cluster_simple", + } + expected = "1\n2\n3\n" + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + create_clickhouse_iceberg_table( + started_cluster, node, namespace, table_name, "(id Int64)" + ) + node.query(f"INSERT INTO {catalog_table} VALUES (1), (2), (3);", settings=table_settings) + + node.query(f"DROP TABLE IF EXISTS {alias_name}") + node.query( + f"CREATE TABLE {alias_name} ENGINE = Alias('{CATALOG_NAME}', '{namespace}.{table_name}')" + ) + + try: + assert node.query( + f"SELECT id FROM {catalog_table} ORDER BY id", + settings=cluster_settings, + ) == expected + + assert node.query( + f"SELECT id FROM {alias_name} ORDER BY id", + settings=table_settings, + ) == expected + + assert node.query( + f"SELECT count() FROM {alias_name}", + settings=cluster_settings, + ) == "3\n" + + assert node.query( + f"SELECT id FROM {alias_name} WHERE id > 0 ORDER BY id", + settings=cluster_settings, + ) == expected + finally: + node.query(f"DROP TABLE IF EXISTS {alias_name}") diff --git a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference new file mode 100644 index 0000000000000..70c90fda37ee0 --- /dev/null +++ b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference @@ -0,0 +1,24 @@ +local-direct-filtered +67 2026-10-07 06:36:00.000000 +local-alias-filtered +67 2026-10-07 06:36:00.000000 +local-alias-limit1 +2026-10-06 14:00:00.000000 s0 +local-alias-trivcount +1000 +osc-direct-filtered +67 2026-10-07 06:36:00.000000 +osc-alias-filtered +67 2026-10-07 06:36:00.000000 +osc-alias-limit1 +2026-10-06 14:00:00.000000 s0 +osc-alias-trivcount +1000 +pr-direct-filtered +67 2026-10-07 06:36:00.000000 +pr-alias-filtered +67 2026-10-07 06:36:00.000000 +pr-alias-limit1 +2026-10-06 14:00:00.000000 s0 +pr-alias-trivcount +1000 diff --git a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh new file mode 100755 index 0000000000000..cb272ca8b49b9 --- /dev/null +++ b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-replicated-database: IcebergLocal is non-replicated. +# +# Customer repro: an Alias over a DataLakeCatalog Iceberg table. +# The catalog is a local REST stub in front of an IcebergLocal table. +# A filtered scan and a LIMIT through the alias must match a direct catalog read +# with no cluster, with object_storage_cluster, and with parallel replicas. +# count() through the alias is the control that already succeeded before the fix. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -euo pipefail + +WH="${CLICKHOUSE_USER_FILES_UNIQUE}/wh" +TABLE_PATH="${WH}/pnl/pnl_t" +ICE_DB="${CLICKHOUSE_DATABASE}_ice" +PNL_DB="${CLICKHOUSE_DATABASE}_pnl" +STUB_PY="${CLICKHOUSE_TMP}/alias_iceberg_stub_${CLICKHOUSE_TEST_UNIQUE_NAME}.py" +STUB_LOG="${CLICKHOUSE_TMP}/alias_iceberg_stub_${CLICKHOUSE_TEST_UNIQUE_NAME}.log" +STUB_PID="" + +STUB_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + +cleanup() { + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PNL_DB}.pnl_t" >/dev/null 2>&1 || true + ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${PNL_DB}" >/dev/null 2>&1 || true + ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${ICE_DB}" >/dev/null 2>&1 || true + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS src" >/dev/null 2>&1 || true + if [[ -n "${STUB_PID}" ]]; then + kill "${STUB_PID}" >/dev/null 2>&1 || true + wait "${STUB_PID}" >/dev/null 2>&1 || true + fi + rm -rf "${WH}" "${STUB_PY}" "${STUB_LOG}" +} +trap cleanup EXIT + +mkdir -p "${WH}/pnl" "$(dirname "${STUB_PY}")" + +cat > "${STUB_PY}" <<'PY' +import glob +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +WH = sys.argv[1] +PORT = int(sys.argv[2]) + + +def tables(ns): + return sorted( + os.path.basename(p) + for p in glob.glob(os.path.join(WH, ns, "*")) + if os.path.isdir(os.path.join(p, "metadata")) + ) + + +def newest_meta(ns, t): + files = glob.glob(os.path.join(WH, ns, t, "metadata", "*.metadata.json")) + files.sort(key=os.path.getmtime) + return files[-1] + + +class H(BaseHTTPRequestHandler): + def _send(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + path = self.path.split("?")[0].rstrip("/") + parts = path.split("/")[2:] if path.startswith("/v1") else None + if parts is None: + return self._send(404, {"error": {"message": "not found", "type": "NotFound", "code": 404}}) + if parts == ["config"]: + return self._send(200, {"defaults": {}, "overrides": {}}) + if parts == ["namespaces"]: + if "parent=" in self.path: + return self._send(200, {"namespaces": []}) + nss = sorted(os.path.basename(p) for p in glob.glob(os.path.join(WH, "*")) if os.path.isdir(p)) + return self._send(200, {"namespaces": [[n] for n in nss]}) + if len(parts) == 2 and parts[0] == "namespaces": + return self._send(200, {"namespace": [parts[1]], "properties": {}}) + if len(parts) == 3 and parts[0] == "namespaces" and parts[2] == "tables": + return self._send(200, {"identifiers": [{"namespace": [parts[1]], "name": t} for t in tables(parts[1])]}) + if len(parts) == 4 and parts[0] == "namespaces" and parts[2] == "tables": + ns, t = parts[1], parts[3] + if t not in tables(ns): + return self._send(404, {"error": {"message": "no table", "type": "NoSuchTableException", "code": 404}}) + m = newest_meta(ns, t) + with open(m) as f: + meta = json.load(f) + if "://" not in meta.get("location", ""): + meta["location"] = "file://" + meta["location"] + return self._send(200, {"metadata-location": "file://" + m, "metadata": meta, "config": {}}) + return self._send(404, {"error": {"message": "not found", "type": "NotFound", "code": 404}}) + + def log_message(self, *a): + pass + + +ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever() +PY + +python3 "${STUB_PY}" "${WH}" "${STUB_PORT}" >"${STUB_LOG}" 2>&1 & +STUB_PID=$! + +for _ in $(seq 1 50); do + if python3 -c "import socket; socket.create_connection(('127.0.0.1', ${STUB_PORT}), 0.2).close()" 2>/dev/null; then + break + fi + sleep 0.1 +done + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg 1 --query " + CREATE TABLE src ( + TimeSlice DateTime64(6, 'UTC'), + TradeDate Date32, + Strategy String, + NetPL Float64 + ) ENGINE = IcebergLocal('${TABLE_PATH}') + PARTITION BY TradeDate +" + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg 1 --query " + INSERT INTO src + SELECT + toDateTime64('2026-10-06 14:00:00', 6, 'UTC') + number * 60, + toDate32('2026-10-06') + (number % 3), + 's' || toString(number % 5), + number / 7 + FROM numbers(1000) +" + +${CLICKHOUSE_CLIENT} --allow_database_iceberg 1 --query " + CREATE DATABASE ${ICE_DB} ENGINE = DataLakeCatalog('http://127.0.0.1:${STUB_PORT}/v1') + SETTINGS + catalog_type = 'rest', + auth_header = 'Authorization: Bearer sandboxonlytesttoken0001', + warehouse = 'warehouse', + vended_credentials = false +" + +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${PNL_DB}" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${PNL_DB}.pnl_t ENGINE = Alias(${ICE_DB}.\`pnl.pnl_t\`)" + +FILT="WHERE TradeDate = '2026-10-06' AND Strategy = 's1'" +OSC="SETTINGS object_storage_cluster = 'test_shard_localhost'" +PR="SETTINGS enable_parallel_replicas = 1, cluster_for_parallel_replicas = 'test_shard_localhost', parallel_replicas_for_cluster_engines = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1" + +run() { + local tag="$1" + local sql="$2" + echo "${tag}" + ${CLICKHOUSE_CLIENT} --query "${sql}" +} + +for mode in local osc pr; do + case "${mode}" in + local) S="" ;; + osc) S="${OSC}" ;; + pr) S="${PR}" ;; + esac + run "${mode}-direct-filtered" "SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM ${ICE_DB}.\`pnl.pnl_t\` ${FILT} ${S}" + run "${mode}-alias-filtered" "SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM ${PNL_DB}.pnl_t ${FILT} ${S}" + run "${mode}-alias-limit1" "SELECT toTimeZone(TimeSlice, 'UTC'), Strategy FROM ${PNL_DB}.pnl_t ORDER BY TimeSlice LIMIT 1 ${S}" + run "${mode}-alias-trivcount" "SELECT count() FROM ${PNL_DB}.pnl_t ${S}" +done From 6b711b230222edce9b2db66f6bd3d5a8cc9c0b24 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 15:38:11 +0200 Subject: [PATCH 36/52] Cover catalog wrappers that re-read Iceberg tables with object_storage_cluster. Alias, Buffer, Merge, materialized views, loop, and Distributed all resolve a fresh catalog storage, so each one is checked the same way for local, query, and database cluster settings. Co-authored-by: Cursor --- .../integration/test_database_iceberg/test.py | 125 +++++++++++++----- 1 file changed, 93 insertions(+), 32 deletions(-) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index c428f8e2bd6b8..a46b335ce85f2 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3090,63 +3090,124 @@ def test_catalog_listing_error_surfaces_in_system_tables(started_cluster): node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") -def test_alias_to_datalake_catalog_object_storage_cluster(started_cluster): +def _create_catalog_wrapper(node, engine, name, catalog_db, namespace, table_name, alias_name): + catalog_table = f"{catalog_db}.`{namespace}.{table_name}`" + qualified_name = f"{namespace}.{table_name}" + if engine == "Alias": + node.query(f"CREATE TABLE {name} ENGINE = Alias('{catalog_db}', '{qualified_name}')") + elif engine == "Buffer": + node.query( + f"CREATE TABLE {name} (id Int64) ENGINE = Buffer('{catalog_db}', '{qualified_name}', " + "1, 86400, 86400, 1000000, 1000000, 1000000000, 1000000000)" + ) + elif engine == "Merge": + node.query( + f"CREATE TABLE {name} (id Int64) ENGINE = Merge('{catalog_db}', '^{namespace}[.]{table_name}$')" + ) + elif engine == "MergeAlias": + node.query(f"CREATE TABLE {name} (id Int64) ENGINE = Merge(currentDatabase(), '^{alias_name}$')") + elif engine == "MaterializedView": + node.query(f"CREATE MATERIALIZED VIEW {name} TO {catalog_table} AS SELECT id FROM {alias_name}") + elif engine == "Distributed": + node.query( + f"CREATE TABLE {name} (id Int64) " + f"ENGINE = Distributed('cluster_simple', currentDatabase(), '{alias_name}')" + ) + else: + raise AssertionError(f"Unexpected engine {engine}") + + +@pytest.mark.parametrize( + "engine", + ["Alias", "Buffer", "Merge", "MergeAlias", "MaterializedView", "Loop", "Distributed"], +) +@pytest.mark.parametrize("cluster_mode", ["local", "query", "database"]) +def test_datalake_catalog_wrapper_object_storage_cluster(started_cluster, engine, cluster_mode): """ - A DataLakeCatalog lookup builds a fresh storage. Query analysis initializes one - copy, then an Alias read looks the table up again. Cluster planning lists files - from that second copy in StorageObjectStorageCluster::getTaskIteratorExtension. + A DataLakeCatalog lookup builds a fresh storage and leaves its configuration + uninitialized. Each engine below resolves that table again, then lists files. - Direct catalog reads and count() through the alias succeed. A distributed scan - through the alias must return the same rows. + cluster_mode chooses where object_storage_cluster comes from: + local keeps it unset, query sets it on the scan, database sets it on the catalog. """ + if engine == "Loop" and cluster_mode != "local": + pytest.skip("loop has no cluster table function") + node = started_cluster.instances["node1"] + node2 = started_cluster.instances["node2"] - test_ref = f"test_alias_osc_{uuid.uuid4().hex[:8]}" + test_ref = f"test_wrapper_osc_{uuid.uuid4().hex[:8]}" namespace = f"{test_ref}_ns" table_name = f"{test_ref}_t" catalog_table = f"{CATALOG_NAME}.`{namespace}.{table_name}`" + cluster_catalog_name = f"{test_ref}_catalog_cluster" + catalog_db = cluster_catalog_name if cluster_mode == "database" else CATALOG_NAME alias_name = f"{test_ref}_alias" + wrapper_name = alias_name if engine == "Alias" else f"{test_ref}_wrapper" table_settings = { "allow_database_iceberg": 1, "allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1, } - cluster_settings = { + settings = table_settings if cluster_mode != "query" else { **table_settings, "object_storage_cluster": "cluster_simple", } - expected = "1\n2\n3\n" + # A clustered Buffer destination reports WithMergeableState, so Buffer plans + # the in-memory branch as a second read of the same query. + distinct = "DISTINCT " if engine == "Buffer" and cluster_mode != "local" else "" + nodes = [node, node2] if engine == "Distributed" else [node] + created = {replica: [] for replica in nodes} create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) create_clickhouse_iceberg_table( started_cluster, node, namespace, table_name, "(id Int64)" ) node.query(f"INSERT INTO {catalog_table} VALUES (1), (2), (3);", settings=table_settings) - - node.query(f"DROP TABLE IF EXISTS {alias_name}") - node.query( - f"CREATE TABLE {alias_name} ENGINE = Alias('{CATALOG_NAME}', '{namespace}.{table_name}')" - ) + if cluster_mode == "database": + create_clickhouse_iceberg_database( + started_cluster, + node, + cluster_catalog_name, + {"object_storage_cluster": "cluster_simple"}, + ) try: - assert node.query( - f"SELECT id FROM {catalog_table} ORDER BY id", - settings=cluster_settings, - ) == expected - - assert node.query( - f"SELECT id FROM {alias_name} ORDER BY id", - settings=table_settings, - ) == expected - - assert node.query( - f"SELECT count() FROM {alias_name}", - settings=cluster_settings, - ) == "3\n" + for replica in nodes: + if replica is node2: + catalog_settings = ( + {"object_storage_cluster": "cluster_simple"} if cluster_mode == "database" else {} + ) + create_clickhouse_iceberg_database( + started_cluster, replica, catalog_db, catalog_settings + ) + if engine in ("Alias", "MergeAlias", "MaterializedView", "Distributed"): + replica.query( + f"CREATE TABLE {alias_name} ENGINE = Alias('{catalog_db}', '{namespace}.{table_name}')" + ) + created[replica].append(alias_name) + if engine != "Alias" and engine != "Loop" and replica is node: + _create_catalog_wrapper( + replica, engine, wrapper_name, catalog_db, namespace, table_name, alias_name + ) + created[replica].append(wrapper_name) + if engine == "Loop": + source = ( + f"(SELECT id FROM loop('{catalog_db}', '{namespace}.{table_name}') " + "WHERE id > 0 LIMIT 3)" + ) + else: + source = wrapper_name assert node.query( - f"SELECT id FROM {alias_name} WHERE id > 0 ORDER BY id", - settings=cluster_settings, - ) == expected + f"SELECT {distinct}id FROM {source} WHERE id > 0 ORDER BY id", + settings=settings, + ) == "1\n2\n3\n" finally: - node.query(f"DROP TABLE IF EXISTS {alias_name}") + for replica, names in created.items(): + for name in reversed(names): + replica.query(f"DROP TABLE IF EXISTS {name}") + if engine == "Distributed": + node2.query(f"DROP DATABASE IF EXISTS {catalog_db}") + if cluster_mode == "database": + node.query(f"DROP DATABASE IF EXISTS {cluster_catalog_name}") From 7b72040dde72cd2d57e94e241193e4d56cbdaefd Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Mon, 12 Oct 2026 11:11:31 -0300 Subject: [PATCH 37/52] Round distances in 02354 so bf16 quantization differences don't fail it Signed-off-by: CarlosFelipeOR --- tests/broken_tests.yaml | 9 --- ...escoring_distance_in_select_list.reference | 36 ++++----- ...arch_rescoring_distance_in_select_list.sql | 76 +++++++++++-------- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/tests/broken_tests.yaml b/tests/broken_tests.yaml index 3e8cac0fc4dee..f8f9043141d02 100644 --- a/tests/broken_tests.yaml +++ b/tests/broken_tests.yaml @@ -144,15 +144,6 @@ check_types: - arm - cpu-aarch64 -- name: 02354_vector_search_rescoring_distance_in_select_list - reason: 'KNOWN: https://github.com/Altinity/ClickHouse/issues/2340 - ARM sanitizer builds have no SimSIMD since #2341; USearch fallback truncates bf16 instead of rounding' - message: '\+7\s0\.1875' - regex: true - check_types: - - asan - - ubsan - - tsan - - msan - name: 01154_move_partition_long reason: INVESTIGATE - Unstable on debug message: 'curl: (28) Operation timed out' diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference index e307b0ef6ae99..0bf9c660e01aa 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.reference @@ -2,32 +2,32 @@ Create tables with Array(Float32) and Array(BFloat16) column Column: Array(Float32) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.203125 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 Column: Array(BFloat16) -- Search vector: Array(Float64) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(Float32) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 -- Search vector: Array(BFloat16) 5 0 -6 0.09375 -7 0.1875 -8 0.296875 +6 0.1 +7 0.2 +8 0.3 diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index c4aa1debd8651..c84a09c2fa03d 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -60,52 +60,68 @@ INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (9, [0.0, 2.4]); -- The nearest neighbours to [0.0, 0.2] are 5,6,7,8 +-- +-- Round the distances: the bf16 index quantization differs between builds +-- (2.2 is stored as 2.203125 with SimSIMD, 2.1875 without it). The inner +-- query keeps the distance function in the SELECT list, as in ClickHouse#85514. SELECT 'Column: Array(Float32)'; SELECT '-- Search vector: Array(Float64)'; -WITH CAST([0.0, 2.0] AS Array(Float64)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_f32 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH CAST([0.0, 2.0] AS Array(Float64)) AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_f32 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; SELECT '-- Search vector: Array(Float32)'; -WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_f32 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_f32 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; SELECT '-- Search vector: Array(BFloat16)'; -WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_f32 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_f32 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; SELECT 'Column: Array(BFloat16)'; SELECT '-- Search vector: Array(Float64)'; -WITH [0.0, 2.0] AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_bf16 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH [0.0, 2.0] AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_bf16 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; SELECT '-- Search vector: Array(Float32)'; -WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_bf16 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH CAST([0.0, 2.0] AS Array(BFloat16)) AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_bf16 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; SELECT '-- Search vector: Array(BFloat16)'; -WITH CAST([0.0, 2.0] AS Array(Float32)) AS reference_vec -SELECT id, L2Distance(vec, reference_vec) -FROM tab_bf16 -ORDER BY L2Distance(vec, reference_vec) -LIMIT 4; +SELECT id, round(d, 1) FROM ( + WITH CAST([0.0, 2.0] AS Array(Float32)) AS reference_vec + SELECT id, L2Distance(vec, reference_vec) AS d + FROM tab_bf16 + ORDER BY L2Distance(vec, reference_vec) + LIMIT 4 +) ORDER BY id; DROP TABLE tab_f32; DROP TABLE tab_bf16; From 3e8533385411498b6ee926ab3e49f3d7eae7f15a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 16:15:42 +0200 Subject: [PATCH 38/52] Move the alias Iceberg catalog scan into the integration tests. The customer filter, limit, and count checks now run against the REST catalog and cluster_simple, including parallel replicas, so the stateless REST stub is no longer needed. Co-authored-by: Cursor --- .../integration/test_database_iceberg/test.py | 92 +++++++++ ...01_alias_iceberg_catalog_cluster.reference | 24 --- .../05301_alias_iceberg_catalog_cluster.sh | 174 ------------------ 3 files changed, 92 insertions(+), 198 deletions(-) delete mode 100644 tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference delete mode 100755 tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index a46b335ce85f2..64bb13d0056cf 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3090,6 +3090,98 @@ def test_catalog_listing_error_surfaces_in_system_tables(started_cluster): node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") +def test_alias_iceberg_catalog_cluster(started_cluster): + """ + Customer repro moved from the stateless suite. An Alias over a catalog Iceberg + table must apply the same filter as a direct catalog read: local, with + object_storage_cluster, and with parallel replicas. count() is the path that + already succeeded before snapshot initialization on the cluster file listing. + """ + node = started_cluster.instances["node1"] + + test_ref = f"test_alias_repro_{uuid.uuid4().hex[:8]}" + namespace = f"{test_ref}_pnl" + table_name = "pnl_t" + catalog_table = f"{CATALOG_NAME}.`{namespace}.{table_name}`" + pnl_db = f"{test_ref}_db" + table_settings = { + "allow_database_iceberg": 1, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + } + modes = { + "local": table_settings, + "osc": {**table_settings, "object_storage_cluster": "cluster_simple"}, + "pr": { + **table_settings, + "enable_parallel_replicas": 1, + "cluster_for_parallel_replicas": "cluster_simple", + "parallel_replicas_for_cluster_engines": 1, + "max_parallel_replicas": 2, + "parallel_replicas_for_non_replicated_merge_tree": 1, + }, + } + filtered = "WHERE TradeDate = '2026-10-06' AND Strategy = 's1'" + filtered_expected = "67\t2026-10-07 06:36:00.000000\n" + limit_expected = "2026-10-06 14:00:00.000000\ts0\n" + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + node.query( + f""" + CREATE TABLE {catalog_table} + ( + TimeSlice DateTime64(6, 'UTC'), + TradeDate Date32, + Strategy String, + NetPL Float64 + ) + ENGINE = IcebergS3('http://minio1:9001/warehouse-rest/{namespace}/', '{minio_access_key}', '{minio_secret_key}') + PARTITION BY TradeDate + """, + settings={ + "allow_experimental_database_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + node.query( + f""" + INSERT INTO {catalog_table} + SELECT + toDateTime64('2026-10-06 14:00:00', 6, 'UTC') + number * 60, + toDate32('2026-10-06') + (number % 3), + 's' || toString(number % 5), + number / 7 + FROM numbers(1000) + """, + settings=table_settings, + ) + node.query(f"CREATE DATABASE {pnl_db}") + node.query(f"CREATE TABLE {pnl_db}.pnl_t ENGINE = Alias({CATALOG_NAME}.`{namespace}.{table_name}`)") + + try: + for settings in modes.values(): + assert node.query( + f"SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM {catalog_table} {filtered}", + settings=settings, + ) == filtered_expected + assert node.query( + f"SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM {pnl_db}.pnl_t {filtered}", + settings=settings, + ) == filtered_expected + assert node.query( + f"SELECT toTimeZone(TimeSlice, 'UTC'), Strategy FROM {pnl_db}.pnl_t ORDER BY TimeSlice LIMIT 1", + settings=settings, + ) == limit_expected + assert node.query( + f"SELECT count() FROM {pnl_db}.pnl_t", + settings=settings, + ) == "1000\n" + finally: + node.query(f"DROP TABLE IF EXISTS {pnl_db}.pnl_t") + node.query(f"DROP DATABASE IF EXISTS {pnl_db}") + drop_clickhouse_iceberg_table(node, namespace, table_name, if_exists=True) + + def _create_catalog_wrapper(node, engine, name, catalog_db, namespace, table_name, alias_name): catalog_table = f"{catalog_db}.`{namespace}.{table_name}`" qualified_name = f"{namespace}.{table_name}" diff --git a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference deleted file mode 100644 index 70c90fda37ee0..0000000000000 --- a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.reference +++ /dev/null @@ -1,24 +0,0 @@ -local-direct-filtered -67 2026-10-07 06:36:00.000000 -local-alias-filtered -67 2026-10-07 06:36:00.000000 -local-alias-limit1 -2026-10-06 14:00:00.000000 s0 -local-alias-trivcount -1000 -osc-direct-filtered -67 2026-10-07 06:36:00.000000 -osc-alias-filtered -67 2026-10-07 06:36:00.000000 -osc-alias-limit1 -2026-10-06 14:00:00.000000 s0 -osc-alias-trivcount -1000 -pr-direct-filtered -67 2026-10-07 06:36:00.000000 -pr-alias-filtered -67 2026-10-07 06:36:00.000000 -pr-alias-limit1 -2026-10-06 14:00:00.000000 s0 -pr-alias-trivcount -1000 diff --git a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh b/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh deleted file mode 100755 index cb272ca8b49b9..0000000000000 --- a/tests/queries/0_stateless/05301_alias_iceberg_catalog_cluster.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, no-replicated-database -# Tag no-replicated-database: IcebergLocal is non-replicated. -# -# Customer repro: an Alias over a DataLakeCatalog Iceberg table. -# The catalog is a local REST stub in front of an IcebergLocal table. -# A filtered scan and a LIMIT through the alias must match a direct catalog read -# with no cluster, with object_storage_cluster, and with parallel replicas. -# count() through the alias is the control that already succeeded before the fix. - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -set -euo pipefail - -WH="${CLICKHOUSE_USER_FILES_UNIQUE}/wh" -TABLE_PATH="${WH}/pnl/pnl_t" -ICE_DB="${CLICKHOUSE_DATABASE}_ice" -PNL_DB="${CLICKHOUSE_DATABASE}_pnl" -STUB_PY="${CLICKHOUSE_TMP}/alias_iceberg_stub_${CLICKHOUSE_TEST_UNIQUE_NAME}.py" -STUB_LOG="${CLICKHOUSE_TMP}/alias_iceberg_stub_${CLICKHOUSE_TEST_UNIQUE_NAME}.log" -STUB_PID="" - -STUB_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') - -cleanup() { - ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PNL_DB}.pnl_t" >/dev/null 2>&1 || true - ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${PNL_DB}" >/dev/null 2>&1 || true - ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${ICE_DB}" >/dev/null 2>&1 || true - ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS src" >/dev/null 2>&1 || true - if [[ -n "${STUB_PID}" ]]; then - kill "${STUB_PID}" >/dev/null 2>&1 || true - wait "${STUB_PID}" >/dev/null 2>&1 || true - fi - rm -rf "${WH}" "${STUB_PY}" "${STUB_LOG}" -} -trap cleanup EXIT - -mkdir -p "${WH}/pnl" "$(dirname "${STUB_PY}")" - -cat > "${STUB_PY}" <<'PY' -import glob -import json -import os -import sys -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -WH = sys.argv[1] -PORT = int(sys.argv[2]) - - -def tables(ns): - return sorted( - os.path.basename(p) - for p in glob.glob(os.path.join(WH, ns, "*")) - if os.path.isdir(os.path.join(p, "metadata")) - ) - - -def newest_meta(ns, t): - files = glob.glob(os.path.join(WH, ns, t, "metadata", "*.metadata.json")) - files.sort(key=os.path.getmtime) - return files[-1] - - -class H(BaseHTTPRequestHandler): - def _send(self, code, obj): - body = json.dumps(obj).encode() - self.send_response(code) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def do_GET(self): - path = self.path.split("?")[0].rstrip("/") - parts = path.split("/")[2:] if path.startswith("/v1") else None - if parts is None: - return self._send(404, {"error": {"message": "not found", "type": "NotFound", "code": 404}}) - if parts == ["config"]: - return self._send(200, {"defaults": {}, "overrides": {}}) - if parts == ["namespaces"]: - if "parent=" in self.path: - return self._send(200, {"namespaces": []}) - nss = sorted(os.path.basename(p) for p in glob.glob(os.path.join(WH, "*")) if os.path.isdir(p)) - return self._send(200, {"namespaces": [[n] for n in nss]}) - if len(parts) == 2 and parts[0] == "namespaces": - return self._send(200, {"namespace": [parts[1]], "properties": {}}) - if len(parts) == 3 and parts[0] == "namespaces" and parts[2] == "tables": - return self._send(200, {"identifiers": [{"namespace": [parts[1]], "name": t} for t in tables(parts[1])]}) - if len(parts) == 4 and parts[0] == "namespaces" and parts[2] == "tables": - ns, t = parts[1], parts[3] - if t not in tables(ns): - return self._send(404, {"error": {"message": "no table", "type": "NoSuchTableException", "code": 404}}) - m = newest_meta(ns, t) - with open(m) as f: - meta = json.load(f) - if "://" not in meta.get("location", ""): - meta["location"] = "file://" + meta["location"] - return self._send(200, {"metadata-location": "file://" + m, "metadata": meta, "config": {}}) - return self._send(404, {"error": {"message": "not found", "type": "NotFound", "code": 404}}) - - def log_message(self, *a): - pass - - -ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever() -PY - -python3 "${STUB_PY}" "${WH}" "${STUB_PORT}" >"${STUB_LOG}" 2>&1 & -STUB_PID=$! - -for _ in $(seq 1 50); do - if python3 -c "import socket; socket.create_connection(('127.0.0.1', ${STUB_PORT}), 0.2).close()" 2>/dev/null; then - break - fi - sleep 0.1 -done - -${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg 1 --query " - CREATE TABLE src ( - TimeSlice DateTime64(6, 'UTC'), - TradeDate Date32, - Strategy String, - NetPL Float64 - ) ENGINE = IcebergLocal('${TABLE_PATH}') - PARTITION BY TradeDate -" - -${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg 1 --query " - INSERT INTO src - SELECT - toDateTime64('2026-10-06 14:00:00', 6, 'UTC') + number * 60, - toDate32('2026-10-06') + (number % 3), - 's' || toString(number % 5), - number / 7 - FROM numbers(1000) -" - -${CLICKHOUSE_CLIENT} --allow_database_iceberg 1 --query " - CREATE DATABASE ${ICE_DB} ENGINE = DataLakeCatalog('http://127.0.0.1:${STUB_PORT}/v1') - SETTINGS - catalog_type = 'rest', - auth_header = 'Authorization: Bearer sandboxonlytesttoken0001', - warehouse = 'warehouse', - vended_credentials = false -" - -${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${PNL_DB}" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${PNL_DB}.pnl_t ENGINE = Alias(${ICE_DB}.\`pnl.pnl_t\`)" - -FILT="WHERE TradeDate = '2026-10-06' AND Strategy = 's1'" -OSC="SETTINGS object_storage_cluster = 'test_shard_localhost'" -PR="SETTINGS enable_parallel_replicas = 1, cluster_for_parallel_replicas = 'test_shard_localhost', parallel_replicas_for_cluster_engines = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1" - -run() { - local tag="$1" - local sql="$2" - echo "${tag}" - ${CLICKHOUSE_CLIENT} --query "${sql}" -} - -for mode in local osc pr; do - case "${mode}" in - local) S="" ;; - osc) S="${OSC}" ;; - pr) S="${PR}" ;; - esac - run "${mode}-direct-filtered" "SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM ${ICE_DB}.\`pnl.pnl_t\` ${FILT} ${S}" - run "${mode}-alias-filtered" "SELECT count(), toTimeZone(max(TimeSlice), 'UTC') FROM ${PNL_DB}.pnl_t ${FILT} ${S}" - run "${mode}-alias-limit1" "SELECT toTimeZone(TimeSlice, 'UTC'), Strategy FROM ${PNL_DB}.pnl_t ORDER BY TimeSlice LIMIT 1 ${S}" - run "${mode}-alias-trivcount" "SELECT count() FROM ${PNL_DB}.pnl_t ${S}" -done From 15eceeb40ba13a11cbcce5e7f90a3d4b82da7e04 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 16:48:40 +0200 Subject: [PATCH 39/52] Format ClickHouse names in the Iceberg alias test docstrings. Related: https://github.com/Altinity/ClickHouse/pull/2550 Co-authored-by: Cursor --- tests/integration/test_database_iceberg/test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 64bb13d0056cf..9e0bbc0e4e79a 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3092,10 +3092,10 @@ def test_catalog_listing_error_surfaces_in_system_tables(started_cluster): def test_alias_iceberg_catalog_cluster(started_cluster): """ - Customer repro moved from the stateless suite. An Alias over a catalog Iceberg - table must apply the same filter as a direct catalog read: local, with - object_storage_cluster, and with parallel replicas. count() is the path that - already succeeded before snapshot initialization on the cluster file listing. + An `Alias` over a catalog `Iceberg` table must apply the same filter as a direct + catalog read: local, with `object_storage_cluster`, and with parallel replicas. + `count` is the path that already succeeded before snapshot initialization on the + cluster file listing. """ node = started_cluster.instances["node1"] @@ -3216,10 +3216,10 @@ def _create_catalog_wrapper(node, engine, name, catalog_db, namespace, table_nam @pytest.mark.parametrize("cluster_mode", ["local", "query", "database"]) def test_datalake_catalog_wrapper_object_storage_cluster(started_cluster, engine, cluster_mode): """ - A DataLakeCatalog lookup builds a fresh storage and leaves its configuration + A `DataLakeCatalog` lookup builds a fresh storage and leaves its configuration uninitialized. Each engine below resolves that table again, then lists files. - cluster_mode chooses where object_storage_cluster comes from: + `cluster_mode` chooses where `object_storage_cluster` comes from: local keeps it unset, query sets it on the scan, database sets it on the catalog. """ if engine == "Loop" and cluster_mode != "local": From 2a09ee317a90d65f72165bbb9f1c4d407ec41c48 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 12 Oct 2026 17:01:29 +0200 Subject: [PATCH 40/52] Format `Buffer` and `WithMergeableState` in the wrapper test comment. Related: https://github.com/Altinity/ClickHouse/pull/2550 Co-authored-by: Cursor --- tests/integration/test_database_iceberg/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 9e0bbc0e4e79a..dc1e4ff5f7c75 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -3245,7 +3245,7 @@ def test_datalake_catalog_wrapper_object_storage_cluster(started_cluster, engine **table_settings, "object_storage_cluster": "cluster_simple", } - # A clustered Buffer destination reports WithMergeableState, so Buffer plans + # A clustered `Buffer` destination reports `WithMergeableState`, so `Buffer` plans # the in-memory branch as a second read of the same query. distinct = "DISTINCT " if engine == "Buffer" and cluster_mode != "local" else "" nodes = [node, node2] if engine == "Distributed" else [node] From 580aa4609b5515706b664e687af66275587b7358 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 12 Oct 2026 14:59:59 -0300 Subject: [PATCH 41/52] safe cast --- src/DataTypes/Utils.cpp | 6 ++ .../tests/gtest_can_be_safely_cast.cpp | 78 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 src/DataTypes/tests/gtest_can_be_safely_cast.cpp diff --git a/src/DataTypes/Utils.cpp b/src/DataTypes/Utils.cpp index 41d576820d522..b1e5e1fd72deb 100644 --- a/src/DataTypes/Utils.cpp +++ b/src/DataTypes/Utils.cpp @@ -33,6 +33,12 @@ bool canBeSafelyCast(const DataTypePtr & from_type, const DataTypePtr & to_type) to_type_unwrapped->getSizeOfValueInMemory() >= from_type->getSizeOfValueInMemory()) return true; + /// An unsigned value fits into a signed integer only if the latter is strictly wider: + /// `UInt8 -> Int16` is lossless, `UInt8 -> Int8` is not. + if (to_which_type.isInt() && + to_type_unwrapped->getSizeOfValueInMemory() > from_type->getSizeOfValueInMemory()) + return true; + if (to_which_type.isString()) return true; diff --git a/src/DataTypes/tests/gtest_can_be_safely_cast.cpp b/src/DataTypes/tests/gtest_can_be_safely_cast.cpp new file mode 100644 index 0000000000000..dcb01bc1badca --- /dev/null +++ b/src/DataTypes/tests/gtest_can_be_safely_cast.cpp @@ -0,0 +1,78 @@ +#include +#include + +#include + +using namespace DB; + +namespace +{ + +bool canCast(const std::string & from, const std::string & to) +{ + auto & factory = DataTypeFactory::instance(); + return canBeSafelyCast(factory.get(from), factory.get(to)); +} + +} + +TEST(CanBeSafelyCast, UnsignedToWiderSigned) +{ + EXPECT_TRUE(canCast("UInt8", "Int16")); + EXPECT_TRUE(canCast("UInt8", "Int32")); + EXPECT_TRUE(canCast("UInt8", "Int64")); + EXPECT_TRUE(canCast("UInt8", "Int128")); + EXPECT_TRUE(canCast("UInt8", "Int256")); + EXPECT_TRUE(canCast("UInt16", "Int32")); + EXPECT_TRUE(canCast("UInt16", "Int64")); + EXPECT_TRUE(canCast("UInt32", "Int64")); + EXPECT_TRUE(canCast("UInt64", "Int128")); + EXPECT_TRUE(canCast("UInt64", "Int256")); + EXPECT_TRUE(canCast("UInt128", "Int256")); +} + +TEST(CanBeSafelyCast, UnsignedToSignedOfSameOrSmallerSizeIsLossy) +{ + EXPECT_FALSE(canCast("UInt8", "Int8")); + EXPECT_FALSE(canCast("UInt16", "Int8")); + EXPECT_FALSE(canCast("UInt16", "Int16")); + EXPECT_FALSE(canCast("UInt32", "Int16")); + EXPECT_FALSE(canCast("UInt32", "Int32")); + EXPECT_FALSE(canCast("UInt64", "Int32")); + EXPECT_FALSE(canCast("UInt64", "Int64")); + EXPECT_FALSE(canCast("UInt128", "Int64")); + EXPECT_FALSE(canCast("UInt128", "Int128")); + EXPECT_FALSE(canCast("UInt256", "Int128")); + EXPECT_FALSE(canCast("UInt256", "Int256")); +} + +TEST(CanBeSafelyCast, SignedToUnsignedIsLossy) +{ + EXPECT_FALSE(canCast("Int8", "UInt8")); + EXPECT_FALSE(canCast("Int8", "UInt16")); + EXPECT_FALSE(canCast("Int8", "UInt64")); + EXPECT_FALSE(canCast("Int32", "UInt64")); + EXPECT_FALSE(canCast("Int128", "UInt256")); +} + +TEST(CanBeSafelyCast, SameSignednessWidening) +{ + EXPECT_TRUE(canCast("UInt8", "UInt8")); + EXPECT_TRUE(canCast("UInt8", "UInt64")); + EXPECT_FALSE(canCast("UInt64", "UInt8")); + EXPECT_TRUE(canCast("Int8", "Int64")); + EXPECT_FALSE(canCast("Int64", "Int8")); +} + +TEST(CanBeSafelyCast, UnsignedToSignedThroughWrappers) +{ + EXPECT_TRUE(canCast("UInt8", "Nullable(Int16)")); + EXPECT_TRUE(canCast("UInt8", "LowCardinality(Int16)")); + EXPECT_TRUE(canCast("Nullable(UInt8)", "Nullable(Int16)")); + EXPECT_TRUE(canCast("Array(UInt8)", "Array(Int16)")); + EXPECT_TRUE(canCast("Tuple(UInt8, UInt32)", "Tuple(Int16, Int64)")); + EXPECT_TRUE(canCast("Map(UInt8, UInt16)", "Map(Int16, Int32)")); + + EXPECT_FALSE(canCast("Array(UInt8)", "Array(Int8)")); + EXPECT_FALSE(canCast("Tuple(UInt8, UInt32)", "Tuple(Int16, Int32)")); +} From abc516e8d5491482d026d54a960c6b272a9db2a7 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 10:03:08 +0200 Subject: [PATCH 42/52] Keep the catalog storage alive for an `Alias` snapshot. `StorageSnapshot` only referenced the target, and `DataLakeCatalog` drops that storage when `getStorageSnapshot` returns. The analyzer then read freed memory under ASan. https: //altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2550&sha=2a09ee317a90d65f72165bbb9f1c4d407ec41c48&name_0=PR&name_1=Integration%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20old%20analyzer%2C%208%2F8%29 Related: https://github.com/Altinity/ClickHouse/pull/2550 Co-authored-by: Cursor --- .../Optimizations/optimizeUseAggregateProjection.cpp | 2 +- .../Optimizations/optimizeUseNormalProjection.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 6 +++--- src/Storages/StorageAlias.cpp | 12 ++++++++++-- src/Storages/StorageSnapshot.cpp | 11 ++++++++++- src/Storages/StorageSnapshot.h | 11 +++++++++++ 7 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index 0a0cbe26fdccc..fb3942b047829 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -873,7 +873,7 @@ std::optional optimizeUseAggregateProjections( { chassert(parent_reading_select_result); auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); + auto proj_snapshot = storage_snapshot->cloneWithMetadata(best_candidate->projection->metadata); auto projection_query_info = query_info; projection_query_info.prewhere_info = nullptr; projection_query_info.row_level_filter = nullptr; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index 14e48cea12a20..fdcf8b0a5d775 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -399,7 +399,7 @@ std::optional optimizeUseNormalProjections( } auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); + auto proj_snapshot = storage_snapshot->cloneWithMetadata(best_candidate->projection->metadata); /// Enables PREWHERE on projections to improve read efficiency and leverage query condition cache. if (query.dag && query.filter_node) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 21b014bd72ff8..12cf6364c255a 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -3927,7 +3927,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, [[ma SingleProjectionIndexReader( std::make_shared( empty_mutations_snapshot, - std::make_shared(storage_snapshot->storage, read_info.projection->metadata), + storage_snapshot->cloneWithMetadata(read_info.projection->metadata), read_info.prewhere_info, actions_settings, reader_settings, @@ -4654,7 +4654,7 @@ void ReadFromMergeTree::createReadTasksForTextIndex(const UsefulSkipIndexes & sk } } - storage_snapshot = std::make_shared(storage_snapshot->storage, std::move(new_metadata)); + storage_snapshot = storage_snapshot->cloneWithMetadata(std::move(new_metadata)); if (output_header != nullptr) { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 2a9552981e0ff..b0acc944dfe1f 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -580,7 +580,7 @@ void StorageObjectStorage::read( { auto stripped = std::make_shared(*storage_snapshot->metadata); stripped->datalake_table_state.reset(); - storage_snapshot = std::make_shared(*this, std::move(stripped)); + storage_snapshot = storage_snapshot->cloneWithMetadata(std::move(stripped)); } }); @@ -616,8 +616,8 @@ void StorageObjectStorage::read( rebuilt->columns, local_context, format_settings, configuration->getPartitionStrategyType())); } - storage_snapshot = std::make_shared( - *this, std::make_shared(std::move(pinned))); + storage_snapshot = storage_snapshot->cloneWithMetadata( + std::make_shared(std::move(pinned))); } } diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index d6d3567665990..54dbd205688b6 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -350,12 +350,20 @@ std::optional StorageAlias::distributedWrite(const ASTInsertQuery StorageSnapshotPtr StorageAlias::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - return getTargetTable()->getStorageSnapshot(metadata_snapshot, query_context); + auto target = getTargetTable(); + auto snapshot = target->getStorageSnapshot(metadata_snapshot, query_context); + /// `DataLakeCatalog` builds a new storage on every lookup. The snapshot keeps only a reference, + /// so it must own that `shared_ptr` or the analyzer reads freed memory. + snapshot->storage_holder = std::move(target); + return snapshot; } StorageSnapshotPtr StorageAlias::getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - return getTargetTable()->getStorageSnapshotWithoutData(metadata_snapshot, query_context); + auto target = getTargetTable(); + auto snapshot = target->getStorageSnapshotWithoutData(metadata_snapshot, query_context); + snapshot->storage_holder = std::move(target); + return snapshot; } void StorageAlias::rename(const String & /* new_path_to_table_data */, const StorageID & new_table_id) diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index 7abaf6688033c..92815364ea605 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -39,7 +39,16 @@ StorageSnapshot::StorageSnapshot( std::shared_ptr StorageSnapshot::clone(DataPtr data_) const { - return std::make_shared(storage, metadata, std::move(data_)); + auto result = std::make_shared(storage, metadata, std::move(data_)); + result->storage_holder = storage_holder; + return result; +} + +std::shared_ptr StorageSnapshot::cloneWithMetadata(StorageMetadataPtr metadata_) const +{ + auto result = std::make_shared(storage, std::move(metadata_)); + result->storage_holder = storage_holder; + return result; } ColumnsDescription StorageSnapshot::getAllColumnsDescription() const diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h index a3635fbf64414..0480608d0ea6c 100644 --- a/src/Storages/StorageSnapshot.h +++ b/src/Storages/StorageSnapshot.h @@ -1,6 +1,9 @@ #pragma once +#include #include +#include + namespace DB { @@ -19,6 +22,11 @@ struct StorageSnapshot const IStorage & storage; const StorageMetadataPtr metadata; + /// Owns `storage` when nothing else does. Empty for catalog-resident tables. + /// `Alias` over `DataLakeCatalog` fills it: each lookup returns a new storage, + /// and this snapshot is the only object that outlives that lookup. + StoragePtr storage_holder; + /// Additional data, on which set of columns may depend. /// E.g. data parts in MergeTree, list of blocks in Memory, etc. struct Data @@ -40,6 +48,9 @@ struct StorageSnapshot std::shared_ptr clone(DataPtr data_) const; + /// Same `storage` and `storage_holder`, replaced metadata. Drops `data`. + std::shared_ptr cloneWithMetadata(StorageMetadataPtr metadata_) const; + /// Get columns description ColumnsDescription getAllColumnsDescription() const; From bd359eba7db709ed5bdca5244d71cd747aba8b22 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 10:23:57 +0200 Subject: [PATCH 43/52] Revert "Keep the catalog storage alive for an `Alias` snapshot." This reverts commit abc516e8d5491482d026d54a960c6b272a9db2a7. --- .../Optimizations/optimizeUseAggregateProjection.cpp | 2 +- .../Optimizations/optimizeUseNormalProjection.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 6 +++--- src/Storages/StorageAlias.cpp | 12 ++---------- src/Storages/StorageSnapshot.cpp | 11 +---------- src/Storages/StorageSnapshot.h | 11 ----------- 7 files changed, 10 insertions(+), 38 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index fb3942b047829..0a0cbe26fdccc 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -873,7 +873,7 @@ std::optional optimizeUseAggregateProjections( { chassert(parent_reading_select_result); auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = storage_snapshot->cloneWithMetadata(best_candidate->projection->metadata); + auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); auto projection_query_info = query_info; projection_query_info.prewhere_info = nullptr; projection_query_info.row_level_filter = nullptr; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index fdcf8b0a5d775..14e48cea12a20 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -399,7 +399,7 @@ std::optional optimizeUseNormalProjections( } auto storage_snapshot = reading->getStorageSnapshot(); - auto proj_snapshot = storage_snapshot->cloneWithMetadata(best_candidate->projection->metadata); + auto proj_snapshot = std::make_shared(storage_snapshot->storage, best_candidate->projection->metadata); /// Enables PREWHERE on projections to improve read efficiency and leverage query condition cache. if (query.dag && query.filter_node) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 12cf6364c255a..21b014bd72ff8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -3927,7 +3927,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, [[ma SingleProjectionIndexReader( std::make_shared( empty_mutations_snapshot, - storage_snapshot->cloneWithMetadata(read_info.projection->metadata), + std::make_shared(storage_snapshot->storage, read_info.projection->metadata), read_info.prewhere_info, actions_settings, reader_settings, @@ -4654,7 +4654,7 @@ void ReadFromMergeTree::createReadTasksForTextIndex(const UsefulSkipIndexes & sk } } - storage_snapshot = storage_snapshot->cloneWithMetadata(std::move(new_metadata)); + storage_snapshot = std::make_shared(storage_snapshot->storage, std::move(new_metadata)); if (output_header != nullptr) { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index b0acc944dfe1f..2a9552981e0ff 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -580,7 +580,7 @@ void StorageObjectStorage::read( { auto stripped = std::make_shared(*storage_snapshot->metadata); stripped->datalake_table_state.reset(); - storage_snapshot = storage_snapshot->cloneWithMetadata(std::move(stripped)); + storage_snapshot = std::make_shared(*this, std::move(stripped)); } }); @@ -616,8 +616,8 @@ void StorageObjectStorage::read( rebuilt->columns, local_context, format_settings, configuration->getPartitionStrategyType())); } - storage_snapshot = storage_snapshot->cloneWithMetadata( - std::make_shared(std::move(pinned))); + storage_snapshot = std::make_shared( + *this, std::make_shared(std::move(pinned))); } } diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index 54dbd205688b6..d6d3567665990 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -350,20 +350,12 @@ std::optional StorageAlias::distributedWrite(const ASTInsertQuery StorageSnapshotPtr StorageAlias::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - auto target = getTargetTable(); - auto snapshot = target->getStorageSnapshot(metadata_snapshot, query_context); - /// `DataLakeCatalog` builds a new storage on every lookup. The snapshot keeps only a reference, - /// so it must own that `shared_ptr` or the analyzer reads freed memory. - snapshot->storage_holder = std::move(target); - return snapshot; + return getTargetTable()->getStorageSnapshot(metadata_snapshot, query_context); } StorageSnapshotPtr StorageAlias::getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - auto target = getTargetTable(); - auto snapshot = target->getStorageSnapshotWithoutData(metadata_snapshot, query_context); - snapshot->storage_holder = std::move(target); - return snapshot; + return getTargetTable()->getStorageSnapshotWithoutData(metadata_snapshot, query_context); } void StorageAlias::rename(const String & /* new_path_to_table_data */, const StorageID & new_table_id) diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index 92815364ea605..7abaf6688033c 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -39,16 +39,7 @@ StorageSnapshot::StorageSnapshot( std::shared_ptr StorageSnapshot::clone(DataPtr data_) const { - auto result = std::make_shared(storage, metadata, std::move(data_)); - result->storage_holder = storage_holder; - return result; -} - -std::shared_ptr StorageSnapshot::cloneWithMetadata(StorageMetadataPtr metadata_) const -{ - auto result = std::make_shared(storage, std::move(metadata_)); - result->storage_holder = storage_holder; - return result; + return std::make_shared(storage, metadata, std::move(data_)); } ColumnsDescription StorageSnapshot::getAllColumnsDescription() const diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h index 0480608d0ea6c..a3635fbf64414 100644 --- a/src/Storages/StorageSnapshot.h +++ b/src/Storages/StorageSnapshot.h @@ -1,9 +1,6 @@ #pragma once -#include #include -#include - namespace DB { @@ -22,11 +19,6 @@ struct StorageSnapshot const IStorage & storage; const StorageMetadataPtr metadata; - /// Owns `storage` when nothing else does. Empty for catalog-resident tables. - /// `Alias` over `DataLakeCatalog` fills it: each lookup returns a new storage, - /// and this snapshot is the only object that outlives that lookup. - StoragePtr storage_holder; - /// Additional data, on which set of columns may depend. /// E.g. data parts in MergeTree, list of blocks in Memory, etc. struct Data @@ -48,9 +40,6 @@ struct StorageSnapshot std::shared_ptr clone(DataPtr data_) const; - /// Same `storage` and `storage_holder`, replaced metadata. Drops `data`. - std::shared_ptr cloneWithMetadata(StorageMetadataPtr metadata_) const; - /// Get columns description ColumnsDescription getAllColumnsDescription() const; From 619d64a350af18701e3c72f6bfd6967d7b14774e Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 10:29:17 +0200 Subject: [PATCH 44/52] Backport the `Alias` snapshot holder from master. `StorageSnapshot` keeps a bare reference to its storage, and `Alias` returned that snapshot after the temporary from `getTargetTable` was destroyed. This brings `withStorageHolder` from upstream, cloning snapshot `Data` because it is still a `unique_ptr` on this branch. https: //github.com/ClickHouse/ClickHouse/pull/120098 Co-authored-by: Cursor --- src/Storages/MergeTree/MergeTreeData.h | 9 + src/Storages/StorageAlias.cpp | 11 +- src/Storages/StorageDistributed.h | 7 + src/Storages/StorageMemory.h | 8 + src/Storages/StorageSnapshot.cpp | 33 +++- src/Storages/StorageSnapshot.h | 16 +- .../gtest_storage_snapshot_storage_holder.cpp | 171 ++++++++++++++++++ ...torage_alias_chain_join_snapshot.reference | 10 + ...5213_storage_alias_chain_join_snapshot.sql | 49 +++++ 9 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp create mode 100644 tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.reference create mode 100644 tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index df774e74778ab..acb9d4777cf3f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -664,6 +664,15 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr RangesInDataPartsPtr parts; MutationsSnapshotPtr mutations_snapshot; + + std::unique_ptr clone() const override + { + auto copy = std::make_unique(); + copy->storage = storage; + copy->parts = parts; + copy->mutations_snapshot = mutations_snapshot; + return copy; + } }; StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index d6d3567665990..3fd41dd2de904 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -350,12 +350,19 @@ std::optional StorageAlias::distributedWrite(const ASTInsertQuery StorageSnapshotPtr StorageAlias::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - return getTargetTable()->getStorageSnapshot(metadata_snapshot, query_context); + /// Bind the target to a named local so that one owning pointer spans both calls, and hand it to the + /// snapshot: `getTargetTable` resolves through `DatabaseCatalog` and returns a temporary, while our + /// caller owns and share-locks this alias rather than the target the snapshot refers to. + StoragePtr target = getTargetTable(); + auto snapshot = target->getStorageSnapshot(metadata_snapshot, query_context); + return snapshot->withStorageHolder(std::move(target)); } StorageSnapshotPtr StorageAlias::getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { - return getTargetTable()->getStorageSnapshotWithoutData(metadata_snapshot, query_context); + StoragePtr target = getTargetTable(); + auto snapshot = target->getStorageSnapshotWithoutData(metadata_snapshot, query_context); + return snapshot->withStorageHolder(std::move(target)); } void StorageAlias::rename(const String & /* new_path_to_table_data */, const StorageID & new_table_id) diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index d867e72ef907e..a81256b05f012 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -249,6 +249,13 @@ class StorageDistributed final : public IStorage, WithContext struct HybridSnapshotData : public StorageSnapshot::Data { MultiVersion::Version watermark_snapshot; + + std::unique_ptr clone() const override + { + auto copy = std::make_unique(); + copy->watermark_snapshot = watermark_snapshot; + return copy; + } }; /// Per-query Hybrid pruning verdict. Recomputed in both `getQueryProcessingStage()` diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 8a281acdf73c9..f7ee873814df9 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -46,6 +46,14 @@ friend class MemorySink; { std::shared_ptr blocks; size_t rows_approx = 0; + + std::unique_ptr clone() const override + { + auto copy = std::make_unique(); + copy->blocks = blocks; + copy->rows_approx = rows_approx; + return copy; + } }; StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp index 7abaf6688033c..797c48f21ec13 100644 --- a/src/Storages/StorageSnapshot.cpp +++ b/src/Storages/StorageSnapshot.cpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include @@ -17,6 +19,7 @@ namespace ErrorCodes extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int COLUMN_QUERIED_MORE_THAN_ONCE; + extern const int LOGICAL_ERROR; } StorageSnapshot::StorageSnapshot( @@ -39,7 +42,35 @@ StorageSnapshot::StorageSnapshot( std::shared_ptr StorageSnapshot::clone(DataPtr data_) const { - return std::make_shared(storage, metadata, std::move(data_)); + auto res = std::make_shared(storage, metadata, std::move(data_)); + res->storage_holder = storage_holder; + return res; +} + +std::shared_ptr StorageSnapshot::withStorageHolder(ConstStoragePtr holder) const +{ + /// Returning a new snapshot rather than installing the holder in place: a snapshot can be shared + /// between call sites within one query (the query metadata cache and the pinned snapshot of + /// `CREATE MATERIALIZED VIEW ... POPULATE` both hand the same object to several readers). + DataPtr cloned_data; + if (data) + cloned_data = data->clone(); + if (data && !cloned_data) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Storage snapshot data cannot be cloned"); + + auto res = std::make_shared(storage, metadata, std::move(cloned_data)); + if (storage_holder) + { + /// An alias chain resolves innermost-first, so an existing holder already owns the storage this + /// snapshot references, while `holder` is only an intermediate alias on the way to it. + res->storage_holder = storage_holder; + } + else + { + chassert(holder && holder.get() == &storage); + res->storage_holder = std::move(holder); + } + return res; } ColumnsDescription StorageSnapshot::getAllColumnsDescription() const diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h index a3635fbf64414..d27b84da77610 100644 --- a/src/Storages/StorageSnapshot.h +++ b/src/Storages/StorageSnapshot.h @@ -1,10 +1,10 @@ #pragma once +#include #include namespace DB { -class IStorage; class ICompressionCodec; using CompressionCodecPtr = std::shared_ptr; @@ -24,6 +24,10 @@ struct StorageSnapshot struct Data { virtual ~Data() = default; + + /// `master` shares `Data` through `shared_ptr`. Here it is still a `unique_ptr`, + /// so a second snapshot has to clone it instead of copying the pointer. + virtual std::unique_ptr clone() const { return nullptr; } }; using DataPtr = std::unique_ptr; @@ -40,6 +44,11 @@ struct StorageSnapshot std::shared_ptr clone(DataPtr data_) const; + /// Returns an equivalent snapshot that additionally owns `holder`, whose referent must be `storage`. + /// Ownership is what keeps that storage alive: `DatabaseCatalog::getTablesToDrop` treats a dropped + /// table as unused as soon as its only remaining `shared_ptr` is the catalog's own. + std::shared_ptr withStorageHolder(ConstStoragePtr holder) const; + /// Get columns description ColumnsDescription getAllColumnsDescription() const; @@ -65,6 +74,11 @@ struct StorageSnapshot /// Get default expression for a column. /// Takes into account physical and virtual columns. std::optional getDefault(const String & column_name) const; + +private: + /// Empty on almost every snapshot: one that is consumed inside the scope of an owning + /// `StoragePtr` must not delay the drop of its table. + ConstStoragePtr storage_holder; }; using StorageSnapshotPtr = std::shared_ptr; diff --git a/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp new file mode 100644 index 0000000000000..d2d0b922f0d35 --- /dev/null +++ b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp @@ -0,0 +1,171 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// StorageSnapshot refers to its storage without owning it, which is sound only while the caller that +/// took the snapshot keeps a StoragePtr. StorageAlias breaks that: it hands out a snapshot of the table +/// it points at, which its own caller neither owns nor locks, so the target could be dropped and +/// destroyed under a running query (heap-use-after-free in IStorage::getStorageID(), STID 2350-8243). +/// withStorageHolder() is the opt-in ownership token that closes it. These tests assert the token +/// mechanism through weak_ptr expiry, which is deterministic and needs no sanitizer. + +namespace +{ + +using namespace DB; + +/// The invariant is a property of StorageSnapshot itself and holds for any engine, so the cheapest +/// storage that can serve the base getStorageSnapshot() is enough; a real engine would only add a +/// Context and a disk to the fixture. +class StorageForSnapshotHolderTest final : public IStorage +{ +public: + explicit StorageForSnapshotHolderTest(const StorageID & id) : IStorage(id) { } + std::string getName() const override { return "StorageForSnapshotHolderTest"; } +}; + +StoragePtr makeStorage(const String & database, const String & table) +{ + return std::make_shared(StorageID{database, table}); +} + +StoragePtr makeStorage(const String & table) +{ + return makeStorage("test_db", table); +} + +StorageSnapshotPtr takeSnapshot(const StoragePtr & storage) +{ + return storage->getStorageSnapshot(std::make_shared(), nullptr); +} + +} + +TEST(StorageSnapshotStorageHolder, PinsTheReferentUntilTheLastSnapshotIsGone) +{ + auto storage = makeStorage("pinned"); + std::weak_ptr weak = storage; + const String expected_name = storage->getStorageID().getFullTableName(); + + auto original = takeSnapshot(storage); + auto pinned = original->withStorageHolder(storage); + + original.reset(); + storage.reset(); + + /// The token is the only owner left, so the referent is still alive and readable. + ASSERT_FALSE(weak.expired()); + EXPECT_EQ(pinned->storage.getStorageID().getFullTableName(), expected_name); + + auto cloned = pinned->clone(nullptr); + pinned.reset(); + + /// clone() has to carry the token, or every projection and metadata re-wrap would silently drop it. + ASSERT_FALSE(weak.expired()); + EXPECT_EQ(cloned->storage.getStorageID().getFullTableName(), expected_name); + + cloned.reset(); + + /// And the token has to be released with the last snapshot, or the table would never be dropped. + EXPECT_TRUE(weak.expired()); +} + +TEST(StorageSnapshotStorageHolder, DoesNotInstallTheTokenInTheSourceSnapshot) +{ + auto storage = makeStorage("shared"); + std::weak_ptr weak = storage; + + auto original = takeSnapshot(storage); + auto pinned = original->withStorageHolder(storage); + + /// One snapshot is shared between the readers of a query (the query metadata cache and the pinned + /// snapshot of CREATE MATERIALIZED VIEW ... POPULATE both hand out the same object), so the token + /// must land on a new snapshot instead of being written into shared state. + EXPECT_NE(original.get(), pinned.get()); + + storage.reset(); + pinned.reset(); + + /// `original` is what such a cache would still be handing out, and it must be exactly as + /// non-owning as it was before the call. It is deliberately not dereferenced here. + EXPECT_TRUE(weak.expired()); +} + +TEST(StorageSnapshotStorageHolder, IsNonOwningWithoutAToken) +{ + auto storage = makeStorage("unpinned"); + std::weak_ptr weak = storage; + + auto snapshot = takeSnapshot(storage); + storage.reset(); + + /// The token is opt-in: a snapshot consumed inside the scope of an owning StoragePtr must not + /// delay the drop of its table. This is also the control for the assertions above. + EXPECT_TRUE(weak.expired()); +} + +TEST(StorageSnapshotStorageHolder, AliasChainKeepsTheInnermostReferentPinned) +{ + auto base = makeStorage("base"); + auto intermediate = makeStorage("intermediate_alias"); + std::weak_ptr weak_base = base; + std::weak_ptr weak_intermediate = intermediate; + const String expected_name = base->getStorageID().getFullTableName(); + + /// An `outer_alias -> inner_alias -> base` chain is supported (04946_storage_alias_attach_stored + /// _definition.sh), and it resolves innermost-first: the inner link pins the referent it took the + /// snapshot from, then the outer link offers its own target, which is the inner alias instead. + auto inner = takeSnapshot(base)->withStorageHolder(base); + auto outer = inner->withStorageHolder(intermediate); + + inner.reset(); + base.reset(); + intermediate.reset(); + + /// The first token wins, so the referent stays pinned. An unconditional overwrite would pin the + /// intermediate alias instead and leave the referent to be freed under the query. + ASSERT_FALSE(weak_base.expired()); + EXPECT_EQ(outer->storage.getStorageID().getFullTableName(), expected_name); + + /// The intermediate is dropped rather than kept, so a chain does not accumulate pins. + EXPECT_TRUE(weak_intermediate.expired()); +} + +TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetItHandsOut) +{ + auto context = getContext().context; + const String database_name = "test_storage_alias_holder_db"; + + auto database = std::make_shared(database_name, context); + DatabaseCatalog::instance().attachDatabase(database_name, database); + SCOPE_EXIT({ DatabaseCatalog::instance().detachDatabase(context, database_name, false, false); }); + + auto target = makeStorage(database_name, "target"); + std::weak_ptr weak_target = target; + database->attachTable(context, "target", target, {}); + target.reset(); + + /// An alias owns nothing and resolves its target through DatabaseCatalog on every access, so this + /// is the whole ownership state a query has when it reads through one. + auto alias = std::make_shared(StorageID{database_name, "alias"}, context, database_name, "target"); + auto snapshot = alias->getStorageSnapshot(std::make_shared(), context); + + /// Detaching releases the catalog's own StoragePtr, which is what DROP does before the background + /// worker destroys the table: DatabaseCatalog::getTablesToDrop() frees it as soon as no one else + /// owns it. So from here the snapshot's token is the only thing that can keep the target alive. + database->detachTable(context, "target"); + + ASSERT_FALSE(weak_target.expired()); + EXPECT_EQ(snapshot->storage.getStorageID().getFullTableName(), database_name + ".target"); + + snapshot.reset(); + EXPECT_TRUE(weak_target.expired()); +} diff --git a/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.reference b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.reference new file mode 100644 index 0000000000000..7856cdbc19c8d --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.reference @@ -0,0 +1,10 @@ +alias_chain_join +1 a A +2 b B +distributed_alias_join +1 a A +1 a A +2 b B +2 b B +distributed_alias_join_explain_ok +1 diff --git a/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql new file mode 100644 index 0000000000000..27f64e8d8d876 --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql @@ -0,0 +1,49 @@ +-- Tags: distributed + +-- An Alias hands out a storage snapshot of the table it points at, which the query neither owns nor +-- share-locks. Reading one inside a JOIN drives the join-reordering pass over that snapshot, and the +-- pass dereferences the storage behind it to build the hash-table cache key, so the snapshot has to +-- keep its referent alive. The Alias-of-Alias arm additionally covers a chain, where the snapshot +-- must stay bound to the innermost referent rather than to the intermediate Alias. + +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS outer_alias_05213; +DROP TABLE IF EXISTS inner_alias_05213; +DROP TABLE IF EXISTS dist_alias_05213; +DROP TABLE IF EXISTS dist_05213; +DROP TABLE IF EXISTS base_05213; +DROP TABLE IF EXISTS right_05213; + +CREATE TABLE base_05213 (k UInt32, v String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE right_05213 (k UInt32, w String) ENGINE = MergeTree ORDER BY k; + +-- CREATE rejects a target that already resolves to an Alias, so the chain is built target-last: +-- while inner_alias_05213 does not exist yet there is nothing to reject. +CREATE TABLE outer_alias_05213 ENGINE = Alias(currentDatabase(), inner_alias_05213); +CREATE TABLE inner_alias_05213 ENGINE = Alias(currentDatabase(), base_05213); + +CREATE TABLE dist_05213 AS base_05213 +ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), base_05213); +CREATE TABLE dist_alias_05213 ENGINE = Alias(currentDatabase(), dist_05213); + +INSERT INTO base_05213 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +INSERT INTO right_05213 VALUES (1, 'A'), (2, 'B'), (4, 'D'); + +SELECT 'alias_chain_join'; +SELECT l.k, l.v, r.w FROM outer_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k ORDER BY ALL; + +-- An Alias of a Distributed table is the shape that reached the freed storage: a Distributed table +-- in the query makes the planner build a throwaway analysis plan, which optimizes joins as well. +SELECT 'distributed_alias_join'; +SELECT l.k, l.v, r.w FROM dist_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k ORDER BY ALL; + +SELECT 'distributed_alias_join_explain_ok'; +SELECT count() > 0 FROM (EXPLAIN SELECT l.k, r.w FROM dist_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k); + +DROP TABLE outer_alias_05213; +DROP TABLE inner_alias_05213; +DROP TABLE dist_alias_05213; +DROP TABLE dist_05213; +DROP TABLE base_05213; +DROP TABLE right_05213; From dd25f85d22c054a542453ed3222ba37556d46c3e Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 11:08:56 +0200 Subject: [PATCH 45/52] Cover both `Alias` snapshot accessors and cloned snapshot data. `getStorageSnapshotWithoutData` had no pin test, and the data clone was only exercised with an empty snapshot. The stateless join now goes through a `View`, because an `Alias` of a `Distributed` table does not itself request filter collection. Related: https://github.com/Altinity/ClickHouse/pull/2557 Co-authored-by: Cursor --- src/Storages/StorageDistributed.h | 36 +++---- .../gtest_storage_snapshot_storage_holder.cpp | 99 ++++++++++++++++++- ...5213_storage_alias_chain_join_snapshot.sql | 32 ++++-- 3 files changed, 135 insertions(+), 32 deletions(-) diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index a81256b05f012..1a180f48d58ad 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -204,6 +204,24 @@ class StorageDistributed final : public IStorage, WithContext bool initializeDiskOnConfigChange(const std::set & new_added_disks) override; + /// Hybrid-specific snapshot-time state attached to `StorageSnapshot::data`. Populated + /// once in `StorageDistributed::getStorageSnapshot` so the watermark values seen by + /// `getQueryProcessingStage` and `read` cannot diverge mid-query under a concurrent + /// `ALTER MODIFY SETTING hybrid_watermark_*`. Without this, two independent + /// `MultiVersion::get` calls could observe different versions and produce inconsistent + /// pruning verdicts (e.g. a `Complete`-stage plan unioned without final merge). + struct HybridSnapshotData : public StorageSnapshot::Data + { + MultiVersion::Version watermark_snapshot; + + std::unique_ptr clone() const override + { + auto copy = std::make_unique(); + copy->watermark_snapshot = watermark_snapshot; + return copy; + } + }; + private: void renameOnDisk(const String & new_path_to_table_data); @@ -240,24 +258,6 @@ class StorageDistributed final : public IStorage, WithContext ASTPtr predicate_ast, const MultiVersion::Version & watermarks); - /// Hybrid-specific snapshot-time state attached to `StorageSnapshot::data`. Populated - /// once in `StorageDistributed::getStorageSnapshot()` so the watermark values seen by - /// `getQueryProcessingStage()` and `read()` cannot diverge mid-query under a concurrent - /// `ALTER MODIFY SETTING hybrid_watermark_*`. Without this, two independent - /// `MultiVersion::get()` calls could observe different versions and produce inconsistent - /// pruning verdicts (e.g. a `Complete`-stage plan unioned without final merge). - struct HybridSnapshotData : public StorageSnapshot::Data - { - MultiVersion::Version watermark_snapshot; - - std::unique_ptr clone() const override - { - auto copy = std::make_unique(); - copy->watermark_snapshot = watermark_snapshot; - return copy; - } - }; - /// Per-query Hybrid pruning verdict. Recomputed in both `getQueryProcessingStage()` /// (to drive the stage decision and empty `optimized_cluster` when the base is pruned) /// and `read()` (to skip planning of pruned additional segments). The verdict is diff --git a/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp index d2d0b922f0d35..1eded406ae419 100644 --- a/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp +++ b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp @@ -1,15 +1,22 @@ #include +#include #include #include +#include #include #include +#include #include +#include #include +#include #include #include #include +#include + /// StorageSnapshot refers to its storage without owning it, which is sound only while the caller that /// took the snapshot keeps a StoragePtr. StorageAlias breaks that: it hands out a snapshot of the table /// it points at, which its own caller neither owns nor locks, so the target could be dropped and @@ -139,10 +146,17 @@ TEST(StorageSnapshotStorageHolder, AliasChainKeepsTheInnermostReferentPinned) EXPECT_TRUE(weak_intermediate.expired()); } -TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetItHandsOut) +namespace +{ + +/// Both `StorageAlias` accessors have to pin, so the two arms below differ only in which one they call. +/// `take_snapshot` receives the alias rather than capturing it, because the alias must be constructed +/// after the target is attached. +void assertStorageAliasPinsItsTarget( + const String & database_name, + const std::function & take_snapshot) { auto context = getContext().context; - const String database_name = "test_storage_alias_holder_db"; auto database = std::make_shared(database_name, context); DatabaseCatalog::instance().attachDatabase(database_name, database); @@ -156,10 +170,10 @@ TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetItHandsOut) /// An alias owns nothing and resolves its target through DatabaseCatalog on every access, so this /// is the whole ownership state a query has when it reads through one. auto alias = std::make_shared(StorageID{database_name, "alias"}, context, database_name, "target"); - auto snapshot = alias->getStorageSnapshot(std::make_shared(), context); + auto snapshot = take_snapshot(*alias, context); /// Detaching releases the catalog's own StoragePtr, which is what DROP does before the background - /// worker destroys the table: DatabaseCatalog::getTablesToDrop() frees it as soon as no one else + /// worker destroys the table: DatabaseCatalog::getTablesToDrop frees it as soon as no one else /// owns it. So from here the snapshot's token is the only thing that can keep the target alive. database->detachTable(context, "target"); @@ -169,3 +183,80 @@ TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetItHandsOut) snapshot.reset(); EXPECT_TRUE(weak_target.expired()); } + +/// `withStorageHolder` must clone `data` rather than steal it: the source snapshot can already be +/// shared through the query metadata cache. +template +std::pair pinPopulatedSnapshot(const StoragePtr & storage, std::unique_ptr data) +{ + auto source = std::make_shared(*storage, std::make_shared(), std::move(data)); + auto pinned = source->withStorageHolder(storage); + return {std::move(source), std::move(pinned)}; +} + +} + +TEST(StorageSnapshotStorageHolder, WithStorageHolderClonesPopulatedData) +{ + auto storage = makeStorage("populated"); + + auto parts = std::make_shared(); + { + auto data = std::make_unique(); + data->storage = storage; + data->parts = parts; + auto [source, pinned] = pinPopulatedSnapshot(storage, std::move(data)); + + const auto * source_data = assert_cast(source->data.get()); + const auto * pinned_data = assert_cast(pinned->data.get()); + EXPECT_NE(source->data.get(), pinned->data.get()); + EXPECT_EQ(source_data->storage.get(), storage.get()); + EXPECT_EQ(pinned_data->storage.get(), source_data->storage.get()); + EXPECT_EQ(pinned_data->parts.get(), parts.get()); + EXPECT_EQ(pinned_data->mutations_snapshot.get(), source_data->mutations_snapshot.get()); + } + + auto blocks = std::make_shared(Blocks{Block{}}); + { + auto data = std::make_unique(); + data->blocks = blocks; + data->rows_approx = 3; + auto [source, pinned] = pinPopulatedSnapshot(storage, std::move(data)); + + const auto * source_data = assert_cast(source->data.get()); + const auto * pinned_data = assert_cast(pinned->data.get()); + EXPECT_NE(source->data.get(), pinned->data.get()); + EXPECT_EQ(source_data->blocks.get(), blocks.get()); + EXPECT_EQ(pinned_data->blocks.get(), source_data->blocks.get()); + EXPECT_EQ(pinned_data->rows_approx, source_data->rows_approx); + } + + auto watermarks = std::make_shared(StorageDistributed::WatermarkParams{{"ts", "1"}}); + { + auto data = std::make_unique(); + data->watermark_snapshot = watermarks; + auto [source, pinned] = pinPopulatedSnapshot(storage, std::move(data)); + + const auto * source_data = assert_cast(source->data.get()); + const auto * pinned_data = assert_cast(pinned->data.get()); + EXPECT_NE(source->data.get(), pinned->data.get()); + EXPECT_EQ(source_data->watermark_snapshot.get(), watermarks.get()); + EXPECT_EQ(pinned_data->watermark_snapshot.get(), source_data->watermark_snapshot.get()); + } +} + +TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetItHandsOut) +{ + assertStorageAliasPinsItsTarget( + "test_storage_alias_holder_db", + [](const StorageAlias & alias, ContextPtr context) + { return alias.getStorageSnapshot(std::make_shared(), context); }); +} + +TEST(StorageSnapshotStorageHolder, StorageAliasPinsTheTargetForASnapshotWithoutData) +{ + assertStorageAliasPinsItsTarget( + "test_storage_alias_holder_without_data_db", + [](const StorageAlias & alias, ContextPtr context) + { return alias.getStorageSnapshotWithoutData(std::make_shared(), context); }); +} diff --git a/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql index 27f64e8d8d876..01d3ba90bc5f8 100644 --- a/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql +++ b/tests/queries/0_stateless/05213_storage_alias_chain_join_snapshot.sql @@ -1,10 +1,15 @@ -- Tags: distributed - --- An Alias hands out a storage snapshot of the table it points at, which the query neither owns nor --- share-locks. Reading one inside a JOIN drives the join-reordering pass over that snapshot, and the --- pass dereferences the storage behind it to build the hash-table cache key, so the snapshot has to --- keep its referent alive. The Alias-of-Alias arm additionally covers a chain, where the snapshot --- must stay bound to the innermost referent rather than to the intermediate Alias. +-- Random settings limits: query_plan_optimize_join_order_limit=(1, None) + +-- An `Alias` hands out a storage snapshot of the table it points at, while the table expression that +-- receives that snapshot owns and share-locks the `Alias` rather than the target. Filter collection is +-- decided on that table expression's storage, so an `Alias` of a `Distributed` table does not by itself +-- build the analysis plan. The `View` on the right side does, and once any table expression asks for +-- it the whole query, including the `Alias`, is planned through `StorageDummy`. These queries are smoke +-- coverage of that shape. Freeing the target under a live plan needs a race with the background drop +-- worker, so the use-after-free itself is asserted by the unit test rather than here. +-- The Alias-of-Alias arm additionally covers a chain, where the token must stay bound to the innermost +-- referent rather than to the intermediate Alias. SET enable_analyzer = 1; @@ -12,6 +17,7 @@ DROP TABLE IF EXISTS outer_alias_05213; DROP TABLE IF EXISTS inner_alias_05213; DROP TABLE IF EXISTS dist_alias_05213; DROP TABLE IF EXISTS dist_05213; +DROP VIEW IF EXISTS view_right_05213; DROP TABLE IF EXISTS base_05213; DROP TABLE IF EXISTS right_05213; @@ -27,23 +33,29 @@ CREATE TABLE dist_05213 AS base_05213 ENGINE = Distributed('test_cluster_two_shards_localhost', currentDatabase(), base_05213); CREATE TABLE dist_alias_05213 ENGINE = Alias(currentDatabase(), dist_05213); +-- Filter collection is decided per table expression, and an Alias is neither a Distributed nor a View +-- to that decision even when its target is, so the query needs a second table expression that asks for +-- it. Once any one does, the whole query is analysed through dummy tables, the Alias included. +CREATE VIEW view_right_05213 AS SELECT k, w FROM right_05213; + INSERT INTO base_05213 VALUES (1, 'a'), (2, 'b'), (3, 'c'); INSERT INTO right_05213 VALUES (1, 'A'), (2, 'B'), (4, 'D'); SELECT 'alias_chain_join'; SELECT l.k, l.v, r.w FROM outer_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k ORDER BY ALL; --- An Alias of a Distributed table is the shape that reached the freed storage: a Distributed table --- in the query makes the planner build a throwaway analysis plan, which optimizes joins as well. +-- An Alias of a Distributed table is the shape that reached the freed storage, so this is the arm that +-- carries an Alias-produced snapshot of a Distributed into the join-reordering pass. SELECT 'distributed_alias_join'; -SELECT l.k, l.v, r.w FROM dist_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k ORDER BY ALL; +SELECT l.k, l.v, r.w FROM dist_alias_05213 AS l JOIN view_right_05213 AS r ON l.k = r.k ORDER BY ALL; SELECT 'distributed_alias_join_explain_ok'; -SELECT count() > 0 FROM (EXPLAIN SELECT l.k, r.w FROM dist_alias_05213 AS l JOIN right_05213 AS r ON l.k = r.k); +SELECT count() > 0 FROM (EXPLAIN SELECT l.k, r.w FROM dist_alias_05213 AS l JOIN view_right_05213 AS r ON l.k = r.k); DROP TABLE outer_alias_05213; DROP TABLE inner_alias_05213; DROP TABLE dist_alias_05213; DROP TABLE dist_05213; +DROP VIEW view_right_05213; DROP TABLE base_05213; DROP TABLE right_05213; From fc346d8ed81b8c0d9d38c1abb2ab542617523ca5 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 13 Oct 2026 11:43:40 +0200 Subject: [PATCH 46/52] Give the MergeTree snapshot clone a real `mutations_snapshot`. A null snapshot made the pointer comparison pass even when `clone` dropped the field. Related: https://github.com/Altinity/ClickHouse/pull/2557 Co-authored-by: Cursor --- .../gtest_storage_snapshot_storage_holder.cpp | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp index 1eded406ae419..c7a2470b36c21 100644 --- a/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp +++ b/src/Storages/tests/gtest_storage_snapshot_storage_holder.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -184,6 +185,60 @@ void assertStorageAliasPinsItsTarget( EXPECT_TRUE(weak_target.expired()); } +/// `clone` only copies the `shared_ptr`. These virtuals are never called; the object exists so the +/// fixture can hold a non-null `mutations_snapshot`. +struct MutationsSnapshotForCloneTest final : MergeTreeData::IMutationsSnapshot +{ + void addPatches(MergeTreeData::DataPartsVector) override + { + } + + MutationCommands getOnFlyMutationCommandsForPart(const MergeTreeData::DataPartPtr &) const override + { + return {}; + } + + PatchParts getPatchesForPart(const MergeTreeData::DataPartPtr &) const override + { + return {}; + } + + std::shared_ptr cloneEmpty() const override + { + return std::make_shared(); + } + + NameSet getAllUpdatedColumns() const override + { + return {}; + } + + bool hasPatchParts() const override + { + return false; + } + + bool hasDataMutations() const override + { + return false; + } + + bool hasAlterMutations() const override + { + return false; + } + + bool hasMetadataMutations() const override + { + return false; + } + + bool hasLightweightDeletedMask() const override + { + return false; + } +}; + /// `withStorageHolder` must clone `data` rather than steal it: the source snapshot can already be /// shared through the query metadata cache. template @@ -201,10 +256,13 @@ TEST(StorageSnapshotStorageHolder, WithStorageHolderClonesPopulatedData) auto storage = makeStorage("populated"); auto parts = std::make_shared(); + /// A default-null `mutations_snapshot` would make the pointer comparison pass even if `clone` dropped the field. + MergeTreeData::MutationsSnapshotPtr mutations = std::make_shared(); { auto data = std::make_unique(); data->storage = storage; data->parts = parts; + data->mutations_snapshot = mutations; auto [source, pinned] = pinPopulatedSnapshot(storage, std::move(data)); const auto * source_data = assert_cast(source->data.get()); @@ -213,7 +271,8 @@ TEST(StorageSnapshotStorageHolder, WithStorageHolderClonesPopulatedData) EXPECT_EQ(source_data->storage.get(), storage.get()); EXPECT_EQ(pinned_data->storage.get(), source_data->storage.get()); EXPECT_EQ(pinned_data->parts.get(), parts.get()); - EXPECT_EQ(pinned_data->mutations_snapshot.get(), source_data->mutations_snapshot.get()); + EXPECT_EQ(source_data->mutations_snapshot.get(), mutations.get()); + EXPECT_EQ(pinned_data->mutations_snapshot.get(), mutations.get()); } auto blocks = std::make_shared(Blocks{Block{}}); From a58cb450ad52fecb86c37c602cda7aede6bbf0bc Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 14 Oct 2026 00:10:09 +0200 Subject: [PATCH 47/52] Remove CAS from generic native copy test --- tests/integration/test_backup_restore_s3/test.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 564c62607fd5f..f69cc50fe41ad 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -553,7 +553,6 @@ def test_backup_to_s3_multipart(cluster): "policy_s3", "policy_s3_other_bucket", "policy_s3_plain_rewritable", - "policy_cas", ], ) def test_backup_to_s3_native_copy(cluster, storage_policy): @@ -562,20 +561,6 @@ def test_backup_to_s3_native_copy(cluster, storage_policy): (backup_events, restore_events) = check_backup_and_restore( cluster, storage_policy, backup_destination ) - - if storage_policy in CAS_POLICIES: - assert ( - backup_events.get("S3CopyObject", 0) - + backup_events.get("S3UploadPartCopy", 0) - > 0 - ) - assert ( - restore_events.get("S3CopyObject", 0) - + restore_events.get("S3UploadPartCopy", 0) - == 0 - ) - return - # single part upload assert backup_events["S3CopyObject"] > 0 assert restore_events["S3CopyObject"] > 0 From fd6d02fd72649732cbbd09c4bd5b5d4b8d86078b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 14 Oct 2026 11:36:02 +0200 Subject: [PATCH 48/52] Exclude CAS logs from backup all test --- tests/integration/test_backup_restore_new/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 6c8a5124afca5..5f856a4d7a2bd 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1765,6 +1765,8 @@ def test_backup_all(exclude_system_log_tables): "query_metric_log", "query_thread_log", "part_log", + "cas_gc_log", + "cas_log", "trace_log", "crash_log", "text_log", From 43952dafab9f5b1926d9d81f35fdbe28d635b181 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 14 Oct 2026 11:44:43 +0200 Subject: [PATCH 49/52] Keep the unwrapped storage alive in `parallelReplicasEnabledForStorage`. `getTargetTable` returns a `StoragePtr` that can be the only owner of a catalog storage. Destroying it before `isMergeTree` was a heap-use-after-free under AddressSanitizer. Related: https://github.com/Altinity/ClickHouse/pull/2550 https: //altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2550&sha=bb444f10dd1de59c9ae3adcf33540371cedf6c5b&name_0=PR&name_1=Integration%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20old%20analyzer%2C%208%2F8%29 Co-authored-by: Cursor --- src/Planner/PlannerJoinTree.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 1633d044ab470..c67eca26efccf 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -991,13 +991,16 @@ std::unique_ptr createComputeAliasColumnsStep( bool parallelReplicasEnabledForStorage(const StoragePtr & current_storage, const ContextPtr & context, const Settings & query_settings) { const auto * table_ptr = current_storage.get(); + /// `getTargetTable` and `getUnderlyingMergeTreeStorageForParallelReplicas` return a `StoragePtr` by value. + /// A catalog lookup can be the only owner of that storage, so it has to stay alive for the checks below. + StoragePtr underlying_storage; if (query_settings[Setting::parallel_replicas_allow_view_over_mergetree]) { const auto * view = typeid_cast(current_storage.get()); if (view) { - auto underlying_storage = view->getUnderlyingMergeTreeStorageForParallelReplicas(context); + underlying_storage = view->getUnderlyingMergeTreeStorageForParallelReplicas(context); if (!underlying_storage) return false; @@ -1015,7 +1018,8 @@ bool parallelReplicasEnabledForStorage(const StoragePtr & current_storage, const if (mv->isRefreshable()) return false; - table_ptr = mv->getTargetTable().get(); + underlying_storage = mv->getTargetTable(); + table_ptr = underlying_storage.get(); } if (!table_ptr->isMergeTree()) From 20d68b0d4aea62a8977f154edd004e553fc5d034 Mon Sep 17 00:00:00 2001 From: MyroTk <44327070+MyroTk@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:45:27 -0400 Subject: [PATCH 50/52] Bump version to 26.6.8.20002.altinityantalya --- cmake/autogenerated_versions.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index a7a9560604f35..61a2688755d4d 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -7,10 +7,10 @@ SET(VERSION_MAJOR 26) SET(VERSION_MINOR 6) SET(VERSION_PATCH 8) SET(VERSION_GITHASH bc7f520bc366fac69d3ff7323fecc35197a9b2c6) -SET(VERSION_DESCRIBE v26.6.8.20001.altinityantalya) -SET(VERSION_STRING 26.6.8.20001.altinityantalya) +SET(VERSION_DESCRIBE v26.6.8.20002.altinityantalya) +SET(VERSION_STRING 26.6.8.20002.altinityantalya) # end of autochange -SET(VERSION_TWEAK 20001) +SET(VERSION_TWEAK 20002) SET(VERSION_FLAVOUR altinityantalya) From b302425b5dd51c471c6ed7574b39b6170e7ca7dc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Oct 2026 08:55:48 -0300 Subject: [PATCH 51/52] add integ tests --- .../test.py | 91 ++++++++++++++++++ .../test_schema_match.py | 92 +++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py index 053dec9670364..990493de451df 100644 --- a/tests/integration/test_export_merge_tree_part_to_iceberg/test.py +++ b/tests/integration/test_export_merge_tree_part_to_iceberg/test.py @@ -1220,6 +1220,97 @@ def test_export_part_with_castable_widening(cluster): node.query(f"DROP TABLE IF EXISTS {iceberg}") +def test_export_part_with_unsigned_to_wider_signed_widening(cluster): + """An unsigned column exported into a strictly wider signed column (UInt8 -> Int16, + UInt16 -> Int32, UInt32 -> Int64, Nullable(UInt8) -> Nullable(Int16)) is a lossless + cast, so it is accepted without export_merge_tree_part_allow_lossy_cast and the + maximum values of the source types round-trip unchanged.""" + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_unsigned_widen_{sfx}" + iceberg = f"iceberg_unsigned_widen_{sfx}" + + make_mt(node, mt, "u8 UInt8, u16 UInt16, u32 UInt32, opt Nullable(UInt8), year Int32", "year") + make_iceberg_s3(node, iceberg, "u8 Int16, u16 Int32, u32 Int64, opt Nullable(Int16), year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (255, 65535, 4294967295, 255, 2020), (0, 0, 0, NULL, 2020)") + part_2020 = get_part(node, mt, "2020") + + export_part(node, mt, part_2020, iceberg) + wait_for_export_part(node, mt, part_2020) + + result = node.query(f"SELECT u8, u16, u32, opt, year FROM {iceberg} ORDER BY u8").strip() + assert result == "0\t0\t0\t\\N\t2020\n255\t65535\t4294967295\t255\t2020", ( + f"Unexpected data after unsigned -> wider signed export:\n{result}" + ) + + assert_part_log(node, mt, part_2020) + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +def test_export_part_unsigned_partition_column_widened_to_signed(cluster): + """A partition column that is unsigned in the source (year UInt16) and a strictly wider + signed integer in the destination (year Int32) passes the lossy-cast gate without the + opt-in, and the partition value is preserved.""" + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_unsigned_pcol_{sfx}" + iceberg = f"iceberg_unsigned_pcol_{sfx}" + + make_mt(node, mt, "id Int32, year UInt16", "year") + make_iceberg_s3(node, iceberg, "id Int32, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020), (2, 2020)") + part_2020 = get_part(node, mt, "2020") + + export_part(node, mt, part_2020, iceberg) + wait_for_export_part(node, mt, part_2020) + + result = node.query(f"SELECT id, year FROM {iceberg} ORDER BY id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data after unsigned partition column export:\n{result}" + + assert_part_log(node, mt, part_2020) + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + +@pytest.mark.parametrize( + "source_type,destination_type", + [("UInt16", "Int16"), ("UInt32", "Int32"), ("UInt64", "Int64")], +) +def test_export_part_unsigned_to_same_size_signed_rejected_without_optin(cluster, source_type, destination_type): + """An unsigned column exported into a signed column of the same size is lossy (values above + the signed maximum wrap), so it is rejected synchronously with INCOMPATIBLE_COLUMNS unless + export_merge_tree_part_allow_lossy_cast is set.""" + node = cluster.instances["node1"] + sfx = unique_suffix() + mt = f"mt_unsigned_same_size_{sfx}" + iceberg = f"iceberg_unsigned_same_size_{sfx}" + + make_mt(node, mt, f"id {source_type}, year Int32", "year") + make_iceberg_s3(node, iceberg, f"id {destination_type}, year Int32", "year") + + node.query(f"INSERT INTO {mt} VALUES (1, 2020)") + part_2020 = get_part(node, mt, "2020") + + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PART '{part_2020}' TO TABLE {iceberg} " + f"SETTINGS allow_experimental_export_merge_tree_part = 1, " + f"allow_experimental_insert_into_iceberg = 1" + ) + assert "INCOMPATIBLE_COLUMNS" in error, f"Expected INCOMPATIBLE_COLUMNS, got: {error!r}" + assert "lossy cast" in error, f"Expected 'lossy cast' in error, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg}").strip()) + assert count == 0, f"Expected no rows after a rejected export, got {count}" + + node.query(f"DROP TABLE IF EXISTS {mt} SYNC") + node.query(f"DROP TABLE IF EXISTS {iceberg}") + + def test_export_part_with_castable_narrowing_values_fit(cluster): """A lossy narrowing (id Int64 -> Int32) succeeds once the user opts in via export_merge_tree_part_allow_lossy_cast.""" diff --git a/tests/integration/test_export_partition_to_iceberg/test_schema_match.py b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py index 755b1095816c0..32de3888ec755 100644 --- a/tests/integration/test_export_partition_to_iceberg/test_schema_match.py +++ b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py @@ -628,6 +628,98 @@ def test_export_partition_with_castable_widening(cluster, source_engine): ) +def test_export_partition_with_unsigned_to_wider_signed_widening(cluster, source_engine): + """An unsigned column exported into a strictly wider signed column (UInt8 -> Int16, + UInt16 -> Int32, UInt32 -> Int64, Nullable(UInt8) -> Nullable(Int16)) is a lossless + cast, so it is accepted without export_merge_tree_part_allow_lossy_cast and the + maximum values of the source types round-trip unchanged.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_unsigned_widen_{uid}" + iceberg_table = f"iceberg_unsigned_widen_{uid}" + + make_source( + node, mt_table, "u8 UInt8, u16 UInt16, u32 UInt32, opt Nullable(UInt8), year Int32", "year", + replica_name="replica1", engine=source_engine, + ) + node.query(f"INSERT INTO {mt_table} VALUES (255, 65535, 4294967295, 255, 2020), (0, 0, 0, NULL, 2020)") + + make_iceberg_s3( + node, iceberg_table, "u8 Int16, u16 Int32, u32 Int64, opt Nullable(Int16), year Int32", partition_by="year" + ) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + result = node.query(f"SELECT u8, u16, u32, opt, year FROM {iceberg_table} ORDER BY u8").strip() + assert result == "0\t0\t0\t\\N\t2020\n255\t65535\t4294967295\t255\t2020", ( + f"Unexpected data after unsigned -> wider signed export:\n{result}" + ) + + +def test_export_partition_unsigned_partition_column_widened_to_signed(cluster, source_engine): + """A partition column that is unsigned in the source (year UInt16) and a strictly wider + signed integer in the destination (year Int32) passes the lossy-cast gate without the + opt-in, and the partition value is preserved.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_unsigned_pcol_{uid}" + iceberg_table = f"iceberg_unsigned_pcol_{uid}" + + make_source(node, mt_table, "id Int32, year UInt16", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + result = node.query(f"SELECT id, year FROM {iceberg_table} ORDER BY id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data after unsigned partition column export:\n{result}" + + +@pytest.mark.parametrize( + "source_type,destination_type", + [("UInt16", "Int16"), ("UInt32", "Int32"), ("UInt64", "Int64")], +) +def test_export_partition_unsigned_to_same_size_signed_rejected_without_optin( + cluster, source_engine, source_type, destination_type +): + """An unsigned column exported into a signed column of the same size is lossy (values above + the signed maximum wrap), so it is rejected synchronously with INCOMPATIBLE_COLUMNS unless + export_merge_tree_part_allow_lossy_cast is set.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_unsigned_same_size_{uid}" + iceberg_table = f"iceberg_unsigned_same_size_{uid}" + + make_source( + node, mt_table, f"id {source_type}, year Int32", "year", replica_name="replica1", engine=source_engine + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020)") + + make_iceberg_s3(node, iceberg_table, f"id {destination_type}, year Int32", partition_by="year") + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} " + f"SETTINGS allow_insert_into_iceberg = 1" + ) + assert "INCOMPATIBLE_COLUMNS" in error, f"Expected INCOMPATIBLE_COLUMNS, got: {error!r}" + assert "lossy cast" in error, f"Expected 'lossy cast' in error, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected no rows after a rejected export, got {count}" + + def test_export_partition_with_castable_narrowing_values_fit(cluster, source_engine): """A lossy narrowing (id Int64 -> Int32) succeeds once the user opts in via export_merge_tree_part_allow_lossy_cast.""" From 47ef4b9bfdd16c49fb6a42909ca9056c653ddd87 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Fri, 16 Oct 2026 11:20:13 -0400 Subject: [PATCH 52/52] Nightly runs skip in-progress runs --- .github/workflows/scheduled_runs.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scheduled_runs.yml b/.github/workflows/scheduled_runs.yml index 24d403f2293be..579f35015529b 100644 --- a/.github/workflows/scheduled_runs.yml +++ b/.github/workflows/scheduled_runs.yml @@ -68,17 +68,33 @@ jobs: --jq '.[0].sha') echo "Tip of ${BRANCH}: ${tip}" + unfinished=0 + unfinished_ids="" + for status in in_progress queued waiting requested pending; do + payload=$(gh api -X GET \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -f branch="$BRANCH" \ + -f status="$status" \ + -F per_page=5 \ + "/repos/Altinity/ClickHouse/actions/workflows/${WORKFLOW}/runs") + count=$(jq '.total_count // 0' <<<"$payload") + ids=$(jq -r '[.workflow_runs // [] | .[].id | tostring] | join(" ")' <<<"$payload") + echo "Unfinished ${WORKFLOW} on ${BRANCH} (${status}): ${count}" + unfinished=$((unfinished + count)) + if [ -n "$ids" ]; then + unfinished_ids="${unfinished_ids} ${ids}" + fi + done + runs=$(gh api -X GET \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ -f branch="$BRANCH" \ + -f status=completed \ -F per_page=10 \ "/repos/Altinity/ClickHouse/actions/workflows/${WORKFLOW}/runs") - active=$(jq ' - [.workflow_runs[] - | select(.status == "queued" or .status == "in_progress" or .status == "waiting" or .status == "requested" or .status == "pending")] - | length' <<<"$runs") latest_sha=$(jq -r '.workflow_runs[0].head_sha // empty' <<<"$runs") latest_id=$(jq -r '.workflow_runs[0].id // empty' <<<"$runs") latest_status=$(jq -r '.workflow_runs[0].status // empty' <<<"$runs") @@ -86,8 +102,8 @@ jobs: dispatch=false previous_run_id="" - if [ "$active" -gt 0 ]; then - echo "${WORKFLOW} already has an unfinished run on ${BRANCH}" + if [ "$unfinished" -gt 0 ]; then + echo "${WORKFLOW} already has an unfinished run on ${BRANCH}:${unfinished_ids}" elif [ -z "$latest_sha" ]; then echo "No previous ${WORKFLOW} run on ${BRANCH}" dispatch=true