diff --git a/.github/workflows/regression-reusable-suite.yml b/.github/workflows/regression-reusable-suite.yml index 233ec08d26d0f..51d064ea0f011 100644 --- a/.github/workflows/regression-reusable-suite.yml +++ b/.github/workflows/regression-reusable-suite.yml @@ -93,6 +93,7 @@ jobs: build_sha: ${{ inputs.build_sha }} pr_number: ${{ github.event.number }} artifacts: builds + regression_args: ${{ inputs.regression_args }} # Args args: --test-to-end --no-colors diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 7d0935d60befe..ffcd7283f0808 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -126,7 +126,7 @@ jobs: strategy: fail-fast: false matrix: - SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] + SUITE: [aes_encryption, atomic_insert, attach, base_58, clickhouse_keeper_failover,data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, jwt_authentication, kafka, kerberos, key_value, memory, part_moves_between_shards, selects, session_timezone, settings, version, window_functions] uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -143,6 +143,40 @@ jobs: job_name: ${{ matrix.SUITE }} secrets: inherit + LightweightDelete: + if: | + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'common') + strategy: + fail-fast: false + matrix: + include: + - PART: 1 + FILTER: --only "/lightweight delete/disk space/*" + - PART: 2 + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: lightweight_delete + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 300 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: lightweight_delete + extra_args: ${{ matrix.FILTER }} + secrets: inherit + AggregateFunctions: if: | fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || @@ -181,6 +215,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -189,7 +225,7 @@ jobs: suite_executable: regression.py output_format: new-fails flags: --with-analyzer - timeout_minutes: ${{ inputs.timeout_minutes }} + timeout_minutes: 300 runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} storage_path: /${{ matrix.ONLY }}_partition @@ -534,10 +570,56 @@ jobs: secrets: inherit # CAS (content-addressed storage) is only available in Antalya builds >= 26.6. - # These jobs run the suites that support the `--cas` or `--cas-s3-cache` flag with - # a content-addressed disk as the default MergeTree storage. Gated on the Antalya - # version so they are never scheduled on upstream release builds where the `cas` - # metadata type does not exist. + # The dedicated `cas` suite, plus other suites run with `--cas` or `--cas-s3-cache`. + # Gated on the Antalya version so they are never scheduled on upstream release + # builds where the `cas` metadata type does not exist. + CASSuite: + if: | + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && + ( + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'cas') + ) + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: cas + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 180 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: cas + secrets: inherit + + CASS3CacheSuite: + if: | + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && + ( + fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).workflow_config.custom_data.ci_regression_jobs, 'cas') + ) + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: cas + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: 180 + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: cas_s3_cache + regression_args: --cas-s3-cache + secrets: inherit + CAS: if: | contains(fromJson(inputs.workflow_config).workflow_config.custom_data.version.string, 'antalya') && @@ -668,7 +750,11 @@ jobs: - PART: 1 FILTER: --only "/lightweight delete/disk space/*" - PART: 2 - FILTER: --skip "/lightweight delete/disk space/*" + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -702,7 +788,11 @@ jobs: - PART: 1 FILTER: --only "/lightweight delete/disk space/*" - PART: 2 - FILTER: --skip "/lightweight delete/disk space/*" + FILTER: --only "/lightweight delete/hard restart/*" + - PART: 3 + FILTER: --skip "/lightweight delete/disk space/*" "/lightweight delete/hard restart/*" "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" + - PART: 4 + FILTER: --only "/lightweight delete/alter after delete/*" "/lightweight delete/concurrent alter and delete/*" "/lightweight delete/concurrent delete/*" "/lightweight delete/basic checks/*" "/lightweight delete/performance/*" "/lightweight delete/ontime tests/*" uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -738,6 +828,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -750,6 +842,7 @@ jobs: runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} part: ${{ matrix.PART }} + storage_path: /${{ matrix.ONLY }}_partition build_sha: ${{ inputs.build_sha }} set_commit_status: true job_name: cas_alter_${{ matrix.ONLY }} @@ -773,6 +866,8 @@ jobs: PART: 1 - ONLY: attach PART: 2 + - ONLY: attach + PART: 3 uses: ./.github/workflows/regression-reusable-suite.yml with: ref: ${{ inputs.commit }} @@ -785,6 +880,7 @@ jobs: runner_arch: ${{ inputs.arch }} runner_type: ${{ inputs.runner_type }} part: ${{ matrix.PART }} + storage_path: /${{ matrix.ONLY }}_partition build_sha: ${{ inputs.build_sha }} set_commit_status: true job_name: cas_s3_cache_alter_${{ matrix.ONLY }} diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 1aa1d56cc648c..522e3031d305f 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -163,7 +163,8 @@ def started_cluster(): "c2.s0_0_1", main_configs=["configs/cluster.xml", "configs/named_collections.xml", "configs/hidden_clusters.xml"], user_configs=["configs/users.xml"], - macros={"replica": "replica2", "shard": "shard1"}, + # Must be unique per instance: the database disk prefix is .../test-{shard}|{replica} + macros={"replica": "replica2_c2", "shard": "shard1"}, with_zookeeper=True, stay_alive=True, ) diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh index fc5df9b541da8..804b1849d3165 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_basic.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + mt_table="mt_table_${RANDOM}" mt_table_partition_expression_with_function="mt_table_partition_expression_with_function_${RANDOM}" s3_table="s3_table_${RANDOM}" @@ -46,8 +49,8 @@ echo "---- Export 3: Export 2020_1_1_0 and 2021_2_2_0 to wildcard table with par query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART 'cb217c742dc7d143b61583011996a160_1_1_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" query "ALTER TABLE $mt_table_partition_expression_with_function EXPORT PART '3be6d49ecf9749a383964bc6fab22d10_2_2_0' TO TABLE $s3_table_wildcard_partition_expression_with_function SETTINGS allow_experimental_export_merge_tree_part = 1" -# below exports are using parts that were exported in export 1 and export 2, so we need to wait for them to complete -sleep 5 +# wait until part_log has 6 ExportPart records +wait_for_exports 6 echo "---- Export 4: Export the same part again, it should be idempotent" query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" @@ -55,8 +58,8 @@ query "ALTER TABLE $mt_table EXPORT PART '2020_1_1_0' TO TABLE $s3_table SETTING echo "---- Export 5: Export the same part again to wildcard, it should be idempotent" query "ALTER TABLE $mt_table EXPORT PART '2022_3_3_0' TO TABLE $s3_table_wildcard SETTINGS allow_experimental_export_merge_tree_part = 1" -# ONE BIG SLEEP after all exports -sleep 15 +# wait until part_log has 8 ExportPart records (6 from the previous exports) +wait_for_exports 8 # ============================================================================ # ALL SELECTS/VERIFICATIONS HAPPEN HERE diff --git a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh index 0164dd70c4e01..1dc66f49e66bc 100755 --- a/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh +++ b/tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + mt_alias="mt_alias_${RANDOM}" mt_materialized="mt_materialized_${RANDOM}" s3_alias_export="s3_alias_export_${RANDOM}" @@ -112,7 +115,8 @@ query "ALTER TABLE $mt_mixed EXPORT PART '$mixed_part_2' TO TABLE FUNCTION s3(s3 query "ALTER TABLE $mt_complex_expr EXPORT PART '$complex_expr_part' TO TABLE $s3_complex_expr_export SETTINGS allow_experimental_export_merge_tree_part = 1" # ONE BIG SLEEP after all exports -sleep 20 +# wait until part_log has 6 ExportPart records +wait_for_exports 6 # ============================================================================ # ALL SELECTS/VERIFICATIONS HAPPEN HERE diff --git a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh index 12b47f4f26648..fa8b1c7c4dafa 100755 --- a/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh +++ b/tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh @@ -6,6 +6,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +# shellcheck source=./export_part.lib +. "$CURDIR"/export_part.lib + R=$RANDOM mt="mt_${R}" dest1="fp_dest1_${R}" @@ -27,21 +30,24 @@ query "CREATE TABLE $dest3 (id UInt64, year UInt16) ENGINE = S3(s3_conn, filenam echo "---- Test: Default pattern {part_name}_{checksum}" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest1 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{part_name}_{checksum}'" -sleep 3 +# wait until part_log has 1 ExportPart record +wait_for_exports 1 query "SELECT * FROM $dest1 ORDER BY id" echo "---- Verify filename matches 2020_1_1_0_*.1.parquet" query "SELECT count() FROM s3(s3_conn, filename='$dest1/**/2020_1_1_0_*.1.parquet', format='One')" echo "---- Test: Custom prefix pattern" query "ALTER TABLE $mt EXPORT PART '2021_2_2_0' TO TABLE $dest2 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = 'myprefix_{part_name}'" -sleep 3 +# wait until part_log has 2 ExportPart records (1 from the previous export) +wait_for_exports 2 query "SELECT * FROM $dest2 ORDER BY id" echo "---- Verify filename matches myprefix_2021_2_2_0.1.parquet" query "SELECT count() FROM s3(s3_conn, filename='$dest2/**/myprefix_2021_2_2_0.1.parquet', format='One')" echo "---- Test: Pattern with macros" query "ALTER TABLE $mt EXPORT PART '2020_1_1_0' TO TABLE $dest3 SETTINGS allow_experimental_export_merge_tree_part = 1, export_merge_tree_part_filename_pattern = '{database}_{table}_{part_name}'" -sleep 3 +# wait until part_log has 3 ExportPart records (2 from the previous exports) +wait_for_exports 3 query "SELECT * FROM $dest3 ORDER BY id" echo "---- Verify macros expanded (no literal braces in parquet filenames, that's the best we can do for stateless tests)" query "SELECT count() = 0 FROM s3(s3_conn, filename='$dest3/**/*.1.parquet', format='One') WHERE _file LIKE '%{%'" diff --git a/tests/queries/0_stateless/export_part.lib b/tests/queries/0_stateless/export_part.lib new file mode 100644 index 0000000000000..453bef86410d7 --- /dev/null +++ b/tests/queries/0_stateless/export_part.lib @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# EXPORT PART runs in the background, so wait for it instead of sleeping. +# $1 is how many exports should have completed by this point, counted from the start of +# the test (part_log is not cleared during a run, so the number is cumulative). +function wait_for_exports() +{ + local expected=$1 + local database=${2:-$CLICKHOUSE_DATABASE} + local timeout=${3:-120} + + while [[ $timeout -gt 0 ]] + do + ${CLICKHOUSE_CLIENT} --query="SYSTEM FLUSH LOGS" > /dev/null 2>&1 + res=$(${CLICKHOUSE_CLIENT} --query="SELECT count() FROM system.part_log WHERE event_type = 'ExportPart' AND database = '$database'") + [[ $res -ge $expected ]] && return 0 + + sleep 1 + timeout=$((timeout - 1)) + done + + echo "Timed out while waiting for $expected export(s) to complete!" >&2 + ${CLICKHOUSE_CLIENT} --query="SELECT source_table, part_name, elapsed FROM system.exports WHERE source_database = '$database'" + return 2 +} + +# vi: ft=bash