From 451461694b20f012e921625270851544e0312c17 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 12:53:08 +0530 Subject: [PATCH 1/6] adding iterations for perf test --- .github/workflows/test-runner.yml | 39 +++++++++++++++--- .github/workflows/windows-benchmark.yml | 55 ++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b3aa1e7dbd..2f4cea2af2 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -114,6 +114,14 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-benchmark-existing cancel-in-progress: true + # Runs *after* windows-benchmark-bq rather than alongside it. Both shards + # benchmark against the same BigQuery project, so running them concurrently + # made each one's numbers depend on the other's API load -- and when that + # tripped rate limits, retry backoff turned the contention into + # multi-second steps. The dependency deliberately does not require the + # BqDriver shard to succeed: '!cancelled()' plus the windows-cmake guard + # preserves the previous run conditions, so the existing-driver baseline is + # still produced when that shard fails or is skipped. if: | !cancelled() && github.event_name == 'workflow_dispatch' && @@ -169,9 +177,19 @@ jobs: import re def parse_gtest_output(filepath): - results = {} + # The suite is run several times (--gtest_repeat), so each test + # appears once per repetition. Take the median of its timings. + # + # These benchmarks talk to a live BigQuery service and fan list + # calls out concurrently, so a single run samples the latency + # tail and moved tens of percent between runs for no code + # reason. The median rejects a single outlier repetition, + # including the cold-cache first one. Files with only one run per + # test (a --gtest_repeat=1 run, or a baseline recorded before + # this change) still work: the median of one sample is itself. + samples = {} if not os.path.exists(filepath): - return results + return {} pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') try: @@ -179,11 +197,22 @@ jobs: for line in f: match = pattern.search(line) if match: - test_name = match.group(1) - time_taken = match.group(2) - results[test_name] = time_taken + ms = parse_time_to_ms(match.group(2)) + if ms is not None: + samples.setdefault(match.group(1), []).append(ms) except Exception as e: print(f'Error reading {filepath}: {e}') + + results = {} + for test_name, values in samples.items(): + values.sort() + n = len(values) + median = (values[n // 2] if n % 2 == 1 + else (values[n // 2 - 1] + values[n // 2]) / 2.0) + results[test_name] = f'{median}ms' + if n > 1: + print(f'{test_name}: median={median:.0f}ms of {n} runs ' + f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)') return results def parse_time_to_ms(time_str): diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index a78ecc4300..f3ce9d0336 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -12,9 +12,14 @@ on: description: "The driver shard to test (Core or BqDriver)" type: string default: "BqDriver" - secrets: - BUILD_CACHE_KEY: - required: true + benchmark_iterations: + required: false + description: >- + How many times to run the whole suite (gtest --gtest_repeat). The + results table reports the median per test, which rejects a single + outlier run. Total runtime scales linearly with this. + type: string + default: "3" workflow_dispatch: inputs: build_shard: @@ -25,6 +30,11 @@ on: - BqDriver - Core default: "BqDriver" + benchmark_iterations: + description: "How many times to run the suite; median is reported" + required: false + type: string + default: "3" permissions: contents: read @@ -33,11 +43,16 @@ jobs: run-benchmarks: name: Run ODBC Performance Benchmarks (${{ inputs.build_shard }}) runs-on: windows-2022 - timeout-minutes: 60 + # The suite is now repeated benchmark_iterations times so the results + # table can take a median, so it takes proportionally longer. The timeout + # is sized for that and, more importantly, bounds a hang: without it a + # stuck run holds the runner until GitHub's 6-hour default. + timeout-minutes: 180 env: DRIVER_ARCH: x64 BUILD_SHARD: ${{ inputs.build_shard }} ODBC_GOOGLE_DRIVER_VERSION: 99.99.99 + BENCHMARK_ITERATIONS: ${{ inputs.benchmark_iterations || '3' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -143,9 +158,37 @@ jobs: EXE_PATH="c:/b/google/cloud/odbc/integration_tests/Release/performance_test.exe" echo "Running performance benchmark executable against $BUILD_SHARD..." + echo " repeats=${BENCHMARK_ITERATIONS}" set +e - "$EXE_PATH" > "$RESULTS_FILE" 2>&1 - TEST_EXIT_CODE=$? + # Run the suite BENCHMARK_ITERATIONS times and let the results parser + # take the median of each test's timings. A single run is dominated by + # BigQuery/service latency variance, so one sample per test moved tens + # of percent between runs for no code reason. + # + # Deliberately separate processes rather than --gtest_repeat: repeating + # in-process re-allocates and frees SQL_HANDLE_ENV once per test, so + # the Driver Manager loads and unloads the driver DLL on every test. + # Tripling that churn crashed the Simba driver mid-run (abort, exit 3). + # A fresh process per repetition keeps that count identical to a + # single-shot run, and an iteration that dies still leaves the other + # iterations' timings in the results file. + # + # tee -a appends each run; the file is truncated first. tee so progress + # is visible in the live job log -- previously all output went to a + # file only uploaded at the end, making a slow run indistinguishable + # from a hung one. PIPESTATUS keeps the executable's exit code, not + # tee's. + : > "$RESULTS_FILE" + TEST_EXIT_CODE=0 + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" | tee -a "$RESULTS_FILE" + "$EXE_PATH" 2>&1 | tee -a "$RESULTS_FILE" + RUN_EXIT=${PIPESTATUS[0]} + if [ $RUN_EXIT -ne 0 ]; then + echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}" + TEST_EXIT_CODE=$RUN_EXIT + fi + done set -e echo "Uploading results to GCS..." From ab3f36bd685fbd4771d08f4a0ce83fed013b4daf Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 13:08:41 +0530 Subject: [PATCH 2/6] minor changes --- .github/workflows/test-runner.yml | 3 ++- .github/workflows/windows-benchmark.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 2f4cea2af2..2e83278521 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -126,7 +126,7 @@ jobs: !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_benchmark_existing == true - needs: [pre-flight] + needs: [pre-flight, windows-benchmark-bq] uses: ./.github/workflows/windows-benchmark.yml with: checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }} @@ -308,3 +308,4 @@ jobs: run: | gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/ echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/benchmark_summary_table.txt" + \ No newline at end of file diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index f3ce9d0336..99dca2e8ee 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -20,6 +20,9 @@ on: outlier run. Total runtime scales linearly with this. type: string default: "3" + secrets: + BUILD_CACHE_KEY: + required: true workflow_dispatch: inputs: build_shard: @@ -198,3 +201,4 @@ jobs: echo "ERROR: Benchmark executable failed with exit code $TEST_EXIT_CODE." exit $TEST_EXIT_CODE fi + \ No newline at end of file From b8e9754da965ebec2714438f003ada47229f64b8 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 15:30:21 +0530 Subject: [PATCH 3/6] reverted the console output --- .github/workflows/test-runner.yml | 1 - .github/workflows/windows-benchmark.yml | 13 +++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 2e83278521..3107e75467 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -308,4 +308,3 @@ jobs: run: | gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/ echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/benchmark_summary_table.txt" - \ No newline at end of file diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index 99dca2e8ee..d1fe3784fa 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -175,18 +175,12 @@ jobs: # A fresh process per repetition keeps that count identical to a # single-shot run, and an iteration that dies still leaves the other # iterations' timings in the results file. - # - # tee -a appends each run; the file is truncated first. tee so progress - # is visible in the live job log -- previously all output went to a - # file only uploaded at the end, making a slow run indistinguishable - # from a hung one. PIPESTATUS keeps the executable's exit code, not - # tee's. : > "$RESULTS_FILE" TEST_EXIT_CODE=0 for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do - echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" | tee -a "$RESULTS_FILE" - "$EXE_PATH" 2>&1 | tee -a "$RESULTS_FILE" - RUN_EXIT=${PIPESTATUS[0]} + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" >> "$RESULTS_FILE" + "$EXE_PATH" >> "$RESULTS_FILE" 2>&1 + RUN_EXIT=$? if [ $RUN_EXIT -ne 0 ]; then echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}" TEST_EXIT_CODE=$RUN_EXIT @@ -201,4 +195,3 @@ jobs: echo "ERROR: Benchmark executable failed with exit code $TEST_EXIT_CODE." exit $TEST_EXIT_CODE fi - \ No newline at end of file From 00a486e15cdcb41d150ccd2cc81db14fcfda68ee Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Thu, 13 Aug 2026 03:43:03 +0000 Subject: [PATCH 4/6] minor: modified the comments and perf results parsing --- .github/workflows/test-runner.yml | 41 ++---- .github/workflows/windows-benchmark.yml | 21 +-- .../examples/catalog_performance_example.cc | 131 ++++++------------ 3 files changed, 64 insertions(+), 129 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 3107e75467..cbab28fd7a 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -114,19 +114,11 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-benchmark-existing cancel-in-progress: true - # Runs *after* windows-benchmark-bq rather than alongside it. Both shards - # benchmark against the same BigQuery project, so running them concurrently - # made each one's numbers depend on the other's API load -- and when that - # tripped rate limits, retry backoff turned the contention into - # multi-second steps. The dependency deliberately does not require the - # BqDriver shard to succeed: '!cancelled()' plus the windows-cmake guard - # preserves the previous run conditions, so the existing-driver baseline is - # still produced when that shard fails or is skipped. if: | !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_benchmark_existing == true - needs: [pre-flight, windows-benchmark-bq] + needs: [pre-flight] uses: ./.github/workflows/windows-benchmark.yml with: checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }} @@ -176,17 +168,18 @@ jobs: import os import re + def clean_test_name(name): + # GTest names follow [Instantiation/]TestSuite.TestCase[/Param] + # Extract everything after '.' to generically remove TestSuite/Instantiation prefix + if '.' in name: + name = name.split('.', 1)[1] + # Strip legacy parameter suffix if comparing against older baselines + name = re.sub(r'/(?:With|Without)HTAPI$', '', name) + return name + def parse_gtest_output(filepath): - # The suite is run several times (--gtest_repeat), so each test + # The suite is run several times across iterations, so each test # appears once per repetition. Take the median of its timings. - # - # These benchmarks talk to a live BigQuery service and fan list - # calls out concurrently, so a single run samples the latency - # tail and moved tens of percent between runs for no code - # reason. The median rejects a single outlier repetition, - # including the cold-cache first one. Files with only one run per - # test (a --gtest_repeat=1 run, or a baseline recorded before - # this change) still work: the median of one sample is itself. samples = {} if not os.path.exists(filepath): return {} @@ -197,9 +190,10 @@ jobs: for line in f: match = pattern.search(line) if match: + test_name = clean_test_name(match.group(1)) ms = parse_time_to_ms(match.group(2)) if ms is not None: - samples.setdefault(match.group(1), []).append(ms) + samples.setdefault(test_name, []).append(ms) except Exception as e: print(f'Error reading {filepath}: {e}') @@ -245,11 +239,6 @@ jobs: else: return ' (0%)' - def clean_test_name(name): - name = name.replace('HTAPIVariations/CatalogPerformanceHtapiParamTest.', '') - name = name.replace('DataFetchPerformanceParamTest.', '') - return name - existing_data = parse_gtest_output('./benchmark_results/current_core.txt') current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt') main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt') @@ -259,8 +248,6 @@ jobs: rows = [] for test in sorted_tests: - cleaned_name = clean_test_name(test) - existing_raw = existing_data.get(test, 'N/A') cur_bq_raw = current_bq_data.get(test, 'N/A') main_bq_raw = main_bq_data.get(test, 'N/A') @@ -275,7 +262,7 @@ jobs: cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}' main_bq_val = f'{main_bq_raw}{main_bq_pct}' - rows.append((cleaned_name, existing_raw, cur_bq_val, main_bq_val)) + rows.append((test, existing_raw, cur_bq_val, main_bq_val)) # Define headers h1 = 'Test Case (HTAPI ON/OFF)' diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index d1fe3784fa..100ac01754 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -15,9 +15,9 @@ on: benchmark_iterations: required: false description: >- - How many times to run the whole suite (gtest --gtest_repeat). The - results table reports the median per test, which rejects a single - outlier run. Total runtime scales linearly with this. + How many times to run the whole suite. The results table reports + the median per test, which rejects a single outlier run. Total + runtime scales linearly with this. type: string default: "3" secrets: @@ -163,18 +163,9 @@ jobs: echo "Running performance benchmark executable against $BUILD_SHARD..." echo " repeats=${BENCHMARK_ITERATIONS}" set +e - # Run the suite BENCHMARK_ITERATIONS times and let the results parser - # take the median of each test's timings. A single run is dominated by - # BigQuery/service latency variance, so one sample per test moved tens - # of percent between runs for no code reason. - # - # Deliberately separate processes rather than --gtest_repeat: repeating - # in-process re-allocates and frees SQL_HANDLE_ENV once per test, so - # the Driver Manager loads and unloads the driver DLL on every test. - # Tripling that churn crashed the Simba driver mid-run (abort, exit 3). - # A fresh process per repetition keeps that count identical to a - # single-shot run, and an iteration that dies still leaves the other - # iterations' timings in the results file. + # Run the suite in a separate process for each iteration to avoid DLL + # reload churn across tests. The results parser takes the median of + # timings across iterations. : > "$RESULTS_FILE" TEST_EXIT_CODE=0 for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do diff --git a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc index 93c65a5061..3e442a03a1 100644 --- a/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc +++ b/google/cloud/odbc/integration_tests/odbc_driver_tests/examples/catalog_performance_example.cc @@ -28,23 +28,10 @@ namespace google::cloud::odbc_tests { -class CatalogPerformanceHtapiParamTest : public ::testing::TestWithParam { - protected: - static std::string GetConnectionString(std::string const& base_conn_str, - bool use_htapi) { - std::string htapi_str = - use_htapi ? ";AllowHtapiForLargeResults=1;HTAPI_ActivationThreshold=0;" - : ";AllowHtapiForLargeResults=0;"; - return base_conn_str + htapi_str; - } -}; - // Primary Keys Performance Tests -TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetPrimaryKeysExactTable) { +TEST(CatalogPerformanceTest, BenchmarkGetPrimaryKeysExactTable) { auto conn = std::make_shared(); - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); CreateTableDirect(conn, kTableWithPKSchema); @@ -55,11 +42,9 @@ TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetPrimaryKeysExactTable) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetPrimaryKeysNoPKTable) { +TEST(CatalogPerformanceTest, BenchmarkGetPrimaryKeysNoPKTable) { auto conn = std::make_shared(); - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); CreateTableDirect(conn, kTableWithOutPKSchema); @@ -71,11 +56,9 @@ TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetPrimaryKeysNoPKTable) { } // Foreign Keys Performance Tests -TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysPkAndFkTables) { +TEST(CatalogPerformanceTest, BenchmarkGetForeignKeysPkAndFkTables) { auto conn = std::make_shared(); - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); CreateTableDirect(conn, kTableCustomerSchema); CreateTableDirect(conn, kTableOrdersSchema); @@ -87,11 +70,9 @@ TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysPkAndFkTables) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysPkTableOnly) { +TEST(CatalogPerformanceTest, BenchmarkGetForeignKeysPkTableOnly) { auto conn = std::make_shared(); - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); CreateTableDirect(conn, kTableCustomerSchema); CreateTableDirect(conn, kTableOrdersSchema); @@ -103,11 +84,9 @@ TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysPkTableOnly) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysFkTableOnly) { +TEST(CatalogPerformanceTest, BenchmarkGetForeignKeysFkTableOnly) { auto conn = std::make_shared(); - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); CreateTableDirect(conn, kTableCustomerSchema); CreateTableDirect(conn, kTableOrdersSchema); @@ -120,16 +99,13 @@ TEST_P(CatalogPerformanceHtapiParamTest, BenchmarkGetForeignKeysFkTableOnly) { } // SQLTables Performance Tests -TEST_P(CatalogPerformanceHtapiParamTest, - SQLTablesFullCatalogEnumerationTableAndView) { +TEST(CatalogPerformanceTest, SQLTablesFullCatalogEnumerationTableAndView) { auto conn = std::make_shared(); std::string catalog_pattern = "%"; std::string table_types = "TABLE,VIEW"; - std::string conn_str = - GetConnectionString(kDefaultConnectionString, GetParam()); - ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); + ASSERT_EQ(Connect(kDefaultConnectionString, conn), SQL_SUCCESS); SQLRETURN status = SQLSetStmtAttr(conn->hstmt, SQL_ATTR_METADATA_ID, reinterpret_cast(SQL_FALSE), 0); @@ -142,14 +118,12 @@ TEST_P(CatalogPerformanceHtapiParamTest, EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, - SQLTablesFullCatalogEnumerationWildcardCatalog) { +TEST(CatalogPerformanceTest, SQLTablesFullCatalogEnumerationWildcardCatalog) { auto conn = std::make_shared(); std::string catalog_pattern = "%"; - std::string base_conn_str = + std::string conn_str = kDefaultConnectionString + ";FilterTablesOnDefaultDataset=0;"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -164,13 +138,12 @@ TEST_P(CatalogPerformanceHtapiParamTest, EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesDatasetLevelEnumeration) { +TEST(CatalogPerformanceTest, SQLTablesDatasetLevelEnumeration) { auto conn = std::make_shared(); std::string dataset = "ODBC_TEST_DATASET"; - std::string base_conn_str = kDefaultConnectionString + - ";DefaultDataset=" + dataset + - ";FilterTablesOnDefaultDataset=0;"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); + std::string conn_str = kDefaultConnectionString + + ";DefaultDataset=" + dataset + + ";FilterTablesOnDefaultDataset=0;"; ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -185,14 +158,13 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesDatasetLevelEnumeration) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesExactTableLookup) { +TEST(CatalogPerformanceTest, SQLTablesExactTableLookup) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_name = "new_timestamp_table"; - std::string base_conn_str = kDefaultConnectionString + - ";DefaultDataset=" + dataset + - ";FilterTablesOnDefaultDataset=0;"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); + std::string conn_str = kDefaultConnectionString + + ";DefaultDataset=" + dataset + + ";FilterTablesOnDefaultDataset=0;"; ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -207,14 +179,13 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesExactTableLookup) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesWildcardTableSearch) { +TEST(CatalogPerformanceTest, SQLTablesWildcardTableSearch) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_pattern = "%timestamp%"; - std::string base_conn_str = kDefaultConnectionString + - ";DefaultDataset=" + dataset + - ";FilterTablesOnDefaultDataset=0;"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); + std::string conn_str = kDefaultConnectionString + + ";DefaultDataset=" + dataset + + ";FilterTablesOnDefaultDataset=0;"; ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -230,13 +201,12 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLTablesWildcardTableSearch) { } // SQLColumns Performance Tests -TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsFullMetadataFetch) { +TEST(CatalogPerformanceTest, SQLColumnsFullMetadataFetch) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_name = "new_timestamp_table"; - std::string base_conn_str = + std::string conn_str = kDefaultConnectionString + ";DefaultDataset=" + dataset + ";"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -247,14 +217,13 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsFullMetadataFetch) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsExactColumnLookup) { +TEST(CatalogPerformanceTest, SQLColumnsExactColumnLookup) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_name = "new_timestamp_table"; std::string column_name = "timestamp_col_1"; - std::string base_conn_str = + std::string conn_str = kDefaultConnectionString + ";DefaultDataset=" + dataset + ";"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -266,14 +235,13 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsExactColumnLookup) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsWildcardColumnSearch) { +TEST(CatalogPerformanceTest, SQLColumnsWildcardColumnSearch) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_name = "new_timestamp_table"; std::string column_pattern = "%timestamp%"; - std::string base_conn_str = + std::string conn_str = kDefaultConnectionString + ";DefaultDataset=" + dataset + ";"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -285,13 +253,12 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsWildcardColumnSearch) { EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); } -TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsLargeSchemaMetadataFetch) { +TEST(CatalogPerformanceTest, SQLColumnsLargeSchemaMetadataFetch) { auto conn = std::make_shared(); std::string dataset = "kirltest"; std::string table_name = "300_column_timestamp"; - std::string base_conn_str = + std::string conn_str = kDefaultConnectionString + ";DefaultDataset=" + dataset + ";"; - std::string conn_str = GetConnectionString(base_conn_str, GetParam()); ASSERT_EQ(Connect(conn_str, conn), SQL_SUCCESS); @@ -303,15 +270,14 @@ TEST_P(CatalogPerformanceHtapiParamTest, SQLColumnsLargeSchemaMetadataFetch) { } // FilterTablesOnDefaultDataset (ON/OFF) Performance Tests -TEST_P(CatalogPerformanceHtapiParamTest, - SQLTablesFullCatalogEnumerationFilterOnOff) { +TEST(CatalogPerformanceTest, SQLTablesFullCatalogEnumerationFilterOnOff) { auto conn = std::make_shared(); std::string default_dataset = "ODBC_TEST_DATASET"; std::string base_conn_str = kDefaultConnectionString + ";DefaultDataset=" + default_dataset; - std::string conn_str_unfiltered = GetConnectionString( - base_conn_str + ";FilterTablesOnDefaultDataset=0;", GetParam()); + std::string conn_str_unfiltered = + base_conn_str + ";FilterTablesOnDefaultDataset=0;"; ASSERT_EQ(Connect(conn_str_unfiltered, conn), SQL_SUCCESS); SQLRETURN status = SQLSetStmtAttr(conn->hstmt, SQL_ATTR_METADATA_ID, reinterpret_cast(SQL_FALSE), 0); @@ -322,8 +288,8 @@ TEST_P(CatalogPerformanceHtapiParamTest, EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); - std::string conn_str_filtered = GetConnectionString( - base_conn_str + ";FilterTablesOnDefaultDataset=1;", GetParam()); + std::string conn_str_filtered = + base_conn_str + ";FilterTablesOnDefaultDataset=1;"; ASSERT_EQ(Connect(conn_str_filtered, conn), SQL_SUCCESS); status = SQLSetStmtAttr(conn->hstmt, SQL_ATTR_METADATA_ID, reinterpret_cast(SQL_FALSE), 0); @@ -336,15 +302,14 @@ TEST_P(CatalogPerformanceHtapiParamTest, } #if defined(BQ_DRIVER_INTEGRATION_TESTS) -TEST_P(CatalogPerformanceHtapiParamTest, - SQLColumnsColumnMetadataEnumerationFilterOnOff) { +TEST(CatalogPerformanceTest, SQLColumnsColumnMetadataEnumerationFilterOnOff) { auto conn = std::make_shared(); std::string default_dataset = "ODBC_TEST_DATASET"; std::string base_conn_str = kDefaultConnectionString + ";DefaultDataset=" + default_dataset; - std::string conn_str_unfiltered = GetConnectionString( - base_conn_str + ";FilterTablesOnDefaultDataset=0;", GetParam()); + std::string conn_str_unfiltered = + base_conn_str + ";FilterTablesOnDefaultDataset=0;"; ASSERT_EQ(Connect(conn_str_unfiltered, conn), SQL_SUCCESS); SQLRETURN status = SQLSetStmtAttr(conn->hstmt, SQL_ATTR_METADATA_ID, reinterpret_cast(SQL_FALSE), 0); @@ -355,8 +320,8 @@ TEST_P(CatalogPerformanceHtapiParamTest, EXPECT_EQ(Disconnect(conn), SQL_SUCCESS); - std::string conn_str_filtered = GetConnectionString( - base_conn_str + ";FilterTablesOnDefaultDataset=1;", GetParam()); + std::string conn_str_filtered = + base_conn_str + ";FilterTablesOnDefaultDataset=1;"; ASSERT_EQ(Connect(conn_str_filtered, conn), SQL_SUCCESS); status = SQLSetStmtAttr(conn->hstmt, SQL_ATTR_METADATA_ID, reinterpret_cast(SQL_FALSE), 0); @@ -369,14 +334,6 @@ TEST_P(CatalogPerformanceHtapiParamTest, } #endif -INSTANTIATE_TEST_SUITE_P( - HTAPIVariations, CatalogPerformanceHtapiParamTest, - ::testing::Values(true, false), - [](::testing::TestParamInfo< - CatalogPerformanceHtapiParamTest::ParamType> const& info) { - return info.param ? "WithHTAPI" : "WithoutHTAPI"; - }); - using DataFetchParams = std::tuple; class DataFetchPerformanceParamTest From d49a496749b873a94a68d8d52b85bda08efa153f Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Thu, 13 Aug 2026 03:44:54 +0000 Subject: [PATCH 5/6] fix: minor fix for windows tests --- google/cloud/odbc/testing/odbc_utils/commons.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/google/cloud/odbc/testing/odbc_utils/commons.cc b/google/cloud/odbc/testing/odbc_utils/commons.cc index 5492753546..85387cc0a9 100644 --- a/google/cloud/odbc/testing/odbc_utils/commons.cc +++ b/google/cloud/odbc/testing/odbc_utils/commons.cc @@ -1714,18 +1714,17 @@ std::string Utf16ToUtf8(std::wstring const& utf_16_str, } #ifdef _WIN32 // https://learn.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-widechartomultibyte - int utf8Length = WideCharToMultiByte(code_page, 0, utf_16_str.c_str(), -1, + int utf8Length = WideCharToMultiByte(code_page, 0, utf_16_str.data(), + static_cast(utf_16_str.length()), NULL, 0, NULL, NULL); if (utf8Length == 0) { throw std::runtime_error( "Error determining buffer size while converting wstring to string"); } - if (sizeof(SQLWCHAR) == 2) { - utf8Length = utf8Length * sizeof(SQLWCHAR); - } std::string utf8Str(utf8Length, 0); // https://learn.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-widechartomultibyte - int result = WideCharToMultiByte(code_page, 0, utf_16_str.c_str(), -1, + int result = WideCharToMultiByte(code_page, 0, utf_16_str.data(), + static_cast(utf_16_str.length()), &utf8Str[0], utf8Length, NULL, NULL); if (result == 0) { throw std::runtime_error("Error while converting wstring to string"); @@ -1773,14 +1772,16 @@ std::wstring Utf8ToUtf16(std::string const& utf_8_str) { #ifdef _WIN32 // https://learn.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-multibytetowidechar int utf16Length = - MultiByteToWideChar(CP_UTF8, 0, utf_8_str.c_str(), -1, NULL, 0); + MultiByteToWideChar(CP_UTF8, 0, utf_8_str.data(), + static_cast(utf_8_str.length()), NULL, 0); if (utf16Length == 0) { throw std::runtime_error( "Error determining buffer size while converting string to wstring"); } std::wstring utf16Str(utf16Length, 0); // https://learn.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-multibytetowidechar - int result = MultiByteToWideChar(CP_UTF8, 0, utf_8_str.c_str(), -1, + int result = MultiByteToWideChar(CP_UTF8, 0, utf_8_str.data(), + static_cast(utf_8_str.length()), &utf16Str[0], utf16Length); if (result == 0) { throw std::runtime_error("Error while converting string to wstring"); From 1e1b9a4517bb67a4349435116b0f992488fb6952 Mon Sep 17 00:00:00 2001 From: Sachin Purohit Date: Thu, 13 Aug 2026 03:58:48 +0000 Subject: [PATCH 6/6] minor: reducing the timeout to 120 mins --- .github/workflows/test-runner.yml | 10 +++++++--- .github/workflows/windows-benchmark.yml | 6 +----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index cbab28fd7a..a53c94b8ba 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -163,11 +163,15 @@ jobs: gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true - name: Parse Results and Generate Table + env: + BRANCH_NAME: ${{ github.ref_name }} run: | python3 <<'EOF' import os import re + branch_name = os.environ.get('BRANCH_NAME', 'Current') + def clean_test_name(name): # GTest names follow [Instantiation/]TestSuite.TestCase[/Param] # Extract everything after '.' to generically remove TestSuite/Instantiation prefix @@ -265,9 +269,9 @@ jobs: rows.append((test, existing_raw, cur_bq_val, main_bq_val)) # Define headers - h1 = 'Test Case (HTAPI ON/OFF)' + h1 = 'Test Case' h2 = 'Existing Driver (Current)' - h3 = 'Google Driver (Current)' + h3 = f'Google Driver ({branch_name})' h4 = 'Google Driver (Main)' # Calculate dynamic widths for formatting @@ -277,7 +281,7 @@ jobs: w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4) # Construct the Markdown table - table = "*Percentages in **Google Driver (Current)** show change relative to **Existing Driver (Current)**. Percentages in **Google Driver (Main)** show change relative to **Google Driver (Current)**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" + table = f"*Percentages in **{h3}** show change relative to **{h2}**. Percentages in **{h4}** show change relative to **{h3}**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n" table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n' table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n' diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index 100ac01754..11c2596258 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -46,11 +46,7 @@ jobs: run-benchmarks: name: Run ODBC Performance Benchmarks (${{ inputs.build_shard }}) runs-on: windows-2022 - # The suite is now repeated benchmark_iterations times so the results - # table can take a median, so it takes proportionally longer. The timeout - # is sized for that and, more importantly, bounds a hang: without it a - # stuck run holds the runner until GitHub's 6-hour default. - timeout-minutes: 180 + timeout-minutes: 120 env: DRIVER_ARCH: x64 BUILD_SHARD: ${{ inputs.build_shard }}