test: adding iterations in perf testing

.github/workflows/test-runner.yml

@@ -163,27 +163,54 @@ jobs:
           gcloud storage cp gs://bq-dev-tools-testing-drivers/odbc-perf/main/results/performance_benchmark_results_BqDriver.txt ./benchmark_results/main_bq.txt || true
 
       - name: Parse Results and Generate Table
+        env:
+          BRANCH_NAME: ${{ github.ref_name }}
         run: |
           python3 <<'EOF'
           import os
           import re
 
+          branch_name = os.environ.get('BRANCH_NAME', 'Current')
+
+          def clean_test_name(name):
+              # GTest names follow [Instantiation/]TestSuite.TestCase[/Param]
+              # Extract everything after '.' to generically remove TestSuite/Instantiation prefix
+              if '.' in name:
+                  name = name.split('.', 1)[1]
+              # Strip legacy parameter suffix if comparing against older baselines
+              name = re.sub(r'/(?:With|Without)HTAPI$', '', name)
+              return name
+
           def parse_gtest_output(filepath):
-              results = {}
+              # The suite is run several times across iterations, so each test
+              # appears once per repetition. Take the median of its timings.
+              samples = {}
               if not os.path.exists(filepath):
-                  return results
+                  return {}
 
               pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)')
               try:
                   with open(filepath, 'r') as f:
                       for line in f:
                           match = pattern.search(line)
                           if match:
-                              test_name = match.group(1)
-                              time_taken = match.group(2)
-                              results[test_name] = time_taken
+                              test_name = clean_test_name(match.group(1))
+                              ms = parse_time_to_ms(match.group(2))
+                              if ms is not None:
+                                  samples.setdefault(test_name, []).append(ms)
               except Exception as e:
                   print(f'Error reading {filepath}: {e}')
+
+              results = {}
+              for test_name, values in samples.items():
+                  values.sort()
+                  n = len(values)
+                  median = (values[n // 2] if n % 2 == 1
+                            else (values[n // 2 - 1] + values[n // 2]) / 2.0)
+                  results[test_name] = f'{median}ms'
+                  if n > 1:
+                      print(f'{test_name}: median={median:.0f}ms of {n} runs '
+                            f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)')
               return results
 
           def parse_time_to_ms(time_str):
@@ -216,11 +243,6 @@ jobs:
               else:
                   return ' (0%)'
 
-          def clean_test_name(name):
-              name = name.replace('HTAPIVariations/CatalogPerformanceHtapiParamTest.', '')
-              name = name.replace('DataFetchPerformanceParamTest.', '')
-              return name
-
           existing_data = parse_gtest_output('./benchmark_results/current_core.txt')
           current_bq_data = parse_gtest_output('./benchmark_results/current_bq.txt')
           main_bq_data = parse_gtest_output('./benchmark_results/main_bq.txt')
@@ -230,8 +252,6 @@ jobs:
 
           rows = []
           for test in sorted_tests:
-              cleaned_name = clean_test_name(test)
-
               existing_raw = existing_data.get(test, 'N/A')
               cur_bq_raw = current_bq_data.get(test, 'N/A')
               main_bq_raw = main_bq_data.get(test, 'N/A')
@@ -246,12 +266,12 @@ jobs:
               cur_bq_val = f'{cur_bq_raw}{cur_bq_pct}'
               main_bq_val = f'{main_bq_raw}{main_bq_pct}'
 
-              rows.append((cleaned_name, existing_raw, cur_bq_val, main_bq_val))
+              rows.append((test, existing_raw, cur_bq_val, main_bq_val))
 
           # Define headers
-          h1 = 'Test Case (HTAPI ON/OFF)'
+          h1 = 'Test Case'
           h2 = 'Existing Driver (Current)'
-          h3 = 'Google Driver (Current)'
+          h3 = f'Google Driver ({branch_name})'
           h4 = 'Google Driver (Main)'
 
           # Calculate dynamic widths for formatting
@@ -261,7 +281,7 @@ jobs:
           w4 = max([len(h4)] + [len(r[3]) for r in rows]) if rows else len(h4)
 
           # Construct the Markdown table
-          table = "*Percentages in **Google Driver (Current)** show change relative to **Existing Driver (Current)**. Percentages in **Google Driver (Main)** show change relative to **Google Driver (Current)**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n"
+          table = f"*Percentages in **{h3}** show change relative to **{h2}**. Percentages in **{h4}** show change relative to **{h3}**. Negative values indicate improvement (faster test execution), positive values indicate degradation (slower).*\n\n"
           table += f'| {h1.ljust(w1)} | {h2.ljust(w2)} | {h3.ljust(w3)} | {h4.ljust(w4)} |\n'
           table += '|-' + ('-' * w1) + '-|-' + ('-' * w2) + '-|-' + ('-' * w3) + '-|-' + ('-' * w4) + '-|\n'
 

.github/workflows/windows-benchmark.yml

@@ -12,6 +12,14 @@ on:
         description: "The driver shard to test (Core or BqDriver)"
         type: string
         default: "BqDriver"
+      benchmark_iterations:
+        required: false
+        description: >-
+          How many times to run the whole suite. The results table reports
+          the median per test, which rejects a single outlier run. Total
+          runtime scales linearly with this.
+        type: string
+        default: "3"
     secrets:
       BUILD_CACHE_KEY:
         required: true
@@ -25,6 +33,11 @@ on:
           - BqDriver
           - Core
         default: "BqDriver"
+      benchmark_iterations:
+        description: "How many times to run the suite; median is reported"
+        required: false
+        type: string
+        default: "3"
 
 permissions:
   contents: read
@@ -33,11 +46,12 @@ jobs:
   run-benchmarks:
     name: Run ODBC Performance Benchmarks (${{ inputs.build_shard }})
     runs-on: windows-2022
-    timeout-minutes: 60
+    timeout-minutes: 120
     env:
       DRIVER_ARCH: x64
       BUILD_SHARD: ${{ inputs.build_shard }}
       ODBC_GOOGLE_DRIVER_VERSION: 99.99.99
+      BENCHMARK_ITERATIONS: ${{ inputs.benchmark_iterations || '3' }}
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
@@ -143,9 +157,22 @@ jobs:
         EXE_PATH="c:/b/google/cloud/odbc/integration_tests/Release/performance_test.exe"
 
         echo "Running performance benchmark executable against $BUILD_SHARD..."     
+        echo "  repeats=${BENCHMARK_ITERATIONS}"
         set +e
-        "$EXE_PATH" > "$RESULTS_FILE" 2>&1
-        TEST_EXIT_CODE=$?
+        # Run the suite in a separate process for each iteration to avoid DLL
+        # reload churn across tests. The results parser takes the median of
+        # timings across iterations.
+        : > "$RESULTS_FILE"
+        TEST_EXIT_CODE=0
+        for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do
+          echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" >> "$RESULTS_FILE"
+          "$EXE_PATH" >> "$RESULTS_FILE" 2>&1
+          RUN_EXIT=$?
+          if [ $RUN_EXIT -ne 0 ]; then
+            echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}"
+            TEST_EXIT_CODE=$RUN_EXIT
+          fi
+        done
         set -e
 
         echo "Uploading results to GCS..."