From 155fad380f68c23c47c0036e2fbe410996406377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Sat, 12 Sep 2026 17:01:16 +0200 Subject: [PATCH 1/3] fix(ci): correct Bencher reporting and benchmark failures --- .github/workflows/internal-benchmark.yml | 16 +++++++--- benchmarks/README.md | 28 +++++++++++++++++ benchmarks/benchmarks-utils.mjs | 40 +++++++++++------------- renovate.json | 6 ++++ 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/.github/workflows/internal-benchmark.yml b/.github/workflows/internal-benchmark.yml index f7ee5d83a..e6c38b1a2 100644 --- a/.github/workflows/internal-benchmark.yml +++ b/.github/workflows/internal-benchmark.yml @@ -23,12 +23,20 @@ jobs: - name: Setup Deno uses: denoland/setup-deno@v2 with: - deno-version: v2.x + deno-version: latest - uses: bencherdev/bencher@main - name: Run benchmark + run: deno task benchmark:tinybench + - name: Publish benchmark + shell: bash run: | + args=() + if [[ "$GITHUB_REF" != refs/heads/master ]]; then + args+=(--dry-run) + fi bencher run \ - --branch master \ + "${args[@]}" \ + --branch "$GITHUB_REF_NAME" \ --threshold-measure latency \ --threshold-test t_test \ --threshold-max-sample-size 64 \ @@ -41,6 +49,4 @@ jobs: --threshold-upper-boundary _ \ --thresholds-reset \ --file benchmark-report.json \ - --err \ - --github-actions ${{ secrets.GITHUB_TOKEN }} \ - "deno task benchmark:tinybench" + --err diff --git a/benchmarks/README.md b/benchmarks/README.md index 88142e8fc..37d6d20c6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -38,3 +38,31 @@ poolifier cloned repository and run: - `bun ./benchmarks/internal/bench.mjs -t tinybench` ### [Results](https://bencher.dev/perf/poolifier-web-worker) + +With `CI=true`, the Tinybench command writes `benchmark-report.json` in Bencher +Metric Format after both pool suites complete. Latency is in nanoseconds and +throughput is in operations per second; bounds represent mean ± one standard +deviation. Failed or incomplete benchmarks fail the command instead of producing +partial results. + +The internal benchmark workflow runs on the dedicated self-hosted runner with +Deno `latest`. Benchmark execution and publication are separate steps: an +execution failure prevents publication and retains the original error. Renovate +waits one day before selecting npm releases in Deno manifests, matching Deno’s +default minimum dependency age without disabling that protection. + +Only runs on `master` publish results. Manual workflow dispatches on other refs +validate the report with Bencher’s `--dry-run`, without changing stored results. + +The workflow retains the `self-hosted` testbed. Its historical latency values +were milliseconds labeled as nanoseconds. Corrected reports use nanoseconds; +without a historical data migration, latency comparisons mix units and can +produce artificial regression alerts. This workflow does not migrate or delete +historical data. Throughput units remain unchanged. + +Bencher applies a one-sided Student’s t-test to at most 64 historical samples: +the upper 0.99 boundary detects latency increases, and the lower 0.99 boundary +detects throughput decreases. Alerts still fail the publication step. These +correlated measures are not independent evidence, and no multiple-testing +correction is applied across benchmark scenarios. An alert identifies a +departure from the baseline, not which code or environment change caused it. diff --git a/benchmarks/benchmarks-utils.mjs b/benchmarks/benchmarks-utils.mjs index fcac03a92..2e70643d9 100644 --- a/benchmarks/benchmarks-utils.mjs +++ b/benchmarks/benchmarks-utils.mjs @@ -292,7 +292,7 @@ export const runPoolifierBenchmarkTinyBench = async ( const bmfResults = {} let pool try { - const bench = new Bench() + const bench = new Bench({ throws: true }) pool = buildPoolifierPool(workerType, poolType, poolSize) for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) { @@ -364,30 +364,26 @@ export const runPoolifierBenchmarkTinyBench = async ( console.table(bench.table()) for (const task of tasks) { - if ( - task.result?.state === 'completed' || - task.result?.state === 'aborted-with-statistics' - ) { - bmfResults[task.name] = { - latency: { - value: task.result.latency.mean, - lower_value: task.result.latency.mean - task.result.latency.sd, - upper_value: task.result.latency.mean + task.result.latency.sd, - }, - throughput: { - value: task.result.throughput.mean, - lower_value: task.result.throughput.mean - - task.result.throughput.sd, - upper_value: task.result.throughput.mean + - task.result.throughput.sd, - }, - } + if (task.result?.state !== 'completed') { + throw new Error(`Benchmark did not complete: ${task.name}`) + } + bmfResults[task.name] = { + latency: { + // Tinybench reports milliseconds; Bencher latency uses nanoseconds. + value: task.result.latency.mean * 1e6, + lower_value: (task.result.latency.mean - task.result.latency.sd) * + 1e6, + upper_value: (task.result.latency.mean + task.result.latency.sd) * + 1e6, + }, + throughput: { + value: task.result.throughput.mean, + lower_value: task.result.throughput.mean - task.result.throughput.sd, + upper_value: task.result.throughput.mean + task.result.throughput.sd, + }, } } return bmfResults - } catch (error) { - console.error(error) - return bmfResults } finally { if (pool != null) { await pool.destroy() diff --git a/renovate.json b/renovate.json index cb6debed1..5e3617832 100644 --- a/renovate.json +++ b/renovate.json @@ -7,6 +7,12 @@ ":maintainLockFilesWeekly" ], "packageRules": [ + { + "matchDatasources": ["npm"], + "matchFileNames": ["**/deno.json"], + "minimumReleaseAge": "1 day", + "internalChecksFilter": "strict" + }, { "matchFileNames": ["**/package.json"], "matchDepTypes": [ From 455cc2a4748df1306e68b92c843e4e60f0358f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Sat, 12 Sep 2026 17:16:42 +0200 Subject: [PATCH 2/3] docs: remove unnecessary benchmark README additions --- benchmarks/README.md | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 37d6d20c6..88142e8fc 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -38,31 +38,3 @@ poolifier cloned repository and run: - `bun ./benchmarks/internal/bench.mjs -t tinybench` ### [Results](https://bencher.dev/perf/poolifier-web-worker) - -With `CI=true`, the Tinybench command writes `benchmark-report.json` in Bencher -Metric Format after both pool suites complete. Latency is in nanoseconds and -throughput is in operations per second; bounds represent mean ± one standard -deviation. Failed or incomplete benchmarks fail the command instead of producing -partial results. - -The internal benchmark workflow runs on the dedicated self-hosted runner with -Deno `latest`. Benchmark execution and publication are separate steps: an -execution failure prevents publication and retains the original error. Renovate -waits one day before selecting npm releases in Deno manifests, matching Deno’s -default minimum dependency age without disabling that protection. - -Only runs on `master` publish results. Manual workflow dispatches on other refs -validate the report with Bencher’s `--dry-run`, without changing stored results. - -The workflow retains the `self-hosted` testbed. Its historical latency values -were milliseconds labeled as nanoseconds. Corrected reports use nanoseconds; -without a historical data migration, latency comparisons mix units and can -produce artificial regression alerts. This workflow does not migrate or delete -historical data. Throughput units remain unchanged. - -Bencher applies a one-sided Student’s t-test to at most 64 historical samples: -the upper 0.99 boundary detects latency increases, and the lower 0.99 boundary -detects throughput decreases. Alerts still fail the publication step. These -correlated measures are not independent evidence, and no multiple-testing -correction is applied across benchmark scenarios. An alert identifies a -departure from the baseline, not which code or environment change caused it. From c66a0646bac3618f40118ee7902e18d4812f8563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Sat, 12 Sep 2026 17:31:57 +0200 Subject: [PATCH 3/3] refactor(benchmarks): use Tinybench unit conversion helper --- benchmarks/benchmarks-utils.mjs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmarks/benchmarks-utils.mjs b/benchmarks/benchmarks-utils.mjs index 2e70643d9..5c53ed6d1 100644 --- a/benchmarks/benchmarks-utils.mjs +++ b/benchmarks/benchmarks-utils.mjs @@ -7,7 +7,7 @@ import { rmSync, writeFileSync, } from 'node:fs' -import { Bench } from 'tinybench' +import { Bench, mToNs } from 'tinybench' import { DynamicThreadPool, FixedThreadPool, @@ -370,11 +370,9 @@ export const runPoolifierBenchmarkTinyBench = async ( bmfResults[task.name] = { latency: { // Tinybench reports milliseconds; Bencher latency uses nanoseconds. - value: task.result.latency.mean * 1e6, - lower_value: (task.result.latency.mean - task.result.latency.sd) * - 1e6, - upper_value: (task.result.latency.mean + task.result.latency.sd) * - 1e6, + value: mToNs(task.result.latency.mean), + lower_value: mToNs(task.result.latency.mean - task.result.latency.sd), + upper_value: mToNs(task.result.latency.mean + task.result.latency.sd), }, throughput: { value: task.result.throughput.mean,