diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 101ce7600..688fda74f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: test-e2e: name: E2E test (web) - # Not yet a required check, matching mutation.yml's own precedent: the suite is new (ExaDev/documents.js#932) and hasn't run unattended in CI before, so it needs a run of real-world flakiness data before gating merges on it. + # Promoted to a required check (ExaDev/documents.js#1194): the suite has run unattended on every pull request and main push since it landed (ExaDev/documents.js#932), and its only observed red run tracked a regression on that pull request's own branch and cleared when the branch was fixed -- a genuine catch, not flakiness, which is exactly the run history gating merges on it requires. Requiredness itself lives in the repository ruleset ("main required checks"), not in this file; this job always runs on both triggers, so requiring it can never strand a pull request waiting for a check that does not report. runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -704,9 +704,18 @@ jobs: deploy-site: name: Build and deploy the web UI to Pages - # After release, so the deploy is built from the release commit the orchestrator just pushed (it bumps packages/web/package.json and tags it). For a commit that releases nothing, no new tag is created and this builds the current tip of main. + # After release, so the deploy is built from the release commit the orchestrator just pushed (it bumps packages/web/package.json and tags it). For a commit that releases nothing, no new tag is created and this builds the current tip of main. test-e2e is in the gate so a main-branch e2e failure blocks deploying the site the suite drives a real browser through, matching the required role e2e already plays on pull requests via the ruleset. needs: - [commitlint, lint, typecheck, test, test-workers, test-smoke, release] + [ + commitlint, + lint, + typecheck, + test, + test-workers, + test-smoke, + test-e2e, + release, + ] if: github.ref == 'refs/heads/main' && github.event_name == 'push' runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 0fd7848e6..7451e04c7 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -1,6 +1,6 @@ name: Mutation testing -# Repo-wide, sharded, incrementally-cached mutation testing via Stryker. Deliberately not yet a required check (see the ruleset docs / repo-setup skill's stryker-mutation-testing reference): no package here has a measured thresholds.break yet, and stryker.shared.ts's packageStrykerConfig sets none, so this job reports real mutation scores without ever failing the build on them -- the correct default until a real baseline exists to gate against. Promote it once scores are measured and a break threshold is deliberately chosen per package. +# Repo-wide, sharded, incrementally-cached mutation testing via Stryker. Each package is gated by its own stryker.config.ts break threshold (derived from a completed run's measured baseline -- see stryker.shared.ts's breakThreshold), and the mutation-result job at the bottom aggregates the shards into the one stable check context a branch ruleset can require: a required status check must report on every pull request, and the shard matrix's width varies with the affected set, so no individual shard context can be required without stranding a pull request that produced fewer shards than the ruleset names. Requiredness itself lives in the repository ruleset ("main required checks"), never in this file -- a pull request can only prepare the job the ruleset points at, and the job is only ready to be required once every package completes a green run behind its threshold. on: pull_request: push: @@ -88,3 +88,28 @@ jobs: name: mutation-report-shard-${{ matrix.index }} path: packages/*/reports/mutation/mutation.html if-no-files-found: ignore + + # The single check context to list as required in the repository ruleset. The mutation-test job above is a matrix whose width the plan job computes per run from the affected package set (three affected packages produce three shards, not eight), so requiring any shard context by name would leave a pull request that produced fewer shards forever waiting on a check that never reports. This job always runs -- `if: always()` keeps it reporting even when planning fails or produces no packages, where the matrix job is skipped -- and derives one conclusion from the outcomes it aggregates, so it is stable across every shard count. + mutation-result: + name: Mutation testing result + needs: [plan, mutation-test] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Aggregate the shard outcomes into one conclusion + # A plan failure is a real failure (nothing can be said about the mutation state); a plan that found no affected packages is a pass (nothing to mutate); anything else gates on the matrix's own aggregate result, which is failure when any shard failed (a package fell below its break threshold or its run errored). + run: | + if [ "${{ needs.plan.result }}" != "success" ]; then + echo "::error::Mutation shard planning failed; see the Plan mutation shards job." + exit 1 + fi + if [ "${{ needs.plan.outputs.has-packages }}" != "true" ]; then + echo "No mutation-affected packages in this change; nothing to gate." + exit 0 + fi + if [ "${{ needs.mutation-test.result }}" != "success" ]; then + echo "::error::One or more mutation shards failed: a package's score fell below its break threshold, or its run failed." + exit 1 + fi + echo "Every mutation shard succeeded." diff --git a/README.md b/README.md index b6ca767c4..c7d77c391 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,8 @@ pnpm test:coverage # the same suites with coverage pnpm test:workers # the same code inside workerd, the real Cloudflare Workers runtime pnpm test:smoke # each package's built dist/ exercised as a real artifact pnpm test:corpus # the real-world conformance corpora (gitignored, so local only) -pnpm test:mutation # Stryker mutation testing, sharded and cached in CI; not yet a required check (no package has a measured break threshold yet) -pnpm test:e2e # the web UI driven through a real browser (Playwright); not yet a required check (new suite, no unattended CI run history yet) +pnpm test:mutation # Stryker mutation testing, sharded and cached in CI, each package gated by the break threshold derived from its measured baseline (per-package stryker.config.ts); a required check once every package completes a green run behind its threshold +pnpm test:e2e # the web UI driven through a real browser (Playwright); a required check (named in the repository ruleset alongside Lint/Test and siblings) ``` Every one of these runs through turbo, so a package whose inputs have not changed replays a cached result rather than re-running. @@ -163,7 +163,7 @@ Every alias name's trusted publisher is registered against this repository and w ## CI -`.github/workflows/ci.yml` holds one job per task — Commitlint, Lint, Typecheck, Test, Test (workerd), Smoke test — each running that task once across the workspace through turbo, followed by Release, its three post-release republish/attestation matrices (see Releases above), and the web UI's Pages deploy on `main`. On a pull request every turbo task runs with `--affected`, restricting work to the packages the branch changed and their dependents; on `main` the full workspace runs, so the caches later runs restore from are complete and the release gate covers everything. Each job restores turbo's cache keyed by task, so an unchanged package costs a cache replay rather than a rebuild. The Typecheck job additionally runs `attw --pack` across every published package after building it, checking that each package's declared types resolve under every module resolution mode — the web UI is excluded, since it publishes nothing and exposes no types. +`.github/workflows/ci.yml` holds one job per task — Commitlint, Lint, Typecheck, Test, Test (workerd), Smoke test, E2E test — each running that task once across the workspace through turbo, followed by Release, its three post-release republish/attestation matrices (see Releases above), and the web UI's Pages deploy on `main`. Mutation testing runs in its own `.github/workflows/mutation.yml` (sharded, incrementally cached, serialised through one concurrency queue so it can never starve these jobs of runners), reporting one aggregate "Mutation testing result" check the repository ruleset can require once every package's run is green behind its threshold. On a pull request every turbo task runs with `--affected`, restricting work to the packages the branch changed and their dependents; on `main` the full workspace runs, so the caches later runs restore from are complete and the release gate covers everything. Each job restores turbo's cache keyed by task, so an unchanged package costs a cache replay rather than a rebuild. The Typecheck job additionally runs `attw --pack` across every published package after building it, checking that each package's declared types resolve under every module resolution mode — the web UI is excluded, since it publishes nothing and exposes no types. Dependabot covers the root manifest and every package's, batching minor and patch updates into one pull request and leaving majors individual; `.github/workflows/dependabot-auto-merge.yml` auto-merges the former once CI is green. The cross-repository `sibling-released` dispatch the separate repositories used to propagate version bumps between themselves is gone: the orchestrator does that inside a single run now, in dependency order, without a pull request per bump. diff --git a/packages/archive-codec/stryker.config.ts b/packages/archive-codec/stryker.config.ts index f90f7ba75..dff23a5ed 100644 --- a/packages/archive-codec/stryker.config.ts +++ b/packages/archive-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 76.26% of 1954 valid mutants, timeout share 2.5% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 73, }); diff --git a/packages/byte-codec/stryker.config.ts b/packages/byte-codec/stryker.config.ts index 88581839d..49342f7f9 100644 --- a/packages/byte-codec/stryker.config.ts +++ b/packages/byte-codec/stryker.config.ts @@ -1,3 +1,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; -export default packageStrykerConfig(); +export default packageStrykerConfig({ + // First CI-measured baseline: 63.99% of 636 valid mutants, timeout share 2.0% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 60, +}); diff --git a/packages/doc-codec/stryker.config.ts b/packages/doc-codec/stryker.config.ts index f90f7ba75..5613e3888 100644 --- a/packages/doc-codec/stryker.config.ts +++ b/packages/doc-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 71.12% of 3758 valid mutants, timeout share 1.8% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 69, }); diff --git a/packages/document-cli/stryker.config.ts b/packages/document-cli/stryker.config.ts index 223947c07..611b74d77 100644 --- a/packages/document-cli/stryker.config.ts +++ b/packages/document-cli/stryker.config.ts @@ -8,4 +8,6 @@ export default packageStrykerConfig({ "!src/**/*.test.tsx", ], vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 33.33% of 6241 valid mutants, timeout share 0.02% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 32, }); diff --git a/packages/document-compute.js/stryker.config.ts b/packages/document-compute.js/stryker.config.ts index 88581839d..1b75801cc 100644 --- a/packages/document-compute.js/stryker.config.ts +++ b/packages/document-compute.js/stryker.config.ts @@ -1,3 +1,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; -export default packageStrykerConfig(); +export default packageStrykerConfig({ + // First CI-measured baseline: 64.48% of 518 valid mutants, timeout share 0.4% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 63, +}); diff --git a/packages/document-mcp/README.md b/packages/document-mcp/README.md index 1c9692d5c..f5ad09b0d 100644 --- a/packages/document-mcp/README.md +++ b/packages/document-mcp/README.md @@ -228,7 +228,7 @@ Every tool that takes or produces document bytes goes through the same two hybri | `odb_tables` | Lists every table an embedded `.odb` database declares — column names, types, and row data — across every storage tier `documents.js` supports (HSQLDB TEXT/CACHED/BINARY, Firebird gbak backups). | | `odb_forms` | Lists every form an `.odb` database declares, with each form's own data source and field-bound controls. | | `odb_reports` | Lists every report an `.odb` database declares, with each report's own data-source command, band/group structure, and `rpt:` formula expressions. | -| `odb_query` | Runs a bounded `SELECT` (with optional JOINs of any kind, table aliases, a derived table in `FROM`, and `IN`/`EXISTS` subqueries) over an embedded `.odb` database's extracted tables — given directly as SQL or by naming a saved query. No database engine involved; no column aliases; an unsupported construct is reported as a tool error naming it, never silently ignored. | +| `odb_query` | Runs a bounded `SELECT` (with optional JOINs of any kind, table aliases, a derived table in `FROM`, and `IN`/`EXISTS` subqueries) over an embedded `.odb` database's extracted tables — given directly as SQL or by naming a saved query. No database engine involved; no column aliases; an unsupported construct is reported as a tool error naming it, never silently ignored. | | `odb_to_csv` | Extracts exactly one named table from an embedded `.odb` database as CSV bytes. The table name is required whenever the database declares more than one table. | | `odb_to_xlsx` | Extracts every table an embedded `.odb` database declares into one xlsx workbook, one sheet per table. | | `odb_render_report` | Resolves one of an `.odb` database's own reports — its data-bound command run through the bounded SQL engine, its `rpt:` formulas evaluated, its bands laid out — and renders the result to docx, odt, or pdf. | diff --git a/packages/document-mcp/stryker.config.ts b/packages/document-mcp/stryker.config.ts index f90f7ba75..8500502ba 100644 --- a/packages/document-mcp/stryker.config.ts +++ b/packages/document-mcp/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 44.55% of 550 valid mutants, timeout share 0% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 43, }); diff --git a/packages/document-outline.js/stryker.config.ts b/packages/document-outline.js/stryker.config.ts index 88581839d..2d805a715 100644 --- a/packages/document-outline.js/stryker.config.ts +++ b/packages/document-outline.js/stryker.config.ts @@ -1,3 +1,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; -export default packageStrykerConfig(); +export default packageStrykerConfig({ + // First CI-measured baseline: 63.87% of 1885 valid mutants, timeout share 1.9% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 61, +}); diff --git a/packages/document-schema.js/stryker.config.ts b/packages/document-schema.js/stryker.config.ts index 4f42dd81f..ee2152b56 100644 --- a/packages/document-schema.js/stryker.config.ts +++ b/packages/document-schema.js/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.unit.config.ts", + // First CI-measured baseline: 23.03% of 3822 valid mutants, timeout share 0.2% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 22, }); diff --git a/packages/epub-codec/stryker.config.ts b/packages/epub-codec/stryker.config.ts index f90f7ba75..f729f0d2c 100644 --- a/packages/epub-codec/stryker.config.ts +++ b/packages/epub-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 66.23% of 2150 valid mutants, timeout share 1.3% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 64, }); diff --git a/packages/excel-number-format/stryker.config.ts b/packages/excel-number-format/stryker.config.ts index 88581839d..6c5aa3215 100644 --- a/packages/excel-number-format/stryker.config.ts +++ b/packages/excel-number-format/stryker.config.ts @@ -1,3 +1,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; -export default packageStrykerConfig(); +export default packageStrykerConfig({ + // First CI-measured baseline: 78.15% of 302 valid mutants, timeout share 7.9% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 70, +}); diff --git a/packages/markdown-codec/stryker.config.ts b/packages/markdown-codec/stryker.config.ts index f90f7ba75..9285c7182 100644 --- a/packages/markdown-codec/stryker.config.ts +++ b/packages/markdown-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 76.32% of 4529 valid mutants, timeout share 4.9% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 71, }); diff --git a/packages/odf.js/stryker.config.ts b/packages/odf.js/stryker.config.ts index f90f7ba75..2f5854b88 100644 --- a/packages/odf.js/stryker.config.ts +++ b/packages/odf.js/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 71.31% of 8442 valid mutants, timeout share 1.3% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 69, }); diff --git a/packages/ooxml.js/stryker.config.ts b/packages/ooxml.js/stryker.config.ts index f90f7ba75..10fa317f4 100644 --- a/packages/ooxml.js/stryker.config.ts +++ b/packages/ooxml.js/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 64.55% of 6823 valid mutants, timeout share 0.4% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 63, }); diff --git a/packages/pdf-codec/src/math-stretch.test.ts b/packages/pdf-codec/src/math-stretch.test.ts index 60f2b420a..3d78f31fa 100644 --- a/packages/pdf-codec/src/math-stretch.test.ts +++ b/packages/pdf-codec/src/math-stretch.test.ts @@ -164,7 +164,7 @@ describe("MathVariants parsing against the real STIX Two Math font", () => { ]); }); - // Enumerating the whole 0x1FFFF codepoint range is cheap uninstrumented (well under 200ms), but `pnpm test:coverage`'s v8 coverage instrumentation adds per-call overhead to every one of the ~131,000 glyphId() calls below, which is enough to clear vitest's 5000ms default on a slower CI runner -- an explicit timeout, not a change to what this test checks. A CI runner busy with a large `--affected` set has been observed taking this test itself over 30s (ExaDev/documents.js#1002), so the budget below leaves real headroom above the observed worst case rather than matching it. + // Enumerating the whole 0x1FFFF codepoint range is cheap uninstrumented (well under 200ms), but instrumentation multiplies the per-call cost of every one of the ~131,000 glyphId() calls below: `pnpm test:coverage`'s v8 coverage has been observed taking this test over 30s on a busy CI runner (ExaDev/documents.js#1002), and Stryker's mutant instrumentation is an order of magnitude heavier still, measuring ~28s for this test on a fast local machine and exceeding 90s on a GitHub mutation runner (ExaDev/documents.js#1194). An explicit timeout, not a change to what this test checks: the budget below leaves headroom above the worst instrumented case (a fully-instrumented mutation dry run on a loaded runner) rather than matching it. it("names glyphs that no Unicode code point reaches, which is why drawing a construction needs glyph IDs rather than text", () => { const font = loadMathFont(); const encoded = new Set(); @@ -198,7 +198,7 @@ describe("MathVariants parsing against the real STIX Two Math font", () => { .assembly!.parts) { expect(encoded.has(part.glyphId)).toBe(false); } - }, 90_000); + }, 300_000); it("reads the radical sign's own vertical construction", () => { const construction = verticalConstruction(RADICAL); diff --git a/packages/pdf-codec/stryker.config.ts b/packages/pdf-codec/stryker.config.ts index f90f7ba75..06152deea 100644 --- a/packages/pdf-codec/stryker.config.ts +++ b/packages/pdf-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // 15 minutes, not Stryker's default 5: the instrumented unit suite's single heaviest test (math-stretch.test.ts's whole-Unicode-range glyphId enumeration) alone measures ~28s instrumented on a fast local machine and has exceeded 90s on a GitHub runner -- the same instrumented suite that finishes the plain unit run in seconds needs several minutes of dry-run budget under mutation instrumentation, and the default left no room for the rest of the suite on top of it. + dryRunTimeoutMinutes: 15, }); diff --git a/packages/ppt-codec/stryker.config.ts b/packages/ppt-codec/stryker.config.ts index f90f7ba75..45bf2fd9a 100644 --- a/packages/ppt-codec/stryker.config.ts +++ b/packages/ppt-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 71.28% of 1497 valid mutants, timeout share 1.3% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 69, }); diff --git a/packages/rtf-codec/stryker.config.ts b/packages/rtf-codec/stryker.config.ts index f90f7ba75..27ba9ba4c 100644 --- a/packages/rtf-codec/stryker.config.ts +++ b/packages/rtf-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 70.65% of 3275 valid mutants, timeout share 2.1% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 67, }); diff --git a/packages/web/vitest.mutation.config.ts b/packages/web/vitest.mutation.config.ts index cbfb6194d..69c32589a 100644 --- a/packages/web/vitest.mutation.config.ts +++ b/packages/web/vitest.mutation.config.ts @@ -1,12 +1,13 @@ import { defineConfig } from "vitest/config"; import baseConfig from "./vite.config"; -// Isolates the "unit" project out of vite.config.ts's multi-project test config for Stryker's vitest-runner, which loads one plain config file and has no equivalent of --project to select among several. test is replaced outright with the unit project's own include glob (an explicit key in an object literal always overrides whatever the earlier spread carried for that same key), so a stale projects/coverage key from the base config's own test block can't survive into this one -- Stryker never picks up the smoke/workers suites (which import from dist/ or need a different runtime and are not meaningful per-mutant) or fight over coverage instrumentation, which Stryker's own runner disables unconditionally anyway. vite.config.ts's default export is a config function (base needs vite's own real command/serve discriminator to stay off the GitHub Pages subpath outside a production build), not a plain object -- resolved here the same way vite itself would, with mode "test" since that's what this config is actually for. +// Isolates the "unit" project out of vite.config.ts's multi-project test config for Stryker's vitest-runner, which loads one plain config file and has no equivalent of --project to select among several. test is replaced outright with the unit project's own include glob (an explicit key in an object literal always overrides whatever the earlier spread carried for that same key), so a stale projects/coverage key from the base config's own test block can't survive into this one -- Stryker never picks up the smoke/workers suites (which import from dist/ or need a different runtime and are not meaningful per-mutant) or fight over coverage instrumentation, which Stryker's own runner disables unconditionally anyway. Replacing test outright also drops the unit project's own `environment: "jsdom"` (it lives inside the projects entry, not at test's top level), so it is restated here explicitly: without it the runner's DOM-rendering tests execute in plain node and the very first one fails the whole dry run with "document is not defined", which is exactly how this package's mutation runs were failing before the key was restated. vite.config.ts's default export is a config function (base needs vite's own real command/serve discriminator to stay off the GitHub Pages subpath outside a production build), not a plain object -- resolved here the same way vite itself would, with mode "test" since that's what this config is actually for. const base = baseConfig({ command: "serve", mode: "test" }); export default defineConfig({ ...base, test: { include: ["src/**/*.test.ts", "src/**/*.test.tsx"], + environment: "jsdom", }, }); diff --git a/packages/wpd-codec/stryker.config.ts b/packages/wpd-codec/stryker.config.ts index f90f7ba75..6ffcc00a5 100644 --- a/packages/wpd-codec/stryker.config.ts +++ b/packages/wpd-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 66.07% of 2013 valid mutants, timeout share 1.2% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 64, }); diff --git a/packages/xls-codec/stryker.config.ts b/packages/xls-codec/stryker.config.ts index f90f7ba75..58d88343b 100644 --- a/packages/xls-codec/stryker.config.ts +++ b/packages/xls-codec/stryker.config.ts @@ -2,4 +2,6 @@ import { packageStrykerConfig } from "../../stryker.shared.ts"; export default packageStrykerConfig({ vitestConfigFile: "vitest.mutation.config.ts", + // First CI-measured baseline: 68.92% of 4166 valid mutants, timeout share 1.6% -- break = floor(score) minus the timeout share rounded up to whole points (minimum one), per the derivation rule on PackageStrykerOptions.breakThreshold. + breakThreshold: 66, }); diff --git a/stryker.shared.ts b/stryker.shared.ts index d9c4b33a1..45f225496 100644 --- a/stryker.shared.ts +++ b/stryker.shared.ts @@ -18,6 +18,10 @@ export interface PackageStrykerOptions { tsconfigFile?: string; // Path to the vitest config Stryker's vitest-runner should load. Left undefined for a package with no multi-project split (Vitest's own zero-config discovery already finds exactly the right test files, matching what its plain `vitest run` _test script does); set to "vitest.mutation.config.ts" (generated alongside this file) for a package whose real vitest.config.ts (or vite.config.ts) splits unit/smoke/workers into named projects, since Stryker's vitest-runner has no --project-equivalent selector and would otherwise also try to run the smoke suite (which imports from a dist/ Stryker's sandboxed copy never builds). vitestConfigFile?: string; + // Whole-dry-run budget in minutes, passed straight through to Stryker's own option of the same name (default 5). Only a package whose INSTRUMENTED unit suite can legitimately approach the default needs this: instrumentation multiplies per-call cost far beyond what the plain or v8-coverage-instrumented suite costs, so a package with one pathologically call-heavy test (pdf-codec's whole-Unicode-range font enumeration is the measured case: ~28s instrumented on a fast local machine, several multiples of that on a GitHub runner) can burn most of the default budget on a single test. Passed by a package only once measured, never speculatively -- the default 5 minutes fits every package whose dry run has actually completed within it. + dryRunTimeoutMinutes?: number; + // Stryker's thresholds.break for this package alone: the mutation score below which stryker exits non-zero and fails the mutation CI job. Derived, never picked -- the rule is documented alongside the thresholds key in packageStrykerConfig below, and a package that has never completed a full mutation run passes nothing and stays ungated until it does (a break guessed without a measured baseline is exactly the magic number this workspace refuses). + breakThreshold?: number; } /** @@ -32,6 +36,8 @@ export function packageStrykerConfig( mutate = ["src/**/*.ts", "!src/**/*.test.ts", "!src/**/*.test.tsx"], tsconfigFile = "tsconfig.json", vitestConfigFile, + dryRunTimeoutMinutes, + breakThreshold, } = options; return { @@ -54,8 +60,12 @@ export function packageStrykerConfig( incremental: true, // Static mutants (module-load-time code) can only be killed by a test that fails on IMPORT, so each one re-runs its ENTIRE related suite -- measured directly against document-schema.js: Stryker's own MutantTestPlanner reported 2989 of 5217 mutants (57%) as static, estimated at 92% of the run's total time. Dropping them is what makes a cold run (no incremental cache to restore -- documents.js and pdf-codec, this workspace's two largest packages, will hit this on their very first CI run) finish inside mutation.yml's own job timeout at all. This can only RAISE a package's score (static mutants are disproportionately survived/no-coverage, never killed), so it never needs revisiting once a package's baseline is eventually measured. ignoreStatic: true, - // high/low colour-code the HTML/clear-text report; deliberately no `break`. No package in this workspace has a measured baseline mutation score yet, and a `break` threshold picked without one would be an arbitrary number rather than a derived one -- exactly the magic-number failure mode to avoid. mutation.yml runs this workspace-wide, sharded and cached, purely to gather real per-package scores; once a package's own baseline is measured, add a `break` to that package's own stryker.config.ts (never here, since a workspace-wide `break` would either be too strict for the workspace's smallest, least-tested package or too lax for its most mature one) and consider promoting the CI job to a required check at that point. - thresholds: { high: 80, low: 60 }, + // high/low colour-code the HTML/clear-text report. `break` is deliberately per-package (a workspace-wide break would be too strict for the least-tested package or too lax for the most mature one) and derived, never picked: take the package's first CI-measured mutation score from mutation.yml's per-shard HTML reports, floor it to whole points, and subtract a noise margin of that package's own Timeout-classified share of its valid mutants, rounded up to whole points with a floor of one. Timeout is the one mutant classification that legitimately flaps between runs -- runner load alone decides whether the same mutant times out (counted detected) or survives -- so that margin keeps even every timeout in a package re-classifying from tripping the break, while a drop beyond it is a real regression. A package whose mutation run has never completed passes no breakThreshold and stays ungated until one does; picking its number without the measurement would be exactly the arbitrary magic number this rule exists to avoid. + thresholds: { + high: 80, + low: 60, + ...(breakThreshold === undefined ? {} : { break: breakThreshold }), + }, // dist/coverage/.turbo are build/tooling output Stryker would otherwise copy into every mutant's own sandbox for nothing -- none of it is ever read by a test. ignorePatterns: ["dist", "coverage", ".turbo"], reporters: ["progress", "clear-text", "html"], @@ -63,5 +73,6 @@ export function packageStrykerConfig( cleanTempDir: true, concurrency: 4, timeoutMS: 30000, + ...(dryRunTimeoutMinutes === undefined ? {} : { dryRunTimeoutMinutes }), }; }