From ceb86edaa6e0ea7818f8c08877c3c3f199838062 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 03:41:25 +0100 Subject: [PATCH 1/6] Repair backend Stryker mutation lane Signed-off-by: Chris0Jeky --- .github/workflows/mutation-testing.yml | 6 +- backend/stryker-config.json | 4 +- scripts/ci/Test-StrykerConfig.ps1 | 101 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 scripts/ci/Test-StrykerConfig.ps1 diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index d3660082f..f5d18948e 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -46,7 +46,11 @@ jobs: dotnet-version: 8.0.x - name: Install Stryker.NET - run: dotnet tool install --global dotnet-stryker + run: dotnet tool install --global dotnet-stryker --version 4.16.0 + + - name: Validate Stryker.NET configuration + shell: pwsh + run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest - name: Restore solution run: dotnet restore backend/Taskdeck.sln diff --git a/backend/stryker-config.json b/backend/stryker-config.json index b7694f1ba..95261e705 100644 --- a/backend/stryker-config.json +++ b/backend/stryker-config.json @@ -19,7 +19,7 @@ }, "language-version": "latest", "verbosity": "info", - "ignored-methods": [], - "excluded-mutations": [] + "ignore-methods": [], + "ignore-mutations": [] } } diff --git a/scripts/ci/Test-StrykerConfig.ps1 b/scripts/ci/Test-StrykerConfig.ps1 new file mode 100644 index 000000000..e16764621 --- /dev/null +++ b/scripts/ci/Test-StrykerConfig.ps1 @@ -0,0 +1,101 @@ +[CmdletBinding()] +param( + [string]$ConfigPath, + [switch]$SelfTest +) + +$ErrorActionPreference = 'Stop' + +if ([string]::IsNullOrWhiteSpace($ConfigPath)) { + $repositoryRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + $ConfigPath = Join-Path (Join-Path $repositoryRoot 'backend') 'stryker-config.json' +} + +function Test-StrykerConfig { + param( + [Parameter(Mandatory)] + [string]$Path + ) + + $resolvedPath = Resolve-Path -LiteralPath $Path -ErrorAction Stop + + try { + $config = Get-Content -LiteralPath $resolvedPath -Raw | ConvertFrom-Json -ErrorAction Stop + } catch { + throw "Stryker configuration '$resolvedPath' is not valid JSON: $($_.Exception.Message)" + } + + $strykerConfig = $config.'stryker-config' + if ($null -eq $strykerConfig) { + throw "Stryker configuration '$resolvedPath' must contain a stryker-config object." + } + + foreach ($obsoleteKey in @('ignored-methods', 'excluded-mutations')) { + if ($null -ne $strykerConfig.PSObject.Properties[$obsoleteKey]) { + $replacementKey = $obsoleteKey -replace '^excluded-', 'ignore-' -replace '^ignored-', 'ignore-' + throw "Stryker configuration '$resolvedPath' uses obsolete '$obsoleteKey'. Use '$replacementKey' instead." + } + } + + foreach ($requiredKey in @('ignore-methods', 'ignore-mutations')) { + $property = $strykerConfig.PSObject.Properties[$requiredKey] + if ($null -eq $property) { + throw "Stryker configuration '$resolvedPath' must contain '$requiredKey' as an array." + } + + if ($property.Value -isnot [System.Array]) { + throw "Stryker configuration '$resolvedPath' must define '$requiredKey' as an array." + } + } +} + +function Invoke-StrykerConfigSelfTest { + $temporaryDirectory = Join-Path ([System.IO.Path]::GetTempPath()) ("taskdeck-stryker-config-$([guid]::NewGuid().ToString('N'))") + $validPath = Join-Path $temporaryDirectory 'valid.json' + + try { + New-Item -ItemType Directory -Path $temporaryDirectory -ErrorAction Stop | Out-Null + Copy-Item -LiteralPath $ConfigPath -Destination $validPath -ErrorAction Stop + Test-StrykerConfig -Path $validPath + + foreach ($obsoleteKey in @('ignored-methods', 'excluded-mutations')) { + $obsoletePath = Join-Path $temporaryDirectory "$obsoleteKey.json" + @" +{ + "stryker-config": { + "ignore-methods": [], + "ignore-mutations": [], + "$obsoleteKey": [] + } +} +"@ | Set-Content -LiteralPath $obsoletePath -Encoding UTF8 + + $obsoleteKeyWasRejected = $false + try { + Test-StrykerConfig -Path $obsoletePath + } catch { + if ($_.Exception.Message -notmatch "obsolete '$obsoleteKey'") { + throw + } + + $obsoleteKeyWasRejected = $true + } + + if (-not $obsoleteKeyWasRejected) { + throw "The Stryker configuration preflight accepted obsolete key '$obsoleteKey'." + } + } + } finally { + if (Test-Path -LiteralPath $temporaryDirectory) { + Remove-Item -LiteralPath $temporaryDirectory -Recurse -Force + } + } +} + +Test-StrykerConfig -Path $ConfigPath + +if ($SelfTest) { + Invoke-StrykerConfigSelfTest +} + +Write-Host "Stryker configuration preflight passed: $ConfigPath" From 6c9ccb5dfb9f68f3a368690f7cfed65d207fbb03 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 03:41:29 +0100 Subject: [PATCH 2/6] Record backend mutation lane failure Signed-off-by: Chris0Jeky --- docs/agentic/FAILURE_LEDGER.md | 1 + docs/agentic/failure_ledger.jsonl | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/agentic/FAILURE_LEDGER.md b/docs/agentic/FAILURE_LEDGER.md index 5ec2c0c5c..75d82be6d 100644 --- a/docs/agentic/FAILURE_LEDGER.md +++ b/docs/agentic/FAILURE_LEDGER.md @@ -26,6 +26,7 @@ Rows sharing a surface and first tracking issue in `future_fix` show only their | 2026-07-26 | non_blocking_risk | agent/tool-command-composition | Resolution record for the repeated #1490 PowerShell/native command-composition failures; the original recurrence remains in append-only history | No workaround remains; use the copyable safe forms and classify future failures through the ledger process | #1490 resolved by PR #1491 merge 61f718af941c56c31b8b7595333b9debd6e47da8: the MCP tooling guide now covers collected foreach output, optional rg exits, safe mu... | resolved | | 2026-07-26 | blocker | agent/python-launcher | Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the ... | Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows | #1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue clo... | open | | 2026-07-26 | blocker | agent/powershell-deny-floor | The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy | Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception | #1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception | open | +| 2026-07-27 | non_blocking_risk | ci/mutation-testing-backend | All 16 recorded scheduled Mutation Testing runs failed to produce backend Stryker evidence; the latest runs 30191449339 and 29676656172 both stopped because Str... | Treat the frontend lane as independent evidence, but do not claim backend mutation coverage until a backend-only exact-head run completes and uploads a non-empt... | #1500 pins Stryker.NET 4.16.0, uses ignore-methods and ignore-mutations, and adds a local preflight that rejects obsolete keys; resolve this row only after the ... | open | ## Classification diff --git a/docs/agentic/failure_ledger.jsonl b/docs/agentic/failure_ledger.jsonl index aed23731f..d431db90c 100644 --- a/docs/agentic/failure_ledger.jsonl +++ b/docs/agentic/failure_ledger.jsonl @@ -25,3 +25,4 @@ {"ts":"2026-07-25T23:48:15Z","class":"blocker","surface":"agent/python-launcher","failure":"Native-Windows python and python3 commands resolve to unusable Microsoft Store aliases, so configured agent hooks and documented failure-ledger utilities do not run even though CPython is installed behind py -3","workaround":"Invoke py -3 -B explicitly on Windows and python3 -B on POSIX; do not install another interpreter or mutate global PATH","future_fix":"#1487: make platform launchers explicit across configured handlers, permissions, smoke execution, and mirrored guidance","status":"open"} {"ts":"2026-07-26T10:56:00Z","class":"blocker","surface":"agent/python-launcher","failure":"Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the active sys.executable -B","workaround":"Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows","future_fix":"#1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue closure","status":"open"} {"ts":"2026-07-26T11:36:55Z","class":"blocker","surface":"agent/powershell-deny-floor","failure":"The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy","workaround":"Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception","future_fix":"#1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception","status":"open"} +{"ts":"2026-07-27T00:00:00Z","class":"non_blocking_risk","surface":"ci/mutation-testing-backend","failure":"All 16 recorded scheduled Mutation Testing runs failed to produce backend Stryker evidence; the latest runs 30191449339 and 29676656172 both stopped because Stryker.NET 4.16.0 rejects ignored-methods and excluded-mutations as invalid schema keys.","workaround":"Treat the frontend lane as independent evidence, but do not claim backend mutation coverage until a backend-only exact-head run completes and uploads a non-empty report artifact.","future_fix":"#1500 pins Stryker.NET 4.16.0, uses ignore-methods and ignore-mutations, and adds a local preflight that rejects obsolete keys; resolve this row only after the PR exact-head backend-only dispatch completes mutation execution with a non-empty Stryker artifact.","status":"open"} From b3e5629fbb64ce76a812564fd891b1b26565e447 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 05:06:15 +0100 Subject: [PATCH 3/6] Bound backend mutation execution Signed-off-by: Chris0Jeky --- .github/workflows/mutation-testing.yml | 7 +- backend/stryker-config.json | 4 - docs/TESTING_GUIDE.md | 23 +- docs/testing/MUTATION_TESTING_POLICY.md | 25 ++- scripts/ci/Test-StrykerConfig.ps1 | 272 +++++++++++++++++++++--- 5 files changed, 275 insertions(+), 56 deletions(-) diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index f5d18948e..e3963238a 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -35,7 +35,7 @@ jobs: name: Backend Mutation (Stryker.NET) if: ${{ !inputs.frontend_only }} runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 360 steps: - name: Checkout uses: actions/checkout@v7 @@ -56,8 +56,8 @@ jobs: run: dotnet restore backend/Taskdeck.sln - name: Run Stryker.NET - working-directory: backend - run: dotnet stryker --config-file stryker-config.json + working-directory: backend/tests/Taskdeck.Domain.Tests + run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput - name: Upload Stryker report if: always() @@ -65,6 +65,7 @@ jobs: with: name: stryker-net-report path: backend/StrykerOutput/**/reports/ + if-no-files-found: error retention-days: 30 frontend-mutation: diff --git a/backend/stryker-config.json b/backend/stryker-config.json index 95261e705..511f20120 100644 --- a/backend/stryker-config.json +++ b/backend/stryker-config.json @@ -1,10 +1,6 @@ { "stryker-config": { "project": "Taskdeck.Domain.csproj", - "test-projects": [ - "tests/Taskdeck.Domain.Tests/Taskdeck.Domain.Tests.csproj" - ], - "solution": "Taskdeck.sln", "mutation-level": "Standard", "reporters": [ "html", diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index f1e111d22..af023e546 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -2,7 +2,7 @@ This is the active testing guide for Taskdeck. -Last Updated: 2026-07-26 +Last Updated: 2026-07-27 Companion Active Docs: - `docs/STATUS.md` - `docs/IMPLEMENTATION_MASTERPLAN.md` @@ -642,18 +642,22 @@ CI: `reusable-visual-regression.yml` in extended CI (testing/visual label). Uplo ### Mutation Testing (TST-05, `#90`/`#796`) -Backend (Stryker.NET): targets `Taskdeck.Domain` with `Taskdeck.Domain.Tests`. Thresholds: break=60, high=80. +Backend (Stryker.NET 4.16.0): targets `Taskdeck.Domain` from the `Taskdeck.Domain.Tests` project context. Thresholds: break=0, low=60, high=80. The checked-in preflight rejects obsolete config keys and solution-context/workflow drift before the long mutation run. Frontend (Stryker JS): targets `captureStore`, `boardStore`, and `board/*.ts` submodules with vitest runner. Run commands: ```bash # Backend -cd backend && dotnet tool install dotnet-stryker && dotnet stryker +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7/Linux runner equivalent: pwsh -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +dotnet tool install --global dotnet-stryker --version 4.16.0 +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput # Frontend cd frontend/taskdeck-web && npm run mutation:test ``` -CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. Non-blocking, reports uploaded as artifacts. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. +CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. The backend job has a finite 360-minute ceiling for the full Domain mutation set and fails if no report artifact exists. Mutation score remains non-blocking. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. ### Container Integration Tests (TST-06, `#91`/`#804`) @@ -812,9 +816,12 @@ Mutation testing is available as a non-blocking quality signal for detecting wea ### Running locally ```bash -# Backend (requires dotnet-stryker global tool) -cd backend -dotnet stryker --config-file stryker-config.json +# Backend (from the repository root; uses the workflow-pinned tool version) +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7/Linux runner equivalent: pwsh -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +dotnet tool install --global dotnet-stryker --version 4.16.0 +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput # Frontend cd frontend/taskdeck-web @@ -823,7 +830,7 @@ npm run mutation:test ### CI -Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. Reports uploaded as artifacts. +Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. The backend job has a 360-minute ceiling and missing backend reports fail artifact upload. ### Policy and triage diff --git a/docs/testing/MUTATION_TESTING_POLICY.md b/docs/testing/MUTATION_TESTING_POLICY.md index dcaf258f9..bac454b36 100644 --- a/docs/testing/MUTATION_TESTING_POLICY.md +++ b/docs/testing/MUTATION_TESTING_POLICY.md @@ -1,6 +1,6 @@ # Mutation Testing Policy -Last Updated: 2026-04-09 +Last Updated: 2026-07-27 ## Purpose @@ -14,8 +14,11 @@ This is a **quality signal**, not a gatekeeping mechanism. Mutation testing comp - **Target**: `Taskdeck.Domain` project - **Test project**: `Taskdeck.Domain.Tests` -- **Rationale**: Domain contains core business logic (entity state machines, validation rules, invariants) where surviving mutants have the highest impact. Domain is pure C# with no infrastructure dependencies, making mutation runs fast and deterministic. +- **Tool contract**: Stryker.NET `4.16.0` +- **Rationale**: Domain contains core business logic (entity state machines, validation rules, invariants) where surviving mutants have the highest impact. Domain is pure C# with no infrastructure dependencies, making it the narrowest deterministic backend target; its full mutation set is still a long-running workload. - **Config**: `backend/stryker-config.json` +- **Execution context**: run from `backend/tests/Taskdeck.Domain.Tests`; do not add `solution` or `test-projects` to the config because solution context takes precedence and discovers unrelated tests +- **Preflight**: `scripts/ci/Test-StrykerConfig.ps1 -SelfTest` rejects obsolete schema keys, solution-context selectors, and workflow/artifact drift before the long mutation run starts ### Frontend (Stryker JS/TS) @@ -51,12 +54,17 @@ After the first 3-4 runs: ### Backend (local) ```bash -# Install Stryker.NET as a global tool (once) -dotnet tool install --global dotnet-stryker +# From the repository root, validate the checked-in schema contract. +# Native Windows PowerShell: +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7 on Linux/macOS/Windows uses the equivalent `pwsh -File ...` form. -# Run from the backend/ directory -cd backend -dotnet stryker --config-file stryker-config.json +# Install the workflow-pinned Stryker.NET version as a global tool (once). +dotnet tool install --global dotnet-stryker --version 4.16.0 + +# Run from the Domain test project so Stryker discovers only that test project. +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput ``` Report: `backend/StrykerOutput//reports/mutation-report.html` @@ -77,6 +85,7 @@ The mutation testing workflow runs: - **On demand**: via `workflow_dispatch` from the Actions tab Reports are uploaded as GitHub Actions artifacts with 30-day retention. +The backend job has a finite 360-minute ceiling for the full Domain mutation set, and artifact upload fails when no report was produced. ## Interpreting Reports @@ -105,7 +114,7 @@ When mutation testing reveals surviving mutants: 2. **Categorize** surviving mutants by triage priority (see above) 3. **Bundle fixes**: Group related assertion improvements into a single PR per module rather than one PR per mutant 4. **Do not chase 100%**: Some surviving mutants are acceptable (e.g., log messages, cosmetic formatting). Document intentional exclusions: - - **Backend (Stryker.NET)**: Use `excluded-mutations` or `ignored-methods` in `backend/stryker-config.json` + - **Backend (Stryker.NET 4.16.0)**: Use `ignore-mutations` or `ignore-methods` in `backend/stryker-config.json`; the preflight deliberately rejects the obsolete `excluded-mutations` and `ignored-methods` spellings - **Frontend (Stryker JS)**: Adjust `mutate` glob patterns in `stryker.config.mjs` or use inline `// Stryker disable` comments in source files ## Scope Expansion Roadmap diff --git a/scripts/ci/Test-StrykerConfig.ps1 b/scripts/ci/Test-StrykerConfig.ps1 index e16764621..9bea7a0a6 100644 --- a/scripts/ci/Test-StrykerConfig.ps1 +++ b/scripts/ci/Test-StrykerConfig.ps1 @@ -1,16 +1,23 @@ [CmdletBinding()] param( [string]$ConfigPath, + [string]$WorkflowPath, [switch]$SelfTest ) $ErrorActionPreference = 'Stop' +$repositoryRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + if ([string]::IsNullOrWhiteSpace($ConfigPath)) { - $repositoryRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) $ConfigPath = Join-Path (Join-Path $repositoryRoot 'backend') 'stryker-config.json' } +if ([string]::IsNullOrWhiteSpace($WorkflowPath)) { + $workflowsDirectory = Join-Path (Join-Path $repositoryRoot '.github') 'workflows' + $WorkflowPath = Join-Path $workflowsDirectory 'mutation-testing.yml' +} + function Test-StrykerConfig { param( [Parameter(Mandatory)] @@ -37,54 +44,252 @@ function Test-StrykerConfig { } } - foreach ($requiredKey in @('ignore-methods', 'ignore-mutations')) { - $property = $strykerConfig.PSObject.Properties[$requiredKey] - if ($null -eq $property) { - throw "Stryker configuration '$resolvedPath' must contain '$requiredKey' as an array." + foreach ($solutionContextKey in @('solution', 'test-projects')) { + if ($null -ne $strykerConfig.PSObject.Properties[$solutionContextKey]) { + throw "Stryker configuration '$resolvedPath' must omit '$solutionContextKey' so the workflow stays in Taskdeck.Domain.Tests context." + } + } + + $projectProperty = $strykerConfig.PSObject.Properties['project'] + if ($null -eq $projectProperty -or $projectProperty.Value -cne 'Taskdeck.Domain.csproj') { + throw "Stryker configuration '$resolvedPath' must target project 'Taskdeck.Domain.csproj'." + } + + $mutationLevelProperty = $strykerConfig.PSObject.Properties['mutation-level'] + if ($null -eq $mutationLevelProperty -or $mutationLevelProperty.Value -cne 'Standard') { + throw "Stryker configuration '$resolvedPath' must preserve mutation-level 'Standard'." + } + + $reportersProperty = $strykerConfig.PSObject.Properties['reporters'] + if ($null -eq $reportersProperty -or $reportersProperty.Value -isnot [System.Array]) { + throw "Stryker configuration '$resolvedPath' must define reporters as an array." + } + + $reporters = @($reportersProperty.Value) + if (($reporters -join ',') -cne 'html,json,progress,cleartext') { + throw "Stryker configuration '$resolvedPath' must preserve html, json, progress, and cleartext reporters." + } + + $thresholdsProperty = $strykerConfig.PSObject.Properties['thresholds'] + $thresholds = $thresholdsProperty.Value + if ($null -eq $thresholdsProperty -or + $null -eq $thresholds -or + $thresholds.high -ne 80 -or + $thresholds.low -ne 60 -or + $thresholds.break -ne 0) { + throw "Stryker configuration '$resolvedPath' must preserve thresholds high=80, low=60, break=0." + } + + foreach ($requiredEmptyKey in @('ignore-methods', 'ignore-mutations')) { + $property = $strykerConfig.PSObject.Properties[$requiredEmptyKey] + if ($null -eq $property -or $property.Value -isnot [System.Array]) { + throw "Stryker configuration '$resolvedPath' must contain '$requiredEmptyKey' as an array." } - if ($property.Value -isnot [System.Array]) { - throw "Stryker configuration '$resolvedPath' must define '$requiredKey' as an array." + if ($property.Value.Count -ne 0) { + throw "Stryker configuration '$resolvedPath' must preserve the empty '$requiredEmptyKey' list." } } } +function Assert-ExactWorkflowLine { + param( + [Parameter(Mandatory)] + [string]$Block, + [Parameter(Mandatory)] + [string]$ExpectedLine, + [Parameter(Mandatory)] + [string]$ContractName + ) + + $count = @(($Block -split "`n") | Where-Object { $_ -ceq $ExpectedLine }).Count + if ($count -ne 1) { + throw "Mutation workflow must contain exactly one $ContractName line '$ExpectedLine' in backend-mutation; found $count." + } +} + +function Assert-ExactWorkflowFragment { + param( + [Parameter(Mandatory)] + [string]$Block, + [Parameter(Mandatory)] + [string]$ExpectedFragment, + [Parameter(Mandatory)] + [string]$ContractName + ) + + $count = 0 + $searchIndex = 0 + while ($true) { + $matchIndex = $Block.IndexOf($ExpectedFragment, $searchIndex, [System.StringComparison]::Ordinal) + if ($matchIndex -lt 0) { + break + } + + $count++ + $searchIndex = $matchIndex + $ExpectedFragment.Length + } + + if ($count -ne 1) { + throw "Mutation workflow must contain exactly one $ContractName block in backend-mutation; found $count." + } +} + +function Test-MutationWorkflowContract { + param( + [Parameter(Mandatory)] + [string]$Path + ) + + $resolvedPath = Resolve-Path -LiteralPath $Path -ErrorAction Stop + $workflow = [System.IO.File]::ReadAllText($resolvedPath) + $normalizedWorkflow = $workflow.Replace("`r`n", "`n").Replace("`r", "`n") + $backendMarker = " backend-mutation:`n" + $frontendMarker = " frontend-mutation:`n" + $backendStart = $normalizedWorkflow.IndexOf($backendMarker, [System.StringComparison]::Ordinal) + $frontendStart = $normalizedWorkflow.IndexOf($frontendMarker, [System.StringComparison]::Ordinal) + + if ($backendStart -lt 0 -or $frontendStart -le $backendStart) { + throw "Mutation workflow '$resolvedPath' must contain backend-mutation before frontend-mutation." + } + + $backendBlock = $normalizedWorkflow.Substring($backendStart, $frontendStart - $backendStart) + $requiredLines = [ordered]@{ + 'finite timeout' = ' timeout-minutes: 360' + 'pinned tool install' = ' run: dotnet tool install --global dotnet-stryker --version 4.16.0' + 'configuration self-test' = ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest' + } + + foreach ($entry in $requiredLines.GetEnumerator()) { + Assert-ExactWorkflowLine -Block $backendBlock -ExpectedLine $entry.Value -ContractName $entry.Key + } + + $strykerStep = @( + ' - name: Run Stryker.NET', + ' working-directory: backend/tests/Taskdeck.Domain.Tests', + ' run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput' + ) -join "`n" + Assert-ExactWorkflowFragment -Block $backendBlock -ExpectedFragment $strykerStep -ContractName 'test-project Stryker step' + + $artifactStep = @( + ' - name: Upload Stryker report', + ' if: always()', + ' uses: actions/upload-artifact@v7', + ' with:', + ' name: stryker-net-report', + ' path: backend/StrykerOutput/**/reports/', + ' if-no-files-found: error', + ' retention-days: 30' + ) -join "`n" + Assert-ExactWorkflowFragment -Block $backendBlock -ExpectedFragment $artifactStep -ContractName 'fail-closed report artifact step' + + $strykerCommandCount = [regex]::Matches( + $backendBlock, + '(?m)^\s+run:\s+(?:dotnet stryker|(?:\S+/)?dotnet-stryker)(?:\s|$)' + ).Count + if ($strykerCommandCount -ne 1) { + throw "Mutation workflow '$resolvedPath' must invoke Stryker exactly once in backend-mutation; found $strykerCommandCount." + } +} + +function Assert-Rejected { + param( + [Parameter(Mandatory)] + [scriptblock]$Action, + [Parameter(Mandatory)] + [string]$Scenario, + [Parameter(Mandatory)] + [string]$ExpectedMessage + ) + + try { + & $Action + } catch { + if (-not $_.Exception.Message.Contains($ExpectedMessage)) { + throw "The Stryker preflight rejected '$Scenario' for an unexpected reason: $($_.Exception.Message)" + } + + return + } + + throw "The Stryker preflight accepted invalid scenario '$Scenario'." +} + +function Write-TextVariant { + param( + [Parameter(Mandatory)] + [string]$Source, + [Parameter(Mandatory)] + [string]$Destination, + [Parameter(Mandatory)] + [string]$Expected, + [Parameter(Mandatory)] + [string]$Replacement + ) + + $content = [System.IO.File]::ReadAllText($Source) + $firstMatch = $content.IndexOf($Expected, [System.StringComparison]::Ordinal) + $lastMatch = $content.LastIndexOf($Expected, [System.StringComparison]::Ordinal) + if ($firstMatch -lt 0 -or $firstMatch -ne $lastMatch) { + throw "Self-test fixture expected exactly one source fragment '$Expected'." + } + + [System.IO.File]::WriteAllText($Destination, $content.Replace($Expected, $Replacement)) +} + function Invoke-StrykerConfigSelfTest { $temporaryDirectory = Join-Path ([System.IO.Path]::GetTempPath()) ("taskdeck-stryker-config-$([guid]::NewGuid().ToString('N'))") - $validPath = Join-Path $temporaryDirectory 'valid.json' + $validConfigPath = Join-Path $temporaryDirectory 'valid.json' + $validWorkflowPath = Join-Path $temporaryDirectory 'valid.yml' try { New-Item -ItemType Directory -Path $temporaryDirectory -ErrorAction Stop | Out-Null - Copy-Item -LiteralPath $ConfigPath -Destination $validPath -ErrorAction Stop - Test-StrykerConfig -Path $validPath - - foreach ($obsoleteKey in @('ignored-methods', 'excluded-mutations')) { - $obsoletePath = Join-Path $temporaryDirectory "$obsoleteKey.json" - @" -{ - "stryker-config": { - "ignore-methods": [], - "ignore-mutations": [], - "$obsoleteKey": [] - } -} -"@ | Set-Content -LiteralPath $obsoletePath -Encoding UTF8 + Copy-Item -LiteralPath $ConfigPath -Destination $validConfigPath -ErrorAction Stop + Copy-Item -LiteralPath $WorkflowPath -Destination $validWorkflowPath -ErrorAction Stop + Test-StrykerConfig -Path $validConfigPath + Test-MutationWorkflowContract -Path $validWorkflowPath - $obsoleteKeyWasRejected = $false - try { - Test-StrykerConfig -Path $obsoletePath - } catch { - if ($_.Exception.Message -notmatch "obsolete '$obsoleteKey'") { - throw - } + $configVariants = @( + @('obsolete ignored-methods key', ' "ignore-methods": [],', ' "ignored-methods": [],', "uses obsolete 'ignored-methods'"), + @('obsolete excluded-mutations key', ' "ignore-mutations": []', ' "excluded-mutations": []', "uses obsolete 'excluded-mutations'"), + @('solution context', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "solution": "Taskdeck.sln",') -join "`n"), "must omit 'solution'"), + @('test-project selector', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "test-projects": ["tests/Taskdeck.Domain.Tests/Taskdeck.Domain.Tests.csproj"],') -join "`n"), "must omit 'test-projects'"), + @('wrong mutation target', ' "project": "Taskdeck.Domain.csproj",', ' "project": "Taskdeck.Application.csproj",', "must target project 'Taskdeck.Domain.csproj'"), + @('changed empty ignore semantics', ' "ignore-mutations": []', ' "ignore-mutations": ["string"]', "must preserve the empty 'ignore-mutations' list"), + @('changed mutation level', ' "mutation-level": "Standard",', ' "mutation-level": "Advanced",', "must preserve mutation-level 'Standard'"), + @('missing JSON reporter', ' "json",', ' "dashboard",', 'must preserve html, json, progress, and cleartext reporters'), + @('changed score threshold', ' "break": 0', ' "break": 60', 'must preserve thresholds high=80, low=60, break=0') + ) - $obsoleteKeyWasRejected = $true + for ($configIndex = 0; $configIndex -lt $configVariants.Count; $configIndex++) { + $configVariant = $configVariants[$configIndex] + $variantPath = Join-Path $temporaryDirectory "config-$configIndex.json" + Write-TextVariant -Source $ConfigPath -Destination $variantPath -Expected $configVariant[1] -Replacement $configVariant[2] + Assert-Rejected -Scenario $configVariant[0] -ExpectedMessage $configVariant[3] -Action { + Test-StrykerConfig -Path $variantPath } + } - if (-not $obsoleteKeyWasRejected) { - throw "The Stryker configuration preflight accepted obsolete key '$obsoleteKey'." + $workflowVariants = @( + @('timeout', ' timeout-minutes: 360', ' timeout-minutes: 60', 'finite timeout'), + @('tool version', ' run: dotnet tool install --global dotnet-stryker --version 4.16.0', ' run: dotnet tool install --global dotnet-stryker --version 4.17.0', 'pinned tool install'), + @('preflight mode', ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest', ' run: ./scripts/ci/Test-StrykerConfig.ps1', 'configuration self-test'), + @('working-directory', ' working-directory: backend/tests/Taskdeck.Domain.Tests', ' working-directory: backend', 'test-project Stryker step'), + @('config path', ' run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput', ' run: dotnet stryker --config-file stryker-config.json', 'test-project Stryker step'), + @('artifact path', ' path: backend/StrykerOutput/**/reports/', ' path: backend/tests/Taskdeck.Domain.Tests/StrykerOutput/**/reports/', 'fail-closed report artifact step'), + @('missing artifact policy', ' if-no-files-found: error', ' if-no-files-found: warn', 'fail-closed report artifact step') + ) + + foreach ($workflowVariant in $workflowVariants) { + $variantPath = Join-Path $temporaryDirectory "$($workflowVariant[0]).yml" + Write-TextVariant -Source $WorkflowPath -Destination $variantPath -Expected $workflowVariant[1] -Replacement $workflowVariant[2] + Assert-Rejected -Scenario "workflow $($workflowVariant[0]) drift" -ExpectedMessage $workflowVariant[3] -Action { + Test-MutationWorkflowContract -Path $variantPath } } + + $rejectedFixtureCount = $configVariants.Count + $workflowVariants.Count + Write-Host "Stryker preflight self-test passed: $($rejectedFixtureCount + 2) checks (2 valid contracts; $rejectedFixtureCount rejected drift fixtures)." } finally { if (Test-Path -LiteralPath $temporaryDirectory) { Remove-Item -LiteralPath $temporaryDirectory -Recurse -Force @@ -93,9 +298,10 @@ function Invoke-StrykerConfigSelfTest { } Test-StrykerConfig -Path $ConfigPath +Test-MutationWorkflowContract -Path $WorkflowPath if ($SelfTest) { Invoke-StrykerConfigSelfTest } -Write-Host "Stryker configuration preflight passed: $ConfigPath" +Write-Host "Stryker configuration and workflow preflight passed: $ConfigPath; $WorkflowPath" From 307add004fbe142321a6ec11be21fab708824d5d Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 05:09:00 +0100 Subject: [PATCH 4/6] Cap focused backend mutation runs Signed-off-by: Chris0Jeky --- .github/workflows/mutation-testing.yml | 2 +- docs/TESTING_GUIDE.md | 4 ++-- docs/testing/MUTATION_TESTING_POLICY.md | 2 +- scripts/ci/Test-StrykerConfig.ps1 | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index e3963238a..34846f18c 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -35,7 +35,7 @@ jobs: name: Backend Mutation (Stryker.NET) if: ${{ !inputs.frontend_only }} runs-on: ubuntu-latest - timeout-minutes: 360 + timeout-minutes: 180 steps: - name: Checkout uses: actions/checkout@v7 diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index af023e546..086cce5dd 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -657,7 +657,7 @@ dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOut cd frontend/taskdeck-web && npm run mutation:test ``` -CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. The backend job has a finite 360-minute ceiling for the full Domain mutation set and fails if no report artifact exists. Mutation score remains non-blocking. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. +CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. The backend job has a finite 180-minute ceiling for the full Domain mutation set and fails if no report artifact exists. Mutation score remains non-blocking. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. ### Container Integration Tests (TST-06, `#91`/`#804`) @@ -830,7 +830,7 @@ npm run mutation:test ### CI -Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. The backend job has a 360-minute ceiling and missing backend reports fail artifact upload. +Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. The backend job has a 180-minute ceiling and missing backend reports fail artifact upload. ### Policy and triage diff --git a/docs/testing/MUTATION_TESTING_POLICY.md b/docs/testing/MUTATION_TESTING_POLICY.md index bac454b36..b32952cf3 100644 --- a/docs/testing/MUTATION_TESTING_POLICY.md +++ b/docs/testing/MUTATION_TESTING_POLICY.md @@ -85,7 +85,7 @@ The mutation testing workflow runs: - **On demand**: via `workflow_dispatch` from the Actions tab Reports are uploaded as GitHub Actions artifacts with 30-day retention. -The backend job has a finite 360-minute ceiling for the full Domain mutation set, and artifact upload fails when no report was produced. +The backend job has a finite 180-minute ceiling for the full Domain mutation set, and artifact upload fails when no report was produced. ## Interpreting Reports diff --git a/scripts/ci/Test-StrykerConfig.ps1 b/scripts/ci/Test-StrykerConfig.ps1 index 9bea7a0a6..526168f94 100644 --- a/scripts/ci/Test-StrykerConfig.ps1 +++ b/scripts/ci/Test-StrykerConfig.ps1 @@ -155,7 +155,7 @@ function Test-MutationWorkflowContract { $backendBlock = $normalizedWorkflow.Substring($backendStart, $frontendStart - $backendStart) $requiredLines = [ordered]@{ - 'finite timeout' = ' timeout-minutes: 360' + 'finite timeout' = ' timeout-minutes: 180' 'pinned tool install' = ' run: dotnet tool install --global dotnet-stryker --version 4.16.0' 'configuration self-test' = ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest' } @@ -271,7 +271,7 @@ function Invoke-StrykerConfigSelfTest { } $workflowVariants = @( - @('timeout', ' timeout-minutes: 360', ' timeout-minutes: 60', 'finite timeout'), + @('timeout', ' timeout-minutes: 180', ' timeout-minutes: 60', 'finite timeout'), @('tool version', ' run: dotnet tool install --global dotnet-stryker --version 4.16.0', ' run: dotnet tool install --global dotnet-stryker --version 4.17.0', 'pinned tool install'), @('preflight mode', ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest', ' run: ./scripts/ci/Test-StrykerConfig.ps1', 'configuration self-test'), @('working-directory', ' working-directory: backend/tests/Taskdeck.Domain.Tests', ' working-directory: backend', 'test-project Stryker step'), From 89e5f9b1ce86823ae155e5fad25e0fae54cab2d5 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 05:18:27 +0100 Subject: [PATCH 5/6] Record bounded backend mutation evidence Signed-off-by: Chris0Jeky --- docs/IMPLEMENTATION_MASTERPLAN.md | 4 ++++ docs/STATUS.md | 3 +++ docs/TESTING_GUIDE.md | 2 ++ docs/agentic/FAILURE_LEDGER.md | 2 +- docs/agentic/failure_ledger.jsonl | 1 + docs/testing/MUTATION_TESTING_POLICY.md | 2 ++ 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/IMPLEMENTATION_MASTERPLAN.md b/docs/IMPLEMENTATION_MASTERPLAN.md index b37d752aa..8720ed11f 100644 --- a/docs/IMPLEMENTATION_MASTERPLAN.md +++ b/docs/IMPLEMENTATION_MASTERPLAN.md @@ -10,6 +10,10 @@ Companion Active Docs: - `docs/MANUAL_TEST_CHECKLIST.md` - `docs/GOLDEN_PRINCIPLES.md` +## Delivery update (2026-07-27, mutation-testing runway) + +- **Bounded backend mutation evidence (`#1500`, PR `#1503`, maintainer-held):** repaired the Domain Stryker.NET lane by pinning 4.16.0, validating its config/workflow contract with an 18-check preflight, running from the single Domain test-project context, capping the job at 180 minutes, and failing artifact upload when no report exists. Backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds over 3,682 mutants (2,351 killed, 576 survived, 2 timed out, 753 skipped), for a 70.75% baseline. Its two-file `stryker-net-report` artifact is 874,386 bytes with SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`. The score is calibration evidence, not a merge gate; expansion/ratcheting remains deferred until the existing 3-4-run policy baseline exists. + ## Delivery update (2026-07-26, agentic governance) - **Failure-ledger projection gate (`#1492`):** Required Docs Governance now pins Python 3.12 and runs the existing JSONL↔Markdown synchronization unittest before the governance checks, so a JSONL-only change with stale generated Markdown fails Required CI without regeneration masking it. Local agentic update workflows use the distinct render-then-test order so hook-appended JSONL can be projected, and the smoke contract pins both sides of that distinction. diff --git a/docs/STATUS.md b/docs/STATUS.md index 59b269bbb..7ba6e8c15 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2,6 +2,9 @@ Last Updated: 2026-07-27 +Backend mutation-testing runway repair (2026-07-27, `#1500`): +- **The scheduled Domain mutation lane now reaches a bounded, artifact-backed result instead of failing before execution.** Stryker.NET is pinned to 4.16.0; an 18-check PowerShell preflight rejects obsolete schema keys, wrong solution/test context, workflow timeout drift, and non-failing artifact uploads; and the workflow runs from `Taskdeck.Domain.Tests` with a 180-minute ceiling. Backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds: 3,682 mutants created, 2,351 killed, 576 survived, 2 timed out, 753 skipped, score 70.75%. The non-empty two-file `stryker-net-report` artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). The score remains a non-blocking calibration signal; PR `#1503` and merge remain maintainer-held because the workflow changes operational CI posture. + Required Docs Governance hardening (2026-07-26, `#1492`): - **Required CI now enforces failure-ledger projection synchronization.** The reusable Docs Governance job pins Python 3.12 and runs the existing `failure_ledger.jsonl` ↔ `FAILURE_LEDGER.md` synchronization unittest before its governance checks, so a stale checked-in projection fails without any renderer masking it. Local agentic update workflows intentionally render first and then test so a valid hook-appended JSONL entry can be projected; the smoke contract keeps that distinction from drifting. diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index 086cce5dd..490567cd0 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -659,6 +659,8 @@ cd frontend/taskdeck-web && npm run mutation:test CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. The backend job has a finite 180-minute ceiling for the full Domain mutation set and fails if no report artifact exists. Mutation score remains non-blocking. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. +Verified baseline: backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds. Stryker created 3,682 mutants; 2,351 were killed, 576 survived, 2 timed out, and 753 were skipped, for a 70.75% score. The uploaded two-file `stryker-net-report` artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). + ### Container Integration Tests (TST-06, `#91`/`#804`) New `Taskdeck.Integration.Tests` project using `Testcontainers.PostgreSql` for ephemeral database isolation. Each test method gets a fresh PostgreSQL database. Requires Docker. diff --git a/docs/agentic/FAILURE_LEDGER.md b/docs/agentic/FAILURE_LEDGER.md index 75d82be6d..d13c6b590 100644 --- a/docs/agentic/FAILURE_LEDGER.md +++ b/docs/agentic/FAILURE_LEDGER.md @@ -26,7 +26,7 @@ Rows sharing a surface and first tracking issue in `future_fix` show only their | 2026-07-26 | non_blocking_risk | agent/tool-command-composition | Resolution record for the repeated #1490 PowerShell/native command-composition failures; the original recurrence remains in append-only history | No workaround remains; use the copyable safe forms and classify future failures through the ledger process | #1490 resolved by PR #1491 merge 61f718af941c56c31b8b7595333b9debd6e47da8: the MCP tooling guide now covers collected foreach output, optional rg exits, safe mu... | resolved | | 2026-07-26 | blocker | agent/python-launcher | Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the ... | Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows | #1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue clo... | open | | 2026-07-26 | blocker | agent/powershell-deny-floor | The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy | Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception | #1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception | open | -| 2026-07-27 | non_blocking_risk | ci/mutation-testing-backend | All 16 recorded scheduled Mutation Testing runs failed to produce backend Stryker evidence; the latest runs 30191449339 and 29676656172 both stopped because Str... | Treat the frontend lane as independent evidence, but do not claim backend mutation coverage until a backend-only exact-head run completes and uploads a non-empt... | #1500 pins Stryker.NET 4.16.0, uses ignore-methods and ignore-mutations, and adds a local preflight that rejects obsolete keys; resolve this row only after the ... | open | +| 2026-07-27 | non_blocking_risk | ci/mutation-testing-backend | Resolution record for the 16-run backend mutation evidence outage; the original failures remain in append-only history. | No workaround remains for producing bounded Domain mutation evidence; retain the non-blocking score posture while collecting the policy's 3-4-run calibration wi... | #1500 / PR #1503 repaired the lane at exact workflow head 307add004fbe142321a6ec11be21fab708824d5d. Backend-only run 30236307062 completed in 192 seconds with a... | resolved | ## Classification diff --git a/docs/agentic/failure_ledger.jsonl b/docs/agentic/failure_ledger.jsonl index d431db90c..a3de08491 100644 --- a/docs/agentic/failure_ledger.jsonl +++ b/docs/agentic/failure_ledger.jsonl @@ -26,3 +26,4 @@ {"ts":"2026-07-26T10:56:00Z","class":"blocker","surface":"agent/python-launcher","failure":"Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the active sys.executable -B","workaround":"Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows","future_fix":"#1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue closure","status":"open"} {"ts":"2026-07-26T11:36:55Z","class":"blocker","surface":"agent/powershell-deny-floor","failure":"The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy","workaround":"Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception","future_fix":"#1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception","status":"open"} {"ts":"2026-07-27T00:00:00Z","class":"non_blocking_risk","surface":"ci/mutation-testing-backend","failure":"All 16 recorded scheduled Mutation Testing runs failed to produce backend Stryker evidence; the latest runs 30191449339 and 29676656172 both stopped because Stryker.NET 4.16.0 rejects ignored-methods and excluded-mutations as invalid schema keys.","workaround":"Treat the frontend lane as independent evidence, but do not claim backend mutation coverage until a backend-only exact-head run completes and uploads a non-empty report artifact.","future_fix":"#1500 pins Stryker.NET 4.16.0, uses ignore-methods and ignore-mutations, and adds a local preflight that rejects obsolete keys; resolve this row only after the PR exact-head backend-only dispatch completes mutation execution with a non-empty Stryker artifact.","status":"open"} +{"ts":"2026-07-27T04:12:28Z","class":"non_blocking_risk","surface":"ci/mutation-testing-backend","failure":"Resolution record for the 16-run backend mutation evidence outage; the original failures remain in append-only history.","workaround":"No workaround remains for producing bounded Domain mutation evidence; retain the non-blocking score posture while collecting the policy's 3-4-run calibration window.","future_fix":"#1500 / PR #1503 repaired the lane at exact workflow head 307add004fbe142321a6ec11be21fab708824d5d. Backend-only run 30236307062 completed in 192 seconds with a 70.75% score over 3,682 mutants and uploaded a non-empty 874,386-byte two-file report artifact (SHA-256 0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8).","status":"resolved"} diff --git a/docs/testing/MUTATION_TESTING_POLICY.md b/docs/testing/MUTATION_TESTING_POLICY.md index b32952cf3..e15445398 100644 --- a/docs/testing/MUTATION_TESTING_POLICY.md +++ b/docs/testing/MUTATION_TESTING_POLICY.md @@ -87,6 +87,8 @@ The mutation testing workflow runs: Reports are uploaded as GitHub Actions artifacts with 30-day retention. The backend job has a finite 180-minute ceiling for the full Domain mutation set, and artifact upload fails when no report was produced. +The first repaired backend-only baseline, [run 30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d`, completed in 192 seconds. It created 3,682 mutants: 2,351 killed, 576 survived, 2 timed out, and 753 skipped, for a 70.75% score. The non-empty two-file report artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). Keep the 0% break threshold until the policy's 3-4-run calibration window exists. + ## Interpreting Reports ### Mutant statuses From b18b9a5c54d7337fe4ba6aaf1eaa134896e63439 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Mon, 27 Jul 2026 05:34:24 +0100 Subject: [PATCH 6/6] Validate mutation exclusion entries Signed-off-by: Chris0Jeky --- docs/MANUAL_TEST_CHECKLIST.md | 6 ++++- docs/testing/MUTATION_TESTING_POLICY.md | 4 +-- scripts/ci/Test-StrykerConfig.ps1 | 35 ++++++++++++++++++++----- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/MANUAL_TEST_CHECKLIST.md b/docs/MANUAL_TEST_CHECKLIST.md index d4f2a6afe..3a891f733 100644 --- a/docs/MANUAL_TEST_CHECKLIST.md +++ b/docs/MANUAL_TEST_CHECKLIST.md @@ -1054,7 +1054,11 @@ Status legend: `[ ]` = not yet performed, `[x]` = verified. 19. [ ] Verify mutation testing workflow is NOT listed in `ci-required.yml` (non-blocking, nightly only). 20. [ ] Verify `StrykerOutput` directories are in `.gitignore`. 21. [ ] Run `npm run mutation:test` locally to validate frontend Stryker setup (after `npm install`). -22. [ ] Run `dotnet stryker --config-file stryker-config.json` locally in `backend/` to validate backend setup. +22. [ ] Validate the backend Stryker setup from the repository root. + - Run: `powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest` + - Run: `cd backend/tests/Taskdeck.Domain.Tests` + - Run: `dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput` + - Expected: the preflight passes and Stryker uses only `Taskdeck.Domain.Tests` for the `Taskdeck.Domain` mutation run. ### Z8. Platform: PWA and Offline Readiness (PR #802 — merged) diff --git a/docs/testing/MUTATION_TESTING_POLICY.md b/docs/testing/MUTATION_TESTING_POLICY.md index e15445398..2fbe2db68 100644 --- a/docs/testing/MUTATION_TESTING_POLICY.md +++ b/docs/testing/MUTATION_TESTING_POLICY.md @@ -112,11 +112,11 @@ The first repaired backend-only baseline, [run 30236307062](https://github.com/C When mutation testing reveals surviving mutants: -1. **File an issue** with the label `test-hardening` and link to the mutation report artifact +1. **File an issue** with the existing `testing` and `hardening` labels and link to the mutation report artifact 2. **Categorize** surviving mutants by triage priority (see above) 3. **Bundle fixes**: Group related assertion improvements into a single PR per module rather than one PR per mutant 4. **Do not chase 100%**: Some surviving mutants are acceptable (e.g., log messages, cosmetic formatting). Document intentional exclusions: - - **Backend (Stryker.NET 4.16.0)**: Use `ignore-mutations` or `ignore-methods` in `backend/stryker-config.json`; the preflight deliberately rejects the obsolete `excluded-mutations` and `ignored-methods` spellings + - **Backend (Stryker.NET 4.16.0)**: Add Stryker patterns as non-empty string entries in the `ignore-mutations` or `ignore-methods` arrays in `backend/stryker-config.json`; the preflight accepts empty arrays and non-empty string entries while rejecting scalar/invalid entries and the obsolete `excluded-mutations` and `ignored-methods` spellings - **Frontend (Stryker JS)**: Adjust `mutate` glob patterns in `stryker.config.mjs` or use inline `// Stryker disable` comments in source files ## Scope Expansion Roadmap diff --git a/scripts/ci/Test-StrykerConfig.ps1 b/scripts/ci/Test-StrykerConfig.ps1 index 526168f94..f950fe015 100644 --- a/scripts/ci/Test-StrykerConfig.ps1 +++ b/scripts/ci/Test-StrykerConfig.ps1 @@ -80,14 +80,17 @@ function Test-StrykerConfig { throw "Stryker configuration '$resolvedPath' must preserve thresholds high=80, low=60, break=0." } - foreach ($requiredEmptyKey in @('ignore-methods', 'ignore-mutations')) { - $property = $strykerConfig.PSObject.Properties[$requiredEmptyKey] + foreach ($exclusionKey in @('ignore-methods', 'ignore-mutations')) { + $property = $strykerConfig.PSObject.Properties[$exclusionKey] if ($null -eq $property -or $property.Value -isnot [System.Array]) { - throw "Stryker configuration '$resolvedPath' must contain '$requiredEmptyKey' as an array." + throw "Stryker configuration '$resolvedPath' must contain '$exclusionKey' as an array." } - if ($property.Value.Count -ne 0) { - throw "Stryker configuration '$resolvedPath' must preserve the empty '$requiredEmptyKey' list." + for ($entryIndex = 0; $entryIndex -lt $property.Value.Count; $entryIndex++) { + $entry = $property.Value[$entryIndex] + if ($entry -isnot [string] -or [string]::IsNullOrWhiteSpace($entry)) { + throw "Stryker configuration '$resolvedPath' entry $entryIndex in '$exclusionKey' must be a non-empty string." + } } } } @@ -249,13 +252,30 @@ function Invoke-StrykerConfigSelfTest { Test-StrykerConfig -Path $validConfigPath Test-MutationWorkflowContract -Path $validWorkflowPath + $validExclusionVariants = @( + @('ignore-methods', ' "ignore-methods": [],', ' "ignore-methods": ["ToString", "Console.Write*"],'), + @('ignore-mutations', ' "ignore-mutations": []', ' "ignore-mutations": ["string", "logical"]') + ) + + for ($validIndex = 0; $validIndex -lt $validExclusionVariants.Count; $validIndex++) { + $validVariant = $validExclusionVariants[$validIndex] + $validVariantPath = Join-Path $temporaryDirectory "valid-$validIndex.json" + Write-TextVariant -Source $ConfigPath -Destination $validVariantPath -Expected $validVariant[1] -Replacement $validVariant[2] + Test-StrykerConfig -Path $validVariantPath + } + $configVariants = @( @('obsolete ignored-methods key', ' "ignore-methods": [],', ' "ignored-methods": [],', "uses obsolete 'ignored-methods'"), @('obsolete excluded-mutations key', ' "ignore-mutations": []', ' "excluded-mutations": []', "uses obsolete 'excluded-mutations'"), @('solution context', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "solution": "Taskdeck.sln",') -join "`n"), "must omit 'solution'"), @('test-project selector', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "test-projects": ["tests/Taskdeck.Domain.Tests/Taskdeck.Domain.Tests.csproj"],') -join "`n"), "must omit 'test-projects'"), @('wrong mutation target', ' "project": "Taskdeck.Domain.csproj",', ' "project": "Taskdeck.Application.csproj",', "must target project 'Taskdeck.Domain.csproj'"), - @('changed empty ignore semantics', ' "ignore-mutations": []', ' "ignore-mutations": ["string"]', "must preserve the empty 'ignore-mutations' list"), + @('ignore-methods scalar', ' "ignore-methods": [],', ' "ignore-methods": "ToString",', "must contain 'ignore-methods' as an array"), + @('ignore-mutations scalar', ' "ignore-mutations": []', ' "ignore-mutations": "string"', "must contain 'ignore-mutations' as an array"), + @('ignore-methods null entry', ' "ignore-methods": [],', ' "ignore-methods": [null],', "entry 0 in 'ignore-methods' must be a non-empty string"), + @('ignore-mutations non-string entry', ' "ignore-mutations": []', ' "ignore-mutations": [42]', "entry 0 in 'ignore-mutations' must be a non-empty string"), + @('ignore-methods empty entry', ' "ignore-methods": [],', ' "ignore-methods": [""],', "entry 0 in 'ignore-methods' must be a non-empty string"), + @('ignore-mutations whitespace entry', ' "ignore-mutations": []', ' "ignore-mutations": [" "]', "entry 0 in 'ignore-mutations' must be a non-empty string"), @('changed mutation level', ' "mutation-level": "Standard",', ' "mutation-level": "Advanced",', "must preserve mutation-level 'Standard'"), @('missing JSON reporter', ' "json",', ' "dashboard",', 'must preserve html, json, progress, and cleartext reporters'), @('changed score threshold', ' "break": 0', ' "break": 60', 'must preserve thresholds high=80, low=60, break=0') @@ -288,8 +308,9 @@ function Invoke-StrykerConfigSelfTest { } } + $validContractCount = 2 + $validExclusionVariants.Count $rejectedFixtureCount = $configVariants.Count + $workflowVariants.Count - Write-Host "Stryker preflight self-test passed: $($rejectedFixtureCount + 2) checks (2 valid contracts; $rejectedFixtureCount rejected drift fixtures)." + Write-Host "Stryker preflight self-test passed: $($validContractCount + $rejectedFixtureCount) checks ($validContractCount valid contracts; $rejectedFixtureCount rejected drift fixtures)." } finally { if (Test-Path -LiteralPath $temporaryDirectory) { Remove-Item -LiteralPath $temporaryDirectory -Recurse -Force