fix: require passing evidence for supported claims

.github/actionlint.yaml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+    - openadapt-parallels

.github/workflows/ci.yml

@@ -374,8 +374,17 @@ jobs:
           pytest -q --ignore=tests/e2e \
             --ignore=tests/test_install_playwright_browser.py \
             --basetemp=runs/ci \
+            --junitxml=runs/unit-claims-junit.xml \
             --cov=openadapt_flow --cov-report=
 
+      # A supported claim is not a function of file existence. Bind every
+      # cited unit-test file to this job's real Pytest result. Missing and
+      # all-skipped files fail closed even when another test passed.
+      - name: Validate passing unit claim evidence
+        run: |
+          python scripts/validate_claims.py --check \
+            --ci-job test --junit runs/unit-claims-junit.xml
+
       # Enforced RATCHET floor on the safety-critical path (branch-inclusive).
       # Scoped via --include to exactly the safety modules (compiler, identity,
       # effects, policy, replayer, resolver). 85% sits just under the fast
@@ -400,6 +409,7 @@ jobs:
             runs/**/BENCH.md
             runs/**/report.json
             runs/**/*.png
+            runs/unit-claims-junit.xml
           if-no-files-found: warn
 
   # --- Browser record -> compile -> replay E2E (REQUIRED on PRs) -----------
@@ -470,8 +480,16 @@ jobs:
           mkdir -p runs
           pytest -q tests/e2e \
             --ignore=tests/e2e/test_free_path_e2e.py \
+            --junitxml=runs/e2e-claims-junit.xml \
             --basetemp=runs/ci
 
+      # The browser maturity claims cite the record/compile/replay E2E file.
+      # Require that file to appear and pass in this required job's own JUnit.
+      - name: Validate passing browser claim evidence
+        run: |
+          python scripts/validate_claims.py --check \
+            --ci-job e2e-browser --junit runs/e2e-claims-junit.xml
+
       - name: Upload run artifacts
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -482,6 +500,7 @@ jobs:
             runs/**/BENCH.md
             runs/**/report.json
             runs/**/*.png
+            runs/e2e-claims-junit.xml
           if-no-files-found: warn
 
   # --- Native Linux: real GTK3 + AT-SPI under isolated X11 (REQUIRED) ------

.github/workflows/validate-claims.yml

@@ -2,18 +2,20 @@ name: Validate claims
 
 # Make every public maturity claim a FUNCTION of automated evidence.
 #
-#   - pull_request: run the fast claim->evidence GATE
-#     (scripts/validate_claims.py --check) on every PR. This is a
+#   - pull_request: run the fast claim->evidence STRUCTURE GATE
+#     (scripts/validate_claims.py --check --structure-only) on every PR. This is a
 #     required-check CANDIDATE: wire it into branch protection as the context
 #     "gate" (the actual CheckRun job name from the "Validate claims" workflow)
 #     to block a PR that overclaims (a `supported` claim whose only backing is
 #     an opt-in/infra-gated test, a `field` result labeled `supported`, or a
-#     missing evidence path). It NEVER touches infra.
-#   - schedule + workflow_dispatch: the infra-gated leg that provisions a real
-#     Windows VM (via oa-vm) and runs the OPT-IN desktop + Citrix e2e that back
-#     the `validating` tiers, then regenerates the evidence report. Where the
-#     infra/secrets are absent it SKIPS cleanly and says so -- it never fakes a
-#     pass (a fabricated green would defeat the whole harness).
+#     missing evidence path). The required CI `test` and `e2e-browser` jobs
+#     separately bind every supported test file to their real JUnit results.
+#     It NEVER touches infra.
+#   - schedule + workflow_dispatch: the infra-gated leg runs only on a labeled
+#     self-hosted macOS runner with the declared local Parallels substrate. It
+#     binds its two selected tests to their real JUnit results and emits an
+#     exactly scoped report. Without that runner configuration the job SKIPS;
+#     it never publishes a fabricated green.
 #
 # GitHub Actions are pinned to full commit SHAs (supply-chain control); the
 # trailing comment records the human-readable version, matching ci.yml.
@@ -25,13 +27,15 @@ on:
   workflow_dispatch:
     inputs:
       run_infra:
-        description: "Provision a Windows VM and run the opt-in desktop/Citrix e2e"
+        description: "Run the configured macOS + Parallels evidence refresh"
         type: boolean
         default: false
 
 concurrency:
   group: validate-claims-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  # A pull-request structure check is safe to supersede. A physical Parallels
+  # run is not: cancellation could interrupt its exact-snapshot cleanup.
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -55,8 +59,8 @@ jobs:
       - name: Install PyYAML
         run: pip install "pyyaml>=6"
 
-      - name: Claim -> evidence gate
-        run: python scripts/validate_claims.py --check
+      - name: Claim -> evidence structure gate
+        run: python scripts/validate_claims.py --check --structure-only
 
       # Prove the doc is regenerable and in sync with the registry: regenerate
       # docs/VERIFICATION.md + docs/verification.json into a temp dir would be
@@ -84,91 +88,115 @@ jobs:
           pip install "pytest>=8"
           pytest -q tests/test_validate_claims.py
 
-  # --- Infra-gated: refresh the VALIDATING-tier evidence on a real VM -------
-  # Provisions a Windows VM via oa-vm and runs the OPT-IN desktop + Citrix e2e
-  # that ground the `windows-desktop-validating` / `citrix-pixel-validating`
-  # claims (see tests/e2e/test_parallels_desktop_e2e.py and
-  # tests/e2e/test_citrix_pixel_e2e.py). NOT run on PRs and NOT a required
-  # check. It documents the human, infra half the PR gate cannot self-generate.
+  # --- Infra-gated: scoped validating evidence on its exact substrate -------
+  # These selected tests require macOS, local Parallels, a running configured
+  # Windows VM, and a known base snapshot. They cannot run on ubuntu-latest or
+  # against an unrelated hosted VM endpoint. A repository owner enables the
+  # job only after a self-hosted runner has all four labels below and the two
+  # required repository variables are configured.
   #
-  # IMPORTANT: when the VM credentials/runner are absent this job SKIPS every
-  # infra step (never fails, never fabricates a pass). A green here means the
-  # opt-in e2e actually ran on real infra; a skip means "not proven this run".
+  # A skipped job means "not proven this run" and produces no report. A green
+  # job means BOTH named files contained a real pass and no failure/error/skip.
+  # RDP, Citrix stand-in, and retained OpenEMR evidence remain separate scopes;
+  # this job never implies that those files ran.
   refresh-validating-evidence:
-    if: ${{ github.event_name != 'pull_request' }}
-    runs-on: ubuntu-latest
+    if: >-
+      ${{ vars.OPENADAPT_PARALLELS_VALIDATION_ENABLED == 'true' &&
+          (github.event_name == 'schedule' || inputs.run_infra == true) }}
+    runs-on: [self-hosted, macos, arm64, openadapt-parallels]
+    # The live-test step has a shorter timeout. This job reserve leaves time for
+    # its unconditional recovery step after a test timeout.
+    timeout-minutes: 110
+    concurrency:
+      group: openadapt-parallels-validation
+      cancel-in-progress: false
+    env:
+      OAFLOW_PARALLELS_VM_UUID: ${{ vars.OAFLOW_PARALLELS_VM_UUID }}
+      OAFLOW_PARALLELS_BASE_SNAPSHOT_ID: ${{ vars.OAFLOW_PARALLELS_BASE_SNAPSHOT_ID }}
+      OAFLOW_PARALLELS_STORAGE_PATH: ${{ github.workspace }}
+      # This path is outside the checkout, so checkout cleaning and a runner
+      # restart cannot erase an unresolved exact-base recovery record.
+      OAFLOW_PARALLELS_RECOVERY_JOURNAL: ${{ github.workspace }}/../.openadapt-flow/parallels-recovery.json
+      OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT: ${{ github.sha }}
+      OAFLOW_WINDOWS_UIA_MATRIX_ID: claims-${{ github.run_id }}-${{ github.run_attempt }}
+      OAFLOW_WINDOWS_UIA_EVIDENCE: ${{ github.workspace }}/runs/windows-uia-evidence.jsonl
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
-      # Gate the whole infra leg on a secret being present. Secrets are not
-      # available to forks/PRs, so this cannot be tricked into running from an
-      # untrusted PR, and a repo without infra configured simply skips.
-      - name: Detect infra availability
-        id: infra
-        env:
-          OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }}
+      - name: Verify exact macOS + Parallels substrate
         run: |
-          if [ -n "${OA_VM_TOKEN}" ] && \
-             { [ "${{ github.event_name }}" = "schedule" ] || \
-               [ "${{ github.event.inputs.run_infra }}" = "true" ]; }; then
-            echo "available=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "available=false" >> "$GITHUB_OUTPUT"
-            echo "::notice::Windows VM infra not available (no OA_VM_TOKEN or" \
-              "run_infra not requested) -- SKIPPING the opt-in desktop/Citrix" \
-              "e2e. This is a clean skip, NOT a pass. The validating-tier" \
-              "claims remain grounded by their committed opt-in tests."
-          fi
-
-      # The following steps are the DOCUMENTED infra path. They only run when a
-      # real VM is reachable; otherwise they are skipped by the `if:` guard.
-      - name: Provision Windows VM (oa-vm)
-        if: steps.infra.outputs.available == 'true'
-        env:
-          OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }}
+          [ "$(uname -s)" = "Darwin" ] || {
+            echo "::error::validating evidence requires macOS"; exit 1;
+          }
+          command -v prlctl >/dev/null || {
+            echo "::error::Parallels prlctl is unavailable"; exit 1;
+          }
+          [ -n "$OAFLOW_PARALLELS_VM_UUID" ] || {
+            echo "::error::OAFLOW_PARALLELS_VM_UUID is not configured"; exit 1;
+          }
+          [ -n "$OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" ] || {
+            echo "::error::OAFLOW_PARALLELS_BASE_SNAPSHOT_ID is not configured"; exit 1;
+          }
+
+      - name: Install validation dependencies
+        run: python -m pip install -e .[dev]
+
+      - name: Reconcile a prior interrupted run before new mutation
+        run: >-
+          python scripts/reconcile_parallels_recovery.py
+          --journal "$OAFLOW_PARALLELS_RECOVERY_JOURNAL"
+
+      - name: Prove configured VM and base are current before mutation
         run: |
-          pip install "pyyaml>=6"
-          pip install -e .[dev]
-          # oa-vm brings up a WAA pool VM; the win_agent + Parallels/RDP target
-          # is what the opt-in e2e drive. Terminate is in the always() step.
-          oa-vm pool-create --name claims-validation
-          oa-vm pool-wait --name claims-validation
-
-      - name: Run opt-in desktop + Citrix e2e (the validating-tier proofs)
-        if: steps.infra.outputs.available == 'true'
+          python -c '
+          import os
+          from openadapt_flow.backends.parallels_vm import ParallelsVM
+          vm = ParallelsVM(os.environ["OAFLOW_PARALLELS_VM_UUID"])
+          vm.require_host_free_space(
+              storage_path=os.environ["OAFLOW_PARALLELS_STORAGE_PATH"]
+          )
+          vm.require_current_snapshot(
+              os.environ["OAFLOW_PARALLELS_BASE_SNAPSHOT_ID"]
+          )
+          '
+
+      - name: Run scoped desktop and pixel evidence
+        timeout-minutes: 90
         env:
           OAFLOW_PARALLELS_E2E: "1"
           OAFLOW_CITRIX_PIXEL_E2E: "1"
         run: |
           mkdir -p runs
-          pytest -q \
-            tests/e2e/test_parallels_desktop_e2e.py \
+          python -m pytest -q \
             tests/e2e/test_citrix_pixel_e2e.py \
+            tests/e2e/test_parallels_desktop_e2e.py \
             --junitxml=runs/validating-junit.xml --basetemp=runs/ci
 
-      - name: Regenerate evidence report with the live green-check
-        if: steps.infra.outputs.available == 'true'
+      - name: Restore exact base after success, failure, or test timeout
+        if: ${{ always() }}
+        timeout-minutes: 15
+        run: >-
+          python scripts/reconcile_parallels_recovery.py
+          --journal "$OAFLOW_PARALLELS_RECOVERY_JOURNAL"
+
+      - name: Require every selected evidence file to pass
         run: |
           python scripts/validate_claims.py --report \
-            --junit runs/validating-junit.xml
+            --ci-job validating --junit runs/validating-junit.xml \
+            --evidence-path tests/e2e/test_parallels_desktop_e2e.py \
+            --evidence-path tests/e2e/test_citrix_pixel_e2e.py
 
-      - name: Upload refreshed evidence
-        if: steps.infra.outputs.available == 'true'
+      - name: Upload scoped passing evidence
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: verification-evidence
+          name: verification-evidence-macos-parallels
           path: |
             docs/VERIFICATION.md
             docs/verification.json
             runs/validating-junit.xml
+            runs/windows-uia-evidence.jsonl
           if-no-files-found: warn
-
-      - name: Terminate VM (always)
-        if: always() && steps.infra.outputs.available == 'true'
-        env:
-          OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }}
-        run: oa-vm pool-cleanup --name claims-validation || true

claims.yaml

@@ -13,7 +13,8 @@
 # -----------------------------------------------------
 # Evidence strength is derived from the repo, not asserted here:
 #   * a test file with NO module-level env skipif, that exists      -> "supported"
-#     (it actually runs, and can be green, in CI)
+#     candidate evidence; its required `test` or `e2e-browser` job must bind the
+#     file to at least one passing case in that job's real JUnit result
 #   * a test file gated by a module-level `pytestmark` env skipif    -> "validating"
 #     (opt-in / infra-gated: grounded, but never runs on default CI)
 #   * a doc / benchmark artifact (.md or a benchmark dir)            -> "roadmap"
@@ -24,7 +25,7 @@
 # result that is not CI-reproducible is never presented as "supported").
 #
 # TIERS
-#   supported  — CI-proven today on the default (non-opt-in) suite.
+#   supported  — bound to a real passing result in required default CI.
 #   validating — grounded in a REAL opt-in/infra-gated proof or a field test;
 #                being validated (incl. with design partners). NOT "supported".
 #   roadmap    — designed/specified, not yet proven by a running test.