From 872fe84984dec4053182ee5d64a9e62944363a40 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 12:40:49 -0400 Subject: [PATCH 1/5] fix: require passing claim evidence --- .github/workflows/ci.yml | 19 ++ .github/workflows/validate-claims.yml | 14 +- claims.yaml | 5 +- docs/VERIFICATION.md | 34 ++-- docs/verification.json | 75 ++++++++ public-artifacts.json | 6 +- scripts/validate_claims.py | 261 ++++++++++++++++++++------ tests/test_ci_workflow_contract.py | 20 ++ tests/test_validate_claims.py | 76 +++++++- 9 files changed, 418 insertions(+), 92 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 432cefab..54161205 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -374,8 +374,17 @@ jobs: pytest -q --ignore=tests/e2e \ --ignore=tests/test_install_playwright_browser.py \ --basetemp=runs/ci \ + --junitxml=runs/unit-claims-junit.xml \ --cov=openadapt_flow --cov-report= + # A supported claim is not a function of file existence. Bind every + # cited unit-test file to this job's real Pytest result. Missing and + # all-skipped files fail closed even when another test passed. + - name: Validate passing unit claim evidence + run: | + python scripts/validate_claims.py --check \ + --ci-job test --junit runs/unit-claims-junit.xml + # Enforced RATCHET floor on the safety-critical path (branch-inclusive). # Scoped via --include to exactly the safety modules (compiler, identity, # effects, policy, replayer, resolver). 85% sits just under the fast @@ -400,6 +409,7 @@ jobs: runs/**/BENCH.md runs/**/report.json runs/**/*.png + runs/unit-claims-junit.xml if-no-files-found: warn # --- Browser record -> compile -> replay E2E (REQUIRED on PRs) ----------- @@ -470,8 +480,16 @@ jobs: mkdir -p runs pytest -q tests/e2e \ --ignore=tests/e2e/test_free_path_e2e.py \ + --junitxml=runs/e2e-claims-junit.xml \ --basetemp=runs/ci + # The browser maturity claims cite the record/compile/replay E2E file. + # Require that file to appear and pass in this required job's own JUnit. + - name: Validate passing browser claim evidence + run: | + python scripts/validate_claims.py --check \ + --ci-job e2e-browser --junit runs/e2e-claims-junit.xml + - name: Upload run artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -482,6 +500,7 @@ jobs: runs/**/BENCH.md runs/**/report.json runs/**/*.png + runs/e2e-claims-junit.xml if-no-files-found: warn # --- Native Linux: real GTK3 + AT-SPI under isolated X11 (REQUIRED) ------ diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index aaf3d480..6130f043 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -2,13 +2,15 @@ name: Validate claims # Make every public maturity claim a FUNCTION of automated evidence. # -# - pull_request: run the fast claim->evidence GATE -# (scripts/validate_claims.py --check) on every PR. This is a +# - pull_request: run the fast claim->evidence STRUCTURE GATE +# (scripts/validate_claims.py --check --structure-only) on every PR. This is a # required-check CANDIDATE: wire it into branch protection as the context # "gate" (the actual CheckRun job name from the "Validate claims" workflow) # to block a PR that overclaims (a `supported` claim whose only backing is # an opt-in/infra-gated test, a `field` result labeled `supported`, or a -# missing evidence path). It NEVER touches infra. +# missing evidence path). The required CI `test` and `e2e-browser` jobs +# separately bind every supported test file to their real JUnit results. +# It NEVER touches infra. # - schedule + workflow_dispatch: the infra-gated leg that provisions a real # Windows VM (via oa-vm) and runs the OPT-IN desktop + Citrix e2e that back # the `validating` tiers, then regenerates the evidence report. Where the @@ -55,8 +57,8 @@ jobs: - name: Install PyYAML run: pip install "pyyaml>=6" - - name: Claim -> evidence gate - run: python scripts/validate_claims.py --check + - name: Claim -> evidence structure gate + run: python scripts/validate_claims.py --check --structure-only # Prove the doc is regenerable and in sync with the registry: regenerate # docs/VERIFICATION.md + docs/verification.json into a temp dir would be @@ -154,7 +156,7 @@ jobs: if: steps.infra.outputs.available == 'true' run: | python scripts/validate_claims.py --report \ - --junit runs/validating-junit.xml + --ci-job validating --junit runs/validating-junit.xml - name: Upload refreshed evidence if: steps.infra.outputs.available == 'true' diff --git a/claims.yaml b/claims.yaml index 653992d5..349b8f58 100644 --- a/claims.yaml +++ b/claims.yaml @@ -13,7 +13,8 @@ # ----------------------------------------------------- # Evidence strength is derived from the repo, not asserted here: # * a test file with NO module-level env skipif, that exists -> "supported" -# (it actually runs, and can be green, in CI) +# candidate evidence; its required `test` or `e2e-browser` job must bind the +# file to at least one passing case in that job's real JUnit result # * a test file gated by a module-level `pytestmark` env skipif -> "validating" # (opt-in / infra-gated: grounded, but never runs on default CI) # * a doc / benchmark artifact (.md or a benchmark dir) -> "roadmap" @@ -24,7 +25,7 @@ # result that is not CI-reproducible is never presented as "supported"). # # TIERS -# supported — CI-proven today on the default (non-opt-in) suite. +# supported — bound to a real passing result in required default CI. # validating — grounded in a REAL opt-in/infra-gated proof or a field test; # being validated (incl. with design partners). NOT "supported". # roadmap — designed/specified, not yet proven by a running test. diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index a34f294e..28aaee27 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -3,20 +3,21 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. - Generated at: **committed registry state (regenerate: scripts/validate_claims.py --report)** -- Green-check against a junit artifact: **not run** (no `--junit` artifact supplied) -- Gate: `python scripts/validate_claims.py --check` (a claim whose tier outranks its strongest backing evidence fails CI). +- Green-check against a junit artifact: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) +- Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). +- Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. -**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test that actually runs on the default (non-opt-in) CI suite; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. +**What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test file that has a real passing case in its required default CI job; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. -## What is CI-proven today vs. being validated +## What is bound to required CI vs. being validated -- **CI-proven today (11):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported` +- **Bound to required CI pass evidence (11):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported` - **Being validated — opt-in / infra-gated or field (8):** `effect-verifier-kit`, `windows-desktop-validating`, `macos-native-validating`, `linux-native-validating`, `rdp-validating`, `desktop-recording-validating`, `citrix-pixel-validating`, `openemr-field-benchmark` - **Roadmap / research (1):** `win32-window-replay-roadmap` ## Claims -### `web-supported` — supported — CI-proven today +### `web-supported` — supported — bound to required CI pass evidence > Web (browser) workflows are supported today: record a GUI workflow once, then replay it deterministically and locally. @@ -34,7 +35,7 @@ - "Supported" is scoped to the reference headless-browser backend in this registry. Desktop and remote-display workflows use the separately scoped acceptance and code-qualified claims below. - The full record->compile->replay browser suite runs in the required e2e-browser PR gate and repeats in the weekly compatibility matrix. -### `deterministic-zero-model-replay` — supported — CI-proven today +### `deterministic-zero-model-replay` — supported — bound to required CI pass evidence > Replay is deterministic, local, and $0 — zero model calls on either the record or the replay side. @@ -51,7 +52,7 @@ - A VLM rung exists as the lowest fallback of the ladder; it is not invoked on a clean deterministic replay, and never on the record side. -### `effect-verification-silent-writes` — supported — CI-proven today +### `effect-verification-silent-writes` — supported — bound to required CI pass evidence > Effect verification against the system of record catches silent wrong writes the screen-only oracle misses (5 of 7 transactional fault classes). @@ -90,7 +91,7 @@ - CONTRACT-PROVEN, not live-proven: the SQL verifier is exercised against sqlite fixtures (no production MariaDB/Postgres), the SFTP arrival path against a fake transport (no real SFTP server), and the worked-example configs are templates — only the FHIR substrate has an additional opt-in live-OpenEMR test. Per-verifier claims are exactly as strong as these tests. -### `identity-gate-halt-armed` — supported — CI-proven today +### `identity-gate-halt-armed` — supported — bound to required CI pass evidence > On ambiguity the runtime halts instead of guessing, and an identity gate refuses a wrong-entity (wrong-patient) click — on ARMED steps only. @@ -108,7 +109,7 @@ - Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. -### `identifier-crop-compile-emission` — supported — CI-proven today +### `identifier-crop-compile-emission` — supported — bound to required CI pass evidence > The compiler emits a pixel identifier crop (anchor.identifier_crop) for identity-armed steps without structured identity — and for any step whose identifying region is marked at record time (--identifier) — so the pixel-compare identity tier arms on remote-display/pixel replays; every crop-less identity-applicable step records an explicit degrade reason, and lint surfaces per-bundle pixel-identity coverage. @@ -125,7 +126,7 @@ - Mechanism proven on synthetic fixtures and the bundled demo app only — this does NOT re-qualify any desktop/remote-display substrate, and the pixel tier remains MISMATCH-or-ABSTAIN (it can add a safe halt on a wrong identifier, never authorize a match; PIXEL_VERIFY_ENABLED stays off pending a jitter-robust distance). - Automatic emission requires an OCR-readable identity band at compile time; icon-only or unreadable rows still compile crop-less (with the recorded reason) and fall back to the OCR band tier at replay. -### `halt-teach-promote` — supported — CI-proven today +### `halt-teach-promote` — supported — bound to required CI pass evidence > A halt is learnable: halt -> teach a correction -> promote a guarded branch, refusing underdetermined fixes. @@ -141,7 +142,7 @@ - Promotion is governed by a regression gate that blocks identity- weakening corrections; the loop refuses to guess when the fix is underdetermined. -### `typed-business-decisions-supported` — supported — CI-proven today +### `typed-business-decisions-supported` — supported — bound to required CI pass evidence > A qualification client can add or update a typed finite business-policy decision without editing an internal manifest. The compiled workflow can then pause at that decision, accept one authorized and attributed choice, retain a signed durable receipt, revalidate the live application, and continue only through the certified successor branch. @@ -158,7 +159,7 @@ - The engine validates a principal and its roles; it does not authenticate a self-asserted user. Desktop, Cloud, or a customer-local identity route must supply the authenticated principal. - A human choice is control authority only. It cannot satisfy entity identity, a postcondition, or effect verification, and it cannot turn a screen statement or a human assertion into VERIFIED. -### `reviewed-judgment-cases-supported` — supported — CI-proven today +### `reviewed-judgment-cases-supported` — supported — bound to required CI pass evidence > A qualification project can bind typed local facts, evidence hashes, reviewer provenance, and the exact decision contract to reviewed examples and counterfactuals. It can retain permanent human authority, record a reviewed automatic-rule candidate, or refuse certification until more evidence exists, without converting one historical choice into policy. @@ -176,7 +177,7 @@ - An automatic-rule case names only a reviewed rule identifier and finite option. Flow does not infer or generate executable policy from the case; the rule must be authored and qualified through the normal program path. - A retained human-node case preserves runtime human authority. A more-evidence-required case refuses certification instead of guessing. -### `qualified-remote-decision-v2-supported` — supported — CI-proven today +### `qualified-remote-decision-v2-supported` — supported — bound to required CI pass evidence > A negotiated V2 attended task binds optional reviewed entity wording to the exact qualification, bundle, step, policy, and pause authority. An unqualified or unrecognized entity class stays local and the remote task uses the signed neutral record or item fallback. @@ -194,7 +195,7 @@ - V2 requires openadapt-types 0.10.x and explicit peer negotiation. The dependency does not upgrade a V1 consumer, and the byte-compatible V1 path remains available. - The entity class is presentation metadata only. Actual record identity remains inside the customer-controlled runner and is revalidated before any resumed action. -### `mockmed-benchmark-ci-reproducible` — supported — CI-proven today +### `mockmed-benchmark-ci-reproducible` — supported — bound to required CI pass evidence > The MockMed benchmark is CI-reproducible: compiled replay finishes ~4.9s p50 with zero model calls versus ~37.5s p50 (~$0.27/run) for the agent. Measured 2026-07-08 on Flow 0.1.0, a pre-v0.2.0 source build; not re-measured on a later release. @@ -289,7 +290,7 @@ - Acceptance covers the named 1280x800 Windows Run/file task and exact VM environment, not arbitrary applications, identity policies, or Citrix. - The multi-window campaign uses deterministic synthetic applications. It exercises the real FreeRDP pixel and input path but does not qualify a named customer application or environment. -### `substrate-runtime-validation-supported` — supported — CI-proven today +### `substrate-runtime-validation-supported` — supported — bound to required CI pass evidence > Runtime-validation v3 binds Standard and Regulated hosted activation to the exact governed authorization template reproduced from the sealed qualification. It also binds the successful run's resolved target kind (`web|windows|macos|linux|rdp|citrix`), artifact, compiler, parameter schema, strict lint, policy, risk, report, and opaque environment. Web retains its exact HTTPS/host boundary. Native and remote execution emits no app, window, host, readiness, or backend-hint values to Cloud. Local verification retains signed v1 and v2 compatibility. @@ -385,4 +386,3 @@ - The historical result uses OCR evidence from a visible saved message row. It does not use an out-of-band OpenEMR system-of-record read. - FIELD TEST, NOT CI-reproducible: the head-to-head ran against a SHARED public demo that other users mutate and that resets daily; the sample is small (10 agent runs). Only the verifier + task-prompt units run in CI. `reproducibility: field` forbids ever labeling this `supported`. - diff --git a/docs/verification.json b/docs/verification.json index f47e56f5..f01c9349 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -1,6 +1,7 @@ { "generated_at": "committed registry state (regenerate: scripts/validate_claims.py --report)", "green_check_run": false, + "green_check_job": null, "ok": true, "claims": [ { @@ -27,6 +28,7 @@ "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, + "ci_job": "e2e-browser", "junit_status": null, "proves": "Records the MockMed browser demo once, compiles it, and replays it under baseline + theme/move/rename drift and parameter substitution through the headless-browser Backend." }, @@ -38,6 +40,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The reference browser demo app and its drift screens render deterministically (no CSS transitions), so replay is repeatable." }, @@ -49,6 +52,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The deterministic replayer resolves steps, substitutes parameters, enforces postconditions and the risk gate \u2014 no model in the loop." } @@ -78,6 +82,7 @@ "gating": "ci (required PR gate (e2e-browser))", "node": null, "node_found": null, + "ci_job": "e2e-browser", "junit_status": null, "proves": "Baseline replay x3 resolves every step on the `template` rung with 0 heals and 0 model calls; healed bundles replay clean afterward." }, @@ -89,6 +94,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The replayer is a deterministic resolution ladder; the VLM rung is a fallback, not on the default path \u2014 the core runtime is model-free." }, @@ -100,6 +106,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "CI-reproducible MockMed run artifact: 100/100 compiled replays at 4.9s p50 with zero model calls, measured 2026-07-08 on Flow 0.1.0 (pre-v0.2.0 source build); field/artifact reference." } @@ -128,6 +135,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The REST/FHIR/document-hash verifiers confirm exactly-one write and refute duplicate / missing / partial / collateral-loss writes." }, @@ -139,6 +147,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Each silent-wrong-action class is silent under the screen-only oracle and is CAUGHT once effects are declared and verified." }, @@ -150,6 +159,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The FHIR effect verifier's read/confirm/refute/indeterminate contract (the live-OpenEMR smoke is a separate opt-in function in this file)." }, @@ -161,6 +171,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The 7-class transactional fault-model study: 5/7 classes silently mishandled by screen-only, all 5 halt through the real replayer." } @@ -188,6 +199,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The read-only SQL verifier's whitelist (mutation/stacking/comment smuggling refused at construction) and full verdict contract, plus the exact table-delta audit promoted from the Frappe Lending reference matrix \u2014 against LOCAL sqlite fixtures only." }, @@ -199,6 +211,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "File-arrival verdicts (pattern / size>0 / mtime freshness / content probe / duplicate export) against real temp directories, and the SFTP path against an in-memory fake paramiko-shaped transport." }, @@ -210,6 +223,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The declarative construction path: env-var auth references fail loud when unset, run-parameter references resolve at build (and refuse to construct unresolved), and pre-kit configs build byte-identically." }, @@ -221,6 +235,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI constructs and preflights all reviewed candidates before input, pins the strongest verifier and its pre-state, preserves the binding through durable resume, and refuses an unavailable selected verifier rather than falling back after an action." }, @@ -232,6 +247,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "lint warns per consequential step lacking an effect contract and reports coverage %; certify fails the same gap only when the policy sets require_effects_for_irreversible (warn-vs-fail is policy-configurable)." }, @@ -243,6 +259,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Every escalated verification failure emits a typed ReconciliationTask carrying the one-way contract hash and verdict evidence \u2014 never the resolved selector values." }, @@ -254,6 +271,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The operator-facing kit contract: config reference plus Frappe (REST/SQL) and OpenEMR (FHIR/SQL) worked examples targeting the reference fixtures." } @@ -282,6 +300,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Wrong-entity / near-name sibling / generic-band / param-in-band probes: the identity check refuses to verify a mismatched target and never arms on non-discriminative bands." }, @@ -293,6 +312,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Underdetermined targets surface as explicit disambiguation rather than a guessed click." }, @@ -304,6 +324,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Public property-based checks preserve the never-false-accept invariant for collapsible identifiers while requiring clean identifiers to remain verifiable." }, @@ -315,6 +336,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The identity resolution ladder (structural/template/OCR) escalation." } @@ -343,6 +365,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Synthetic-fixture proof of the emission mechanism: a pixel-only recording compiles WITH a crop under templates/identifiers/ (sealed + manifest-hashed like every image crop); a structured recording writes NO identity pixels and records why; explicit --identifier markings (event rect / meta region) win; encrypted save seals the crop and the sealed crop still reaches the tier; the compiled crop drives a wrong-MRN MISMATCH through the real replayer ladder while pixel VERIFY stays hard-gated off (zero-false-accept preserved)." }, @@ -354,6 +377,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Lint coverage surfacing: identity-armed steps with vs without crops (missing_identifier_crop findings carry the compiler's degrade reason; warn on band-only identity, info under structured identity)." } @@ -382,6 +406,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "A surprise modal halts and emits a learnable trace; teaching promotes a guarded dismiss branch that replays without regression; an underdetermined correction is refused." }, @@ -393,6 +418,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The `teach` CLI resolves a halt via a correction spec or a recording, refuses an underdetermined fix, and errors on a non-halted run." } @@ -421,6 +447,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers typed qualification authoring and replacement, qualification-revision and certification invalidation, ambiguous insertion refusal, the closed runtime decision contract, role and evidence checks, expiry, renewal, idempotency, crash recovery, signed branch authority, scoped outputs, fresh live-state revalidation, normal downstream action gates, and repair-bypass refusal." }, @@ -432,6 +459,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public engine contract separates a declared business choice from an operational halt and defines the Desktop, Cloud, and customer-local integration boundary." } @@ -460,6 +488,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers closed fact schemas, exact workflow and decision bindings, local-only evidence references, conflict refusal, reciprocal contrast coverage for automatic-rule candidates, retained human authority, unresolved-evidence refusal, and the invariant that case evaluation never edits or synthesizes the executable rule." }, @@ -471,6 +500,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI binds the reviewed local evidence bytes and review note into certification, reproduces that exact contract when a qualified workflow is saved, and refuses changed case or evidence digests." }, @@ -482,6 +512,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public contract separates qualification-time judgment capture from one live runtime answer and describes the scriptable authoring and judgment-case interfaces." } @@ -510,6 +541,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the V1-compatible and V2 signed envelopes, provenance binding, remote-safe entity vocabulary, idempotent decisions, and fresh revalidation before resumed actuation." }, @@ -521,6 +553,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI requires the current exact qualification contract before the V2 producer can emit its reviewed presentation metadata." }, @@ -532,6 +565,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The delivery contract defines explicit V2 negotiation, neutral V1 fallback, and the local boundary for actual entity identifiers." } @@ -560,6 +594,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The MockMed app + drift screens are deterministic and reachable, the substrate the benchmark replays against." }, @@ -571,6 +606,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The benchmark harness / report accounting that produces the numbers." }, @@ -582,6 +618,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Full numbers, methodology, and caveats for the CI-reproducible MockMed comparison (dated run artifact)." } @@ -612,6 +649,7 @@ "gating": "opt-in (OAFLOW_PARALLELS_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "OPT-IN live proof on a real Win11-ARM VM: record->compile->replay through WindowsBackend with the structural rung resolving every click by AutomationId (armed_coverage == 1.0)." }, @@ -623,6 +661,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The 4-method Backend protocol on Windows against a MOCKED WAA server (runs in CI post-merge; proves the wire, not a live desktop)." }, @@ -634,6 +673,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Desktop drift benchmark: the structural rung resolved 21/21 targets where visual replay alone managed 6/21." }, @@ -645,6 +685,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Accepted candidate 20260717-candidate-56759c8-v2: 3/3 exact task and SQLite effects, 12 UIA receipts, and 3/3 stale plus 3/3 ambiguity refusals with zero silent incorrect success, over-halt, or model calls." } @@ -675,6 +716,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI covers unique-window binding, foreground/focus proof, exact-element text delivery, and refusal before physical input when proof fails." }, @@ -686,6 +728,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI validates the qualification oracle, failure taxonomy, evidence hashes, cleanup adjudication, and ambiguity-refusal accounting." }, @@ -697,6 +740,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Candidate b1b61a5 completed 3/3 exact-byte TextEdit effects and refused two ambiguous windows without modifying either file." } @@ -726,6 +770,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Default CI covers exact app/window binding, bounded candidate enumeration, ambiguity and stale-target refusal, native invoke/focus and editable-text delivery, window-scoped capture, physical-input opt-in, and Wayland portal refusal through an injected Linux client." }, @@ -737,6 +782,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Default CI enforces the live qualification's fixed three-trial matrix, independent exact-file and absence oracles, and separate silent-incorrect-success, over-halt, and refusal-failure accounting." }, @@ -748,6 +794,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The required linux-atspi-x11 job drives the real GTK3 fixture through AT-SPI on an isolated Xvfb display and session D-Bus, emitting a per-trial JSON evidence artifact." }, @@ -759,6 +806,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The operator contract, X11 and Wayland boundaries, configuration, delivery-only receipt semantics, and required real GTK/AT-SPI CI qualification contract." } @@ -789,6 +837,7 @@ "gating": "opt-in (OAFLOW_PARALLELS_RDP_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "Opt-in snapshot-safe real RDP qualification harness with exactly three trials, independent guest-tools oracle, failure taxonomy, and cleanup." }, @@ -800,6 +849,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "CI covers the backend/transport contract, framebuffer conversion, pointer and keyboard delivery, and record-compile-replay conformance." }, @@ -811,6 +861,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the bounded 27-trial FreeRDP campaign contract, independent SQLite, CSV, and Maildir oracles, fail-closed visual action preflight, uncertain-delivery handling, and result accounting." }, @@ -822,6 +873,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public synthetic multi-window campaign defines the workflow, faults, independent persisted surfaces, and acceptance denominator." }, @@ -833,6 +885,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "Candidate 82a658a passed 3/3 at 51.845, 10.467, and 7.477 seconds, with zero failures, silent incorrect successes, over-halts, or model calls." } @@ -862,6 +915,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI exercises all six target kinds, validates v3 against its closed JSON Schema, refuses a Standard or Regulated activation without an exact template, binds the template SHA to the HMAC, preserves web URL/host checks, keeps native/remote execution empty, verifies the canonical HMAC vector, and retains signed v1/v2 compatibility." }, @@ -873,6 +927,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI carries the resolved web, Windows, and Citrix backend token through durable resume into the new run report rather than trusting an upload-time relabel." }, @@ -884,6 +939,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The public closed envelope requires target_kind and the governed authorization template SHA, and conditionally permits browser execution fields only for web." } @@ -914,6 +970,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The live-capture orchestration + CLI wiring (record --backend windows|macos|linux|rdp|citrix -> openadapt-capture -> convert_capture), and that a desktop-shaped recording COMPILES into a bundle and REPLAYS to completion through the desktop backend path, resolving each click to its recorded target (runs on default CI)." }, @@ -925,6 +982,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The capture->recording bridge over a REAL openadapt-capture session (its own event-processing pipeline + frame extraction) feeding the UNMODIFIED compiler (runs on default CI: the `test` job installs the `capture` extra; openadapt-capture >=0.5.4 imports clean headless)." }, @@ -936,6 +994,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The capture-assisted and live-observer paths, fail-closed secret handling, RDP coordinate binding, and reuse of openadapt-capture + the capture adapter." } @@ -967,6 +1026,7 @@ "gating": "opt-in (OAFLOW_CITRIX_PIXEL_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "OPT-IN pixel-only proof: structural_armed_coverage == 0, replay resolves on template/ocr/geometry only, on-screen OCR read-back verifies the write, the identity gate HALTs on a look-alike patient, and render drift triggers halt-on-ambiguity." }, @@ -978,6 +1038,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The pixel-substrate identity probe behind the on-pixels identity gate." }, @@ -989,6 +1050,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI covers the dedicated backend preset, exact owner/title overrides, readiness probe construction, pixel-only capability boundary, backend factory, and product CLI selectors." }, @@ -1000,6 +1062,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI refuses governed Citrix execution before action when its readiness binding is absent or blank, admits a sealed bundle carrying a recorded readiness binding, and reports only the resolved backend token rather than sensitive target strings." }, @@ -1011,6 +1074,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI restores the recorded Citrix owner, exact title, and readiness binding through durable approve/resume, and refuses before backend construction when resumed configuration is incomplete." }, @@ -1022,6 +1086,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI binds a completed local Citrix report to the closed `citrix` execution token without copying target owner, title, or readiness values into the hosted summary." }, @@ -1033,6 +1098,7 @@ "gating": "opt-in (OAFLOW_CITRIX_STANDIN_E2E)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "The dedicated Citrix backend passes three healthy effect-confirmed record->compile->replay trials and three severe-drift safe-halts over a no-DOM surface, with zero model calls, false completion, or silent incorrect success." }, @@ -1044,6 +1110,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "Required CI validates the public real-ICA campaign preflight, distinct authority keys, executable and oracle attestations, one-use nonce journal, crash recovery, uncertain dispatch, and fail-closed report contract without provisioning infrastructure." }, @@ -1055,6 +1122,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The retained six-trial record reports code_readiness_accepted=true and ica_hdx_accepted=false, keeping driver readiness distinct from a counted live ICA/HDX qualification." }, @@ -1066,6 +1134,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The Citrix driver model, reusable evidence, exact-deployment acceptance contract, independent-effect boundary, and customer-controlled posture." } @@ -1093,6 +1162,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "MOCK-ONLY conformance: the win32 WindowClient implements the remote-display WindowClient seam (exact process/title/class selection with ambiguity halt, client-area coordinate mapping under per-monitor DPI awareness, PrintWindow-then-BitBlt capture fallback, focus-verification refusal, and the UIPI elevation guard) against a scripted Win32 API \u2014 never against a real Windows host." }, @@ -1104,6 +1174,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The remote-display contract the client implements and the evidence ladder for qualifying it (Parallels/RDP client-window analog first, then a counted live Citrix host run)." } @@ -1134,6 +1205,7 @@ "gating": "ci (required PR gate (test))", "node": null, "node_found": null, + "ci_job": "test", "junit_status": null, "proves": "The CI-reproducible half: the saved-message-row verifier, its unsaved-entry-form rejection, and the intent-not-coordinates task prompt that both benchmark arms use." }, @@ -1145,6 +1217,7 @@ "gating": "opt-in (OPENADAPT_OPENEMR_FINALS_DIR)", "node": null, "node_found": null, + "ci_job": "validating", "junit_status": null, "proves": "When the local-only retained final frames are mounted, the guard replays all 30 with the current verifier and requires 19/20 compiled, 10/10 agent, and exactly one corrected legacy false success." }, @@ -1156,6 +1229,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The corrected field-test numbers, saved-row screen-oracle contract, methodology, and cost caps for the 20-vs-10 head-to-head run." }, @@ -1167,6 +1241,7 @@ "gating": "artifact (doc/benchmark)", "node": null, "node_found": null, + "ci_job": null, "junit_status": null, "proves": "The correctness-only field findings (fresh browsers, zero model calls, closed-loop scrolling)." } diff --git a/public-artifacts.json b/public-artifacts.json index 77606d83..f3ca931c 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -121,7 +121,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "085faf897b2c9f31afe4b1297454606bec8af3ac0165616a6351f06e9ce1fe6a" + "sha256": "7030f250fe66f36e71f3b44d52713e7f550a10e04c10f37ebdc745fa9a97c69a" }, { "path": ".pre-commit-config.yaml", @@ -601,7 +601,7 @@ }, { "path": "claims.yaml", - "sha256": "bf33b2ef84654c8563937828e38e389e065efcea5ec2654d4888423c1ede0142" + "sha256": "5423f4f10ef289dc3e232630c8558f12a76b3f11ebbb497feb923a928dc230b7" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1809,7 +1809,7 @@ }, { "path": "docs/verification.json", - "sha256": "b27d6f6a848d1dd75149155bc5f33dc4b797634def65da5ffe4f62fe0053f44e" + "sha256": "c8275336c9581532a3841a57913228b7584ab1487cdcc9c7b7ab884326769842" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 97a69f4b..224946f8 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -4,30 +4,34 @@ `scripts/check_consistency.py` stops the README from carrying stale *strings*. This script stops it from carrying stale *maturity claims*. It reads the machine-readable registry `claims.yaml` (each claim -> a `tier` -> the backing -test(s)/benchmark(s)) and enforces a tier<->evidence contract, so a "supported" -claim whose proof is only an opt-in/infra-gated test — or is missing entirely — -is a hard CI failure instead of a thing a design partner discovers. +test(s)/benchmark(s)) and enforces a tier<->evidence contract. A "supported" +claim whose proof is only an opt-in/infra-gated test, is missing, is absent from +the required job's JUnit result, is skipped, or fails is a hard CI failure. The evidence STRENGTH of each artifact is derived from the repo, never asserted by the registry (which therefore cannot lie about it): * a test file with NO module-level env skipif, that exists -> ``supported`` - (it actually runs, and can be green, on the default CI suite) + candidate evidence (the required ``test`` or ``e2e-browser`` job must also + supply a JUnit result that proves the cited file actually passed) * a test file gated by a module-level ``pytestmark`` env skipif -> ``validating`` (opt-in / infra-gated: grounded in a real proof, but never on default CI) * a doc / benchmark artifact (``.md`` or a benchmark dir) -> ``roadmap`` (design / field evidence; cannot by itself prove a running capability) -A claim FAILS when its ``tier`` OUTRANKS its strongest evidence, or when a -claim marked ``reproducibility: field`` is labeled ``supported`` (a result that -is not CI-reproducible is never presented as "supported"), or when any evidence -path is missing (registry rot). +A claim FAILS when its ``tier`` OUTRANKS its strongest evidence, when a claim +marked ``reproducibility: field`` is labeled ``supported`` (a result that is not +CI-reproducible is never presented as "supported"), or when any evidence path +is missing (registry rot). In a required test job, it also fails unless every +supported test assigned to that job has at least one passing case, no failing +case, and a real entry in that job's JUnit result. Usage:: - python scripts/validate_claims.py --check # gate (exit 1 on violation) + python scripts/validate_claims.py --check --structure-only + python scripts/validate_claims.py --check --ci-job test \ + --junit runs/unit-claims-junit.xml python scripts/validate_claims.py --report # (re)write docs/VERIFICATION.md + .json - python scripts/validate_claims.py --check --junit runs/ci/junit.xml # + green-check The public functions are importable so ``tests/test_validate_claims.py`` can drive them with controlled registries (catching registry rot before CI does). @@ -58,10 +62,11 @@ VALID_TIERS = set(TIER_RANK) # Evidence strength labels reuse the tier vocabulary (same rank scale). -STRENGTH_CI = "supported" # non-opt-in test that exists -> runs on default CI +STRENGTH_CI = "supported" # eligible test; required-job JUnit must prove its pass STRENGTH_OPTIN = "validating" # opt-in / infra-gated test -> grounded, not on CI STRENGTH_DOC = "roadmap" # doc/benchmark artifact -> design/field evidence only STRENGTH_MISSING = "research" # nothing backing it +CI_JOBS = {"test", "e2e-browser", "validating"} # --------------------------------------------------------------------------- # @@ -200,7 +205,8 @@ class EvidenceResult: gating: str # human-readable: "ci (required PR gate)", "opt-in (ENV)", ... node: Optional[str] = None node_found: Optional[bool] = None - junit_status: Optional[str] = None # "passed" | "failed" | None (unknown) + ci_job: Optional[str] = None + junit_status: Optional[str] = None # "passed" | "failed" | "skipped" | "unknown" @dataclass @@ -240,8 +246,19 @@ def load_registry(path: Path = REGISTRY) -> dict[str, Any]: return data +def _required_ci_job(path: str, strength: str) -> Optional[str]: + if strength == STRENGTH_OPTIN: + return "validating" + if strength != STRENGTH_CI: + return None + return "e2e-browser" if path.startswith("tests/e2e/") else "test" + + def _classify_evidence( - ev: dict[str, Any], repo_root: Path, junit: Optional[dict[str, str]] + ev: dict[str, Any], + repo_root: Path, + junit: Optional[dict[str, str]], + ci_job: Optional[str], ) -> EvidenceResult: path = str(ev["path"]) kind = ev.get("kind") or _infer_kind(path) @@ -253,6 +270,7 @@ def _classify_evidence( strength = STRENGTH_MISSING gating = "missing" node_found: Optional[bool] = None + required_ci_job: Optional[str] = None junit_status: Optional[str] = None if not exists: @@ -275,8 +293,9 @@ def _classify_evidence( else "required PR gate (test)" ) gating = f"ci ({stage})" - if junit is not None: - junit_status = junit.get(Path(path).name, "unknown") + required_ci_job = _required_ci_job(path, strength) + if junit is not None and required_ci_job == ci_job: + junit_status = junit.get(path, "unknown") else: # doc / benchmark artifact: design or field evidence, never a run proof. strength = STRENGTH_DOC @@ -291,6 +310,7 @@ def _classify_evidence( gating=gating, node=str(node) if node else None, node_found=node_found, + ci_job=required_ci_job, junit_status=junit_status, ) @@ -299,6 +319,7 @@ def validate_claim( raw: dict[str, Any], repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, + ci_job: Optional[str] = None, ) -> ClaimResult: """Validate a single registry entry, returning a ClaimResult with errors.""" cid = str(raw.get("id", "")) @@ -320,7 +341,7 @@ def validate_claim( return result for ev in raw.get("evidence", []) or []: - result.evidence.append(_classify_evidence(ev, repo_root, junit)) + result.evidence.append(_classify_evidence(ev, repo_root, junit, ci_job)) # 1) registry rot: every evidence path must exist. for e in result.evidence: @@ -346,13 +367,27 @@ def validate_claim( f"'supported' (result is not CI-reproducible)" ) - # 4) green-check (only when a junit artifact is supplied): a supported - # claim's CI tests must not be red. - if junit is not None and tier == "supported": + # 4) Required-job proof. File existence makes a test eligible to back a + # supported claim. It does not prove that the test ran. The job-scoped + # JUnit result must contain a real passing case from every cited file. + if junit is not None and ci_job is not None and tier == "supported": for e in result.evidence: - if e.strength == STRENGTH_CI and e.junit_status == "failed": + if e.strength != STRENGTH_CI or e.ci_job != ci_job: + continue + if e.junit_status == "failed": + result.errors.append( + f"[{cid}] supported claim's backing test is RED in " + f"{ci_job} JUnit: {e.path}" + ) + elif e.junit_status == "skipped": + result.errors.append( + f"[{cid}] supported claim's backing test was SKIPPED in " + f"{ci_job} JUnit: {e.path}" + ) + elif e.junit_status != "passed": result.errors.append( - f"[{cid}] supported claim's backing test is RED in junit: {e.path}" + f"[{cid}] supported claim's backing test is ABSENT from " + f"{ci_job} JUnit: {e.path}" ) return result @@ -378,37 +413,89 @@ def validate_all( registry: dict[str, Any], repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, + ci_job: Optional[str] = None, ) -> list[ClaimResult]: return [ - validate_claim(raw, repo_root=repo_root, junit=junit) + validate_claim(raw, repo_root=repo_root, junit=junit, ci_job=ci_job) for raw in registry.get("claims", []) ] # --------------------------------------------------------------------------- # -# optional junit parse (confirm supported claims are green) +# JUnit parse (prove supported claims passed in their required CI job) # --------------------------------------------------------------------------- # -def parse_junit(path: Path) -> dict[str, str]: - """Map test-file basename -> "passed"|"failed" from a junit XML artifact. +class JunitEvidenceError(ValueError): + """A required CI result is absent, malformed, or carries no test cases.""" + + +def _junit_case_path(case: Any, repo_root: Path) -> Optional[str]: + """Return a repo-relative Python test path from one JUnit testcase.""" + + file_attr = str(case.get("file") or "").replace("\\", "/").lstrip("./") + if file_attr: + if "/tests/" in file_attr: + file_attr = "tests/" + file_attr.split("/tests/", 1)[1] + candidate = Path(file_attr) + if candidate.is_absolute(): + try: + candidate = candidate.relative_to(repo_root) + except ValueError: + return None + normalized = candidate.as_posix() + if normalized.startswith("tests/") and normalized.endswith(".py"): + return normalized + + classname = str(case.get("classname") or "") + parts = classname.split(".") if classname else [] + while len(parts) >= 2: + candidate = "/".join(parts) + ".py" + if candidate.startswith("tests/") and (repo_root / candidate).is_file(): + return candidate + parts.pop() + return None + + +def parse_junit(path: Path, repo_root: Path = REPO_ROOT) -> dict[str, str]: + """Map repo-relative test file -> passed, failed, or skipped. - Best-effort and coarse (file granularity): if ANY case in a file failed or - errored, the file is "failed". Used only to red-flag a `supported` claim. + Pytest's default xUnit2 output omits the ``file`` attribute. Resolve its + dotted ``classname`` against the repository instead of silently returning + an empty map. A file is failed when any case failed or errored. It is passed + when at least one case passed and no case failed. It is skipped only when + every mapped case was skipped. """ import xml.etree.ElementTree as ET + if not path.is_file(): + raise JunitEvidenceError(f"required JUnit artifact is missing: {path}") + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError) as exc: + raise JunitEvidenceError( + f"required JUnit artifact cannot be read: {path}: {exc}" + ) from exc + status: dict[str, str] = {} - root = ET.parse(path).getroot() + rank = {"skipped": 1, "passed": 2, "failed": 3} for case in root.iter("testcase"): - file_attr = case.get("file") or case.get("classname", "") - name = Path(file_attr).name if file_attr else "" - if not name.endswith(".py"): + test_path = _junit_case_path(case, repo_root) + if test_path is None: continue - failed = any(child.tag in ("failure", "error") for child in case) - prev = status.get(name) - if failed: - status[name] = "failed" - elif prev != "failed": - status[name] = "passed" + child_tags = {child.tag.rsplit("}", 1)[-1] for child in case} + case_status = ( + "failed" + if child_tags & {"failure", "error"} + else "skipped" + if "skipped" in child_tags + else "passed" + ) + previous = status.get(test_path) + if previous is None or rank[case_status] > rank[previous]: + status[test_path] = case_status + if not status: + raise JunitEvidenceError( + f"required JUnit artifact contains no repository test cases: {path}" + ) return status @@ -440,14 +527,19 @@ def resolve_now(explicit: Optional[str]) -> str: # report generation # --------------------------------------------------------------------------- # _TIER_BADGE = { - "supported": "supported — CI-proven today", + "supported": "supported — bound to required CI pass evidence", "validating": "validating — opt-in / infra-gated or field test", "roadmap": "roadmap — designed, not yet proven", "research": "research — open question", } -def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> str: +def render_markdown( + results: list[ClaimResult], + now: str, + junit_used: bool, + junit_job: Optional[str] = None, +) -> str: lines: list[str] = [] lines.append("# VERIFICATION — maturity claims backed by tests") lines.append("") @@ -459,18 +551,29 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s lines.append(f"- Generated at: **{now}**") lines.append( "- Green-check against a junit artifact: " - + ("**run**" if junit_used else "**not run** (no `--junit` artifact supplied)") + + ( + f"**run for `{junit_job}`**" + if junit_used and junit_job + else "**not embedded in this generated registry view** " + "(required CI jobs enforce pass evidence)" + ) + ) + lines.append( + "- Structure gate: `python scripts/validate_claims.py --check " + "--structure-only` (a claim whose tier outranks its strongest backing " + "evidence fails CI)." ) lines.append( - "- Gate: `python scripts/validate_claims.py --check` " - "(a claim whose tier outranks its strongest backing evidence fails CI)." + "- Pass gates: required `test` and `e2e-browser` jobs supply their own " + "JUnit files; an absent, all-skipped, or failed supported evidence file " + "fails that required job." ) lines.append("") lines.append( "**What this harness does and does not do.** It makes each public " "maturity claim a *function* of automated evidence: a `supported` claim " - "must be backed by a test that actually runs on the default (non-opt-in) " - "CI suite; a `validating` claim must be grounded in a REAL opt-in / " + "must be backed by a test file that has a real passing case in its " + "required default CI job; a `validating` claim must be grounded in a REAL opt-in / " "infra-gated proof or a field test, and is never presented as " "supported. It does not replace workflow- and deployment-specific " "acceptance: application controls, identity rules, effect oracles, and " @@ -482,10 +585,11 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s ci = [r for r in results if r.tier == "supported"] val = [r for r in results if r.tier == "validating"] other = [r for r in results if r.tier in ("roadmap", "research")] - lines.append("## What is CI-proven today vs. being validated") + lines.append("## What is bound to required CI vs. being validated") lines.append("") lines.append( - f"- **CI-proven today ({len(ci)}):** " + ", ".join(f"`{r.id}`" for r in ci) + f"- **Bound to required CI pass evidence ({len(ci)}):** " + + ", ".join(f"`{r.id}`" for r in ci) ) lines.append( f"- **Being validated — opt-in / infra-gated or field ({len(val)}):** " @@ -537,15 +641,19 @@ def render_markdown(results: list[ClaimResult], now: str, junit_used: bool) -> s lines.append(f"- ❌ {err}") lines.append("") - return "\n".join(lines) + "\n" + return "\n".join(lines).rstrip() + "\n" def render_json( - results: list[ClaimResult], now: str, junit_used: bool + results: list[ClaimResult], + now: str, + junit_used: bool, + junit_job: Optional[str] = None, ) -> dict[str, Any]: return { "generated_at": now, "green_check_run": junit_used, + "green_check_job": junit_job, "ok": all(r.ok for r in results), "claims": [ { @@ -565,6 +673,7 @@ def render_json( "gating": e.gating, "node": e.node, "node_found": e.node_found, + "ci_job": e.ci_job, "junit_status": e.junit_status, "proves": e.proves, } @@ -583,11 +692,7 @@ def render_json( def _collect_junit(junit_path: Optional[str]) -> Optional[dict[str, str]]: if not junit_path: return None - p = Path(junit_path) - if not p.exists(): - print(f"warning: --junit artifact not found, skipping green-check: {p}") - return None - return parse_junit(p) + return parse_junit(Path(junit_path)) def main(argv: Optional[list[str]] = None) -> int: @@ -602,7 +707,18 @@ def main(argv: Optional[list[str]] = None) -> int: parser.add_argument( "--junit", default=None, - help="optional junit XML to confirm supported claims are green", + help="JUnit XML from the required job named by --ci-job", + ) + parser.add_argument( + "--ci-job", + choices=sorted(CI_JOBS), + default=None, + help="required CI job that produced --junit", + ) + parser.add_argument( + "--structure-only", + action="store_true", + help="check registry structure without claiming that supported tests passed", ) parser.add_argument( "--now", @@ -614,18 +730,39 @@ def main(argv: Optional[list[str]] = None) -> int: if not (args.check or args.report): args.check = True # default action is the gate + if args.structure_only and (args.junit or args.ci_job): + print("Claims gate FAILED: --structure-only cannot consume a JUnit result") + return 1 + if bool(args.junit) != bool(args.ci_job): + print("Claims gate FAILED: --junit and --ci-job must be supplied together") + return 1 + if args.check and not args.structure_only and not args.junit: + print( + "Claims gate FAILED: a supported-tier check requires --junit and " + "--ci-job; use --structure-only only for the separate registry-shape gate" + ) + return 1 + registry = load_registry(Path(args.registry)) - junit = _collect_junit(args.junit) - results = validate_all(registry, junit=junit) + try: + junit = _collect_junit(args.junit) + except JunitEvidenceError as exc: + print(f"Claims gate FAILED: {exc}") + return 1 + results = validate_all(registry, junit=junit, ci_job=args.ci_job) now = resolve_now(args.now) if args.report: DOC_OUT.parent.mkdir(parents=True, exist_ok=True) DOC_OUT.write_text( - render_markdown(results, now, junit is not None), encoding="utf-8" + render_markdown(results, now, junit is not None, args.ci_job), + encoding="utf-8", ) JSON_OUT.write_text( - json.dumps(render_json(results, now, junit is not None), indent=2) + "\n", + json.dumps( + render_json(results, now, junit is not None, args.ci_job), indent=2 + ) + + "\n", encoding="utf-8", ) print( @@ -642,8 +779,12 @@ def main(argv: Optional[list[str]] = None) -> int: n = len(results) proven = sum(1 for r in results if r.tier == "supported") print( - f"Claims gate passed: {n} claims, {proven} supported (CI-proven), " - "each tier backed by evidence of at least equal strength." + f"Claims gate passed: {n} claims, {proven} marked supported; " + + ( + f"all {args.ci_job} claim evidence passed." + if args.ci_job + else "registry structure is consistent; no live pass was claimed." + ) ) return 1 if errors else 0 diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index be4a0ab3..8a6a63ec 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -295,6 +295,26 @@ def test_exhaustive_identity_ladder_corpus_runs_in_the_slow_lane_only() -> None: assert workflow.count(f'{flag}: "1"') == 1 +def test_supported_claims_consume_their_required_jobs_real_junit() -> None: + """The two required test jobs must fail when cited evidence did not run.""" + + workflow = CI.read_text(encoding="utf-8") + unit_start = workflow.index("- name: Test (fast unit suite)") + unit_end = workflow.index("- name: Coverage (whole-package visibility)") + unit = workflow[unit_start:unit_end] + assert "--junitxml=runs/unit-claims-junit.xml" in unit + assert "--ci-job test --junit runs/unit-claims-junit.xml" in unit + + browser_start = workflow.index("- name: E2E (browser record -> compile -> replay)") + browser_end = workflow.index("- name: Upload run artifacts", browser_start) + browser = workflow[browser_start:browser_end] + assert "--junitxml=runs/e2e-claims-junit.xml" in browser + assert "--ci-job e2e-browser --junit runs/e2e-claims-junit.xml" in browser + + claims = VALIDATE_CLAIMS.read_text(encoding="utf-8") + assert "validate_claims.py --check --structure-only" in claims + + def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: workflow = QUICKSTART.read_text(encoding="utf-8") lifecycle_start = workflow.index(" lifecycle:") diff --git a/tests/test_validate_claims.py b/tests/test_validate_claims.py index 94817aa0..9d42da93 100644 --- a/tests/test_validate_claims.py +++ b/tests/test_validate_claims.py @@ -196,7 +196,7 @@ def test_unknown_tier_fails() -> None: # --------------------------------------------------------------------------- # -# green-check via a junit artifact (optional path) +# required-job proof via a JUnit artifact # --------------------------------------------------------------------------- # def test_junit_green_check_flags_red_supported_test(tmp_path: Path) -> None: junit = tmp_path / "junit.xml" @@ -210,10 +210,71 @@ def test_junit_green_check_flags_red_supported_test(tmp_path: Path) -> None: encoding="utf-8", ) parsed = vc.parse_junit(junit) - assert parsed.get("test_replayer.py") == "failed" - result = vc.validate_claim(_claim(), junit=parsed) + assert parsed.get("tests/test_replayer.py") == "failed" + result = vc.validate_claim(_claim(), junit=parsed, ci_job="test") assert not result.ok - assert any("RED in junit" in e for e in result.errors) + assert any("RED in test JUnit" in e for e in result.errors) + + +def test_pytest_xunit2_classname_is_mapped_without_file_attribute( + tmp_path: Path, +) -> None: + junit = tmp_path / "junit.xml" + junit.write_text( + """ + + + """, + encoding="utf-8", + ) + + assert vc.parse_junit(junit) == {"tests/test_replayer.py": "passed"} + + +def test_supported_evidence_must_be_present_and_not_all_skipped() -> None: + missing = vc.validate_claim(_claim(), junit={}, ci_job="test") + skipped = vc.validate_claim(_claim(), junit={CI_TEST: "skipped"}, ci_job="test") + passed = vc.validate_claim(_claim(), junit={CI_TEST: "passed"}, ci_job="test") + + assert any("ABSENT" in error for error in missing.errors) + assert any("SKIPPED" in error for error in skipped.errors) + assert passed.ok, passed.errors + + +def test_junit_file_is_passed_when_one_case_passes_and_an_optional_case_skips( + tmp_path: Path, +) -> None: + junit = tmp_path / "junit.xml" + junit.write_text( + """ + + + + """, + encoding="utf-8", + ) + + assert vc.parse_junit(junit)[CI_TEST] == "passed" + + +def test_missing_or_empty_junit_fails_closed(tmp_path: Path) -> None: + missing = tmp_path / "missing.xml" + empty = tmp_path / "empty.xml" + empty.write_text("", encoding="utf-8") + + for path in (missing, empty): + try: + vc.parse_junit(path) + except vc.JunitEvidenceError: + pass + else: + raise AssertionError(f"{path} did not fail closed") + + +def test_cli_requires_junit_for_non_structural_supported_check() -> None: + assert vc.main(["--check"]) == 1 + assert vc.main(["--check", "--structure-only"]) == 0 + assert vc.main(["--check", "--junit", "missing.xml"]) == 1 def test_report_renders_without_crashing() -> None: @@ -221,6 +282,13 @@ def test_report_renders_without_crashing() -> None: md = vc.render_markdown(results, now="2026-07-14T00:00:00Z", junit_used=False) assert "VERIFICATION" in md assert "web-supported" in md + assert md.endswith("\n") and not md.endswith("\n\n") blob = vc.render_json(results, now="2026-07-14T00:00:00Z", junit_used=False) assert blob["ok"] is True + assert blob["green_check_job"] is None + assert all( + "ci_job" in evidence + for claim in blob["claims"] + for evidence in claim["evidence"] + ) assert {c["id"] for c in blob["claims"]} # non-empty From 1da27358542524b3b4187ec0d54b1fa46e7936e5 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 14:42:42 -0400 Subject: [PATCH 2/5] fix: bind validating evidence to its substrate --- .github/workflows/validate-claims.yml | 119 ++++++++---------- docs/VERIFICATION.md | 3 +- docs/verification.json | 2 + public-artifacts.json | 4 +- scripts/validate_claims.py | 171 ++++++++++++++++++++++---- tests/test_ci_workflow_contract.py | 21 ++++ tests/test_validate_claims.py | 120 +++++++++++++++++- 7 files changed, 343 insertions(+), 97 deletions(-) diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index 6130f043..13659cad 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -11,11 +11,11 @@ name: Validate claims # missing evidence path). The required CI `test` and `e2e-browser` jobs # separately bind every supported test file to their real JUnit results. # It NEVER touches infra. -# - schedule + workflow_dispatch: the infra-gated leg that provisions a real -# Windows VM (via oa-vm) and runs the OPT-IN desktop + Citrix e2e that back -# the `validating` tiers, then regenerates the evidence report. Where the -# infra/secrets are absent it SKIPS cleanly and says so -- it never fakes a -# pass (a fabricated green would defeat the whole harness). +# - schedule + workflow_dispatch: the infra-gated leg runs only on a labeled +# self-hosted macOS runner with the declared local Parallels substrate. It +# binds its two selected tests to their real JUnit results and emits an +# exactly scoped report. Without that runner configuration the job SKIPS; +# it never publishes a fabricated green. # # GitHub Actions are pinned to full commit SHAs (supply-chain control); the # trailing comment records the human-readable version, matching ci.yml. @@ -27,7 +27,7 @@ on: workflow_dispatch: inputs: run_infra: - description: "Provision a Windows VM and run the opt-in desktop/Citrix e2e" + description: "Run the configured macOS + Parallels evidence refresh" type: boolean default: false @@ -86,19 +86,30 @@ jobs: pip install "pytest>=8" pytest -q tests/test_validate_claims.py - # --- Infra-gated: refresh the VALIDATING-tier evidence on a real VM ------- - # Provisions a Windows VM via oa-vm and runs the OPT-IN desktop + Citrix e2e - # that ground the `windows-desktop-validating` / `citrix-pixel-validating` - # claims (see tests/e2e/test_parallels_desktop_e2e.py and - # tests/e2e/test_citrix_pixel_e2e.py). NOT run on PRs and NOT a required - # check. It documents the human, infra half the PR gate cannot self-generate. + # --- Infra-gated: scoped validating evidence on its exact substrate ------- + # These selected tests require macOS, local Parallels, a running configured + # Windows VM, and a known base snapshot. They cannot run on ubuntu-latest or + # against an unrelated hosted VM endpoint. A repository owner enables the + # job only after a self-hosted runner has all three labels below and the two + # required repository variables are configured. # - # IMPORTANT: when the VM credentials/runner are absent this job SKIPS every - # infra step (never fails, never fabricates a pass). A green here means the - # opt-in e2e actually ran on real infra; a skip means "not proven this run". + # A skipped job means "not proven this run" and produces no report. A green + # job means BOTH named files contained a real pass and no failure/error/skip. + # RDP, Citrix stand-in, and retained OpenEMR evidence remain separate scopes; + # this job never implies that those files ran. refresh-validating-evidence: - if: ${{ github.event_name != 'pull_request' }} - runs-on: ubuntu-latest + if: >- + ${{ vars.OPENADAPT_PARALLELS_VALIDATION_ENABLED == 'true' && + (github.event_name == 'schedule' || inputs.run_infra == true) }} + runs-on: [self-hosted, macos, arm64] + timeout-minutes: 90 + env: + OAFLOW_PARALLELS_VM_UUID: ${{ vars.OAFLOW_PARALLELS_VM_UUID }} + OAFLOW_PARALLELS_BASE_SNAPSHOT_ID: ${{ vars.OAFLOW_PARALLELS_BASE_SNAPSHOT_ID }} + OAFLOW_PARALLELS_STORAGE_PATH: ${{ github.workspace }} + OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT: ${{ github.sha }} + OAFLOW_WINDOWS_UIA_MATRIX_ID: claims-${{ github.run_id }}-${{ github.run_attempt }} + OAFLOW_WINDOWS_UIA_EVIDENCE: ${{ github.workspace }}/runs/windows-uia-evidence.jsonl steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -106,71 +117,49 @@ jobs: with: python-version: "3.12" - # Gate the whole infra leg on a secret being present. Secrets are not - # available to forks/PRs, so this cannot be tricked into running from an - # untrusted PR, and a repo without infra configured simply skips. - - name: Detect infra availability - id: infra - env: - OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }} - run: | - if [ -n "${OA_VM_TOKEN}" ] && \ - { [ "${{ github.event_name }}" = "schedule" ] || \ - [ "${{ github.event.inputs.run_infra }}" = "true" ]; }; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::notice::Windows VM infra not available (no OA_VM_TOKEN or" \ - "run_infra not requested) -- SKIPPING the opt-in desktop/Citrix" \ - "e2e. This is a clean skip, NOT a pass. The validating-tier" \ - "claims remain grounded by their committed opt-in tests." - fi - - # The following steps are the DOCUMENTED infra path. They only run when a - # real VM is reachable; otherwise they are skipped by the `if:` guard. - - name: Provision Windows VM (oa-vm) - if: steps.infra.outputs.available == 'true' - env: - OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }} + - name: Verify exact macOS + Parallels substrate run: | - pip install "pyyaml>=6" - pip install -e .[dev] - # oa-vm brings up a WAA pool VM; the win_agent + Parallels/RDP target - # is what the opt-in e2e drive. Terminate is in the always() step. - oa-vm pool-create --name claims-validation - oa-vm pool-wait --name claims-validation - - - name: Run opt-in desktop + Citrix e2e (the validating-tier proofs) - if: steps.infra.outputs.available == 'true' + [ "$(uname -s)" = "Darwin" ] || { + echo "::error::validating evidence requires macOS"; exit 1; + } + command -v prlctl >/dev/null || { + echo "::error::Parallels prlctl is unavailable"; exit 1; + } + [ -n "$OAFLOW_PARALLELS_VM_UUID" ] || { + echo "::error::OAFLOW_PARALLELS_VM_UUID is not configured"; exit 1; + } + [ -n "$OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" ] || { + echo "::error::OAFLOW_PARALLELS_BASE_SNAPSHOT_ID is not configured"; exit 1; + } + + - name: Install validation dependencies + run: python -m pip install -e .[dev] + + - name: Run scoped desktop and pixel evidence env: OAFLOW_PARALLELS_E2E: "1" OAFLOW_CITRIX_PIXEL_E2E: "1" run: | mkdir -p runs - pytest -q \ + python -m pytest -q \ tests/e2e/test_parallels_desktop_e2e.py \ tests/e2e/test_citrix_pixel_e2e.py \ --junitxml=runs/validating-junit.xml --basetemp=runs/ci - - name: Regenerate evidence report with the live green-check - if: steps.infra.outputs.available == 'true' + - name: Require every selected evidence file to pass run: | python scripts/validate_claims.py --report \ - --ci-job validating --junit runs/validating-junit.xml + --ci-job validating --junit runs/validating-junit.xml \ + --evidence-path tests/e2e/test_parallels_desktop_e2e.py \ + --evidence-path tests/e2e/test_citrix_pixel_e2e.py - - name: Upload refreshed evidence - if: steps.infra.outputs.available == 'true' + - name: Upload scoped passing evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: verification-evidence + name: verification-evidence-macos-parallels path: | docs/VERIFICATION.md docs/verification.json runs/validating-junit.xml + runs/windows-uia-evidence.jsonl if-no-files-found: warn - - - name: Terminate VM (always) - if: always() && steps.infra.outputs.available == 'true' - env: - OA_VM_TOKEN: ${{ secrets.OA_VM_TOKEN }} - run: oa-vm pool-cleanup --name claims-validation || true diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 28aaee27..bcf06ab7 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -3,9 +3,10 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. - Generated at: **committed registry state (regenerate: scripts/validate_claims.py --report)** -- Green-check against a junit artifact: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) +- JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). - Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. +- Scoped validating refreshes name each selected evidence file; each selected file must pass on its declared substrate. Unselected validating evidence is not represented as checked. **What this harness does and does not do.** It makes each public maturity claim a *function* of automated evidence: a `supported` claim must be backed by a test file that has a real passing case in its required default CI job; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. It does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. diff --git a/docs/verification.json b/docs/verification.json index f01c9349..6c5c8cb3 100644 --- a/docs/verification.json +++ b/docs/verification.json @@ -2,7 +2,9 @@ "generated_at": "committed registry state (regenerate: scripts/validate_claims.py --report)", "green_check_run": false, "green_check_job": null, + "green_check_scope": [], "ok": true, + "validation_errors": [], "claims": [ { "id": "web-supported", diff --git a/public-artifacts.json b/public-artifacts.json index f3ca931c..c16a82d4 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -121,7 +121,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "7030f250fe66f36e71f3b44d52713e7f550a10e04c10f37ebdc745fa9a97c69a" + "sha256": "75265f62b9606ee7eca54a989f72d09dd7a0f16016e5883fd723a09d825d990b" }, { "path": ".pre-commit-config.yaml", @@ -1809,7 +1809,7 @@ }, { "path": "docs/verification.json", - "sha256": "c8275336c9581532a3841a57913228b7584ab1487cdcc9c7b7ab884326769842" + "sha256": "538fc0a2dc392a10617da96522191d03aa3d3cbe0288bd4891cd14e1de274f97" }, { "path": "openadapt_flow/console/static/console.css", diff --git a/scripts/validate_claims.py b/scripts/validate_claims.py index 224946f8..73d95c56 100644 --- a/scripts/validate_claims.py +++ b/scripts/validate_claims.py @@ -31,6 +31,9 @@ python scripts/validate_claims.py --check --structure-only python scripts/validate_claims.py --check --ci-job test \ --junit runs/unit-claims-junit.xml + python scripts/validate_claims.py --report --ci-job validating \ + --junit runs/validating-junit.xml \ + --evidence-path tests/e2e/test_parallels_desktop_e2e.py python scripts/validate_claims.py --report # (re)write docs/VERIFICATION.md + .json The public functions are importable so ``tests/test_validate_claims.py`` can @@ -320,6 +323,7 @@ def validate_claim( repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, ci_job: Optional[str] = None, + evidence_scope: Optional[set[str]] = None, ) -> ClaimResult: """Validate a single registry entry, returning a ClaimResult with errors.""" cid = str(raw.get("id", "")) @@ -368,25 +372,37 @@ def validate_claim( ) # 4) Required-job proof. File existence makes a test eligible to back a - # supported claim. It does not prove that the test ran. The job-scoped - # JUnit result must contain a real passing case from every cited file. - if junit is not None and ci_job is not None and tier == "supported": + # claim. It does not prove that the test ran. Required supported jobs + # enforce every assigned file. A substrate-specific validating run + # enforces every file in its explicit scope and makes no claim about the + # other validating evidence. + if junit is not None and ci_job is not None: for e in result.evidence: - if e.strength != STRENGTH_CI or e.ci_job != ci_job: + if e.ci_job != ci_job or e.strength not in { + STRENGTH_CI, + STRENGTH_OPTIN, + }: + continue + required = ( + e.path in evidence_scope + if evidence_scope is not None + else tier == "supported" + ) + if not required: continue if e.junit_status == "failed": result.errors.append( - f"[{cid}] supported claim's backing test is RED in " + f"[{cid}] {tier} claim's backing test is RED in " f"{ci_job} JUnit: {e.path}" ) elif e.junit_status == "skipped": result.errors.append( - f"[{cid}] supported claim's backing test was SKIPPED in " + f"[{cid}] {tier} claim's backing test was SKIPPED in " f"{ci_job} JUnit: {e.path}" ) elif e.junit_status != "passed": result.errors.append( - f"[{cid}] supported claim's backing test is ABSENT from " + f"[{cid}] {tier} claim's backing test is ABSENT from " f"{ci_job} JUnit: {e.path}" ) @@ -414,13 +430,41 @@ def validate_all( repo_root: Path = REPO_ROOT, junit: Optional[dict[str, str]] = None, ci_job: Optional[str] = None, + evidence_scope: Optional[set[str]] = None, ) -> list[ClaimResult]: return [ - validate_claim(raw, repo_root=repo_root, junit=junit, ci_job=ci_job) + validate_claim( + raw, + repo_root=repo_root, + junit=junit, + ci_job=ci_job, + evidence_scope=evidence_scope, + ) for raw in registry.get("claims", []) ] +def validate_evidence_scope( + results: list[ClaimResult], + ci_job: Optional[str], + evidence_scope: Optional[set[str]], +) -> list[str]: + """Reject a scoped live check that names evidence outside its exact job.""" + + if evidence_scope is None: + return [] + eligible = { + evidence.path + for result in results + for evidence in result.evidence + if evidence.ci_job == ci_job + } + return [ + f"[{ci_job}] scoped evidence is not registered for this job: {path}" + for path in sorted(evidence_scope - eligible) + ] + + # --------------------------------------------------------------------------- # # JUnit parse (prove supported claims passed in their required CI job) # --------------------------------------------------------------------------- # @@ -448,9 +492,12 @@ def _junit_case_path(case: Any, repo_root: Path) -> Optional[str]: classname = str(case.get("classname") or "") parts = classname.split(".") if classname else [] while len(parts) >= 2: - candidate = "/".join(parts) + ".py" - if candidate.startswith("tests/") and (repo_root / candidate).is_file(): - return candidate + candidate_path = "/".join(parts) + ".py" + if ( + candidate_path.startswith("tests/") + and (repo_root / candidate_path).is_file() + ): + return candidate_path parts.pop() return None @@ -539,6 +586,8 @@ def render_markdown( now: str, junit_used: bool, junit_job: Optional[str] = None, + junit_scope: Optional[set[str]] = None, + validation_errors: Optional[list[str]] = None, ) -> str: lines: list[str] = [] lines.append("# VERIFICATION — maturity claims backed by tests") @@ -549,15 +598,19 @@ def render_markdown( ) lines.append("") lines.append(f"- Generated at: **{now}**") - lines.append( - "- Green-check against a junit artifact: " - + ( - f"**run for `{junit_job}`**" - if junit_used and junit_job - else "**not embedded in this generated registry view** " + if junit_used and junit_job: + scope_text = ( + "; exact evidence scope: " + + ", ".join(f"`{path}`" for path in sorted(junit_scope)) + if junit_scope + else "; all supported evidence assigned to this job" + ) + lines.append(f"- JUnit pass check: **run for `{junit_job}`{scope_text}**") + else: + lines.append( + "- JUnit pass check: **not embedded in this generated registry view** " "(required CI jobs enforce pass evidence)" ) - ) lines.append( "- Structure gate: `python scripts/validate_claims.py --check " "--structure-only` (a claim whose tier outranks its strongest backing " @@ -568,6 +621,17 @@ def render_markdown( "JUnit files; an absent, all-skipped, or failed supported evidence file " "fails that required job." ) + lines.append( + "- Scoped validating refreshes name each selected evidence file; each " + "selected file must pass on its declared substrate. Unselected validating " + "evidence is not represented as checked." + ) + if validation_errors: + lines.append("") + lines.append("## Validation errors") + lines.append("") + for error in validation_errors: + lines.append(f"- ❌ {error}") lines.append("") lines.append( "**What this harness does and does not do.** It makes each public " @@ -649,12 +713,17 @@ def render_json( now: str, junit_used: bool, junit_job: Optional[str] = None, + junit_scope: Optional[set[str]] = None, + validation_errors: Optional[list[str]] = None, ) -> dict[str, Any]: + validation_errors = validation_errors or [] return { "generated_at": now, "green_check_run": junit_used, "green_check_job": junit_job, - "ok": all(r.ok for r in results), + "green_check_scope": sorted(junit_scope or []), + "ok": all(r.ok for r in results) and not validation_errors, + "validation_errors": validation_errors, "claims": [ { "id": r.id, @@ -715,6 +784,15 @@ def main(argv: Optional[list[str]] = None) -> int: default=None, help="required CI job that produced --junit", ) + parser.add_argument( + "--evidence-path", + action="append", + default=[], + help=( + "exact validating evidence path selected by this substrate-specific " + "run; repeat for each selected file" + ), + ) parser.add_argument( "--structure-only", action="store_true", @@ -730,8 +808,11 @@ def main(argv: Optional[list[str]] = None) -> int: if not (args.check or args.report): args.check = True # default action is the gate - if args.structure_only and (args.junit or args.ci_job): - print("Claims gate FAILED: --structure-only cannot consume a JUnit result") + if args.structure_only and (args.junit or args.ci_job or args.evidence_path): + print( + "Claims gate FAILED: --structure-only cannot consume a JUnit result " + "or evidence scope" + ) return 1 if bool(args.junit) != bool(args.ci_job): print("Claims gate FAILED: --junit and --ci-job must be supplied together") @@ -742,6 +823,19 @@ def main(argv: Optional[list[str]] = None) -> int: "--ci-job; use --structure-only only for the separate registry-shape gate" ) return 1 + evidence_scope = set(args.evidence_path) if args.evidence_path else None + if args.ci_job == "validating" and evidence_scope is None: + print( + "Claims gate FAILED: --ci-job validating requires one or more exact " + "--evidence-path values" + ) + return 1 + if args.ci_job in {"test", "e2e-browser"} and evidence_scope is not None: + print( + "Claims gate FAILED: required supported CI jobs derive their complete " + "evidence scope from claims.yaml" + ) + return 1 registry = load_registry(Path(args.registry)) try: @@ -749,18 +843,39 @@ def main(argv: Optional[list[str]] = None) -> int: except JunitEvidenceError as exc: print(f"Claims gate FAILED: {exc}") return 1 - results = validate_all(registry, junit=junit, ci_job=args.ci_job) + results = validate_all( + registry, + junit=junit, + ci_job=args.ci_job, + evidence_scope=evidence_scope, + ) + scope_errors = validate_evidence_scope(results, args.ci_job, evidence_scope) now = resolve_now(args.now) if args.report: DOC_OUT.parent.mkdir(parents=True, exist_ok=True) DOC_OUT.write_text( - render_markdown(results, now, junit is not None, args.ci_job), + render_markdown( + results, + now, + junit is not None, + args.ci_job, + evidence_scope, + scope_errors, + ), encoding="utf-8", ) JSON_OUT.write_text( json.dumps( - render_json(results, now, junit is not None, args.ci_job), indent=2 + render_json( + results, + now, + junit is not None, + args.ci_job, + evidence_scope, + scope_errors, + ), + indent=2, ) + "\n", encoding="utf-8", @@ -769,7 +884,7 @@ def main(argv: Optional[list[str]] = None) -> int: f"wrote {DOC_OUT.relative_to(REPO_ROOT)} and {JSON_OUT.relative_to(REPO_ROOT)}" ) - errors = [err for r in results for err in r.errors] + errors = [err for r in results for err in r.errors] + scope_errors if args.check: if errors: print(f"Claims gate FAILED ({len(errors)} violation(s)):") @@ -781,7 +896,11 @@ def main(argv: Optional[list[str]] = None) -> int: print( f"Claims gate passed: {n} claims, {proven} marked supported; " + ( - f"all {args.ci_job} claim evidence passed." + ( + f"all scoped {args.ci_job} claim evidence passed." + if evidence_scope is not None + else f"all {args.ci_job} claim evidence passed." + ) if args.ci_job else "registry structure is consistent; no live pass was claimed." ) diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index 8a6a63ec..a2a8dfda 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -315,6 +315,27 @@ def test_supported_claims_consume_their_required_jobs_real_junit() -> None: assert "validate_claims.py --check --structure-only" in claims +def test_validating_refresh_uses_exact_macos_parallels_substrate_and_scope() -> None: + """A green validating report cannot come from Ubuntu or skipped evidence.""" + + claims = VALIDATE_CLAIMS.read_text(encoding="utf-8") + start = claims.index(" refresh-validating-evidence:") + refresh = claims[start:] + + assert "runs-on: [self-hosted, macos, arm64]" in refresh + assert "OPENADAPT_PARALLELS_VALIDATION_ENABLED" in refresh + assert "runs-on: ubuntu-latest" not in refresh + assert "oa-vm" not in refresh + assert "command -v prlctl" in refresh + assert "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" in refresh + assert "--ci-job validating --junit runs/validating-junit.xml" in refresh + for path in ( + "tests/e2e/test_parallels_desktop_e2e.py", + "tests/e2e/test_citrix_pixel_e2e.py", + ): + assert f"--evidence-path {path}" in refresh + + def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: workflow = QUICKSTART.read_text(encoding="utf-8") lifecycle_start = workflow.index(" lifecycle:") diff --git a/tests/test_validate_claims.py b/tests/test_validate_claims.py index 9d42da93..0ab56aaa 100644 --- a/tests/test_validate_claims.py +++ b/tests/test_validate_claims.py @@ -33,6 +33,7 @@ # they are ever moved this test fails loudly rather than testing a fiction. OPTIN_TEST = "tests/e2e/test_citrix_pixel_e2e.py" INLINE_OPTIN_TEST = "tests/e2e/test_citrix_workspace_standin_e2e.py" +PARALLELS_TEST = "tests/e2e/test_parallels_desktop_e2e.py" CI_TEST = "tests/test_replayer.py" DOC_ARTIFACT = "docs/desktop/CITRIX_PIXEL.md" @@ -78,9 +79,7 @@ def test_every_real_evidence_path_exists() -> None: def test_optin_detector_flags_the_real_optin_tests() -> None: citrix = (REPO_ROOT / OPTIN_TEST).read_text(encoding="utf-8") citrix_standin = (REPO_ROOT / INLINE_OPTIN_TEST).read_text(encoding="utf-8") - parallels = (REPO_ROOT / "tests/e2e/test_parallels_desktop_e2e.py").read_text( - encoding="utf-8" - ) + parallels = (REPO_ROOT / PARALLELS_TEST).read_text(encoding="utf-8") assert vc.detect_optin_env(citrix) == "OAFLOW_CITRIX_PIXEL_E2E" assert vc.detect_optin_env(citrix_standin) == "OAFLOW_CITRIX_STANDIN_E2E" assert vc.detect_optin_env(parallels) == "OAFLOW_PARALLELS_E2E" @@ -241,6 +240,107 @@ def test_supported_evidence_must_be_present_and_not_all_skipped() -> None: assert passed.ok, passed.errors +def test_scoped_validating_evidence_must_pass_on_its_exact_substrate() -> None: + claim = _claim( + tier="validating", + evidence=[{"path": OPTIN_TEST, "proves": "x"}], + ) + scope = {OPTIN_TEST} + absent = vc.validate_claim( + claim, + junit={}, + ci_job="validating", + evidence_scope=scope, + ) + skipped = vc.validate_claim( + claim, + junit={OPTIN_TEST: "skipped"}, + ci_job="validating", + evidence_scope=scope, + ) + failed = vc.validate_claim( + claim, + junit={OPTIN_TEST: "failed"}, + ci_job="validating", + evidence_scope=scope, + ) + passed = vc.validate_claim( + claim, + junit={OPTIN_TEST: "passed"}, + ci_job="validating", + evidence_scope=scope, + ) + + assert any("ABSENT" in error for error in absent.errors) + assert any("SKIPPED" in error for error in skipped.errors) + assert any("RED" in error for error in failed.errors) + assert passed.ok, passed.errors + + +def test_scoped_validating_report_is_not_ok_when_selected_test_skips() -> None: + scope = {OPTIN_TEST} + results = [ + vc.validate_claim( + _claim( + tier="validating", + evidence=[{"path": OPTIN_TEST, "proves": "x"}], + ), + junit={OPTIN_TEST: "skipped"}, + ci_job="validating", + evidence_scope=scope, + ) + ] + + blob = vc.render_json( + results, + now="2026-07-14T00:00:00Z", + junit_used=True, + junit_job="validating", + junit_scope=scope, + ) + assert blob["green_check_run"] is True + assert blob["green_check_job"] == "validating" + assert blob["green_check_scope"] == [OPTIN_TEST] + assert blob["ok"] is False + + +def test_real_validating_scope_checks_only_its_two_declared_files() -> None: + scope = {PARALLELS_TEST, OPTIN_TEST} + results = vc.validate_all( + vc.load_registry(), + junit={PARALLELS_TEST: "passed", OPTIN_TEST: "passed"}, + ci_job="validating", + evidence_scope=scope, + ) + errors = [error for result in results for error in result.errors] + + assert errors == [] + assert vc.validate_evidence_scope(results, "validating", scope) == [] + unselected = { + evidence.path: evidence.junit_status + for result in results + for evidence in result.evidence + if evidence.ci_job == "validating" and evidence.path not in scope + } + assert unselected + assert set(unselected.values()) == {"unknown"} + + +def test_scoped_validating_evidence_rejects_unregistered_path() -> None: + results = vc.validate_all(vc.load_registry()) + + errors = vc.validate_evidence_scope( + results, + "validating", + {"tests/e2e/test_not_registered.py"}, + ) + + assert errors == [ + "[validating] scoped evidence is not registered for this job: " + "tests/e2e/test_not_registered.py" + ] + + def test_junit_file_is_passed_when_one_case_passes_and_an_optional_case_skips( tmp_path: Path, ) -> None: @@ -275,6 +375,18 @@ def test_cli_requires_junit_for_non_structural_supported_check() -> None: assert vc.main(["--check"]) == 1 assert vc.main(["--check", "--structure-only"]) == 0 assert vc.main(["--check", "--junit", "missing.xml"]) == 1 + assert ( + vc.main( + [ + "--check", + "--ci-job", + "validating", + "--junit", + "missing.xml", + ] + ) + == 1 + ) def test_report_renders_without_crashing() -> None: @@ -286,6 +398,8 @@ def test_report_renders_without_crashing() -> None: blob = vc.render_json(results, now="2026-07-14T00:00:00Z", junit_used=False) assert blob["ok"] is True assert blob["green_check_job"] is None + assert blob["green_check_scope"] == [] + assert blob["validation_errors"] == [] assert all( "ci_job" in evidence for claim in blob["claims"] From 460cdb16759bb8d9f85e5f1dea5b922f7229d0b7 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 14:59:15 -0400 Subject: [PATCH 3/5] fix: make validating VM lifecycle fail closed --- .github/actionlint.yaml | 3 + .github/workflows/validate-claims.yml | 27 ++++- docs/desktop_windows_runbook.md | 37 ++++--- openadapt_flow/backends/parallels_vm.py | 23 ++++ public-artifacts.json | 6 +- tests/e2e/test_citrix_pixel_e2e.py | 139 +++++++++++++++++++----- tests/e2e/test_parallels_desktop_e2e.py | 6 + tests/test_ci_workflow_contract.py | 9 +- tests/test_parallels_vm.py | 111 +++++++++++++++++++ 9 files changed, 312 insertions(+), 49 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..c737ab0c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - openadapt-parallels diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index 13659cad..687b9391 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -33,7 +33,9 @@ on: concurrency: group: validate-claims-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # A pull-request structure check is safe to supersede. A physical Parallels + # run is not: cancellation could interrupt its exact-snapshot cleanup. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -90,7 +92,7 @@ jobs: # These selected tests require macOS, local Parallels, a running configured # Windows VM, and a known base snapshot. They cannot run on ubuntu-latest or # against an unrelated hosted VM endpoint. A repository owner enables the - # job only after a self-hosted runner has all three labels below and the two + # job only after a self-hosted runner has all four labels below and the two # required repository variables are configured. # # A skipped job means "not proven this run" and produces no report. A green @@ -101,8 +103,11 @@ jobs: if: >- ${{ vars.OPENADAPT_PARALLELS_VALIDATION_ENABLED == 'true' && (github.event_name == 'schedule' || inputs.run_infra == true) }} - runs-on: [self-hosted, macos, arm64] + runs-on: [self-hosted, macos, arm64, openadapt-parallels] timeout-minutes: 90 + concurrency: + group: openadapt-parallels-validation + cancel-in-progress: false env: OAFLOW_PARALLELS_VM_UUID: ${{ vars.OAFLOW_PARALLELS_VM_UUID }} OAFLOW_PARALLELS_BASE_SNAPSHOT_ID: ${{ vars.OAFLOW_PARALLELS_BASE_SNAPSHOT_ID }} @@ -135,6 +140,20 @@ jobs: - name: Install validation dependencies run: python -m pip install -e .[dev] + - name: Prove configured VM and base are current before mutation + run: | + python -c ' + import os + from openadapt_flow.backends.parallels_vm import ParallelsVM + vm = ParallelsVM(os.environ["OAFLOW_PARALLELS_VM_UUID"]) + vm.require_host_free_space( + storage_path=os.environ["OAFLOW_PARALLELS_STORAGE_PATH"] + ) + vm.require_current_snapshot( + os.environ["OAFLOW_PARALLELS_BASE_SNAPSHOT_ID"] + ) + ' + - name: Run scoped desktop and pixel evidence env: OAFLOW_PARALLELS_E2E: "1" @@ -142,8 +161,8 @@ jobs: run: | mkdir -p runs python -m pytest -q \ - tests/e2e/test_parallels_desktop_e2e.py \ tests/e2e/test_citrix_pixel_e2e.py \ + tests/e2e/test_parallels_desktop_e2e.py \ --junitxml=runs/validating-junit.xml --basetemp=runs/ci - name: Require every selected evidence file to pass diff --git a/docs/desktop_windows_runbook.md b/docs/desktop_windows_runbook.md index 5cf247c3..6df43d92 100644 --- a/docs/desktop_windows_runbook.md +++ b/docs/desktop_windows_runbook.md @@ -2,8 +2,8 @@ How to bring the **desktop** path to parity with the **web** path — record → compile → replay with the structural (UIA) + vision ladder, identity, and -effect verification — and run the **snapshot-safe live proof** in one pass on a -machine with the Parallels Windows 11 VM. +effect verification — and run the **snapshot-safe live proof** in three counted +trials on a machine with the Parallels Windows 11 VM. This is the operator runbook. The design/rationale lives in `openadapt_flow/backends/win_agent/README.md` (the agent) and @@ -126,25 +126,34 @@ fire. --- -## 4. Run the snapshot-safe live proof (one pass) +## 4. Run the snapshot-safe live proof (three counted trials) The opt-in e2e (`tests/e2e/test_parallels_desktop_e2e.py`) does the whole loop -against the built-in Windows **Calculator** (deterministic, no PHI): snapshot → -ensure VM up → launch the agent in session 1 → record→compile→replay via -`WindowsBackend` → assert the UIA structural rung fires and the run completes → -**revert to the snapshot**. +against the deterministic Patient Notes fixture: prove one exact configured +base snapshot is current → revert to that base → ensure the VM is running → +create one exact trial snapshot → launch the typed agent in session 1 → +record→compile→replay via `WindowsBackend` → assert the UIA structural rung and +independent database oracle → restore and verify the configured base → delete +only the exact trial snapshot → suspend and verify the VM. It repeats this +contract three times. ```bash # On the Mac with the Parallels VM: -OAFLOW_PARALLELS_E2E=1 pytest -q tests/e2e/test_parallels_desktop_e2e.py -# Optional: OAFLOW_PARALLELS_VM_UUID='{...}' to target a different VM. +export OAFLOW_PARALLELS_E2E=1 +export OAFLOW_PARALLELS_VM_UUID='{exact-vm-uuid}' +export OAFLOW_PARALLELS_BASE_SNAPSHOT_ID='{exact-base-snapshot-uuid}' +export OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT='40-character-git-commit' +export OAFLOW_PARALLELS_STORAGE_PATH="$PWD" +pytest -q tests/e2e/test_parallels_desktop_e2e.py ``` -**Snapshot safety, guaranteed:** the test takes a **fresh** snapshot before it -touches the guest and reverts to it in a `finally` block. It **never deletes** -the VM or any snapshot, and it is **skipped entirely** unless -`OAFLOW_PARALLELS_E2E=1`. Always confirm your VM has a known-good snapshot before -running anything against it. +**Snapshot safety contract:** the test refuses before mutation unless the exact +configured base exists and is current. It retains each created trial snapshot +ID in memory. Cleanup restores and verifies the base before it deletes only +that exact ID. A restore, verification, deletion, evidence-write, or final +suspend failure rejects the trial. The harness never uses a snapshot name, +wildcard, child-recursive delete, or VM delete operation. It is skipped entirely +unless `OAFLOW_PARALLELS_E2E=1`. --- diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py index 2949abb9..a3233f34 100644 --- a/openadapt_flow/backends/parallels_vm.py +++ b/openadapt_flow/backends/parallels_vm.py @@ -278,6 +278,29 @@ def list_snapshots(self) -> list[SnapshotInfo]: out.append(SnapshotInfo(snapshot_id=sid, current=bool(star))) return out + def require_current_snapshot(self, snapshot_id: str) -> None: + """Prove one exact preserved snapshot exists and is current. + + Qualification callers use this read-only check before they resume, + focus, snapshot, deploy, or actuate on a dedicated VM. It prevents a + run from preserving and later deleting an unrelated current state. + """ + + snapshot_pattern = ( + r"\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}" + ) + if re.fullmatch(snapshot_pattern, snapshot_id) is None: + raise ValueError("snapshot_id must be one exact braced UUID") + snapshots = self.list_snapshots() + match = next( + (item for item in snapshots if item.snapshot_id == snapshot_id), None + ) + if match is None: + raise ParallelsError("preserved base snapshot is missing") + if not match.current: + raise ParallelsError("preserved base snapshot is not current") + def revert(self, snapshot_id: str) -> None: """Revert to a snapshot (the per-run clean-state reset).""" self._run(["snapshot-switch", self.uuid, "-i", snapshot_id]) diff --git a/public-artifacts.json b/public-artifacts.json index c16a82d4..6b2aaf35 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -71,6 +71,10 @@ "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "sha256": "bb032fd5942477be558870dbf2c6615d258b6600ac2b389c5c4bf7c55b856da6" }, + { + "path": ".github/actionlint.yaml", + "sha256": "f58f146d55a022c530f1dd35e5da6a2e9ea0cd5cbcac0e6b0e50995a8b45cf20" + }, { "path": ".github/dependabot.yml", "sha256": "40a11fc00e1010db80d8788ee8c6125b51b14d642be018cddb761ea209f66095" @@ -121,7 +125,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "75265f62b9606ee7eca54a989f72d09dd7a0f16016e5883fd723a09d825d990b" + "sha256": "368accbf68954e8e481fc6d7d36229b9cb70c267ec302e01657aa6537900a9e6" }, { "path": ".pre-commit-config.yaml", diff --git a/tests/e2e/test_citrix_pixel_e2e.py b/tests/e2e/test_citrix_pixel_e2e.py index a3f31dab..e62007a6 100644 --- a/tests/e2e/test_citrix_pixel_e2e.py +++ b/tests/e2e/test_citrix_pixel_e2e.py @@ -34,11 +34,14 @@ * Requires macOS with Screen-Recording AND Accessibility granted to the driving app; SKIPS (never fails, never fabricates) when input cannot be delivered — a dropped synthetic click must never look like success. - * SNAPSHOT-FIRST, REVERT-AFTER; NEVER deletes the VM or ANY snapshot. - * Requires the Parallels VM window to be open + resumable; SKIPS otherwise. - -Env overrides: ``OAFLOW_PARALLELS_VM_UUID``, ``OAFLOW_CITRIX_WINDOW_TITLE`` -(default "Windows 11"). + * Requires one exact preserved base snapshot to be current before mutation. + * Creates one exact owned snapshot, restores the base, verifies it, and + deletes only that owned snapshot. A cleanup failure fails the proof. + * Requires the Parallels VM window to be open + resumable. + +Env overrides: ``OAFLOW_PARALLELS_VM_UUID``, +``OAFLOW_PARALLELS_BASE_SNAPSHOT_ID``, ``OAFLOW_PARALLELS_STORAGE_PATH``, and +``OAFLOW_CITRIX_WINDOW_TITLE`` (default "Windows 11"). """ from __future__ import annotations @@ -63,6 +66,8 @@ _SCRIPT_DIR = Path(__file__).resolve().parents[2] / "scripts" / "desktop" WINDOW_TITLE = os.environ.get("OAFLOW_CITRIX_WINDOW_TITLE", "Windows 11") +BASE_SNAPSHOT_ENV = "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" +HOST_STORAGE_PATH_ENV = "OAFLOW_PARALLELS_STORAGE_PATH" # -- environment guards (skip cleanly, never fail spuriously) ---------------- @@ -141,6 +146,52 @@ def _db_get(vm, pid: int) -> dict: return json.loads((r.stdout or "null").strip() or "null") +def _restore_pixel_vm( + vm, + *, + base_snapshot_id: str, + owned_snapshot_id: str | None, + vm_touched: bool, +) -> None: + """Restore the exact base, delete only the owned snapshot, and suspend.""" + + from openadapt_flow.backends.parallels_vm import ParallelsError + + if owned_snapshot_id is None and not vm_touched: + return + cleanup_errors: list[Exception] = [] + if owned_snapshot_id is not None: + try: + vm.restore_base_and_delete_owned_snapshot( + base_snapshot_id=base_snapshot_id, + owned_snapshot_id=owned_snapshot_id, + ) + except Exception as exc: # noqa: BLE001 + cleanup_errors.append(exc) + elif vm_touched: + try: + vm.revert(base_snapshot_id) + except Exception as exc: # noqa: BLE001 + cleanup_errors.append(exc) + try: + state = vm.status() + if state in {"running", "paused"}: + vm.suspend() + state = vm.status() + if state != "suspended": + raise ParallelsError( + f"preserved base did not finish suspended (state={state!r})" + ) + vm.require_current_snapshot(base_snapshot_id) + except Exception as exc: # noqa: BLE001 + cleanup_errors.append(exc) + if cleanup_errors: + details = "; ".join(repr(error) for error in cleanup_errors) + raise RuntimeError( + f"pixel proof failed to restore its exact base: {details}" + ) from cleanup_errors[0] + + # -- the proof --------------------------------------------------------------- @@ -151,7 +202,10 @@ def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: record_desktop_demo, structural_armed_coverage, ) - from openadapt_flow.backends.parallels_vm import DEFAULT_VM_UUID, ParallelsVM + from openadapt_flow.backends.parallels_vm import ( + DEFAULT_VM_UUID, + ParallelsVM, + ) from openadapt_flow.backends.remote_display import ( RemoteDisplayBackend, RemoteDisplayError, @@ -162,29 +216,46 @@ def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: from openadapt_flow.runtime.effects.onscreen import OnScreenReadbackVerifier uuid = os.environ.get("OAFLOW_PARALLELS_VM_UUID", DEFAULT_VM_UUID) + base_snapshot_id = os.environ.get(BASE_SNAPSHOT_ENV) + if not base_snapshot_id: + raise RuntimeError( + f"{BASE_SNAPSHOT_ENV} is required for snapshot-safe qualification" + ) vm = ParallelsVM(uuid) - - # The VM must already be running (Parallels Standard cannot prlctl-start; a - # suspended VM is resumed by the operator or the GUI play button). Skip - # cleanly rather than fail if it is not reachable. - if vm.status() != "running" or not _guest_ready(vm): - pytest.skip("Parallels VM not running / guest tools unreachable") - - backend = RemoteDisplayBackend( - owner_substr="Parallels Desktop", title_substr=WINDOW_TITLE - ) + snap_id: str | None = None + vm_touched = False + active_error: BaseException | None = None try: - backend.ensure_foreground() - except RemoteDisplayError as e: - pytest.skip(f"remote-display client window not foregroundable: {e}") + storage_path = os.environ.get(HOST_STORAGE_PATH_ENV, os.getcwd()) + vm.require_host_free_space(storage_path=storage_path) + # This read-only check must pass before the first mutation. Reverting + # the same current id then binds the live working state to that exact + # base before resume, focus, snapshot, deployment, or actuation. + vm.require_current_snapshot(base_snapshot_id) + vm_touched = True + vm.revert(base_snapshot_id) + vm.require_current_snapshot(base_snapshot_id) + vm.ensure_running() + if vm.status() != "running" or not _guest_ready(vm): + raise RuntimeError("Parallels VM or guest tools did not become ready") + + backend = RemoteDisplayBackend( + owner_substr="Parallels Desktop", title_substr=WINDOW_TITLE + ) + try: + backend.ensure_foreground() + except RemoteDisplayError as exc: + raise RuntimeError( + "remote-display client window is not foregroundable" + ) from exc + + snap_id = vm.snapshot( + f"oaflow-citrix-{int(time.time())}", description="citrix pixel e2e" + ) + # A snapshot on a running VM briefly disturbs guest tools. + if not _guest_ready(vm): + raise RuntimeError("guest tools did not recover after pixel snapshot") - # SNAPSHOT FIRST — reverted in finally (never deleted). - snap_id = vm.snapshot( - f"oaflow-citrix-{int(time.time())}", description="citrix pixel e2e" - ) - # A snapshot on a running VM briefly disturbs guest tools; wait for it back. - _guest_ready(vm) - try: # ---- deploy + launch the stand-in clinical app (pixel target) ------ _deploy_and_launch(vm, drift="none") backend.ensure_foreground() @@ -323,8 +394,18 @@ def driver(rec) -> None: assert not amb_report.success, ( "run must HALT under render drift rather than click a guessed target" ) + except BaseException as exc: + active_error = exc + raise finally: try: - vm.revert(snap_id) # never delete — revert only - except Exception as e: # noqa: BLE001 - print(f"[citrix] WARNING: revert to {snap_id} failed: {e}") + _restore_pixel_vm( + vm, + base_snapshot_id=base_snapshot_id, + owned_snapshot_id=snap_id, + vm_touched=vm_touched, + ) + except Exception as cleanup_error: + if active_error is None: + raise + active_error.add_note(f"cleanup error: {cleanup_error!r}") diff --git a/tests/e2e/test_parallels_desktop_e2e.py b/tests/e2e/test_parallels_desktop_e2e.py index 1bd15bfc..57fdbaa5 100644 --- a/tests/e2e/test_parallels_desktop_e2e.py +++ b/tests/e2e/test_parallels_desktop_e2e.py @@ -416,7 +416,13 @@ def test_desktop_record_compile_replay_structural(tmp_path, trial: int) -> None: evidence_row["host_free_bytes_before"] = vm.require_host_free_space( storage_path=storage_path ) + # Refuse before the first mutation when this dedicated VM is not already + # bound to the reviewed base. Reverting that same current id then proves + # the live working state starts from the exact preserved snapshot. + vm.require_current_snapshot(base_snapshot_id) vm_touched = True + vm.revert(base_snapshot_id) + vm.require_current_snapshot(base_snapshot_id) vm.ensure_running() # Snapshot before any guest deployment/recording. This id is retained # in memory and is the only snapshot the trial may later delete. diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index a2a8dfda..24df345e 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -322,13 +322,20 @@ def test_validating_refresh_uses_exact_macos_parallels_substrate_and_scope() -> start = claims.index(" refresh-validating-evidence:") refresh = claims[start:] - assert "runs-on: [self-hosted, macos, arm64]" in refresh + assert "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" in claims + assert "runs-on: [self-hosted, macos, arm64, openadapt-parallels]" in refresh assert "OPENADAPT_PARALLELS_VALIDATION_ENABLED" in refresh + assert "group: openadapt-parallels-validation" in refresh + assert "cancel-in-progress: false" in refresh assert "runs-on: ubuntu-latest" not in refresh assert "oa-vm" not in refresh assert "command -v prlctl" in refresh assert "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" in refresh + assert "vm.require_current_snapshot" in refresh assert "--ci-job validating --junit runs/validating-junit.xml" in refresh + assert refresh.index("tests/e2e/test_citrix_pixel_e2e.py") < refresh.index( + "tests/e2e/test_parallels_desktop_e2e.py" + ) for path in ( "tests/e2e/test_parallels_desktop_e2e.py", "tests/e2e/test_citrix_pixel_e2e.py", diff --git a/tests/test_parallels_vm.py b/tests/test_parallels_vm.py index 2cc2e105..dec36ecc 100644 --- a/tests/test_parallels_vm.py +++ b/tests/test_parallels_vm.py @@ -17,6 +17,7 @@ ParallelsVM, SnapshotInfo, ) +from tests.e2e.test_citrix_pixel_e2e import _restore_pixel_vm UUID = "{d4f9c29a-52e1-4793-9334-7e971c3d0ab3}" @@ -171,6 +172,116 @@ def test_restore_refusal_never_deletes_when_base_is_not_current(monkeypatch): assert deleted == [] +def test_require_current_snapshot_accepts_only_exact_current_base(monkeypatch): + vm = ParallelsVM(UUID) + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + other = "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" + monkeypatch.setattr( + vm, + "list_snapshots", + lambda: [SnapshotInfo(base, True), SnapshotInfo(other, False)], + ) + + vm.require_current_snapshot(base) + + with pytest.raises(ParallelsError, match="not current"): + vm.require_current_snapshot(other) + with pytest.raises(ParallelsError, match="missing"): + vm.require_current_snapshot("{b8199a93-ef44-439f-a70b-eaf5bd1d771b}") + with pytest.raises(ValueError, match="exact braced UUID"): + vm.require_current_snapshot("base") + + +class _PixelLifecycleVM: + def __init__(self, *, restore_error: Exception | None = None) -> None: + self.calls: list[tuple[str, ...]] = [] + self.state = "running" + self.restore_error = restore_error + + def restore_base_and_delete_owned_snapshot( + self, *, base_snapshot_id: str, owned_snapshot_id: str + ) -> None: + self.calls.append(("restore-delete", base_snapshot_id, owned_snapshot_id)) + if self.restore_error is not None: + raise self.restore_error + + def revert(self, snapshot_id: str) -> None: + self.calls.append(("revert", snapshot_id)) + + def status(self) -> str: + self.calls.append(("status", self.state)) + return self.state + + def suspend(self) -> None: + self.calls.append(("suspend",)) + self.state = "suspended" + + def require_current_snapshot(self, snapshot_id: str) -> None: + self.calls.append(("require-current", snapshot_id)) + + +def test_pixel_cleanup_restores_base_deletes_only_owned_and_verifies() -> None: + vm = _PixelLifecycleVM() + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + owned = "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" + + _restore_pixel_vm( + vm, + base_snapshot_id=base, + owned_snapshot_id=owned, + vm_touched=True, + ) + + assert vm.calls == [ + ("restore-delete", base, owned), + ("status", "running"), + ("suspend",), + ("status", "suspended"), + ("require-current", base), + ] + + +def test_pixel_cleanup_failure_cannot_produce_a_passing_test() -> None: + vm = _PixelLifecycleVM(restore_error=ParallelsError("restore failed")) + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + owned = "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" + + with pytest.raises(RuntimeError, match="failed to restore its exact base"): + _restore_pixel_vm( + vm, + base_snapshot_id=base, + owned_snapshot_id=owned, + vm_touched=True, + ) + + +def test_pixel_cleanup_without_owned_snapshot_reverts_exact_base() -> None: + vm = _PixelLifecycleVM() + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + + _restore_pixel_vm( + vm, + base_snapshot_id=base, + owned_snapshot_id=None, + vm_touched=True, + ) + + assert ("revert", base) in vm.calls + + +def test_pixel_cleanup_does_not_touch_vm_after_read_only_preflight_refusal() -> None: + vm = _PixelLifecycleVM() + + _restore_pixel_vm( + vm, + base_snapshot_id="{35dba943-a22d-473c-b1b0-44fa6326e626}", + owned_snapshot_id=None, + vm_touched=False, + ) + + assert vm.calls == [] + + def test_guest_ip_skips_apipa(monkeypatch): ipconfig = ( " Autoconfiguration IPv4 Address. . : 169.254.83.107\n" From bf7ccd54a9a8b8b96279042d74af9819891d9f5d Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 15:14:13 -0400 Subject: [PATCH 4/5] fix: recover interrupted Parallels validation --- .github/workflows/validate-claims.yml | 20 ++- docs/desktop_windows_runbook.md | 27 +-- openadapt_flow/backends/parallels_vm.py | 209 ++++++++++++++++++++++-- public-artifacts.json | 4 +- scripts/reconcile_parallels_recovery.py | 25 +++ tests/e2e/test_citrix_pixel_e2e.py | 89 ++++------ tests/e2e/test_parallels_desktop_e2e.py | 77 +++------ tests/test_ci_workflow_contract.py | 14 ++ tests/test_parallels_vm.py | 151 ++++++++++------- 9 files changed, 422 insertions(+), 194 deletions(-) create mode 100644 scripts/reconcile_parallels_recovery.py diff --git a/.github/workflows/validate-claims.yml b/.github/workflows/validate-claims.yml index 687b9391..48be76cc 100644 --- a/.github/workflows/validate-claims.yml +++ b/.github/workflows/validate-claims.yml @@ -104,7 +104,9 @@ jobs: ${{ vars.OPENADAPT_PARALLELS_VALIDATION_ENABLED == 'true' && (github.event_name == 'schedule' || inputs.run_infra == true) }} runs-on: [self-hosted, macos, arm64, openadapt-parallels] - timeout-minutes: 90 + # The live-test step has a shorter timeout. This job reserve leaves time for + # its unconditional recovery step after a test timeout. + timeout-minutes: 110 concurrency: group: openadapt-parallels-validation cancel-in-progress: false @@ -112,6 +114,9 @@ jobs: OAFLOW_PARALLELS_VM_UUID: ${{ vars.OAFLOW_PARALLELS_VM_UUID }} OAFLOW_PARALLELS_BASE_SNAPSHOT_ID: ${{ vars.OAFLOW_PARALLELS_BASE_SNAPSHOT_ID }} OAFLOW_PARALLELS_STORAGE_PATH: ${{ github.workspace }} + # This path is outside the checkout, so checkout cleaning and a runner + # restart cannot erase an unresolved exact-base recovery record. + OAFLOW_PARALLELS_RECOVERY_JOURNAL: ${{ github.workspace }}/../.openadapt-flow/parallels-recovery.json OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT: ${{ github.sha }} OAFLOW_WINDOWS_UIA_MATRIX_ID: claims-${{ github.run_id }}-${{ github.run_attempt }} OAFLOW_WINDOWS_UIA_EVIDENCE: ${{ github.workspace }}/runs/windows-uia-evidence.jsonl @@ -140,6 +145,11 @@ jobs: - name: Install validation dependencies run: python -m pip install -e .[dev] + - name: Reconcile a prior interrupted run before new mutation + run: >- + python scripts/reconcile_parallels_recovery.py + --journal "$OAFLOW_PARALLELS_RECOVERY_JOURNAL" + - name: Prove configured VM and base are current before mutation run: | python -c ' @@ -155,6 +165,7 @@ jobs: ' - name: Run scoped desktop and pixel evidence + timeout-minutes: 90 env: OAFLOW_PARALLELS_E2E: "1" OAFLOW_CITRIX_PIXEL_E2E: "1" @@ -165,6 +176,13 @@ jobs: tests/e2e/test_parallels_desktop_e2e.py \ --junitxml=runs/validating-junit.xml --basetemp=runs/ci + - name: Restore exact base after success, failure, or test timeout + if: ${{ always() }} + timeout-minutes: 15 + run: >- + python scripts/reconcile_parallels_recovery.py + --journal "$OAFLOW_PARALLELS_RECOVERY_JOURNAL" + - name: Require every selected evidence file to pass run: | python scripts/validate_claims.py --report \ diff --git a/docs/desktop_windows_runbook.md b/docs/desktop_windows_runbook.md index 6df43d92..6c84e0be 100644 --- a/docs/desktop_windows_runbook.md +++ b/docs/desktop_windows_runbook.md @@ -130,12 +130,12 @@ fire. The opt-in e2e (`tests/e2e/test_parallels_desktop_e2e.py`) does the whole loop against the deterministic Patient Notes fixture: prove one exact configured -base snapshot is current → revert to that base → ensure the VM is running → -create one exact trial snapshot → launch the typed agent in session 1 → -record→compile→replay via `WindowsBackend` → assert the UIA structural rung and -independent database oracle → restore and verify the configured base → delete -only the exact trial snapshot → suspend and verify the VM. It repeats this -contract three times. +base snapshot is current → persist one private exact-base recovery journal → +revert to that base → ensure the VM is running → launch the typed agent in +session 1 → record→compile→replay via `WindowsBackend` → assert the UIA +structural rung and independent database oracle → restore and verify the +configured base → suspend and verify the VM. It repeats this contract three +times. It creates no ephemeral trial snapshot. ```bash # On the Mac with the Parallels VM: @@ -144,16 +144,19 @@ export OAFLOW_PARALLELS_VM_UUID='{exact-vm-uuid}' export OAFLOW_PARALLELS_BASE_SNAPSHOT_ID='{exact-base-snapshot-uuid}' export OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT='40-character-git-commit' export OAFLOW_PARALLELS_STORAGE_PATH="$PWD" +export OAFLOW_PARALLELS_RECOVERY_JOURNAL='/absolute/private/path/parallels-recovery.json' pytest -q tests/e2e/test_parallels_desktop_e2e.py ``` **Snapshot safety contract:** the test refuses before mutation unless the exact -configured base exists and is current. It retains each created trial snapshot -ID in memory. Cleanup restores and verifies the base before it deletes only -that exact ID. A restore, verification, deletion, evidence-write, or final -suspend failure rejects the trial. The harness never uses a snapshot name, -wildcard, child-recursive delete, or VM delete operation. It is skipped entirely -unless `OAFLOW_PARALLELS_E2E=1`. +configured base exists and is current. It writes the recovery journal before +the first VM mutation. Normal cleanup, the workflow's unconditional cleanup +step, and the next run all use that record to revert the exact VM to the exact +base and verify its suspended state. The record remains after a failed recovery +and disappears only after all checks pass. A restore, verification, +evidence-write, or final suspend failure rejects the trial. The harness never +uses a snapshot name, wildcard, snapshot delete, child-recursive delete, or VM +delete operation. It is skipped entirely unless `OAFLOW_PARALLELS_E2E=1`. --- diff --git a/openadapt_flow/backends/parallels_vm.py b/openadapt_flow/backends/parallels_vm.py index a3233f34..e2d40367 100644 --- a/openadapt_flow/backends/parallels_vm.py +++ b/openadapt_flow/backends/parallels_vm.py @@ -28,14 +28,18 @@ import functools import http.server +import json import os import re import shutil import socketserver +import stat import subprocess +import tempfile import threading import time from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: # pragma: no cover - typing only @@ -115,6 +119,150 @@ class ParallelsError(RuntimeError): """A ``prlctl`` invocation failed.""" +_EXACT_UUID_PATTERN = re.compile( + r"\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}" +) + + +def _require_exact_uuid(value: str, *, field: str) -> None: + if _EXACT_UUID_PATTERN.fullmatch(value) is None: + raise ValueError(f"{field} must be one exact braced UUID") + + +@dataclass(frozen=True) +class ParallelsRecoveryRecord: + """Durable binding for one VM that must return to one preserved base.""" + + vm_uuid: str + base_snapshot_id: str + + +class ParallelsRecoveryJournal: + """Persist exact VM recovery authority before the first VM mutation. + + The validating runner stores this file outside the checked-out repository. + A test process, job timeout, runner restart, or manual cancellation can then + leave the record for the reserved cleanup step or the next run. The record + never grants snapshot deletion authority; recovery only switches to the + configured preserved base and suspends that exact VM. + """ + + VERSION = 1 + + def __init__(self, path: str | os.PathLike[str]) -> None: + candidate = Path(path) + if not candidate.is_absolute(): + raise ValueError("recovery journal path must be absolute") + self.path = Path(os.path.abspath(candidate)) + + @staticmethod + def _fsync_directory(path: Path) -> None: + if os.name == "nt": # Windows cannot open directories this way. + return + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def begin(self, *, vm_uuid: str, base_snapshot_id: str) -> None: + """Create one durable record without replacing an earlier recovery.""" + + _require_exact_uuid(vm_uuid, field="vm_uuid") + _require_exact_uuid(base_snapshot_id, field="base_snapshot_id") + parent = self.path.parent + parent.mkdir(parents=True, exist_ok=True, mode=0o700) + payload = ( + json.dumps( + { + "version": self.VERSION, + "vm_uuid": vm_uuid, + "base_snapshot_id": base_snapshot_id, + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{self.path.name}.", dir=parent + ) + temporary_path = Path(temporary_name) + try: + if hasattr(os, "fchmod"): + os.fchmod(descriptor, 0o600) + else: # pragma: no cover - Windows compatibility for unit imports + os.chmod(temporary_path, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + try: + # A hard link publishes the complete record atomically and + # refuses to overwrite an unresolved record from an older run. + os.link(temporary_path, self.path) + except FileExistsError as exc: + raise ParallelsError( + f"unresolved Parallels recovery journal exists: {self.path}" + ) from exc + self._fsync_directory(parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary_path.unlink(missing_ok=True) + + def load(self) -> ParallelsRecoveryRecord | None: + """Read and validate one closed-schema, owner-private record.""" + + try: + metadata = self.path.lstat() + except FileNotFoundError: + return None + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise ParallelsError("recovery journal must be a regular file") + if hasattr(os, "getuid") and metadata.st_uid != os.getuid(): + raise ParallelsError("recovery journal must be owned by the runner user") + if os.name != "nt" and metadata.st_mode & 0o077: + raise ParallelsError("recovery journal must not grant group/other access") + if metadata.st_size > 4096: + raise ParallelsError("recovery journal is too large") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(self.path, flags) + with os.fdopen(descriptor, encoding="utf-8") as stream: + value = json.load(stream) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ParallelsError("recovery journal is unreadable") from exc + expected_keys = {"version", "vm_uuid", "base_snapshot_id"} + if not isinstance(value, dict) or set(value) != expected_keys: + raise ParallelsError("recovery journal schema is invalid") + if value["version"] != self.VERSION: + raise ParallelsError("recovery journal version is unsupported") + vm_uuid = value["vm_uuid"] + base_snapshot_id = value["base_snapshot_id"] + if not isinstance(vm_uuid, str) or not isinstance(base_snapshot_id, str): + raise ParallelsError("recovery journal identifiers must be strings") + try: + _require_exact_uuid(vm_uuid, field="vm_uuid") + _require_exact_uuid(base_snapshot_id, field="base_snapshot_id") + except ValueError as exc: + raise ParallelsError("recovery journal identifier is invalid") from exc + return ParallelsRecoveryRecord( + vm_uuid=vm_uuid, + base_snapshot_id=base_snapshot_id, + ) + + def clear(self) -> None: + """Remove a record only after exact recovery verification succeeds.""" + + self.path.unlink() + self._fsync_directory(self.path.parent) + + class ParallelsVM: """Thin, fully-programmatic wrapper over ``prlctl`` for one VM. @@ -162,12 +310,12 @@ def _run( def status(self) -> str: """Return the VM power state (running/paused/suspended/stopped).""" - proc = self._run(["list", "--all", "-o", "status,uuid"], check=False) + proc = self._run(["list", "--all", "-o", "status,uuid"]) for line in proc.stdout.splitlines(): if self.uuid in line: return line.split()[0].strip() # Fall back to name match. - proc = self._run(["list", "--all"], check=False) + proc = self._run(["list", "--all"]) for line in proc.stdout.splitlines(): if self.uuid in line: return line.split()[1].strip() @@ -265,7 +413,7 @@ def snapshot(self, name: str, description: str = "") -> str: def list_snapshots(self) -> list[SnapshotInfo]: """Return all snapshots (``*`` marks the current one).""" - proc = self._run(["snapshot-list", self.uuid], check=False) + proc = self._run(["snapshot-list", self.uuid]) out: list[SnapshotInfo] = [] for line in proc.stdout.splitlines(): # Columns are PARENT_SNAPSHOT_ID then SNAPSHOT_ID; the current @@ -286,12 +434,7 @@ def require_current_snapshot(self, snapshot_id: str) -> None: run from preserving and later deleting an unrelated current state. """ - snapshot_pattern = ( - r"\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}" - ) - if re.fullmatch(snapshot_pattern, snapshot_id) is None: - raise ValueError("snapshot_id must be one exact braced UUID") + _require_exact_uuid(snapshot_id, field="snapshot_id") snapshots = self.list_snapshots() match = next( (item for item in snapshots if item.snapshot_id == snapshot_id), None @@ -313,12 +456,7 @@ def delete_owned_snapshot(self, snapshot_id: str) -> None: and must first switch to their preserved base with :meth:`restore_base_and_delete_owned_snapshot`. """ - snapshot_pattern = ( - r"\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" - r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}" - ) - if re.fullmatch(snapshot_pattern, snapshot_id) is None: - raise ValueError("snapshot_id must be one exact braced UUID") + _require_exact_uuid(snapshot_id, field="snapshot_id") self._run(["snapshot-delete", self.uuid, "-i", snapshot_id]) def restore_base_and_delete_owned_snapshot( @@ -707,3 +845,44 @@ def _agent_alive(self, url: str, *, fingerprint: Optional[str] = None) -> bool: return r.status_code == 200 and r.json().get("status") == "ok" except Exception: # noqa: BLE001 return False + + +def recover_parallels_vm( + journal_path: str | os.PathLike[str], + *, + prlctl: str = DEFAULT_PRLCTL, +) -> bool: + """Restore and suspend the exact VM in a durable recovery record. + + Returns ``False`` when no record exists. The record remains present after + any command or verification failure, so a reserved cleanup step or a later + run can retry. It is removed only after the exact base is current and the + VM has reported a verified suspended state through successful ``prlctl`` + commands. + """ + + journal = ParallelsRecoveryJournal(journal_path) + record = journal.load() + if record is None: + return False + vm = ParallelsVM(record.vm_uuid, prlctl=prlctl) + snapshots = vm.list_snapshots() + if not any(item.snapshot_id == record.base_snapshot_id for item in snapshots): + raise ParallelsError("preserved recovery base snapshot is missing") + # Switch even when Parallels marks this snapshot current. The guest can have + # modified working state after that marker, and recovery must discard it. + vm.revert(record.base_snapshot_id) + vm.require_current_snapshot(record.base_snapshot_id) + state = vm.status() + if state in {"running", "paused"}: + vm.suspend() + elif state == "stopped": + vm.start() + vm.suspend() + elif state != "suspended": + raise ParallelsError(f"cannot recover VM from state {state!r}") + if vm.status() != "suspended": + raise ParallelsError("recovered VM did not report suspended state") + vm.require_current_snapshot(record.base_snapshot_id) + journal.clear() + return True diff --git a/public-artifacts.json b/public-artifacts.json index 6b2aaf35..07120c45 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -93,7 +93,7 @@ }, { "path": ".github/workflows/ci.yml", - "sha256": "2910e528f63e328b281d233ccacf3a4483601cf147d7e9336edbd07f33cb8d4a" + "sha256": "c562a53c3222a5db05a123d78875379162c056bc2573f7d01f03dcf4df9e42f2" }, { "path": ".github/workflows/citrix-workspace-standin.yml", @@ -125,7 +125,7 @@ }, { "path": ".github/workflows/validate-claims.yml", - "sha256": "368accbf68954e8e481fc6d7d36229b9cb70c267ec302e01657aa6537900a9e6" + "sha256": "5cfeab8e231b79bbeacb106f069a331ee46f007631b9d28f9aca4239553afdbc" }, { "path": ".pre-commit-config.yaml", diff --git a/scripts/reconcile_parallels_recovery.py b/scripts/reconcile_parallels_recovery.py new file mode 100644 index 00000000..21ad8c5f --- /dev/null +++ b/scripts/reconcile_parallels_recovery.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Recover one exact Parallels validation VM from its durable journal.""" + +from __future__ import annotations + +import argparse + +from openadapt_flow.backends.parallels_vm import recover_parallels_vm + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--journal", + required=True, + help="Absolute host path to the private recovery journal", + ) + args = parser.parse_args() + recovered = recover_parallels_vm(args.journal) + print("recovered exact Parallels base" if recovered else "no recovery pending") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/test_citrix_pixel_e2e.py b/tests/e2e/test_citrix_pixel_e2e.py index e62007a6..e364d6c2 100644 --- a/tests/e2e/test_citrix_pixel_e2e.py +++ b/tests/e2e/test_citrix_pixel_e2e.py @@ -35,13 +35,15 @@ driving app; SKIPS (never fails, never fabricates) when input cannot be delivered — a dropped synthetic click must never look like success. * Requires one exact preserved base snapshot to be current before mutation. - * Creates one exact owned snapshot, restores the base, verifies it, and - deletes only that owned snapshot. A cleanup failure fails the proof. + * Writes a durable recovery journal before mutation, restores the exact + base, and verifies the final suspended state. A cleanup failure fails the + proof and leaves the journal for a reserved cleanup step or the next run. * Requires the Parallels VM window to be open + resumable. Env overrides: ``OAFLOW_PARALLELS_VM_UUID``, ``OAFLOW_PARALLELS_BASE_SNAPSHOT_ID``, ``OAFLOW_PARALLELS_STORAGE_PATH``, and -``OAFLOW_CITRIX_WINDOW_TITLE`` (default "Windows 11"). +``OAFLOW_PARALLELS_RECOVERY_JOURNAL``, and ``OAFLOW_CITRIX_WINDOW_TITLE`` +(default "Windows 11"). """ from __future__ import annotations @@ -68,6 +70,7 @@ WINDOW_TITLE = os.environ.get("OAFLOW_CITRIX_WINDOW_TITLE", "Windows 11") BASE_SNAPSHOT_ENV = "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" HOST_STORAGE_PATH_ENV = "OAFLOW_PARALLELS_STORAGE_PATH" +RECOVERY_JOURNAL_ENV = "OAFLOW_PARALLELS_RECOVERY_JOURNAL" # -- environment guards (skip cleanly, never fail spuriously) ---------------- @@ -147,64 +150,32 @@ def _db_get(vm, pid: int) -> dict: def _restore_pixel_vm( - vm, + journal_path: str, *, - base_snapshot_id: str, - owned_snapshot_id: str | None, - vm_touched: bool, + journal_started: bool, ) -> None: - """Restore the exact base, delete only the owned snapshot, and suspend.""" + """Run the same durable recovery used by workflow and next-run cleanup.""" - from openadapt_flow.backends.parallels_vm import ParallelsError + from openadapt_flow.backends.parallels_vm import recover_parallels_vm - if owned_snapshot_id is None and not vm_touched: - return - cleanup_errors: list[Exception] = [] - if owned_snapshot_id is not None: - try: - vm.restore_base_and_delete_owned_snapshot( - base_snapshot_id=base_snapshot_id, - owned_snapshot_id=owned_snapshot_id, - ) - except Exception as exc: # noqa: BLE001 - cleanup_errors.append(exc) - elif vm_touched: - try: - vm.revert(base_snapshot_id) - except Exception as exc: # noqa: BLE001 - cleanup_errors.append(exc) - try: - state = vm.status() - if state in {"running", "paused"}: - vm.suspend() - state = vm.status() - if state != "suspended": - raise ParallelsError( - f"preserved base did not finish suspended (state={state!r})" - ) - vm.require_current_snapshot(base_snapshot_id) - except Exception as exc: # noqa: BLE001 - cleanup_errors.append(exc) - if cleanup_errors: - details = "; ".join(repr(error) for error in cleanup_errors) - raise RuntimeError( - f"pixel proof failed to restore its exact base: {details}" - ) from cleanup_errors[0] + recovered = recover_parallels_vm(journal_path) + if journal_started and not recovered: + raise RuntimeError("durable Parallels recovery record disappeared") # -- the proof --------------------------------------------------------------- def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: - _require_macos_input() - from openadapt_flow.adapters.desktop_recorder import ( record_desktop_demo, structural_armed_coverage, ) from openadapt_flow.backends.parallels_vm import ( DEFAULT_VM_UUID, + ParallelsRecoveryJournal, ParallelsVM, + recover_parallels_vm, ) from openadapt_flow.backends.remote_display import ( RemoteDisplayBackend, @@ -217,13 +188,23 @@ def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: uuid = os.environ.get("OAFLOW_PARALLELS_VM_UUID", DEFAULT_VM_UUID) base_snapshot_id = os.environ.get(BASE_SNAPSHOT_ENV) + journal_path = os.environ.get(RECOVERY_JOURNAL_ENV) + if not journal_path: + raise RuntimeError( + f"{RECOVERY_JOURNAL_ENV} is required for crash-safe qualification" + ) + # A prior timeout, cancellation, or runner restart leaves this record. The + # next run must reconcile it before it makes a new mutation. + recover_parallels_vm(journal_path) + # Resolve stale recovery before this environment guard can skip the test. + _require_macos_input() if not base_snapshot_id: raise RuntimeError( f"{BASE_SNAPSHOT_ENV} is required for snapshot-safe qualification" ) vm = ParallelsVM(uuid) - snap_id: str | None = None - vm_touched = False + journal = ParallelsRecoveryJournal(journal_path) + journal_started = False active_error: BaseException | None = None try: storage_path = os.environ.get(HOST_STORAGE_PATH_ENV, os.getcwd()) @@ -232,7 +213,8 @@ def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: # the same current id then binds the live working state to that exact # base before resume, focus, snapshot, deployment, or actuation. vm.require_current_snapshot(base_snapshot_id) - vm_touched = True + journal.begin(vm_uuid=uuid, base_snapshot_id=base_snapshot_id) + journal_started = True vm.revert(base_snapshot_id) vm.require_current_snapshot(base_snapshot_id) vm.ensure_running() @@ -249,13 +231,6 @@ def test_citrix_pixel_only_record_replay_identity_verify_halt(tmp_path) -> None: "remote-display client window is not foregroundable" ) from exc - snap_id = vm.snapshot( - f"oaflow-citrix-{int(time.time())}", description="citrix pixel e2e" - ) - # A snapshot on a running VM briefly disturbs guest tools. - if not _guest_ready(vm): - raise RuntimeError("guest tools did not recover after pixel snapshot") - # ---- deploy + launch the stand-in clinical app (pixel target) ------ _deploy_and_launch(vm, drift="none") backend.ensure_foreground() @@ -400,10 +375,8 @@ def driver(rec) -> None: finally: try: _restore_pixel_vm( - vm, - base_snapshot_id=base_snapshot_id, - owned_snapshot_id=snap_id, - vm_touched=vm_touched, + journal_path, + journal_started=journal_started, ) except Exception as cleanup_error: if active_error is None: diff --git a/tests/e2e/test_parallels_desktop_e2e.py b/tests/e2e/test_parallels_desktop_e2e.py index 57fdbaa5..ce76381b 100644 --- a/tests/e2e/test_parallels_desktop_e2e.py +++ b/tests/e2e/test_parallels_desktop_e2e.py @@ -38,10 +38,11 @@ * OPT-IN ONLY. Skipped unless ``OAFLOW_PARALLELS_E2E=1`` -- it is collected but never runs on CI or any machine without the env var, so ``pytest --ignore=tests/e2e`` (macOS CI) and a plain run both stay green. - * SNAPSHOT-FIRST, RESTORE-BASE, DELETE-OWNED. A fresh per-trial snapshot is - taken before anything touches the guest. ``finally`` switches to the - explicitly named preserved base, proves it is current, and deletes only - the exact snapshot id this trial created. No snapshot accumulates. + * RECOVERY-JOURNAL-FIRST. Before any VM mutation, the harness writes one + private durable record that binds the exact VM to its preserved base. + ``finally``, a reserved workflow step, and the next run all use the same + recovery function. It reverts the exact base and verifies suspension. + The validating tests create no ephemeral snapshot. * A host-free-space preflight runs before VM mutation. The harness never deletes the VM or a pre-existing snapshot, and never runs unless the maintainer explicitly opts in. CI never does. @@ -92,6 +93,7 @@ BASE_SNAPSHOT_ENV = "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" HOST_STORAGE_PATH_ENV = "OAFLOW_PARALLELS_STORAGE_PATH" CANDIDATE_COMMIT_ENV = "OAFLOW_WINDOWS_UIA_CANDIDATE_COMMIT" +RECOVERY_JOURNAL_ENV = "OAFLOW_PARALLELS_RECOVERY_JOURNAL" def _failure_category(error: Exception) -> str: @@ -358,8 +360,9 @@ def test_desktop_record_compile_replay_structural(tmp_path, trial: int) -> None: ) from openadapt_flow.backends.parallels_vm import ( DEFAULT_VM_UUID, - ParallelsError, + ParallelsRecoveryJournal, ParallelsVM, + recover_parallels_vm, ) from openadapt_flow.compiler import compile_recording from openadapt_flow.ir import Workflow @@ -368,7 +371,16 @@ def test_desktop_record_compile_replay_structural(tmp_path, trial: int) -> None: uuid = os.environ.get("OAFLOW_PARALLELS_VM_UUID", DEFAULT_VM_UUID) base_snapshot_id = os.environ.get(BASE_SNAPSHOT_ENV) candidate_commit = os.environ.get(CANDIDATE_COMMIT_ENV) + journal_path = os.environ.get(RECOVERY_JOURNAL_ENV) + if not journal_path: + raise RuntimeError( + f"{RECOVERY_JOURNAL_ENV} is required for crash-safe qualification" + ) + # Recover a record left by a timeout, cancellation, or runner restart before + # this trial makes a new mutation. + recover_parallels_vm(journal_path) vm = ParallelsVM(uuid) + journal = ParallelsRecoveryJournal(journal_path) token = secrets.token_hex(16) started_at = datetime.now(timezone.utc).isoformat() evidence_row: dict[str, Any] = { @@ -392,13 +404,12 @@ def test_desktop_record_compile_replay_structural(tmp_path, trial: int) -> None: "ambiguity_refusal_passed": False, "clean_restore_passed": False, "base_snapshot_current_after": False, - "trial_snapshot_deleted": False, + "ephemeral_snapshot_created": False, "failure_category": None, "failure_categories": [], } - snap_id: str | None = None - vm_touched = False + journal_started = False active_error: Exception | None = None try: if not base_snapshot_id: @@ -420,17 +431,11 @@ def test_desktop_record_compile_replay_structural(tmp_path, trial: int) -> None: # bound to the reviewed base. Reverting that same current id then proves # the live working state starts from the exact preserved snapshot. vm.require_current_snapshot(base_snapshot_id) - vm_touched = True + journal.begin(vm_uuid=uuid, base_snapshot_id=base_snapshot_id) + journal_started = True vm.revert(base_snapshot_id) vm.require_current_snapshot(base_snapshot_id) vm.ensure_running() - # Snapshot before any guest deployment/recording. This id is retained - # in memory and is the only snapshot the trial may later delete. - snap_id = vm.snapshot( - f"oaflow-e2e-t{trial}-{int(time.time())}", - description=f"openadapt-flow typed UIA qualification trial {trial}/3", - ) - evidence_row["trial_snapshot_id"] = snap_id endpoint = vm.launch_agent(token=token) # launch_agent auto-provisions the per-run TLS cert into the guest and # returns the pin fingerprint, so the client is encrypted + pinned end to @@ -600,42 +605,14 @@ def driver(rec) -> None: ) raise finally: - # Restore the explicit preserved base. Only after proving it current may - # the harness delete the exact per-trial snapshot id it just created. - # A failure makes the row rejected and stops cleanup before any broader - # mutation; no names, wildcards, or child-recursive deletion are used. + # Use the same exact-base recovery as the reserved workflow step and the + # next run. A failure retains the journal and rejects this evidence row. cleanup_errors: list[Exception] = [] - if base_snapshot_id and snap_id is not None: - try: - vm.restore_base_and_delete_owned_snapshot( - base_snapshot_id=base_snapshot_id, - owned_snapshot_id=snap_id, - ) - evidence_row["trial_snapshot_deleted"] = True - except Exception as exc: - cleanup_errors.append(exc) - elif base_snapshot_id and vm_touched: - try: - vm.revert(base_snapshot_id) - except Exception as exc: - cleanup_errors.append(exc) try: - state = vm.status() - if state in {"running", "paused"}: - vm.suspend() - state = vm.status() - if state != "suspended": - raise ParallelsError( - f"preserved base did not finish suspended (state={state!r})" - ) - if base_snapshot_id: - snapshots = vm.list_snapshots() - evidence_row["base_snapshot_current_after"] = any( - item.snapshot_id == base_snapshot_id and item.current - for item in snapshots - ) - if not evidence_row["base_snapshot_current_after"]: - raise ParallelsError("preserved base is not current after trial") + recovered = recover_parallels_vm(journal_path) + if journal_started and not recovered: + raise RuntimeError("durable Parallels recovery record disappeared") + evidence_row["base_snapshot_current_after"] = recovered except Exception as exc: cleanup_errors.append(exc) evidence_row["clean_restore_passed"] = not cleanup_errors diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index 24df345e..de54889b 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -11,6 +11,8 @@ CI = ROOT / ".github/workflows/ci.yml" QUICKSTART = ROOT / ".github/workflows/quickstart-lifecycle.yml" VALIDATE_CLAIMS = ROOT / ".github/workflows/validate-claims.yml" +PIXEL_E2E = ROOT / "tests/e2e/test_citrix_pixel_e2e.py" +DESKTOP_E2E = ROOT / "tests/e2e/test_parallels_desktop_e2e.py" def test_playwright_version_probes_are_valid_python() -> None: @@ -331,7 +333,13 @@ def test_validating_refresh_uses_exact_macos_parallels_substrate_and_scope() -> assert "oa-vm" not in refresh assert "command -v prlctl" in refresh assert "OAFLOW_PARALLELS_BASE_SNAPSHOT_ID" in refresh + assert "OAFLOW_PARALLELS_RECOVERY_JOURNAL" in refresh + assert "${{ github.workspace }}/../.openadapt-flow/" in refresh assert "vm.require_current_snapshot" in refresh + assert refresh.count("scripts/reconcile_parallels_recovery.py") == 2 + assert "if: ${{ always() }}" in refresh + assert "timeout-minutes: 110" in refresh + assert "timeout-minutes: 90" in refresh assert "--ci-job validating --junit runs/validating-junit.xml" in refresh assert refresh.index("tests/e2e/test_citrix_pixel_e2e.py") < refresh.index( "tests/e2e/test_parallels_desktop_e2e.py" @@ -342,6 +350,12 @@ def test_validating_refresh_uses_exact_macos_parallels_substrate_and_scope() -> ): assert f"--evidence-path {path}" in refresh + for test_path in (PIXEL_E2E, DESKTOP_E2E): + source = test_path.read_text(encoding="utf-8") + assert "journal.begin(" in source + assert source.index("journal.begin(") < source.index("vm.revert(") + assert "vm.snapshot(" not in source + def test_clean_machine_lifecycle_declares_utf8_on_every_os() -> None: workflow = QUICKSTART.read_text(encoding="utf-8") diff --git a/tests/test_parallels_vm.py b/tests/test_parallels_vm.py index dec36ecc..30d5303f 100644 --- a/tests/test_parallels_vm.py +++ b/tests/test_parallels_vm.py @@ -6,6 +6,7 @@ from __future__ import annotations +import os import subprocess import types @@ -14,8 +15,10 @@ from openadapt_flow.backends import parallels_vm as pv from openadapt_flow.backends.parallels_vm import ( ParallelsError, + ParallelsRecoveryJournal, ParallelsVM, SnapshotInfo, + recover_parallels_vm, ) from tests.e2e.test_citrix_pixel_e2e import _restore_pixel_vm @@ -56,6 +59,14 @@ def test_status_parses_running(monkeypatch): assert ParallelsVM(UUID).status() == "running" +def test_status_rejects_failed_command_even_with_plausible_stdout(monkeypatch): + out = f"STATUS UUID\nrunning {UUID}\n" + _mock_run(monkeypatch, {"list": _completed(out, returncode=1, stderr="failed")}) + + with pytest.raises(ParallelsError, match="rc=1"): + ParallelsVM(UUID).status() + + def test_snapshot_parses_id(monkeypatch): out = ( "Creating the snapshot...\n" @@ -87,6 +98,18 @@ def test_list_snapshots_marks_current(monkeypatch): assert snaps[1].snapshot_id == "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" +def test_snapshot_proof_rejects_failed_command_with_plausible_stdout(monkeypatch): + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + out = f"PARENT_SNAPSHOT_ID SNAPSHOT_ID\n *{base}\n" + _mock_run( + monkeypatch, + {"snapshot-list": _completed(out, returncode=1, stderr="failed")}, + ) + + with pytest.raises(ParallelsError, match="rc=1"): + ParallelsVM(UUID).require_current_snapshot(base) + + def test_host_free_space_preflight_refuses_before_vm_command(monkeypatch): calls = _mock_run(monkeypatch, {}) monkeypatch.setattr( @@ -192,21 +215,50 @@ def test_require_current_snapshot_accepts_only_exact_current_base(monkeypatch): vm.require_current_snapshot("base") -class _PixelLifecycleVM: - def __init__(self, *, restore_error: Exception | None = None) -> None: - self.calls: list[tuple[str, ...]] = [] +def test_recovery_journal_is_private_atomic_and_refuses_overwrite(tmp_path) -> None: + base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + path = tmp_path / "recovery.json" + journal = ParallelsRecoveryJournal(path) + + journal.begin(vm_uuid=UUID, base_snapshot_id=base) + + if os.name != "nt": + assert path.stat().st_mode & 0o777 == 0o600 + assert journal.load() == pv.ParallelsRecoveryRecord(UUID, base) + with pytest.raises(ParallelsError, match="unresolved"): + journal.begin(vm_uuid=UUID, base_snapshot_id=base) + journal.clear() + assert journal.load() is None + + +def test_recovery_journal_rejects_relative_path_and_invalid_identifier( + tmp_path, +) -> None: + with pytest.raises(ValueError, match="absolute"): + ParallelsRecoveryJournal("recovery.json") + journal = ParallelsRecoveryJournal(tmp_path / "recovery.json") + with pytest.raises(ValueError, match="vm_uuid"): + journal.begin(vm_uuid="Windows 11", base_snapshot_id=UUID) + + +class _RecoveryVM: + def __init__(self, base: str, *, revert_error: Exception | None = None) -> None: + self.base = base + self.revert_error = revert_error self.state = "running" - self.restore_error = restore_error + self.calls: list[tuple[str, ...]] = [] - def restore_base_and_delete_owned_snapshot( - self, *, base_snapshot_id: str, owned_snapshot_id: str - ) -> None: - self.calls.append(("restore-delete", base_snapshot_id, owned_snapshot_id)) - if self.restore_error is not None: - raise self.restore_error + def list_snapshots(self) -> list[SnapshotInfo]: + self.calls.append(("list-snapshots",)) + return [SnapshotInfo(self.base, False)] def revert(self, snapshot_id: str) -> None: self.calls.append(("revert", snapshot_id)) + if self.revert_error is not None: + raise self.revert_error + + def require_current_snapshot(self, snapshot_id: str) -> None: + self.calls.append(("require-current", snapshot_id)) def status(self) -> str: self.calls.append(("status", self.state)) @@ -216,70 +268,57 @@ def suspend(self) -> None: self.calls.append(("suspend",)) self.state = "suspended" - def require_current_snapshot(self, snapshot_id: str) -> None: - self.calls.append(("require-current", snapshot_id)) + def start(self) -> None: + self.calls.append(("start",)) + self.state = "running" -def test_pixel_cleanup_restores_base_deletes_only_owned_and_verifies() -> None: - vm = _PixelLifecycleVM() +def test_durable_recovery_reverts_exact_base_suspends_and_clears( + monkeypatch, tmp_path +) -> None: base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" - owned = "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" + path = tmp_path / "recovery.json" + journal = ParallelsRecoveryJournal(path) + journal.begin(vm_uuid=UUID, base_snapshot_id=base) + fake = _RecoveryVM(base) + monkeypatch.setattr(pv, "ParallelsVM", lambda *_a, **_k: fake) - _restore_pixel_vm( - vm, - base_snapshot_id=base, - owned_snapshot_id=owned, - vm_touched=True, - ) + assert recover_parallels_vm(path) is True - assert vm.calls == [ - ("restore-delete", base, owned), + assert fake.calls == [ + ("list-snapshots",), + ("revert", base), + ("require-current", base), ("status", "running"), ("suspend",), ("status", "suspended"), ("require-current", base), ] + assert not path.exists() -def test_pixel_cleanup_failure_cannot_produce_a_passing_test() -> None: - vm = _PixelLifecycleVM(restore_error=ParallelsError("restore failed")) - base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" - owned = "{516f223f-7e3a-48f4-90d0-f69f9aaa7644}" - - with pytest.raises(RuntimeError, match="failed to restore its exact base"): - _restore_pixel_vm( - vm, - base_snapshot_id=base, - owned_snapshot_id=owned, - vm_touched=True, - ) - - -def test_pixel_cleanup_without_owned_snapshot_reverts_exact_base() -> None: - vm = _PixelLifecycleVM() +def test_failed_recovery_retains_journal_for_reserved_or_next_run( + monkeypatch, tmp_path +) -> None: base = "{35dba943-a22d-473c-b1b0-44fa6326e626}" + path = tmp_path / "recovery.json" + journal = ParallelsRecoveryJournal(path) + journal.begin(vm_uuid=UUID, base_snapshot_id=base) + fake = _RecoveryVM(base, revert_error=ParallelsError("restore failed")) + monkeypatch.setattr(pv, "ParallelsVM", lambda *_a, **_k: fake) - _restore_pixel_vm( - vm, - base_snapshot_id=base, - owned_snapshot_id=None, - vm_touched=True, - ) - - assert ("revert", base) in vm.calls + with pytest.raises(ParallelsError, match="restore failed"): + recover_parallels_vm(path) + assert journal.load() == pv.ParallelsRecoveryRecord(UUID, base) -def test_pixel_cleanup_does_not_touch_vm_after_read_only_preflight_refusal() -> None: - vm = _PixelLifecycleVM() - _restore_pixel_vm( - vm, - base_snapshot_id="{35dba943-a22d-473c-b1b0-44fa6326e626}", - owned_snapshot_id=None, - vm_touched=False, - ) +def test_pixel_cleanup_requires_the_expected_durable_record(monkeypatch) -> None: + monkeypatch.setattr(pv, "recover_parallels_vm", lambda _path: False) - assert vm.calls == [] + with pytest.raises(RuntimeError, match="record disappeared"): + _restore_pixel_vm("/private/tmp/recovery.json", journal_started=True) + _restore_pixel_vm("/private/tmp/recovery.json", journal_started=False) def test_guest_ip_skips_apipa(monkeypatch): From 357df9324c33263d35a6b13db1b955597e0600da Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Tue, 18 Aug 2026 16:03:06 -0400 Subject: [PATCH 5/5] test: bound Windows lock release check --- tests/test_install_playwright_browser.py | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_install_playwright_browser.py b/tests/test_install_playwright_browser.py index d134f15b..9326f890 100644 --- a/tests/test_install_playwright_browser.py +++ b/tests/test_install_playwright_browser.py @@ -6,6 +6,7 @@ import signal import subprocess import sys +import time from collections.abc import Sequence from pathlib import Path from typing import Any, cast @@ -734,6 +735,26 @@ def _windows_lock_is_available(lock_path: Path) -> bool: return True +def _wait_for_windows_lock_release(lock_path: Path) -> bool: + """Allow bounded Windows teardown while still rejecting a live child.""" + + deadline = ( + time.monotonic() + install_playwright_browser.PROCESS_EXIT_TIMEOUT_SECONDS + ) + while True: + if _windows_lock_is_available(lock_path): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep( + min( + install_playwright_browser.PROCESS_GROUP_POLL_INTERVAL_SECONDS, + remaining, + ) + ) + + @pytest.mark.skipif(sys.platform != "win32", reason="real Windows Job Object proof") @pytest.mark.parametrize("leader_return_code", [0, 7]) def test_windows_job_kills_child_after_normal_leader_exit( @@ -786,7 +807,7 @@ def test_windows_job_kills_child_after_normal_leader_exit( child_pid = int(pid_path.read_text(encoding="utf-8")) try: assert result == leader_return_code - assert _windows_lock_is_available(lock_path) + assert _wait_for_windows_lock_release(lock_path) finally: subprocess.run( ["taskkill", "/PID", str(child_pid), "/T", "/F"], @@ -845,7 +866,7 @@ def test_windows_job_kills_child_on_timeout(tmp_path: Path) -> None: child_pid = int(pid_path.read_text(encoding="utf-8")) try: assert result == 124 - assert _windows_lock_is_available(lock_path) + assert _wait_for_windows_lock_release(lock_path) finally: subprocess.run( ["taskkill", "/PID", str(child_pid), "/T", "/F"],