From 8149cd8ee1545b94e6ca076370fa50385792bd10 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 16:39:19 +0200 Subject: [PATCH 01/23] perf(ci): parallelize native release qualification and promote exact candidates Preserve native coverage, test the Windows candidate, bind promotion to immutable evidence, harden merge gates, and record safe timing snapshots. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/pytest-timing/action.yml | 43 + .github/instructions/cicd.instructions.md | 110 +- .github/scripts/ci/merge_gate_wait.sh | 58 +- .github/workflows/build-release.yml | 958 ++++------------- .github/workflows/ci-integration.yml | 32 +- .github/workflows/ci-runtime.yml | 2 + .github/workflows/ci.yml | 32 +- .github/workflows/merge-gate.yml | 21 +- .github/workflows/release-platform.yml | 360 +++++++ .../docs/contributing/development-guide.md | 17 +- .../docs/contributing/integration-testing.md | 310 +++--- scripts/build-binary.sh | 7 - scripts/package_release.py | 185 ++++ scripts/release-candidate.cjs | 775 ++++++++++++++ scripts/release-candidate.test.cjs | 990 ++++++++++++++++++ scripts/release-platforms.json | 37 + scripts/test-integration.sh | 87 +- scripts/windows/build-binary.ps1 | 12 +- scripts/windows/test-install-script.ps1 | 338 +++--- tests/integration/conftest.py | 14 + tests/integration/test_mcp_registry_e2e.py | 70 +- .../test_windows_installer_launchers.py | 93 +- tests/quality/test_ci_topology.py | 52 +- tests/unit/scripts/test_package_release.py | 195 ++++ tests/unit/test_ado_live_workflow_contract.py | 17 +- tests/unit/test_mcp_registry_collection.py | 58 + tests/unit/test_merge_gate_wait.py | 309 ++++++ tests/unit/test_platform_contract_workflow.py | 651 ++++++------ tests/unit/test_pytest_timing_workflows.py | 156 +++ tests/unit/test_runtime_prerequisite_gate.py | 55 + tests/unit/test_unix_installer_checksums.py | 3 +- .../unit/test_windows_installer_candidate.py | 265 +++++ tests/unit/test_windows_signing_contract.py | 14 +- tests/unit/test_windows_unit_diagnostics.py | 28 +- tests/utils/windows_installer_candidate.py | 148 +++ 35 files changed, 4931 insertions(+), 1571 deletions(-) create mode 100644 .github/actions/pytest-timing/action.yml create mode 100644 .github/workflows/release-platform.yml create mode 100644 scripts/package_release.py create mode 100644 scripts/release-candidate.cjs create mode 100644 scripts/release-candidate.test.cjs create mode 100644 scripts/release-platforms.json create mode 100644 tests/unit/scripts/test_package_release.py create mode 100644 tests/unit/test_mcp_registry_collection.py create mode 100644 tests/unit/test_merge_gate_wait.py create mode 100644 tests/unit/test_pytest_timing_workflows.py create mode 100644 tests/unit/test_runtime_prerequisite_gate.py create mode 100644 tests/unit/test_windows_installer_candidate.py create mode 100644 tests/utils/windows_installer_candidate.py diff --git a/.github/actions/pytest-timing/action.yml b/.github/actions/pytest-timing/action.yml new file mode 100644 index 0000000000..f5eb71e6c7 --- /dev/null +++ b/.github/actions/pytest-timing/action.yml @@ -0,0 +1,43 @@ +name: Pytest timing hints +description: Restore timing hints only; cached data never selects or skips tests. +inputs: + suite: + description: Test corpus whose timings are being recorded. + required: true + partition: + description: Distinguish simultaneous writers while sharing restore hints. + default: all + snapshot: + description: Publish one immutable per-run snapshot for downstream shards. + default: '' +runs: + using: composite + steps: + - if: inputs.snapshot == '' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .test_durations + key: pytest-times-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.suite }}-${{ github.sha }}-${{ inputs.partition }} + restore-keys: | + pytest-times-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.suite }}- + - if: inputs.snapshot != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .test_durations + key: pytest-times-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.suite }}-${{ github.sha }}-all + restore-keys: | + pytest-times-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.suite }}- + - if: inputs.snapshot != '' + shell: bash + run: | + if [ ! -f .test_durations ]; then + printf '{}\n' > .test_durations + fi + - if: inputs.snapshot != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.snapshot }} + path: .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 7 diff --git a/.github/instructions/cicd.instructions.md b/.github/instructions/cicd.instructions.md index f18d2857fe..2cafd94550 100644 --- a/.github/instructions/cicd.instructions.md +++ b/.github/instructions/cicd.instructions.md @@ -6,13 +6,17 @@ description: "CI/CD Pipeline configuration for PyInstaller binary packaging and # CI/CD Pipeline Instructions ## Workflow Architecture (Tiered + Merge Queue) -Five workflows split by trigger and tier. PRs get fast feedback; the heavy +Core workflows split by trigger and tier. PRs get fast feedback; the heavy integration suite runs only at merge time via GitHub Merge Queue (microsoft/apm#770). -1. **`ci.yml`** - Tier 1, runs on `pull_request` AND `merge_group` - - **Linux-only** (ubuntu-24.04). Combined `build-and-test` job: unit tests + binary build in a single runner. No secrets needed. - - Uploads Linux x86_64 binary artifact for downstream integration testing. +1. **`ci.yml`** - Tier 1, runs on `pull_request`, `merge_group`, and `workflow_call` + - Linux unit shards and combined coverage, lint, architecture ratchets, + self-check and lifecycle smoke; a bounded Windows compatibility lane. + No production secrets needed. + - PR-only binary smoke provides early packaging feedback. Merge queue + builds its own binary; full release candidates call the source checks + and build each native archive separately. - Runs in both PR context (fast feedback for contributors) and merge_group context (against the tentative merge commit before queue auto-merges). 2. **`ci-integration.yml`** - Tier 2, `merge_group` trigger only @@ -26,50 +30,74 @@ integration suite runs only at merge time via GitHub Merge Queue This file holds production secrets (`GH_CLI_PAT`, `ADO_APM_PAT`). Required-check satisfaction at PR time is handled by `merge-gate.yml`, which aggregates all required signals into a single `gate` check. -3. **`merge-gate.yml`** - single-authority PR-time aggregator - - Triggers on `pull_request` only (single trigger - dual-trigger with - `pull_request_target` produces SUCCESS+CANCELLED check-run twins via - `cancel-in-progress` and poisons branch protection's rollup). +3. **`merge-gate.yml`** - single-authority PR and merge-queue aggregator + - Uses event-specific required checks. Never add a parallel + `pull_request_target` gate, which can create duplicate check-run names. - One job named `gate`. Polls the Checks API for all entries in the workflow's `EXPECTED_CHECKS` env var; aggregates pass/fail into a single check-run. - Branch protection requires ONLY this one check (`gate`). Adding, renaming, or removing an underlying check is a `merge-gate.yml` edit, never a ruleset edit. Tide / bors single-authority pattern. - - Recovery if the `pull_request` webhook is dropped: empty commit, - `gh workflow run merge-gate.yml -f pr_number=NNN`, or close+reopen. + - Required results must succeed; skipped or neutral is not passing + evidence. Lint and Test Architecture Ratchets are required in both tiers. - `.github/CODEOWNERS` requires Lead Maintainer review for any change to `.github/workflows/**`. -4. **`build-release.yml`** - `push` to main, tags, schedule, `workflow_dispatch` - - **Linux + Windows** run combined `build-and-test` (unit tests + binary build in one job). Unit tests run on every push for platform-regression signal; **smoke tests are gated to tag/schedule/dispatch only** (promotion boundaries) to avoid duplicating `ci-integration.yml`'s merge-time smoke and to cut redundant codex-binary downloads. - - **macOS Intel** uses `build-and-validate-macos-intel` (root node, runs own unit tests - no dependency on `build-and-test`). Builds the binary on every push for early regression feedback; tag/schedule/dispatch promotion runs add marker-bounded `lifecycle_smoke and not live` integration coverage and isolated release validation. - - **macOS ARM** uses `build-and-validate-macos-arm` (root node, tag/schedule/dispatch only - ARM runners are extremely scarce with 2-4h+ queue waits). Promotion runs retain the full non-live integration corpus and isolated release validation. +4. **`build-release.yml`** - main push, tags, schedule, default-branch `repository_dispatch` + - Calls **`release-platform.yml`** using the canonical platform catalog + in `scripts/release-platforms.json`. Native unit tests and builds are + independent. Integration, isolated validation and the Windows installer + depend only on their own platform's build. + - Ordinary main pushes retain four lanes, including macOS Intel. + Full tag/schedule/manual qualification adds macOS ARM and source CI. + Intel retains `lifecycle_smoke and not live`; ARM retains full non-live + integration coverage. Neither platform drops native unit coverage. + - Tags can promote an exact-SHA, fully qualified trusted candidate; + partial main builds and PR artifacts are not release evidence. + Missing matching candidates require a fresh full build. - Publication jobs do not receive `ADO_APM_PAT`; live ADO PAT acceptance is an explicit `ado_pat_e2e` mode in `auth-acceptance.yml`. Full 5-platform binary output (linux x86_64/arm64, darwin x86_64/arm64, windows x86_64). -5. **`ci-runtime.yml`** - nightly schedule, manual dispatch, path-filtered push +5. **`ci-runtime.yml`** - nightly schedule, default-branch repository dispatch, path-filtered push - **Linux x86_64 only**. Live inference smoke tests (`apm run`) isolated from release pipeline. - Uses `GH_MODELS_PAT` for GitHub Models API access. - Failures do not block releases - annotated as warnings. ## Platform Testing Strategy -- **PR time**: Linux-only combined build-and-test in `ci.yml`. Catches logic bugs and dependency issues before merge. Windows + macOS are tested post-merge (platform-specific issues are rare and the full matrix runs on every push to main). -- **Post-merge**: Full 5-platform matrix (linux x86_64/arm64, darwin x86_64/arm64, windows x86_64) catches remaining platform-specific issues on main. -- **Rationale**: ci.yml has always been Linux-only - Windows and macOS are covered by `build-release.yml` on every push to main. This keeps PR feedback fast while still catching platform issues before release. +- **PR time**: Linux source checks and packaging smoke, plus a bounded + Windows compatibility gate. Platform-specific failures remain possible. +- **Post-merge**: Four native unit/build lanes; full qualification includes + all five architectures and every applicable integration/validation gate. +- **Publication**: Requires successful source and native qualification. + Parallelizing these checks must never remove a required result. ## PyInstaller Binary Packaging - **CRITICAL**: Uses `--onedir` mode (NOT `--onefile`) for faster CLI startup performance - **Binary Structure**: Creates `dist/{binary_name}/apm` (nested directory containing executable + dependencies) - **Platform Naming**: `apm-{platform}-{arch}` (e.g., `apm-darwin-arm64`, `apm-linux-x86_64`) -- **Spec File**: `build/apm.spec` handles data bundling, hidden imports, and UPX compression +- **Spec File**: `build/apm.spec` handles data bundling and hidden imports. + Do not provision UPX: the pinned PyInstaller disables it on non-Windows, + while the spec disables it on Windows to avoid antivirus false positives. ## Artifact Flow Quirks -- **Upload**: Artifacts include both binary directory + test scripts for isolation testing -- **Download**: GitHub Actions creates nested structure: `{artifact_name}/dist/{binary_name}/apm` -- **Release Prep**: Extract binary from nested path using `tar -czf "${binary}.tar.gz" -C "${artifact_dir}/dist" "${binary}"` +- **Package once**: `scripts/package_release.py` creates each final archive + after signing and verifies the embedded version/build SHA. +- **Upload**: Each native artifact contains `release-assets/` (archive, + checksum, metadata) and selected validation scripts. +- **Native consumers**: Verify hashes and identity, then extract the same + archive into `dist/{binary_name}`. Do not test an installed older release. +- **Promotion**: Verify complete immutable run/artifact evidence and copy + the ten public archive/checksum files without repackaging. The write-token + publisher has no checkout and executes no candidate scripts. +- **Reruns**: Candidate artifacts are attempt-scoped and promotion downloads + exact artifact IDs. Use **Re-run all jobs** to regenerate qualification; + a partial rerun must not combine old archives with new test evidence. ## Critical Testing Phases 1. **Integration Tests**: Full source code access for comprehensive testing 2. **Release Validation**: ISOLATION testing - no source checkout, validates exact shipped binary experience -3. **Path Resolution**: Use symlinks and PATH manipulation for isolated binary testing +3. **Path Resolution**: Set `APM_BINARY_PATH` explicitly for pytest, so an + editable install or ambient PATH cannot silently replace the candidate +4. **Installer Tests**: New installs and upgrade destinations use the + current candidate; an older binary may only seed the upgrade source ## Inference Testing (Decoupled) - Live inference tests (`apm run`) are **isolated** in `ci-runtime.yml` - they do NOT gate releases @@ -78,16 +106,23 @@ integration suite runs only at merge time via GitHub Merge Queue - Rationale: 8 inference executions x 2% failure rate = 14.9% false-negative per release; APM core UVPs require zero live inference ## Release Flow Dependencies -- **PR workflow**: Tier 1 only - ci.yml (build-and-test, Linux-only) provides fast feedback. Tier 2 does not run until enqueued. -- **Merge queue workflow**: ci.yml (Tier 1 against tentative merge ref) + ci-integration.yml (Tier 2: build -> smoke-test -> integration-tests -> release-validation). Queue auto-merges on success; ejects on failure. -- **Push/Release workflow (Linux + Windows)**: build-and-test -> integration-tests -> release-validation -> create-release -> publish-pypi (gh-aw-compat runs in parallel, informational) +- **PR workflow**: Tier 1 provides fast feedback; Tier 2 waits for enqueue. +- **Merge queue workflow**: Tier 1 plus Tier 2's own build. Integration and + isolated validation no longer form a serial fan-in; the gate still + requires all applicable results. +- **Fresh release**: Independent native units/build -> per-platform + integration/validation/installer -> complete qualification -> verified + assets -> publication. Source CI runs alongside the native lanes. +- **Qualified same-SHA release**: Trusted candidate lookup -> verification + -> publication. Qualification cost occurred before tagging, not vanished. - **Homebrew distribution**: `microsoft/homebrew-apm` polls the latest public APM release and updates its formula with its own `GITHUB_TOKEN`; this workflow must not send a cross-repository dispatch or hold a tap credential. -- **Push/Release workflow (macOS Intel)**: build-and-validate-macos-intel (root node: unit tests + build always + conditional `lifecycle_smoke and not live` integration/release-validation) -> create-release -- **Push/Release workflow (macOS ARM)**: build-and-validate-macos-arm (root node, tag/schedule/dispatch only; full non-live integration corpus and all phases run) -> create-release - **Live ADO PAT acceptance**: manual `auth-acceptance.yml` dispatch with `ado_pat_e2e: true` -> exact `live and requires_ado_pat` marker intersection. Invalid credentials fail with production auth diagnostics but do not block publication. -- **Tag Triggers**: Only `v*.*.*` tags trigger full release pipeline +- **Tag Triggers**: `v*` tags enter release planning; promotion must verify + that the tag and candidate version identify the same release - **Artifact Retention**: 30 days for debugging failed releases -- **Cross-workflow artifacts**: ci-integration.yml builds the binary inline (no cross-workflow artifact transfer); build-release.yml jobs share artifacts within the same workflow run. +- **Cross-run artifacts**: Only the release candidate protocol may reuse + release artifacts, bound to the exact trusted source run and commit. + `ci-integration.yml` continues to build its own binary. ## Branch Protection & Required Checks - **Single required check**: branch protection (`main-protection` ruleset id 9294522) requires exactly one status check context: `gate` from `merge-gate.yml`. All other PR-time signals are aggregated by that workflow's poll loop. @@ -107,13 +142,16 @@ integration suite runs only at merge time via GitHub Merge Queue - `APM_RUN_INFERENCE_TESTS` - When `1`, enables live inference tests in validation scripts ## Performance Considerations -- **Combined build-and-test**: Eliminates ~1.5m runner re-provisioning overhead by running unit tests and binary build in the same job. -- **macOS as root nodes**: macOS consolidated jobs run their own unit tests and start immediately - no dependency on Linux/Windows test completion. +- **Critical path**: Unit tests do not delay native builds. A slow platform + does not delay another platform's integration or isolated validation. + Account for added runner setup and queue waits when measuring savings. - **Native uv caching**: `setup-uv` action with `enable-cache: true` replaces manual `actions/cache@v3` blocks. -- **Targeted setup-node usage**: Node.js is only installed in `ci-runtime.yml`, macOS consolidated jobs, and integration-tests/release-validation phases (for `apm runtime setup copilot` -> npm install). -- **macOS runner consolidation**: Each macOS arch has a single consolidated job (build + integration + release-validation). Intel (`build-and-validate-macos-intel`) runs on every push since Intel runners are plentiful and uses marker-bounded non-live lifecycle integration coverage for promotions. ARM (`build-and-validate-macos-arm`) is gated to tag/schedule/dispatch only since ARM runners are extremely scarce (2-4h+ queue waits) and remains the full non-live corpus macOS authority. This avoids serial re-queuing of runners across multiple jobs and redundant corpus replay on Intel. -- **Unit tests skip macOS**: Python unit tests are platform-agnostic; Linux + Windows coverage is sufficient. macOS-specific validation (binary build, focused Intel integration, full non-live ARM integration, and release validation) still runs via the consolidated jobs. +- **Runtime provisioning**: `APM_TEST_RUNTIMES` limits setup to selected + prerequisites. Strict collection fails rather than silently skipping a + selected test when a required runtime is missing. +- **Duration history**: Persist pytest timings and JUnit outcomes. Timing + caches are scheduling hints, never cached pass results or test allowlists. + Preserve complete shard coverage and existing fixture-affinity groups. - **Tier 2 runs once per merged PR**, not per WIP push, since it triggers on `merge_group` only. Saves the bulk of integration minutes that the previous per-push flow burned. -- UPX compression when available (reduces binary size ~50%) - Python optimization level 2 in PyInstaller - Aggressive module exclusions (tkinter, matplotlib, etc.) \ No newline at end of file diff --git a/.github/scripts/ci/merge_gate_wait.sh b/.github/scripts/ci/merge_gate_wait.sh index 7c14d285c3..025e3a305e 100755 --- a/.github/scripts/ci/merge_gate_wait.sh +++ b/.github/scripts/ci/merge_gate_wait.sh @@ -23,6 +23,10 @@ # EXPECTED_CHECKS required. Comma-separated list of check-run names to # wait for. Whitespace around commas is trimmed. # Example: "Build & Test (Linux),Build (Linux)" +# Only exact-name, exact-SHA GitHub Actions results +# count; colliding latest Actions names fail closed. +# Workflow definitions still form the trust boundary: +# app identity alone does not authenticate a workflow. # EVENT_NAME optional. The triggering event ('pull_request', # 'merge_group', 'workflow_dispatch'). Used only to # emit the right recovery instructions on timeout. @@ -31,8 +35,8 @@ # POLL_SEC optional. Poll interval in seconds. Default: 30. # # Exit codes: -# 0 all expected checks completed with success | skipped | neutral -# 1 at least one expected check completed with a failing conclusion +# 0 all expected checks completed with success +# 1 at least one expected check is ambiguous or completed without success # 2 at least one expected check never appeared within TIMEOUT_MIN # (THE BUG we catch -- dropped 'pull_request' webhook) # 3 at least one expected check appeared but did not complete in time @@ -95,40 +99,56 @@ while [ "$(date +%s)" -lt "$deadline" ]; do poll_count=$((poll_count + 1)) pending_count=0 + # Fetch one paginated snapshot per poll, not one request per check. + # This also bounds the cost of rechecking previously successful jobs. + payload=$(gh api \ + -H "Accept: application/vnd.github+json" \ + --paginate --slurp \ + "repos/${REPO}/commits/${SHA}/check-runs?filter=latest&per_page=100" \ + 2>/dev/null) || payload='[]' + for i in "${!checks[@]}"; do c="${checks[i]}" - [ "${check_status[i]}" = "pending" ] || continue - pending_count=$((pending_count + 1)) - - # Filter by check-run name server-side, asking GitHub for only the - # latest run per name (avoids client-side sort / pagination races - # when a check has been re-run on the same SHA). - encoded=$(jq -rn --arg n "$c" '$n|@uri') - payload=$(gh api \ - -H "Accept: application/vnd.github+json" \ - "repos/${REPO}/commits/${SHA}/check-runs?check_name=${encoded}&filter=latest&per_page=10" \ - 2>/dev/null) || payload='{"check_runs":[]}' - - total=$(echo "$payload" | jq '.check_runs | length' 2>/dev/null || echo 0) + # Recheck successful results too: a rerun on the same SHA can replace + # an earlier pass while the gate is still waiting for other checks. + check_status[i]="pending" + check_url[i]="" + + # Check identity within the snapshot. Never let another + # app's same-named result satisfy a GitHub Actions requirement, or + # silently choose a passing result from colliding Actions workflows. + candidates=$(echo "$payload" | jq -c --arg name "$c" --arg sha "$SHA" \ + '[.[] | .check_runs[] | select( + .name == $name and .head_sha == $sha and .app.slug == "github-actions" + )]' 2>/dev/null) || candidates='[]' + + total=$(echo "$candidates" | jq 'length' 2>/dev/null || echo 0) case "$total" in ''|*[!0-9]*) total=0 ;; esac if [ "$total" -eq 0 ]; then + pending_count=$((pending_count + 1)) echo "[merge-gate] poll #${poll_count}: '${c}' not yet present" continue fi - status=$(echo "$payload" | jq -r '.check_runs | sort_by(.started_at) | reverse | .[0].status') - conclusion=$(echo "$payload" | jq -r '.check_runs | sort_by(.started_at) | reverse | .[0].conclusion') - url=$(echo "$payload" | jq -r '.check_runs | sort_by(.started_at) | reverse | .[0].html_url') + if [ "$total" -gt 1 ]; then + echo "::error title=Ambiguous required check::'${c}' has ${total} latest GitHub Actions results. Required check names must be unique." + exit 1 + fi + + status=$(echo "$candidates" | jq -r '.[0].status') + conclusion=$(echo "$candidates" | jq -r '.[0].conclusion') + url=$(echo "$candidates" | jq -r '.[0].html_url // empty') check_url[i]="$url" if [ "$status" != "completed" ]; then + pending_count=$((pending_count + 1)) echo "[merge-gate] poll #${poll_count}: '${c}' status=${status}" continue fi case "$conclusion" in - success|skipped|neutral) + success) check_status[i]="ok" echo "[merge-gate] poll #${poll_count}: '${c}' OK (${conclusion})" ;; diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index b16710dad3..7ba0540bf3 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -1,805 +1,248 @@ name: CI/CD Pipeline -env: - PYTHON_VERSION: '3.12' # Standardize on 3.12 for Ubuntu 24.04 LTS compatibility - on: push: - branches: [ main ] - tags: [ "v*" ] + branches: [main] + tags: ["v*"] paths-ignore: - - 'docs/**' - - '.gitignore' - - 'LICENSE' + - docs/** + - .gitignore + - LICENSE schedule: - # Run daily at 04:00 UTC (6 AM CEST) - cron: '0 4 * * *' repository_dispatch: types: [manual-build-release] -# Restrict default GITHUB_TOKEN permissions for all jobs (principle of least privilege) permissions: contents: read jobs: - # Unit tests + binary build combined (Linux + Windows). - # Merging test+build eliminates ~1.5m runner re-provisioning overhead per platform. - # macOS runners are scarce and have their own consolidated jobs below. - build-and-test: - name: Build & Test - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-24.04 - arch: x86_64 - platform: linux - binary_name: apm-linux-x86_64 - - os: ubuntu-24.04-arm - arch: arm64 - platform: linux - binary_name: apm-linux-arm64 - - os: windows-latest - arch: x86_64 - platform: windows - binary_name: apm-windows-x86_64 - - runs-on: ${{ matrix.os }} + plan: + name: Resolve Release Candidate + runs-on: ubuntu-24.04 permissions: contents: read - + actions: read + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + full_validation: ${{ steps.plan.outputs.full_validation }} + candidate_run_id: ${{ steps.plan.outputs.candidate_run_id }} + candidate_artifact_ids: ${{ steps.plan.outputs.candidate_artifact_ids }} + candidate_evidence_artifact_id: ${{ steps.plan.outputs.candidate_evidence_artifact_id }} + is_prerelease: ${{ steps.plan.outputs.is_prerelease }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - - name: Install dependencies - run: uv sync --frozen --extra dev --extra build - - # Keep the installer gate ahead of the broad unit suite so an unrelated - # platform test failure cannot hide installer-specific Windows evidence. - - name: Test install.ps1 end-to-end (Windows) - if: matrix.platform == 'windows' - shell: pwsh - env: - APM_E2E_TESTS: "1" - run: >- - uv run --frozen pytest - tests/integration/test_windows_installer_launchers.py - -vv -ra --tb=short - - # Unit tests run on every push for fast platform-regression signal. - # Smoke is intentionally NOT included here: it duplicates ci-integration.yml's - # merge-time smoke gate and burns a real codex binary download per platform - # per push (~15 redundant runs/day). Smoke is gated to promotion boundaries - # below (tag/schedule/dispatch) where it actually serves as a pre-ship gate. - - name: Run unit tests - if: matrix.platform != 'windows' - env: - GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal - - # Named starts/outcomes identify unfinished tests without dumping captures. - # This bounds an opaque hang; it does not turn a timeout into success. - - name: Run unit tests (Windows diagnostics) - if: matrix.platform == 'windows' - timeout-minutes: 60 - env: - PYTHONUNBUFFERED: "1" - GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: >- - uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal - -vv --tb=short --show-capture=no --no-showlocals - - # Smoke runs only at promotion boundaries: - # - tags (pre-ship release gate; only place tag-cut releases get smoke validation) - # - schedule (nightly regression catch for upstream codex URL drift) - # - repository_dispatch (default-branch manual safety net) - - name: Run smoke tests - if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch' - env: - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: uv run pytest tests/integration/test_runtime_smoke.py -v - - - name: Install UPX (Linux) - if: matrix.platform == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y upx-ucl - - - name: Build binary (Unix) - if: matrix.platform != 'windows' - run: | - chmod +x scripts/build-binary.sh - uv run ./scripts/build-binary.sh - - - name: Build binary (Windows) - if: matrix.platform == 'windows' - shell: pwsh - run: | - uv run pwsh scripts/windows/build-binary.ps1 - - - name: Sign Windows binary (Authenticode) - if: matrix.platform == 'windows' && env.WINDOWS_CERT_PFX != '' - shell: pwsh - env: - WINDOWS_CERT_PFX: ${{ secrets.WINDOWS_CERT_PFX }} - WINDOWS_CERT_PASSWORD: ${{ secrets.WINDOWS_CERT_PASSWORD }} - BINARY_DIR: dist\apm-windows-x86_64 - run: | - uv run pwsh scripts/windows/sign-binary.ps1 - - - name: Upload binary as workflow artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + persist-credentials: false + - name: Resolve exact-SHA qualification or build fresh + id: plan + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - name: ${{ matrix.binary_name }} - path: | - ./dist/${{ matrix.binary_name }} - ./dist/${{ matrix.binary_name }}.sha256 - ./scripts/test-release-validation.sh - ./scripts/windows/test-release-validation.ps1 - ./scripts/test-dependency-integration.sh - ./scripts/windows/test-dependency-integration.ps1 - ./scripts/github-token-helper.sh - ./scripts/windows/github-token-helper.ps1 - include-hidden-files: true # Required to include .apm directories - retention-days: 30 - if-no-files-found: error - - # Consolidated macOS Intel build + validation. - # Intel runners are plentiful (<1 min queue) so this runs on every push for - # early macOS build-regression feedback. Integration and release-validation - # phases are conditional on tag/schedule/dispatch. - # Root node - runs its own unit tests instead of waiting for build-and-test. - build-and-validate-macos-intel: - name: Build & Validate (macOS x86_64) - runs-on: macos-15-intel + script: | + const { plan } = require('./scripts/release-candidate.cjs'); + await plan({ github, context, core }); + + # The existing source CI is also a required candidate authority. Calling the + # same workflow keeps lint, architecture, coverage, red-team and lifecycle + # gates from drifting into a second, weaker release-only implementation. + candidate-checks: + name: Candidate Source Checks + needs: [plan] + if: needs.plan.outputs.full_validation == 'true' && needs.plan.outputs.candidate_run_id == '' + uses: ./.github/workflows/ci.yml permissions: contents: read + # Each reusable invocation has its own build -> integration/validation edges. + # A slow Windows unit or installer test cannot hold Linux integration idle. + platforms: + name: ${{ matrix.binary_name }} + needs: [plan] + if: needs.plan.outputs.candidate_run_id == '' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + uses: ./.github/workflows/release-platform.yml + with: + runner: ${{ matrix.runner }} + platform: ${{ matrix.platform }} + binary-name: ${{ matrix.binary_name }} + integration-markers: ${{ matrix.integration_markers }} + full-validation: ${{ needs.plan.outputs.full_validation == 'true' }} + secrets: + GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + GH_MODELS_PAT: ${{ secrets.GH_MODELS_PAT }} + WINDOWS_CERT_PFX: ${{ secrets.WINDOWS_CERT_PFX }} + WINDOWS_CERT_PASSWORD: ${{ secrets.WINDOWS_CERT_PASSWORD }} + + candidate-ready: + name: Qualify Release Candidate + needs: [plan, platforms, candidate-checks] + if: >- + always() && needs.plan.result == 'success' && + needs.plan.outputs.full_validation == 'true' && + needs.plan.outputs.candidate_run_id == '' && + needs.platforms.result == 'success' && + needs.candidate-checks.result == 'success' + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + outputs: + candidate_artifact_ids: ${{ steps.qualification.outputs.candidate_artifact_ids }} + candidate_evidence_artifact_id: ${{ steps.evidence.outputs.artifact-id }} steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - node-version: '24' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Setup build environment - run: | - sudo xcode-select --install 2>/dev/null || true - until xcode-select -p >/dev/null 2>&1; do sleep 5; done - - - name: Install UPX - run: brew install upx - - - name: Install Unix shell startup test runtimes - run: | - brew list bash >/dev/null 2>&1 || brew install bash - brew list fish >/dev/null 2>&1 || brew install fish - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + pattern: candidate-${{ github.run_attempt }}-apm-* + path: candidate-artifacts + - name: Record complete candidate evidence + id: qualification + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - enable-cache: true - - - name: Install dependencies - run: uv sync --frozen --extra dev --extra build - - - name: Run unit tests - run: uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal - - # PHASE 1: BUILD BINARY - - name: Build binary - run: | - chmod +x scripts/build-binary.sh - uv run ./scripts/build-binary.sh - - - name: Test macOS non-shell binary startup - env: - APM_E2E_TESTS: "1" - APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-darwin-x86_64/apm - run: | - test -x "$APM_BINARY_PATH" - uv run --frozen pytest \ - tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_version_runs \ - tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_rich_table_runs \ - -vv -ra --tb=short - - - name: Upload binary as workflow artifact + script: | + const { qualify } = require('./scripts/release-candidate.cjs'); + await qualify({ github, context, core, artifactRoot: 'candidate-artifacts' }); + - id: evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: apm-darwin-x86_64 - path: | - ./dist/apm-darwin-x86_64 - ./dist/apm-darwin-x86_64.sha256 - ./scripts/test-release-validation.sh - ./scripts/test-dependency-integration.sh - ./scripts/github-token-helper.sh - include-hidden-files: true - retention-days: 30 + name: release-candidate-evidence-${{ github.run_attempt }} + path: release-candidate-evidence.json if-no-files-found: error + retention-days: 30 - # PHASE 2: FOCUSED INTEGRATION TESTS (tag/schedule/dispatch only) - - name: Run focused Intel integration tests - if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch' - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - # Intel validates the promoted lifecycle subset against its native - # binary. Live credential acceptance belongs to auth-acceptance.yml. - # loadgroup preserves HOME-mutating test serialization. - PYTEST_MARK_EXPR: "lifecycle_smoke and not live" - PYTEST_EXTRA_ARGS: "-n 4 --dist loadgroup" - run: | - chmod +x scripts/test-integration.sh - uv run ./scripts/test-integration.sh - timeout-minutes: 30 - - # PHASE 3: RELEASE VALIDATION (tag/schedule/dispatch only) - - name: Prepare isolated release-validation environment - if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch' - run: | - mkdir -p /tmp/apm-isolated-test/dist - cp -R ./dist/apm-darwin-x86_64 /tmp/apm-isolated-test/dist/ - cp ./dist/apm-darwin-x86_64.sha256 /tmp/apm-isolated-test/dist/ 2>/dev/null || true - cp ./scripts/test-release-validation.sh /tmp/apm-isolated-test/ - cp ./scripts/test-dependency-integration.sh /tmp/apm-isolated-test/ 2>/dev/null || true - cp ./scripts/github-token-helper.sh /tmp/apm-isolated-test/ 2>/dev/null || true - - cd /tmp/apm-isolated-test - chmod +x ./dist/apm-darwin-x86_64/apm - ln -s "$(pwd)/dist/apm-darwin-x86_64/apm" "$(pwd)/apm" - echo "/tmp/apm-isolated-test" >> "$GITHUB_PATH" - - - name: Run release validation tests - if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch' - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: | - cd /tmp/apm-isolated-test - chmod +x test-release-validation.sh - ./test-release-validation.sh - timeout-minutes: 20 - - # Consolidated macOS ARM build + validation. - # ARM runners (macos-latest) are extremely scarce - 2-4+ hour queue waits are - # common. This job is gated to tag/schedule/dispatch only so push-to-main never - # blocks on ARM availability. All phases run unconditionally since the job itself - # is already release-gated. - # Root node - runs its own unit tests instead of waiting for build-and-test. - build-and-validate-macos-arm: - name: Build & Validate (macOS arm64) - if: github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch' - runs-on: macos-latest + verify-candidate: + name: Verify Publication Inputs + needs: [plan, platforms, candidate-checks, candidate-ready] + if: >- + always() && !cancelled() && github.event_name == 'push' && github.ref_type == 'tag' && + needs.plan.result == 'success' && + (needs.plan.outputs.candidate_run_id != '' || + (needs.platforms.result == 'success' && needs.candidate-checks.result == 'success' && + needs.candidate-ready.result == 'success')) + runs-on: ubuntu-24.04 permissions: contents: read - + actions: read steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '24' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install UPX - run: brew install upx - - - name: Install Unix shell startup test runtimes - run: | - brew list bash >/dev/null 2>&1 || brew install bash - brew list fish >/dev/null 2>&1 || brew install fish - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - enable-cache: true - - - name: Install dependencies - run: uv sync --frozen --extra dev --extra build - - - name: Run unit tests - run: uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal - - # PHASE 1: BUILD BINARY - - name: Build binary - run: | - chmod +x scripts/build-binary.sh - uv run ./scripts/build-binary.sh - - - name: Test macOS non-shell binary startup + persist-credentials: false + - name: Require exact immutable download identities + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - APM_E2E_TESTS: "1" - APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-darwin-arm64/apm - run: | - test -x "$APM_BINARY_PATH" - uv run --frozen pytest \ - tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_version_runs \ - tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_rich_table_runs \ - -vv -ra --tb=short - - - name: Upload binary as workflow artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + ARTIFACT_IDS: ${{ needs.plan.outputs.candidate_artifact_ids || needs.candidate-ready.outputs.candidate_artifact_ids }} + EVIDENCE_ID: ${{ needs.plan.outputs.candidate_evidence_artifact_id || needs.candidate-ready.outputs.candidate_evidence_artifact_id }} with: - name: apm-darwin-arm64 - path: | - ./dist/apm-darwin-arm64 - ./dist/apm-darwin-arm64.sha256 - ./scripts/test-release-validation.sh - ./scripts/test-dependency-integration.sh - ./scripts/github-token-helper.sh - include-hidden-files: true - retention-days: 30 - if-no-files-found: error - - # PHASE 2: INTEGRATION TESTS - - name: Run integration tests - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - # Scarce ARM runners force the full suite onto one runner, so - # parallelise the non-live corpus in-process with xdist. - # --dist loadgroup honors the - # home_env xdist_group markers that keep HOME-mutating tests - # serialized on a single worker. - PYTEST_MARK_EXPR: "not live" - PYTEST_EXTRA_ARGS: "-n 4 --dist loadgroup" - run: | - chmod +x scripts/test-integration.sh - uv run ./scripts/test-integration.sh - timeout-minutes: 60 - - # PHASE 3: RELEASE VALIDATION - - name: Prepare isolated release-validation environment - run: | - mkdir -p /tmp/apm-isolated-test/dist - cp -R ./dist/apm-darwin-arm64 /tmp/apm-isolated-test/dist/ - cp ./dist/apm-darwin-arm64.sha256 /tmp/apm-isolated-test/dist/ 2>/dev/null || true - cp ./scripts/test-release-validation.sh /tmp/apm-isolated-test/ - cp ./scripts/test-dependency-integration.sh /tmp/apm-isolated-test/ 2>/dev/null || true - cp ./scripts/github-token-helper.sh /tmp/apm-isolated-test/ 2>/dev/null || true - - cd /tmp/apm-isolated-test - chmod +x ./dist/apm-darwin-arm64/apm - ln -s "$(pwd)/dist/apm-darwin-arm64/apm" "$(pwd)/apm" - echo "/tmp/apm-isolated-test" >> "$GITHUB_PATH" - - - name: Run release validation tests - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: | - cd /tmp/apm-isolated-test - chmod +x test-release-validation.sh - ./test-release-validation.sh - timeout-minutes: 20 - - # Integration tests with full source code access (Linux + Windows). - # macOS integration tests run inside the consolidated macOS jobs to avoid re-queuing scarce runners. - # Skip on push-to-main: already validated by ci-integration.yml during the PR. - # Run on tags (release gate), schedule (regression), and dispatch (manual). - integration-tests: - name: Integration Tests - if: > - always() && - (github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch') && - (github.event_name == 'schedule' || needs.build-and-test.result == 'success') - needs: [build-and-test] - strategy: - matrix: - include: - - os: ubuntu-24.04 - arch: x86_64 - platform: linux - binary_name: apm-linux-x86_64 - - os: ubuntu-24.04-arm - arch: arm64 - platform: linux - binary_name: apm-linux-arm64 - - os: windows-latest - arch: x86_64 - platform: windows - binary_name: apm-windows-x86_64 - - runs-on: ${{ matrix.os }} - permissions: - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Download APM binary from build artifacts - continue-on-error: ${{ github.event_name == 'schedule' }} + script: | + const ids = process.env.ARTIFACT_IDS.split(','); + if (ids.length !== 5 || new Set(ids).size !== 5 || + ids.some(id => !/^[1-9][0-9]*$/.test(id)) || + !/^[1-9][0-9]*$/.test(process.env.EVIDENCE_ID)) { + throw new Error('Complete immutable candidate artifact IDs are required before download'); + } + - name: Download qualified evidence uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ matrix.binary_name }} - path: ./ - - - name: Check downloaded binary artifact - id: artifact - if: always() - shell: bash - run: | - if [ -d "./dist/${{ matrix.binary_name }}" ]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::warning::No ${{ matrix.binary_name }} artifact; skipping scheduled integration validation for this platform." - fi - - - name: Set up Node.js - if: github.event_name != 'schedule' || steps.artifact.outputs.available == 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + artifact-ids: ${{ needs.plan.outputs.candidate_evidence_artifact_id || needs.candidate-ready.outputs.candidate_evidence_artifact_id }} + run-id: ${{ needs.plan.outputs.candidate_run_id || github.run_id }} + github-token: ${{ github.token }} + path: candidate-evidence + - name: Download exact candidate artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - node-version: '24' - - - name: Set up Python - if: github.event_name != 'schedule' || steps.artifact.outputs.available == 'true' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + artifact-ids: ${{ needs.plan.outputs.candidate_artifact_ids || needs.candidate-ready.outputs.candidate_artifact_ids }} + run-id: ${{ needs.plan.outputs.candidate_run_id || github.run_id }} + github-token: ${{ github.token }} + path: candidate-artifacts + - name: Verify identity and archive bytes before publication + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CANDIDATE_RUN_ID: ${{ needs.plan.outputs.candidate_run_id || github.run_id }} with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - if: github.event_name != 'schedule' || steps.artifact.outputs.available == 'true' - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + script: | + const { verify } = require('./scripts/release-candidate.cjs'); + await verify({ + github, context, core, + runId: process.env.CANDIDATE_RUN_ID, + artifactRoot: 'candidate-artifacts', + evidencePath: 'candidate-evidence', + outputRoot: 'release-assets' + }); + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - enable-cache: true - - - name: Install Unix shell startup test runtimes - if: matrix.platform != 'windows' && (github.event_name != 'schedule' || steps.artifact.outputs.available == 'true') - run: | - sudo apt-get update - sudo apt-get install -y fish zsh - - - name: Install test dependencies - if: github.event_name != 'schedule' || steps.artifact.outputs.available == 'true' - run: uv sync --frozen --extra dev - - - name: Run integration tests (Unix) - if: matrix.platform != 'windows' && (github.event_name != 'schedule' || steps.artifact.outputs.available == 'true') - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} # Primary: APM module access - # The tag gate runs the non-live corpus on one runner. Four workers stay - # below the merge gate's aggregate width while loadgroup preserves - # xdist_group serialization for HOME-mutating tests. - PYTEST_MARK_EXPR: "not live" - PYTEST_EXTRA_ARGS: "-n 4 --dist loadgroup" - run: | - chmod +x scripts/test-integration.sh - uv run ./scripts/test-integration.sh - timeout-minutes: 60 - - - name: Run integration tests (Windows) - if: matrix.platform == 'windows' && (github.event_name != 'schedule' || steps.artifact.outputs.available == 'true') - shell: pwsh - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: | - uv run pwsh scripts/windows/test-integration.ps1 -SkipBuild - timeout-minutes: 20 - - # Release validation tests - Final pre-release validation of shipped binary (Linux + Windows). - # macOS release validation runs inside the consolidated macOS jobs. - release-validation: - name: Release Validation - if: > - always() && - (github.ref_type == 'tag' || github.event_name == 'schedule' || github.event_name == 'repository_dispatch') && - (github.event_name == 'schedule' || - (needs.build-and-test.result == 'success' && needs.integration-tests.result == 'success')) - needs: [build-and-test, integration-tests] - strategy: - matrix: - include: - - os: ubuntu-24.04 - arch: x86_64 - platform: linux - binary_name: apm-linux-x86_64 - - os: ubuntu-24.04-arm - arch: arm64 - platform: linux - binary_name: apm-linux-arm64 - - os: windows-latest - arch: x86_64 - platform: windows - binary_name: apm-windows-x86_64 + name: verified-release-assets + path: release-assets/ + if-no-files-found: error + retention-days: 7 - runs-on: ${{ matrix.os }} + create-release: + name: Create GitHub Release + needs: [plan, verify-candidate] + if: >- + always() && !cancelled() && github.event_name == 'push' && github.ref_type == 'tag' && + needs.plan.result == 'success' && needs.verify-candidate.result == 'success' + runs-on: ubuntu-24.04 permissions: - contents: read - + contents: write + outputs: + is_prerelease: ${{ needs.plan.outputs.is_prerelease }} + is_private_repo: ${{ github.event.repository.private }} steps: - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '24' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Download APM binary from build artifacts - continue-on-error: ${{ github.event_name == 'schedule' }} + # The write-token job neither checks out nor executes candidate code. + - name: Download verified release archives uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ matrix.binary_name }} - path: ${{ matrix.platform == 'windows' && 'D:\apm-isolated-test' || '/tmp/apm-isolated-test/' }} - - - name: Check downloaded binary artifact (Unix) - id: artifact_unix - if: always() && matrix.platform != 'windows' - run: | - if [ -d "/tmp/apm-isolated-test/dist/${{ matrix.binary_name }}" ]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::warning::No ${{ matrix.binary_name }} artifact; skipping scheduled release validation for this platform." - fi - - - name: Check downloaded binary artifact (Windows) - id: artifact_windows - if: always() && matrix.platform == 'windows' - shell: pwsh - run: | - if (Test-Path "D:\apm-isolated-test\dist\${{ matrix.binary_name }}") { - "available=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - } else { - "available=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - Write-Warning "No ${{ matrix.binary_name }} artifact; skipping scheduled release validation for this platform." - } - - - name: Make binary executable and verify isolation (Unix) - if: matrix.platform != 'windows' && (github.event_name != 'schedule' || steps.artifact_unix.outputs.available == 'true') - run: | - cd /tmp/apm-isolated-test - - # Debug: List the downloaded structure - echo "Downloaded structure:" - find . -name "apm" -type f - ls -la ./dist/ - - # Make the binary executable - chmod +x ./dist/${{ matrix.binary_name }}/apm - - - name: Create APM symlink for testing (Unix) - if: matrix.platform != 'windows' && (github.event_name != 'schedule' || steps.artifact_unix.outputs.available == 'true') - run: | - cd /tmp/apm-isolated-test - ln -s "$(pwd)/dist/${{ matrix.binary_name }}/apm" "$(pwd)/apm" - echo "/tmp/apm-isolated-test" >> "$GITHUB_PATH" - - - name: Verify binary and add to PATH (Windows) - if: matrix.platform == 'windows' && (github.event_name != 'schedule' || steps.artifact_windows.outputs.available == 'true') - shell: pwsh - run: | - cd D:\apm-isolated-test - - # Debug: List the downloaded structure - Write-Host "Downloaded structure:" - Get-ChildItem -Recurse -Filter "apm.exe" - Get-ChildItem .\dist\ - - # Add binary directory to PATH - $binDir = "D:\apm-isolated-test\dist\${{ matrix.binary_name }}" - echo $binDir | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Run release validation tests (Unix) - if: matrix.platform != 'windows' && (github.event_name != 'schedule' || steps.artifact_unix.outputs.available == 'true') - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: | - cd /tmp/apm-isolated-test - chmod +x scripts/test-release-validation.sh - ./scripts/test-release-validation.sh - timeout-minutes: 20 - - - name: Run release validation tests (Windows) - if: matrix.platform == 'windows' && (github.event_name != 'schedule' || steps.artifact_windows.outputs.available == 'true') - shell: pwsh - env: - APM_E2E_TESTS: "1" - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: | - cd D:\apm-isolated-test - .\scripts\windows\test-release-validation.ps1 - timeout-minutes: 20 + name: verified-release-assets + path: release-assets + - name: Create GitHub Release + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + with: + generate_release_notes: true + fail_on_unmatched_files: true + prerelease: ${{ needs.plan.outputs.is_prerelease == 'true' }} + files: | + release-assets/apm-linux-x86_64.tar.gz + release-assets/apm-linux-x86_64.tar.gz.sha256 + release-assets/apm-linux-arm64.tar.gz + release-assets/apm-linux-arm64.tar.gz.sha256 + release-assets/apm-darwin-x86_64.tar.gz + release-assets/apm-darwin-x86_64.tar.gz.sha256 + release-assets/apm-darwin-arm64.tar.gz + release-assets/apm-darwin-arm64.tar.gz.sha256 + release-assets/apm-windows-x86_64.zip + release-assets/apm-windows-x86_64.zip.sha256 daily-release-smoke-signal: name: Daily Release Smoke Signal - needs: - - build-and-test - - build-and-validate-macos-intel - - build-and-validate-macos-arm - - integration-tests - - release-validation + needs: [plan, platforms, candidate-checks, candidate-ready] if: always() && github.event_name == 'schedule' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: read contents: read issues: write steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: File or recover daily smoke issue uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | const { main } = require('./scripts/daily-release-smoke-signal.cjs'); await main({ github, context, core }); - - create-release: - name: Create GitHub Release - needs: [build-and-test, build-and-validate-macos-intel, build-and-validate-macos-arm, integration-tests, release-validation] - if: github.event_name == 'push' && github.ref_type == 'tag' # Only pushed tags create GitHub releases - runs-on: ubuntu-latest - permissions: - contents: write # Required for release creation - outputs: - is_prerelease: ${{ steps.release_type.outputs.is_prerelease }} - is_private_repo: ${{ github.event.repository.private }} - - steps: - - name: Download all build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: ./dist - - - name: Prepare release binaries - run: | - # Debug: Show the actual downloaded structure - echo "Directory listing:" - ls -la ./dist - echo "" - - # Create tar.gz archives from directory structure for release and Homebrew - cd dist - for binary in apm-linux-x86_64 apm-linux-arm64 apm-darwin-x86_64 apm-darwin-arm64; do - # With artifacts containing both scripts and dist/, the binary is in artifact/dist/binary/ - artifact_dir="${binary}" - binary_dir="${artifact_dir}/dist/${binary}" - - if [ -d "$binary_dir" ] && [ -f "$binary_dir/apm" ]; then - echo "Processing $binary_dir directory..." - # Ensure the binary is executable before archiving - chmod +x "${binary_dir}/apm" - # Create tar.gz with preserved permissions, archiving ONLY the binary directory for release - tar -czf "${binary}.tar.gz" -C "${artifact_dir}/dist" "${binary}" - if command -v sha256sum &> /dev/null; then - sha256sum "${binary}.tar.gz" > "${binary}.tar.gz.sha256" - elif command -v shasum &> /dev/null; then - shasum -a 256 "${binary}.tar.gz" > "${binary}.tar.gz.sha256" - fi - echo "Created ${binary}.tar.gz" - else - echo "ERROR: Binary directory $binary_dir not found or $binary_dir/apm missing" - echo "Artifact directory contents:" - ls -la "$artifact_dir/" || echo "Directory $artifact_dir does not exist" - if [ -d "$artifact_dir/dist" ]; then - echo "Dist directory contents:" - ls -la "$artifact_dir/dist/" - fi - exit 1 - fi - done - - binary="apm-windows-x86_64" - artifact_dir="${binary}" - binary_dir="${artifact_dir}/dist/${binary}" - if [ -d "$binary_dir" ] && [ -f "$binary_dir/apm.exe" ]; then - echo "Processing $binary_dir directory..." - ( - cd "${artifact_dir}/dist" - zip -qr "../../${binary}.zip" "${binary}" - ) - if command -v sha256sum &> /dev/null; then - sha256sum "${binary}.zip" > "${binary}.zip.sha256" - elif command -v shasum &> /dev/null; then - shasum -a 256 "${binary}.zip" > "${binary}.zip.sha256" - fi - echo "Created ${binary}.zip" - else - echo "ERROR: Binary directory $binary_dir not found or $binary_dir/apm.exe missing" - echo "Artifact directory contents:" - ls -la "$artifact_dir/" || echo "Directory $artifact_dir does not exist" - if [ -d "$artifact_dir/dist" ]; then - echo "Dist directory contents:" - ls -la "$artifact_dir/dist/" - fi - exit 1 - fi - - - name: Determine release type - id: release_type - run: | - TAG_NAME="${{ github.ref_name }}" - - # Check if tag matches stable semver pattern (e.g., v1.2.3, v0.2.0) - # Stable: starts with v, followed by digits.digits.digits, then end of string - if [[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "is_prerelease=false" >> "$GITHUB_OUTPUT" - echo "Detected stable release: $TAG_NAME" - # Check PEP 440 prerelease patterns: v1.2.3a1, v1.2.3b1, v1.2.3rc1, etc. - elif [[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(a|b|rc)[0-9]+$ ]]; then - echo "is_prerelease=true" >> "$GITHUB_OUTPUT" - echo "Detected PEP 440 prerelease: $TAG_NAME" - else - echo "is_prerelease=true" >> "$GITHUB_OUTPUT" - echo "Detected other prerelease: $TAG_NAME" - fi - - - name: Create GitHub Release - id: release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 - with: - generate_release_notes: true - prerelease: ${{ steps.release_type.outputs.is_prerelease == 'true' }} - files: | - ./dist/apm-linux-x86_64.tar.gz - ./dist/apm-linux-x86_64.tar.gz.sha256 - ./dist/apm-linux-arm64.tar.gz - ./dist/apm-linux-arm64.tar.gz.sha256 - ./dist/apm-darwin-x86_64.tar.gz - ./dist/apm-darwin-x86_64.tar.gz.sha256 - ./dist/apm-darwin-arm64.tar.gz - ./dist/apm-darwin-arm64.tar.gz.sha256 - ./dist/apm-windows-x86_64.zip - ./dist/apm-windows-x86_64.zip.sha256 - - # Deploy the docs site after the release is published. - # Invoked here (rather than via the docs workflow's `release: published` - # trigger) because release events created by GITHUB_TOKEN do not fire - # downstream workflow runs -- a documented Actions safeguard against - # recursion. Calling docs.yml directly preserves the standard release -> - # docs deploy chain without needing a PAT or GitHub App token. deploy-docs: name: Deploy Docs needs: [create-release] - if: github.event_name == 'push' && github.ref_type == 'tag' && needs.create-release.outputs.is_prerelease != 'true' + if: >- + always() && !cancelled() && needs.create-release.result == 'success' && + github.event_name == 'push' && github.ref_type == 'tag' && + needs.create-release.outputs.is_prerelease != 'true' uses: ./.github/workflows/docs.yml permissions: contents: read @@ -808,15 +251,12 @@ jobs: with: is_prerelease: false - # GH-AW Compatibility - validates the released binary works in the - # exact flow GitHub Agentic Workflows uses (isolated install + pack, no token). - # Informational: runs as continue-on-error because it depends on external services - # (GitHub API, npm, microsoft/apm-action, microsoft/apm-sample-package) that can - # fail independently of APM code quality (~4% false-negative rate per execution). gh-aw-compat: name: GH-AW Compatibility needs: [create-release] - if: github.event_name == 'push' && github.ref_type == 'tag' + if: >- + always() && !cancelled() && needs.create-release.result == 'success' && + github.event_name == 'push' && github.ref_type == 'tag' runs-on: ubuntu-24.04 continue-on-error: true steps: @@ -832,40 +272,34 @@ jobs: target: claude apm-version: ${{ github.ref_name }} working-directory: /tmp/gh-aw-compat-test - - name: Verify bundle - run: | - test -f "${{ steps.pack.outputs.bundle-path }}" - echo "[+] GH-AW compatibility test passed" - + env: + BUNDLE_PATH: ${{ steps.pack.outputs.bundle-path }} + run: test -f "$BUNDLE_PATH" - name: Warn on failure if: failure() - run: | - echo "::warning::GH-AW compatibility test failed. This is informational - external dependency may be unavailable. Investigate before next release." + run: echo "::warning::GH-AW compatibility failed; investigate the external dependency before the next release." build-pypi-distributions: name: Build PyPI Distributions - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [create-release] - if: github.event_name == 'push' && github.ref_type == 'tag' && needs.create-release.outputs.is_private_repo != 'true' && needs.create-release.outputs.is_prerelease != 'true' + if: >- + always() && !cancelled() && needs.create-release.result == 'success' && + github.event_name == 'push' && github.ref_type == 'tag' && + needs.create-release.outputs.is_private_repo != 'true' && + needs.create-release.outputs.is_prerelease != 'true' permissions: contents: read - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install locked build dependencies run: uv sync --frozen --extra dev - - name: Build Python package run: uv build --no-build-isolation --no-sources - - name: Upload Python distributions uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -875,25 +309,23 @@ jobs: dist/*.tar.gz if-no-files-found: error - # Publish to PyPI (only stable releases from public repo) publish-pypi: name: Publish to PyPI - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [build-pypi-distributions] + if: always() && !cancelled() && needs.build-pypi-distributions.result == 'success' environment: name: pypi url: https://pypi.org/p/apm-cli permissions: actions: read - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - + id-token: write steps: - name: Download Python distributions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: pypi-distributions path: dist - - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 1faf89a27e..1936c49700 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -62,11 +62,6 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra dev --extra build - - name: Install UPX - run: | - sudo apt-get update - sudo apt-get install -y upx-ucl - - name: Build binary run: | chmod +x scripts/build-binary.sh @@ -83,6 +78,12 @@ jobs: retention-days: 7 if-no-files-found: error + - name: Freeze integration scheduling hints + uses: ./.github/actions/pytest-timing + with: + suite: integration + snapshot: integration-duration-snapshot + smoke-test: name: Smoke Test (Linux) needs: [build] @@ -204,6 +205,11 @@ jobs: - name: Install test dependencies run: uv sync --frozen --extra dev + - name: Download shared integration scheduling snapshot + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: integration-duration-snapshot + # APM resolves its cache dir to ~/.cache/apm by default on Linux # (XDG_CACHE_HOME unset -> $HOME/.cache/apm). actions/cache restores # straight into that path, so the apm binary picks it up natively @@ -233,7 +239,7 @@ jobs: # --dist loadgroup is required: it is the only xdist scheduler # that honors pytest.mark.xdist_group, which we use to keep # HOME-mutating tests serialized on a single worker. - PYTEST_EXTRA_ARGS: "--splits 4 --group ${{ matrix.shard }} -n 2 --dist loadgroup --maxfail 10 --cov --cov-fail-under=0" + PYTEST_EXTRA_ARGS: "--splits 4 --group ${{ matrix.shard }} -n 2 --dist loadgroup --maxfail 10 --cov --cov-fail-under=0 --durations=50 --store-durations --junitxml=test-results/integration.xml" run: | chmod +x scripts/test-integration.sh uv run ./scripts/test-integration.sh @@ -241,6 +247,18 @@ jobs: # suite so runner variance does not turn into an unbounded job. timeout-minutes: 20 + - name: Upload test timings and outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-integration-shard-${{ matrix.shard }} + path: | + test-results/ + .test_durations + include-hidden-files: true + retention-days: 30 + if-no-files-found: error + - name: Rename coverage data for shard if: always() run: | @@ -331,7 +349,7 @@ jobs: release-validation: name: Release Validation (Linux) - needs: [build, integration-tests] + needs: [build] runs-on: ubuntu-24.04 steps: - name: Checkout test scripts diff --git a/.github/workflows/ci-runtime.yml b/.github/workflows/ci-runtime.yml index 40aaf8f180..363e1d163b 100644 --- a/.github/workflows/ci-runtime.yml +++ b/.github/workflows/ci-runtime.yml @@ -76,6 +76,8 @@ jobs: - name: Run smoke tests env: + APM_E2E_TESTS: "1" + APM_RUN_INTEGRATION_TESTS: "1" GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} run: uv run pytest tests/integration/test_runtime_smoke.py -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f0b0e7e3..93c02c3bca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ env: PYTHON_VERSION: '3.12' on: + workflow_call: pull_request: branches: [ main ] # Tier 1 also runs in merge queue context so the same unit + build checks @@ -224,9 +225,8 @@ jobs: # enforces the global 80% floor, just off the PR-time critical path, # and is escalated to required on `merge_group`. # - # First runs use pytest-split's naive file-based split (no .test_durations - # committed yet); commit a durations file in a follow-up PR after one - # green run to balance shards better. + # Record timings without independently restoring mutable shard histories. + # All shards must partition the same collection with identical inputs. build-and-test-shard: name: Build & Test Shard ${{ matrix.shard }} (Linux) runs-on: ubuntu-24.04 @@ -253,6 +253,10 @@ jobs: packages/apm-contributor-dashboard/tests/logic.test.mjs packages/apm-contributor-dashboard/tests/server.test.mjs + - name: Run release candidate trust tests + if: matrix.shard == 1 + run: node --test scripts/release-candidate.test.cjs + - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -271,6 +275,9 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra dev + # Record timings, but do not independently restore a rolling cache here: + # different snapshots could assign the same test to neither shard. + # Keep PR selection deterministic without adding a runner-allocation gate. # --splits 2 --group N matches the layout used by ci-integration.yml. # -n 2 --dist worksteal parallelises within the shard. # @@ -285,8 +292,21 @@ jobs: uv run pytest tests/unit tests/test_console.py tests/red_team \ --splits 2 --group ${{ matrix.shard }} \ -n 2 --dist worksteal \ + --durations=50 --store-durations --junitxml=test-results/unit.xml \ --cov --cov-report= --cov-fail-under=60 + - name: Upload test timings and outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-unit-shard-${{ matrix.shard }} + path: | + test-results/ + .test_durations + include-hidden-files: true + retention-days: 30 + if-no-files-found: error + - name: Rename coverage data for shard if: always() run: | @@ -373,6 +393,7 @@ jobs: # and in build-release.yml on release. pr-binary-smoke: name: PR Binary Smoke (Linux) + if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 permissions: contents: read @@ -393,11 +414,6 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra dev --extra build - - name: Install UPX - run: | - sudo apt-get update - sudo apt-get install -y upx-ucl - - name: Build binary run: | chmod +x scripts/build-binary.sh diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 914454d6a7..8afc016858 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -112,7 +112,10 @@ jobs: # Keep this in sync with the underlying workflows. # NOTE: 'gate' (this job) MUST NOT appear here -- it would # deadlock waiting for itself. - # PR context: only the shard jobs gate. Coverage Combine runs + # Lint and Test Architecture Ratchets are required in BOTH + # contexts: the sole required 'gate' check must not let their + # failures merge, even when no human review is required. + # PR context: Coverage Combine runs # in parallel as a non-required signal so global 80% drift is # visible in the PR UI without paying ~3 min of runner- # allocation latency on the critical path. @@ -133,18 +136,16 @@ jobs: # gate the class of Windows-only regressions fixed in # microsoft/apm#2233 rather than surfacing only after landing # on main. - EXPECTED_CHECKS: ${{ github.event_name == 'merge_group' && 'Build & Test Shard 1 (Linux),Build & Test Shard 2 (Linux),Coverage Combine (Linux),Windows Compatibility Gate,APM Self-Check,NOTICE Drift Check,Lifecycle Smoke (Linux),Build (Linux),Smoke Test (Linux),Integration Tests (Linux),Release Validation (Linux)' || 'Build & Test Shard 1 (Linux),Build & Test Shard 2 (Linux),Windows Compatibility Gate,APM Self-Check,NOTICE Drift Check,Lifecycle Smoke (Linux)' }} - # Poll budget: ci-integration.yml chains Build -> Smoke -> - # Integration (timeout 20m) -> Release Validation (timeout 20m). - # Theoretical worst case ~50m; observed today ~5m end-to-end. - # Sized at 55m to absorb growth without false-failing the gate. + EXPECTED_CHECKS: ${{ github.event_name == 'merge_group' && 'Lint,Test Architecture Ratchets,Build & Test Shard 1 (Linux),Build & Test Shard 2 (Linux),Coverage Combine (Linux),Windows Compatibility Gate,APM Self-Check,NOTICE Drift Check,Lifecycle Smoke (Linux),Build (Linux),Smoke Test (Linux),Integration Tests (Linux),Release Validation (Linux)' || 'Lint,Test Architecture Ratchets,Build & Test Shard 1 (Linux),Build & Test Shard 2 (Linux),Windows Compatibility Gate,APM Self-Check,NOTICE Drift Check,Lifecycle Smoke (Linux)' }} + # Integration waits for Build -> Smoke; isolated validation waits + # only for Build. Retain the existing 55m budget for runner queues + # and slow native checks rather than trading correctness for speed. TIMEOUT_MIN: '55' - # 5s poll keeps perceived gate latency low. With ~3 expected checks - # per poll = ~36 gh API calls/min, well under the 5000/hr quota. + # Each 5s poll fetches one paginated snapshot for all checks + # (~12 API calls/min for one page), including earlier successes + # so a rerun cannot reuse a stale pass. # cancel-in-progress concurrency bounds parallel polling per PR. POLL_SEC: '5' run: | chmod +x .github/scripts/ci/merge_gate_wait.sh .github/scripts/ci/merge_gate_wait.sh - - diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml new file mode 100644 index 0000000000..8a036614fb --- /dev/null +++ b/.github/workflows/release-platform.yml @@ -0,0 +1,360 @@ +name: Native release candidate + +on: + workflow_call: + inputs: + runner: + type: string + required: true + platform: + type: string + required: true + binary-name: + type: string + required: true + integration-markers: + type: string + default: not live + full-validation: + type: boolean + required: true + secrets: + GH_CLI_PAT: + required: false + GH_MODELS_PAT: + required: false + WINDOWS_CERT_PFX: + required: false + WINDOWS_CERT_PASSWORD: + required: false + +permissions: + contents: read + +env: + PYTHON_VERSION: '3.12' + +jobs: + # Native unit evidence is independent of the frozen artifact. Never put this + # in the build's needs list: publication, not compilation, consumes its verdict. + unit-tests: + name: Unit Tests + runs-on: ${{ inputs.runner }} + timeout-minutes: 65 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev + - uses: ./.github/actions/pytest-timing + with: + suite: unit + - name: Run unit tests + if: inputs.platform != 'windows' + env: + GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} + GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} + run: >- + uv run --frozen pytest tests/unit tests/test_console.py + -n auto --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit.xml + - name: Run unit tests (Windows diagnostics) + if: inputs.platform == 'windows' + timeout-minutes: 60 + env: + PYTHONUNBUFFERED: '1' + GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + run: >- + uv run --frozen pytest tests/unit tests/test_console.py + -n auto --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit.xml + -vv --tb=short --show-capture=no --no-showlocals + - name: Upload unit timings and outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ inputs.binary-name }}-unit + path: | + test-results/ + .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + + build: + name: Build Candidate + runs-on: ${{ inputs.runner }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev --extra build + - name: Build binary (Unix) + if: inputs.platform != 'windows' + run: uv run --frozen bash scripts/build-binary.sh + - name: Build binary (Windows) + if: inputs.platform == 'windows' + run: uv run --frozen pwsh scripts/windows/build-binary.ps1 + - name: Sign Windows binary (Authenticode) + if: inputs.platform == 'windows' && env.WINDOWS_CERT_PFX != '' + env: + WINDOWS_CERT_PFX: ${{ secrets.WINDOWS_CERT_PFX }} + WINDOWS_CERT_PASSWORD: ${{ secrets.WINDOWS_CERT_PASSWORD }} + BINARY_DIR: dist\apm-windows-x86_64 + run: uv run --frozen pwsh scripts/windows/sign-binary.ps1 + - name: Test native binary startup and core contracts + env: + PYTHONUTF8: '1' + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/${{ inputs.platform == 'windows' && 'apm.exe' || 'apm' }} + run: >- + uv run --frozen pytest tests/integration/test_core_smoke.py + -vv -ra --tb=short --junitxml=test-results/core.xml + - name: Package exact candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: ${{ inputs.binary-name }} + shell: bash + run: >- + python scripts/package_release.py pack + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Upload binary as workflow artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + path: | + release-assets/ + scripts/package_release.py + scripts/release-platforms.json + scripts/test-release-validation.sh + scripts/windows/test-release-validation.ps1 + scripts/test-dependency-integration.sh + scripts/windows/test-dependency-integration.ps1 + scripts/github-token-helper.sh + scripts/windows/github-token-helper.ps1 + include-hidden-files: true + retention-days: 30 + if-no-files-found: error + - name: Upload native smoke outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ inputs.binary-name }}-core + path: test-results/ + if-no-files-found: error + retention-days: 30 + + integration-tests: + name: Integration Tests + needs: [build] + if: inputs.full-validation + runs-on: ${{ inputs.runner }} + timeout-minutes: 65 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install Linux shell test runtimes + if: inputs.platform == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y fish zsh + - name: Install macOS shell test runtimes + if: inputs.platform == 'darwin' + run: | + brew list bash >/dev/null 2>&1 || brew install bash + brew list fish >/dev/null 2>&1 || brew install fish + - name: Install dependencies + run: uv sync --frozen --extra dev + - name: Extract checked candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: ${{ inputs.binary-name }} + shell: bash + run: >- + python scripts/package_release.py verify-extract + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - uses: ./.github/actions/pytest-timing + with: + suite: integration + - name: Run integration tests (Unix) + if: inputs.platform != 'windows' + timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} + env: + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} + APM_TEST_RUNTIMES: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 'none' || 'copilot' }} + PYTEST_EXTRA_ARGS: -n 4 --dist loadgroup --durations=50 --store-durations --junitxml=test-results/integration.xml + run: uv run --frozen bash scripts/test-integration.sh + - name: Run integration tests (Windows) + if: inputs.platform == 'windows' + timeout-minutes: 20 + env: + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm.exe + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + run: uv run --frozen pwsh scripts/windows/test-integration.ps1 -SkipBuild + - name: Upload integration timings and outcomes + if: always() && inputs.platform != 'windows' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ inputs.binary-name }}-integration + path: | + test-results/ + .test_durations + include-hidden-files: true + retention-days: 30 + if-no-files-found: error + + # No checkout or editable package: validate the actual archive on a fresh + # native runner, independently of integration and its mutable HOME/caches. + release-validation: + name: Isolated Release Validation + needs: [build] + if: inputs.full-validation + runs-on: ${{ inputs.runner }} + timeout-minutes: 25 + steps: + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + - name: Extract checked candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: ${{ inputs.binary-name }} + shell: bash + run: >- + python scripts/package_release.py verify-extract + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Run release validation tests (Unix) + if: inputs.platform != 'windows' + timeout-minutes: 20 + env: + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + APM_E2E_TESTS: '1' + BINARY_NAME: ${{ inputs.binary-name }} + run: bash scripts/test-release-validation.sh "$PWD/dist/$BINARY_NAME/apm" + - name: Run release validation tests (Windows) + if: inputs.platform == 'windows' + shell: pwsh + timeout-minutes: 20 + env: + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + APM_E2E_TESTS: '1' + run: | + $env:PATH = "$PWD\dist\apm-windows-x86_64;$env:PATH" + .\scripts\windows\test-release-validation.ps1 + + windows-installer: + name: Candidate Windows Installer + needs: [build] + if: inputs.platform == 'windows' + runs-on: ${{ inputs.runner }} + timeout-minutes: 20 + env: + APM_BASELINE_VERSION: v0.28.0 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev + - name: Download historical upgrade source once + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download $env:APM_BASELINE_VERSION --repo microsoft/apm ` + --pattern apm-windows-x86_64.zip --pattern apm-windows-x86_64.zip.sha256 ` + --dir installer-baseline + if ($LASTEXITCODE -ne 0) { throw "Could not download the historical upgrade source" } + $sidecar = (Get-Content installer-baseline/apm-windows-x86_64.zip.sha256 -Raw).Trim() + if ($sidecar -notmatch '^([a-fA-F0-9]{64})\s+\*?apm-windows-x86_64\.zip$') { + throw "Invalid historical upgrade source checksum" + } + $expected = $Matches[1].ToLowerInvariant() + $actual = (Get-FileHash installer-baseline/apm-windows-x86_64.zip -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -ne $expected) { throw "Historical upgrade source checksum mismatch" } + "APM_BASELINE_SHA256=$actual" >> $env:GITHUB_ENV + - name: Test install.ps1 end-to-end (Windows) + shell: pwsh + env: + APM_E2E_TESTS: '1' + APM_CANDIDATE_ARCHIVE: ${{ github.workspace }}/release-assets/apm-windows-x86_64.zip + APM_BASELINE_ARCHIVE: ${{ github.workspace }}/installer-baseline/apm-windows-x86_64.zip + run: | + $candidate = Get-Content release-assets/apm-windows-x86_64.json -Raw | ConvertFrom-Json + $env:APM_CANDIDATE_VERSION = "v$($candidate.version)" + $env:APM_CANDIDATE_SHA256 = $candidate.archive_sha256 + uv run --frozen pytest tests/integration/test_windows_installer_launchers.py -vv -ra --tb=short --junitxml=test-results/installer.xml + - name: Upload installer outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ inputs.binary-name }}-installer + path: test-results/ + retention-days: 30 + if-no-files-found: error + + gate: + name: Native Candidate Gate + needs: [unit-tests, build, integration-tests, release-validation, windows-installer] + if: always() + runs-on: ubuntu-24.04 + steps: + - name: Require every applicable native result + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RESULTS: ${{ toJSON(needs) }} + FULL: ${{ inputs.full-validation }} + PLATFORM: ${{ inputs.platform }} + with: + script: | + const results = JSON.parse(process.env.RESULTS); + const required = ['unit-tests', 'build']; + if (process.env.FULL === 'true') required.push('integration-tests', 'release-validation'); + if (process.env.PLATFORM === 'windows') required.push('windows-installer'); + for (const name of required) { + if (results[name]?.result !== 'success') core.setFailed(`${name} did not succeed`); + } diff --git a/docs/src/content/docs/contributing/development-guide.md b/docs/src/content/docs/contributing/development-guide.md index 596f5753da..73c6ed5471 100644 --- a/docs/src/content/docs/contributing/development-guide.md +++ b/docs/src/content/docs/contributing/development-guide.md @@ -5,7 +5,7 @@ sidebar: order: 1 --- -Thank you for considering contributing to APM! This document outlines the process for contributing to the project. +Contributor setup, tests, CI gates, and pull requests. ## Code of Conduct @@ -52,7 +52,20 @@ Enhancement suggestions are welcome! Please: 2. Ensure your PR addresses only one concern (one feature, one bug fix). 3. Include tests for new functionality. 4. Update documentation if needed. -5. PRs must pass all CI checks before they can be merged. +5. PRs must pass the required `gate` check before they can be merged. + +### Required CI and release gates + +`Merge Gate / gate` is the required PR and merge-queue check. It accepts only +successful underlying GitHub Actions checks; skipped, neutral, missing, +duplicate, failed, cancelled, or timed-out checks fail closed. It requires Lint +and Test Architecture Ratchets in both contexts. + +Release candidates never reuse PR artifacts. Tags publish only fresh full +qualification or a same-SHA, trusted-main candidate with matching evidence. +Ordinary `main` green status is not enough. See +[Integration Testing](../integration-testing/) for the native platform matrix, +runtime selections, and archive-promotion model. ### Workflow dependency updates diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 64bbdaf30c..e645650352 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -4,22 +4,21 @@ sidebar: order: 3 --- -This document describes APM's integration testing strategy to ensure runtime setup scripts work correctly and the golden scenario from the README functions as expected. +APM integration testing covers PR checks, merge queue runs, runtime smoke, and release qualification. ## Testing Strategy APM uses a tiered approach to integration testing: -### 1. **Smoke Tests** (merge queue, runtime changes, and releases) -- **Location**: `tests/integration/test_runtime_smoke.py` -- **Purpose**: Fast verification that runtime setup scripts work -- **Scope**: - - Runtime installation (codex, llm) - - Binary functionality (`--version`, `--help`) - - APM runtime detection - - Workflow compilation without execution -- **Duration**: ~2-3 minutes per platform -- **Trigger**: merge queue integration workflow, runtime-code pushes, scheduled/manual runs, and release validation +### 1. **Core Smoke** (merge queue and release builds) +- **Location**: `tests/integration/test_core_smoke.py` +- **Purpose**: fast verification that the built native binary starts and satisfies core CLI contracts +- **Scope**: binary startup, `--version`, `--help`, and network-free core behavior +- **Trigger**: merge queue integration workflow and each native release-platform build + +Runtime installation smoke remains in `tests/integration/test_runtime_smoke.py` +and runs only in the runtime workflow or full integration selections that opt in +to the required runtimes. ### 2. **End-to-End Golden Scenario Tests** (merge queue and promotion runs) - **Location**: `tests/integration/test_golden_scenario_e2e.py` @@ -29,9 +28,9 @@ APM uses a tiered approach to integration testing: - Project initialization (`apm init`) - Dependency installation (`apm install`) - Real API calls to GitHub Models - - Copilot, Codex, LLM, and Gemini runtime execution -- **Duration**: ~10-15 minutes per platform (with 20-minute timeout) -- **Trigger**: merge queue integration workflow, plus tag, schedule, and manual promotion runs + - Runtime-specific execution when the selected lane provisions that runtime +- **Duration**: captured in hosted timing artifacts +- **Trigger**: merge queue integration workflow, plus tag, schedule, and repository-dispatch release qualification runs ### 3. **Lifecycle Smoke** (PR-time required check) - **Location**: selected declaratively via `lifecycle_smoke and not lifecycle_merge_group`. Tests marked `lifecycle_merge_group` remain outside the bounded required set. @@ -50,16 +49,21 @@ APM uses a tiered approach to integration testing: - **Run it locally** (the exact command CI runs): ```bash APM_E2E_TESTS=1 uv run --extra dev pytest -p no:cacheprovider -q --strict-markers \ + -n 2 --dist loadgroup \ -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration ``` -### 4. **Live Guardrailing Hero** (scheduled/manual) +### 4. **Live Guardrailing Hero** (scheduled/repository dispatch) - **Location**: `tests/integration/test_guardrailing_hero_e2e.py` - **Purpose**: Preserve the real remote, token-gated packaged CLI hero without multiplying it across the default packaged platform matrix - **Scope**: project initialization, two GitHub-backed installs, compile/deploy, and prompt startup through the built Linux x64 binary -- **Trigger**: one `ci-runtime.yml` invocation on schedule or manual dispatch that fails the workflow on error (`continue-on-error` is not set); never pull requests or the generic integration script +- **Trigger**: one `ci-runtime.yml` invocation on schedule or repository dispatch that fails the workflow on hero or runtime-smoke errors; never pull requests or the generic integration script - **Selection mechanism**: the explicit test node with `-m live`; collection gates remain owned by `tests/integration/conftest.py` +Nightly runtime smoke sets the required live opt-ins explicitly +(`APM_E2E_TESTS=1`, `APM_RUN_INTEGRATION_TESTS=1`, and tokens). The separate +inference-validation step is annotated but non-blocking. + ## Running Tests Locally Integration tests live under `tests/integration/` and run via `pytest` @@ -86,10 +90,15 @@ what the test family you want actually requires. | `requires_runtime_llm` | The `llm` runtime installed under `~/.apm/runtimes/` | `apm runtime setup llm` | | `live` | Tests that hit real third-party repositories; deselected by default | Override the deselect: `pytest -m live tests/integration -v` | -Without any of those env vars or runtimes a `pytest tests/integration` -invocation is silent rather than red: every test is collected and -reported as `SKIPPED` with a one-line reason, so you can see exactly -what is missing and why. +Without any of those env vars or runtimes, a local `pytest tests/integration` +invocation is silent rather than red: every test is collected and reported as +`SKIPPED` with a one-line reason, so you can see exactly what is missing and +why. + +CI uses `--strict-runtime-prerequisites` for provisioned selections. If a +selected runtime test would otherwise skip because its runtime is missing, +collection fails instead. That prevents green builds caused by omitted runtime +setup. ### Three marker axes @@ -328,23 +337,24 @@ Packaged executable tests must continue to use `apm_binary_path`. ### CI orchestrator: `scripts/test-integration.sh` -`scripts/test-integration.sh` is the thin orchestrator the CI -integration job invokes. Its sole responsibilities are: resolve -GitHub / ADO tokens, detect platform, locate or build the apm -PyInstaller binary, install runtimes (codex / copilot / llm), -install python test dependencies, and run -`pytest tests/integration/` once. All per-test gating lives in the -marker registry described above. New integration tests dropped into -`tests/integration/` are picked up automatically; add the right -`requires_*` marker and the registry will skip the test when its -precondition is missing. - -Release promotion sets `PYTEST_MARK_EXPR="not live"`, so an expiring -third-party credential cannot block publication. Live ADO PAT coverage is -owned by the **Auth Acceptance Tests** workflow: dispatch it with -`ado_pat_e2e: true` and an `ado_repo` acceptance fixture. That workflow -selects `live and requires_ado_pat`, requires -`AUTH_TEST_ADO_APM_PAT`, and fails with the normal actionable auth +`scripts/test-integration.sh` is the thin orchestrator the CI integration job +invokes. Its responsibilities are: resolve GitHub / ADO tokens, detect platform, +locate or build the apm PyInstaller binary, install the requested runtimes, +install Python test dependencies, and run `pytest tests/integration/` once. All +per-test gating lives in the marker registry described above. + +`APM_TEST_RUNTIMES` controls which external runtimes the orchestrator installs: + +- unset locally: install `copilot codex llm` +- `copilot`: install only Copilot CLI for full non-live release qualification +- `none`: install no external runtime for focused lifecycle selections + +Release promotion sets `PYTEST_MARK_EXPR` instead of editing test lists. Full +non-live qualification uses `not live`; the focused macOS Intel lifecycle lane +uses `lifecycle_smoke and not live`. Live ADO PAT coverage is owned by the +**Auth Acceptance Tests** workflow: dispatch it with `ado_pat_e2e: true` and an +`ado_repo` acceptance fixture. That workflow selects `live and requires_ado_pat`, +requires `AUTH_TEST_ADO_APM_PAT`, and fails with the normal actionable auth diagnostic when the PAT is rejected. Run the same focused acceptance locally with: @@ -361,32 +371,112 @@ environment end-to-end; for local iteration prefer the direct ## CI/CD Integration -### GitHub Actions Workflow - -**On PR and merge queue:** -1. PR-time unit checks and the hermetic Lifecycle Smoke gate run first; merge queue adds Linux smoke, integration, and release-validation gates. - -The required Windows compatibility gate selects `windows_compat` tests. Its collection guard requires a non-empty subset, not a fixed test count, so adding marked regressions does not require raising a ceiling. The workflow's test roots and timeout bound scope and runtime. +### Pull requests and merge queue + +The required `gate` check is the single merge authority. It polls the exact +check names expected for the event SHA and accepts only `success`; skipped, +neutral, missing, duplicate, failed, cancelled, or timed-out checks fail closed. + +At PR time the gate requires lint, test architecture ratchets, Linux unit +shards, Windows compatibility, APM self-check, NOTICE drift, and Lifecycle +Smoke. In merge queue context it also requires the Linux build, core smoke, +full integration, release validation, and the coverage combine gate. + +### Release qualification + +`build-release.yml` qualifies native candidates by platform through +`release-platform.yml`: + +1. Resolve whether to reuse a qualified candidate or build fresh. +2. For fresh full validation, call source CI so lint, architecture ratchets, + coverage, red-team tests, and lifecycle smoke stay in the same authority as + normal CI. +3. Run independent native unit and build jobs for each platform. +4. Let platform integration, isolated archive validation, and the Windows + installer job depend only on that platform's build artifact. +5. Promote only the exact archives that were packaged once, verified, unpacked, + and tested. + +The publication path never repacks candidate archives. `scripts/package_release.py` +packages each native archive once, records archive and executable SHA-256 +digests, verifies archive member safety, checks the embedded version/build SHA, +extracts into a fresh destination, and rechecks the extracted executable before +release upload. +Candidate artifacts are attempt-scoped (`candidate--`; +evidence is `release-candidate-evidence-`), so partial reruns must +not reuse old-attempt artifacts; use **Re-run all jobs** to regenerate complete +qualification. + +### Candidate reuse on tags + +A pushed tag may reuse an existing candidate only when release evidence proves +the same commit SHA already completed a successful, trusted-main full +qualification with immutable run identity, artifact identity, and matching +digests. Ordinary `main` green status is not enough, and PR artifacts are never +publication inputs. An absent or expired candidate triggers a fresh full build. +API errors or inconsistent candidate evidence fail the release rather than +silently falling back. + +### Platform matrix + +Full release qualification runs on five native platforms for tags, schedules, +and `repository_dispatch` `manual-build-release` runs: + +- Linux x86_64: `ubuntu-24.04` +- Linux arm64: `ubuntu-24.04-arm` +- Windows x86_64: `windows-latest` +- macOS Intel: `macos-15-intel` +- macOS Apple Silicon: `macos-latest` + +Ordinary `main` pushes keep the existing four-platform policy and exclude macOS +Apple Silicon. macOS Intel runs the focused `lifecycle_smoke and not live` +integration selection; full non-live lanes use `not live`. + +### Windows installer candidate testing + +The Windows installer job consumes the candidate archive produced by the build +job. It does not depend on integration, release validation, or a pinned GitHub +Release destination. Tests may use a previous binary only as an upgrade source; +the destination under test is the freshly built candidate archive. + +Coverage has two paths. The fresh-candidate path checks install, native launch, +junction cleanup, tamper rejection, and same-version reinstall; a canary proves +the old candidate release tree is replaced. The older-source path installs the +baseline once, then runs the real `apm self-update` lifecycle to the candidate +and performs the cross-version upgrade assertions there. + +The wrapper requires six workflow-supplied inputs and has no published-release +fallback: + +```text +APM_CANDIDATE_ARCHIVE +APM_CANDIDATE_VERSION +APM_CANDIDATE_SHA256 +APM_BASELINE_ARCHIVE +APM_BASELINE_VERSION +APM_BASELINE_SHA256 +``` -Linux Lifecycle Smoke runs the required marker subset with `-n 2 --dist loadgroup`. Grouped tests stay on one worker, and the six-minute job limit remains unchanged. +Archive inputs are ZIP paths. SHA-256 inputs are hex digest strings, not +`.sha256` file paths. Version inputs accept either the packager metadata form +`X.Y.Z` or a historical `vX.Y.Z`; the wrapper normalizes to a leading `v`, and +the baseline version must be older than the candidate version. Missing or bad +candidate metadata fails the Windows E2E run; it does not skip or fall back to a +published release. -**On pushed version tag releases:** -1. Unit tests + Smoke tests -2. Build binaries (cross-platform) -3. **E2E golden scenario tests** (using built binaries). Linux and macOS Apple Silicon retain the full non-live integration corpus; macOS Intel runs the marker-bounded `lifecycle_smoke and not live` subset plus native startup and isolated release validation. -4. Create GitHub Release -5. Publish to PyPI +The job entrypoint is pytest, not the PowerShell helper. Set `APM_E2E_TESTS=1` +and the six variables above, then run: -**Daily scheduled release smoke:** -- Runs the same promotion validation path once per day. -- If a build matrix leg fails, downstream Linux/Windows integration and release-validation jobs still run for any platform artifact that was produced. -- Failing scheduled runs update one `ci/daily-smoke` tracking issue; a later recovered run closes it. -- This signal is advisory and is not a required PR check. +```bash +uv run --frozen --extra dev pytest -q tests/integration/test_windows_installer_launchers.py +``` -**Manual workflow dispatch:** -- Test builds (uploads as workflow artifacts) -- Allows testing the full build pipeline without creating a release, even when dispatched from a tag ref -- Useful for validating changes before tagging +Python owns the verified local HTTP server and mandatory suite arguments. It +serves the verified archives and the current checkout's `install.ps1` bytes on +`127.0.0.1` through the production mirror variables (`APM_RELEASE_BASE_URL`, +`APM_INSTALLER_BASE_URL`, `APM_NO_DIRECT_FALLBACK=1`). Production `install.ps1` +is unchanged. The test also compares the installed candidate executable hash +with the executable member inside the candidate ZIP. ### Windows unit hang diagnostics @@ -398,7 +488,9 @@ uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal -vv --tb `PYTHONUNBUFFERED=1` keeps named test starts and outcomes visible while the suite runs. Captured test output and local-variable dumps remain disabled. These diagnostics identify candidate unfinished tests, not stack frames or the exact blocked phase; an outcome can appear before fixture teardown completes. -The 60-minute step limit fails closed on a hang; it is not proof that tests pass or a root-cause fix. Test selection and parallelism are unchanged. The PR-time `windows_compat` gate exercises live name visibility during setup, call, and teardown after a failed call. +The 60-minute step limit fails closed on hangs. Test selection and parallelism +are unchanged. The PR-time `windows_compat` gate exercises name visibility +during setup, call, and teardown after a failed call. ### GitHub Actions Authentication @@ -414,58 +506,8 @@ E2E tests require proper GitHub Models API access: Runtime setup prefers `GITHUB_TOKEN` for GitHub Models and falls back to `GITHUB_APM_PAT` when no user-scoped token is present. -### Release Pipeline Sequencing - -The workflow ensures quality gates at each step: - -1. **build-and-test** jobs - Unit tests plus binary builds -2. **integration-tests** job - Comprehensive non-live runtime scenarios -3. **release-validation** job - Final shipped-binary validation -4. **create-release** job - GitHub release creation -5. **publish-pypi** job - PyPI package publication - -Each publication stage must succeed before proceeding to the next. Live -third-party credential acceptance remains a failing gate in the opt-in Auth -Acceptance workflow rather than a dependency of release publication. - -The [`microsoft/homebrew-apm`](https://github.com/microsoft/homebrew-apm) tap updates independently: it polls the latest APM release and commits formula updates with its own repository-scoped `GITHUB_TOKEN`. The release pipeline does not hold a cross-repository Homebrew credential. - -### Test Matrix - -Promotion integration tests run on: -- **Linux**: ubuntu-24.04 (x86_64), ubuntu-24.04-arm (arm64) -- **Windows**: windows-latest (x86_64) -- **macOS Intel**: macos-15-intel (x86_64), with marker-bounded `lifecycle_smoke and not live` integration coverage, native binary startup, and isolated release validation -- **macOS Apple Silicon**: macos-latest (arm64), with the full non-live integration corpus - -**Python Version**: 3.12 (standardized across all environments) -**Package Manager**: uv (for fast dependency management and virtual environments) - -## What the Tests Verify - -### Smoke Tests Verify: -- [+] Runtime setup scripts execute successfully -- [+] Binaries are downloaded and installed correctly -- [+] Binaries respond to basic commands -- [+] APM can detect installed runtimes -- [+] Configuration files are created properly -- [+] Workflow compilation works (without execution) - -### E2E Tests Verify: -- [+] Complete golden scenario from README works -- [+] `apm runtime setup copilot` installs and configures GitHub Copilot CLI -- [+] `apm runtime setup codex` installs and configures Codex -- [+] `apm runtime setup llm` installs and configures LLM -- [+] `apm init my-hello-world` creates project correctly -- [+] `apm install` handles dependencies -- [+] `apm run start --param name="Tester"` executes successfully -- [+] Real API calls to GitHub Models work -- [+] Parameter substitution works correctly -- [+] MCP integration functions (GitHub tools) -- [+] Binary artifacts work across platforms -- [+] Release pipeline integrity (GitHub Release -> PyPI) - -### Lifecycle Smoke Verifies: +### Lifecycle Smoke verifies + - Install content-hash roundtrip (Consume contract) - Virtual-skill lock convergence (Produce contract, adjacent to the #2226 ADO lock-coordinate fix) - Policy pinned-constraint enforcement (Govern contract) @@ -474,39 +516,41 @@ Promotion integration tests run on: - Prune's merged-hook and ownership-sidecar reconciliation for the `claude` target (the direct #2249 regression -- an orphaned package's merged hook entries and sidecar markers must be cleaned up, not left pointing at deleted scripts) - No network, no credentials, no built binary required for any of the above -## Benefits +### Timing and performance capture -### **Speed vs Confidence Balance** -- **Smoke tests**: Fast feedback (2-3 min) on every change -- **E2E tests**: High confidence (15 min) only when shipping +CI pytest lanes record `--durations=50`, JUnit XML, and `.test_durations`; +`test-results-*` artifacts contain the files that lane writes. -### **Cost Efficiency** -- Smoke tests use no API credits -- E2E tests only run on releases (minimizing API usage) -- Manual workflow dispatch for test builds without publishing +Do not restore mutable per-shard timing histories: different partition maps can +omit tests. Merge-queue integration freezes one immutable `.test_durations` +artifact in the existing build job, with no extra CI job; all four shards +download that identical snapshot. PR unit shards record timings and JUnit only. +They do not restore a rolling cache or present the shards as timing-balanced: +the shared planner job required for that can cost more runner allocation time +than it saves in balancing. -### **Platform Coverage** -- Tests run on all supported platforms -- Catches platform-specific runtime issues +Native full-suite release lanes may restore timing history through +`.github/actions/pytest-timing`, keyed by OS, architecture, suite, commit SHA, +and partition. It remains a hint only: no test-result cache, test selection, or +allowlist. Grouped tests still use `--dist loadgroup` so HOME-mutating fixtures +stay on one worker. -### **Release Confidence** -- E2E tests must pass before any publishing steps -- Multi-stage release pipeline ensures quality gates -- Guarantees shipped releases work end-to-end -- Users can trust the README golden scenario -- Cross-platform binary verification +Inspect expensive fixtures by opening the slowest `--durations=50` entries and +correlating fixture-heavy node IDs with JUnit XML for the same OS, architecture, +suite, and shard. Do not claim release speedups until optimized hosted data +exists; the current tag baseline is 39m16s. ## Debugging Test Failures ### Smoke Test Failures -- Check runtime setup script output -- Verify platform compatibility -- Check network connectivity for downloads +- Check the core smoke or runtime smoke step that failed. +- Verify the native binary path and platform match the selected lane. +- For runtime smoke, verify the relevant opt-in and runtime setup logs. ### E2E Test Failures - **Use the unified integration script first**: Run `./scripts/test-integration.sh` to reproduce the exact CI environment locally - Verify `GITHUB_TOKEN` has required permissions (`models:read`) -- Ensure both `GITHUB_TOKEN` and `GITHUB_MODELS_KEY` environment variables are set +- Ensure the required GitHub and ADO tokens for the selected markers are set - Check GitHub Models API availability - Review actual vs expected output - Test locally with same environment diff --git a/scripts/build-binary.sh b/scripts/build-binary.sh index 702d61f910..db1b7bb2ba 100755 --- a/scripts/build-binary.sh +++ b/scripts/build-binary.sh @@ -55,13 +55,6 @@ if ! uv run pyinstaller --version &> /dev/null; then exit 1 fi -# Check if UPX is available (optional, for compression) -if command -v upx &> /dev/null; then - echo -e "${GREEN}UPX found - binary will be compressed${NC}" -else - echo -e "${YELLOW}UPX not found - binary will not be compressed (install with: brew install upx)${NC}" -fi - # Inject build SHA into version.py VERSION_FILE="src/apm_cli/version.py" BUILD_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "") diff --git a/scripts/package_release.py b/scripts/package_release.py new file mode 100644 index 0000000000..1d9a821359 --- /dev/null +++ b/scripts/package_release.py @@ -0,0 +1,185 @@ +"""Package once, then verify and extract the exact native release candidate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import subprocess +import tarfile +import zipfile +from pathlib import Path + +import tomllib + +BINARY_NAMES = tuple( + row["binary_name"] + for row in json.loads(Path(__file__).with_name("release-platforms.json").read_text("ascii")) +) + + +def file_digest(path: Path) -> str: + """Hash a file without loading the binary bundle into memory.""" + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def archive_name(binary_name: str) -> str: + """Return the public archive name for one supported native platform.""" + if binary_name not in BINARY_NAMES: + raise ValueError(f"Unsupported release binary: {binary_name}") + suffix = ".zip" if binary_name == "apm-windows-x86_64" else ".tar.gz" + return binary_name + suffix + + +def executable_name(binary_name: str) -> str: + """Return the platform's native executable filename.""" + return "apm.exe" if binary_name == "apm-windows-x86_64" else "apm" + + +def require_sha(sha: str) -> None: + """Reject abbreviated or malformed candidate identities.""" + if re.fullmatch(r"[0-9a-f]{40}", sha) is None: + raise ValueError("Candidate SHA must be a full lowercase Git commit SHA") + + +def require_binary_identity(executable: Path, version: str, sha: str) -> None: + """Reject an old or unbound executable even when its archive has valid hashes.""" + result = subprocess.run( # noqa: S603 - executes only this trusted native build. + [str(executable.resolve()), "--version"], + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + check=True, + env={**os.environ, "NO_COLOR": "1", "COLUMNS": "200"}, + ) + match = re.search(rf"\bversion {re.escape(version)}\s+\(([0-9a-f]{{7,40}})\)", result.stdout) + if match is None or not sha.startswith(match.group(1)): + raise ValueError("Candidate executable version/build SHA does not match the checkout") + + +def package( + binary_name: str, dist: Path, output: Path, sha: str, pyproject: Path +) -> dict[str, str | int]: + """Create the archive that both native validation and publication consume.""" + require_sha(sha) + name = archive_name(binary_name) + bundle = dist / binary_name + executable = bundle / executable_name(binary_name) + if not executable.is_file(): + raise FileNotFoundError(f"Candidate executable not found: {executable}") + if executable.name == "apm": + executable.chmod(executable.stat().st_mode | 0o111) + version = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"]["version"] + require_binary_identity(executable, version, sha) + output.mkdir(parents=True, exist_ok=True) + archive = output / name + if archive.suffix == ".zip": + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as target: + for path in sorted(bundle.rglob("*")): + target.write(path, path.relative_to(dist).as_posix()) + else: + with tarfile.open(archive, "w:gz") as target: + target.add(bundle, arcname=binary_name) + digest = file_digest(archive) + metadata: dict[str, str | int] = { + "schema_version": 1, + "sha": sha, + "version": version, + "binary_name": binary_name, + "archive": name, + "archive_sha256": digest, + "executable_sha256": file_digest(executable), + } + (output / f"{name}.sha256").write_text(f"{digest} {name}\n", encoding="ascii") + (output / f"{binary_name}.json").write_text( + json.dumps(metadata, sort_keys=True, indent=2) + "\n", encoding="ascii" + ) + # Signing can change the executable after the build script wrote its checksum. + (dist / f"{binary_name}.sha256").write_text( + f"{metadata['executable_sha256']} {binary_name}/{executable.name}\n", + encoding="ascii", + ) + return metadata + + +def verify_extract(binary_name: str, archives: Path, destination: Path, sha: str) -> Path: + """Fail closed on identity/hash drift before extracting a candidate archive.""" + require_sha(sha) + name = archive_name(binary_name) + metadata = json.loads((archives / f"{binary_name}.json").read_text(encoding="ascii")) + for key, expected in ( + ("schema_version", 1), + ("sha", sha), + ("binary_name", binary_name), + ("archive", name), + ): + if metadata.get(key) != expected: + raise ValueError(f"Candidate {key} does not match {expected!r}") + archive = archives / name + digest = file_digest(archive) + if metadata.get("archive_sha256") != digest: + raise ValueError(f"Candidate archive checksum mismatch: {name}") + checksum = (archives / f"{name}.sha256").read_text(encoding="ascii") + if checksum != f"{digest} {name}\n": + raise ValueError(f"Candidate checksum sidecar mismatch: {name}") + bundle = destination / binary_name + if bundle.exists(): + raise FileExistsError(f"Candidate extraction destination already exists: {bundle}") + destination.mkdir(parents=True, exist_ok=True) + if archive.suffix == ".zip": + with zipfile.ZipFile(archive) as source: + members = source.infolist() + for item in members: + require_member_path(item.filename, binary_name) + if stat.S_ISLNK(item.external_attr >> 16): + raise ValueError("Candidate ZIP contains an unsupported symbolic link") + for item in members: + source.extract(item, destination) + else: + with tarfile.open(archive, "r:gz") as source: + for item in source.getmembers(): + require_member_path(item.name, binary_name) + if item.issym() or item.islnk(): + parent = (destination / item.name).parent if item.issym() else destination + if not (parent / item.linkname).resolve().is_relative_to(bundle.resolve()): + raise ValueError("Candidate archive link escapes its native bundle") + source.extractall(destination, filter="data") + executable = bundle / executable_name(binary_name) + if file_digest(executable) != metadata.get("executable_sha256"): + raise ValueError("Extracted candidate executable checksum mismatch") + require_binary_identity(executable, metadata["version"], sha) + return executable + + +def require_member_path(name: str, binary_name: str) -> None: + """Keep every archive entry inside its one native bundle on either OS.""" + parts = name.rstrip("/").split("/") + if parts[0] != binary_name or any( + part in {"", ".", ".."} or "\\" in part or ":" in part for part in parts + ): + raise ValueError("Candidate archive contains an unsafe member path") + + +def main() -> None: + """Expose packaging and isolated candidate validation to release jobs.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("pack", "verify-extract")) + parser.add_argument("--binary-name", required=True, choices=BINARY_NAMES) + parser.add_argument("--sha", required=True) + parser.add_argument("--dist", type=Path, default=Path("dist")) + parser.add_argument("--archives", type=Path, default=Path("release-assets")) + parser.add_argument("--pyproject", type=Path, default=Path("pyproject.toml")) + args = parser.parse_args() + if args.operation == "pack": + package(args.binary_name, args.dist, args.archives, args.sha, args.pyproject) + else: + verify_extract(args.binary_name, args.archives, args.dist, args.sha) + + +if __name__ == "__main__": + main() diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs new file mode 100644 index 0000000000..5d936730a6 --- /dev/null +++ b/scripts/release-candidate.cjs @@ -0,0 +1,775 @@ +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); + +const REPO = "microsoft/apm"; +const MAIN_BRANCH = "main"; +const WORKFLOW_PATH = ".github/workflows/build-release.yml"; +const EVIDENCE_BASENAME = "release-candidate-evidence"; +const TRUSTED_REUSE_EVENTS = new Set(["schedule", "repository_dispatch"]); +const CURRENT_RUN_EVENTS = new Set(["push", "schedule", "repository_dispatch"]); +const MAX_CANDIDATE_AGE_DAYS = 30; +const RERUN_ALL_JOBS_HINT = "Use Re-run all jobs to regenerate complete candidate evidence for this attempt"; +const CONFIG_HASH_FILES = [ + ".github/workflows/build-release.yml", + ".github/workflows/release-platform.yml", + ".github/workflows/ci.yml", + "scripts/package_release.py", + "scripts/release-candidate.cjs", + "scripts/release-platforms.json", +]; +const REQUIRED_SOURCE_JOB_NAMES = [ + "Candidate Source Checks / Lint", + "Candidate Source Checks / Windows Compatibility Gate", + "Candidate Source Checks / Test Architecture Ratchets", + "Candidate Source Checks / Build & Test Shard 1 (Linux)", + "Candidate Source Checks / Build & Test Shard 2 (Linux)", + "Candidate Source Checks / Coverage Combine (Linux)", + "Candidate Source Checks / APM Self-Check", + "Candidate Source Checks / Lifecycle Smoke (Linux)", +]; +const REQUIRED_SOURCE_JOB_FRAGMENTS = REQUIRED_SOURCE_JOB_NAMES; +const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +class CandidateUnavailableError extends Error {} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function loadCatalog() { + return readJson(path.join(__dirname, "release-platforms.json")); +} + +function archiveName(binaryName) { + return `${binaryName}${binaryName === "apm-windows-x86_64" ? ".zip" : ".tar.gz"}`; +} + +function assertFullSha(sha, label = "sha") { + if (!/^[0-9a-f]{40}$/.test(String(sha || ""))) { + throw new Error(`${label} must be a full lowercase 40-character commit SHA`); + } +} + +function repoFullName(context) { + if (context?.repo?.owner && context?.repo?.repo) { + return `${context.repo.owner}/${context.repo.repo}`; + } + return context?.payload?.repository?.full_name || process.env.GITHUB_REPOSITORY || ""; +} + +function repoParts(context) { + const full = repoFullName(context); + const [owner, repo] = full.split("/"); + return { owner, repo }; +} + +function eventName(context) { + return context?.eventName || context?.event_name || process.env.GITHUB_EVENT_NAME || ""; +} + +function refName(ref) { + if (!ref) return ""; + if (ref.startsWith("refs/heads/")) return ref.slice("refs/heads/".length); + if (ref.startsWith("refs/tags/")) return ref.slice("refs/tags/".length); + return ref.split("/").pop() || ""; +} + +function isTagPush(context) { + return eventName(context) === "push" && String(context?.ref || "").startsWith("refs/tags/v"); +} + +function isPrereleaseTag(context) { + return isTagPush(context) && refName(context.ref).includes("-"); +} + +function tagVersion(context) { + if (!isTagPush(context)) return null; + return refName(context.ref).replace(/^v/, ""); +} + +function requiresFullValidation(context) { + const event = eventName(context); + return isTagPush(context) || event === "schedule" || event === "repository_dispatch"; +} + +function matrixForContext(context, catalog = loadCatalog()) { + const full = requiresFullValidation(context); + return { + matrix: { include: full ? catalog : catalog.filter((row) => row.on_main !== false) }, + fullValidation: full, + }; +} + +function digestFile(file) { + const hash = crypto.createHash("sha256"); + hash.update(fs.readFileSync(file)); + return hash.digest("hex"); +} + +function configHashes(root = process.cwd()) { + const hashes = {}; + for (const relative of CONFIG_HASH_FILES) { + const file = path.join(root, relative); + if (!fs.existsSync(file)) { + throw new Error(`Required release config file is missing: ${relative}`); + } + hashes[relative] = `sha256:${digestFile(file)}`; + } + return hashes; +} + +function walkFiles(root) { + if (!fs.existsSync(root)) return []; + const files = []; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const full = path.join(root, entry.name); + if (entry.isDirectory()) { + files.push(...walkFiles(full)); + } else if (entry.isFile()) { + files.push(full); + } + } + return files; +} + +function findOneFile(root, filename) { + const matches = walkFiles(root).filter((file) => path.basename(file) === filename); + if (matches.length !== 1) { + throw new Error(`Expected exactly one ${filename} under ${root}, found ${matches.length}`); + } + return matches[0]; +} + +function candidateArtifactName(binaryName, runAttempt) { + return `candidate-${requireRunAttempt(runAttempt)}-${binaryName}`; +} + +function evidenceArtifactName(runAttempt) { + return `${EVIDENCE_BASENAME}-${requireRunAttempt(runAttempt)}`; +} + +function requireRunAttempt(attempt) { + if (!/^[1-9][0-9]*$/.test(String(attempt || ""))) { + throw new Error("Workflow run attempt must be present and numeric"); + } + return Number(attempt); +} + +function requireArtifactDigest(digest, label) { + if (!ARTIFACT_DIGEST_PATTERN.test(String(digest || ""))) { + throw new Error(`${label} is missing a valid immutable sha256 digest`); + } +} + +function requireFreshArtifact(artifact, label) { + if (artifact.expired === true) throw new CandidateUnavailableError(`${label} artifact is expired`); + requireArtifactDigest(artifact.digest, label); +} + +function findOneArtifactByName(artifacts, name) { + const matches = artifacts.filter((artifact) => artifact.name === name); + if (matches.length !== 1) { + const ErrorType = matches.length === 0 ? CandidateUnavailableError : Error; + throw new ErrorType( + `Expected exactly one workflow artifact named ${name}, found ${matches.length}. ${RERUN_ALL_JOBS_HINT}.`, + ); + } + return matches[0]; +} + +function addAttemptHint(error) { + if (String(error?.message || "").includes(RERUN_ALL_JOBS_HINT)) return error; + return new Error(`${error.message}. ${RERUN_ALL_JOBS_HINT}.`); +} + +async function paginate(github, method, params, collectionName) { + if (typeof github.paginate === "function") { + return await github.paginate(method, params); + } + const response = await method(params); + return response?.data?.[collectionName] || []; +} + +async function listWorkflowRuns(github, params) { + return await paginate( + github, + github.rest.actions.listWorkflowRuns, + params, + "workflow_runs", + ); +} + +async function listArtifacts(github, params) { + return await paginate( + github, + github.rest.actions.listWorkflowRunArtifacts, + params, + "artifacts", + ); +} + +async function listJobs(github, params) { + const actions = github.rest.actions; + if (params.attempt_number && actions.listJobsForWorkflowRunAttempt) { + return await paginate(github, actions.listJobsForWorkflowRunAttempt, params, "jobs"); + } + const { attempt_number: _attempt, ...withoutAttempt } = params; + return await paginate(github, actions.listJobsForWorkflowRun, withoutAttempt, "jobs"); +} + +async function getWorkflowRun(github, params) { + const response = await github.rest.actions.getWorkflowRun(params); + return response.data; +} + +function validateTrustedReusableRun( + run, + expectedSha, + { now = Date.now(), maxAgeDays = MAX_CANDIDATE_AGE_DAYS } = {}, +) { + assertFullSha(expectedSha, "expectedSha"); + if (!run) throw new Error("Candidate run was not found"); + requireRunAttempt(run.run_attempt); + if (run.event === "workflow_dispatch" || run.event === "pull_request") { + throw new Error(`Candidate run event ${run.event} is not trusted for release promotion`); + } + if (!run.path) throw new Error("Candidate run is missing workflow path metadata"); + if (run.path !== WORKFLOW_PATH) { + throw new Error(`Candidate run workflow path ${run.path} is not ${WORKFLOW_PATH}`); + } + if (!TRUSTED_REUSE_EVENTS.has(run.event)) { + throw new Error(`Candidate run event ${run.event} is not promotable`); + } + if (run.head_sha !== expectedSha) { + throw new Error("Candidate run SHA does not match tag SHA"); + } + if (!run.head_branch) throw new Error("Candidate run is missing head_branch metadata"); + if (run.head_branch !== MAIN_BRANCH) { + throw new Error(`Candidate run branch ${run.head_branch} is not ${MAIN_BRANCH}`); + } + if (run.repository?.full_name && run.repository.full_name !== REPO) { + throw new Error(`Candidate run repository ${run.repository.full_name} is not ${REPO}`); + } + if (run.status !== "completed" || run.conclusion !== "success") { + throw new Error("Candidate run did not complete successfully"); + } + const completedAt = Date.parse(run.updated_at || run.run_started_at || run.created_at || ""); + if (!Number.isFinite(completedAt)) { + throw new Error("Candidate run is missing timestamp metadata"); + } + if (now - completedAt > maxAgeDays * 24 * 60 * 60 * 1000) { + throw new CandidateUnavailableError(`Candidate run is older than ${maxAgeDays} days`); + } +} + +async function candidateFromRun({ + github, + owner, + repo, + run, + expectedSha, + catalog, +}) { + validateTrustedReusableRun(run, expectedSha); + const artifacts = await listArtifacts(github, { + owner, + repo, + run_id: run.id, + per_page: 100, + }); + const attempt = requireRunAttempt(run.run_attempt); + const evidence = findOneArtifactByName(artifacts, evidenceArtifactName(attempt)); + requireFreshArtifact(evidence, evidence.name); + + const platformArtifacts = []; + for (const row of catalog) { + const artifact = findOneArtifactByName(artifacts, candidateArtifactName(row.binary_name, attempt)); + requireFreshArtifact(artifact, artifact.name); + platformArtifacts.push(artifact); + } + return { + run, + evidence, + artifacts: platformArtifacts, + }; +} + +async function discoverCandidateRun({ github, context, sha, catalog = loadCatalog(), core }) { + assertFullSha(sha, "tag SHA"); + const { owner, repo } = repoParts(context); + const createdSince = new Date(Date.now() - MAX_CANDIDATE_AGE_DAYS * 24 * 60 * 60 * 1000) + .toISOString() + .slice(0, 10); + const runs = await listWorkflowRuns(github, { + owner, + repo, + workflow_id: path.basename(WORKFLOW_PATH), + branch: MAIN_BRANCH, + head_sha: sha, + created: `>=${createdSince}`, + status: "completed", + per_page: 100, + }); + const sameSha = runs + .filter((run) => run.head_sha === sha) + .filter((run) => TRUSTED_REUSE_EVENTS.has(run.event)) + .filter((run) => run.conclusion === "success" && run.status === "completed") + .sort((a, b) => String(b.run_started_at || b.created_at || "").localeCompare(String(a.run_started_at || a.created_at || ""))); + + if (sameSha.length === 0) return null; + try { + return await candidateFromRun({ + github, + owner, + repo, + run: sameSha[0], + expectedSha: sha, + catalog, + }); + } catch (error) { + if (!(error instanceof CandidateUnavailableError)) throw error; + core?.info?.(`Prior candidate unavailable; building a fresh full candidate: ${error.message}`); + return null; + } +} + +async function resolveExplicitCandidateRun({ github, context, runId, sha, catalog = loadCatalog() }) { + const { owner, repo } = repoParts(context); + if (!/^[1-9][0-9]*$/.test(String(runId || ""))) { + throw new Error("candidate_run_id must be a numeric workflow run id"); + } + const run = await getWorkflowRun(github, { owner, repo, run_id: Number(runId) }); + const candidate = await candidateFromRun({ github, owner, repo, run, expectedSha: sha, catalog }); + if (!candidate) { + throw new Error(`Candidate run ${runId} does not contain release-candidate evidence`); + } + return candidate; +} + +function collectArchiveInventory({ artifactRoot, catalog, sha }) { + assertFullSha(sha, "candidate SHA"); + const platforms = {}; + for (const row of catalog) { + const binaryName = row.binary_name; + const archive = archiveName(binaryName); + const archivePath = findOneFile(artifactRoot, archive); + const sidecarPath = findOneFile(artifactRoot, `${archive}.sha256`); + const metadataPath = findOneFile(artifactRoot, `${binaryName}.json`); + const metadata = readJson(metadataPath); + for (const [key, expected] of Object.entries({ + schema_version: 1, + sha, + binary_name: binaryName, + archive, + })) { + if (metadata[key] !== expected) { + throw new Error(`Candidate ${binaryName} metadata ${key} does not match ${expected}`); + } + } + const digest = digestFile(archivePath); + if (metadata.archive_sha256 !== digest) { + throw new Error(`Candidate archive digest mismatch for ${archive}`); + } + if (typeof metadata.version !== "string" || metadata.version.length === 0) { + throw new Error(`Candidate ${binaryName} metadata version is missing`); + } + if (!/^[0-9a-f]{64}$/.test(String(metadata.executable_sha256 || ""))) { + throw new Error(`Candidate ${binaryName} executable digest is missing or malformed`); + } + const sidecar = fs.readFileSync(sidecarPath, "utf8"); + if (sidecar !== `${digest} ${archive}\n`) { + throw new Error(`Candidate ${archive} checksum sidecar does not match`); + } + platforms[binaryName] = { + runner: row.runner, + platform: row.platform, + arch: row.arch || null, + version: metadata.version, + executable_sha256: metadata.executable_sha256, + artifact_id: null, + artifact_name: null, + artifact_size: null, + artifact_digest: null, + archive, + archive_sha256: digest, + sidecar: `${archive}.sha256`, + metadata: `${binaryName}.json`, + }; + } + return platforms; +} + +function candidateVersion(platforms, context) { + const versions = new Set(Object.values(platforms).map((platform) => platform.version)); + if (versions.size !== 1) { + throw new Error("Candidate platform metadata versions do not match"); + } + const version = [...versions][0]; + const expectedTagVersion = tagVersion(context); + if (expectedTagVersion && version !== expectedTagVersion) { + throw new Error(`Candidate version ${version} does not match tag v${expectedTagVersion}`); + } + return version; +} + +function attachArtifactMetadata(platforms, catalog, artifacts, runAttempt) { + const attempt = requireRunAttempt(runAttempt); + for (const row of catalog) { + const binaryName = row.binary_name; + const artifact = findOneArtifactByName(artifacts, candidateArtifactName(binaryName, attempt)); + requireFreshArtifact(artifact, artifact.name); + platforms[binaryName].artifact_id = artifact.id; + platforms[binaryName].artifact_name = artifact.name; + platforms[binaryName].artifact_size = artifact.size_in_bytes || null; + platforms[binaryName].artifact_digest = artifact.digest; + } +} + +function normalizeRequiredJobs(jobs) { + return (jobs || []).map((job) => { + if (typeof job === "string") return { name: job, conclusion: null, status: null }; + return { + name: String(job.name || ""), + conclusion: job.conclusion || null, + status: job.status || null, + }; + }); +} + +function selectRequiredJobs(jobs, catalog) { + const normalized = normalizeRequiredJobs(jobs); + const selected = []; + function select(expectedName) { + const matches = normalized.filter((job) => job.name === expectedName); + if (matches.length !== 1) { + const states = matches + .map((job) => `${job.name}:${job.status || "unknown"}/${job.conclusion || "none"}`) + .join(", "); + throw new Error(`Candidate required job evidence expected exactly one ${expectedName}${states ? ` (${states})` : ""}`); + } + const [job] = matches; + if (job.status !== "completed" || job.conclusion !== "success") { + throw new Error( + `Candidate required job ${expectedName} was ${job.status || "unknown"}/${job.conclusion || "none"}`, + ); + } + selected.push({ name: job.name, status: "completed", conclusion: "success" }); + } + for (const jobName of REQUIRED_SOURCE_JOB_NAMES) select(jobName); + for (const row of catalog) select(`${row.binary_name} / Native Candidate Gate`); + return selected; +} + +function assertRequiredJobs(requiredJobs, catalog) { + const jobs = normalizeRequiredJobs(requiredJobs); + if (jobs.length === 0) throw new Error("Candidate manifest contains no required job evidence"); + selectRequiredJobs(jobs, catalog); +} + +async function qualify({ + github, + context, + core, + artifactRoot = "candidate-artifacts", + evidencePath = "release-candidate-evidence.json", +}) { + const catalog = loadCatalog(); + const { owner, repo } = repoParts(context); + const runId = Number(context.runId || process.env.GITHUB_RUN_ID); + const runAttempt = Number(context.runAttempt || process.env.GITHUB_RUN_ATTEMPT || 1); + const sha = context.sha; + assertFullSha(sha, "candidate SHA"); + if (repoFullName(context) !== REPO) throw new Error(`Release candidates must be produced in ${REPO}`); + if (!CURRENT_RUN_EVENTS.has(eventName(context))) { + throw new Error(`Event ${eventName(context)} cannot produce a release candidate manifest`); + } + + let platforms; + try { + platforms = collectArchiveInventory({ artifactRoot, catalog, sha }); + } catch (error) { + throw addAttemptHint(error); + } + const version = candidateVersion(platforms, context); + const artifacts = await listArtifacts(github, { owner, repo, run_id: runId, per_page: 100 }); + try { + attachArtifactMetadata(platforms, catalog, artifacts, runAttempt); + } catch (error) { + throw addAttemptHint(error); + } + const jobs = await listJobs(github, { + owner, + repo, + run_id: runId, + attempt_number: runAttempt, + per_page: 100, + }); + const requiredJobs = selectRequiredJobs(jobs, catalog); + + const evidence = { + schema_version: 1, + repo: REPO, + workflow_path: WORKFLOW_PATH, + event: eventName(context), + ref: context.ref, + head_branch: String(context.ref || "").startsWith("refs/heads/") ? refName(context.ref) : null, + head_sha: sha, + version, + run_id: runId, + run_attempt: runAttempt, + full_validation: true, + generated_at: new Date().toISOString(), + config_hashes: configHashes(process.cwd()), + required_jobs: requiredJobs, + platforms, + }; + fs.writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + core?.setOutput?.( + "candidate_artifact_ids", + Object.values(platforms).map((platform) => String(platform.artifact_id)).join(","), + ); + core?.setOutput?.("release_version", version); + core?.setOutput?.("candidate_sha", sha); + core?.info?.(`Recorded release candidate evidence for ${sha}`); + return evidence; +} + +function assertEvidenceIdentity(evidence, { context, runId }) { + if (evidence.schema_version !== 1) throw new Error("Unsupported candidate evidence schema"); + if (evidence.repo !== REPO) throw new Error("Candidate evidence repo mismatch"); + if (evidence.workflow_path !== WORKFLOW_PATH) throw new Error("Candidate evidence workflow mismatch"); + if (Number(evidence.run_id) !== Number(runId)) throw new Error("Candidate evidence run_id mismatch"); + if (evidence.head_sha !== context.sha) throw new Error("Candidate evidence SHA does not match tag SHA"); + if (evidence.full_validation !== true) throw new Error("Candidate evidence is not full validation"); + const expectedTagVersion = tagVersion(context); + if (!expectedTagVersion) throw new Error("Release candidate verification requires a v* tag context"); + if (evidence.version !== expectedTagVersion) { + throw new Error(`Candidate evidence version ${evidence.version} does not match tag v${expectedTagVersion}`); + } +} + +function assertConfigHashes(evidence, root = process.cwd()) { + const actual = configHashes(root); + for (const [file, digest] of Object.entries(actual)) { + if (evidence.config_hashes?.[file] !== digest) { + throw new Error(`Candidate config hash mismatch for ${file}`); + } + } +} + +function assertRunMatchesEvidence(run, evidence, context) { + const sameRun = Number(evidence.run_id) === Number(context.runId || process.env.GITHUB_RUN_ID); + if (!run) throw new Error("Workflow run metadata was not found"); + requireRunAttempt(run.run_attempt); + if (!run.path) throw new Error("Workflow run is missing workflow path metadata"); + if (run.path !== WORKFLOW_PATH) throw new Error(`Workflow run path ${run.path} is not ${WORKFLOW_PATH}`); + if (!run.head_branch) throw new Error("Workflow run is missing head_branch metadata"); + if (run.repository?.full_name && run.repository.full_name !== REPO) { + throw new Error(`Workflow run repository ${run.repository.full_name} is not ${REPO}`); + } + if (run.head_sha !== evidence.head_sha) throw new Error("Workflow run SHA does not match evidence"); + if (run.event !== evidence.event) throw new Error("Workflow run event does not match evidence"); + if (Number(run.run_attempt) !== Number(evidence.run_attempt)) { + throw new Error("Workflow run attempt does not match evidence"); + } + if (sameRun) { + if (!CURRENT_RUN_EVENTS.has(evidence.event)) { + throw new Error(`Current run event ${evidence.event} cannot publish a candidate`); + } + if (!["in_progress", "completed"].includes(run.status)) { + throw new Error(`Current workflow run status ${run.status} cannot publish a candidate`); + } + if (evidence.event !== eventName(context) || evidence.ref !== context.ref) { + throw new Error("Current run evidence does not match this workflow invocation"); + } + return; + } + if (evidence.ref !== `refs/heads/${MAIN_BRANCH}` || evidence.head_branch !== MAIN_BRANCH) { + throw new Error(`Reusable candidate evidence must come from ${MAIN_BRANCH}`); + } + validateTrustedReusableRun(run, evidence.head_sha); +} + +function assertArtifactMetadata(evidence, catalog, artifacts) { + const expectedPlatformNames = catalog.map((row) => row.binary_name).sort(); + const actualPlatformNames = Object.keys(evidence.platforms || {}).sort(); + if (JSON.stringify(actualPlatformNames) !== JSON.stringify(expectedPlatformNames)) { + throw new Error("Candidate platform inventory is not exactly the supported release platforms"); + } + const names = new Set(); + for (const row of catalog) { + const platform = evidence.platforms?.[row.binary_name]; + if (!platform) throw new Error(`Candidate evidence missing platform ${row.binary_name}`); + const expectedArtifactName = candidateArtifactName(row.binary_name, evidence.run_attempt); + if (platform.artifact_name !== expectedArtifactName) { + throw new Error(`Candidate artifact name for ${row.binary_name} is not ${expectedArtifactName}`); + } + const artifact = artifacts.find((item) => Number(item.id) === Number(platform.artifact_id)); + if (!artifact) throw new Error(`Workflow artifact ${platform.artifact_id} not found for ${row.binary_name}`); + if (artifact.expired === true) throw new Error(`Workflow artifact ${artifact.name} is expired`); + if (artifact.name !== platform.artifact_name) { + throw new Error(`Workflow artifact name mismatch for ${row.binary_name}`); + } + if (platform.artifact_size != null && Number(artifact.size_in_bytes) !== Number(platform.artifact_size)) { + throw new Error(`Workflow artifact size mismatch for ${row.binary_name}`); + } + requireArtifactDigest(artifact.digest, `Workflow artifact ${artifact.name}`); + requireArtifactDigest(platform.artifact_digest, `Candidate evidence artifact ${artifact.name}`); + if (artifact.digest !== platform.artifact_digest) { + throw new Error(`Workflow artifact digest mismatch for ${row.binary_name}`); + } + names.add(platform.archive); + } + const expectedArchives = new Set(catalog.map((row) => archiveName(row.binary_name))); + if (names.size !== expectedArchives.size || [...names].some((name) => !expectedArchives.has(name))) { + throw new Error("Candidate archive inventory is not exactly the supported release platforms"); + } +} + +function verifyLocalArchiveBytes({ evidence, catalog, artifactRoot, outputRoot }) { + fs.mkdirSync(outputRoot, { recursive: true }); + const expectedReleaseFiles = new Set( + catalog.flatMap((row) => { + const archive = archiveName(row.binary_name); + return [archive, `${archive}.sha256`, `${row.binary_name}.json`]; + }), + ); + const unexpectedReleaseFiles = walkFiles(artifactRoot) + .map((file) => path.basename(file)) + .filter((name) => /^apm-/.test(name)) + .filter((name) => /\.(tar\.gz|zip|sha256|json)$/.test(name)) + .filter((name) => !expectedReleaseFiles.has(name)); + if (unexpectedReleaseFiles.length) { + throw new Error(`Candidate contains unexpected release files: ${unexpectedReleaseFiles.join(", ")}`); + } + const localInventory = collectArchiveInventory({ + artifactRoot, + catalog, + sha: evidence.head_sha, + }); + const localVersion = candidateVersion(localInventory, { eventName: "push", ref: `refs/tags/v${evidence.version}` }); + if (localVersion !== evidence.version) { + throw new Error("Candidate local archive versions do not match evidence"); + } + for (const row of catalog) { + const binaryName = row.binary_name; + const platform = evidence.platforms[binaryName]; + const archive = platform.archive; + if (localInventory[binaryName].archive_sha256 !== platform.archive_sha256) { + throw new Error(`Candidate archive digest mismatch for ${archive}`); + } + if (localInventory[binaryName].version !== platform.version) { + throw new Error(`Candidate version mismatch for ${archive}`); + } + if (localInventory[binaryName].executable_sha256 !== platform.executable_sha256) { + throw new Error(`Candidate executable digest mismatch for ${archive}`); + } + const archivePath = findOneFile(artifactRoot, archive); + const sidecarPath = findOneFile(artifactRoot, `${archive}.sha256`); + const digest = digestFile(archivePath); + if (platform.archive_sha256 !== digest) { + throw new Error(`Candidate archive digest mismatch for ${archive}`); + } + const sidecar = fs.readFileSync(sidecarPath, "utf8"); + if (sidecar !== `${digest} ${archive}\n`) { + throw new Error(`Candidate checksum sidecar mismatch for ${archive}`); + } + fs.copyFileSync(archivePath, path.join(outputRoot, archive)); + fs.copyFileSync(sidecarPath, path.join(outputRoot, `${archive}.sha256`)); + } +} + +function resolveEvidencePath(evidencePath) { + if (fs.existsSync(evidencePath) && fs.statSync(evidencePath).isDirectory()) { + return findOneFile(evidencePath, "release-candidate-evidence.json"); + } + return evidencePath; +} + +async function verify({ + github, + context, + core, + runId = process.env.CANDIDATE_RUN_ID || context.runId || process.env.GITHUB_RUN_ID, + artifactRoot = "candidate-artifacts", + evidencePath = "candidate-evidence", + outputRoot = "release-assets", +}) { + const catalog = loadCatalog(); + const { owner, repo } = repoParts(context); + const evidence = readJson(resolveEvidencePath(evidencePath)); + assertEvidenceIdentity(evidence, { context, runId }); + assertConfigHashes(evidence, process.cwd()); + assertRequiredJobs(evidence.required_jobs, catalog); + + const run = await getWorkflowRun(github, { owner, repo, run_id: Number(runId) }); + assertRunMatchesEvidence(run, evidence, context); + const jobs = await listJobs(github, { + owner, + repo, + run_id: Number(runId), + attempt_number: Number(evidence.run_attempt), + per_page: 100, + }); + selectRequiredJobs(jobs, catalog); + const artifacts = await listArtifacts(github, { + owner, + repo, + run_id: Number(runId), + per_page: 100, + }); + assertArtifactMetadata(evidence, catalog, artifacts); + verifyLocalArchiveBytes({ evidence, catalog, artifactRoot, outputRoot }); + core?.info?.(`Verified release candidate ${runId} for ${context.sha}`); + return evidence; +} + +async function plan({ github, context, core }) { + const catalog = loadCatalog(); + const { matrix, fullValidation } = matrixForContext(context, catalog); + let candidateRunId = ""; + let candidateArtifactIds = ""; + let candidateEvidenceArtifactId = ""; + if (isTagPush(context)) { + const candidate = await discoverCandidateRun({ github, context, sha: context.sha, catalog, core }); + if (candidate) { + candidateRunId = String(candidate.run.id); + candidateArtifactIds = candidate.artifacts.map((artifact) => String(artifact.id)).join(","); + candidateEvidenceArtifactId = String(candidate.evidence.id); + } + } + core.setOutput("matrix", JSON.stringify(matrix)); + core.setOutput("full_validation", String(fullValidation)); + core.setOutput("candidate_run_id", candidateRunId); + core.setOutput("candidate_artifact_ids", candidateArtifactIds); + core.setOutput("candidate_evidence_artifact_id", candidateEvidenceArtifactId); + core.setOutput("is_prerelease", String(isPrereleaseTag(context))); +} + +module.exports = { + CONFIG_HASH_FILES, + REQUIRED_SOURCE_JOB_NAMES, + REQUIRED_SOURCE_JOB_FRAGMENTS, + TRUSTED_REUSE_EVENTS, + WORKFLOW_PATH, + archiveName, + assertRequiredJobs, + candidateFromRun, + candidateVersion, + collectArchiveInventory, + configHashes, + discoverCandidateRun, + evidenceArtifactName, + matrixForContext, + plan, + qualify, + resolveEvidencePath, + resolveExplicitCandidateRun, + selectRequiredJobs, + candidateArtifactName, + validateTrustedReusableRun, + verify, +}; diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs new file mode 100644 index 0000000000..78963b6916 --- /dev/null +++ b/scripts/release-candidate.test.cjs @@ -0,0 +1,990 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { after, describe, it } = require("node:test"); + +const rc = require("./release-candidate.cjs"); + +const ROOT = path.resolve(__dirname, ".."); +const SHA = "a".repeat(40); +const OTHER_SHA = "b".repeat(40); +const NOW = new Date().toISOString(); + +function catalog() { + return JSON.parse(fs.readFileSync(path.join(__dirname, "release-platforms.json"), "ascii")); +} + +function digestBytes(bytes) { + return crypto.createHash("sha256").update(bytes).digest("hex"); +} + +function makeTemp() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "apm-release-candidate-")); + after(() => fs.rmSync(root, { recursive: true, force: true })); + return root; +} + +function writeRequiredConfig(root) { + for (const relative of rc.CONFIG_HASH_FILES) { + const file = path.join(root, relative); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const source = path.join(ROOT, relative); + fs.writeFileSync( + file, + fs.existsSync(source) ? fs.readFileSync(source) : `test fixture for ${relative}\n`, + ); + } +} + +function writeCandidateFiles(root, rows = catalog(), sha = SHA, version = "1.2.3") { + fs.mkdirSync(root, { recursive: true }); + for (const row of rows) { + const archive = rc.archiveName(row.binary_name); + const dir = path.join(root, row.binary_name, "release-assets"); + fs.mkdirSync(dir, { recursive: true }); + const archiveBytes = Buffer.from(`${row.binary_name}:${sha}`); + const digest = digestBytes(archiveBytes); + fs.writeFileSync(path.join(dir, archive), archiveBytes); + fs.writeFileSync(path.join(dir, `${archive}.sha256`), `${digest} ${archive}\n`, "ascii"); + fs.writeFileSync( + path.join(dir, `${row.binary_name}.json`), + `${JSON.stringify( + { + schema_version: 1, + sha, + version, + binary_name: row.binary_name, + archive, + archive_sha256: digest, + executable_sha256: digestBytes(Buffer.from(`exe:${row.binary_name}`)), + }, + null, + 2, + )}\n`, + "ascii", + ); + } +} + +function artifactMetadata(rows = catalog(), attempt = 1, options = {}) { + let id = options.startId || 100; + const artifacts = []; + const attemptNames = options.attemptNames !== false; + if (options.includeEvidence !== false) { + const evidenceName = attemptNames + ? `release-candidate-evidence-${attempt}` + : "release-candidate-evidence"; + artifacts.push({ + id: id++, + name: evidenceName, + size_in_bytes: 512, + expired: options.expiredEvidence || false, + digest: options.evidenceDigest || `sha256:${digestBytes(Buffer.from(evidenceName))}`, + }); + } + for (const row of rows) { + const name = attemptNames ? `candidate-${attempt}-${row.binary_name}` : row.binary_name; + artifacts.push({ + id: id++, + name, + size_in_bytes: 1000 + id, + expired: options.expiredBinary === row.binary_name, + digest: + options.digestPrefix === null + ? null + : `sha256:${digestBytes(Buffer.from(`${options.digestPrefix || ""}${name}`))}`, + }); + } + return artifacts; +} + +function requiredJobs(rows = catalog(), mutate = (jobs) => jobs) { + const jobs = [ + ...rc.REQUIRED_SOURCE_JOB_FRAGMENTS.map((name) => ({ + name, + status: "completed", + conclusion: "success", + })), + ...rows.map((row) => ({ + name: `${row.binary_name} / Native Candidate Gate`, + status: "completed", + conclusion: "success", + })), + ]; + return mutate(jobs); +} + +function context(overrides = {}) { + return { + eventName: "push", + ref: "refs/tags/v1.2.3", + sha: SHA, + runId: 999, + runAttempt: 1, + repo: { owner: "microsoft", repo: "apm" }, + payload: { repository: { full_name: "microsoft/apm" } }, + ...overrides, + }; +} + +function fakeGithub({ runs = [], artifactsByRun = {}, jobsByRun = {}, runById = {}, calls = [] } = {}) { + return { + rest: { + actions: { + listWorkflowRuns: async (params) => { + calls.push(["listWorkflowRuns", params]); + return { data: { workflow_runs: runs } }; + }, + listWorkflowRunArtifacts: async (params) => { + calls.push(["listWorkflowRunArtifacts", params]); + return { data: { artifacts: artifactsByRun[Number(params.run_id)] || [] } }; + }, + listJobsForWorkflowRun: async (params) => { + calls.push(["listJobsForWorkflowRun", params]); + return { data: { jobs: jobsByRun[Number(params.run_id)] || [] } }; + }, + listJobsForWorkflowRunAttempt: async (params) => { + calls.push(["listJobsForWorkflowRunAttempt", params]); + return { data: { jobs: jobsByRun[Number(params.run_id)] || [] } }; + }, + getWorkflowRun: async (params) => { + calls.push(["getWorkflowRun", params]); + return { data: runById[Number(params.run_id)] }; + }, + }, + }, + }; +} + +function withCwd(dir, fn) { + const old = process.cwd(); + process.chdir(dir); + return Promise.resolve() + .then(fn) + .finally(() => process.chdir(old)); +} + +describe("release candidate planning", () => { + for (const unavailable of ["missing evidence", "missing platform", "expired evidence", "expired platform", "old run"]) { + it(`builds fresh when automatic discovery finds ${unavailable}`, async () => { + const outputs = {}; + const messages = []; + const artifacts = artifactMetadata(catalog(), 1, { + includeEvidence: unavailable !== "missing evidence", + expiredEvidence: unavailable === "expired evidence", + expiredBinary: unavailable === "expired platform" ? "apm-linux-x86_64" : undefined, + }); + if (unavailable === "missing platform") artifacts.pop(); + const run = { + id: 123, event: "schedule", path: rc.WORKFLOW_PATH, head_sha: SHA, + head_branch: "main", status: "completed", conclusion: "success", run_attempt: 1, + run_started_at: unavailable === "old run" + ? new Date(Date.now() - 31 * 86400000).toISOString() : NOW, + }; + await rc.plan({ + github: fakeGithub({ runs: [run], artifactsByRun: { 123: artifacts } }), + context: context(), + core: { + setOutput(name, value) { outputs[name] = value; }, + info(message) { messages.push(message); }, + }, + }); + assert.equal(outputs.full_validation, "true"); + assert.equal(JSON.parse(outputs.matrix).include.length, 5); + assert.equal(outputs.candidate_run_id, ""); + assert.equal(outputs.candidate_artifact_ids, ""); + assert.equal(outputs.candidate_evidence_artifact_id, ""); + assert.match(messages[0], /building a fresh full candidate/); + }); + } + + it("keeps main push matrix partial and non-promotable", () => { + const { matrix, fullValidation } = rc.matrixForContext( + context({ eventName: "push", ref: "refs/heads/main" }), + catalog(), + ); + + assert.equal(fullValidation, false); + assert.equal(matrix.include.length, 4); + assert.deepEqual( + matrix.include.filter((row) => row.on_main === false).map((row) => row.binary_name), + [], + ); + }); + + it("uses all platforms for schedule, dispatch, and tag full validation", () => { + for (const ctx of [ + context({ eventName: "schedule", ref: "refs/heads/main" }), + context({ eventName: "repository_dispatch", ref: "refs/heads/main" }), + context({ eventName: "push", ref: "refs/tags/v1.2.3" }), + ]) { + const { matrix, fullValidation } = rc.matrixForContext(ctx, catalog()); + assert.equal(fullValidation, true); + assert.equal(matrix.include.length, 5); + } + }); + + it("outputs prior candidate run and artifact IDs for exact-SHA tag promotion", async () => { + const outputs = {}; + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + const github = fakeGithub({ + runs: [run], + artifactsByRun: { 123: artifactMetadata(catalog()) }, + }); + + await rc.plan({ + github, + context: context(), + core: { setOutput(name, value) { outputs[name] = value; } }, + }); + + assert.equal(outputs.candidate_run_id, "123"); + assert.match(outputs.candidate_evidence_artifact_id, /^[0-9]+$/); + assert.equal(outputs.candidate_artifact_ids.split(",").length, 5); + }); +}); + +describe("trusted candidate discovery", () => { + it("discovers exact-SHA schedule candidates with complete immutable artifact metadata", async () => { + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + const github = fakeGithub({ + runs: [run], + artifactsByRun: { 123: artifactMetadata(catalog()) }, + }); + + const selected = await rc.discoverCandidateRun({ github, context: context(), sha: SHA }); + + assert.equal(selected.run.id, 123); + }); + + it("narrows candidate lookup by exact SHA and bounded creation window", async () => { + const calls = []; + const github = fakeGithub({ calls }); + + await rc.discoverCandidateRun({ github, context: context(), sha: SHA }); + + const [, params] = calls.find(([name]) => name === "listWorkflowRuns"); + assert.equal(params.workflow_id, "build-release.yml"); + assert.equal(params.branch, "main"); + assert.equal(params.head_sha, SHA); + assert.match(params.created, /^>=\d{4}-\d{2}-\d{2}$/); + }); + + it("ignores ordinary push-main artifacts even when they share the tag SHA", async () => { + const run = { + id: 123, + event: "push", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + const github = fakeGithub({ + runs: [run], + artifactsByRun: { 123: artifactMetadata(catalog()) }, + }); + + const selected = await rc.discoverCandidateRun({ github, context: context(), sha: SHA }); + + assert.equal(selected, null); + }); + + it("hard-fails when an explicitly selected candidate is older than the allowed window", async () => { + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: "2000-01-01T00:00:00Z", + }; + const github = fakeGithub({ + runById: { 123: run }, + artifactsByRun: { 123: artifactMetadata(catalog()) }, + }); + + await assert.rejects( + () => rc.resolveExplicitCandidateRun({ github, context: context(), runId: 123, sha: SHA }), + /older than/, + ); + }); + + for (const invalid of ["missing timestamp", "duplicate evidence", "duplicate platform", "missing digest", "API error"]) { + it(`never silently rebuilds after ${invalid}`, async () => { + const run = { + id: 123, event: "schedule", path: rc.WORKFLOW_PATH, head_sha: SHA, + head_branch: "main", status: "completed", conclusion: "success", run_attempt: 1, + run_started_at: NOW, + }; + const artifacts = artifactMetadata(catalog()); + if (invalid === "missing timestamp") delete run.run_started_at; + if (invalid === "duplicate evidence") artifacts.push({ ...artifacts[0], id: 200 }); + if (invalid === "duplicate platform") artifacts.push({ ...artifacts[1], id: 200 }); + if (invalid === "missing digest") delete artifacts[0].digest; + const github = fakeGithub({ runs: [run], artifactsByRun: { 123: artifacts } }); + if (invalid === "API error") { + github.rest.actions.listWorkflowRunArtifacts = async () => { + throw new Error("API request denied"); + }; + } + const outputs = {}; + await assert.rejects(() => rc.plan({ + github, context: context(), + core: { setOutput(name, value) { outputs[name] = value; } }, + })); + assert.deepEqual(outputs, {}); + }); + } + + it("rejects workflow_dispatch as a release-promotion source", () => { + assert.throws( + () => + rc.validateTrustedReusableRun( + { + id: 123, + event: "workflow_dispatch", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + }, + SHA, + ), + /not trusted/, + ); + }); + + it("rejects same-SHA candidates from the wrong workflow path", () => { + assert.throws( + () => + rc.validateTrustedReusableRun( + { + id: 123, + event: "schedule", + path: ".github/workflows/other.yml", + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }, + SHA, + ), + /workflow path/, + ); + }); + + it("rejects missing reusable run identity fields", () => { + for (const [field, pattern] of [ + ["path", /workflow path/], + ["head_branch", /head_branch/], + ["run_attempt", /attempt/], + ]) { + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + delete run[field]; + assert.throws(() => rc.validateTrustedReusableRun(run, SHA), pattern); + } + }); + + it("rejects expired candidate artifacts for explicit candidate handling", async () => { + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + const github = fakeGithub({ + runById: { 123: run }, + artifactsByRun: { + 123: artifactMetadata(catalog(), 1, { expiredBinary: "apm-linux-x86_64" }), + }, + }); + + await assert.rejects( + () => rc.resolveExplicitCandidateRun({ github, context: context(), runId: 123, sha: SHA }), + /expired/, + ); + }); + + it("hard-fails invalid explicit candidate_run_id instead of falling back", async () => { + const github = fakeGithub({ + runById: { + 777: { + id: 777, + event: "workflow_dispatch", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + }, + }, + artifactsByRun: { 777: artifactMetadata(catalog()) }, + }); + + await assert.rejects( + () => rc.resolveExplicitCandidateRun({ github, context: context(), runId: 777, sha: SHA }), + /not trusted/, + ); + }); + + it("chooses newest same-SHA trusted candidate without requiring cross-run byte equality", async () => { + const runs = [ + { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: "2026-09-07T10:00:00Z", + }, + { + id: 124, + event: "repository_dispatch", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: "2026-09-07T11:00:00Z", + }, + ]; + const github = fakeGithub({ + runs, + artifactsByRun: { + 123: artifactMetadata(catalog(), 1, { digestPrefix: "old-" }), + 124: artifactMetadata(catalog(), 1, { digestPrefix: "new-" }), + }, + }); + + const selected = await rc.discoverCandidateRun({ github, context: context(), sha: SHA }); + + assert.equal(selected.run.id, 124); + }); + + it("does not fall back to an older same-SHA candidate when the newest candidate is invalid", async () => { + const runs = [ + { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: "2026-09-07T10:00:00Z", + }, + { + id: 124, + event: "repository_dispatch", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: "2026-09-07T11:00:00Z", + }, + ]; + const github = fakeGithub({ + runs, + artifactsByRun: { + 123: artifactMetadata(catalog(), 1, { digestPrefix: "old-" }), + 124: artifactMetadata(catalog(), 1, { evidenceDigest: "malformed" }), + }, + }); + + await assert.rejects( + () => rc.discoverCandidateRun({ github, context: context(), sha: SHA }), + /valid immutable sha256 digest/, + ); + }); +}); + +describe("candidate manifest and publication verification", () => { + it("qualifies a manifest only with full source and five-platform native gate evidence", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + }); + + await withCwd(root, async () => { + const evidence = await rc.qualify({ + github, + context: context({ eventName: "schedule", ref: "refs/heads/main" }), + core: { info() {} }, + artifactRoot, + }); + + assert.equal(evidence.full_validation, true); + assert.equal(Object.keys(evidence.platforms).length, 5); + assert.equal(fs.existsSync(path.join(root, "release-candidate-evidence.json")), true); + }); + }); + + it("rejects legacy bare artifact names and tells operators to rerun all jobs", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog(), 1, { attemptNames: false }) }, + jobsByRun: { 999: requiredJobs() }, + }); + + await withCwd(root, async () => { + await assert.rejects( + () => rc.qualify({ github, context: context(), core: { info() {} }, artifactRoot }), + /Use Re-run all jobs to regenerate complete candidate evidence for this attempt/, + ); + }); + }); + + it("rejects manifests missing source gate evidence", () => { + assert.throws( + () => + rc.assertRequiredJobs( + requiredJobs(catalog(), (jobs) => + jobs.filter((job) => !job.name.includes("Test Architecture Ratchets")), + ), + catalog(), + ), + /Test Architecture Ratchets/, + ); + }); + + it("rejects shadow and duplicate jobs instead of substring-matching a success", () => { + assert.throws( + () => + rc.selectRequiredJobs( + requiredJobs(catalog(), (jobs) => [ + ...jobs.filter((job) => job.name !== "Candidate Source Checks / Lint"), + { + name: "Candidate Source Checks / Not Lint", + status: "completed", + conclusion: "success", + }, + ]), + catalog(), + ), + /exactly one Candidate Source Checks \/ Lint/, + ); + + assert.throws( + () => + rc.selectRequiredJobs( + requiredJobs(catalog(), (jobs) => [ + ...jobs, + { + name: "Candidate Source Checks / Windows Compatibility Gate", + status: "completed", + conclusion: "success", + }, + ]), + catalog(), + ), + /exactly one Candidate Source Checks \/ Windows Compatibility Gate/, + ); + }); + + it("rejects candidate archives whose version does not match the tag", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot, catalog(), SHA, "1.2.4"); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + }); + + await withCwd(root, async () => { + await assert.rejects( + () => rc.qualify({ github, context: context(), core: { info() {} }, artifactRoot }), + /does not match tag v1\.2\.3/, + ); + }); + }); + + it("rejects a reusable manifest whose version does not match the release tag", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + runById: { + 999: { + id: 999, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }, + }, + }); + + await withCwd(root, async () => { + await rc.qualify({ + github, + context: context({ eventName: "schedule", ref: "refs/heads/main" }), + core: { info() {} }, + artifactRoot, + }); + + await assert.rejects( + () => + rc.verify({ + github, + context: context({ ref: "refs/tags/v1.2.4" }), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }), + /version 1\.2\.3 does not match tag v1\.2\.4/, + ); + }); + }); + + it("rejects malformed API artifact digests", async () => { + const run = { + id: 123, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + run_started_at: NOW, + }; + const github = fakeGithub({ + runById: { 123: run }, + artifactsByRun: { 123: artifactMetadata(catalog(), 1, { digestPrefix: null }) }, + }); + + await assert.rejects( + () => rc.resolveExplicitCandidateRun({ github, context: context(), runId: 123, sha: SHA }), + /valid immutable sha256 digest/, + ); + }); + + it("verifies and copies exact archive bytes without repacking", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + runById: { + 999: { + id: 999, + event: "push", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "v1.2.3", + status: "in_progress", + conclusion: null, + run_attempt: 1, + }, + }, + }); + + await withCwd(root, async () => { + await rc.qualify({ github, context: context(), core: { info() {} }, artifactRoot }); + await rc.verify({ + github, + context: context(), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }); + + for (const row of catalog()) { + const archive = rc.archiveName(row.binary_name); + assert.equal( + fs.readFileSync(path.join(outputRoot, archive), "utf8"), + fs.readFileSync(path.join(artifactRoot, row.binary_name, "release-assets", archive), "utf8"), + ); + } + }); + }); + + it("hard-fails when archive bytes differ from manifest digest", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + runById: { + 999: { + id: 999, + event: "push", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "v1.2.3", + status: "in_progress", + conclusion: null, + run_attempt: 1, + }, + }, + }); + + await withCwd(root, async () => { + await rc.qualify({ github, context: context(), core: { info() {} }, artifactRoot }); + const tampered = path.join( + artifactRoot, + "apm-linux-x86_64", + "release-assets", + "apm-linux-x86_64.tar.gz", + ); + fs.writeFileSync(tampered, "tampered"); + + await assert.rejects( + () => + rc.verify({ + github, + context: context(), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }), + /archive digest mismatch/, + ); + }); + }); + + it("hard-fails when downloaded candidate artifacts contain an unexpected release archive", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + runById: { + 999: { + id: 999, + event: "push", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "v1.2.3", + status: "in_progress", + conclusion: null, + run_attempt: 1, + }, + }, + }); + + await withCwd(root, async () => { + await rc.qualify({ github, context: context(), core: { info() {} }, artifactRoot }); + fs.writeFileSync(path.join(artifactRoot, "apm-surprise.tar.gz"), "unexpected"); + + await assert.rejects( + () => + rc.verify({ + github, + context: context(), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }), + /unexpected release files/, + ); + }); + }); + + it("hard-fails when live Actions job state no longer proves a required gate", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const githubForQualify = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + }); + const githubForVerify = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { + 999: requiredJobs(catalog(), (jobs) => + jobs.map((job) => + job.name.includes("Lint") ? { ...job, conclusion: "failure" } : job, + ), + ), + }, + runById: { + 999: { + id: 999, + event: "push", + path: rc.WORKFLOW_PATH, + head_sha: SHA, + head_branch: "v1.2.3", + status: "in_progress", + conclusion: null, + run_attempt: 1, + }, + }, + }); + + await withCwd(root, async () => { + await rc.qualify({ + github: githubForQualify, + context: context(), + core: { info() {} }, + artifactRoot, + }); + + await assert.rejects( + () => + rc.verify({ + github: githubForVerify, + context: context(), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }), + /Candidate Source Checks \/ Lint was completed\/failure/, + ); + }); + }); + + it("hard-fails explicit candidate_run_id verification when SHA mismatches", async () => { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + const outputRoot = path.join(root, "release-assets"); + writeCandidateFiles(artifactRoot); + const github = fakeGithub({ + artifactsByRun: { 999: artifactMetadata(catalog()) }, + jobsByRun: { 999: requiredJobs() }, + runById: { + 999: { + id: 999, + event: "schedule", + path: rc.WORKFLOW_PATH, + head_sha: OTHER_SHA, + head_branch: "main", + status: "completed", + conclusion: "success", + run_attempt: 1, + }, + }, + }); + + await withCwd(root, async () => { + const evidence = { + schema_version: 1, + repo: "microsoft/apm", + workflow_path: rc.WORKFLOW_PATH, + event: "schedule", + ref: "refs/heads/main", + head_branch: "main", + head_sha: OTHER_SHA, + run_id: 999, + run_attempt: 1, + full_validation: true, + config_hashes: rc.configHashes(root), + required_jobs: requiredJobs(), + platforms: rc.collectArchiveInventory({ artifactRoot, catalog: catalog(), sha: SHA }), + }; + fs.writeFileSync( + path.join(root, "release-candidate-evidence.json"), + `${JSON.stringify(evidence, null, 2)}\n`, + ); + + await assert.rejects( + () => + rc.verify({ + github, + context: context(), + core: { info() {} }, + runId: 999, + artifactRoot, + evidencePath: path.join(root, "release-candidate-evidence.json"), + outputRoot, + }), + /SHA does not match tag SHA/, + ); + }); + }); +}); diff --git a/scripts/release-platforms.json b/scripts/release-platforms.json new file mode 100644 index 0000000000..02ad8ab9d8 --- /dev/null +++ b/scripts/release-platforms.json @@ -0,0 +1,37 @@ +[ + { + "runner": "ubuntu-24.04", + "platform": "linux", + "binary_name": "apm-linux-x86_64", + "integration_markers": "not live", + "on_main": true + }, + { + "runner": "ubuntu-24.04-arm", + "platform": "linux", + "binary_name": "apm-linux-arm64", + "integration_markers": "not live", + "on_main": true + }, + { + "runner": "windows-latest", + "platform": "windows", + "binary_name": "apm-windows-x86_64", + "integration_markers": "not live", + "on_main": true + }, + { + "runner": "macos-15-intel", + "platform": "darwin", + "binary_name": "apm-darwin-x86_64", + "integration_markers": "lifecycle_smoke and not live", + "on_main": true + }, + { + "runner": "macos-latest", + "platform": "darwin", + "binary_name": "apm-darwin-arm64", + "integration_markers": "not live", + "on_main": false + } +] diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index ee4d9ae63f..5e7bf2f415 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -216,67 +216,37 @@ setup_binary_for_testing() { # Set up runtimes (codex/llm/copilot) - Integration Testing Coverage! setup_runtimes() { log_info "=== Setting up runtimes for integration tests ===" - - # Set up GitHub Copilot CLI runtime (recommended default) - log_info "Setting up GitHub Copilot CLI runtime..." - if ! ./apm runtime setup copilot; then - log_error "Failed to set up GitHub Copilot CLI runtime" - exit 1 - fi - - # Set up codex runtime - log_info "Setting up Codex runtime..." - if ! ./apm runtime setup codex; then - log_error "Failed to set up Codex runtime" - exit 1 - fi - - # Set up LLM runtime - log_info "Setting up LLM runtime..." - if ! ./apm runtime setup llm; then - log_error "Failed to set up LLM runtime" - exit 1 - fi - - # Add runtime paths to current session PATH - log_info "Adding runtime paths to current session..." RUNTIME_PATH="$HOME/.apm/runtimes" export PATH="$RUNTIME_PATH:$PATH" - - # Verify runtimes are available - log_info "Verifying runtime installations..." - - # Check GitHub Copilot CLI - if command -v copilot >/dev/null 2>&1; then - local copilot_version=$(copilot --version 2>&1 || echo "unknown") - log_success "GitHub Copilot CLI ready: $copilot_version" - else - log_error "GitHub Copilot CLI not found in PATH after setup" - exit 1 + local runtimes="${APM_TEST_RUNTIMES:-copilot codex llm}" + if [[ "$runtimes" == "none" ]]; then + log_info "No external runtime prerequisite for this test selection" + return fi - - # Check codex - if command -v codex >/dev/null 2>&1; then - local codex_version=$(codex --version 2>&1 || echo "unknown") - log_success "Codex runtime ready: $codex_version" - else - log_error "Codex not found in PATH after setup" - echo "PATH: $PATH" - echo "Looking for codex in: $RUNTIME_PATH" - ls -la "$RUNTIME_PATH" || echo "Runtime directory not found" - exit 1 - fi - - # Check LLM wrapper - local llm_path="$HOME/.apm/runtimes/llm" - if [[ -x "$llm_path" ]]; then - log_success "LLM runtime ready at: $llm_path" - else - log_error "LLM runtime not found at: $llm_path" - exit 1 - fi - - log_success "All runtimes configured successfully (Copilot, Codex, LLM)" + local runtime + for runtime in $runtimes; do + case "$runtime" in + copilot|codex|llm) ;; + *) + log_error "Unknown APM_TEST_RUNTIMES entry: $runtime" + exit 1 + ;; + esac + log_info "Setting up $runtime runtime..." + if ! ./apm runtime setup "$runtime"; then + log_error "Failed to set up $runtime runtime" + exit 1 + fi + if ! command -v "$runtime" >/dev/null 2>&1; then + log_error "$runtime runtime is not executable after setup" + exit 1 + fi + if [[ "$runtime" == "llm" && ! -x "$RUNTIME_PATH/llm" ]]; then + log_error "LLM wrapper is not executable after setup: $RUNTIME_PATH/llm" + exit 1 + fi + done + log_success "Required runtimes configured successfully: $runtimes" } # Install test dependencies (like CI does) @@ -369,6 +339,7 @@ run_e2e_tests() { log_info "Pytest marker selection: $PYTEST_MARK_EXPR" fi if pytest tests/integration/ -v --tb=short \ + --strict-runtime-prerequisites \ ${extra_args[@]+"${extra_args[@]}"} \ ${marker_args[@]+"${marker_args[@]}"}; then log_success "Integration test suite passed (collected and ran via pytest discovery)" diff --git a/scripts/windows/build-binary.ps1 b/scripts/windows/build-binary.ps1 index 29d16cdeaf..184c5d31fa 100644 --- a/scripts/windows/build-binary.ps1 +++ b/scripts/windows/build-binary.ps1 @@ -32,20 +32,16 @@ try { exit 1 } -# Check if UPX is available (optional) -if (Get-Command upx -ErrorAction SilentlyContinue) { - Write-Host "UPX found - binary will be compressed" -ForegroundColor Green -} else { - Write-Host "UPX not found - binary will not be compressed" -ForegroundColor Yellow -} - # Inject build SHA into version.py $VersionFile = "src/apm_cli/version.py" $originalContent = Get-Content $VersionFile -Raw $BuildSHA = git rev-parse --short HEAD 2>$null if ($BuildSHA) { Write-Host "Injecting build SHA: $BuildSHA" -ForegroundColor Yellow - $newContent = $originalContent -replace '^__BUILD_SHA__ = None$', "__BUILD_SHA__ = `"$BuildSHA`"" + $newContent = $originalContent -replace '(?m)^__BUILD_SHA__ = None\r?$', "__BUILD_SHA__ = `"$BuildSHA`"" + if ($newContent -eq $originalContent) { + throw "Could not inject the candidate build SHA into version.py" + } Set-Content -Path $VersionFile -Value $newContent -NoNewline } diff --git a/scripts/windows/test-install-script.ps1 b/scripts/windows/test-install-script.ps1 index 46e2b09e16..e683494b87 100644 --- a/scripts/windows/test-install-script.ps1 +++ b/scripts/windows/test-install-script.ps1 @@ -15,15 +15,33 @@ # end-to-end (download + dispatch + new version reported). # # Designed to run on the windows-latest GitHub Actions runner. Performs a -# real install of a pinned APM release into an isolated test prefix and -# leaves the developer's existing apm install untouched. +# real install of the current build candidate from a verified loopback fixture. +# The only historical release is the upgrade SOURCE, never the destination. +# Invoke via tests/integration/test_windows_installer_launchers.py, which owns +# archive verification and the fixture server. param( - [string]$PinnedVersion = "v0.29.0", - [string]$OlderVersion = "v0.28.0" + [Parameter(Mandatory = $true)][string]$CandidateVersion, + [Parameter(Mandatory = $true)][string]$OlderVersion, + [Parameter(Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{64}$')][string]$CandidateExecutableSha256, + [Parameter(Mandatory = $true)][uri]$FixtureBaseUrl, + [Parameter(Mandatory = $true)][string]$TestRoot ) $ErrorActionPreference = "Stop" +if ($FixtureBaseUrl.Scheme -ne "http" -or $FixtureBaseUrl.Host -ne "127.0.0.1" -or + $FixtureBaseUrl.Port -le 0 -or $FixtureBaseUrl.UserInfo -or + $FixtureBaseUrl.AbsolutePath -ne "/" -or $FixtureBaseUrl.Query -or $FixtureBaseUrl.Fragment) { + throw "Installer tests require the verified loopback fixture, not a public release destination" +} +foreach ($tag in @($CandidateVersion, $OlderVersion)) { + if ($tag -cnotmatch '^v[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?$') { + throw "Installer test versions must be exact vX.Y.Z release tags" + } +} +if ($env:OS -ne "Windows_NT") { throw "Native Windows installer tests require Windows" } +$TestRoot = (Resolve-Path -LiteralPath $TestRoot -ErrorAction Stop).Path $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $RepoRoot = Resolve-Path (Join-Path $ScriptDir "..\..") @@ -45,6 +63,21 @@ function Assert-True { } } +function Test-ReportedVersion { + param([string]$Output, [string]$Version) + return $Output -match ("(?m)\bversion\s+" + [regex]::Escape($Version.TrimStart("v")) + "(?=\s|$)") +} + +function Assert-CandidatePayload { + param([string]$Root) + $executable = Join-Path $Root "current\apm.exe" + Assert-True (Test-Path -LiteralPath $executable -PathType Leaf) "Candidate current\apm.exe exists" + if (Test-Path -LiteralPath $executable -PathType Leaf) { + $actual = (Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash.ToLowerInvariant() + Assert-True ($actual -ceq $CandidateExecutableSha256) "Installed executable is byte-identical to the verified candidate" + } +} + # --------------------------------------------------------------------------- # Test 1: Get-Sha256Hex function falls back to .NET when Get-FileHash is gone. # --------------------------------------------------------------------------- @@ -67,7 +100,7 @@ function Test-Sha256Fallback { return } - $tempFile = [System.IO.Path]::GetTempFileName() + $tempFile = Join-Path $TestRoot ([System.IO.Path]::GetRandomFileName()) try { Set-Content -Path $tempFile -Value "the quick brown fox" -NoNewline -Encoding ASCII $expected = (Get-FileHash -Path $tempFile -Algorithm SHA256).Hash.ToLower() @@ -83,7 +116,7 @@ Remove-Module Microsoft.PowerShell.Utility -Force -ErrorAction SilentlyContinue $($match.Value) Write-Output (Get-Sha256Hex -Path '$tempFile') "@ - $childScriptPath = [System.IO.Path]::Combine($env:TEMP, [System.IO.Path]::GetRandomFileName() + ".ps1") + $childScriptPath = Join-Path $TestRoot ([System.IO.Path]::GetRandomFileName() + ".ps1") Set-Content -Path $childScriptPath -Value $childScript -Encoding UTF8 try { $actual = & powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $childScriptPath 2>&1 @@ -203,7 +236,7 @@ Write-Output '---APM-JSON-END---' # Use GetRandomFileName so we don't leak the GetTempFileName-created # zero-byte .tmp companion file every run. - $tempScript = [System.IO.Path]::Combine($env:TEMP, [System.IO.Path]::GetRandomFileName() + ".ps1") + $tempScript = Join-Path $TestRoot ([System.IO.Path]::GetRandomFileName() + ".ps1") try { Set-Content -Path $tempScript -Value $childScript -Encoding UTF8 $raw = & pwsh -NoProfile -NonInteractive -File $tempScript 2>&1 @@ -253,7 +286,11 @@ function Invoke-InstallScript { Remove-Item Env:APM_SKIP_CHECKSUM -ErrorAction SilentlyContinue & powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $InstallScript | Out-Host - return $LASTEXITCODE + $exitCode = $LASTEXITCODE + if ($exitCode -eq 0 -and $Version -ceq $CandidateVersion) { + Assert-CandidatePayload -Root (Split-Path -Parent $BinDir) + } + return $exitCode } finally { if ($null -ne $savedVersion) { $env:VERSION = $savedVersion } else { Remove-Item Env:VERSION -ErrorAction SilentlyContinue } if ($null -ne $savedInstallDir) { $env:APM_INSTALL_DIR = $savedInstallDir } else { Remove-Item Env:APM_INSTALL_DIR -ErrorAction SilentlyContinue } @@ -270,7 +307,7 @@ function Get-ShimVersion { function New-IsolatedPrefix { # Spaces and a cmd metacharacter exercise launcher/path quoting end to end. - $root = Join-Path ([System.IO.Path]::GetTempPath()) ("APM Install Test & Edge " + [System.Guid]::NewGuid().ToString("N")) + $root = Join-Path $TestRoot ("APM Install Test & Edge " + [System.Guid]::NewGuid().ToString("N")) $binDir = Join-Path $root "bin" $tmpDir = Join-Path $root "tmp" New-Item -ItemType Directory -Force -Path $binDir | Out-Null @@ -346,13 +383,13 @@ sys.exit(1) # --------------------------------------------------------------------------- function Test-EndToEndInstall { - Write-Step "Test 3: End-to-end install of APM $PinnedVersion into isolated prefix" + Write-Step "Test 3: End-to-end install of APM $CandidateVersion into isolated prefix" $prefix = New-IsolatedPrefix try { Assert-True ($prefix.Root -match [regex]::Escape(" & ")) "Test prefix exercises spaces and a cmd metacharacter" - Write-Info "Running install.ps1 (VERSION=$PinnedVersion, APM_INSTALL_DIR=$($prefix.BinDir), APM_TEMP_DIR=$($prefix.TmpDir))" - $exitCode = Invoke-InstallScript -Version $PinnedVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir + Write-Info "Running install.ps1 (VERSION=$CandidateVersion, APM_INSTALL_DIR=$($prefix.BinDir), APM_TEMP_DIR=$($prefix.TmpDir))" + $exitCode = Invoke-InstallScript -Version $CandidateVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir Assert-True ($exitCode -eq 0) "install.ps1 exits 0 (got $exitCode)" $shim = Join-Path $prefix.BinDir "apm.cmd" @@ -382,7 +419,7 @@ function Test-EndToEndInstall { $ver = Get-ShimVersion -ShimPath $shim Assert-True ($ver.ExitCode -eq 0) "apm.cmd --version exits 0 (got $($ver.ExitCode); output: $($ver.Output))" - Assert-True ($ver.Output -match $PinnedVersion.TrimStart("v")) "apm.cmd --version reports $PinnedVersion" + Assert-True (Test-ReportedVersion $ver.Output $CandidateVersion) "apm.cmd --version reports $CandidateVersion" } $currentDir = Join-Path $prefix.Root "current" @@ -408,7 +445,7 @@ function Test-EndToEndInstall { $cmdOutput = & cmd.exe /d /c 'set "PATH=%APM_LAUNCH_TEST_PATH%" && apm --version' 2>&1 $cmdExit = $LASTEXITCODE Assert-True ($cmdExit -eq 0) "cmd.exe resolves bare apm (got $cmdExit; output: $cmdOutput)" - Assert-True (($cmdOutput | Out-String) -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "cmd.exe reports $PinnedVersion" + Assert-True (Test-ReportedVersion ($cmdOutput | Out-String) $CandidateVersion) "cmd.exe reports $CandidateVersion" $pythonScript = @' import os @@ -429,7 +466,7 @@ sys.exit(result.returncode) $pythonOutput = & python -c $pythonScript 2>&1 $pythonExit = $LASTEXITCODE Assert-True ($pythonExit -eq 0) "Python subprocess resolves bare apm (got $pythonExit; output: $pythonOutput)" - Assert-True (($pythonOutput | Out-String) -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "Python subprocess reports $PinnedVersion" + Assert-True (Test-ReportedVersion ($pythonOutput | Out-String) $CandidateVersion) "Python subprocess reports $CandidateVersion" Assert-MissingStableExecutableFailsForNativeProcess ` -CurrentDir $currentDir ` @@ -443,7 +480,7 @@ sys.exit(result.returncode) $bashOutput = & $bash.Source -lc 'cd "$1" && PATH="$2:$PATH" && command -v apm && apm --version' bash $bashTestDir $bashCurrentDir 2>&1 $bashExit = $LASTEXITCODE Assert-True ($bashExit -eq 0) "Git Bash resolves bare apm (got $bashExit; output: $bashOutput)" - Assert-True (($bashOutput | Out-String) -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "Git Bash reports $PinnedVersion" + Assert-True (Test-ReportedVersion ($bashOutput | Out-String) $CandidateVersion) "Git Bash reports $CandidateVersion" } } finally { Remove-Item Env:APM_LAUNCH_TEST_CWD -ErrorAction SilentlyContinue @@ -454,6 +491,7 @@ sys.exit(result.returncode) $leftover = Get-ChildItem -Path $prefix.TmpDir -Filter "apm-install-*" -Directory -ErrorAction SilentlyContinue Assert-True (-not $leftover) "No leftover apm-install-* directory in APM_TEMP_DIR" + Test-SameVersionReinstall -Prefix $prefix } finally { Remove-Item -Recurse -Force $prefix.Root -ErrorAction SilentlyContinue } @@ -474,7 +512,7 @@ function Test-NonJunctionCollision { New-Item -ItemType Directory -Force -Path $currentDir | Out-Null Set-Content -Path $canary -Value "preserve me" -Encoding ASCII - $exitCode = Invoke-InstallScript -Version $PinnedVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir + $exitCode = Invoke-InstallScript -Version $CandidateVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir Assert-True ($exitCode -eq 1) "Installer refuses a non-junction current path (got $exitCode)" Assert-True (Test-Path $canary -PathType Leaf) "Non-junction current path preserves its canary file" @@ -486,61 +524,60 @@ function Test-NonJunctionCollision { } } -# --------------------------------------------------------------------------- -# Test 4a: Cross-version upgrade. Install OlderVersion, then PinnedVersion, -# into the same prefix. The shim must end up pointing at PinnedVersion's -# release dir and `apm --version` must report PinnedVersion. -# --------------------------------------------------------------------------- - -function Test-CrossVersionUpgrade { - Write-Step "Test 4a: Cross-version upgrade $OlderVersion -> $PinnedVersion in same prefix" +function Test-CandidateChecksumRejection { + Write-Step "Test 3c: tampered candidate archive is rejected before promotion" $prefix = New-IsolatedPrefix + $savedReleaseBase = $env:APM_RELEASE_BASE_URL try { - Write-Info "Step 1: install $OlderVersion" - $exit1 = Invoke-InstallScript -Version $OlderVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir - Assert-True ($exit1 -eq 0) "Step 1 install.ps1 exits 0 (got $exit1)" - - $shim = Join-Path $prefix.BinDir "apm.cmd" - $ver1 = Get-ShimVersion -ShimPath $shim - Assert-True ($ver1.Output -match $OlderVersion.TrimStart("v")) "Step 1: apm.cmd --version reports $OlderVersion (got: $($ver1.Output))" - - Write-Info "Step 2: install $PinnedVersion over the existing install" - $exit2 = Invoke-InstallScript -Version $PinnedVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir - Assert-True ($exit2 -eq 0) "Step 2 install.ps1 exits 0 (got $exit2)" - - $ver2 = Get-ShimVersion -ShimPath $shim - Assert-True ($ver2.Output -match $PinnedVersion.TrimStart("v")) "Step 2: apm.cmd --version reports $PinnedVersion (got: $($ver2.Output))" - - $shimText = Get-Content $shim -Raw - Assert-True ($shimText -match [regex]::Escape($PinnedVersion)) "Step 2: shim references $PinnedVersion path" - - # The version-stable junction must re-point at the new release so bare - # CreateProcess / Git Bash callers resolve the upgraded apm.exe, not the - # previous release still on disk. A wrong-target junction survives the - # installer's own Test-Path guard, so assert the resolved version too. - $stableExe = Join-Path $prefix.Root "current\apm.exe" - Assert-True (Test-Path $stableExe) "Stable current\apm.exe exists after upgrade" - if (Test-Path $stableExe) { - $stableVer = & $stableExe --version 2>&1 - Assert-True (($stableVer | Out-String) -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "current\apm.exe reports $PinnedVersion after upgrade" - } - $junctionLeftover = Get-ChildItem -Path $prefix.Root -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "current.new-*" -or $_.Name -like "current.old-*" } - Assert-True (-not $junctionLeftover) "No leftover current.new-* / current.old-* junction temps at install root" - - # Both release dirs may coexist (we only replace the matching tag), - # but the staging/backup helper dirs from the second install MUST be - # cleaned up. - $releasesDir = Join-Path $prefix.BinDir "..\releases" | Resolve-Path - $leftoverStaging = Get-ChildItem -Path $releasesDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "*.new-*" -or $_.Name -like "*.old-*" } - Assert-True (-not $leftoverStaging) "No leftover .new-* / .old-* staging/backup dirs after upgrade" + $env:APM_RELEASE_BASE_URL = "$($FixtureBaseUrl.AbsoluteUri.TrimEnd('/'))/tampered" + $exitCode = Invoke-InstallScript -Version $CandidateVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir + Assert-True ($exitCode -eq 1) "Tampered candidate fails checksum verification (got $exitCode)" + Assert-True (-not (Test-Path (Join-Path $prefix.Root "current"))) "Tampered candidate has no promoted current path" + Assert-True (-not (Test-Path (Join-Path $prefix.BinDir "apm.cmd"))) "Tampered candidate has no command shim" + $executables = Get-ChildItem -LiteralPath $prefix.Root -Filter "apm.exe" -Recurse -ErrorAction SilentlyContinue + Assert-True (-not $executables) "Tampered candidate is not extracted or installed" } finally { + $env:APM_RELEASE_BASE_URL = $savedReleaseBase Remove-Item -Recurse -Force $prefix.Root -ErrorAction SilentlyContinue } } +# --------------------------------------------------------------------------- +# Test 4a: Cross-version upgrade assertions on the real self-update scenario. +# Reuse its old -> candidate transition rather than installing both a second time. +# --------------------------------------------------------------------------- + +function Test-CrossVersionUpgrade { + param([Parameter(Mandatory = $true)][hashtable]$Prefix) + Write-Step "Test 4a: Cross-version upgrade $OlderVersion -> $CandidateVersion in same prefix" + + $shim = Join-Path $Prefix.BinDir "apm.cmd" + $ver2 = Get-ShimVersion -ShimPath $shim + Assert-True ($ver2.ExitCode -eq 0) "Upgraded apm.cmd --version exits 0" + Assert-True (Test-ReportedVersion $ver2.Output $CandidateVersion) "Step 2: apm.cmd --version reports $CandidateVersion (got: $($ver2.Output))" + + $shimText = Get-Content $shim -Raw + Assert-True ($shimText -match [regex]::Escape($CandidateVersion)) "Step 2: shim references $CandidateVersion path" + + # A wrong-target junction survives the installer's Test-Path guard. + $stableExe = Join-Path $Prefix.Root "current\apm.exe" + Assert-True (Test-Path $stableExe) "Stable current\apm.exe exists after upgrade" + if (Test-Path $stableExe) { + $stableVer = & $stableExe --version 2>&1 + Assert-True ($LASTEXITCODE -eq 0) "current\apm.exe exits 0 after upgrade" + Assert-True (Test-ReportedVersion ($stableVer | Out-String) $CandidateVersion) "current\apm.exe reports $CandidateVersion after upgrade" + } + $junctionLeftover = Get-ChildItem -Path $Prefix.Root -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "current.new-*" -or $_.Name -like "current.old-*" } + Assert-True (-not $junctionLeftover) "No leftover current.new-* / current.old-* junction temps at install root" + + $releasesDir = Join-Path $Prefix.BinDir "..\releases" | Resolve-Path + $leftoverStaging = Get-ChildItem -Path $releasesDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "*.new-*" -or $_.Name -like "*.old-*" } + Assert-True (-not $leftoverStaging) "No leftover .new-* / .old-* staging/backup dirs after upgrade" +} + # --------------------------------------------------------------------------- # Test 4b: Same-version reinstall. This is the path that exercises the # stage -> move-existing-aside -> promote -> delete-backup branch in @@ -548,69 +585,61 @@ function Test-CrossVersionUpgrade { # --------------------------------------------------------------------------- function Test-SameVersionReinstall { - Write-Step "Test 4b: Same-version reinstall of $PinnedVersion exercises promote/backup branch" - - $prefix = New-IsolatedPrefix - try { - Write-Info "Step 1: install $PinnedVersion" - $exit1 = Invoke-InstallScript -Version $PinnedVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir - Assert-True ($exit1 -eq 0) "Step 1 install.ps1 exits 0 (got $exit1)" - - $releasesDir = Join-Path $prefix.BinDir "..\releases" | Resolve-Path - $releaseDir = Join-Path $releasesDir $PinnedVersion - Assert-True (Test-Path $releaseDir) "Release dir exists after first install ($releaseDir)" - $firstExe = Join-Path $releaseDir "apm.exe" - $firstStamp = (Get-Item $firstExe).LastWriteTimeUtc - - Write-Info "Step 2: reinstall $PinnedVersion (must rename releaseDir aside, promote staging, delete backup)" - $exit2 = Invoke-InstallScript -Version $PinnedVersion -BinDir $prefix.BinDir -TmpDir $prefix.TmpDir - Assert-True ($exit2 -eq 0) "Step 2 install.ps1 exits 0 (got $exit2)" - - Assert-True (Test-Path $releaseDir) "Release dir still exists after reinstall" - $secondExe = Join-Path $releaseDir "apm.exe" - Assert-True (Test-Path $secondExe) "apm.exe present after reinstall" - - # apm.exe must be the freshly staged copy, not the original (the - # promote step renames the old release dir aside and moves the - # staging dir into place, so write time must be >= first stamp). - $secondStamp = (Get-Item $secondExe).LastWriteTimeUtc - Assert-True ($secondStamp -ge $firstStamp) "apm.exe write time advanced after reinstall ($firstStamp -> $secondStamp)" - - $ver = Get-ShimVersion -ShimPath (Join-Path $prefix.BinDir "apm.cmd") - Assert-True ($ver.ExitCode -eq 0) "apm.cmd --version exits 0 after reinstall (got $($ver.ExitCode))" - Assert-True ($ver.Output -match $PinnedVersion.TrimStart("v")) "apm.cmd --version reports $PinnedVersion after reinstall" - - # The junction is rebuilt on every install; after a same-version - # reinstall it must still resolve the freshly promoted apm.exe for bare - # CreateProcess / Git Bash callers. - $stableExe = Join-Path $prefix.Root "current\apm.exe" - Assert-True (Test-Path $stableExe) "Stable current\apm.exe exists after reinstall" - if (Test-Path $stableExe) { - $stableVer = & $stableExe --version 2>&1 - Assert-True (($stableVer | Out-String) -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "current\apm.exe reports $PinnedVersion after reinstall" - } - $junctionLeftover = Get-ChildItem -Path $prefix.Root -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "current.new-*" -or $_.Name -like "current.old-*" } - Assert-True (-not $junctionLeftover) "No leftover current.new-* / current.old-* junction temps at install root after reinstall" - - $leftoverStaging = Get-ChildItem -Path $releasesDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "*.new-*" -or $_.Name -like "*.old-*" } - Assert-True (-not $leftoverStaging) "No leftover .new-* / .old-* dirs after reinstall (rollback path didn't trigger and backup was deleted)" - } finally { - Remove-Item -Recurse -Force $prefix.Root -ErrorAction SilentlyContinue + param([Parameter(Mandatory = $true)][hashtable]$Prefix) + Write-Step "Test 4b: Same-version reinstall of $CandidateVersion exercises promote/backup branch" + + # Reuse the fresh-install scenario's prefix, which its caller cleans up. + $releasesDir = Join-Path $Prefix.BinDir "..\releases" | Resolve-Path + $releaseDir = Join-Path $releasesDir $CandidateVersion + Assert-True (Test-Path $releaseDir) "Release dir exists after first install ($releaseDir)" + $firstExe = Join-Path $releaseDir "apm.exe" + $firstStamp = (Get-Item $firstExe).LastWriteTimeUtc + $replacementCanary = Join-Path $releaseDir "reinstall-canary.txt" + Set-Content -LiteralPath $replacementCanary -Value "old release only" -Encoding ASCII + + Write-Info "Step 2: reinstall $CandidateVersion (must rename releaseDir aside, promote staging, delete backup)" + $exit2 = Invoke-InstallScript -Version $CandidateVersion -BinDir $Prefix.BinDir -TmpDir $Prefix.TmpDir + Assert-True ($exit2 -eq 0) "Step 2 install.ps1 exits 0 (got $exit2)" + + Assert-True (Test-Path $releaseDir) "Release dir still exists after reinstall" + $secondExe = Join-Path $releaseDir "apm.exe" + Assert-True (Test-Path $secondExe) "apm.exe present after reinstall" + Assert-True (-not (Test-Path -LiteralPath $replacementCanary)) "Reinstall replaces the previous release tree, including its canary" + + # Promotion must replace the existing bundle, not leave it in place. + $secondStamp = (Get-Item $secondExe).LastWriteTimeUtc + Assert-True ($secondStamp -ge $firstStamp) "apm.exe write time advanced after reinstall ($firstStamp -> $secondStamp)" + + $ver = Get-ShimVersion -ShimPath (Join-Path $Prefix.BinDir "apm.cmd") + Assert-True ($ver.ExitCode -eq 0) "apm.cmd --version exits 0 after reinstall (got $($ver.ExitCode))" + Assert-True (Test-ReportedVersion $ver.Output $CandidateVersion) "apm.cmd --version reports $CandidateVersion after reinstall" + + $stableExe = Join-Path $Prefix.Root "current\apm.exe" + Assert-True (Test-Path $stableExe) "Stable current\apm.exe exists after reinstall" + if (Test-Path $stableExe) { + $stableVer = & $stableExe --version 2>&1 + Assert-True ($LASTEXITCODE -eq 0) "current\apm.exe exits 0 after reinstall" + Assert-True (Test-ReportedVersion ($stableVer | Out-String) $CandidateVersion) "current\apm.exe reports $CandidateVersion after reinstall" } + $junctionLeftover = Get-ChildItem -Path $Prefix.Root -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "current.new-*" -or $_.Name -like "current.old-*" } + Assert-True (-not $junctionLeftover) "No leftover current.new-* / current.old-* junction temps at install root after reinstall" + + $leftoverStaging = Get-ChildItem -Path $releasesDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "*.new-*" -or $_.Name -like "*.old-*" } + Assert-True (-not $leftoverStaging) "No leftover .new-* / .old-* dirs after reinstall (rollback path didn't trigger and backup was deleted)" } # --------------------------------------------------------------------------- # Test 5: Real `apm self-update` end-to-end. Install OlderVersion, then run -# the installed apm.cmd's self-update command pinned to PinnedVersion. The -# installed apm downloads install.ps1 for that release and runs it, exercising +# the installed apm.cmd's self-update command targeting CandidateVersion. The +# installed apm downloads this checkout's install.ps1 and runs it, exercising # the whole launch path that issue #1389 originally broke. The fresh apm.cmd -# must report PinnedVersion afterwards. +# must report CandidateVersion afterwards. # # The VERSION pin keeps the installed version deterministic and skips the live -# releases/latest lookup in supported release binaries. Retry remains here as a -# guard against transient installer-script download latency. +# releases/latest lookup in supported release binaries. Transport is local and +# deterministic, so retries would only mask regressions. # --------------------------------------------------------------------------- function Test-SelfUpdateCommand { @@ -624,9 +653,10 @@ function Test-SelfUpdateCommand { $shim = Join-Path $prefix.BinDir "apm.cmd" $ver1 = Get-ShimVersion -ShimPath $shim - Assert-True ($ver1.Output -match $OlderVersion.TrimStart("v")) "Step 1: apm.cmd --version reports $OlderVersion (got: $($ver1.Output))" + Assert-True ($ver1.ExitCode -eq 0) "Self-update baseline apm.cmd --version exits 0" + Assert-True (Test-ReportedVersion $ver1.Output $OlderVersion) "Step 1: apm.cmd --version reports $OlderVersion (got: $($ver1.Output))" - Write-Info "Step 2: run apm self-update pinned to $PinnedVersion" + Write-Info "Step 2: run apm self-update targeting $CandidateVersion" # Point the self-update temp file at our isolated prefix so we don't # litter the runner's %LOCALAPPDATA% and so the staged install.ps1 # has a writable temp dir. @@ -639,25 +669,10 @@ function Test-SelfUpdateCommand { $savedVersion = $env:VERSION $env:APM_TEMP_DIR = $prefix.TmpDir $env:APM_INSTALL_DIR = $prefix.BinDir - $env:VERSION = $PinnedVersion + $env:VERSION = $CandidateVersion try { - $output = @() - $selfUpdateExit = 1 - for ($attempt = 1; $attempt -le 3; $attempt++) { - Write-Info "self-update attempt $attempt of 3" - $output = & cmd.exe /c "`"$shim`" self-update" 2>&1 - $selfUpdateExit = $LASTEXITCODE - if ($selfUpdateExit -eq 0) { - break - } - $outputText = ($output | Out-String) - if ($outputText -notmatch "Unable to fetch latest version from (GitHub|remote)" -and $outputText -notmatch "Update failed:") { - break - } - if ($attempt -lt 3) { - Start-Sleep -Seconds (5 * $attempt) - } - } + $output = & cmd.exe /c "`"$shim`" self-update" 2>&1 + $selfUpdateExit = $LASTEXITCODE } finally { if ($null -ne $savedTempDir) { $env:APM_TEMP_DIR = $savedTempDir } else { Remove-Item Env:APM_TEMP_DIR -ErrorAction SilentlyContinue } if ($null -ne $savedInstallDir) { $env:APM_INSTALL_DIR = $savedInstallDir } else { Remove-Item Env:APM_INSTALL_DIR -ErrorAction SilentlyContinue } @@ -672,7 +687,9 @@ function Test-SelfUpdateCommand { $ver2 = Get-ShimVersion -ShimPath $shim Assert-True ($ver2.ExitCode -eq 0) "apm.cmd --version exits 0 after self-update" - Assert-True ($ver2.Output -match [regex]::Escape($PinnedVersion.TrimStart("v"))) "apm.cmd --version reports $PinnedVersion after self-update (got: $($ver2.Output))" + Assert-True (Test-ReportedVersion $ver2.Output $CandidateVersion) "apm.cmd --version reports $CandidateVersion after self-update (got: $($ver2.Output))" + Assert-CandidatePayload -Root $prefix.Root + Test-CrossVersionUpgrade -Prefix $prefix } finally { Remove-Item -Recurse -Force $prefix.Root -ErrorAction SilentlyContinue } @@ -688,14 +705,39 @@ Write-Host " APM install.ps1 Windows integration test " Write-Host "=================================================================" -ForegroundColor Blue Write-Host "" -Test-Sha256Fallback -Test-MoveThenTestOrdering -Test-AntivirusDetector -Test-EndToEndInstall -Test-NonJunctionCollision -Test-CrossVersionUpgrade -Test-SameVersionReinstall -Test-SelfUpdateCommand +$savedUserPath = [Environment]::GetEnvironmentVariable("Path", "User") +$savedEnvironment = @{} +foreach ($name in @( + "Path", "VERSION", "APM_INSTALL_DIR", "APM_TEMP_DIR", "APM_SKIP_CHECKSUM", + "APM_RELEASE_BASE_URL", "APM_INSTALLER_BASE_URL", "APM_RELEASE_METADATA_URL", + "APM_NO_DIRECT_FALLBACK", "APM_LAUNCH_TEST_CWD", "APM_LAUNCH_TEST_PATH", + "NO_COLOR", "COLUMNS" +)) { + $savedEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, "Process") +} +try { + $fixture = $FixtureBaseUrl.AbsoluteUri.TrimEnd('/') + $env:APM_RELEASE_BASE_URL = "$fixture/releases" + $env:APM_INSTALLER_BASE_URL = "$fixture/installer" + $env:APM_RELEASE_METADATA_URL = "$fixture/unexpected-metadata" + $env:APM_NO_DIRECT_FALLBACK = "1" + $env:NO_COLOR = "1" + $env:COLUMNS = "200" + Remove-Item Env:APM_SKIP_CHECKSUM -ErrorAction SilentlyContinue + + Test-Sha256Fallback + Test-MoveThenTestOrdering + Test-AntivirusDetector + Test-EndToEndInstall + Test-NonJunctionCollision + Test-CandidateChecksumRejection + Test-SelfUpdateCommand +} finally { + [Environment]::SetEnvironmentVariable("Path", $savedUserPath, "User") + foreach ($name in $savedEnvironment.Keys) { + [Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process") + } +} Write-Host "" Write-Host "=================================================================" -ForegroundColor Blue diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7bf988fedd..e54040b7b5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -281,6 +281,16 @@ def _has_runtime(name: str) -> bool: } +def pytest_addoption(parser: pytest.Parser) -> None: + """Let provisioned CI selections reject missing runtimes instead of skipping.""" + parser.addoption( + "--strict-runtime-prerequisites", + action="store_true", + help="Fail collection when a selected test requires an unavailable runtime", + ) + + +@pytest.hookimpl(trylast=True) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Auto-skip items whose marker precondition is not met. @@ -291,6 +301,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item for item in items: for marker_name, (check_fn, reason) in _MARKER_CHECKS.items(): if item.get_closest_marker(marker_name) and not check_fn(): + if marker_name.startswith("requires_runtime_") and config.getoption( + "--strict-runtime-prerequisites" + ): + raise pytest.UsageError(f"{item.nodeid}: required runtime missing: {reason}") item.add_marker(pytest.mark.skip(reason=reason)) diff --git a/tests/integration/test_mcp_registry_e2e.py b/tests/integration/test_mcp_registry_e2e.py index ca35f1c48c..bc4386a2a9 100644 --- a/tests/integration/test_mcp_registry_e2e.py +++ b/tests/integration/test_mcp_registry_e2e.py @@ -14,12 +14,11 @@ import json import os -import shutil # noqa: F401 import subprocess import tempfile import time +from functools import lru_cache from pathlib import Path -from unittest import mock # noqa: F401 import pytest import toml @@ -30,6 +29,7 @@ _REGISTRY_TRANSIENT_MARKER = "could not reach mcp registry" +@lru_cache(maxsize=1) def _is_registry_healthy() -> bool: """Check if GitHub MCP server has proper package configuration. @@ -64,9 +64,10 @@ def _is_registry_healthy() -> bool: return False +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + # Skip all tests in this module if not in E2E mode E2E_MODE = os.environ.get("APM_E2E_TESTS", "").lower() in ("1", "true", "yes") -GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") pytestmark = pytest.mark.requires_e2e_mode @@ -159,6 +160,16 @@ def temp_e2e_home(): del os.environ["HOME"] +@pytest.fixture +def healthy_github_mcp_registry(): + """Skip selected package-install tests when the live registry lacks Docker packages.""" + if not _is_registry_healthy(): + pytest.skip( + "GitHub MCP server configured as remote-only (no packages) - " + "skipping installation tests" + ) + + class TestMCPRegistryE2E: """E2E tests for MCP registry functionality.""" @@ -207,11 +218,12 @@ def test_mcp_show_command(self, temp_e2e_home, apm_binary_path): print(f"[OK] MCP show displays server details:\n{result.stdout[:500]}...") @pytest.mark.skipif(not GITHUB_TOKEN, reason="GITHUB_TOKEN required for installation tests") - @pytest.mark.skipif( - not _is_registry_healthy(), - reason="GitHub MCP server configured as remote-only (no packages) - skipping installation tests", - ) - def test_registry_installation_with_codex(self, temp_e2e_home, apm_binary_path): + def test_registry_installation_with_codex( + self, + temp_e2e_home, + apm_binary_path, + healthy_github_mcp_registry, + ): """Test complete registry-based installation flow with Codex runtime.""" print("\n=== Testing Registry Installation with Codex ===") @@ -220,7 +232,6 @@ def test_registry_installation_with_codex(self, temp_e2e_home, apm_binary_path): result = run_command(f"{apm_binary_path} runtime setup codex", timeout=300) assert result.returncode == 0, f"Codex setup failed: {result.stderr}" - # Verify codex config was created codex_config = Path(temp_e2e_home) / ".codex" / "config.toml" assert codex_config.exists(), "Codex configuration not created" @@ -343,11 +354,12 @@ def test_registry_installation_with_codex(self, temp_e2e_home, apm_binary_path): else: pytest.fail("Codex configuration file not found after installation") - @pytest.mark.skipif( - not _is_registry_healthy(), - reason="GitHub MCP server configured as remote-only (no packages) - skipping installation tests", - ) - def test_empty_string_handling_copilot_e2e(self, temp_e2e_home, apm_binary_path): + def test_empty_string_handling_copilot_e2e( + self, + temp_e2e_home, + apm_binary_path, + healthy_github_mcp_registry, + ): """Test end-to-end empty string and defaults handling during installation.""" print("\n=== Testing Empty String and Defaults Handling ===") @@ -465,7 +477,11 @@ def test_empty_string_handling_copilot_e2e(self, temp_e2e_home, apm_binary_path) print("[WARN] Copilot configuration not created (binary may not be available)") # This is OK for testing - we're validating the adapter logic - def test_empty_string_handling_e2e(self, temp_e2e_home, apm_binary_path): + def test_empty_string_handling_e2e( + self, + temp_e2e_home, + apm_binary_path, + ): """Test end-to-end empty string and defaults handling during installation.""" print("\n=== Testing Empty String and Defaults Handling ===") @@ -532,11 +548,12 @@ def test_empty_string_handling_e2e(self, temp_e2e_home, apm_binary_path): print("[OK] Empty string handling verified in configuration") - @pytest.mark.skipif( - not _is_registry_healthy(), - reason="GitHub MCP server configured as remote-only (no packages) - skipping installation tests", - ) - def test_cross_adapter_consistency(self, temp_e2e_home, apm_binary_path): + def test_cross_adapter_consistency( + self, + temp_e2e_home, + apm_binary_path, + healthy_github_mcp_registry, + ): """Test that Codex adapter handles MCP server installation consistently.""" print("\n=== Testing Codex Adapter Consistency ===") @@ -600,11 +617,12 @@ def test_cross_adapter_consistency(self, temp_e2e_home, apm_binary_path): else: pytest.fail("No Codex configuration found to verify") - @pytest.mark.skipif( - not _is_registry_healthy(), - reason="GitHub MCP server configured as remote-only (no packages) - skipping installation tests", - ) - def test_duplication_prevention_e2e(self, temp_e2e_home, apm_binary_path): + def test_duplication_prevention_e2e( + self, + temp_e2e_home, + apm_binary_path, + healthy_github_mcp_registry, + ): """Test that repeated installations don't create duplicate entries.""" print("\n=== Testing Duplication Prevention ===") @@ -724,7 +742,7 @@ def test_qualified_ref_does_not_match_different_namespace(self): try: # 'nonexistent-org/github-mcp-server' shares the slug with # 'io.github.github/github-mcp-server' but belongs to a - # different namespace — boundary matching must reject it. + # different namespace - boundary matching must reject it. result = client.find_server_by_reference("nonexistent-org/github-mcp-server") except Exception: pytest.skip("Registry unavailable") diff --git a/tests/integration/test_windows_installer_launchers.py b/tests/integration/test_windows_installer_launchers.py index f621440cea..c91c5688c4 100644 --- a/tests/integration/test_windows_installer_launchers.py +++ b/tests/integration/test_windows_installer_launchers.py @@ -1,13 +1,29 @@ -"""End-to-end regression coverage for Windows installer launchers.""" +"""End-to-end regression coverage for Windows installer launchers. + +Required inputs: APM_CANDIDATE_{ARCHIVE,VERSION,SHA256} and +APM_BASELINE_{ARCHIVE,VERSION,SHA256}. Versions accept packager metadata +(X.Y.Z) or release tags (vX.Y.Z); the baseline is an older source only. +""" from __future__ import annotations -import subprocess +import os from pathlib import Path +from urllib.parse import urlparse import pytest +from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner +from tests.utils.isolated_apm_environment import IsolatedApmEnvironment +from tests.utils.windows_installer_candidate import ( + ASSET_NAME, + InstallerArchive, + preserve_windows_user_path, + serve_installer_candidate, +) + pytestmark = [ + pytest.mark.e2e, pytest.mark.requires_e2e_mode, pytest.mark.requires_windows, ] @@ -15,23 +31,58 @@ ROOT = Path(__file__).resolve().parents[2] -def test_windows_installer_exposes_stable_executable() -> None: +def test_windows_installer_exposes_stable_executable(tmp_path: Path) -> None: """Install APM and launch bare ``apm`` through Windows process boundaries.""" - result = subprocess.run( - [ - "pwsh", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-File", - str(ROOT / "scripts" / "windows" / "test-install-script.ps1"), - ], - cwd=ROOT, - capture_output=True, - text=True, - timeout=600, - check=False, - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert result.stdout.rstrip().endswith("All install.ps1 integration tests passed.") + candidate = InstallerArchive.from_environment(os.environ, "CANDIDATE") + baseline = InstallerArchive.from_environment(os.environ, "BASELINE") + isolated = IsolatedApmEnvironment.create(tmp_path / "installer", base_env=os.environ) + environment = isolated.subprocess_env() + environment["APPDATA"] = str(isolated.home) + with ( + preserve_windows_user_path(), + serve_installer_candidate(candidate, baseline, ROOT / "install.ps1") as mirror, + ): + environment["APM_TEST_LOOPBACK_PORTS"] = str(urlparse(mirror.base_url).port) + runner = ApmLifecycleRunner( + [ + "pwsh", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(ROOT / "scripts" / "windows" / "test-install-script.ps1"), + ], + timeout_seconds=600, + ) + result = runner.run( + [ + "-CandidateVersion", + candidate.version, + "-OlderVersion", + baseline.version, + "-CandidateExecutableSha256", + candidate.executable_sha256, + "-FixtureBaseUrl", + mirror.base_url, + "-TestRoot", + str(isolated.work_root), + ], + cwd=ROOT, + env=environment, + scenario_id="windows-candidate-installer", + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.rstrip().endswith("All install.ps1 integration tests passed.") + assert not mirror.unexpected_paths + assert mirror.requested_paths.count("/installer/install.ps1") == 1 + for release in (candidate, baseline): + path = f"/releases/{release.version}/{ASSET_NAME}" + assert path in mirror.requested_paths + assert path + ".sha256" in mirror.requested_paths + assert mirror.requested_paths.count(f"/releases/{baseline.version}/{ASSET_NAME}") == 1 + assert 3 <= mirror.requested_paths.count(f"/releases/{candidate.version}/{ASSET_NAME}") <= 4 + tampered_path = f"/tampered/{candidate.version}/{ASSET_NAME}" + assert tampered_path in mirror.requested_paths + assert tampered_path + ".sha256" in mirror.requested_paths diff --git a/tests/quality/test_ci_topology.py b/tests/quality/test_ci_topology.py index 123fb3d2a9..6df00770fd 100644 --- a/tests/quality/test_ci_topology.py +++ b/tests/quality/test_ci_topology.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import shlex import subprocess import sys @@ -48,6 +49,7 @@ "Build & Test Shard 1 (Linux)", "Build & Test Shard 2 (Linux)", ) +REQUIRED_SOURCE_CHECKS = ("Lint", RATCHET_CHECK, *REQUIRED_SHARD_CHECKS) REQUIRED_SHARD_ROOTS = ( "tests/unit", "tests/test_console.py", @@ -251,18 +253,13 @@ def _assert_topology(root: Path, inventory: dict[str, str]) -> None: workflows = root / ".github" / "workflows" ci = load_workflow(workflows / "ci.yml") + assert workflow_job(ci, "lint")["name"] == "Lint" ratchet = workflow_job(ci, RATCHET_JOB) assert ratchet["name"] == RATCHET_CHECK _assert_ci_test_targets(ci, inventory) merge_gate = load_workflow(workflows / "merge-gate.yml") - gate = workflow_job(merge_gate, "gate") - wait_step = workflow_step(gate, "Wait for all required checks") - expected_checks = wait_step["env"]["EXPECTED_CHECKS"] - assert isinstance(expected_checks, str) - assert RATCHET_CHECK not in expected_checks - for required_check in REQUIRED_SHARD_CHECKS: - assert required_check in expected_checks + _assert_required_source_checks(merge_gate) ratchet_callers: list[tuple[str, str]] = [] for workflow_path in sorted(workflows.glob("*.yml")): @@ -285,12 +282,51 @@ def repository_inventory( return _ratchet_test_inventory(repository_python_inventory) -def test_test_architecture_ratchets_remain_non_required( +def _assert_required_source_checks(merge_gate: WorkflowNode) -> None: + """Require exact source-check names in each event's declared gate policy.""" + wait_step = workflow_step(workflow_job(merge_gate, "gate"), "Wait for all required checks") + expression = wait_step["env"]["EXPECTED_CHECKS"] + assert isinstance(expression, str) + match = re.fullmatch( + r"\$\{\{\s*github\.event_name == 'merge_group' && '([^']+)' \|\| '([^']+)'\s*\}\}", + expression, + ) + assert match is not None, "expected explicit merge_group and pull_request check lists" + for event, checks in zip(("merge_group", "pull_request"), match.groups(), strict=True): + names = [name.strip() for name in checks.split(",")] + assert len(names) == len(set(names)), f"{event}: duplicate required check names" + assert "gate" not in names, f"{event}: gate cannot require itself" + for required_check in REQUIRED_SOURCE_CHECKS: + assert required_check in names, f"{event}: missing required check {required_check}" + + +def test_source_checks_and_architecture_ratchets_are_required( repository_inventory: dict[str, str], ) -> None: _assert_topology(REPO_ROOT, repository_inventory) +@pytest.mark.parametrize("required_check", REQUIRED_SOURCE_CHECKS) +@pytest.mark.parametrize("event", ["merge_group", "pull_request"]) +@pytest.mark.parametrize("replacement", ["", "optional "]) +def test_required_source_check_removed_or_renamed_fails( + required_check: str, event: str, replacement: str +) -> None: + """Either event losing a source check must break, even if the other retains it.""" + merge_gate = deepcopy(load_workflow(REPO_ROOT / ".github/workflows/merge-gate.yml")) + wait_step = workflow_step(workflow_job(merge_gate, "gate"), "Wait for all required checks") + branches = wait_step["env"]["EXPECTED_CHECKS"].split(" || ") + index = 0 if event == "merge_group" else 1 + if replacement: + branches[index] = branches[index].replace(required_check, replacement + required_check) + else: + branches[index] = branches[index].replace(required_check + ",", "") + wait_step["env"]["EXPECTED_CHECKS"] = " || ".join(branches) + + with pytest.raises(AssertionError, match=f"{event}: missing required check"): + _assert_required_source_checks(merge_gate) + + def test_repository_ratchet_inventory_is_collected_once( repository_inventory: dict[str, str], ) -> None: diff --git a/tests/unit/scripts/test_package_release.py b/tests/unit/scripts/test_package_release.py new file mode 100644 index 0000000000..e52b06edd3 --- /dev/null +++ b/tests/unit/scripts/test_package_release.py @@ -0,0 +1,195 @@ +"""Release publication uses the same checked archive as native validation.""" + +from __future__ import annotations + +import base64 +import json +import os +import shutil +import subprocess +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from scripts.package_release import ( + BINARY_NAMES, + archive_name, + file_digest, + package, + require_binary_identity, + verify_extract, +) + +pytestmark = pytest.mark.component +SHA = "a" * 40 + + +@pytest.fixture +def candidate(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path, Path]: + """Create a minimal versioned native bundle without an external build.""" + dist = tmp_path / "dist" + archives = tmp_path / "release-assets" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[project]\nversion = "1.2.3"\n', encoding="ascii") + monkeypatch.setattr( + "scripts.package_release.subprocess.run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args, 0, stdout=f"Agent Package Manager (APM) CLI version 1.2.3 ({SHA[:9]})\n" + ), + ) + return dist, archives, pyproject + + +@pytest.mark.parametrize("binary_name", BINARY_NAMES) +def test_archive_round_trip_keeps_candidate_bytes_and_hidden_files( + candidate: tuple[Path, Path, Path], tmp_path: Path, binary_name: str +) -> None: + dist, archives, pyproject = candidate + bundle = dist / binary_name + bundle.mkdir(parents=True) + executable = bundle / ("apm.exe" if binary_name.endswith("windows-x86_64") else "apm") + executable.write_bytes(b"candidate-native-executable") + hidden = bundle / "_internal" / ".apm" + hidden.mkdir(parents=True) + (hidden / "fixture").write_bytes(b"hidden") + metadata = package(binary_name, dist, archives, SHA, pyproject) + + extracted = verify_extract(binary_name, archives, tmp_path / "isolated", SHA) + + assert metadata["version"] == "1.2.3" + assert extracted.read_bytes() == executable.read_bytes() + assert (extracted.parent / "_internal" / ".apm" / "fixture").read_bytes() == b"hidden" + assert (archives / archive_name(binary_name)).is_file() + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("sha", "b" * 40), + ("archive", "../unexpected.zip"), + ("binary_name", "apm-linux-arm64"), + ("archive_sha256", "0" * 64), + ("schema_version", 2), + ], +) +def test_candidate_identity_or_digest_drift_fails_before_extraction( + candidate: tuple[Path, Path, Path], tmp_path: Path, field: str, value: str | int +) -> None: + dist, archives, pyproject = candidate + binary_name = "apm-linux-x86_64" + (dist / binary_name).mkdir(parents=True) + (dist / binary_name / "apm").write_bytes(b"candidate") + metadata = package(binary_name, dist, archives, SHA, pyproject) + metadata[field] = value + (archives / f"{binary_name}.json").write_text(json.dumps(metadata), encoding="ascii") + + with pytest.raises(ValueError, match="Candidate"): + verify_extract(binary_name, archives, tmp_path / "isolated", SHA) + + assert not (tmp_path / "isolated").exists() + + +@pytest.mark.parametrize( + "output", + [ + f"Agent Package Manager (APM) CLI version 1.2.2 ({SHA[:9]})", + "Agent Package Manager (APM) CLI version 1.2.3 (bbbbbbbbb)", + "Agent Package Manager (APM) CLI version 1.2.3", + ], +) +def test_published_or_unbound_binary_cannot_become_candidate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: str +) -> None: + monkeypatch.setattr( + "scripts.package_release.subprocess.run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, stdout=output), + ) + with pytest.raises(ValueError, match="version/build SHA"): + require_binary_identity(tmp_path / "apm", "1.2.3", SHA) + + +def test_tampered_archive_is_not_extracted( + candidate: tuple[Path, Path, Path], tmp_path: Path +) -> None: + dist, archives, pyproject = candidate + binary_name = "apm-linux-x86_64" + (dist / binary_name).mkdir(parents=True) + (dist / binary_name / "apm").write_bytes(b"candidate") + package(binary_name, dist, archives, SHA, pyproject) + (archives / archive_name(binary_name)).write_bytes(b"tampered") + + with pytest.raises(ValueError, match="checksum mismatch"): + verify_extract(binary_name, archives, tmp_path / "isolated", SHA) + + assert not (tmp_path / "isolated").exists() + + +@pytest.mark.parametrize("binary_name", ["apm-linux-x86_64", "apm-windows-x86_64"]) +@pytest.mark.parametrize( + "suffix", ["/../escape", "/nested/../../escape", "/C:stream", r"/..\escape"] +) +def test_matching_digest_does_not_allow_archive_path_escape( + candidate: tuple[Path, Path, Path], tmp_path: Path, binary_name: str, suffix: str +) -> None: + dist, archives, pyproject = candidate + (dist / binary_name).mkdir(parents=True) + executable = "apm.exe" if binary_name.startswith("apm-windows") else "apm" + (dist / binary_name / executable).write_bytes(b"candidate") + metadata = package(binary_name, dist, archives, SHA, pyproject) + archive = archives / archive_name(binary_name) + if archive.suffix == ".zip": + with zipfile.ZipFile(archive, "w") as target: + target.writestr(binary_name + suffix, b"unsafe") + else: + with tarfile.open(archive, "w:gz") as target: + target.addfile(tarfile.TarInfo(binary_name + suffix)) + digest = file_digest(archive) + metadata["archive_sha256"] = digest + (archives / f"{binary_name}.json").write_text(json.dumps(metadata), encoding="ascii") + (archives / f"{archive.name}.sha256").write_text( + f"{digest} {archive.name}\n", encoding="ascii" + ) + + with pytest.raises(ValueError, match="unsafe member path"): + verify_extract(binary_name, archives, tmp_path / "isolated", SHA) + + assert not (tmp_path / "isolated" / binary_name).exists() + + +@pytest.mark.windows_compat +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_windows_build_injects_sha_into_a_multiline_version_module(newline: str) -> None: + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("PowerShell is required for the native build-script regression") + script = Path(__file__).resolve().parents[3] / "scripts/windows/build-binary.ps1" + assignment = next( + line.strip() + for line in script.read_text("utf-8").splitlines() + if line.strip().startswith("$newContent = ") + ) + original = newline.join(['"""Version metadata."""', "__BUILD_SHA__ = None", ""]) + probe = ( + "$ErrorActionPreference = 'Stop'; " + "$originalContent = [Text.Encoding]::UTF8.GetString(" + "[Convert]::FromBase64String($env:PROBE_CONTENT)); " + f"$BuildSHA = '{SHA[:9]}'; {assignment}; " + "$newContent | ConvertTo-Json -Compress" + ) + result = subprocess.run( + [pwsh, "-NoProfile", "-NonInteractive", "-Command", probe], + capture_output=True, + text=True, + check=True, + timeout=30, + env={ + **os.environ, + "PROBE_CONTENT": base64.b64encode(original.encode("utf-8")).decode("ascii"), + }, + ) + updated = json.loads(result.stdout) + assert f'__BUILD_SHA__ = "{SHA[:9]}"' in updated + assert "__BUILD_SHA__ = None" not in updated + assert updated.startswith('"""Version metadata."""') diff --git a/tests/unit/test_ado_live_workflow_contract.py b/tests/unit/test_ado_live_workflow_contract.py index 3f6ffb0768..ff54f404e9 100644 --- a/tests/unit/test_ado_live_workflow_contract.py +++ b/tests/unit/test_ado_live_workflow_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from copy import deepcopy from pathlib import Path from typing import Any @@ -20,15 +21,11 @@ ) ROOT = Path(__file__).resolve().parents[2] -RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "build-release.yml" +RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release-platform.yml" AUTH_WORKFLOW = ROOT / ".github" / "workflows" / "auth-acceptance.yml" INTEGRATION_SCRIPT = ROOT / "scripts" / "test-integration.sh" LIVE_ADO_SELECTOR = "live and requires_ado_pat" -RELEASE_INTEGRATION_STEPS = ( - ("build-and-validate-macos-intel", "Run focused Intel integration tests"), - ("build-and-validate-macos-arm", "Run integration tests"), - ("integration-tests", "Run integration tests (Unix)"), -) +RELEASE_INTEGRATION_STEPS = (("integration-tests", "Run integration tests (Unix)"),) def _walk_nodes(value: Any) -> list[dict[str, Any]]: @@ -56,7 +53,9 @@ def _assert_release_excludes_live_ado(workflow: dict[str, Any]) -> None: step = workflow_step(job, step_name) marker_expression = effective_env(workflow, job, step).get("PYTEST_MARK_EXPR") assert isinstance(marker_expression, str) - assert "not live" in marker_expression + assert marker_expression == "${{ inputs.integration-markers }}" + platforms = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) + assert all("not live" in row["integration_markers"] for row in platforms) windows_step = workflow_step( workflow_job(workflow, "integration-tests"), @@ -156,8 +155,8 @@ def test_release_live_selector_mutation_is_rejected() -> None: """Publication cannot silently regain live external-service nodes.""" workflow = deepcopy(load_workflow(RELEASE_WORKFLOW)) step = workflow_step( - workflow_job(workflow, "build-and-validate-macos-arm"), - "Run integration tests", + workflow_job(workflow, "integration-tests"), + "Run integration tests (Unix)", ) step["env"]["PYTEST_MARK_EXPR"] = "live" diff --git a/tests/unit/test_mcp_registry_collection.py b/tests/unit/test_mcp_registry_collection.py new file mode 100644 index 0000000000..ad48d2d7d8 --- /dev/null +++ b/tests/unit/test_mcp_registry_collection.py @@ -0,0 +1,58 @@ +"""Collection regression tests for live MCP registry integration coverage.""" + +from __future__ import annotations + +import runpy +import socket +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.component + + +def test_mcp_registry_e2e_collection_import_does_not_open_network( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The collection import phase for the live E2E module must not probe the registry.""" + repo_root = Path(__file__).resolve().parents[2] + attempts_log = tmp_path / "network-attempts.log" + home = tmp_path / "home" + home.mkdir() + + def record(kind: str, target: object) -> None: + with attempts_log.open("a", encoding="utf-8") as handle: + handle.write(f"{kind}: {target!r}\n") + + def blocked_connect(self, address): + record("connect", address) + raise RuntimeError("network attempted during collection") + + def blocked_create_connection(address, *args, **kwargs): + record("create_connection", address) + raise RuntimeError("network attempted during collection") + + def blocked_getaddrinfo(host, *args, **kwargs): + record("getaddrinfo", host) + raise RuntimeError("network attempted during collection") + + monkeypatch.setattr(socket.socket, "connect", blocked_connect) + monkeypatch.setattr(socket, "create_connection", blocked_create_connection) + monkeypatch.setattr(socket, "getaddrinfo", blocked_getaddrinfo) + monkeypatch.setenv("APM_NO_CACHE", "1") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("MCP_REGISTRY_CONNECT_TIMEOUT", "0.001") + monkeypatch.setenv("MCP_REGISTRY_READ_TIMEOUT", "0.001") + monkeypatch.delenv("MCP_REGISTRY_URL", raising=False) + monkeypatch.syspath_prepend(str(repo_root / "src")) + sys.modules.pop("test_mcp_registry_e2e", None) + sys.modules.pop("tests.integration.test_mcp_registry_e2e", None) + + runpy.run_path( + str(repo_root / "tests" / "integration" / "test_mcp_registry_e2e.py"), + run_name="__apm_collection_probe__", + ) + + assert not attempts_log.exists(), attempts_log.read_text(encoding="utf-8") diff --git a/tests/unit/test_merge_gate_wait.py b/tests/unit/test_merge_gate_wait.py new file mode 100644 index 0000000000..79af295692 --- /dev/null +++ b/tests/unit/test_merge_gate_wait.py @@ -0,0 +1,309 @@ +"""Hermetic regressions for the required-check aggregator's shell boundary.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = [ + pytest.mark.component, + pytest.mark.skipif(os.name == "nt", reason="The merge gate executes on a POSIX runner"), +] + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github/scripts/ci/merge_gate_wait.sh" +SHA = "a" * 40 +GateRunner = Callable[..., subprocess.CompletedProcess[str]] + +FAKE_TOOLS = """\ +import json +import os +import sys +from pathlib import Path + +root = Path(os.environ["GATE_TEST_ROOT"]) +tool = Path(sys.argv[0]).name +if tool == "date": + clock = root / "clock" + tick = int(clock.read_text()) if clock.exists() else 0 + clock.write_text(str(tick + 1)) + print(tick if tick <= int(os.environ["GATE_TEST_POLLS"]) else 61) +elif tool == "gh": + with (root / "requests").open("a") as stream: + stream.write(json.dumps(sys.argv[1:]) + "\\n") + state_path = root / "calls" + index = int(state_path.read_text()) if state_path.exists() else 0 + state_path.write_text(str(index + 1)) + pages = [] + for rounds in json.loads((root / "responses").read_text()).values(): + response = rounds[min(index, len(rounds) - 1)] + if response == "api-error": + sys.exit(1) + if isinstance(response, str): + print(response) + sys.exit(0) + pages.extend(response) + print(json.dumps(pages)) +""" + + +def _check( + name: str = "Lint", + *, + conclusion: str | None = "success", + status: str = "completed", + app: str = "github-actions", + sha: str = SHA, +) -> dict[str, object]: + """Build one Checks API result with explicit source identity.""" + return { + "name": name, + "head_sha": sha, + "status": status, + "conclusion": conclusion, + "app": {"slug": app}, + "html_url": "https://github.com/microsoft/apm/actions/runs/123/job/456", + } + + +def _pages(*checks: dict[str, object]) -> list[dict[str, object]]: + """Match gh api --paginate --slurp output without making network calls.""" + return [{"check_runs": list(checks)}] + + +@pytest.fixture +def run_gate(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> GateRunner: + """Run the real Bash/JQ logic with a deterministic clock and fake GitHub.""" + assert shutil.which("bash"), "bash is required to exercise the deployed gate" + assert shutil.which("jq"), "jq is required to exercise the deployed gate" + for command in ("gh", "date", "sleep"): + tool = tmp_path / command + tool.write_text(f"#!{sys.executable}\n{FAKE_TOOLS}", encoding="ascii") + tool.chmod(0o755) + monkeypatch.setenv("PATH", f"{tmp_path}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("GATE_TEST_ROOT", str(tmp_path)) + monkeypatch.setenv("GH_TOKEN", "fake-token") + monkeypatch.setenv("REPO", "microsoft/apm") + monkeypatch.setenv("SHA", SHA) + monkeypatch.setenv("TIMEOUT_MIN", "1") + monkeypatch.setenv("POLL_SEC", "0") + + def run( + responses: dict[str, list[object]], + *, + polls: int = 1, + event: str = "pull_request", + script: Path = SCRIPT, + ) -> subprocess.CompletedProcess[str]: + (tmp_path / "responses").write_text(json.dumps(responses), encoding="ascii") + monkeypatch.setenv("EXPECTED_CHECKS", ",".join(responses)) + monkeypatch.setenv("GATE_TEST_POLLS", str(polls)) + monkeypatch.setenv("EVENT_NAME", event) + return subprocess.run( + ["bash", str(script)], + cwd=ROOT, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + return run + + +def test_all_required_successes_pass_in_the_final_poll( + run_gate: GateRunner, tmp_path: Path +) -> None: + """Success must not require another sleep/poll beyond the deadline.""" + names = ("Lint", "Test Architecture Ratchets", "Build & Test Shard 1 (Linux)") + result = run_gate({name: [_pages(_check(name))] for name in names}) + + assert result.returncode == 0, result.stdout + result.stderr + assert "all 3 check(s) completed successfully" in result.stdout + calls = [json.loads(line) for line in (tmp_path / "requests").read_text().splitlines()] + assert len(calls) == 1 + arguments = calls[0] + assert "--paginate" in arguments + assert "--slurp" in arguments + parsed = urlparse(arguments[-1]) + assert parsed.path == f"repos/microsoft/apm/commits/{SHA}/check-runs" + assert parse_qs(parsed.query) == {"filter": ["latest"], "per_page": ["100"]} + + +@pytest.mark.parametrize( + "conclusion", + [ + "skipped", + "neutral", + "failure", + "cancelled", + "timed_out", + "action_required", + "stale", + "startup_failure", + None, + "unknown", + ], +) +def test_required_check_without_success_fails_closed( + run_gate: GateRunner, conclusion: str | None +) -> None: + result = run_gate({"Lint": [_pages(_check(conclusion=conclusion))]}) + + assert result.returncode == 1 + assert "Required check failed" in result.stdout + assert "completed successfully" not in result.stdout + + +@pytest.mark.parametrize("event", ["pull_request", "merge_group"]) +@pytest.mark.parametrize("response", [_pages(), "api-error", "invalid-json"]) +def test_missing_or_unreadable_required_check_blocks_with_recovery( + run_gate: GateRunner, response: object, event: str +) -> None: + result = run_gate({"Lint": [response]}, event=event) + + assert result.returncode == 2 + assert "Required check never started" in result.stderr + assert "Lint" in result.stderr + if event == "merge_group": + assert "Remove the PR from the merge queue and re-add it" in result.stderr + else: + assert "close and reopen the PR" in result.stderr + + +@pytest.mark.parametrize("status", ["queued", "in_progress", "waiting"]) +def test_unfinished_required_check_times_out(run_gate: GateRunner, status: str) -> None: + result = run_gate({"Lint": [_pages(_check(status=status, conclusion=None))]}) + + assert result.returncode == 3 + assert "Required check timeout" in result.stderr + urls = [urlparse(token) for token in result.stderr.split() if "://" in token] + assert [(url.hostname, url.path) for url in urls] == [ + ("github.com", "/microsoft/apm/actions/runs/123/job/456") + ] + + +@pytest.mark.parametrize( + ("first_response", "timeout_code"), + [ + (_pages(), 2), + (_pages(_check("Test Architecture Ratchets", status="queued", conclusion=None)), 3), + ], +) +@pytest.mark.parametrize("polls", [1, 2]) +def test_partial_success_waits_for_every_required_check( + run_gate: GateRunner, first_response: object, timeout_code: int, polls: int +) -> None: + """A late check may recover, but cannot be replaced by another check's pass.""" + result = run_gate( + { + "Lint": [_pages(_check())], + "Test Architecture Ratchets": [ + first_response, + _pages(_check("Test Architecture Ratchets")), + ], + }, + polls=polls, + ) + + assert result.returncode == (timeout_code if polls == 1 else 0) + assert ("completed successfully" in result.stdout) is (polls == 2) + + +@pytest.mark.parametrize( + "override", + [{"app": "another-app"}, {"name": "Other Lint"}, {"sha": "b" * 40}], +) +def test_wrong_check_identity_cannot_satisfy_requirement( + run_gate: GateRunner, override: dict[str, str] +) -> None: + result = run_gate({"Lint": [_pages(_check(**override))]}) + + assert result.returncode == 2 + assert "Required check never started" in result.stderr + + +def test_external_app_collision_does_not_hide_actions_failure(run_gate: GateRunner) -> None: + result = run_gate({"Lint": [_pages(_check(conclusion="failure"), _check(app="another-app"))]}) + + assert result.returncode == 1 + assert "FAILED (failure)" in result.stdout + + +@pytest.mark.parametrize("other_conclusion", ["success", "failure"]) +def test_duplicate_actions_names_on_different_pages_fail_closed( + run_gate: GateRunner, other_conclusion: str +) -> None: + result = run_gate({"Lint": [[*_pages(_check()), *_pages(_check(conclusion=other_conclusion))]]}) + + assert result.returncode == 1 + assert "Ambiguous required check" in result.stdout + + +def test_rerun_cannot_reuse_an_earlier_success(run_gate: GateRunner) -> None: + result = run_gate( + { + "Lint": [_pages(_check()), _pages(_check(conclusion="failure"))], + "Test Architecture Ratchets": [ + _pages(_check("Test Architecture Ratchets", status="in_progress", conclusion=None)), + _pages(_check("Test Architecture Ratchets")), + ], + }, + polls=2, + ) + + assert result.returncode == 1 + assert "FAILED (failure)" in result.stdout + + +@pytest.mark.parametrize("conclusion", ["skipped", "neutral"]) +def test_accepting_non_success_mutation_is_detected( + run_gate: GateRunner, tmp_path: Path, conclusion: str +) -> None: + """A policy-relaxing mutation produces the forbidden green verdict.""" + source = SCRIPT.read_text(encoding="ascii") + assert "\n success)\n" in source + mutant = tmp_path / "mutant.sh" + mutant.write_text( + source.replace("\n success)\n", f"\n success|{conclusion})\n"), + encoding="ascii", + ) + result = run_gate({"Lint": [_pages(_check(conclusion=conclusion))]}, script=mutant) + + with pytest.raises(AssertionError): + assert result.returncode == 1 + + +def test_gate_retains_read_only_token_and_trusted_base_checkout() -> None: + """PRs must not execute their own gate script or receive production secrets.""" + workflow = load_workflow(ROOT / ".github/workflows/merge-gate.yml") + assert set(workflow["on"]) == {"pull_request", "merge_group"} + assert workflow["permissions"] == { + "contents": "read", + "checks": "read", + "pull-requests": "read", + } + gate = workflow_job(workflow, "gate") + checkout = workflow_step(gate, "Checkout trusted gate implementation") + assert checkout["with"]["ref"] == "${{ steps.sha.outputs.trusted_sha }}" + assert checkout["with"]["persist-credentials"] is False + resolver = workflow_step(gate, "Resolve target and trusted SHAs")["run"] + for expression in ( + "github.event.pull_request.base.sha", + "github.event.merge_group.base_sha", + ): + assert expression in resolver + wait = workflow_step(gate, "Wait for all required checks") + assert wait["env"]["GH_TOKEN"] == "${{ github.token }}" + assert wait["env"]["SHA"] == "${{ steps.sha.outputs.target_sha }}" + assert "secrets." not in json.dumps(workflow) diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index 8db41d73ab..ce910c67e8 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -1,7 +1,13 @@ -"""Semantic contracts for hosted PR6 platform evidence.""" +"""Semantic contracts for parallel, native, exact-artifact release evidence.""" from __future__ import annotations +import hashlib +import json +import os +import re +import subprocess +import zipfile from copy import deepcopy from pathlib import Path @@ -20,359 +26,420 @@ ) ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github" / "workflows" / "build-release.yml" -WINDOWS_RELEASE_VALIDATION = ROOT / "scripts" / "windows" / "test-release-validation.ps1" -MACOS_VERSION_TEST_ID = ( - "tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_version_runs" -) -MACOS_RICH_TABLE_TEST_ID = ( - "tests/integration/test_core_smoke.py::TestBinaryStartup::test_apm_rich_table_runs" -) -WINDOWS_TEST_ID = "tests/integration/test_windows_installer_launchers.py" -MACOS_STARTUP_CONTRACTS = ( - ( - "build-and-validate-macos-intel", - "macos-15-intel", - "${{ github.workspace }}/dist/apm-darwin-x86_64/apm", - None, - ), - ( - "build-and-validate-macos-arm", - "macos-latest", - "${{ github.workspace }}/dist/apm-darwin-arm64/apm", - "github.ref_type == 'tag' || github.event_name == 'schedule' || " - "github.event_name == 'repository_dispatch'", - ), -) -NON_LIVE_UNIX_INTEGRATION_STEPS = ( - ("integration-tests", "Run integration tests (Unix)"), - ("build-and-validate-macos-arm", "Run integration tests"), -) -NON_LIVE_UNIX_TIMEOUT_MINUTES = { - ("integration-tests", "Run integration tests (Unix)"): 60, - ("build-and-validate-macos-arm", "Run integration tests"): 60, -} -NON_LIVE_UNIX_PYTEST_ARGS = "-n 4 --dist loadgroup" -NON_LIVE_MARK_EXPRESSION = "not live" -INTEL_FOCUSED_INTEGRATION_STEP = "Run focused Intel integration tests" -INTEL_FOCUSED_MARK_EXPRESSION = "lifecycle_smoke and not live" -RUNTIME_SETUP_STEPS = ( - ("build-and-test", "Run smoke tests"), - ("build-and-validate-macos-intel", INTEL_FOCUSED_INTEGRATION_STEP), - ("build-and-validate-macos-intel", "Run release validation tests"), - ("build-and-validate-macos-arm", "Run integration tests"), - ("build-and-validate-macos-arm", "Run release validation tests"), - ("integration-tests", "Run integration tests (Unix)"), - ("integration-tests", "Run integration tests (Windows)"), - ("release-validation", "Run release validation tests (Unix)"), - ("release-validation", "Run release validation tests (Windows)"), +WORKFLOW = ROOT / ".github/workflows/release-platform.yml" +RELEASE_WORKFLOW = ROOT / ".github/workflows/build-release.yml" +UNIX_TIMEOUT = "${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }}" +UNIX_ARGS = ( + "-n 4 --dist loadgroup --durations=50 --store-durations --junitxml=test-results/integration.xml" ) +SMOKE = "Test native binary startup and core contracts" +INSTALLER = "Test install.ps1 end-to-end (Windows)" def _workflow() -> dict: return load_workflow(WORKFLOW) -def _assert_macos_startup_steps(workflow: dict) -> None: - for job_id, runner, binary_path, job_condition in MACOS_STARTUP_CONTRACTS: - job = workflow_job(workflow, job_id) - step = workflow_step(job, "Test macOS non-shell binary startup") - if job_condition is None: - assert_unconditional(job, label=f"{job_id} job") - else: - assert job.get("if") == job_condition - assert_unconditional(step, label=f"{job_id} startup step") - assert job["runs-on"] == runner - assert effective_env(workflow, job, step).get("GITHUB_TOKEN") is None - assert step["env"] == { - "APM_E2E_TESTS": "1", - "APM_BINARY_PATH": binary_path, - } - tokens = shell_tokens(step) - assert tokens[:3] == ["test", "-x", "$APM_BINARY_PATH"] - assert_exact_command( - shell_commands(step), - [ - "uv", - "run", - "--frozen", - "pytest", - MACOS_VERSION_TEST_ID, - MACOS_RICH_TABLE_TEST_ID, - "-vv", - "-ra", - "--tb=short", - ], - label=f"{job_id} startup step", - ) - assert workflow_step_index(job, "Build binary") < workflow_step_index( - job, - "Test macOS non-shell binary startup", - ) - assert workflow_step_index( - job, - "Test macOS non-shell binary startup", - ) < workflow_step_index(job, "Upload binary as workflow artifact") +def test_candidate_source_authorities_match_the_real_reusable_workflow() -> None: + """Reject invented job names and omitted authorities in promotion evidence.""" + source = load_workflow(ROOT / ".github/workflows/ci.yml") + expected = set() + for job_id, job in source["jobs"].items(): + if job_id == "pr-binary-smoke": + continue + name = job["name"] + shards = job.get("strategy", {}).get("matrix", {}).get("shard", [None]) + for shard in shards: + expected.add( + "Candidate Source Checks / " + name.replace("${{ matrix.shard }}", str(shard)) + ) + completed = subprocess.run( + [ + "node", + "-e", + "process.stdout.write(JSON.stringify(" + "require('./scripts/release-candidate.cjs').REQUIRED_SOURCE_JOB_NAMES));", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + timeout=15, + ) + names = json.loads(completed.stdout) + assert len(names) == len(set(names)) + assert set(names) == expected + + +def _execute_native_gate(results: dict, full: bool, platform: str) -> list[str]: + gate = workflow_job(_workflow(), "gate") + script = workflow_step(gate, "Require every applicable native result")["with"]["script"] + harness = ( + "const failures = [];" + "const core = {setFailed: message => failures.push(message)};" + f"(new Function('core', {json.dumps(script)}))(core);" + "process.stdout.write(JSON.stringify(failures));" + ) + completed = subprocess.run( + ["node", "-e", harness], + check=True, + capture_output=True, + text=True, + timeout=15, + env={ + **os.environ, + "RESULTS": json.dumps(results), + "FULL": str(full).lower(), + "PLATFORM": platform, + }, + ) + return json.loads(completed.stdout) -def _assert_windows_installer_step(workflow: dict) -> None: - job = workflow_job(workflow, "build-and-test") - step = workflow_step(job, "Test install.ps1 end-to-end (Windows)") - assert step.get("if") == "matrix.platform == 'windows'" +@pytest.mark.parametrize( + "job_name", + ["unit-tests", "build", "integration-tests", "release-validation", "windows-installer"], +) +@pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "neutral", None]) +def test_native_gate_executes_fail_closed_for_each_required_job( + job_name: str, result: str | None +) -> None: + gate = workflow_job(_workflow(), "gate") + results = {name: {"result": "success"} for name in gate["needs"]} + if result is None: + del results[job_name] + else: + results[job_name]["result"] = result + assert _execute_native_gate(results, True, "windows") == [f"{job_name} did not succeed"] + + +@pytest.mark.parametrize("full", [False, True]) +@pytest.mark.parametrize("platform", ["linux", "darwin", "windows"]) +def test_native_gate_accepts_only_applicable_successes(full: bool, platform: str) -> None: + gate = workflow_job(_workflow(), "gate") + results = {name: {"result": "success"} for name in gate["needs"]} + if not full: + results["integration-tests"]["result"] = "skipped" + results["release-validation"]["result"] = "skipped" + if platform != "windows": + results["windows-installer"]["result"] = "skipped" + assert _execute_native_gate(results, full, platform) == [] + + +def _assert_native_startup(workflow: dict) -> None: + job = workflow_job(workflow, "build") + assert_unconditional(job, label="native candidate build") + assert job["runs-on"] == "${{ inputs.runner }}" + step = workflow_step(job, SMOKE) + assert_unconditional(step, label="native artifact smoke") assert effective_env(workflow, job, step).get("GITHUB_TOKEN") is None - assert step["env"] == {"APM_E2E_TESTS": "1"} - tokens = shell_tokens(step) - assert tokens[:4] == ["uv", "run", "--frozen", "pytest"] - assert WINDOWS_TEST_ID in tokens - assert "-vv" in tokens - assert "-ra" in tokens - assert "--tb=short" in tokens + assert step["env"] == { + "PYTHONUTF8": "1", + "APM_E2E_TESTS": "1", + "APM_BINARY_PATH": ( + "${{ github.workspace }}/dist/${{ inputs.binary-name }}/" + "${{ inputs.platform == 'windows' && 'apm.exe' || 'apm' }}" + ), + } + assert_exact_command( + shell_commands(step), + [ + "uv", + "run", + "--frozen", + "pytest", + "tests/integration/test_core_smoke.py", + "-vv", + "-ra", + "--tb=short", + "--junitxml=test-results/core.xml", + ], + label="native core smoke", + ) + for build in ("Build binary (Unix)", "Build binary (Windows)"): + assert workflow_step_index(job, build) < workflow_step_index(job, SMOKE) + assert workflow_step_index(job, SMOKE) < workflow_step_index( + job, "Package exact candidate archive" + ) + assert workflow_step_index(job, "Package exact candidate archive") < workflow_step_index( + job, "Upload binary as workflow artifact" + ) -def _assert_standalone_integration_timeouts(workflow: dict) -> None: - job = workflow_job(workflow, "integration-tests") - unix_step = workflow_step(job, "Run integration tests (Unix)") - windows_step = workflow_step(job, "Run integration tests (Windows)") - assert unix_step.get("timeout-minutes") == 60 - assert windows_step.get("timeout-minutes") == 20 - - -def _assert_non_live_unix_integration_parallelism(workflow: dict) -> None: - for job_id, step_name in NON_LIVE_UNIX_INTEGRATION_STEPS: - step = workflow_step(workflow_job(workflow, job_id), step_name) - assert step["env"].get("PYTEST_MARK_EXPR") == NON_LIVE_MARK_EXPRESSION - assert step["env"].get("PYTEST_EXTRA_ARGS") == NON_LIVE_UNIX_PYTEST_ARGS - assert step.get("timeout-minutes") == NON_LIVE_UNIX_TIMEOUT_MINUTES[(job_id, step_name)] - - -def _assert_intel_focused_integration(workflow: dict) -> None: - job = workflow_job(workflow, "build-and-validate-macos-intel") - step = workflow_step(job, INTEL_FOCUSED_INTEGRATION_STEP) - assert step.get("if") == ( - "github.ref_type == 'tag' || github.event_name == 'schedule' || " - "github.event_name == 'repository_dispatch'" +def _assert_windows_installer(workflow: dict) -> None: + job = workflow_job(workflow, "windows-installer") + assert job["needs"] == ["build"] + assert job["if"] == "inputs.platform == 'windows'" + step = workflow_step(job, INSTALLER) + env = effective_env(workflow, job, step) + assert env.get("GITHUB_TOKEN") is None + assert env.get("GH_TOKEN") is None + assert step["env"]["APM_E2E_TESTS"] == "1" + assert step["env"]["APM_CANDIDATE_ARCHIVE"] == ( + "${{ github.workspace }}/release-assets/apm-windows-x86_64.zip" + ) + assert step["env"]["APM_BASELINE_ARCHIVE"] == ( + "${{ github.workspace }}/installer-baseline/apm-windows-x86_64.zip" + ) + assert env["APM_BASELINE_VERSION"] == "v0.28.0" + assert "APM_CANDIDATE_VERSION" in step["run"] + assert "APM_CANDIDATE_SHA256" in step["run"] + baseline = workflow_step(job, "Download historical upgrade source once") + assert baseline["env"] == {"GH_TOKEN": "${{ github.token }}"} + assert "gh release download $env:APM_BASELINE_VERSION" in baseline["run"] + assert "if ($actual -ne $expected)" in baseline["run"] + assert workflow_step_index(job, "Download historical upgrade source once") < ( + workflow_step_index(job, INSTALLER) ) - assert step["env"].get("PYTEST_MARK_EXPR") == INTEL_FOCUSED_MARK_EXPRESSION - assert step["env"].get("PYTEST_EXTRA_ARGS") == NON_LIVE_UNIX_PYTEST_ARGS - assert step.get("timeout-minutes") == 30 assert_exact_command( shell_commands(step), - ["uv", "run", "./scripts/test-integration.sh"], - label="macOS Intel focused integration step", + [ + "uv", + "run", + "--frozen", + "pytest", + "tests/integration/test_windows_installer_launchers.py", + "-vv", + "-ra", + "--tb=short", + "--junitxml=test-results/installer.xml", + ], + label="candidate Windows installer", ) -def test_macos_jobs_run_non_shell_binary_startup_after_build() -> None: - """Both macOS jobs execute the exact generated artifact before upload.""" - _assert_macos_startup_steps(_workflow()) - - -def test_windows_installer_contract_is_windows_only_and_tokenless() -> None: - """The Windows E2E has exact gating and no effective repository token.""" - _assert_windows_installer_step(_workflow()) +def test_installer_workflow_inputs_reach_the_actual_candidate_reader(tmp_path: Path) -> None: + """Catch producer/consumer naming drift, not just two independent string checks.""" + from tests.utils.windows_installer_candidate import EXECUTABLE_MEMBER, InstallerArchive - -def test_standalone_integration_timeouts_are_platform_specific() -> None: - """Standalone Unix has headroom while Windows retains its proven bound.""" - _assert_standalone_integration_timeouts(_workflow()) - - -def test_linux_and_arm_retain_non_live_corpus_grouped_parallelism() -> None: - """Linux and macOS ARM keep the bounded non-live integration corpus.""" - _assert_non_live_unix_integration_parallelism(_workflow()) - - -def test_intel_integration_is_marker_scoped_and_bounded() -> None: - """Intel keeps focused native coverage without replaying the full corpus.""" - _assert_intel_focused_integration(_workflow()) - - -@pytest.mark.parametrize(("job_id", "step_name"), RUNTIME_SETUP_STEPS) -def test_runtime_setup_uses_builtin_github_api_token( - job_id: str, - step_name: str, -) -> None: - """Public runtime metadata must not use the private-module PAT.""" + archive = tmp_path / "candidate.zip" + with zipfile.ZipFile(archive, "w") as target: + target.writestr(EXECUTABLE_MEMBER, b"candidate") + digest = hashlib.sha256(archive.read_bytes()).hexdigest() workflow = _workflow() - step = workflow_step(workflow_job(workflow, job_id), step_name) - assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" - assert step["env"]["GITHUB_APM_PAT"] == "${{ secrets.GH_CLI_PAT }}" + job = workflow_job(workflow, "windows-installer") + step = workflow_step(job, INSTALLER) + environment = dict(effective_env(workflow, job, step)) + for name in environment: + if "CANDIDATE" in name and name.endswith("_ARCHIVE"): + environment[name] = str(archive) + for name in re.findall(r"\$env:(APM_[A-Z0-9_]+)\s*=", step["run"]): + if name.endswith("_VERSION"): + environment[name] = "v9.8.7" + elif name.endswith("_SHA256"): + environment[name] = digest + candidate = InstallerArchive.from_environment(environment, "CANDIDATE") + assert candidate.sha256 == digest + assert candidate.version == "v9.8.7" + + +def _assert_parallel_paths(workflow: dict) -> None: + for name in ("unit-tests", "build"): + assert "needs" not in workflow_job(workflow, name) + for name in ("integration-tests", "release-validation", "windows-installer"): + assert workflow_job(workflow, name)["needs"] == ["build"] + gate = workflow_job(workflow, "gate") + assert gate["if"] == "always()" + assert set(gate["needs"]) == { + "unit-tests", + "build", + "integration-tests", + "release-validation", + "windows-installer", + } + step = workflow_step(gate, "Require every applicable native result") + assert step["env"]["RESULTS"] == "${{ toJSON(needs) }}" + assert "results[name]?.result !== 'success'" in step["with"]["script"] + assert "core.setFailed" in step["with"]["script"] + + +def _assert_integration(workflow: dict) -> None: + job = workflow_job(workflow, "integration-tests") + unix = workflow_step(job, "Run integration tests (Unix)") + assert unix["env"]["PYTEST_MARK_EXPR"] == "${{ inputs.integration-markers }}" + assert unix["env"]["PYTEST_EXTRA_ARGS"] == UNIX_ARGS + assert unix["timeout-minutes"] == UNIX_TIMEOUT + assert unix["env"]["APM_BINARY_PATH"] == ( + "${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm" + ) + windows = workflow_step(job, "Run integration tests (Windows)") + assert windows["timeout-minutes"] == 20 + assert windows["env"]["APM_BINARY_PATH"].endswith("/apm.exe") + for step in (unix, windows): + assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" + assert step["env"]["GITHUB_APM_PAT"] == "${{ secrets.GH_CLI_PAT }}" + assert "APM_RUN_INFERENCE_TESTS" not in effective_env(workflow, job, step) + assert "-IncludeLiveADO" not in shell_tokens(windows) -def test_release_validation_keeps_live_inference_decoupled() -> None: - """Release gates install runtimes but do not invoke paid live inference.""" - workflow = _workflow() - job = workflow_job(workflow, "release-validation") - for step_name in ( - "Run release validation tests (Unix)", - "Run release validation tests (Windows)", - ): - env = effective_env(workflow, job, workflow_step(job, step_name)) - assert "APM_RUN_INFERENCE_TESTS" not in env - assert "GITHUB_TOKEN" not in env +def test_native_startup_runs_exact_frozen_candidate_before_packaging() -> None: + _assert_native_startup(_workflow()) - script = WINDOWS_RELEASE_VALIDATION.read_text(encoding="utf-8") - assert script.count('$env:APM_RUN_INFERENCE_TESTS -eq "1"') >= 2 - assert '$env:APM_RUN_INFERENCE_TESTS -ne "1"' in script - assert "$testsTotal = 4" in script - assert "$env:GITHUB_APM_PAT -or $env:GITHUB_TOKEN" in script +def test_windows_installer_is_candidate_bound_and_tokenless() -> None: + _assert_windows_installer(_workflow()) -@pytest.mark.parametrize( - ("job_id", "step_name"), - NON_LIVE_UNIX_INTEGRATION_STEPS, -) -def test_non_live_unix_serial_integration_mutation_is_rejected( - job_id: str, - step_name: str, -) -> None: - """No non-live release node can silently regress to the serial path.""" - workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, job_id), step_name) - del step["env"]["PYTEST_EXTRA_ARGS"] - - with pytest.raises(AssertionError): - _assert_non_live_unix_integration_parallelism(workflow) +def test_native_jobs_have_no_unit_or_cross_platform_barriers() -> None: + _assert_parallel_paths(_workflow()) -@pytest.mark.parametrize( - ("job_id", "step_name"), - NON_LIVE_UNIX_INTEGRATION_STEPS, -) -def test_non_live_unix_timeout_mutation_is_rejected( - job_id: str, - step_name: str, -) -> None: - """Every non-live Unix release node retains its measured timeout bound.""" - workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, job_id), step_name) - step["timeout-minutes"] = NON_LIVE_UNIX_TIMEOUT_MINUTES[(job_id, step_name)] + 1 - with pytest.raises(AssertionError): - _assert_non_live_unix_integration_parallelism(workflow) +def test_native_integration_preserves_grouping_bounds_and_artifact_identity() -> None: + _assert_integration(_workflow()) -def test_intel_non_live_corpus_mutation_is_rejected() -> None: - """Intel cannot silently regain the redundant non-live corpus.""" - workflow = deepcopy(_workflow()) - step = workflow_step( - workflow_job(workflow, "build-and-validate-macos-intel"), - INTEL_FOCUSED_INTEGRATION_STEP, +def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: + catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) + rows = {row["binary_name"]: row for row in catalog} + assert len(rows) == len(catalog) == 5 + assert {name: row["runner"] for name, row in rows.items()} == { + "apm-linux-x86_64": "ubuntu-24.04", + "apm-linux-arm64": "ubuntu-24.04-arm", + "apm-darwin-x86_64": "macos-15-intel", + "apm-darwin-arm64": "macos-latest", + "apm-windows-x86_64": "windows-latest", + } + assert rows["apm-darwin-x86_64"]["integration_markers"] == "lifecycle_smoke and not live" + assert all( + row["integration_markers"] == "not live" + for name, row in rows.items() + if name != "apm-darwin-x86_64" ) - step["env"]["PYTEST_MARK_EXPR"] = NON_LIVE_MARK_EXPRESSION + assert {name for name, row in rows.items() if not row["on_main"]} == {"apm-darwin-arm64"} - with pytest.raises(AssertionError): - _assert_intel_focused_integration(workflow) - - -def test_arm_focused_subset_mutation_is_rejected() -> None: - """ARM remains the broad non-live macOS release authority.""" - workflow = deepcopy(_workflow()) - step = workflow_step( - workflow_job(workflow, "build-and-validate-macos-arm"), - "Run integration tests", - ) - step["env"]["PYTEST_MARK_EXPR"] = INTEL_FOCUSED_MARK_EXPRESSION - with pytest.raises(AssertionError): - _assert_non_live_unix_integration_parallelism(workflow) +def test_isolated_validation_has_no_checkout_and_uses_checked_archive() -> None: + workflow = _workflow() + job = workflow_job(workflow, "release-validation") + assert all("actions/checkout" not in step.get("uses", "") for step in job["steps"]) + unpack = workflow_step(job, "Extract checked candidate archive") + assert "verify-extract" in shell_tokens(unpack) + for name in ("Run release validation tests (Unix)", "Run release validation tests (Windows)"): + step = workflow_step(job, name) + assert step["timeout-minutes"] == 20 + env = effective_env(workflow, job, step) + assert "GITHUB_TOKEN" not in env + assert "APM_RUN_INFERENCE_TESTS" not in env -def test_unix_integration_twenty_minute_timeout_mutation_is_rejected() -> None: - """The prior Unix timeout cannot satisfy the packaged timing contract.""" +@pytest.mark.parametrize("job", ["unit-tests", "build"]) +def test_unit_build_dependency_mutation_is_rejected(job: str) -> None: workflow = deepcopy(_workflow()) - unix_step = workflow_step( - workflow_job(workflow, "integration-tests"), - "Run integration tests (Unix)", - ) - unix_step["timeout-minutes"] = 20 - + workflow_job(workflow, job)["needs"] = ["another-platform"] with pytest.raises(AssertionError): - _assert_standalone_integration_timeouts(workflow) + _assert_parallel_paths(workflow) -def test_missing_unix_integration_timeout_mutation_is_rejected() -> None: - """Removing the Unix timeout cannot silently make the step unbounded.""" +@pytest.mark.parametrize("job", ["integration-tests", "release-validation", "windows-installer"]) +def test_reintroduced_barrier_is_rejected(job: str) -> None: workflow = deepcopy(_workflow()) - unix_step = workflow_step( - workflow_job(workflow, "integration-tests"), - "Run integration tests (Unix)", - ) - del unix_step["timeout-minutes"] - + workflow_job(workflow, job)["needs"].append("unit-tests") with pytest.raises(AssertionError): - _assert_standalone_integration_timeouts(workflow) + _assert_parallel_paths(workflow) -@pytest.mark.parametrize("scope", ("workflow", "job", "step")) -def test_windows_token_scope_mutations_are_rejected(scope: str) -> None: - """A token inherited from any Actions scope must fail the contract.""" +@pytest.mark.parametrize("scope", ["workflow", "job", "step"]) +@pytest.mark.parametrize("contract", ["smoke", "installer"]) +def test_native_token_scope_mutations_are_rejected(scope: str, contract: str) -> None: workflow = deepcopy(_workflow()) - job = workflow_job(workflow, "build-and-test") - step = workflow_step(job, "Test install.ps1 end-to-end (Windows)") + job = workflow_job(workflow, "build" if contract == "smoke" else "windows-installer") + step = workflow_step(job, SMOKE if contract == "smoke" else INSTALLER) {"workflow": workflow, "job": job, "step": step}[scope].setdefault("env", {})[ "GITHUB_TOKEN" ] = "secret" - with pytest.raises(AssertionError): - _assert_windows_installer_step(workflow) + (_assert_native_startup if contract == "smoke" else _assert_windows_installer)(workflow) -@pytest.mark.parametrize("job_id", [contract[0] for contract in MACOS_STARTUP_CONTRACTS]) -@pytest.mark.parametrize("scope", ("workflow", "job", "step")) -def test_macos_token_scope_mutations_are_rejected(job_id: str, scope: str) -> None: - """A token inherited from any Actions scope must fail the macOS contract.""" +@pytest.mark.parametrize("scope", ["job", "step"]) +def test_disabling_native_startup_is_rejected(scope: str) -> None: workflow = deepcopy(_workflow()) - job = workflow_job(workflow, job_id) - step = workflow_step(job, "Test macOS non-shell binary startup") - {"workflow": workflow, "job": job, "step": step}[scope].setdefault("env", {})[ - "GITHUB_TOKEN" - ] = "secret" + job = workflow_job(workflow, "build") + {"job": job, "step": workflow_step(job, SMOKE)}[scope]["if"] = False + with pytest.raises(AssertionError): + _assert_native_startup(workflow) + +def test_echo_cannot_replace_frozen_smoke() -> None: + workflow = deepcopy(_workflow()) + step = workflow_step(workflow_job(workflow, "build"), SMOKE) + step["run"] = "echo " + step["run"] with pytest.raises(AssertionError): - _assert_macos_startup_steps(workflow) + _assert_native_startup(workflow) -def test_windows_linux_gate_mutation_is_rejected() -> None: - """A Linux condition cannot satisfy the Windows-only platform contract.""" +@pytest.mark.parametrize("field", ["PYTEST_MARK_EXPR", "PYTEST_EXTRA_ARGS", "APM_BINARY_PATH"]) +def test_missing_integration_safety_argument_is_rejected(field: str) -> None: workflow = deepcopy(_workflow()) step = workflow_step( - workflow_job(workflow, "build-and-test"), - "Test install.ps1 end-to-end (Windows)", + workflow_job(workflow, "integration-tests"), "Run integration tests (Unix)" ) - step["if"] = "matrix.platform == 'linux'" - + step["env"][field] = "wrong" with pytest.raises(AssertionError): - _assert_windows_installer_step(workflow) + _assert_integration(workflow) -@pytest.mark.parametrize("job_id", [contract[0] for contract in MACOS_STARTUP_CONTRACTS]) -@pytest.mark.parametrize("scope", ("job", "step")) -def test_macos_disabled_mutations_are_rejected(job_id: str, scope: str) -> None: - """The macOS startup evidence cannot be disabled at job or step scope.""" - workflow = deepcopy(_workflow()) - job = workflow_job(workflow, job_id) - step = workflow_step(job, "Test macOS non-shell binary startup") - {"job": job, "step": step}[scope]["if"] = False +def test_publication_only_consumes_verified_archives() -> None: + workflow = load_workflow(RELEASE_WORKFLOW) + publisher = workflow_job(workflow, "create-release") + assert publisher["needs"] == ["plan", "verify-candidate"] + assert all("run" not in step for step in publisher["steps"]) + assert all("actions/checkout" not in step.get("uses", "") for step in publisher["steps"]) + assert workflow_step(publisher, "Create GitHub Release")["with"]["fail_on_unmatched_files"] - with pytest.raises(AssertionError): - _assert_macos_startup_steps(workflow) - -@pytest.mark.parametrize("job_id", [contract[0] for contract in MACOS_STARTUP_CONTRACTS]) -def test_macos_echo_replacement_mutation_is_rejected(job_id: str) -> None: - """An echo cannot replace the exact frozen pytest invocation.""" - workflow = deepcopy(_workflow()) - step = workflow_step( - workflow_job(workflow, job_id), - "Test macOS non-shell binary startup", - ) - step["run"] = ( - 'test -x "$APM_BINARY_PATH"\n' - "echo uv run --frozen pytest " - f"{MACOS_VERSION_TEST_ID} {MACOS_RICH_TABLE_TEST_ID} " - "-vv -ra --tb=short\n" +def test_native_artifacts_and_downloads_are_attempt_scoped() -> None: + workflow = _workflow() + artifact_name = "candidate-${{ github.run_attempt }}-${{ inputs.binary-name }}" + upload = workflow_step(workflow_job(workflow, "build"), "Upload binary as workflow artifact") + assert upload["with"]["name"] == artifact_name + for name in ("integration-tests", "release-validation", "windows-installer"): + downloads = [ + step + for step in workflow_job(workflow, name)["steps"] + if step.get("uses", "").startswith("actions/download-artifact@") + ] + assert len(downloads) == 1 + assert downloads[0]["with"]["name"] == artifact_name + + +def test_tag_downloads_only_exact_selected_artifact_ids() -> None: + workflow = load_workflow(RELEASE_WORKFLOW) + verifier = workflow_job(workflow, "verify-candidate") + for name in ("Download qualified evidence", "Download exact candidate artifacts"): + download = workflow_step(verifier, name) + assert "artifact-ids" in download["with"] + assert "pattern" not in download["with"] + assert "name" not in download["with"] + assert workflow_step_index(verifier, "Require exact immutable download identities") < ( + workflow_step_index(verifier, name) + ) + assert download["with"]["run-id"] == ( + "${{ needs.plan.outputs.candidate_run_id || github.run_id }}" + ) + qualifier = workflow_job(workflow, "candidate-ready") + assert qualifier["outputs"]["candidate_artifact_ids"] == ( + "${{ steps.qualification.outputs.candidate_artifact_ids }}" ) + evidence = next(step for step in qualifier["steps"] if step.get("id") == "evidence") + assert evidence["with"]["name"] == "release-candidate-evidence-${{ github.run_attempt }}" - with pytest.raises(AssertionError): - _assert_macos_startup_steps(workflow) + +@pytest.mark.parametrize( + ("job_name", "authority"), + [ + ("create-release", "verify-candidate"), + ("deploy-docs", "create-release"), + ("gh-aw-compat", "create-release"), + ("build-pypi-distributions", "create-release"), + ("publish-pypi", "build-pypi-distributions"), + ], +) +def test_warm_release_descendants_require_success_without_skipped_ancestor_poisoning( + job_name: str, authority: str +) -> None: + job = workflow_job(load_workflow(RELEASE_WORKFLOW), job_name) + assert authority in job["needs"] + assert "always()" in job["if"] + assert "!cancelled()" in job["if"] + assert f"needs.{authority}.result == 'success'" in job["if"] diff --git a/tests/unit/test_pytest_timing_workflows.py b/tests/unit/test_pytest_timing_workflows.py new file mode 100644 index 0000000000..efe62ff05f --- /dev/null +++ b/tests/unit/test_pytest_timing_workflows.py @@ -0,0 +1,156 @@ +"""Timing hints must not create inconsistent shard selection.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from copy import deepcopy +from pathlib import Path + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +ROOT = Path(__file__).resolve().parents[2] +ACTION = "./.github/actions/pytest-timing" + + +def assert_shared_integration_snapshot(workflow: dict) -> None: + build = workflow_job(workflow, "build") + snapshot = workflow_step(build, "Freeze integration scheduling hints") + assert snapshot["uses"] == ACTION + assert snapshot["with"] == { + "suite": "integration", + "snapshot": "integration-duration-snapshot", + } + shards = workflow_job(workflow, "integration-tests-shard") + assert "build" in shards["needs"] + assert shards["strategy"]["matrix"]["shard"] == [1, 2, 3, 4] + download = workflow_step(shards, "Download shared integration scheduling snapshot") + assert download["uses"].startswith("actions/download-artifact@") + assert download["with"] == {"name": snapshot["with"]["snapshot"]} + assert not any(step.get("uses") == ACTION for step in shards["steps"]) + + +def test_integration_shards_consume_one_immutable_history_snapshot() -> None: + workflow = load_workflow(ROOT / ".github/workflows/ci-integration.yml") + assert_shared_integration_snapshot(workflow) + + +@pytest.mark.parametrize("mutation", ["missing_dependency", "per_shard_snapshot", "rolling_cache"]) +def test_snapshot_contract_rejects_divergent_shard_history(mutation: str) -> None: + workflow = deepcopy(load_workflow(ROOT / ".github/workflows/ci-integration.yml")) + shards = workflow_job(workflow, "integration-tests-shard") + if mutation == "missing_dependency": + shards["needs"] = [] + elif mutation == "per_shard_snapshot": + workflow_step(shards, "Download shared integration scheduling snapshot")["with"]["name"] = ( + "integration-duration-${{ matrix.shard }}" + ) + else: + shards["steps"].append({"uses": ACTION, "with": {"suite": "integration"}}) + with pytest.raises(AssertionError): + assert_shared_integration_snapshot(workflow) + + +def test_unit_shards_record_timings_without_a_mutable_selection_cache() -> None: + workflow = load_workflow(ROOT / ".github/workflows/ci.yml") + shards = workflow_job(workflow, "build-and-test-shard") + assert not shards.get("needs") + assert shards["strategy"]["matrix"]["shard"] == [1, 2] + assert not any(step.get("uses") == ACTION for step in shards["steps"]) + command = workflow_step(shards, "Run tests with coverage")["run"] + assert "--store-durations" in command + assert "--junitxml=test-results/unit.xml" in command + + +def test_snapshot_publication_is_complete_and_never_writes_the_rolling_cache() -> None: + action = load_workflow(ROOT / ".github/actions/pytest-timing/action.yml") + snapshot_steps = [ + step for step in action["runs"]["steps"] if step.get("if") == "inputs.snapshot != ''" + ] + assert any(step.get("uses", "").startswith("actions/cache/restore@") for step in snapshot_steps) + assert not any( + step.get("uses", "").startswith(("actions/cache@", "actions/cache/save@")) + for step in snapshot_steps + ) + upload = next( + step + for step in snapshot_steps + if step.get("uses", "").startswith("actions/upload-artifact@") + ) + assert upload["with"]["include-hidden-files"] is True + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["path"] == ".test_durations" + + +@pytest.mark.parametrize("weighted", [False, True]) +def test_real_split_and_xdist_preserve_complete_disjoint_coverage( + tmp_path: Path, weighted: bool +) -> None: + """Execute both shards with the deployed plugins and one immutable input.""" + config = tmp_path / "pytest.ini" + config.write_text("[pytest]\nmarkers = xdist_group(name): fixture affinity\n", encoding="ascii") + probe = tmp_path / "test_probe.py" + probe.write_text( + "import pytest\n" + "@pytest.mark.xdist_group('shared_home')\n" + "@pytest.mark.parametrize('index', range(8))\n" + "def test_probe(index):\n" + " assert 0 <= index < 8\n", + encoding="ascii", + ) + expected = {f"test_probe.py::test_probe[{index}]@shared_home" for index in range(8)} + hints = {node: (20.0 if "[0]" in node else 1.0) for node in expected} if weighted else {} + hints["test_removed.py::test_old"] = 200.0 + snapshot = json.dumps(hints) + observed: list[set[str]] = [] + for shard in (1, 2): + durations = tmp_path / f"durations-{shard}.json" + durations.write_text(snapshot, encoding="ascii") + outcomes = tmp_path / f"outcomes-{shard}.xml" + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-c", + str(config), + "--confcutdir", + str(tmp_path), + str(probe), + "-q", + "--splits", + "2", + "--group", + str(shard), + "-n", + "2", + "--dist", + "loadgroup", + "--store-durations", + "--durations-path", + str(durations), + f"--junitxml={outcomes}", + ], + cwd=tmp_path, + env={**os.environ, "PYTEST_ADDOPTS": ""}, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + cases = list(ET.parse(outcomes).iter("testcase")) # noqa: S314 - generated by this probe. + assert cases + assert all(not list(case) for case in cases) + nodes = {f"test_probe.py::{case.attrib['name']}" for case in cases} + observed.append(nodes) + stored = json.loads(durations.read_text("ascii")) + assert nodes <= stored.keys() + assert all(stored[node] >= 0 for node in nodes) + assert observed[0].isdisjoint(observed[1]) + assert observed[0] | observed[1] == expected diff --git a/tests/unit/test_runtime_prerequisite_gate.py b/tests/unit/test_runtime_prerequisite_gate.py new file mode 100644 index 0000000000..fc6f1b31cc --- /dev/null +++ b/tests/unit/test_runtime_prerequisite_gate.py @@ -0,0 +1,55 @@ +"""Reduced CI provisioning must not silently skip runtime regression contracts.""" + +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from tests.integration import conftest + +pytestmark = pytest.mark.component + + +@pytest.mark.parametrize("runtime", ["copilot", "codex", "llm"]) +@pytest.mark.parametrize("strict", [False, True]) +def test_missing_selected_runtime_fails_closed_only_for_provisioned_ci( + monkeypatch: pytest.MonkeyPatch, runtime: str, strict: bool +) -> None: + config = Mock(spec=pytest.Config) + config.getoption.return_value = strict + item = Mock(spec=pytest.Item) + item.nodeid = "tests/integration/test_example.py::test_requires_runtime" + item.get_closest_marker.side_effect = lambda name: ( + Mock() if name == f"requires_runtime_{runtime}" else None + ) + monkeypatch.setattr(conftest, "_has_runtime", lambda name: False) + + if strict: + with pytest.raises(pytest.UsageError, match="required runtime missing"): + conftest.pytest_collection_modifyitems(config, [item]) + item.add_marker.assert_not_called() + else: + conftest.pytest_collection_modifyitems(config, [item]) + assert item.add_marker.call_args.args[0].name == "skip" + + +def test_unselected_runtime_does_not_require_provisioning(monkeypatch: pytest.MonkeyPatch) -> None: + probe = Mock(return_value=False) + monkeypatch.setattr(conftest, "_has_runtime", probe) + config = Mock(spec=pytest.Config) + config.getoption.return_value = True + conftest.pytest_collection_modifyitems(config, []) + probe.assert_not_called() + + +def test_selected_available_runtime_still_executes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(conftest, "_has_runtime", lambda name: True) + config = Mock(spec=pytest.Config) + config.getoption.return_value = True + item = Mock(spec=pytest.Item) + item.get_closest_marker.side_effect = lambda name: ( + Mock() if name == "requires_runtime_copilot" else None + ) + conftest.pytest_collection_modifyitems(config, [item]) + item.add_marker.assert_not_called() diff --git a/tests/unit/test_unix_installer_checksums.py b/tests/unit/test_unix_installer_checksums.py index fc70f71450..a57e50aa73 100644 --- a/tests/unit/test_unix_installer_checksums.py +++ b/tests/unit/test_unix_installer_checksums.py @@ -126,8 +126,9 @@ def _run_installer( stub = ROOT / "tests/utils/unix_installer_stub.py" for tool in tools: wrapper = tmp_path / "bin" / tool + # The stub is stdlib-only; avoid site hooks on every fake tool invocation. wrapper.write_text( - f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(stub))} {tool} "$@"\n', + f'#!/bin/sh\nexec {shlex.quote(sys.executable)} -S {shlex.quote(str(stub))} {tool} "$@"\n', encoding="ascii", ) wrapper.chmod(0o755) diff --git a/tests/unit/test_windows_installer_candidate.py b/tests/unit/test_windows_installer_candidate.py new file mode 100644 index 0000000000..62d4ba671b --- /dev/null +++ b/tests/unit/test_windows_installer_candidate.py @@ -0,0 +1,265 @@ +"""Cross-platform proofs for the candidate-only Windows installer gate.""" + +from __future__ import annotations + +import hashlib +import io +import shutil +import subprocess +import sys +import zipfile +from contextlib import nullcontext +from dataclasses import replace +from pathlib import Path +from unittest.mock import MagicMock +from urllib.error import HTTPError +from urllib.parse import urlparse +from urllib.request import ProxyHandler, build_opener + +import pytest + +from tests.utils.windows_installer_candidate import ( + ASSET_NAME, + EXECUTABLE_MEMBER, + InstallerArchive, + preserve_windows_user_path, + serve_installer_candidate, +) + +pytestmark = [pytest.mark.component, pytest.mark.windows_compat] +ROOT = Path(__file__).resolve().parents[2] +SUITE = ROOT / "scripts/windows/test-install-script.ps1" +PWSH = shutil.which("pwsh") + + +@pytest.fixture +def candidate_environment(tmp_path: Path) -> dict[str, str]: + """Create a ZIP without executing it; the native suite owns runtime identity.""" + archive = tmp_path / ASSET_NAME + with zipfile.ZipFile(archive, "w") as target: + target.writestr(EXECUTABLE_MEMBER, b"MZ-test-candidate") + return { + "APM_CANDIDATE_ARCHIVE": str(archive), + "APM_CANDIDATE_VERSION": "v9.8.7", + "APM_CANDIDATE_SHA256": hashlib.sha256(archive.read_bytes()).hexdigest(), + } + + +@pytest.mark.parametrize("role", ["CANDIDATE", "BASELINE"]) +@pytest.mark.parametrize( + ("version", "tag"), + [("9.8.7", "v9.8.7"), ("v9.8.7", "v9.8.7"), ("9.8.7rc1", "v9.8.7rc1")], +) +def test_fixture_accepts_packager_metadata_version( + candidate_environment: dict[str, str], role: str, version: str, tag: str +) -> None: + """The packager's unprefixed version and historical source tags share one contract.""" + environment = { + key.replace("CANDIDATE", role): value for key, value in candidate_environment.items() + } + environment[f"APM_{role}_VERSION"] = version + archive = InstallerArchive.from_environment(environment, role) + assert archive.version == tag + assert archive.sha256 == candidate_environment["APM_CANDIDATE_SHA256"] + + +@pytest.mark.parametrize("field", ["ARCHIVE", "VERSION", "SHA256"]) +def test_candidate_requires_every_explicit_input( + candidate_environment: dict[str, str], field: str +) -> None: + """Neither missing metadata nor an absent candidate may select a public tag.""" + candidate_environment.pop(f"APM_CANDIDATE_{field}") + with pytest.raises(ValueError, match="no published-release fallback"): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +def test_missing_candidate_file_fails(candidate_environment: dict[str, str]) -> None: + """A stale workflow path must fail before PowerShell or the server starts.""" + Path(candidate_environment["APM_CANDIDATE_ARCHIVE"]).unlink() + with pytest.raises(FileNotFoundError): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +@pytest.mark.parametrize("version", ["v9.8", "@v9.8.7", "v9.8.7/other", "v9.8.7\n", "latest"]) +def test_candidate_rejects_nonexact_tag( + candidate_environment: dict[str, str], version: str +) -> None: + """Fixture route identities may not smuggle paths or select moving tags.""" + candidate_environment["APM_CANDIDATE_VERSION"] = version + with pytest.raises(ValueError, match=r"exact vX\.Y\.Z"): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +@pytest.mark.parametrize("digest", ["0" * 64, "not-a-digest"]) +def test_candidate_rejects_wrong_digest(candidate_environment: dict[str, str], digest: str) -> None: + """A checksum mismatch cannot be replaced with a locally computed trust value.""" + candidate_environment["APM_CANDIDATE_SHA256"] = digest + with pytest.raises(ValueError, match="SHA256"): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +def test_candidate_rejects_tampered_archive(candidate_environment: dict[str, str]) -> None: + """The caller's digest binds the served bytes, not just the archive's name.""" + with Path(candidate_environment["APM_CANDIDATE_ARCHIVE"]).open("ab") as stream: + stream.write(b"tampered") + with pytest.raises(ValueError, match="archive SHA256 mismatch"): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +def test_candidate_requires_native_bundle(candidate_environment: dict[str, str]) -> None: + """An authenticated archive without the Windows executable cannot pass.""" + archive = Path(candidate_environment["APM_CANDIDATE_ARCHIVE"]) + with zipfile.ZipFile(archive, "w") as target: + target.writestr("apm-linux-x86_64/apm", b"wrong platform") + candidate_environment["APM_CANDIDATE_SHA256"] = hashlib.sha256(archive.read_bytes()).hexdigest() + with pytest.raises(ValueError, match="must contain exactly one"): + InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + + +def test_mirror_serves_only_verified_candidate_and_current_installer( + candidate_environment: dict[str, str], tmp_path: Path +) -> None: + """Production download paths serve immutable bytes with original checksums.""" + candidate = InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + baseline = replace(candidate, version="v9.8.6") + installer = tmp_path / "install.ps1" + installer.write_bytes(b"# exact checkout installer\r\n") + opener = build_opener(ProxyHandler({})) + # Replacement after verification cannot change the immutable fixture bytes. + Path(candidate_environment["APM_CANDIDATE_ARCHIVE"]).write_bytes(b"replaced") + with serve_installer_candidate(candidate, baseline, installer) as mirror: + parsed = urlparse(mirror.base_url) + assert (parsed.scheme, parsed.hostname, parsed.path) == ("http", "127.0.0.1", "") + assert parsed.port is not None + for release in (candidate, baseline): + route = f"/releases/{release.version}/{ASSET_NAME}" + with opener.open(mirror.base_url + route, timeout=5) as response: + assert response.read() == release.payload + with opener.open(mirror.base_url + route + ".sha256", timeout=5) as response: + assert response.read() == release.checksum_bytes() + with opener.open(mirror.base_url + "/installer/install.ps1", timeout=5) as response: + assert response.read() == installer.read_bytes() + tampered = f"/tampered/{candidate.version}/{ASSET_NAME}" + with opener.open(mirror.base_url + tampered, timeout=5) as response: + assert hashlib.sha256(response.read()).hexdigest() != candidate.sha256 + with opener.open(mirror.base_url + tampered + ".sha256", timeout=5) as response: + assert response.read() == candidate.checksum_bytes() + assert not mirror.unexpected_paths + with pytest.raises(HTTPError) as failure: + opener.open(mirror.base_url + "/releases/v0.29.0/" + ASSET_NAME, timeout=5) + assert failure.value.code == 404 + assert mirror.unexpected_paths == [f"/releases/v0.29.0/{ASSET_NAME}"] + assert mirror.requested_paths.count("/installer/install.ps1") == 1 + with zipfile.ZipFile(io.BytesIO(candidate.payload)) as archive: + assert ( + candidate.executable_sha256 + == hashlib.sha256(archive.read(EXECUTABLE_MEMBER)).hexdigest() + ) + + +@pytest.mark.parametrize("baseline_version", ["v9.8.7", "v9.8.8"]) +def test_mirror_rejects_baseline_as_destination( + candidate_environment: dict[str, str], baseline_version: str +) -> None: + """The historical binary must be an older upgrade source.""" + candidate = InstallerArchive.from_environment(candidate_environment, "CANDIDATE") + with ( + pytest.raises(ValueError, match="baseline must be older"), + serve_installer_candidate( + candidate, replace(candidate, version=baseline_version), ROOT / "install.ps1" + ), + ): + pytest.fail("The fixture server must not start for a downgrade or reinstall baseline") + + +@pytest.mark.parametrize("original", [None, ("original user path", 2)]) +@pytest.mark.parametrize("interrupted", [False, True]) +def test_user_path_restored_even_after_native_timeout( + monkeypatch: pytest.MonkeyPatch, + original: tuple[str, int] | None, + interrupted: bool, +) -> None: + """The parent owns rollback when killing PowerShell prevents its finally.""" + registry = MagicMock() + key = registry.OpenKey.return_value.__enter__.return_value + registry.QueryValueEx.side_effect = [ + FileNotFoundError() if original is None else original, + ("installer-modified-path", 1), + ] + monkeypatch.setitem(sys.modules, "winreg", registry) + failure = pytest.raises(RuntimeError, match="timed out") if interrupted else nullcontext() + with failure, preserve_windows_user_path(): + if interrupted: + raise RuntimeError("Native process tree timed out") + if original is None: + registry.DeleteValue.assert_called_once_with(key, "Path") + registry.SetValueEx.assert_not_called() + else: + registry.SetValueEx.assert_called_once_with(key, "Path", 0, original[1], original[0]) + registry.DeleteValue.assert_not_called() + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +def test_powershell_candidate_version_rejects_wrong_binary() -> None: + """Run the actual suite's comparison function, including substring twins.""" + command = r""" +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile( + 'scripts/windows/test-install-script.ps1', [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +$function = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq 'Test-ReportedVersion' +}, $true) +. ([scriptblock]::Create($function.Extent.Text)) +if (-not (Test-ReportedVersion 'apm, version 9.8.7 (abcdef123)' 'v9.8.7')) { + throw 'Exact candidate version was rejected' +} +foreach ($wrong in @('9.8.70', '9.8.7rc1', '9.8.6', '19.8.7', '9x8x7')) { + if (Test-ReportedVersion "apm, version $wrong" 'v9.8.7') { + throw "Wrong binary version accepted: $wrong" + } +} +Write-Output 'candidate version negative twins passed' +""" + result = subprocess.run( + [PWSH, "-NoProfile", "-NonInteractive", "-Command", command], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "candidate version negative twins passed" + + +def test_native_gate_retains_integrity_self_update_and_environment_boundaries() -> None: + """Local transport must not replace the actual updater or launcher process.""" + script = SUITE.read_text(encoding="utf-8") + assert "$PinnedVersion" not in script + assert "[Parameter(Mandatory = $true)][string]$CandidateVersion" in script + assert '$env:APM_NO_DIRECT_FALLBACK = "1"' in script + assert "Remove-Item Env:APM_SKIP_CHECKSUM" in script + assert 'cmd.exe /c "`"$shim`" self-update"' in script + assert "for ($attempt" not in script + assert "Test-CandidateChecksumRejection" in script[script.index("# Runner") :] + assert "Assert-CandidatePayload -Root $prefix.Root" in script + assert "Test-ReportedVersion $ver2.Output $CandidateVersion" in script + fresh = script[ + script.index("function Test-EndToEndInstall") : script.index( + "function Test-NonJunctionCollision" + ) + ] + assert "Test-SameVersionReinstall -Prefix $prefix" in fresh + updater = script[script.index("function Test-SelfUpdateCommand") : script.index("# Runner")] + assert "Test-CrossVersionUpgrade -Prefix $prefix" in updater + assert script.count("Invoke-InstallScript -Version $OlderVersion") == 1 + assert "Reinstall replaces the previous release tree, including its canary" in script + assert '[Environment]::SetEnvironmentVariable("Path", $savedUserPath, "User")' in script + assert ( + '[Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process")' + in script + ) diff --git a/tests/unit/test_windows_signing_contract.py b/tests/unit/test_windows_signing_contract.py index 77a22073b7..a3e717204e 100644 --- a/tests/unit/test_windows_signing_contract.py +++ b/tests/unit/test_windows_signing_contract.py @@ -23,7 +23,7 @@ ) ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github" / "workflows" / "build-release.yml" +WORKFLOW = ROOT / ".github" / "workflows" / "release-platform.yml" SIGNING_STEP = "Sign Windows binary (Authenticode)" BUILD_STEP = "Build binary (Windows)" @@ -36,7 +36,7 @@ def _workflow() -> WorkflowNode: def _assert_signing_step(workflow: WorkflowNode) -> None: """Assert the Authenticode signing step is correctly configured.""" - job = workflow_job(workflow, "build-and-test") + job = workflow_job(workflow, "build") step = workflow_step(job, SIGNING_STEP) # Must only run on the Windows matrix entry. @@ -77,7 +77,7 @@ def test_signing_step_exists() -> None: def test_signing_step_gated_on_windows() -> None: """Removing the Windows platform gate must fail the contract.""" workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, "build-and-test"), SIGNING_STEP) + step = workflow_step(workflow_job(workflow, "build"), SIGNING_STEP) step["if"] = "env.WINDOWS_CERT_PFX != ''" # no platform gate with pytest.raises(AssertionError, match="Windows platform"): @@ -87,7 +87,7 @@ def test_signing_step_gated_on_windows() -> None: def test_signing_step_gated_on_secret() -> None: """Removing the secret gate must fail -- signing must skip gracefully.""" workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, "build-and-test"), SIGNING_STEP) + step = workflow_step(workflow_job(workflow, "build"), SIGNING_STEP) step["if"] = "matrix.platform == 'windows'" # no secret gate with pytest.raises(AssertionError, match="WINDOWS_CERT_PFX"): @@ -97,7 +97,7 @@ def test_signing_step_gated_on_secret() -> None: def test_signing_step_references_script() -> None: """Replacing the script reference must fail the contract.""" workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, "build-and-test"), SIGNING_STEP) + step = workflow_step(workflow_job(workflow, "build"), SIGNING_STEP) step["run"] = "echo skipping signing\n" with pytest.raises(AssertionError, match=r"sign-binary\.ps1"): @@ -107,7 +107,7 @@ def test_signing_step_references_script() -> None: def test_signing_step_order() -> None: """Moving the signing step after upload must fail the contract.""" workflow = deepcopy(_workflow()) - job = workflow_job(workflow, "build-and-test") + job = workflow_job(workflow, "build") steps: list = job["steps"] # Find and remove the signing step, then insert it after the upload step. @@ -129,7 +129,7 @@ def test_signing_step_order() -> None: def test_signing_step_requires_password_env() -> None: """Removing WINDOWS_CERT_PASSWORD from env must fail the contract.""" workflow = deepcopy(_workflow()) - step = workflow_step(workflow_job(workflow, "build-and-test"), SIGNING_STEP) + step = workflow_step(workflow_job(workflow, "build"), SIGNING_STEP) env = step.get("env", {}) assert isinstance(env, dict), "env must be a mapping for this mutation test" env.pop("WINDOWS_CERT_PASSWORD", None) diff --git a/tests/unit/test_windows_unit_diagnostics.py b/tests/unit/test_windows_unit_diagnostics.py index 1765f1b7ed..6d7260ba1c 100644 --- a/tests/unit/test_windows_unit_diagnostics.py +++ b/tests/unit/test_windows_unit_diagnostics.py @@ -15,23 +15,35 @@ pytestmark = pytest.mark.component -WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/build-release.yml" -TEST_COMMAND = ["uv", "run", "pytest", "tests/unit", "tests/test_console.py"] -PARALLEL_ARGS = ["-n", "auto", "--dist", "worksteal"] +WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/release-platform.yml" +TEST_COMMAND = ["uv", "run", "--frozen", "pytest", "tests/unit", "tests/test_console.py"] +PARALLEL_ARGS = [ + "-n", + "auto", + "--dist", + "worksteal", + "--durations=50", + "--store-durations", + "--junitxml=test-results/unit.xml", +] DIAGNOSTIC_ARGS = ["-vv", "--tb=short", "--show-capture=no", "--no-showlocals"] def _assert_windows_diagnostics(workflow: dict) -> None: """Windows stays exhaustive, parallel, bounded, and fail-closed.""" - job = workflow_job(workflow, "build-and-test") + job = workflow_job(workflow, "unit-tests") linux = workflow_step(job, "Run unit tests") windows = workflow_step(job, "Run unit tests (Windows diagnostics)") - assert linux["if"] == "matrix.platform != 'windows'" + assert linux["if"] == "inputs.platform != 'windows'" assert shell_commands(linux) == [TEST_COMMAND + PARALLEL_ARGS] - assert windows["if"] == "matrix.platform == 'windows'" + assert windows["if"] == "inputs.platform == 'windows'" assert windows["timeout-minutes"] == 60 assert shell_commands(windows) == [[*TEST_COMMAND, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS]] - assert windows["env"] == {**linux["env"], "PYTHONUNBUFFERED": "1"} + assert windows["env"] == { + "PYTHONUNBUFFERED": "1", + "GITHUB_TOKEN": "${{ secrets.GH_MODELS_PAT }}", + "GITHUB_APM_PAT": "${{ secrets.GH_CLI_PAT }}", + } for node in (job, windows): assert not node.get("continue-on-error", False) @@ -54,7 +66,7 @@ def test_windows_diagnostics_contract_rejects_mutations(key: str, value: object) """A success-shaped replacement or changed safety bound is rejected.""" workflow = deepcopy(load_workflow(WORKFLOW)) step = workflow_step( - workflow_job(workflow, "build-and-test"), "Run unit tests (Windows diagnostics)" + workflow_job(workflow, "unit-tests"), "Run unit tests (Windows diagnostics)" ) step[key] = value with pytest.raises(AssertionError): diff --git a/tests/utils/windows_installer_candidate.py b/tests/utils/windows_installer_candidate.py new file mode 100644 index 0000000000..d5bbf683cf --- /dev/null +++ b/tests/utils/windows_installer_candidate.py @@ -0,0 +1,148 @@ +"""Verified, loopback-only release fixtures for the native Windows installer.""" + +from __future__ import annotations + +import hashlib +import io +import re +import threading +import zipfile +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +from packaging.version import Version + +ASSET_NAME = "apm-windows-x86_64.zip" +EXECUTABLE_MEMBER = "apm-windows-x86_64/apm.exe" + + +@contextmanager +def preserve_windows_user_path() -> Iterator[None]: + """Restore persistent PATH even when the native process tree times out.""" + import winreg + + with winreg.OpenKey( + winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_READ | winreg.KEY_SET_VALUE + ) as key: + try: + original = winreg.QueryValueEx(key, "Path") + except FileNotFoundError: + original = None + try: + yield + finally: + try: + current = winreg.QueryValueEx(key, "Path") + except FileNotFoundError: + current = None + if current != original: + if original is None: + winreg.DeleteValue(key, "Path") + else: + value, kind = original + winreg.SetValueEx(key, "Path", 0, kind, value) + + +@dataclass(frozen=True) +class InstallerArchive: + """Immutable bytes authenticated against the caller's explicit digest.""" + + version: str + payload: bytes + sha256: str + executable_sha256: str + + @classmethod + def from_environment(cls, environment: Mapping[str, str], role: str) -> InstallerArchive: + """Require an explicit candidate/baseline; never resolve a public release.""" + prefix = f"APM_{role}_" + values = {} + for field in ("ARCHIVE", "VERSION", "SHA256"): + name = prefix + field + value = environment.get(name, "") + if not value: + raise ValueError(f"{name} is required; no published-release fallback is allowed") + values[field] = value + version = values["VERSION"] + if re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?", version) is None: + raise ValueError(f"{prefix}VERSION must be an exact vX.Y.Z or X.Y.Z release version") + version = "v" + version.removeprefix("v") + digest = values["SHA256"] + if re.fullmatch(r"[0-9a-fA-F]{64}", digest) is None: + raise ValueError(f"{prefix}SHA256 must be an explicit SHA256 digest") + payload = Path(values["ARCHIVE"]).read_bytes() + digest = digest.lower() + if hashlib.sha256(payload).hexdigest() != digest: + raise ValueError(f"{role} archive SHA256 mismatch") + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + if archive.namelist().count(EXECUTABLE_MEMBER) != 1: + raise ValueError(f"{role} archive must contain exactly one {EXECUTABLE_MEMBER}") + with archive.open(EXECUTABLE_MEMBER) as executable: + executable_digest = hashlib.file_digest(executable, "sha256").hexdigest() + return cls(version, payload, digest, executable_digest) + + def checksum_bytes(self) -> bytes: + """Return the production installer's SHA256 sidecar format.""" + return f"{self.sha256} {ASSET_NAME}\n".encode("ascii") + + +@dataclass +class InstallerMirror: + """One bounded server and its observed production HTTP requests.""" + + base_url: str + requested_paths: list[str] + unexpected_paths: list[str] + + +@contextmanager +def serve_installer_candidate( + candidate: InstallerArchive, + baseline: InstallerArchive, + installer: Path, +) -> Iterator[InstallerMirror]: + """Serve exact verified archives and the unmodified checkout installer.""" + if Version(baseline.version) >= Version(candidate.version): + raise ValueError("The baseline must be older than the candidate destination") + routes = {"/installer/install.ps1": installer.read_bytes()} + for release in (candidate, baseline): + archive_path = f"/releases/{release.version}/{ASSET_NAME}" + routes[archive_path] = release.payload + routes[archive_path + ".sha256"] = release.checksum_bytes() + tampered_path = f"/tampered/{candidate.version}/{ASSET_NAME}" + routes[tampered_path] = candidate.payload + b"\nchecksum-negative-twin\n" + routes[tampered_path + ".sha256"] = candidate.checksum_bytes() + requested: list[str] = [] + unexpected: list[str] = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + requested.append(self.path) + body = routes.get(self.path) + if body is None: + unexpected.append(self.path) + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + """Keep archive requests out of pytest's normal output.""" + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield InstallerMirror(f"http://127.0.0.1:{server.server_port}", requested, unexpected) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + if thread.is_alive(): + raise RuntimeError("Windows installer fixture server did not stop") From 7d867f20448beba7e9edae70c5b8e6089818e115 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 17:24:06 +0200 Subject: [PATCH 02/23] fix(ci): repair collection prerequisites and canonical instruction deployment Register strict runtime options before whole-tree collection and regenerate the CI instruction self-deployment from its canonical source. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .apm/instructions/cicd.instructions.md | 110 +++++++++++++------ apm.lock.yaml | 4 +- tests/conftest.py | 9 ++ tests/integration/conftest.py | 9 -- tests/unit/test_runtime_prerequisite_gate.py | 50 +++++++++ 5 files changed, 135 insertions(+), 47 deletions(-) diff --git a/.apm/instructions/cicd.instructions.md b/.apm/instructions/cicd.instructions.md index f18d2857fe..2cafd94550 100644 --- a/.apm/instructions/cicd.instructions.md +++ b/.apm/instructions/cicd.instructions.md @@ -6,13 +6,17 @@ description: "CI/CD Pipeline configuration for PyInstaller binary packaging and # CI/CD Pipeline Instructions ## Workflow Architecture (Tiered + Merge Queue) -Five workflows split by trigger and tier. PRs get fast feedback; the heavy +Core workflows split by trigger and tier. PRs get fast feedback; the heavy integration suite runs only at merge time via GitHub Merge Queue (microsoft/apm#770). -1. **`ci.yml`** - Tier 1, runs on `pull_request` AND `merge_group` - - **Linux-only** (ubuntu-24.04). Combined `build-and-test` job: unit tests + binary build in a single runner. No secrets needed. - - Uploads Linux x86_64 binary artifact for downstream integration testing. +1. **`ci.yml`** - Tier 1, runs on `pull_request`, `merge_group`, and `workflow_call` + - Linux unit shards and combined coverage, lint, architecture ratchets, + self-check and lifecycle smoke; a bounded Windows compatibility lane. + No production secrets needed. + - PR-only binary smoke provides early packaging feedback. Merge queue + builds its own binary; full release candidates call the source checks + and build each native archive separately. - Runs in both PR context (fast feedback for contributors) and merge_group context (against the tentative merge commit before queue auto-merges). 2. **`ci-integration.yml`** - Tier 2, `merge_group` trigger only @@ -26,50 +30,74 @@ integration suite runs only at merge time via GitHub Merge Queue This file holds production secrets (`GH_CLI_PAT`, `ADO_APM_PAT`). Required-check satisfaction at PR time is handled by `merge-gate.yml`, which aggregates all required signals into a single `gate` check. -3. **`merge-gate.yml`** - single-authority PR-time aggregator - - Triggers on `pull_request` only (single trigger - dual-trigger with - `pull_request_target` produces SUCCESS+CANCELLED check-run twins via - `cancel-in-progress` and poisons branch protection's rollup). +3. **`merge-gate.yml`** - single-authority PR and merge-queue aggregator + - Uses event-specific required checks. Never add a parallel + `pull_request_target` gate, which can create duplicate check-run names. - One job named `gate`. Polls the Checks API for all entries in the workflow's `EXPECTED_CHECKS` env var; aggregates pass/fail into a single check-run. - Branch protection requires ONLY this one check (`gate`). Adding, renaming, or removing an underlying check is a `merge-gate.yml` edit, never a ruleset edit. Tide / bors single-authority pattern. - - Recovery if the `pull_request` webhook is dropped: empty commit, - `gh workflow run merge-gate.yml -f pr_number=NNN`, or close+reopen. + - Required results must succeed; skipped or neutral is not passing + evidence. Lint and Test Architecture Ratchets are required in both tiers. - `.github/CODEOWNERS` requires Lead Maintainer review for any change to `.github/workflows/**`. -4. **`build-release.yml`** - `push` to main, tags, schedule, `workflow_dispatch` - - **Linux + Windows** run combined `build-and-test` (unit tests + binary build in one job). Unit tests run on every push for platform-regression signal; **smoke tests are gated to tag/schedule/dispatch only** (promotion boundaries) to avoid duplicating `ci-integration.yml`'s merge-time smoke and to cut redundant codex-binary downloads. - - **macOS Intel** uses `build-and-validate-macos-intel` (root node, runs own unit tests - no dependency on `build-and-test`). Builds the binary on every push for early regression feedback; tag/schedule/dispatch promotion runs add marker-bounded `lifecycle_smoke and not live` integration coverage and isolated release validation. - - **macOS ARM** uses `build-and-validate-macos-arm` (root node, tag/schedule/dispatch only - ARM runners are extremely scarce with 2-4h+ queue waits). Promotion runs retain the full non-live integration corpus and isolated release validation. +4. **`build-release.yml`** - main push, tags, schedule, default-branch `repository_dispatch` + - Calls **`release-platform.yml`** using the canonical platform catalog + in `scripts/release-platforms.json`. Native unit tests and builds are + independent. Integration, isolated validation and the Windows installer + depend only on their own platform's build. + - Ordinary main pushes retain four lanes, including macOS Intel. + Full tag/schedule/manual qualification adds macOS ARM and source CI. + Intel retains `lifecycle_smoke and not live`; ARM retains full non-live + integration coverage. Neither platform drops native unit coverage. + - Tags can promote an exact-SHA, fully qualified trusted candidate; + partial main builds and PR artifacts are not release evidence. + Missing matching candidates require a fresh full build. - Publication jobs do not receive `ADO_APM_PAT`; live ADO PAT acceptance is an explicit `ado_pat_e2e` mode in `auth-acceptance.yml`. Full 5-platform binary output (linux x86_64/arm64, darwin x86_64/arm64, windows x86_64). -5. **`ci-runtime.yml`** - nightly schedule, manual dispatch, path-filtered push +5. **`ci-runtime.yml`** - nightly schedule, default-branch repository dispatch, path-filtered push - **Linux x86_64 only**. Live inference smoke tests (`apm run`) isolated from release pipeline. - Uses `GH_MODELS_PAT` for GitHub Models API access. - Failures do not block releases - annotated as warnings. ## Platform Testing Strategy -- **PR time**: Linux-only combined build-and-test in `ci.yml`. Catches logic bugs and dependency issues before merge. Windows + macOS are tested post-merge (platform-specific issues are rare and the full matrix runs on every push to main). -- **Post-merge**: Full 5-platform matrix (linux x86_64/arm64, darwin x86_64/arm64, windows x86_64) catches remaining platform-specific issues on main. -- **Rationale**: ci.yml has always been Linux-only - Windows and macOS are covered by `build-release.yml` on every push to main. This keeps PR feedback fast while still catching platform issues before release. +- **PR time**: Linux source checks and packaging smoke, plus a bounded + Windows compatibility gate. Platform-specific failures remain possible. +- **Post-merge**: Four native unit/build lanes; full qualification includes + all five architectures and every applicable integration/validation gate. +- **Publication**: Requires successful source and native qualification. + Parallelizing these checks must never remove a required result. ## PyInstaller Binary Packaging - **CRITICAL**: Uses `--onedir` mode (NOT `--onefile`) for faster CLI startup performance - **Binary Structure**: Creates `dist/{binary_name}/apm` (nested directory containing executable + dependencies) - **Platform Naming**: `apm-{platform}-{arch}` (e.g., `apm-darwin-arm64`, `apm-linux-x86_64`) -- **Spec File**: `build/apm.spec` handles data bundling, hidden imports, and UPX compression +- **Spec File**: `build/apm.spec` handles data bundling and hidden imports. + Do not provision UPX: the pinned PyInstaller disables it on non-Windows, + while the spec disables it on Windows to avoid antivirus false positives. ## Artifact Flow Quirks -- **Upload**: Artifacts include both binary directory + test scripts for isolation testing -- **Download**: GitHub Actions creates nested structure: `{artifact_name}/dist/{binary_name}/apm` -- **Release Prep**: Extract binary from nested path using `tar -czf "${binary}.tar.gz" -C "${artifact_dir}/dist" "${binary}"` +- **Package once**: `scripts/package_release.py` creates each final archive + after signing and verifies the embedded version/build SHA. +- **Upload**: Each native artifact contains `release-assets/` (archive, + checksum, metadata) and selected validation scripts. +- **Native consumers**: Verify hashes and identity, then extract the same + archive into `dist/{binary_name}`. Do not test an installed older release. +- **Promotion**: Verify complete immutable run/artifact evidence and copy + the ten public archive/checksum files without repackaging. The write-token + publisher has no checkout and executes no candidate scripts. +- **Reruns**: Candidate artifacts are attempt-scoped and promotion downloads + exact artifact IDs. Use **Re-run all jobs** to regenerate qualification; + a partial rerun must not combine old archives with new test evidence. ## Critical Testing Phases 1. **Integration Tests**: Full source code access for comprehensive testing 2. **Release Validation**: ISOLATION testing - no source checkout, validates exact shipped binary experience -3. **Path Resolution**: Use symlinks and PATH manipulation for isolated binary testing +3. **Path Resolution**: Set `APM_BINARY_PATH` explicitly for pytest, so an + editable install or ambient PATH cannot silently replace the candidate +4. **Installer Tests**: New installs and upgrade destinations use the + current candidate; an older binary may only seed the upgrade source ## Inference Testing (Decoupled) - Live inference tests (`apm run`) are **isolated** in `ci-runtime.yml` - they do NOT gate releases @@ -78,16 +106,23 @@ integration suite runs only at merge time via GitHub Merge Queue - Rationale: 8 inference executions x 2% failure rate = 14.9% false-negative per release; APM core UVPs require zero live inference ## Release Flow Dependencies -- **PR workflow**: Tier 1 only - ci.yml (build-and-test, Linux-only) provides fast feedback. Tier 2 does not run until enqueued. -- **Merge queue workflow**: ci.yml (Tier 1 against tentative merge ref) + ci-integration.yml (Tier 2: build -> smoke-test -> integration-tests -> release-validation). Queue auto-merges on success; ejects on failure. -- **Push/Release workflow (Linux + Windows)**: build-and-test -> integration-tests -> release-validation -> create-release -> publish-pypi (gh-aw-compat runs in parallel, informational) +- **PR workflow**: Tier 1 provides fast feedback; Tier 2 waits for enqueue. +- **Merge queue workflow**: Tier 1 plus Tier 2's own build. Integration and + isolated validation no longer form a serial fan-in; the gate still + requires all applicable results. +- **Fresh release**: Independent native units/build -> per-platform + integration/validation/installer -> complete qualification -> verified + assets -> publication. Source CI runs alongside the native lanes. +- **Qualified same-SHA release**: Trusted candidate lookup -> verification + -> publication. Qualification cost occurred before tagging, not vanished. - **Homebrew distribution**: `microsoft/homebrew-apm` polls the latest public APM release and updates its formula with its own `GITHUB_TOKEN`; this workflow must not send a cross-repository dispatch or hold a tap credential. -- **Push/Release workflow (macOS Intel)**: build-and-validate-macos-intel (root node: unit tests + build always + conditional `lifecycle_smoke and not live` integration/release-validation) -> create-release -- **Push/Release workflow (macOS ARM)**: build-and-validate-macos-arm (root node, tag/schedule/dispatch only; full non-live integration corpus and all phases run) -> create-release - **Live ADO PAT acceptance**: manual `auth-acceptance.yml` dispatch with `ado_pat_e2e: true` -> exact `live and requires_ado_pat` marker intersection. Invalid credentials fail with production auth diagnostics but do not block publication. -- **Tag Triggers**: Only `v*.*.*` tags trigger full release pipeline +- **Tag Triggers**: `v*` tags enter release planning; promotion must verify + that the tag and candidate version identify the same release - **Artifact Retention**: 30 days for debugging failed releases -- **Cross-workflow artifacts**: ci-integration.yml builds the binary inline (no cross-workflow artifact transfer); build-release.yml jobs share artifacts within the same workflow run. +- **Cross-run artifacts**: Only the release candidate protocol may reuse + release artifacts, bound to the exact trusted source run and commit. + `ci-integration.yml` continues to build its own binary. ## Branch Protection & Required Checks - **Single required check**: branch protection (`main-protection` ruleset id 9294522) requires exactly one status check context: `gate` from `merge-gate.yml`. All other PR-time signals are aggregated by that workflow's poll loop. @@ -107,13 +142,16 @@ integration suite runs only at merge time via GitHub Merge Queue - `APM_RUN_INFERENCE_TESTS` - When `1`, enables live inference tests in validation scripts ## Performance Considerations -- **Combined build-and-test**: Eliminates ~1.5m runner re-provisioning overhead by running unit tests and binary build in the same job. -- **macOS as root nodes**: macOS consolidated jobs run their own unit tests and start immediately - no dependency on Linux/Windows test completion. +- **Critical path**: Unit tests do not delay native builds. A slow platform + does not delay another platform's integration or isolated validation. + Account for added runner setup and queue waits when measuring savings. - **Native uv caching**: `setup-uv` action with `enable-cache: true` replaces manual `actions/cache@v3` blocks. -- **Targeted setup-node usage**: Node.js is only installed in `ci-runtime.yml`, macOS consolidated jobs, and integration-tests/release-validation phases (for `apm runtime setup copilot` -> npm install). -- **macOS runner consolidation**: Each macOS arch has a single consolidated job (build + integration + release-validation). Intel (`build-and-validate-macos-intel`) runs on every push since Intel runners are plentiful and uses marker-bounded non-live lifecycle integration coverage for promotions. ARM (`build-and-validate-macos-arm`) is gated to tag/schedule/dispatch only since ARM runners are extremely scarce (2-4h+ queue waits) and remains the full non-live corpus macOS authority. This avoids serial re-queuing of runners across multiple jobs and redundant corpus replay on Intel. -- **Unit tests skip macOS**: Python unit tests are platform-agnostic; Linux + Windows coverage is sufficient. macOS-specific validation (binary build, focused Intel integration, full non-live ARM integration, and release validation) still runs via the consolidated jobs. +- **Runtime provisioning**: `APM_TEST_RUNTIMES` limits setup to selected + prerequisites. Strict collection fails rather than silently skipping a + selected test when a required runtime is missing. +- **Duration history**: Persist pytest timings and JUnit outcomes. Timing + caches are scheduling hints, never cached pass results or test allowlists. + Preserve complete shard coverage and existing fixture-affinity groups. - **Tier 2 runs once per merged PR**, not per WIP push, since it triggers on `merge_group` only. Saves the bulk of integration minutes that the previous per-push flow burned. -- UPX compression when available (reduces binary size ~50%) - Python optimization level 2 in PyInstaller - Aggressive module exclusions (tkinter, matplotlib, etc.) \ No newline at end of file diff --git a/apm.lock.yaml b/apm.lock.yaml index c4404b22b4..1f734fd1f4 100644 --- a/apm.lock.yaml +++ b/apm.lock.yaml @@ -2800,7 +2800,7 @@ deployments: owners: - . active_owner: . - content_hash: sha256:33201cb88ea2f34b4950a9b52f87dc8dfb682796aaf53068ba7ae406c0c5e2c2 + content_hash: sha256:8194ad1d1f38d65f8b06eea8ffbde41c189a97da0b9d63981e76dfa833a5fb0c - kind: project-relative target: copilot value: .github/instructions/cli.instructions.md @@ -3291,7 +3291,7 @@ local_deployed_file_hashes: .github/agents/test-coverage-expert.agent.md: sha256:c79891a1b23ca5c814e9801d678de395a9dcf1f07e7e9340ffc5ae0891239e59 .github/instructions/architecture.instructions.md: sha256:71c0da596267f223b6a536234c3ac8e8c565fa7f289bbf23d0fe568bd9681859 .github/instructions/changelog.instructions.md: sha256:1e51ec4c74e847967962bd279dc4c6e582c5d3578490b3c28d5f3acd3e05f73e - .github/instructions/cicd.instructions.md: sha256:33201cb88ea2f34b4950a9b52f87dc8dfb682796aaf53068ba7ae406c0c5e2c2 + .github/instructions/cicd.instructions.md: sha256:8194ad1d1f38d65f8b06eea8ffbde41c189a97da0b9d63981e76dfa833a5fb0c .github/instructions/cli.instructions.md: sha256:8e39e8d5047ce88575cb02f87c2bcede584dfef258bd86f7466c7badf136541a .github/instructions/doc-sync.instructions.md: sha256:bb3816254f8df6bffc6faacd556871f36903e9d7f348982f1e2de0339384c696 .github/instructions/encoding.instructions.md: sha256:93db7377dc896f6efecf2c5d8c5d89255a555562f468d034d64c42edd5cf46d5 diff --git a/tests/conftest.py b/tests/conftest.py index 3093b8bc8f..a8e0512824 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,15 @@ _TMP_HOME = Path(tempfile.mkdtemp(prefix="apm-test-home-")) +def pytest_addoption(parser: pytest.Parser) -> None: + """Register CI options before integration conftests load during whole-tree collection.""" + parser.addoption( + "--strict-runtime-prerequisites", + action="store_true", + help="Fail collection when a selected test requires an unavailable runtime", + ) + + def _ensure_home_env(home: Path) -> None: home_str = str(home) os.environ["HOME"] = home_str diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e54040b7b5..473d0f81c9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -281,15 +281,6 @@ def _has_runtime(name: str) -> bool: } -def pytest_addoption(parser: pytest.Parser) -> None: - """Let provisioned CI selections reject missing runtimes instead of skipping.""" - parser.addoption( - "--strict-runtime-prerequisites", - action="store_true", - help="Fail collection when a selected test requires an unavailable runtime", - ) - - @pytest.hookimpl(trylast=True) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Auto-skip items whose marker precondition is not met. diff --git a/tests/unit/test_runtime_prerequisite_gate.py b/tests/unit/test_runtime_prerequisite_gate.py index fc6f1b31cc..c92e006be8 100644 --- a/tests/unit/test_runtime_prerequisite_gate.py +++ b/tests/unit/test_runtime_prerequisite_gate.py @@ -2,6 +2,11 @@ from __future__ import annotations +import os +import shutil +import subprocess +import sys +from pathlib import Path from unittest.mock import Mock import pytest @@ -11,6 +16,51 @@ pytestmark = pytest.mark.component +@pytest.mark.parametrize("strict", [False, True]) +def test_parent_directory_collection_registers_runtime_option_before_child_hooks( + tmp_path: Path, strict: bool +) -> None: + """Whole-tree collection must register options before loading integration hooks.""" + root = Path(__file__).resolve().parents[2] + integration = tmp_path / "tests" / "integration" + integration.mkdir(parents=True) + shutil.copyfile(root / "tests/conftest.py", tmp_path / "tests/conftest.py") + source = (root / "tests/integration/conftest.py").read_text(encoding="utf-8") + (integration / "conftest.py").write_text( + source + "\n\ndef _has_runtime(name):\n return False\n", + encoding="utf-8", + ) + (integration / "test_runtime.py").write_text( + "import pytest\n" + "pytestmark = pytest.mark.requires_runtime_codex\n" + "def test_runtime_case():\n" + " pass\n", + encoding="utf-8", + ) + (tmp_path / "pytest.ini").write_text( + "[pytest]\nmarkers = requires_runtime_codex: requires codex\n", + encoding="utf-8", + ) + command = [sys.executable, "-m", "pytest", "--collect-only", "--strict-markers", "-q"] + if strict: + command.append("--strict-runtime-prerequisites") + result = subprocess.run( + [*command, "tests"], + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": os.pathsep.join((str(root), str(root / "src")))}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + output = result.stdout + result.stderr + assert result.returncode == (4 if strict else 0), output + if strict: + assert "required runtime missing" in output + else: + assert "test_runtime_case" in output + + @pytest.mark.parametrize("runtime", ["copilot", "codex", "llm"]) @pytest.mark.parametrize("strict", [False, True]) def test_missing_selected_runtime_fails_closed_only_for_provisioned_ci( From 05c3aa563e6da7ee6c64a57cbbcf6f85eff039a1 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:24:58 +0200 Subject: [PATCH 03/23] perf(ci): shard ARM qualification and overlap read-only release builds Preserve exact candidate and scenario identity, bind rerun artifacts to attempts, avoid AST work in lexical guards, and add opt-in controlled performance probes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-release.yml | 88 ++-- .github/workflows/ci-integration.yml | 207 +++----- .github/workflows/ci-release-performance.yml | 188 +++++++ .github/workflows/ci-source-performance.yml | 150 ++++++ .github/workflows/ci.yml | 1 + .github/workflows/docs-build.yml | 98 ++++ .github/workflows/docs.yml | 122 ++--- .github/workflows/pypi-distributions.yml | 59 +++ .github/workflows/release-integration.yml | 211 ++++++++ .github/workflows/release-platform.yml | 129 ++--- .../docs/contributing/development-guide.md | 8 + .../docs/contributing/integration-testing.md | 65 ++- pyproject.toml | 1 + .../checks/lexical_shared.py | 14 +- scripts/compare_test_runs.py | 336 ++++++++++++ scripts/pytest_performance_evidence.py | 12 + scripts/release-candidate.cjs | 6 + scripts/release-candidate.test.cjs | 15 + scripts/release-platforms.json | 15 + scripts/test-integration.sh | 13 +- .../test_architecture_owner_rule_mutations.py | 18 + .../test_mcp_env_var_copilot_e2e.py | 1 - tests/quality/test_ci_topology.py | 46 +- .../unit/scripts/test_architecture_runner.py | 70 +++ tests/unit/scripts/test_compare_test_runs.py | 292 +++++++++++ tests/unit/test_ado_live_workflow_contract.py | 39 +- tests/unit/test_docs_contract_workflow.py | 16 +- tests/unit/test_mcp_runtime_prerequisite.py | 95 ++++ tests/unit/test_platform_contract_workflow.py | 219 +++++++- tests/unit/test_pytest_timing_workflows.py | 114 +++- .../unit/test_release_performance_workflow.py | 164 ++++++ .../unit/test_release_publication_overlap.py | 495 ++++++++++++++++++ .../unit/test_security_workflow_contracts.py | 10 +- .../unit/test_windows_compat_gate_workflow.py | 216 +++++++- uv.lock | 2 + 35 files changed, 3123 insertions(+), 412 deletions(-) create mode 100644 .github/workflows/ci-release-performance.yml create mode 100644 .github/workflows/ci-source-performance.yml create mode 100644 .github/workflows/docs-build.yml create mode 100644 .github/workflows/pypi-distributions.yml create mode 100644 .github/workflows/release-integration.yml create mode 100644 scripts/compare_test_runs.py create mode 100644 scripts/pytest_performance_evidence.py create mode 100644 tests/unit/scripts/test_compare_test_runs.py create mode 100644 tests/unit/test_mcp_runtime_prerequisite.py create mode 100644 tests/unit/test_release_performance_workflow.py create mode 100644 tests/unit/test_release_publication_overlap.py diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 7ba0540bf3..c37fa17029 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -68,6 +68,9 @@ jobs: platform: ${{ matrix.platform }} binary-name: ${{ matrix.binary_name }} integration-markers: ${{ matrix.integration_markers }} + integration-shard-count: ${{ matrix.integration_shard_count }} + integration-xdist-workers: ${{ matrix.integration_xdist_workers }} + integration-splitting-algorithm: ${{ matrix.integration_splitting_algorithm }} full-validation: ${{ needs.plan.outputs.full_validation == 'true' }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} @@ -174,11 +177,41 @@ jobs: }); - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: verified-release-assets + name: verified-release-assets-${{ github.run_attempt }} path: release-assets/ + compression-level: 0 if-no-files-found: error retention-days: 7 + build-docs-artifact: + name: Build Docs Artifact + needs: [plan] + if: >- + always() && !cancelled() && needs.plan.result == 'success' && + github.event_name == 'push' && github.ref_type == 'tag' && + needs.plan.outputs.is_prerelease != 'true' + uses: ./.github/workflows/docs-build.yml + permissions: + contents: read + with: + pages_artifact_name: docs-pages-${{ github.run_attempt }} + upload_pages_artifact: true + + build-pypi-distributions: + name: Build PyPI Distributions + needs: [plan] + if: >- + always() && !cancelled() && needs.plan.result == 'success' && + github.event_name == 'push' && github.ref_type == 'tag' && + github.event.repository.private != true && + needs.plan.outputs.is_prerelease != 'true' + uses: ./.github/workflows/pypi-distributions.yml + permissions: + contents: read + with: + artifact_name: pypi-distributions-${{ github.run_attempt }} + upload_artifact: true + create-release: name: Create GitHub Release needs: [plan, verify-candidate] @@ -196,7 +229,7 @@ jobs: - name: Download verified release archives uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: verified-release-assets + name: verified-release-assets-${{ github.run_attempt }} path: release-assets - name: Create GitHub Release uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 @@ -238,18 +271,25 @@ jobs: deploy-docs: name: Deploy Docs - needs: [create-release] + needs: [create-release, build-docs-artifact] if: >- always() && !cancelled() && needs.create-release.result == 'success' && + needs.build-docs-artifact.result == 'success' && github.event_name == 'push' && github.ref_type == 'tag' && needs.create-release.outputs.is_prerelease != 'true' - uses: ./.github/workflows/docs.yml + runs-on: ubuntu-24.04 permissions: - contents: read pages: write id-token: write - with: - is_prerelease: false + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 + with: + artifact_name: docs-pages-${{ github.run_attempt }} gh-aw-compat: name: GH-AW Compatibility @@ -280,40 +320,16 @@ jobs: if: failure() run: echo "::warning::GH-AW compatibility failed; investigate the external dependency before the next release." - build-pypi-distributions: - name: Build PyPI Distributions + publish-pypi: + name: Publish to PyPI runs-on: ubuntu-24.04 - needs: [create-release] + needs: [create-release, build-pypi-distributions] if: >- always() && !cancelled() && needs.create-release.result == 'success' && + needs.build-pypi-distributions.result == 'success' && github.event_name == 'push' && github.ref_type == 'tag' && needs.create-release.outputs.is_private_repo != 'true' && needs.create-release.outputs.is_prerelease != 'true' - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Install locked build dependencies - run: uv sync --frozen --extra dev - - name: Build Python package - run: uv build --no-build-isolation --no-sources - - name: Upload Python distributions - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: pypi-distributions - path: | - dist/*.whl - dist/*.tar.gz - if-no-files-found: error - - publish-pypi: - name: Publish to PyPI - runs-on: ubuntu-24.04 - needs: [build-pypi-distributions] - if: always() && !cancelled() && needs.build-pypi-distributions.result == 'success' environment: name: pypi url: https://pypi.org/p/apm-cli @@ -324,7 +340,7 @@ jobs: - name: Download Python distributions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: pypi-distributions + name: pypi-distributions-${{ github.run_attempt }} path: dist - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 1936c49700..6c72fdfa4b 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -67,22 +67,38 @@ jobs: chmod +x scripts/build-binary.sh uv run ./scripts/build-binary.sh + - name: Package exact candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: apm-linux-x86_64 + run: >- + python scripts/package_release.py pack + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Upload binary uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: apm-mq-linux-x86_64 + name: apm-mq-${{ github.run_attempt }}-linux-x86_64 path: | - ./dist/apm-linux-x86_64 - ./dist/apm-linux-x86_64.sha256 + release-assets/ + scripts/package_release.py + scripts/release-platforms.json + scripts/test-release-validation.sh + scripts/test-dependency-integration.sh + scripts/github-token-helper.sh + scripts/windows/test-release-validation.ps1 + scripts/windows/test-dependency-integration.ps1 + scripts/windows/github-token-helper.ps1 include-hidden-files: true retention-days: 7 if-no-files-found: error + compression-level: 0 - name: Freeze integration scheduling hints uses: ./.github/actions/pytest-timing with: suite: integration - snapshot: integration-duration-snapshot + snapshot: integration-duration-snapshot-${{ github.run_attempt }} smoke-test: name: Smoke Test (Linux) @@ -91,23 +107,24 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Artifact root is ./dist/ (least common ancestor of the uploaded paths - # in the build job), so download to ./dist/ to preserve the - # dist/apm-linux-x86_64/apm layout the conftest expects. - name: Download binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: apm-mq-linux-x86_64 - path: ./dist/ - - - name: Make binary executable - run: chmod +x ./dist/apm-linux-x86_64/apm + name: apm-mq-${{ github.run_attempt }}-linux-x86_64 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Extract checked candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: apm-linux-x86_64 + run: >- + python scripts/package_release.py verify-extract + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: @@ -157,8 +174,7 @@ jobs: integration-tests-shard: name: Integration Tests Shard ${{ matrix.shard }} (Linux) - needs: [build, smoke-test] - runs-on: ubuntu-24.04 + needs: [build] strategy: fail-fast: false matrix: @@ -168,120 +184,34 @@ jobs: # 4 HOME-mutating files carry an xdist_group("home_env") marker # so xdist serializes them on a single worker (race-safe). shard: [1, 2, 3, 4] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # Artifact root is ./dist/ (least common ancestor of the uploaded paths in - # the build job), so download to ./dist/ to preserve dist/apm-linux-x86_64/apm. - - name: Download binary - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: apm-mq-linux-x86_64 - path: ./dist/ - - - name: Make binary executable - run: chmod +x ./dist/apm-linux-x86_64/apm - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '24' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - - name: Install Unix shell startup test runtimes - run: | - sudo apt-get update - sudo apt-get install -y fish zsh - - - name: Install test dependencies - run: uv sync --frozen --extra dev - - - name: Download shared integration scheduling snapshot - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: integration-duration-snapshot - - # APM resolves its cache dir to ~/.cache/apm by default on Linux - # (XDG_CACHE_HOME unset -> $HOME/.cache/apm). actions/cache restores - # straight into that path, so the apm binary picks it up natively - # with no symlink or env override required. - - name: Compute cache week bucket - id: cache-bucket - run: echo "week=$(date -u +'%Y-%U')" >> "$GITHUB_OUTPUT" - - - name: Cache APM resolved git/HTTP fetches - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/apm - key: apm-cache-shard${{ matrix.shard }}-${{ steps.cache-bucket.outputs.week }}-${{ hashFiles('uv.lock') }} - restore-keys: | - apm-cache-shard${{ matrix.shard }}-${{ steps.cache-bucket.outputs.week }}- - apm-cache-shard${{ matrix.shard }}- - - - name: Run integration tests (sharded + parallelized) - env: - APM_E2E_TESTS: "1" - APM_RUN_INTEGRATION_TESTS: "1" - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }} - # Sharded + parallel pytest invocation passed through to - # scripts/test-integration.sh, which still owns runtime setup - # (apm runtime setup copilot/codex/llm) and token resolution. - # --dist loadgroup is required: it is the only xdist scheduler - # that honors pytest.mark.xdist_group, which we use to keep - # HOME-mutating tests serialized on a single worker. - PYTEST_EXTRA_ARGS: "--splits 4 --group ${{ matrix.shard }} -n 2 --dist loadgroup --maxfail 10 --cov --cov-fail-under=0 --durations=50 --store-durations --junitxml=test-results/integration.xml" - run: | - chmod +x scripts/test-integration.sh - uv run ./scripts/test-integration.sh - # Keep an outer cap above the runtime setup and sharded integration - # suite so runner variance does not turn into an unbounded job. - timeout-minutes: 20 - - - name: Upload test timings and outcomes - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: test-results-integration-shard-${{ matrix.shard }} - path: | - test-results/ - .test_durations - include-hidden-files: true - retention-days: 30 - if-no-files-found: error - - - name: Rename coverage data for shard - if: always() - run: | - if [ -f .coverage ]; then - mv .coverage .coverage.shard-${{ matrix.shard }} - fi - - - name: Upload coverage data - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: integration-coverage-shard-${{ matrix.shard }} - path: .coverage.shard-${{ matrix.shard }} - include-hidden-files: true - retention-days: 7 - if-no-files-found: ignore + uses: ./.github/workflows/release-integration.yml + with: + runner: ubuntu-24.04 + platform: linux + binary-name: apm-linux-x86_64 + candidate-artifact-name: apm-mq-${{ github.run_attempt }}-linux-x86_64 + candidate-sha: ${{ github.sha }} + timing-snapshot-artifact: integration-duration-snapshot-${{ github.run_attempt }} + integration-markers: not live + shard-count: 4 + shard-index: ${{ matrix.shard }} + xdist-workers: 2 + timeout-minutes: 20 + artifact-prefix: mq-linux-${{ github.run_attempt }} + runtime-prerequisites: none + run-network-integration-tests: true + cache-apm-fetches: true + coverage-enabled: true + secrets: + GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }} # Fan-in job that preserves the gate-required check name # "Integration Tests (Linux)". Branch protection / merge-gate.yml # require this exact name; shard jobs above are not required directly. integration-tests: name: Integration Tests (Linux) - needs: [integration-tests-shard, scaling-guards] + needs: [integration-tests-shard, scaling-guards, smoke-test] if: always() runs-on: ubuntu-24.04 steps: @@ -302,7 +232,7 @@ jobs: if: always() uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: integration-coverage-shard-* + pattern: integration-coverage-mq-linux-${{ github.run_attempt }}-shard-* path: coverage-shards/ merge-multiple: true continue-on-error: true @@ -347,6 +277,14 @@ jobs: fi echo "Scaling guards passed." + - name: Aggregate smoke-test result + run: | + if [[ "${{ needs.smoke-test.result }}" != "success" ]]; then + echo "Smoke test failed: ${{ needs.smoke-test.result }}" + exit 1 + fi + echo "Smoke test passed." + release-validation: name: Release Validation (Linux) needs: [build] @@ -368,25 +306,30 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - # See note in integration-tests: download to dist/ to preserve layout. - name: Download binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: apm-mq-linux-x86_64 - path: /tmp/apm-isolated-test/dist/ + name: apm-mq-${{ github.run_attempt }}-linux-x86_64 - - name: Make binary executable - run: chmod +x /tmp/apm-isolated-test/dist/apm-linux-x86_64/apm + - name: Extract checked candidate archive + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: apm-linux-x86_64 + run: >- + python repo/scripts/package_release.py verify-extract + --binary-name "$BINARY_NAME" --archives release-assets --sha "$CANDIDATE_SHA" - name: Create APM symlink for testing run: | - cd /tmp/apm-isolated-test + mkdir -p isolated-release-validation + mv dist isolated-release-validation/dist + cd isolated-release-validation ln -s "$(pwd)/dist/apm-linux-x86_64/apm" "$(pwd)/apm" - echo "/tmp/apm-isolated-test" >> $GITHUB_PATH + pwd >> "$GITHUB_PATH" - name: Prepare test scripts run: | - cp -r repo/scripts /tmp/apm-isolated-test/scripts + cp -r repo/scripts isolated-release-validation/scripts - name: Run release validation tests env: @@ -394,7 +337,7 @@ jobs: GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }} run: | - cd /tmp/apm-isolated-test + cd isolated-release-validation chmod +x scripts/test-release-validation.sh ./scripts/test-release-validation.sh timeout-minutes: 20 diff --git a/.github/workflows/ci-release-performance.yml b/.github/workflows/ci-release-performance.yml new file mode 100644 index 0000000000..459a026d21 --- /dev/null +++ b/.github/workflows/ci-release-performance.yml @@ -0,0 +1,188 @@ +name: CI Release Performance Probe + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + +permissions: + contents: read + +concurrency: + group: ci-release-performance-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + candidate: + name: Build ARM performance candidate + if: contains(github.event.pull_request.labels.*.name, 'ci-performance') + runs-on: macos-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Install locked build dependencies + run: uv sync --frozen --extra dev --extra build + - name: Build current ARM candidate + env: + UV_FROZEN: 'true' + UV_NO_SYNC: 'true' + run: uv run --frozen --no-sync bash scripts/build-binary.sh + - name: Exercise actual candidate core + env: + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-darwin-arm64/apm + run: >- + uv run --frozen --no-sync python -m pytest + tests/integration/test_core_smoke.py -q + --junitxml=test-results/core.xml + - name: Package one immutable candidate for every variant + run: >- + python scripts/package_release.py pack + --binary-name apm-darwin-arm64 --sha "$GITHUB_SHA" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-arm-candidate-${{ github.run_attempt }} + path: release-assets/ + compression-level: 0 + if-no-files-found: error + retention-days: 30 + - name: Freeze identical cold scheduling input + run: printf '{}\n' > .test_durations + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-arm-cold-timings-${{ github.run_attempt }} + path: .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Upload candidate startup outcome + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-arm-core-${{ github.run_attempt }} + path: test-results/ + if-no-files-found: error + retention-days: 30 + + integration: + name: ARM Integration (${{ matrix.variant }}, shard ${{ matrix.shard }}) + needs: [candidate] + if: contains(github.event.pull_request.labels.*.name, 'ci-performance') + strategy: + fail-fast: false + matrix: + include: + - variant: baseline + shard: 1 + count: 1 + workers: 4 + - variant: proposed + shard: 1 + count: 2 + workers: 2 + - variant: proposed + shard: 2 + count: 2 + workers: 2 + uses: ./.github/workflows/release-integration.yml + permissions: + contents: read + with: + runner: macos-latest + platform: darwin + binary-name: apm-darwin-arm64 + candidate-artifact-name: benchmark-arm-candidate-${{ github.run_attempt }} + candidate-sha: ${{ github.sha }} + timing-snapshot-artifact: benchmark-arm-cold-timings-${{ github.run_attempt }} + integration-markers: not live + shard-count: ${{ matrix.count }} + shard-index: ${{ matrix.shard }} + xdist-workers: ${{ matrix.workers }} + splitting-algorithm: least_duration + timeout-minutes: 65 + runtime-prerequisites: none + cache-apm-fetches: false + artifact-prefix: arm-probe-${{ matrix.variant }}-${{ github.run_attempt }} + evidence-variant: ${{ matrix.variant }} + evidence-selection: tests/integration/ -m not live + + compare: + name: Compare ARM Integration Performance + needs: [candidate, integration] + if: >- + always() && !cancelled() && + contains(github.event.pull_request.labels.*.name, 'ci-performance') + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require all benchmark executions succeeded + env: + BUILD_RESULT: ${{ needs.candidate.result }} + INTEGRATION_RESULT: ${{ needs.integration.result }} + run: | + test "$BUILD_RESULT" = success + test "$INTEGRATION_RESULT" = success + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Download baseline evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: performance-evidence-arm-probe-baseline-${{ github.run_attempt }}-baseline-shard-1 + path: performance-inputs/baseline + - name: Download proposed shard one evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: performance-evidence-arm-probe-proposed-${{ github.run_attempt }}-proposed-shard-1 + path: performance-inputs/proposed-1 + - name: Download proposed shard two evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: performance-evidence-arm-probe-proposed-${{ github.run_attempt }}-proposed-shard-2 + path: performance-inputs/proposed-2 + - name: Require identical outcomes and a measured execution gain + run: >- + python -m scripts.compare_test_runs compare + --baseline performance-inputs/baseline/baseline-shard-1.json + --proposed performance-inputs/proposed-1/proposed-shard-1.json + performance-inputs/proposed-2/proposed-shard-2.json + --expected-sha "$GITHUB_SHA" --minimum-speedup 0.05 + --output performance-comparison/arm-cold.json + - name: Upload comparison + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: arm-performance-comparison-${{ github.run_attempt }} + path: performance-comparison/ + if-no-files-found: ignore + retention-days: 30 + + python-distributions: + name: Read-only Python distribution build + if: contains(github.event.pull_request.labels.*.name, 'ci-performance') + uses: ./.github/workflows/pypi-distributions.yml + permissions: + contents: read + with: + artifact_name: benchmark-python-distributions-${{ github.run_attempt }} + upload_artifact: true + + docs: + name: Read-only documentation build + if: contains(github.event.pull_request.labels.*.name, 'ci-performance') + uses: ./.github/workflows/docs-build.yml + permissions: + contents: read + with: + pages_artifact_name: benchmark-docs-${{ github.run_attempt }} + upload_pages_artifact: true diff --git a/.github/workflows/ci-source-performance.yml b/.github/workflows/ci-source-performance.yml new file mode 100644 index 0000000000..c0578dede2 --- /dev/null +++ b/.github/workflows/ci-source-performance.yml @@ -0,0 +1,150 @@ +name: CI Source Performance Probe + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + +permissions: + contents: read + +concurrency: + group: ci-source-performance-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + windows-compat-performance: + name: Windows Compat Performance (${{ matrix.variant }}) + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance') }} + runs-on: windows-latest + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - variant: baseline + pytest_extra: '' + - variant: proposed + pytest_extra: '-n 2 --dist loadgroup' + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + - name: Install dependencies + run: uv sync --frozen --extra dev + + - name: Record Windows compatibility selection + shell: pwsh + run: | + New-Item -ItemType Directory -Force "test-results/${{ matrix.variant }}" | Out-Null + "variant=${{ matrix.variant }}" | Set-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" + "pytest_extra=${{ matrix.pytest_extra }}" | Add-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" + "selection=windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py" | Add-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" + + - name: Run Windows compatibility selection + shell: pwsh + run: >- + uv run --frozen --no-sync --extra dev python -m pytest -p no:cacheprovider + -p scripts.pytest_performance_evidence + -v + -m windows_compat + tests/unit + tests/integration/test_lifecycle_workspace_lock.py + ${{ matrix.pytest_extra }} + --durations=50 + --store-durations + --junitxml=test-results/${{ matrix.variant }}/windows.xml + + - name: Capture comparable proof + if: always() + shell: pwsh + run: >- + python -m scripts.compare_test_runs capture + --junit test-results/${{ matrix.variant }}/windows.xml + --output test-results/${{ matrix.variant }}/proof.json + --source-sha "$env:GITHUB_SHA" + --selection 'windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py' + --variant ${{ matrix.variant }} + --shard 1 + --shard-count 1 + + - name: Preserve timing file with mode + if: always() + shell: pwsh + run: | + if (Test-Path ".test_durations") { + Copy-Item ".test_durations" "test-results/${{ matrix.variant }}/.test_durations" + } + + - name: Upload Windows compatibility performance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-compat-performance-${{ matrix.variant }}-${{ github.run_attempt }} + path: test-results/${{ matrix.variant }}/ + include-hidden-files: true + retention-days: 14 + if-no-files-found: error + + compare-windows-compat-performance: + name: Compare Windows Compat Performance + if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'ci-performance') }} + needs: [windows-compat-performance] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require benchmark variants succeeded + run: | + if [ "${{ needs.windows-compat-performance.result }}" != "success" ]; then + echo "::error::Windows compatibility benchmark variants did not both succeed: ${{ needs.windows-compat-performance.result }}" + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Download baseline proof + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: windows-compat-performance-baseline-${{ github.run_attempt }} + path: performance-inputs/baseline + + - name: Download proposed proof + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: windows-compat-performance-proposed-${{ github.run_attempt }} + path: performance-inputs/proposed + + - name: Compare equivalent Windows compatibility runs + run: >- + python -m scripts.compare_test_runs compare + --baseline performance-inputs/baseline/proof.json + --proposed performance-inputs/proposed/proof.json + --expected-sha "$GITHUB_SHA" + --minimum-speedup 0.05 + --output performance-comparison/comparison.json + + - name: Upload Windows compatibility comparison + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-compat-performance-comparison-${{ github.run_attempt }} + path: performance-comparison/ + include-hidden-files: true + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93c02c3bca..9a1f858174 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,7 @@ jobs: - name: Run cross-platform contract family run: >- uv run --extra dev pytest -p no:cacheprovider -v + -n 2 --dist loadgroup -m windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml new file mode 100644 index 0000000000..f79390f9b6 --- /dev/null +++ b/.github/workflows/docs-build.yml @@ -0,0 +1,98 @@ +name: Build Docs Artifact + +on: + workflow_call: + inputs: + pages_artifact_name: + description: 'Name for the uploaded Pages artifact.' + required: false + type: string + default: github-pages + upload_pages_artifact: + description: 'Upload the rendered docs as a Pages artifact.' + required: false + type: boolean + default: false + workflow_dispatch: + inputs: + pages_artifact_name: + description: 'Name for the uploaded Pages artifact.' + required: false + type: string + default: github-pages + upload_pages_artifact: + description: 'Upload the rendered docs as a Pages artifact for read-only timing.' + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: 'docs/package-lock.json' + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + + - name: Install Python test dependencies + run: uv sync --frozen --extra dev + + - name: Install dependencies + working-directory: ./docs + run: npm ci + + - name: Test generated-link checker + working-directory: ./docs + run: npm run test:links + + - name: Test CLI registry contract + run: >- + uv run --frozen pytest + tests/unit/test_cli_docs_contract.py + tests/integration/test_cli_docs_contract.py + -q + + - name: Build documentation + working-directory: ./docs + run: npm run build + + - name: Check CLI registry against rendered pages + run: uv run --frozen python scripts/check_cli_docs.py docs/dist + + - name: Verify schema $id reachability + # Asserts every JSON Schema's declared $id URL resolves to a + # byte-identical file in the build output. Without this, a + # schema move or $id typo would silently break every toolchain + # that pinned to the canonical URL. Uses python3 stdlib only. + run: python3 scripts/check_schema_ids.py + + - name: Upload build artifacts + if: inputs.upload_pages_artifact != false + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + name: ${{ inputs.pages_artifact_name || 'github-pages' }} + path: docs/dist diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 37ccee010d..c1703f5c5d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,19 +3,14 @@ name: Deploy Docs on: # Deploy docs only when a new APM version is released, so the published # site always matches the latest released binary (see microsoft/apm#641). - # Primary entrypoint is workflow_call from the CI/CD Pipeline release job - # (release: published does not fire when the release is created by - # GITHUB_TOKEN -- a documented Actions safeguard against recursion). + # This wrapper delegates all rendering and checks to docs-build.yml. The + # release pipeline calls docs-build.yml directly for read-only prebuilds, + # keeping Pages/OIDC permissions out of that reusable call chain. + # release: published does not fire when the release is created by + # GITHUB_TOKEN -- a documented Actions safeguard against recursion. # The release: published trigger is kept as a safety net for human-cut # releases. PR runs build (no deploy) to catch breakage before merge. # Manual workflow_dispatch is supported for re-publishing the current docs. - workflow_call: - inputs: - is_prerelease: - description: 'Skip deploy when true (build-only). Defaults to false (deploy).' - required: false - type: boolean - default: false release: types: [published] pull_request: @@ -27,7 +22,24 @@ on: - 'tests/unit/test_cli_docs_contract.py' - 'tests/integration/test_cli_docs_contract.py' - '.github/workflows/docs.yml' + - '.github/workflows/docs-build.yml' workflow_dispatch: + inputs: + deploy: + description: 'Deploy after building docs. Set false for read-only timing.' + required: false + type: boolean + default: true + pages_artifact_name: + description: 'Name for the uploaded Pages artifact.' + required: false + type: string + default: github-pages + upload_pages_artifact: + description: 'Upload the rendered docs as a Pages artifact.' + required: false + type: boolean + default: true permissions: contents: read @@ -38,84 +50,26 @@ concurrency: jobs: build: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '22' - cache: 'npm' - cache-dependency-path: 'docs/package-lock.json' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - - name: Install Python test dependencies - run: uv sync --frozen --extra dev - - - name: Install dependencies - working-directory: ./docs - run: npm ci - - - name: Test generated-link checker - working-directory: ./docs - run: npm run test:links - - - name: Test CLI registry contract - run: >- - uv run --frozen pytest - tests/unit/test_cli_docs_contract.py - tests/integration/test_cli_docs_contract.py - -q - - - name: Build documentation - working-directory: ./docs - run: npm run build - - - name: Check CLI registry against rendered pages - run: uv run --frozen python scripts/check_cli_docs.py docs/dist - - - name: Verify schema $id reachability - # Asserts every JSON Schema's declared $id URL resolves to a - # byte-identical file in the build output. Without this, a - # schema move or $id typo would silently break every toolchain - # that pinned to the canonical URL. Uses python3 stdlib only. - run: python3 scripts/check_schema_ids.py - - - name: Upload build artifacts - # NOTE: in a reusable workflow, github.event_name reflects the CALLER's - # event (here: 'push' of a v* tag from build-release.yml), NOT - # 'workflow_call'. Detect the workflow_call invocation via the tag-push - # context instead. PR runs (event_name == 'pull_request') correctly - # build-only because none of the three branches match. - if: | - github.event_name == 'workflow_dispatch' || - (github.event_name == 'release' && github.event.release.prerelease == false) || - (github.event_name == 'push' && github.ref_type == 'tag' && inputs.is_prerelease == false) - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 - with: - path: docs/dist + name: Build Docs Artifact + uses: ./.github/workflows/docs-build.yml + permissions: + contents: read + with: + pages_artifact_name: ${{ inputs.pages_artifact_name || 'github-pages' }} + upload_pages_artifact: >- + ${{ + (github.event_name == 'workflow_dispatch' && inputs.upload_pages_artifact != false) || + (github.event_name == 'release' && github.event.release.prerelease == false) + }} deploy: needs: build # Only stable releases (or manual dispatch) update the public docs site, # so prerelease tags (vX.Y.Z-rc1, etc.) don't clobber published docs. - # NOTE: in a reusable workflow, github.event_name reflects the CALLER's - # event ('push' of a v* tag from build-release.yml), NOT 'workflow_call'. - if: | - github.event_name == 'workflow_dispatch' || - (github.event_name == 'release' && github.event.release.prerelease == false) || - (github.event_name == 'push' && github.ref_type == 'tag' && inputs.is_prerelease == false) + if: >- + always() && !cancelled() && needs.build.result == 'success' && + ((github.event_name == 'workflow_dispatch' && inputs.deploy != false) || + (github.event_name == 'release' && github.event.release.prerelease == false)) runs-on: ubuntu-latest permissions: pages: write @@ -127,3 +81,5 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 + with: + artifact_name: ${{ inputs.pages_artifact_name || 'github-pages' }} diff --git a/.github/workflows/pypi-distributions.yml b/.github/workflows/pypi-distributions.yml new file mode 100644 index 0000000000..99fd5a666f --- /dev/null +++ b/.github/workflows/pypi-distributions.yml @@ -0,0 +1,59 @@ +name: Build PyPI Distributions + +on: + workflow_call: + inputs: + artifact_name: + description: 'Name for the uploaded Python distribution artifact.' + required: false + type: string + default: pypi-distributions + upload_artifact: + description: 'Upload the built distributions for a later publisher job.' + required: false + type: boolean + default: true + workflow_dispatch: + inputs: + artifact_name: + description: 'Name for the uploaded Python distribution artifact.' + required: false + type: string + default: pypi-distributions + upload_artifact: + description: 'Upload the built distributions for read-only timing.' + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + build: + name: Build PyPI Distributions + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install locked build dependencies + run: uv sync --frozen --extra dev + - name: Build Python package + run: uv build --no-build-isolation --no-sources + - name: Upload Python distributions + if: inputs.upload_artifact != false + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact_name || 'pypi-distributions' }} + path: | + dist/*.whl + dist/*.tar.gz + compression-level: 0 + if-no-files-found: error diff --git a/.github/workflows/release-integration.yml b/.github/workflows/release-integration.yml new file mode 100644 index 0000000000..0c17be8c1f --- /dev/null +++ b/.github/workflows/release-integration.yml @@ -0,0 +1,211 @@ +name: Release integration shard + +on: + workflow_call: + inputs: + runner: + type: string + required: true + platform: + type: string + required: true + binary-name: + type: string + required: true + candidate-artifact-name: + type: string + required: true + candidate-sha: + type: string + required: true + timing-snapshot-artifact: + type: string + required: true + integration-markers: + type: string + default: not live + shard-count: + type: number + default: 1 + shard-index: + type: number + default: 1 + xdist-workers: + type: number + default: 4 + splitting-algorithm: + type: string + default: duration_based_chunks + timeout-minutes: + type: number + default: 60 + artifact-prefix: + type: string + required: true + runtime-prerequisites: + type: string + default: none + run-network-integration-tests: + type: boolean + default: false + cache-apm-fetches: + type: boolean + default: false + coverage-enabled: + type: boolean + default: false + evidence-variant: + type: string + default: '' + evidence-selection: + type: string + default: '' + secrets: + GH_CLI_PAT: + required: false + ADO_APM_PAT: + required: false + +permissions: + contents: read + +env: + PYTHON_VERSION: '3.12' + +jobs: + integration-tests: + name: Integration Tests Shard ${{ inputs.shard-index }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.candidate-artifact-name }} + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install Linux shell test runtimes + if: inputs.platform == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y fish zsh + - name: Install macOS shell test runtimes + if: inputs.platform == 'darwin' + run: | + brew list bash >/dev/null 2>&1 || brew install bash + brew list fish >/dev/null 2>&1 || brew install fish + - name: Install dependencies + run: uv sync --frozen --extra dev + - name: Extract checked candidate archive + env: + CANDIDATE_SHA: ${{ inputs.candidate-sha }} + BINARY_NAME: ${{ inputs.binary-name }} + shell: bash + run: >- + python scripts/package_release.py verify-extract + --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Download shared integration scheduling snapshot + if: inputs.platform != 'windows' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.timing-snapshot-artifact }} + - name: Compute cache week bucket + if: inputs.cache-apm-fetches + id: cache-bucket + run: echo "week=$(date -u +'%Y-%U')" >> "$GITHUB_OUTPUT" + - name: Cache APM resolved git/HTTP fetches + if: inputs.cache-apm-fetches + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/apm + key: apm-cache-shard${{ inputs.shard-index }}-${{ steps.cache-bucket.outputs.week }}-${{ hashFiles('uv.lock') }} + restore-keys: | + apm-cache-shard${{ inputs.shard-index }}-${{ steps.cache-bucket.outputs.week }}- + apm-cache-shard${{ inputs.shard-index }}- + - name: Run integration tests (Unix) + if: inputs.platform != 'windows' + env: + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }} + APM_RUN_INTEGRATION_TESTS: ${{ inputs.run-network-integration-tests && '1' || '' }} + PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} + APM_TEST_RUNTIMES: ${{ inputs.runtime-prerequisites }} + PYTEST_EXTRA_ARGS: >- + ${{ inputs.evidence-variant != '' && '-p scripts.pytest_performance_evidence' || '' }} + --splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} + --splitting-algorithm ${{ inputs.splitting-algorithm }} + -n ${{ inputs.xdist-workers }} --dist loadgroup + --durations=50 --store-durations + --junitxml=test-results/integration-shard-${{ inputs.shard-index }}.xml + ${{ inputs.coverage-enabled && '--cov --cov-fail-under=0' || '' }} + run: uv run --frozen bash scripts/test-integration.sh + - name: Run integration tests (Windows) + if: inputs.platform == 'windows' + timeout-minutes: 20 + env: + APM_E2E_TESTS: '1' + APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm.exe + GITHUB_API_TOKEN: ${{ github.token }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + run: uv run --frozen pwsh scripts/windows/test-integration.ps1 -SkipBuild + - name: Capture integration performance evidence + if: always() && inputs.platform != 'windows' && inputs.evidence-variant != '' + env: + EVIDENCE_SELECTION: ${{ inputs.evidence-selection }} + PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} + run: | + mkdir -p performance-evidence + selection="${EVIDENCE_SELECTION:-tests/integration/ -m $PYTEST_MARK_EXPR}" + python -m scripts.compare_test_runs capture \ + --junit "test-results/integration-shard-${{ inputs.shard-index }}.xml" \ + --output "performance-evidence/${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }}.json" \ + --source-sha "${{ inputs.candidate-sha }}" \ + --selection "$selection" \ + --variant "${{ inputs.evidence-variant }}" \ + --shard "${{ inputs.shard-index }}" \ + --shard-count "${{ inputs.shard-count }}" \ + --candidate-metadata "release-assets/${{ inputs.binary-name }}.json" + - name: Upload integration timings and outcomes + if: always() && inputs.platform != 'windows' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ inputs.artifact-prefix }}-integration-shard-${{ inputs.shard-index }} + path: | + test-results/ + .test_durations + include-hidden-files: true + retention-days: 30 + if-no-files-found: error + - name: Upload integration performance evidence + if: always() && inputs.platform != 'windows' && inputs.evidence-variant != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-evidence-${{ inputs.artifact-prefix }}-${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }} + path: performance-evidence/ + retention-days: 30 + if-no-files-found: error + - name: Rename coverage data for shard + if: always() && inputs.coverage-enabled + run: | + if [ -f .coverage ]; then + mv .coverage .coverage.shard-${{ inputs.shard-index }} + fi + - name: Upload coverage data + if: always() && inputs.coverage-enabled + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-coverage-${{ inputs.artifact-prefix }}-shard-${{ inputs.shard-index }} + path: .coverage.shard-${{ inputs.shard-index }} + include-hidden-files: true + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml index 8a036614fb..009a6b5106 100644 --- a/.github/workflows/release-platform.yml +++ b/.github/workflows/release-platform.yml @@ -15,6 +15,15 @@ on: integration-markers: type: string default: not live + integration-shard-count: + type: number + default: 1 + integration-xdist-workers: + type: number + default: 4 + integration-splitting-algorithm: + type: string + default: duration_based_chunks full-validation: type: boolean required: true @@ -79,7 +88,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-results-${{ inputs.binary-name }}-unit + name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit path: | test-results/ .test_durations @@ -130,6 +139,12 @@ jobs: run: >- python scripts/package_release.py pack --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" + - name: Freeze integration scheduling hints + if: inputs.full-validation && inputs.platform != 'windows' + uses: ./.github/actions/pytest-timing + with: + suite: integration + snapshot: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} - name: Upload binary as workflow artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -147,90 +162,58 @@ jobs: include-hidden-files: true retention-days: 30 if-no-files-found: error + compression-level: 0 - name: Upload native smoke outcomes if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-results-${{ inputs.binary-name }}-core + name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-core path: test-results/ if-no-files-found: error retention-days: 30 - integration-tests: - name: Integration Tests + integration-tests-shard: + name: Integration Tests Shard ${{ matrix.shard }} needs: [build] if: inputs.full-validation - runs-on: ${{ inputs.runner }} - timeout-minutes: 65 + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.integration-shard-count == 2 && '{"shard":[1,2]}' || '{"shard":[1]}') }} + uses: ./.github/workflows/release-integration.yml + with: + runner: ${{ inputs.runner }} + platform: ${{ inputs.platform }} + binary-name: ${{ inputs.binary-name }} + candidate-artifact-name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + candidate-sha: ${{ github.sha }} + timing-snapshot-artifact: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} + integration-markers: ${{ inputs.integration-markers }} + shard-count: ${{ inputs.integration-shard-count }} + shard-index: ${{ matrix.shard }} + xdist-workers: ${{ inputs.integration-xdist-workers }} + splitting-algorithm: ${{ inputs.integration-splitting-algorithm }} + timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} + artifact-prefix: ${{ github.run_attempt }}-${{ inputs.binary-name }} + runtime-prerequisites: none + secrets: + GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + + integration-tests: + name: Integration Tests + needs: [integration-tests-shard] + if: always() && inputs.full-validation + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '24' - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - name: Install Linux shell test runtimes - if: inputs.platform == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y fish zsh - - name: Install macOS shell test runtimes - if: inputs.platform == 'darwin' - run: | - brew list bash >/dev/null 2>&1 || brew install bash - brew list fish >/dev/null 2>&1 || brew install fish - - name: Install dependencies - run: uv sync --frozen --extra dev - - name: Extract checked candidate archive - env: - CANDIDATE_SHA: ${{ github.sha }} - BINARY_NAME: ${{ inputs.binary-name }} - shell: bash - run: >- - python scripts/package_release.py verify-extract - --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" - - uses: ./.github/actions/pytest-timing - with: - suite: integration - - name: Run integration tests (Unix) - if: inputs.platform != 'windows' - timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} - env: - APM_E2E_TESTS: '1' - APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} - APM_TEST_RUNTIMES: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 'none' || 'copilot' }} - PYTEST_EXTRA_ARGS: -n 4 --dist loadgroup --durations=50 --store-durations --junitxml=test-results/integration.xml - run: uv run --frozen bash scripts/test-integration.sh - - name: Run integration tests (Windows) - if: inputs.platform == 'windows' - timeout-minutes: 20 + - name: Require every applicable integration shard + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - APM_E2E_TESTS: '1' - APM_BINARY_PATH: ${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm.exe - GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: uv run --frozen pwsh scripts/windows/test-integration.ps1 -SkipBuild - - name: Upload integration timings and outcomes - if: always() && inputs.platform != 'windows' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + RESULTS: ${{ toJSON(needs) }} with: - name: test-results-${{ inputs.binary-name }}-integration - path: | - test-results/ - .test_durations - include-hidden-files: true - retention-days: 30 - if-no-files-found: error + script: | + const results = JSON.parse(process.env.RESULTS); + if (results['integration-tests-shard']?.result !== 'success') { + core.setFailed('integration-tests-shard did not succeed'); + } # No checkout or editable package: validate the actual archive on a fresh # native runner, independently of integration and its mutable HOME/caches. @@ -332,7 +315,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-results-${{ inputs.binary-name }}-installer + name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-installer path: test-results/ retention-days: 30 if-no-files-found: error diff --git a/docs/src/content/docs/contributing/development-guide.md b/docs/src/content/docs/contributing/development-guide.md index 73c6ed5471..522f70b1cd 100644 --- a/docs/src/content/docs/contributing/development-guide.md +++ b/docs/src/content/docs/contributing/development-guide.md @@ -67,6 +67,14 @@ Ordinary `main` green status is not enough. See [Integration Testing](../integration-testing/) for the native platform matrix, runtime selections, and archive-promotion model. +Maintainers can apply the `ci-performance` PR label to run read-only native +and Windows before/after comparisons. These probes require identical test +outcomes and report execution time separately from runner setup and queue +delays. They neither publish releases nor replace required qualification. +Remove the label before unrelated follow-up pushes to avoid rerunning them. +See [performance capture](../integration-testing/#timing-and-performance-capture) +for artifact names and acceptance criteria. + ### Workflow dependency updates When updating actions in generated `.github/workflows/*.lock.yml` files, diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index e645650352..6a73e37ad1 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -40,7 +40,7 @@ to the required runtimes. - **Duration**: the required expression must remain inside its hard 6-minute job timeout; hosted duration is authoritative. - **Trigger**: every pull request and merge queue run (`ci.yml`'s `lifecycle-smoke` job, required via `merge-gate.yml`) - **Selection mechanism**: `pytest --strict-markers -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration` -- declarative, not a file/node-id list. No central count or membership list is maintained. -- **Full-coverage path**: merge-group workflow `ci-integration.yml`, job `integration-tests-shard`, step `Run integration tests (sharded + parallelized)`, calls `uv run ./scripts/test-integration.sh`; that script runs unfiltered `pytest tests/integration/`, so the complete lifecycle family remains exercised. +- **Full-coverage path**: merge-group workflow `ci-integration.yml` calls the shared `release-integration.yml` shard runner. Its Unix step invokes `scripts/test-integration.sh` over `tests/integration/`, so the complete lifecycle family remains exercised. - **Drift guard**: `tests/quality/test_ci_topology.py` independently collects the full, merge-group-only, and required selections; verifies their set partition; and preserves the required expression, full-integration execution path, step-level `APM_E2E_TESTS: "1"` binding, network/credential prohibitions, and required-check membership. - **Fixture controls**: lifecycle helpers set `APM_TEST_LOOPBACK_PORTS` for a port-scoped local registry and `APM_TEST_FAIL_LOCK_REPLACE=1` for atomic-write fault injection. These are internal test controls, not user-facing APM settings. - **Learning ledger**: `tests/fixtures/lifecycle_bug_ledger.json` maps representative escaped defects to generalized laws, oracle tiers, phases, and executable regression node IDs, including coverage of already-correct behavior. Use same-workspace transitions to prove survivor ownership and scoped cleanup. It is not a bug-count census; `tests/quality/test_lifecycle_bug_ledger.py` validates its taxonomy and links. @@ -346,8 +346,11 @@ per-test gating lives in the marker registry described above. `APM_TEST_RUNTIMES` controls which external runtimes the orchestrator installs: - unset locally: install `copilot codex llm` -- `copilot`: install only Copilot CLI for full non-live release qualification -- `none`: install no external runtime for focused lifecycle selections +- `copilot`: install only Copilot CLI when explicitly needed for local reproduction +- `none`: install no external runtime up front; native and merge-queue CI use this selection + +Config-rendering cases do not require Copilot itself. Tests whose purpose is +runtime installation still execute their own setup in isolated environments. Release promotion sets `PYTEST_MARK_EXPR` instead of editing test lists. Full non-live qualification uses `not live`; the focused macOS Intel lifecycle lane @@ -381,6 +384,9 @@ At PR time the gate requires lint, test architecture ratchets, Linux unit shards, Windows compatibility, APM self-check, NOTICE drift, and Lifecycle Smoke. In merge queue context it also requires the Linux build, core smoke, full integration, release validation, and the coverage combine gate. +The Windows compatibility selection uses two workers with `--dist loadgroup`. +Merge-queue smoke, integration shards, and isolated validation can start after +the build independently; all remain required at the final gate. ### Release qualification @@ -397,6 +403,16 @@ full integration, release validation, and the coverage combine gate. 5. Promote only the exact archives that were packaged once, verified, unpacked, and tested. +Unix integration uses the shared `release-integration.yml` runner. macOS ARM +uses two outer shards with two workers each; other native Unix lanes use one +shard with four workers. ARM uses `least_duration` assignment, preserving +relative order within each shard. Every shard must succeed. + +For stable tags, read-only docs and wheel builds also start after release +planning. Their publishers wait for successful GitHub Release creation and +their own build artifacts. Prebuilding does not grant publishing permissions +or make failed qualification deployable. + The publication path never repacks candidate archives. `scripts/package_release.py` packages each native archive once, records archive and executable SHA-256 digests, verifies archive member safety, checks the embedded version/build SHA, @@ -480,7 +496,7 @@ with the executable member inside the candidate ZIP. ### Windows unit hang diagnostics -The Windows full-unit step in `.github/workflows/build-release.yml` runs: +The Windows full-unit step in `.github/workflows/release-platform.yml` runs: ```sh uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal -vv --tb=short --show-capture=no --no-showlocals @@ -513,6 +529,7 @@ Runtime setup prefers `GITHUB_TOKEN` for GitHub Models and falls back to `GITHUB - Policy pinned-constraint enforcement (Govern contract) - The virtual/manifestless lifecycle matrix: install, lock, frozen-install, update, and audit stay consistent (the direct #2240 regression) - The ADO lock-coordinate single-owner guard (the direct #2226 regression) +- Architecture registry coverage: every registered guard has exactly one mutation case. Only the cheap completeness assertion runs in PR smoke; the full mutation matrix remains in integration qualification. - Prune's merged-hook and ownership-sidecar reconciliation for the `claude` target (the direct #2249 regression -- an orphaned package's merged hook entries and sidecar markers must be cleaned up, not left pointing at deleted scripts) - No network, no credentials, no built binary required for any of the above @@ -529,16 +546,42 @@ They do not restore a rolling cache or present the shards as timing-balanced: the shared planner job required for that can cost more runner allocation time than it saves in balancing. -Native full-suite release lanes may restore timing history through -`.github/actions/pytest-timing`, keyed by OS, architecture, suite, commit SHA, -and partition. It remains a hint only: no test-result cache, test selection, or -allowlist. Grouped tests still use `--dist loadgroup` so HOME-mutating fixtures -stay on one worker. +Native builds also freeze any available timing history through +`.github/actions/pytest-timing`; all shards consume that one immutable +snapshot. Measured shard histories are retained as artifacts, not independently +restored or written back by each shard. Hints never replace test execution or +act as an allowlist. Grouped tests still use `--dist loadgroup` so +HOME-mutating fixtures stay on one worker within each isolated job. Inspect expensive fixtures by opening the slowest `--durations=50` entries and correlating fixture-heavy node IDs with JUnit XML for the same OS, architecture, -suite, and shard. Do not claim release speedups until optimized hosted data -exists; the current tag baseline is 39m16s. +suite, and shard. Historical release run `34104956270` completed in 39m16s; +compare a later complete release before claiming an end-to-end improvement. + +#### Opt-in performance comparisons + +Apply the `ci-performance` PR label to run `ci-release-performance.yml` and +`ci-source-performance.yml`. The native probe builds one current macOS ARM +candidate, then compares one four-worker run against two two-worker shards. +All three consume the same verified archive and the same empty timing +snapshot, testing cold-history behavior rather than assuming a warm cache. +The Windows probe compares serial and two-worker execution of the unchanged +`windows_compat` selection. The probes use read-only permissions and no +production secrets; they cannot publish or qualify a release candidate. + +`scripts.compare_test_runs` checks original pytest node IDs, outcomes, complete +disjoint shard coverage, source/candidate identity, and runtime/lockfile +fingerprints before accepting a measured execution improvement of at least +5%. New skips, missing tests, failures, or different environments fail the +comparison. JUnit controller elapsed time is reported separately from summed +runner time; neither includes runner allocation or setup. + +Download `arm-performance-comparison-` or +`windows-compat-performance-comparison-` for the JSON verdict, and +the corresponding per-variant artifacts for raw JUnit and timings. Use Actions +job timestamps to include setup and queue delays. Remove the opt-in label +before unrelated pushes; these measurements are evidence, not required +release gates or cached pass results. ## Debugging Test Failures diff --git a/pyproject.toml b/pyproject.toml index 00b88a5720..ce749a9029 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "pylint>=3.0.0", "hypothesis>=6.0.0", "mutmut==3.6.0", + "packaging>=24.0", ] build = [ "pyinstaller>=6.0.0", diff --git a/scripts/architecture_linter/checks/lexical_shared.py b/scripts/architecture_linter/checks/lexical_shared.py index 46b52e7813..74f77b4b92 100644 --- a/scripts/architecture_linter/checks/lexical_shared.py +++ b/scripts/architecture_linter/checks/lexical_shared.py @@ -250,15 +250,21 @@ def duplicate_definition_lines( message: str, respect_exempt: bool, ) -> list[Violation]: - """Flag matching definitions outside their canonical owner.""" + """Flag matching definition-shaped lines outside their canonical owner. + + This is a legacy grep-equivalent check: callers provide the exact lexical + pattern that constitutes a duplicate for their owner decision. Keep the + scan on cached source lines so broad duplicate checks do not accidentally + parse and index every Python file under the searched prefix. + """ findings: list[Violation] = [] for path in python_paths(provider, under=prefix): if path == owner: continue - facts = provider.file_facts(path) - if facts.read_error is not None: + lines, read_error = provider.lexical_lines(path) + if read_error is not None: continue - for number, line in enumerate(facts.lines, start=1): + for number, line in enumerate(lines, start=1): if respect_exempt and EXEMPT_MARKER in line: continue match = pattern.search(line) diff --git a/scripts/compare_test_runs.py b/scripts/compare_test_runs.py new file mode 100644 index 0000000000..e20940d9d3 --- /dev/null +++ b/scripts/compare_test_runs.py @@ -0,0 +1,336 @@ +"""Compare pytest execution time only after proving equivalent test evidence.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + +from scripts.package_release import archive_name, file_digest, require_sha + +OUTCOMES = {"passed", "skipped", "xfailed", "failed", "error"} + + +class _JUnitTreeBuilder(ET.TreeBuilder): + def doctype(self, name: str, pubid: str | None, system: str | None) -> None: + """Reject DTDs before the XML parser can expand custom entities.""" + raise ValueError("JUnit document type declarations are forbidden") + + +@dataclass(frozen=True) +class Report: + """One pytest invocation, including its explicit source and shard identity.""" + + identity: dict[str, object] + variant: str + shard: int + shard_count: int + elapsed_seconds: float + cases: dict[tuple[str, str], str] + + def payload(self) -> dict[str, object]: + """Return a deterministic, inspectable evidence document.""" + return { + "schema_version": 1, + "identity": self.identity, + "variant": self.variant, + "shard": self.shard, + "shard_count": self.shard_count, + "elapsed_seconds": self.elapsed_seconds, + "cases": [ + {"id": list(node), "outcome": outcome} + for node, outcome in sorted(self.cases.items()) + ], + } + + +def _environment() -> dict[str, str]: + return { + "python": platform.python_version(), + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "cpu_count": str(os.cpu_count()), + "runner_image": os.environ.get("ImageVersion", ""), # noqa: SIM112 - runner-images contract. + "dependency_lock": file_digest(Path(__file__).resolve().parents[1] / "uv.lock"), + } + + +def _identity( + source_sha: str, selection: str, candidate: object, environment: object +) -> dict[str, object]: + require_sha(source_sha) + if not isinstance(selection, str) or not selection.strip(): + raise ValueError("A nonempty test selection is required") + if ( + not isinstance(environment, dict) + or set(environment) + != { + "python", + "system", + "release", + "machine", + "cpu_count", + "runner_image", + "dependency_lock", + } + or not all(isinstance(value, str) for value in environment.values()) + or any(not value for key, value in environment.items() if key != "runner_image") + ): + raise ValueError("Explicit runtime/runner environment evidence is required") + if candidate is not None: + if not isinstance(candidate, dict) or type(candidate.get("schema_version")) is not int: + raise ValueError("Invalid candidate metadata schema") + if candidate["schema_version"] != 1 or candidate.get("sha") != source_sha: + raise ValueError("Candidate metadata does not identify the expected source") + binary_name = candidate.get("binary_name") + if not isinstance(binary_name, str) or candidate.get("archive") != archive_name( + binary_name + ): + raise ValueError("Candidate metadata has an invalid archive identity") + if not isinstance(candidate.get("version"), str) or not candidate["version"]: + raise ValueError("Candidate metadata has no version") + for field in ("archive_sha256", "executable_sha256"): + value = candidate.get(field) + if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise ValueError(f"Invalid candidate {field}") + return { + "source_sha": source_sha, + "selection": selection, + "candidate": candidate, + "environment": environment, + } + + +def _positive_seconds(value: object) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value <= 0 + ): + raise ValueError("Elapsed time must be finite and positive") + return float(value) + + +def _shard_identity(variant: str, shard: int, count: int) -> None: + if variant not in ("baseline", "proposed"): + raise ValueError("Variant must be baseline or proposed") + if type(shard) is not int or type(count) is not int or not 1 <= shard <= count <= 16: + raise ValueError("Require a complete bounded shard identity: 1 <= shard <= count <= 16") + + +def capture( + junit: Path, + source_sha: str, + selection: str, + variant: str, + shard: int, + shard_count: int, + candidate_metadata: Path | None = None, +) -> Report: + """Read one real pytest JUnit report without equating skips with passes.""" + _shard_identity(variant, shard, shard_count) + candidate = json.loads(candidate_metadata.read_text("utf-8")) if candidate_metadata else None + identity = _identity(source_sha, selection, candidate, _environment()) + parser = ET.XMLParser(target=_JUnitTreeBuilder()) # noqa: S314 - rejects all DTD declarations. + root = ET.parse(junit, parser=parser).getroot() # noqa: S314 - DTDs rejected before expansion. + suites = [root] if root.tag == "testsuite" else list(root) + if len(suites) != 1 or suites[0].tag != "testsuite": + raise ValueError("Expected the single-suite JUnit output of one pytest invocation") + suite = suites[0] + elapsed = _positive_seconds(float(suite.attrib["time"])) + cases = {} + for node in suite.findall("testcase"): + identities = node.findall("./properties/property[@name='apm_performance_nodeid']") + if len(identities) > 1: + raise ValueError("Duplicate original node identity in JUnit") + key = ( + ("pytest-nodeid", identities[0].attrib["value"]) + if identities + else (node.attrib.get("classname", ""), node.attrib["name"]) + ) + if not key[1] or key in cases: + raise ValueError(f"Empty or duplicate JUnit case identity: {key!r}") + failures = [child for child in node if child.tag in ("failure", "error", "skipped")] + if len(failures) > 1: + raise ValueError(f"Ambiguous JUnit outcome: {key!r}") + if not failures: + outcome = "passed" + elif failures[0].tag == "skipped": + outcome = "xfailed" if failures[0].get("type") == "pytest.xfail" else "skipped" + else: + outcome = "failed" if failures[0].tag == "failure" else "error" + cases[key] = outcome + counts = { + "tests": len(cases), + "failures": sum(value == "failed" for value in cases.values()), + "errors": sum(value == "error" for value in cases.values()), + "skipped": sum(value in ("skipped", "xfailed") for value in cases.values()), + } + if not cases or any(int(suite.attrib[field]) != count for field, count in counts.items()): + raise ValueError("JUnit summary counts do not match the nonempty case inventory") + return Report(identity, variant, shard, shard_count, elapsed, cases) + + +def read_report(path: Path) -> Report: + """Reject incomplete or malformed proof rather than silently dropping a shard.""" + data = json.loads(path.read_text("utf-8")) + if not isinstance(data, dict) or type(data.get("schema_version")) is not int: + raise ValueError(f"Invalid report schema: {path}") + if data["schema_version"] != 1 or not isinstance(data.get("identity"), dict): + raise ValueError(f"Invalid report identity: {path}") + raw_identity = data["identity"] + identity = _identity( + raw_identity["source_sha"], + raw_identity["selection"], + raw_identity["candidate"], + raw_identity["environment"], + ) + _shard_identity(data["variant"], data["shard"], data["shard_count"]) + elapsed = _positive_seconds(data["elapsed_seconds"]) + entries = data["cases"] + if not isinstance(entries, list) or not entries: + raise ValueError("Report must contain a nonempty case inventory") + cases = {} + for entry in entries: + if not isinstance(entry, dict): + raise ValueError("Invalid case evidence") + node, outcome = entry.get("id"), entry.get("outcome") + if ( + not isinstance(node, list) + or len(node) != 2 + or not all(isinstance(part, str) for part in node) + or not node[1] + or not isinstance(outcome, str) + or outcome not in OUTCOMES + ): + raise ValueError("Invalid case identity or outcome") + key = (node[0], node[1]) + if key in cases: + raise ValueError(f"Duplicate case evidence: {key!r}") + cases[key] = outcome + return Report(identity, data["variant"], data["shard"], data["shard_count"], elapsed, cases) + + +def _group(reports: list[Report], variant: str) -> dict[tuple[str, str], str]: + if not reports: + raise ValueError(f"No {variant} reports") + count = reports[0].shard_count + if len(reports) != count or {report.shard for report in reports} != set(range(1, count + 1)): + raise ValueError(f"Missing or duplicate {variant} shard") + cases = {} + for report in reports: + if report.variant != variant or report.shard_count != count: + raise ValueError(f"Inconsistent {variant} shard identity") + if set(cases) & report.cases.keys(): + raise ValueError(f"Overlapping {variant} test partitions") + if set(report.cases.values()) & {"failed", "error"}: + raise ValueError(f"{variant} contains failing tests") + cases.update(report.cases) + if "passed" not in cases.values(): + raise ValueError(f"{variant} did not execute any passing tests") + return cases + + +def compare( + baseline: list[Report], proposed: list[Report], expected_sha: str, minimum_speedup: float +) -> dict[str, object]: + """Prove identity, complete selection and outcomes before comparing execution time.""" + require_sha(expected_sha) + if not math.isfinite(minimum_speedup) or not 0 <= minimum_speedup < 1: + raise ValueError("Minimum speedup must be finite and in [0, 1)") + old_cases, new_cases = _group(baseline, "baseline"), _group(proposed, "proposed") + identity = baseline[0].identity + if identity["source_sha"] != expected_sha or any( + report.identity != identity for report in (*baseline, *proposed) + ): + raise ValueError( + "Source, candidate bytes, environment or test selection differ between runs" + ) + if old_cases != new_cases: + missing, extra = old_cases.keys() - new_cases.keys(), new_cases.keys() - old_cases.keys() + changed = sum(old_cases[node] != new_cases[node] for node in old_cases.keys() & new_cases) + raise ValueError( + f"Scenario parity failed: {len(missing)} missing, {len(extra)} extra, " + f"{changed} changed outcomes" + ) + before = max(report.elapsed_seconds for report in baseline) + after = max(report.elapsed_seconds for report in proposed) + speedup = 1 - after / before + return { + "schema_version": 1, + "identity": identity, + "scenario_count": len(old_cases), + "outcomes": { + outcome: list(old_cases.values()).count(outcome) for outcome in sorted(OUTCOMES) + }, + "same_scenarios_and_outcomes": True, + "baseline_pytest_critical_seconds": before, + "proposed_pytest_critical_seconds": after, + "baseline_pytest_runner_seconds": sum(report.elapsed_seconds for report in baseline), + "proposed_pytest_runner_seconds": sum(report.elapsed_seconds for report in proposed), + "speedup_fraction": speedup, + "minimum_speedup_fraction": minimum_speedup, + "meets_threshold": speedup >= minimum_speedup, + "scope": "pytest controller elapsed time; excludes runner setup and queue delays", + } + + +def main() -> int: + """Write durable JSON proof; fail visibly on mismatched evidence or a missed target.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + record = commands.add_parser("capture") + record.add_argument("--junit", type=Path, required=True) + record.add_argument("--source-sha", required=True) + record.add_argument("--selection", required=True) + record.add_argument("--variant", choices=("baseline", "proposed"), required=True) + record.add_argument("--shard", type=int, required=True) + record.add_argument("--shard-count", type=int, required=True) + record.add_argument("--candidate-metadata", type=Path) + check = commands.add_parser("compare") + check.add_argument("--baseline", type=Path, nargs="+", required=True) + check.add_argument("--proposed", type=Path, nargs="+", required=True) + check.add_argument("--expected-sha", required=True) + check.add_argument("--minimum-speedup", type=float, default=0.05) + for command in (record, check): + command.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.operation == "capture": + result = capture( + args.junit, + args.source_sha, + args.selection, + args.variant, + args.shard, + args.shard_count, + args.candidate_metadata, + ).payload() + else: + result = compare( + [read_report(path) for path in args.baseline], + [read_report(path) for path in args.proposed], + args.expected_sha, + args.minimum_speedup, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii" + ) + except (ValueError, KeyError, TypeError, OSError, ET.ParseError) as error: + parser.error(f"Invalid performance evidence: {error}") + if args.operation == "compare" and not result["meets_threshold"]: + parser.exit(1, f"Performance target missed; inspect {args.output}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pytest_performance_evidence.py b/scripts/pytest_performance_evidence.py new file mode 100644 index 0000000000..413aed1c81 --- /dev/null +++ b/scripts/pytest_performance_evidence.py @@ -0,0 +1,12 @@ +"""Preserve original pytest node IDs before xdist appends scheduling groups.""" + +from __future__ import annotations + +import pytest + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Record a stable identity without changing selection, grouping, or outcomes.""" + for item in items: + item.user_properties.append(("apm_performance_nodeid", item.nodeid)) diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index 5d936730a6..2bfba03844 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -15,7 +15,13 @@ const RERUN_ALL_JOBS_HINT = "Use Re-run all jobs to regenerate complete candidat const CONFIG_HASH_FILES = [ ".github/workflows/build-release.yml", ".github/workflows/release-platform.yml", + ".github/workflows/release-integration.yml", + ".github/workflows/docs-build.yml", + ".github/workflows/pypi-distributions.yml", ".github/workflows/ci.yml", + ".github/actions/pytest-timing/action.yml", + "scripts/test-integration.sh", + "scripts/github-token-helper.sh", "scripts/package_release.py", "scripts/release-candidate.cjs", "scripts/release-platforms.json", diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs index 78963b6916..bcd60b10ed 100644 --- a/scripts/release-candidate.test.cjs +++ b/scripts/release-candidate.test.cjs @@ -14,6 +14,21 @@ const SHA = "a".repeat(40); const OTHER_SHA = "b".repeat(40); const NOW = new Date().toISOString(); +it("fingerprints every local reusable workflow reachable from release qualification", () => { + const pending = [rc.WORKFLOW_PATH]; + const visited = new Set(); + while (pending.length) { + const file = pending.pop(); + if (visited.has(file)) continue; + visited.add(file); + assert.ok(rc.CONFIG_HASH_FILES.includes(file), `Missing workflow fingerprint: ${file}`); + const source = fs.readFileSync(path.join(ROOT, file), "utf8"); + for (const match of source.matchAll(/uses:\s*\.\/(\.github\/workflows\/[\w.-]+\.ya?ml)/g)) { + pending.push(match[1]); + } + } +}); + function catalog() { return JSON.parse(fs.readFileSync(path.join(__dirname, "release-platforms.json"), "ascii")); } diff --git a/scripts/release-platforms.json b/scripts/release-platforms.json index 02ad8ab9d8..d3aa91fae7 100644 --- a/scripts/release-platforms.json +++ b/scripts/release-platforms.json @@ -4,6 +4,9 @@ "platform": "linux", "binary_name": "apm-linux-x86_64", "integration_markers": "not live", + "integration_shard_count": 1, + "integration_xdist_workers": 4, + "integration_splitting_algorithm": "duration_based_chunks", "on_main": true }, { @@ -11,6 +14,9 @@ "platform": "linux", "binary_name": "apm-linux-arm64", "integration_markers": "not live", + "integration_shard_count": 1, + "integration_xdist_workers": 4, + "integration_splitting_algorithm": "duration_based_chunks", "on_main": true }, { @@ -18,6 +24,9 @@ "platform": "windows", "binary_name": "apm-windows-x86_64", "integration_markers": "not live", + "integration_shard_count": 1, + "integration_xdist_workers": 4, + "integration_splitting_algorithm": "duration_based_chunks", "on_main": true }, { @@ -25,6 +34,9 @@ "platform": "darwin", "binary_name": "apm-darwin-x86_64", "integration_markers": "lifecycle_smoke and not live", + "integration_shard_count": 1, + "integration_xdist_workers": 4, + "integration_splitting_algorithm": "duration_based_chunks", "on_main": true }, { @@ -32,6 +44,9 @@ "platform": "darwin", "binary_name": "apm-darwin-arm64", "integration_markers": "not live", + "integration_shard_count": 2, + "integration_xdist_workers": 2, + "integration_splitting_algorithm": "least_duration", "on_main": false } ] diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 5e7bf2f415..66562e5e6e 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -45,15 +45,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/github-token-helper.sh" log_info() { - echo -e "${BLUE}ℹ️ $1${NC}" + echo -e "${BLUE}[i] $1${NC}" } log_success() { - echo -e "${GREEN}✅ $1${NC}" + echo -e "${GREEN}[+] $1${NC}" } log_error() { - echo -e "${RED}❌ $1${NC}" + echo -e "${RED}[x] $1${NC}" } # Check prerequisites @@ -193,6 +193,7 @@ setup_binary_for_testing() { # The binary is located at ./dist/$BINARY_NAME/apm (directory structure) BINARY_PATH="./dist/$BINARY_NAME/apm" + export APM_BINARY_PATH="$(pwd)/dist/$BINARY_NAME/apm" # Make binary executable (like CI does) chmod +x "$BINARY_PATH" @@ -333,12 +334,16 @@ run_e2e_tests() { # bare "${arr[@]}" on an empty array raises an unbound-variable error. # shellcheck disable=SC2206 extra_args=(${PYTEST_EXTRA_ARGS:-}) + pytest_command=(pytest) + if [[ " ${extra_args[*]} " == *" scripts.pytest_performance_evidence "* ]]; then + pytest_command=(python -m pytest) + fi marker_args=() if [[ -n "${PYTEST_MARK_EXPR:-}" ]]; then marker_args=(-m "$PYTEST_MARK_EXPR") log_info "Pytest marker selection: $PYTEST_MARK_EXPR" fi - if pytest tests/integration/ -v --tb=short \ + if "${pytest_command[@]}" tests/integration/ -v --tb=short \ --strict-runtime-prerequisites \ ${extra_args[@]+"${extra_args[@]}"} \ ${marker_args[@]+"${marker_args[@]}"}; then diff --git a/tests/integration/test_architecture_owner_rule_mutations.py b/tests/integration/test_architecture_owner_rule_mutations.py index 79e0372803..aee6dcdf24 100644 --- a/tests/integration/test_architecture_owner_rule_mutations.py +++ b/tests/integration/test_architecture_owner_rule_mutations.py @@ -181,6 +181,15 @@ class MutationCase: new="actual_hex = _local_policy_hash(raw_bytes, algo)", intent="Policy verification bypasses the canonical SHA-2 digest owner.", ), + MutationCase( + guard_id="contracts-tooling-policy-identity", + rule_id="contracts-tooling-policy-identity", + path="src/apm_cli/policy/matcher.py", + old=" normalized = normalize_package_policy_identity(\n", + new=" normalized = normalize_package_repo_url(\n", + intent="Dependency policy matching skips the source-owned identity normalizer.", + replace_all=True, + ), MutationCase( guard_id="contracts-tooling-project-yaml-write-delegation", rule_id="contracts-tooling-project-yaml-write-delegation", @@ -189,6 +198,14 @@ class MutationCase: new=" write_text_lf(\n", intent="The atomic project YAML writer bypasses the canonical atomic writer.", ), + MutationCase( + guard_id="contracts-tooling-python-artifact-membership", + rule_id="contracts-tooling-python-artifact-membership", + path="src/apm_cli/install/deployed_paths.py", + old="is_generated_python_artifact(relative)", + new="False", + intent="Deployed-file inventory stops routing bytecode membership through security/gate.py.", + ), MutationCase( guard_id="contracts-tooling-root-context-write-eligibility", rule_id="contracts-tooling-root-context-write-eligibility", @@ -1074,6 +1091,7 @@ def baseline_violated_rule_ids() -> frozenset[str]: return frozenset(violation.rule_id for violation in report.violations) +@pytest.mark.lifecycle_smoke def test_matrix_covers_every_registry_guard_exactly_once() -> None: """The matrix guard set must equal the registry guard set, one case each.""" matrix_guard_ids = [case.guard_id for case in MUTATIONS] diff --git a/tests/integration/test_mcp_env_var_copilot_e2e.py b/tests/integration/test_mcp_env_var_copilot_e2e.py index 0e475c38ef..538a50cf61 100644 --- a/tests/integration/test_mcp_env_var_copilot_e2e.py +++ b/tests/integration/test_mcp_env_var_copilot_e2e.py @@ -25,7 +25,6 @@ pytestmark = [ pytest.mark.requires_apm_binary, - pytest.mark.requires_runtime_copilot, # Mutates os.environ["HOME"]; must be serialized on a single xdist worker. # Requires --dist loadgroup in the xdist invocation (the only # scheduler that honors xdist_group); without it the marker is diff --git a/tests/quality/test_ci_topology.py b/tests/quality/test_ci_topology.py index 6df00770fd..570618d9e8 100644 --- a/tests/quality/test_ci_topology.py +++ b/tests/quality/test_ci_topology.py @@ -76,8 +76,9 @@ f"{LIFECYCLE_SMOKE_FULL_EXPRESSION} and not {LIFECYCLE_SMOKE_MERGE_GROUP_MARKER}" ) MERGE_GROUP_INTEGRATION_WORKFLOW = REPO_ROOT / ".github/workflows/ci-integration.yml" +SHARED_INTEGRATION_WORKFLOW = REPO_ROOT / ".github/workflows/release-integration.yml" MERGE_GROUP_INTEGRATION_JOB = "integration-tests-shard" -MERGE_GROUP_INTEGRATION_STEP = "Run integration tests (sharded + parallelized)" +MERGE_GROUP_INTEGRATION_STEP = "Run integration tests (Unix)" MERGE_GROUP_INTEGRATION_SCRIPT = REPO_ROOT / "scripts/test-integration.sh" PYPROJECT = REPO_ROOT / "pyproject.toml" FORBIDDEN_CREDENTIAL_ENV = ("GITHUB_APM_PAT", "ADO_APM_PAT", "GITHUB_TOKEN") @@ -715,11 +716,48 @@ def _assert_merge_group_full_lifecycle_path() -> None: """Pin the merge-group workflow path that executes the full integration root.""" workflow = load_workflow(MERGE_GROUP_INTEGRATION_WORKFLOW) job = workflow_job(workflow, MERGE_GROUP_INTEGRATION_JOB) - step = workflow_step(job, MERGE_GROUP_INTEGRATION_STEP) + assert "if" not in job + assert job["uses"] == "./.github/workflows/release-integration.yml" + assert job["with"]["platform"] == "linux" + assert job["with"]["integration-markers"] == "not live" + shared = load_workflow(SHARED_INTEGRATION_WORKFLOW) + executor = workflow_job(shared, "integration-tests") + assert "if" not in executor + step = workflow_step(executor, MERGE_GROUP_INTEGRATION_STEP) + assert step["if"] == "inputs.platform != 'windows'" assert step["env"][LIFECYCLE_SMOKE_E2E_ENV] == LIFECYCLE_SMOKE_E2E_VALUE - assert "uv run ./scripts/test-integration.sh" in step["run"] + assert step["env"]["PYTEST_MARK_EXPR"] == "${{ inputs.integration-markers }}" + assert ["uv", "run", "--frozen", "bash", "scripts/test-integration.sh"] in shell_commands(step) script = MERGE_GROUP_INTEGRATION_SCRIPT.read_text(encoding="utf-8") - assert "pytest tests/integration/ -v" in script + assert "pytest_command=(pytest)" in script + assert re.search( + r'^\s*if "\$\{pytest_command\[@\]\}" tests/integration/ -v\b', script, re.MULTILINE + ) + + +@pytest.mark.parametrize( + "fault", ["disabled-caller", "focused-marker", "disabled-runner", "no-e2e"] +) +def test_merge_group_full_lifecycle_delegation_mutations_fail( + monkeypatch: pytest.MonkeyPatch, fault: str +) -> None: + workflows = { + path: load_workflow(path) + for path in (MERGE_GROUP_INTEGRATION_WORKFLOW, SHARED_INTEGRATION_WORKFLOW) + } + caller = workflow_job(workflows[MERGE_GROUP_INTEGRATION_WORKFLOW], MERGE_GROUP_INTEGRATION_JOB) + executor = workflow_job(workflows[SHARED_INTEGRATION_WORKFLOW], "integration-tests") + if fault == "disabled-caller": + caller["if"] = False + elif fault == "focused-marker": + caller["with"]["integration-markers"] = "lifecycle_smoke" + elif fault == "disabled-runner": + executor["if"] = False + else: + workflow_step(executor, MERGE_GROUP_INTEGRATION_STEP)["env"].pop(LIFECYCLE_SMOKE_E2E_ENV) + monkeypatch.setattr(sys.modules[__name__], "load_workflow", workflows.__getitem__) + with pytest.raises((AssertionError, KeyError)): + _assert_merge_group_full_lifecycle_path() def test_lifecycle_smoke_is_required_and_hermetic() -> None: diff --git a/tests/unit/scripts/test_architecture_runner.py b/tests/unit/scripts/test_architecture_runner.py index 59f7b4ed61..af804e4a8c 100644 --- a/tests/unit/scripts/test_architecture_runner.py +++ b/tests/unit/scripts/test_architecture_runner.py @@ -36,6 +36,7 @@ import pytest from scripts.architecture_linter import runner +from scripts.architecture_linter.checks.lexical_shared import duplicate_definition_lines from scripts.architecture_linter.diagnostics import ( LEGACY_AC_ALIASES, DiagnosticCollector, @@ -117,6 +118,75 @@ def test_source_cache_allows_only_safe_test_scoped_overrides(tmp_path: Path) -> assert "unsafe repository-relative path" in (error or "") +def test_duplicate_definition_lines_uses_lexical_reads_not_python_parse(tmp_path: Path) -> None: + """Duplicate scans are grep-equivalent and must not parse every candidate.""" + owner = "src/apm_cli/owner.py" + duplicate = "src/apm_cli/duplicate.py" + (tmp_path / "src/apm_cli").mkdir(parents=True) + (tmp_path / owner).write_text("def clone():\n return 1\n", encoding="utf-8") + # Syntactically invalid, but the duplicate line is still lexically visible. + (tmp_path / duplicate).write_text("def clone(\n", encoding="utf-8") + provider = FactsProvider(tmp_path, (owner, duplicate), registry=None) + + findings = duplicate_definition_lines( + provider, + rule_id="rule", + prefix="src/apm_cli/", + pattern=re.compile(r"^def clone\("), + owner=owner, + message="duplicate clone", + respect_exempt=False, + ) + + assert [(item.path, item.line, item.column, item.message) for item in findings] == [ + (duplicate, 1, 1, "duplicate clone") + ] + assert provider.source_cache.read_attempts == 1 + assert provider.parse_cache.parse_attempts == 0 + assert provider.tree_index_builds == 0 + + +def test_duplicate_definition_lines_respects_exemptions_overrides_and_read_errors( + tmp_path: Path, +) -> None: + """Lexical duplicate scans keep their exemption, override, and error behavior.""" + owner = "src/apm_cli/owner.py" + exempt = "src/apm_cli/exempt.py" + override = "src/apm_cli/override.py" + unreadable = "src/apm_cli/unreadable.py" + (tmp_path / "src/apm_cli").mkdir(parents=True) + (tmp_path / owner).write_text("def clone():\n return 1\n", encoding="utf-8") + (tmp_path / exempt).write_text( + "def clone(): # architecture-authority-exempt: covered elsewhere\n", + encoding="utf-8", + ) + (tmp_path / override).write_text("def not_a_clone():\n return 1\n", encoding="utf-8") + provider = FactsProvider( + tmp_path, + (owner, exempt, override, unreadable), + registry=None, + source_overrides={ + override: "def clone():\n return 2\n", + unreadable: b"\xff\xfe", + }, + ) + + findings = duplicate_definition_lines( + provider, + rule_id="rule", + prefix="src/apm_cli/", + pattern=re.compile(r"^def clone\("), + owner=owner, + message="duplicate clone", + respect_exempt=True, + ) + + assert [(item.path, item.line, item.column) for item in findings] == [(override, 1, 1)] + assert provider.source_cache.read_attempts == 3 + assert provider.source_cache.read_errors == 1 + assert provider.parse_cache.parse_attempts == 0 + + def test_parse_cache_parses_each_source_once_including_syntax_errors() -> None: """A valid source and a syntactically invalid source each parse once.""" cache = ParseCache() diff --git a/tests/unit/scripts/test_compare_test_runs.py b/tests/unit/scripts/test_compare_test_runs.py new file mode 100644 index 0000000000..ab1c116675 --- /dev/null +++ b/tests/unit/scripts/test_compare_test_runs.py @@ -0,0 +1,292 @@ +"""Performance claims require identical, complete, successful test evidence.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from scripts.compare_test_runs import Report, capture, compare, read_report + +pytestmark = pytest.mark.component +SHA = "a" * 40 +IDENTITY = { + "source_sha": SHA, + "selection": "not live", + "candidate": None, + "environment": { + "python": "3.12.12", + "system": "Darwin", + "release": "24.6.0", + "machine": "arm64", + "cpu_count": "3", + "runner_image": "20260901", + "dependency_lock": "e" * 64, + }, +} +CASES = {("suite", "test_a"): "passed", ("suite", "test_b"): "skipped"} + + +def _baseline() -> Report: + return Report(IDENTITY, "baseline", 1, 1, 10.0, CASES) + + +def _proposed() -> list[Report]: + return [ + Report(IDENTITY, "proposed", 1, 2, 5.0, {("suite", "test_a"): "passed"}), + Report(IDENTITY, "proposed", 2, 2, 6.0, {("suite", "test_b"): "skipped"}), + ] + + +def test_complete_disjoint_shards_preserve_skips_and_report_wall_separately_from_cost() -> None: + result = compare([_baseline()], _proposed(), SHA, 0.05) + assert result["scenario_count"] == 2 + assert result["outcomes"] == {"passed": 1, "skipped": 1, "xfailed": 0, "failed": 0, "error": 0} + assert result["baseline_pytest_critical_seconds"] == 10.0 + assert result["proposed_pytest_critical_seconds"] == 6.0 + assert result["proposed_pytest_runner_seconds"] == 11.0 + assert result["speedup_fraction"] == pytest.approx(0.4) + assert result["meets_threshold"] is True + + +@pytest.mark.parametrize( + "fault", ["missing", "duplicate", "overlap", "identity", "selection", "sha", "environment"] +) +def test_rejects_missing_overlap_and_identity_drift(fault: str) -> None: + proposed = _proposed() + expected = SHA + if fault == "missing": + proposed.pop() + elif fault == "duplicate": + proposed[1] = proposed[0] + elif fault == "overlap": + proposed[1] = replace(proposed[1], cases=proposed[0].cases) + elif fault == "identity": + proposed[1] = replace( + proposed[1], identity={**IDENTITY, "candidate": {"digest": "different"}} + ) + elif fault == "selection": + proposed[1] = replace(proposed[1], identity={**IDENTITY, "selection": "smoke"}) + elif fault == "environment": + proposed[1] = replace( + proposed[1], + identity={ + **IDENTITY, + "environment": {**IDENTITY["environment"], "runner_image": "new"}, + }, + ) + else: + expected = "b" * 40 + with pytest.raises(ValueError): + compare([_baseline()], proposed, expected, 0.05) + + +@pytest.mark.parametrize("outcome", ["skipped", "xfailed", "failed", "error"]) +def test_cannot_gain_speed_by_skipping_or_failing_a_previously_passing_case(outcome: str) -> None: + proposed = _proposed() + proposed[0] = replace(proposed[0], cases={("suite", "test_a"): outcome}) + with pytest.raises(ValueError): + compare([_baseline()], proposed, SHA, 0.05) + + +@pytest.mark.parametrize("cases", [{("suite", "test_extra"): "passed"}, {}]) +def test_rejects_extra_or_dropped_scenarios(cases: dict[tuple[str, str], str]) -> None: + proposed = _proposed() + proposed[0] = replace(proposed[0], cases=cases) + with pytest.raises(ValueError): + compare([_baseline()], proposed, SHA, 0.05) + + +def test_missed_speed_target_remains_explicit_in_durable_result() -> None: + proposed = [replace(_baseline(), variant="proposed", elapsed_seconds=10.1)] + result = compare([_baseline()], proposed, SHA, 0.05) + assert result["same_scenarios_and_outcomes"] is True + assert result["meets_threshold"] is False + assert result["speedup_fraction"] < 0 + + +@pytest.mark.parametrize("seconds", [float("nan"), float("inf"), 0, -1, True]) +def test_rejects_unusable_timings(tmp_path: Path, seconds: float) -> None: + payload = _baseline().payload() + payload["elapsed_seconds"] = seconds + path = tmp_path / "proof.json" + path.write_text(json.dumps(payload), encoding="ascii") + with pytest.raises(ValueError, match="finite and positive"): + read_report(path) + + +@pytest.mark.parametrize( + "fault", ["duplicate", "unknown-outcome", "empty", "missing-field", "bool-shard"] +) +def test_rejects_malformed_inventory(tmp_path: Path, fault: str) -> None: + payload = _baseline().payload() + if fault == "duplicate": + payload["cases"] = [{"id": ["suite", "a"], "outcome": "passed"}] * 2 + elif fault == "unknown-outcome": + payload["cases"] = [{"id": ["suite", "a"], "outcome": "unknown"}] + elif fault == "empty": + payload["cases"] = [] + elif fault == "missing-field": + del payload["identity"] + else: + payload["shard"] = True + path = tmp_path / "proof.json" + path.write_text(json.dumps(payload), encoding="ascii") + with pytest.raises(ValueError): + read_report(path) + + +def test_actual_junit_capture_roundtrip_keeps_xfail_identity(tmp_path: Path) -> None: + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '' + '' + "", + encoding="ascii", + ) + report = capture(junit, SHA, "not live", "baseline", 1, 1) + assert report.elapsed_seconds == 7.5 + assert report.cases == {("suite", "test_a"): "passed", ("suite", "test_b"): "xfailed"} + path = tmp_path / "proof.json" + path.write_text(json.dumps(report.payload()), encoding="ascii") + assert read_report(path) == report + + +@pytest.mark.parametrize("fault", ["none", "source", "digest", "binary", "version", "schema"]) +def test_candidate_evidence_is_bound_to_the_source_and_archive(tmp_path: Path, fault: str) -> None: + metadata = { + "schema_version": 1, + "sha": SHA, + "version": "0.30.0", + "binary_name": "apm-darwin-arm64", + "archive": "apm-darwin-arm64.tar.gz", + "archive_sha256": "c" * 64, + "executable_sha256": "d" * 64, + } + if fault == "source": + metadata["sha"] = "b" * 40 + elif fault == "digest": + metadata["archive_sha256"] = "not-a-digest" + elif fault == "binary": + metadata["binary_name"] = "apm-windows-x86_64" + elif fault == "version": + metadata["version"] = "" + elif fault == "schema": + metadata["schema_version"] = True + path = tmp_path / "candidate.json" + path.write_text(json.dumps(metadata), encoding="ascii") + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '', + encoding="ascii", + ) + if fault != "none": + with pytest.raises(ValueError): + capture(junit, SHA, "not live", "baseline", 1, 1, path) + else: + report = capture(junit, SHA, "not live", "baseline", 1, 1, path) + assert report.identity["candidate"] == metadata + + +@pytest.mark.parametrize( + "xml", + [ + '', + '' + '', + '' + '', + "", + ], +) +def test_rejects_incomplete_or_ambiguous_junit(tmp_path: Path, xml: str) -> None: + path = tmp_path / "junit.xml" + path.write_text(xml, encoding="ascii") + with pytest.raises(ValueError): + capture(path, SHA, "not live", "baseline", 1, 1) + + +@pytest.mark.parametrize( + "declaration", + [ + ']>', + '', + ], +) +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +def test_junit_rejects_dtds_before_expanding_entities( + tmp_path: Path, declaration: str, encoding: str +) -> None: + path = tmp_path / "junit.xml" + path.write_text( + f'{declaration}' + '' + '', + encoding=encoding, + ) + with pytest.raises(ValueError, match="document type declarations are forbidden"): + capture(path, SHA, "not live", "baseline", 1, 1) + + +def test_real_pytest_xdist_reports_preserve_original_node_ids_and_outcomes(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[3] + (tmp_path / "pytest.ini").write_text( + "[pytest]\nmarkers = xdist_group: scheduling affinity\n", encoding="ascii" + ) + (tmp_path / "test_example.py").write_text( + "import pytest\n" + "@pytest.mark.xdist_group('home')\n" + "@pytest.mark.parametrize('value', ['literal@home', 'other', 'xfail', 'skip'])\n" + "def test_case(value):\n" + " if value == 'skip': pytest.skip('known prerequisite')\n" + " if value == 'xfail': pytest.xfail('known failure')\n" + " assert value in ['literal@home', 'other']\n", + encoding="ascii", + ) + reports = [] + for variant, shard in (("baseline", 1), ("proposed", 1), ("proposed", 2)): + xml = tmp_path / f"{variant}-{shard}.xml" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "scripts.pytest_performance_evidence", + "--dist", + "loadgroup", + "-n", + "0" if variant == "baseline" else "2", + f"--junitxml={xml}", + "test_example.py", + ] + if variant == "proposed": + command.extend(["--splits", "2", "--group", str(shard)]) + result = subprocess.run( + command, + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(root)}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + reports.append( + capture(xml, SHA, "example", variant, shard, 1 if variant == "baseline" else 2) + ) + # Ignore overhead-dominated toy timings; this case proves actual plugin interoperability. + baseline = replace(reports[0], elapsed_seconds=10) + proposed = [replace(report, elapsed_seconds=5) for report in reports[1:]] + proof = compare([baseline], proposed, SHA, 0.05) + assert proof["scenario_count"] == 4 + assert proof["outcomes"]["passed"] == 2 + assert ("pytest-nodeid", "test_example.py::test_case[literal@home]") in baseline.cases diff --git a/tests/unit/test_ado_live_workflow_contract.py b/tests/unit/test_ado_live_workflow_contract.py index ff54f404e9..c8be8ff9df 100644 --- a/tests/unit/test_ado_live_workflow_contract.py +++ b/tests/unit/test_ado_live_workflow_contract.py @@ -12,7 +12,6 @@ from tests.integration import test_ado_e2e from tests.workflow_contracts import ( assert_exact_command, - effective_env, load_workflow, shell_commands, shell_tokens, @@ -25,7 +24,7 @@ AUTH_WORKFLOW = ROOT / ".github" / "workflows" / "auth-acceptance.yml" INTEGRATION_SCRIPT = ROOT / "scripts" / "test-integration.sh" LIVE_ADO_SELECTOR = "live and requires_ado_pat" -RELEASE_INTEGRATION_STEPS = (("integration-tests", "Run integration tests (Unix)"),) +RELEASE_INTEGRATION_JOBS = ("integration-tests-shard",) def _walk_nodes(value: Any) -> list[dict[str, Any]]: @@ -48,17 +47,18 @@ def _assert_release_excludes_live_ado(workflow: dict[str, Any]) -> None: if isinstance(env, dict): assert "ADO_APM_PAT" not in env - for job_id, step_name in RELEASE_INTEGRATION_STEPS: + for job_id in RELEASE_INTEGRATION_JOBS: job = workflow_job(workflow, job_id) - step = workflow_step(job, step_name) - marker_expression = effective_env(workflow, job, step).get("PYTEST_MARK_EXPR") + marker_expression = job["with"].get("integration-markers") assert isinstance(marker_expression, str) assert marker_expression == "${{ inputs.integration-markers }}" + assert job["secrets"] == {"GH_CLI_PAT": "${{ secrets.GH_CLI_PAT }}"} platforms = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) assert all("not live" in row["integration_markers"] for row in platforms) + reusable = load_workflow(ROOT / ".github/workflows/release-integration.yml") windows_step = workflow_step( - workflow_job(workflow, "integration-tests"), + workflow_job(reusable, "integration-tests"), "Run integration tests (Windows)", ) assert "-IncludeLiveADO" not in shell_tokens(windows_step) @@ -133,6 +133,21 @@ def test_integration_script_owns_explicit_marker_selection() -> None: ) +def test_integration_script_exports_explicit_candidate_binary_path() -> None: + """The integration harness must not rely on PATH-only candidate discovery.""" + script = INTEGRATION_SCRIPT.read_text(encoding="utf-8") + assert 'export APM_BINARY_PATH="$(pwd)/dist/$BINARY_NAME/apm"' in script + + +def test_integration_script_uses_module_pytest_for_performance_plugin() -> None: + """The proof plugin must load through repo-root module imports, not console pytest.""" + script = INTEGRATION_SCRIPT.read_text(encoding="utf-8") + assert "pytest_command=(pytest)" in script + assert 'scripts.pytest_performance_evidence "*' in script + assert "pytest_command=(python -m pytest)" in script + assert 'if "${pytest_command[@]}" tests/integration/ -v --tb=short \\' in script + + def test_auth_acceptance_explicitly_selects_live_ado_nodes() -> None: """The opt-in auth workflow remains the live PAT acceptance owner.""" _assert_auth_acceptance_selects_live_ado(load_workflow(AUTH_WORKFLOW)) @@ -141,11 +156,7 @@ def test_auth_acceptance_explicitly_selects_live_ado_nodes() -> None: def test_release_ado_secret_mutation_is_rejected() -> None: """Adding the expiring ADO PAT back to publication must fail the contract.""" workflow = deepcopy(load_workflow(RELEASE_WORKFLOW)) - step = workflow_step( - workflow_job(workflow, "integration-tests"), - "Run integration tests (Unix)", - ) - step["env"]["ADO_APM_PAT"] = "secret" + workflow_job(workflow, "integration-tests-shard")["secrets"]["ADO_APM_PAT"] = "secret" with pytest.raises(AssertionError): _assert_release_excludes_live_ado(workflow) @@ -154,11 +165,7 @@ def test_release_ado_secret_mutation_is_rejected() -> None: def test_release_live_selector_mutation_is_rejected() -> None: """Publication cannot silently regain live external-service nodes.""" workflow = deepcopy(load_workflow(RELEASE_WORKFLOW)) - step = workflow_step( - workflow_job(workflow, "integration-tests"), - "Run integration tests (Unix)", - ) - step["env"]["PYTEST_MARK_EXPR"] = "live" + workflow_job(workflow, "integration-tests-shard")["with"]["integration-markers"] = "live" with pytest.raises(AssertionError): _assert_release_excludes_live_ado(workflow) diff --git a/tests/unit/test_docs_contract_workflow.py b/tests/unit/test_docs_contract_workflow.py index cf45ebbbfc..7dd4122715 100644 --- a/tests/unit/test_docs_contract_workflow.py +++ b/tests/unit/test_docs_contract_workflow.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github" / "workflows" / "docs.yml" +BUILD_WORKFLOW = ROOT / ".github" / "workflows" / "docs-build.yml" UNIT_CONTRACT = "tests/unit/test_cli_docs_contract.py" SUBPROCESS_CONTRACT = "tests/integration/test_cli_docs_contract.py" @@ -25,6 +26,10 @@ def _workflow() -> dict: return load_workflow(WORKFLOW) +def _build_workflow() -> dict: + return load_workflow(BUILD_WORKFLOW) + + def _assert_checker_test_step(workflow: dict) -> None: build = workflow_job(workflow, "build") step = workflow_step(build, "Test CLI registry contract") @@ -46,6 +51,7 @@ def test_docs_contract_runs_for_every_cli_registration_owner() -> None: "src/apm_cli/cli.py", "src/apm_cli/commands/**", "scripts/check_cli_docs.py", + ".github/workflows/docs-build.yml", UNIT_CONTRACT, SUBPROCESS_CONTRACT, } @@ -54,11 +60,15 @@ def test_docs_contract_runs_for_every_cli_registration_owner() -> None: def test_docs_build_has_read_only_permissions() -> None: """Pages write and OIDC minting belong only to the deploy job.""" workflow = _workflow() + build_workflow = _build_workflow() build = workflow_job(workflow, "build") deploy = workflow_job(workflow, "deploy") assert workflow["permissions"] == {"contents": "read"} - assert "permissions" not in build + assert build["uses"] == "./.github/workflows/docs-build.yml" + assert build["permissions"] == {"contents": "read"} + assert build_workflow["permissions"] == {"contents": "read"} + assert workflow_job(build_workflow, "build")["permissions"] == {"contents": "read"} assert deploy["permissions"] == { "pages": "write", "id-token": "write", @@ -67,7 +77,7 @@ def test_docs_build_has_read_only_permissions() -> None: def test_docs_python_operations_are_frozen() -> None: """The docs owner must consume the reviewed lock without mutation.""" - workflow = _workflow() + workflow = _build_workflow() build = workflow_job(workflow, "build") install = workflow_step(build, "Install Python test dependencies") checker = workflow_step(build, "Check CLI registry against rendered pages") @@ -87,7 +97,7 @@ def test_docs_python_operations_are_frozen() -> None: @pytest.mark.parametrize("mutation", ("step-disabled", "unit-omitted", "subprocess-commented")) def test_docs_checker_step_mutations_are_rejected(mutation: str) -> None: """Disabled, omitted, or commented contract execution must fail.""" - workflow = deepcopy(_workflow()) + workflow = deepcopy(_build_workflow()) step = workflow_step(workflow_job(workflow, "build"), "Test CLI registry contract") if mutation == "step-disabled": step["if"] = False diff --git a/tests/unit/test_mcp_runtime_prerequisite.py b/tests/unit/test_mcp_runtime_prerequisite.py new file mode 100644 index 0000000000..ba3f05c2b2 --- /dev/null +++ b/tests/unit/test_mcp_runtime_prerequisite.py @@ -0,0 +1,95 @@ +"""MCP config-only E2E coverage must not require runtime binaries.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.integration import test_mcp_env_var_copilot_e2e as mcp_env_e2e + +pytestmark = pytest.mark.component + +REPO_ROOT = Path(__file__).resolve().parents[2] +MCP_ENV_E2E = "tests/integration/test_mcp_env_var_copilot_e2e.py" +EXPECTED_CASES = { + f"{MCP_ENV_E2E}::TestMcpEnvVarHeadersCopilot::" + "test_self_defined_http_server_translates_env_vars_not_resolves", + f"{MCP_ENV_E2E}::TestMcpEnvVarHeadersCopilot::" + "test_self_defined_stdio_server_translates_env_vars_in_args", + f"{MCP_ENV_E2E}::TestMcpEnvVarHeadersCursor::test_cursor_still_resolves_env_vars_to_literal", +} + + +def _module_pytest_marks() -> list[pytest.Mark]: + """Return normalized module-level pytest marks for the config-only E2E module.""" + raw_marks = mcp_env_e2e.pytestmark + if not isinstance(raw_marks, list): + raw_marks = [raw_marks] + return raw_marks + + +def test_config_only_mcp_e2e_keeps_binary_gate_without_copilot_runtime() -> None: + """The E2E still needs candidate apm, but not the Copilot CLI binary.""" + marks = _module_pytest_marks() + names = [mark.name for mark in marks] + + assert "requires_apm_binary" in names + assert "requires_runtime_copilot" not in names + assert any( + mark.name == "xdist_group" and mark.kwargs.get("name") == "home_env" for mark in marks + ) + + +def test_config_only_mcp_e2e_collects_without_copilot_under_strict_prerequisites( + tmp_path: Path, +) -> None: + """Strict runtime prereqs must not deselect config-only Copilot rendering tests.""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_apm = fake_bin / "apm" + fake_apm.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake_apm.chmod(0o755) + fake_home = tmp_path / "home" + fake_home.mkdir() + + env = os.environ.copy() + env["APM_BINARY_PATH"] = str(fake_apm) + env["HOME"] = str(fake_home) + env["PATH"] = str(fake_bin) + git_executable = shutil.which("git") + if git_executable is not None: + env["GIT_PYTHON_GIT_EXECUTABLE"] = git_executable + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "no:cacheprovider", + "--collect-only", + "--strict-runtime-prerequisites", + "-q", + MCP_ENV_E2E, + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + output = result.stdout + result.stderr + collected = { + line.strip() for line in result.stdout.splitlines() if line.startswith(f"{MCP_ENV_E2E}::") + } + + assert result.returncode == 0, output + assert collected == EXPECTED_CASES + assert "required runtime missing" not in output + assert not (fake_home / ".apm" / "runtimes" / "copilot").exists() diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index ce910c67e8..d5bfcf2c63 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -27,10 +27,21 @@ ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github/workflows/release-platform.yml" +INTEGRATION_WORKFLOW = ROOT / ".github/workflows/release-integration.yml" RELEASE_WORKFLOW = ROOT / ".github/workflows/build-release.yml" UNIX_TIMEOUT = "${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }}" UNIX_ARGS = ( - "-n 4 --dist loadgroup --durations=50 --store-durations --junitxml=test-results/integration.xml" + "${{ inputs.evidence-variant != '' && '-p scripts.pytest_performance_evidence' || '' }} " + "--splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} " + "--splitting-algorithm ${{ inputs.splitting-algorithm }} " + "-n ${{ inputs.xdist-workers }} --dist loadgroup " + "--durations=50 --store-durations " + "--junitxml=test-results/integration-shard-${{ inputs.shard-index }}.xml " + "${{ inputs.coverage-enabled && '--cov --cov-fail-under=0' || '' }}" +) +SHARD_MATRIX = ( + "${{ fromJSON(inputs.integration-shard-count == 2 && '{\"shard\":[1,2]}' " + "|| '{\"shard\":[1]}') }}" ) SMOKE = "Test native binary startup and core contracts" INSTALLER = "Test install.ps1 end-to-end (Windows)" @@ -40,6 +51,10 @@ def _workflow() -> dict: return load_workflow(WORKFLOW) +def _integration_workflow() -> dict: + return load_workflow(INTEGRATION_WORKFLOW) + + def test_candidate_source_authorities_match_the_real_reusable_workflow() -> None: """Reject invented job names and omitted authorities in promotion evidence.""" source = load_workflow(ROOT / ".github/workflows/ci.yml") @@ -96,6 +111,29 @@ def _execute_native_gate(results: dict, full: bool, platform: str) -> list[str]: return json.loads(completed.stdout) +def _execute_integration_gate(results: dict) -> list[str]: + gate = workflow_job(_workflow(), "integration-tests") + script = workflow_step(gate, "Require every applicable integration shard")["with"]["script"] + harness = ( + "const failures = [];" + "const core = {setFailed: message => failures.push(message)};" + f"(new Function('core', {json.dumps(script)}))(core);" + "process.stdout.write(JSON.stringify(failures));" + ) + completed = subprocess.run( + ["node", "-e", harness], + check=True, + capture_output=True, + text=True, + timeout=15, + env={ + **os.environ, + "RESULTS": json.dumps(results), + }, + ) + return json.loads(completed.stdout) + + @pytest.mark.parametrize( "job_name", ["unit-tests", "build", "integration-tests", "release-validation", "windows-installer"], @@ -126,6 +164,16 @@ def test_native_gate_accepts_only_applicable_successes(full: bool, platform: str assert _execute_native_gate(results, full, platform) == [] +@pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "neutral", None]) +def test_native_integration_fan_in_requires_shard_matrix_success(result: str | None) -> None: + results = {"integration-tests-shard": {"result": "success"}} + if result is None: + del results["integration-tests-shard"] + else: + results["integration-tests-shard"]["result"] = result + assert _execute_integration_gate(results) == ["integration-tests-shard did not succeed"] + + def _assert_native_startup(workflow: dict) -> None: job = workflow_job(workflow, "build") assert_unconditional(job, label="native candidate build") @@ -236,7 +284,11 @@ def test_installer_workflow_inputs_reach_the_actual_candidate_reader(tmp_path: P def _assert_parallel_paths(workflow: dict) -> None: for name in ("unit-tests", "build"): assert "needs" not in workflow_job(workflow, name) - for name in ("integration-tests", "release-validation", "windows-installer"): + shard = workflow_job(workflow, "integration-tests-shard") + assert shard["needs"] == ["build"] + assert shard["strategy"]["fail-fast"] is False + assert workflow_job(workflow, "integration-tests")["needs"] == ["integration-tests-shard"] + for name in ("release-validation", "windows-installer"): assert workflow_job(workflow, name)["needs"] == ["build"] gate = workflow_job(workflow, "gate") assert gate["if"] == "always()" @@ -253,23 +305,95 @@ def _assert_parallel_paths(workflow: dict) -> None: assert "core.setFailed" in step["with"]["script"] -def _assert_integration(workflow: dict) -> None: - job = workflow_job(workflow, "integration-tests") - unix = workflow_step(job, "Run integration tests (Unix)") +def _assert_integration(workflow: dict, integration: dict) -> None: + job = workflow_job(workflow, "integration-tests-shard") + assert job["uses"] == "./.github/workflows/release-integration.yml" + assert job["strategy"]["fail-fast"] is False + assert job["strategy"]["matrix"] == SHARD_MATRIX + assert job["with"]["candidate-artifact-name"] == ( + "candidate-${{ github.run_attempt }}-${{ inputs.binary-name }}" + ) + assert job["with"]["candidate-sha"] == "${{ github.sha }}" + assert job["with"]["timing-snapshot-artifact"] == ( + "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}" + ) + assert job["with"]["integration-markers"] == "${{ inputs.integration-markers }}" + assert job["with"]["shard-count"] == "${{ inputs.integration-shard-count }}" + assert job["with"]["shard-index"] == "${{ matrix.shard }}" + assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" + assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" + assert job["with"]["timeout-minutes"] == UNIX_TIMEOUT + assert job["with"]["artifact-prefix"] == "${{ github.run_attempt }}-${{ inputs.binary-name }}" + assert job["with"]["runtime-prerequisites"] == "none" + build = workflow_job(workflow, "build") + snapshot = workflow_step(build, "Freeze integration scheduling hints") + assert snapshot["if"] == "inputs.full-validation && inputs.platform != 'windows'" + assert snapshot["with"]["snapshot"] == ( + "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}" + ) + + integration_job = workflow_job(integration, "integration-tests") + unpack = workflow_step(integration_job, "Extract checked candidate archive") + assert "verify-extract" in shell_tokens(unpack) + assert unpack["env"] == { + "CANDIDATE_SHA": "${{ inputs.candidate-sha }}", + "BINARY_NAME": "${{ inputs.binary-name }}", + } + unix = workflow_step(integration_job, "Run integration tests (Unix)") assert unix["env"]["PYTEST_MARK_EXPR"] == "${{ inputs.integration-markers }}" assert unix["env"]["PYTEST_EXTRA_ARGS"] == UNIX_ARGS - assert unix["timeout-minutes"] == UNIX_TIMEOUT assert unix["env"]["APM_BINARY_PATH"] == ( "${{ github.workspace }}/dist/${{ inputs.binary-name }}/apm" ) - windows = workflow_step(job, "Run integration tests (Windows)") + assert unix["env"]["APM_TEST_RUNTIMES"] == "${{ inputs.runtime-prerequisites }}" + assert unix["env"]["APM_RUN_INTEGRATION_TESTS"] == ( + "${{ inputs.run-network-integration-tests && '1' || '' }}" + ) + windows = workflow_step(integration_job, "Run integration tests (Windows)") assert windows["timeout-minutes"] == 20 assert windows["env"]["APM_BINARY_PATH"].endswith("/apm.exe") for step in (unix, windows): assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" assert step["env"]["GITHUB_APM_PAT"] == "${{ secrets.GH_CLI_PAT }}" - assert "APM_RUN_INFERENCE_TESTS" not in effective_env(workflow, job, step) + assert "APM_RUN_INFERENCE_TESTS" not in effective_env(integration, integration_job, step) assert "-IncludeLiveADO" not in shell_tokens(windows) + capture = workflow_step(integration_job, "Capture integration performance evidence") + assert capture["if"] == ( + "always() && inputs.platform != 'windows' && inputs.evidence-variant != ''" + ) + assert capture["env"] == { + "EVIDENCE_SELECTION": "${{ inputs.evidence-selection }}", + "PYTEST_MARK_EXPR": "${{ inputs.integration-markers }}", + } + capture_tokens = shell_tokens(capture) + for token in ( + "scripts.compare_test_runs", + "capture", + "--junit", + "test-results/integration-shard-${{ inputs.shard-index }}.xml", + "--source-sha", + "${{ inputs.candidate-sha }}", + "--variant", + "${{ inputs.evidence-variant }}", + "--shard-count", + "${{ inputs.shard-count }}", + "--candidate-metadata", + "release-assets/${{ inputs.binary-name }}.json", + ): + assert token in capture_tokens + proof_upload = workflow_step(integration_job, "Upload integration performance evidence") + assert proof_upload["if"] == capture["if"] + assert proof_upload["with"]["name"] == ( + "performance-evidence-${{ inputs.artifact-prefix }}-" + "${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }}" + ) + timings_upload = workflow_step(integration_job, "Upload integration timings and outcomes") + assert timings_upload["if"] == "always() && inputs.platform != 'windows'" + assert timings_upload["with"]["if-no-files-found"] == "error" + coverage_upload = workflow_step(integration_job, "Upload coverage data") + assert coverage_upload["with"]["name"] == ( + "integration-coverage-${{ inputs.artifact-prefix }}-shard-${{ inputs.shard-index }}" + ) def test_native_startup_runs_exact_frozen_candidate_before_packaging() -> None: @@ -285,7 +409,7 @@ def test_native_jobs_have_no_unit_or_cross_platform_barriers() -> None: def test_native_integration_preserves_grouping_bounds_and_artifact_identity() -> None: - _assert_integration(_workflow()) + _assert_integration(_workflow(), _integration_workflow()) def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: @@ -300,6 +424,16 @@ def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: "apm-windows-x86_64": "windows-latest", } assert rows["apm-darwin-x86_64"]["integration_markers"] == "lifecycle_smoke and not live" + assert rows["apm-darwin-arm64"]["integration_shard_count"] == 2 + assert rows["apm-darwin-arm64"]["integration_xdist_workers"] == 2 + assert rows["apm-darwin-arm64"]["integration_splitting_algorithm"] == "least_duration" + assert all( + row["integration_shard_count"] == 1 + and row["integration_xdist_workers"] == 4 + and row["integration_splitting_algorithm"] == "duration_based_chunks" + for name, row in rows.items() + if name != "apm-darwin-arm64" + ) assert all( row["integration_markers"] == "not live" for name, row in rows.items() @@ -330,7 +464,14 @@ def test_unit_build_dependency_mutation_is_rejected(job: str) -> None: _assert_parallel_paths(workflow) -@pytest.mark.parametrize("job", ["integration-tests", "release-validation", "windows-installer"]) +@pytest.mark.parametrize( + "job", + [ + "integration-tests-shard", + "release-validation", + "windows-installer", + ], +) def test_reintroduced_barrier_is_rejected(job: str) -> None: workflow = deepcopy(_workflow()) workflow_job(workflow, job)["needs"].append("unit-tests") @@ -370,13 +511,14 @@ def test_echo_cannot_replace_frozen_smoke() -> None: @pytest.mark.parametrize("field", ["PYTEST_MARK_EXPR", "PYTEST_EXTRA_ARGS", "APM_BINARY_PATH"]) def test_missing_integration_safety_argument_is_rejected(field: str) -> None: - workflow = deepcopy(_workflow()) + workflow = _workflow() + integration = deepcopy(_integration_workflow()) step = workflow_step( - workflow_job(workflow, "integration-tests"), "Run integration tests (Unix)" + workflow_job(integration, "integration-tests"), "Run integration tests (Unix)" ) step["env"][field] = "wrong" with pytest.raises(AssertionError): - _assert_integration(workflow) + _assert_integration(workflow, integration) def test_publication_only_consumes_verified_archives() -> None: @@ -390,10 +532,32 @@ def test_publication_only_consumes_verified_archives() -> None: def test_native_artifacts_and_downloads_are_attempt_scoped() -> None: workflow = _workflow() + integration = _integration_workflow() artifact_name = "candidate-${{ github.run_attempt }}-${{ inputs.binary-name }}" upload = workflow_step(workflow_job(workflow, "build"), "Upload binary as workflow artifact") assert upload["with"]["name"] == artifact_name - for name in ("integration-tests", "release-validation", "windows-installer"): + assert upload["with"]["compression-level"] == 0 + unit_upload = workflow_step( + workflow_job(workflow, "unit-tests"), "Upload unit timings and outcomes" + ) + assert unit_upload["with"]["name"] == ( + "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit" + ) + core_upload = workflow_step(workflow_job(workflow, "build"), "Upload native smoke outcomes") + assert core_upload["with"]["name"] == ( + "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-core" + ) + installer_upload = workflow_step( + workflow_job(workflow, "windows-installer"), "Upload installer outcomes" + ) + assert installer_upload["with"]["name"] == ( + "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-installer" + ) + assert ( + workflow_job(workflow, "integration-tests-shard")["with"]["candidate-artifact-name"] + == artifact_name + ) + for name in ("release-validation", "windows-installer"): downloads = [ step for step in workflow_job(workflow, name)["steps"] @@ -401,6 +565,13 @@ def test_native_artifacts_and_downloads_are_attempt_scoped() -> None: ] assert len(downloads) == 1 assert downloads[0]["with"]["name"] == artifact_name + download = next( + step + for step in workflow_job(integration, "integration-tests")["steps"] + if step.get("uses", "").startswith("actions/download-artifact@") + and step.get("with", {}).get("name") == "${{ inputs.candidate-artifact-name }}" + ) + assert download["with"]["name"] == "${{ inputs.candidate-artifact-name }}" def test_tag_downloads_only_exact_selected_artifact_ids() -> None: @@ -426,20 +597,22 @@ def test_tag_downloads_only_exact_selected_artifact_ids() -> None: @pytest.mark.parametrize( - ("job_name", "authority"), + ("job_name", "authorities"), [ - ("create-release", "verify-candidate"), - ("deploy-docs", "create-release"), - ("gh-aw-compat", "create-release"), - ("build-pypi-distributions", "create-release"), - ("publish-pypi", "build-pypi-distributions"), + ("create-release", ["verify-candidate"]), + ("build-docs-artifact", ["plan"]), + ("deploy-docs", ["create-release", "build-docs-artifact"]), + ("gh-aw-compat", ["create-release"]), + ("build-pypi-distributions", ["plan"]), + ("publish-pypi", ["create-release", "build-pypi-distributions"]), ], ) def test_warm_release_descendants_require_success_without_skipped_ancestor_poisoning( - job_name: str, authority: str + job_name: str, authorities: list[str] ) -> None: job = workflow_job(load_workflow(RELEASE_WORKFLOW), job_name) - assert authority in job["needs"] assert "always()" in job["if"] assert "!cancelled()" in job["if"] - assert f"needs.{authority}.result == 'success'" in job["if"] + for authority in authorities: + assert authority in job["needs"] + assert f"needs.{authority}.result == 'success'" in job["if"] diff --git a/tests/unit/test_pytest_timing_workflows.py b/tests/unit/test_pytest_timing_workflows.py index efe62ff05f..f7a5b54a77 100644 --- a/tests/unit/test_pytest_timing_workflows.py +++ b/tests/unit/test_pytest_timing_workflows.py @@ -16,44 +16,117 @@ ROOT = Path(__file__).resolve().parents[2] ACTION = "./.github/actions/pytest-timing" +RELEASE_INTEGRATION = "./.github/workflows/release-integration.yml" +SHARD_MATRIX = ( + "${{ fromJSON(inputs.integration-shard-count == 2 && '{\"shard\":[1,2]}' " + "|| '{\"shard\":[1]}') }}" +) -def assert_shared_integration_snapshot(workflow: dict) -> None: +def assert_shared_integration_snapshot(workflow: dict, integration: dict) -> None: build = workflow_job(workflow, "build") + mq_candidate = "apm-mq-${{ github.run_attempt }}-linux-x86_64" + assert workflow_step(build, "Upload binary")["with"]["name"] == mq_candidate snapshot = workflow_step(build, "Freeze integration scheduling hints") assert snapshot["uses"] == ACTION assert snapshot["with"] == { "suite": "integration", - "snapshot": "integration-duration-snapshot", + "snapshot": "integration-duration-snapshot-${{ github.run_attempt }}", } shards = workflow_job(workflow, "integration-tests-shard") assert "build" in shards["needs"] + assert "smoke-test" not in shards["needs"] assert shards["strategy"]["matrix"]["shard"] == [1, 2, 3, 4] - download = workflow_step(shards, "Download shared integration scheduling snapshot") + assert shards["uses"] == RELEASE_INTEGRATION + assert shards["with"]["candidate-artifact-name"] == mq_candidate + assert shards["with"]["timing-snapshot-artifact"] == snapshot["with"]["snapshot"] + assert shards["with"]["shard-count"] == 4 + assert shards["with"]["xdist-workers"] == 2 + assert "splitting-algorithm" not in shards["with"] + assert shards["with"]["artifact-prefix"] == "mq-linux-${{ github.run_attempt }}" + assert shards["with"]["runtime-prerequisites"] == "none" + assert shards["with"]["run-network-integration-tests"] is True + assert shards["with"]["coverage-enabled"] is True + fan_in = workflow_job(workflow, "integration-tests") + assert set(fan_in["needs"]) == {"integration-tests-shard", "scaling-guards", "smoke-test"} + assert ( + workflow_step(workflow_job(workflow, "smoke-test"), "Download binary")["with"]["name"] + == mq_candidate + ) + assert ( + workflow_step(workflow_job(workflow, "release-validation"), "Download binary")["with"][ + "name" + ] + == mq_candidate + ) + coverage_download = workflow_step(fan_in, "Download shard coverage") + assert coverage_download["with"]["pattern"] == ( + "integration-coverage-mq-linux-${{ github.run_attempt }}-shard-*" + ) + release_job = workflow_job(integration, "integration-tests") + download = workflow_step(release_job, "Download shared integration scheduling snapshot") assert download["uses"].startswith("actions/download-artifact@") - assert download["with"] == {"name": snapshot["with"]["snapshot"]} - assert not any(step.get("uses") == ACTION for step in shards["steps"]) + assert download["with"] == {"name": "${{ inputs.timing-snapshot-artifact }}"} + assert not any(step.get("uses") == ACTION for step in release_job["steps"]) def test_integration_shards_consume_one_immutable_history_snapshot() -> None: workflow = load_workflow(ROOT / ".github/workflows/ci-integration.yml") - assert_shared_integration_snapshot(workflow) + integration = load_workflow(ROOT / ".github/workflows/release-integration.yml") + assert_shared_integration_snapshot(workflow, integration) @pytest.mark.parametrize("mutation", ["missing_dependency", "per_shard_snapshot", "rolling_cache"]) def test_snapshot_contract_rejects_divergent_shard_history(mutation: str) -> None: workflow = deepcopy(load_workflow(ROOT / ".github/workflows/ci-integration.yml")) + integration = deepcopy(load_workflow(ROOT / ".github/workflows/release-integration.yml")) shards = workflow_job(workflow, "integration-tests-shard") if mutation == "missing_dependency": shards["needs"] = [] elif mutation == "per_shard_snapshot": - workflow_step(shards, "Download shared integration scheduling snapshot")["with"]["name"] = ( - "integration-duration-${{ matrix.shard }}" - ) + shards["with"]["timing-snapshot-artifact"] = "integration-duration-${{ matrix.shard }}" else: - shards["steps"].append({"uses": ACTION, "with": {"suite": "integration"}}) + workflow_job(integration, "integration-tests")["steps"].append( + {"uses": ACTION, "with": {"suite": "integration"}} + ) + with pytest.raises(AssertionError): + assert_shared_integration_snapshot(workflow, integration) + + +def assert_release_platform_snapshot(workflow: dict) -> None: + build = workflow_job(workflow, "build") + snapshot = workflow_step(build, "Freeze integration scheduling hints") + assert snapshot["uses"] == ACTION + assert snapshot["with"] == { + "suite": "integration", + "snapshot": "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}", + } + job = workflow_job(workflow, "integration-tests-shard") + assert job["uses"] == RELEASE_INTEGRATION + assert job["needs"] == ["build"] + assert job["strategy"]["fail-fast"] is False + assert job["strategy"]["matrix"] == SHARD_MATRIX + assert job["with"]["timing-snapshot-artifact"] == snapshot["with"]["snapshot"] + assert job["with"]["shard-count"] == "${{ inputs.integration-shard-count }}" + assert job["with"]["shard-index"] == "${{ matrix.shard }}" + assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" + assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" + assert job["with"]["artifact-prefix"] == "${{ github.run_attempt }}-${{ inputs.binary-name }}" + assert job["with"]["runtime-prerequisites"] == "none" + + +def test_release_platform_freezes_one_snapshot_before_native_shards() -> None: + workflow = load_workflow(ROOT / ".github/workflows/release-platform.yml") + assert_release_platform_snapshot(workflow) + + +def test_release_platform_snapshot_mutation_is_rejected() -> None: + workflow = deepcopy(load_workflow(ROOT / ".github/workflows/release-platform.yml")) + workflow_job(workflow, "integration-tests-shard")["with"]["timing-snapshot-artifact"] = ( + "integration-duration-${{ matrix.shard }}" + ) with pytest.raises(AssertionError): - assert_shared_integration_snapshot(workflow) + assert_release_platform_snapshot(workflow) def test_unit_shards_record_timings_without_a_mutable_selection_cache() -> None: @@ -103,7 +176,7 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( " assert 0 <= index < 8\n", encoding="ascii", ) - expected = {f"test_probe.py::test_probe[{index}]@shared_home" for index in range(8)} + expected = {f"test_probe.py::test_probe[{index}]" for index in range(8)} hints = {node: (20.0 if "[0]" in node else 1.0) for node in expected} if weighted else {} hints["test_removed.py::test_old"] = 200.0 snapshot = json.dumps(hints) @@ -117,6 +190,8 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( sys.executable, "-m", "pytest", + "-p", + "scripts.pytest_performance_evidence", "-c", str(config), "--confcutdir", @@ -127,6 +202,8 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( "2", "--group", str(shard), + "--splitting-algorithm", + "least_duration", "-n", "2", "--dist", @@ -136,7 +213,7 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( str(durations), f"--junitxml={outcomes}", ], - cwd=tmp_path, + cwd=ROOT, env={**os.environ, "PYTEST_ADDOPTS": ""}, capture_output=True, text=True, @@ -146,11 +223,14 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( assert result.returncode == 0, result.stdout + result.stderr cases = list(ET.parse(outcomes).iter("testcase")) # noqa: S314 - generated by this probe. assert cases - assert all(not list(case) for case in cases) - nodes = {f"test_probe.py::{case.attrib['name']}" for case in cases} + nodes = { + identity.attrib["value"] + for case in cases + for identity in case.findall("./properties/property[@name='apm_performance_nodeid']") + } observed.append(nodes) stored = json.loads(durations.read_text("ascii")) - assert nodes <= stored.keys() - assert all(stored[node] >= 0 for node in nodes) + assert stored + assert all(duration >= 0 for duration in stored.values()) assert observed[0].isdisjoint(observed[1]) assert observed[0] | observed[1] == expected diff --git a/tests/unit/test_release_performance_workflow.py b/tests/unit/test_release_performance_workflow.py new file mode 100644 index 0000000000..d2b28d325c --- /dev/null +++ b/tests/unit/test_release_performance_workflow.py @@ -0,0 +1,164 @@ +"""Native performance evidence must use the same candidate and complete test selection.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = pytest.mark.component +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ci-release-performance.yml" + + +def _assert_probe(workflow: dict) -> None: + assert workflow["on"] == { + "pull_request": {"types": ["opened", "reopened", "synchronize", "labeled"]} + } + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is True + for job in workflow["jobs"].values(): + assert "ci-performance" in job["if"] + assert "secrets" not in job + assert "environment" not in job + + candidate = workflow_job(workflow, "candidate") + assert candidate["runs-on"] == "macos-latest" + assert candidate["timeout-minutes"] == 15 + build = workflow_step(candidate, "Build current ARM candidate") + assert "scripts/build-binary.sh" in build["run"] + assert build["env"] == {"UV_FROZEN": "true", "UV_NO_SYNC": "true"} + pack = workflow_step(candidate, "Package one immutable candidate for every variant") + assert 'scripts/package_release.py pack --binary-name apm-darwin-arm64 --sha "$GITHUB_SHA"' in ( + " ".join(pack["run"].split()) + ) + snapshot = workflow_step(candidate, "Freeze identical cold scheduling input") + assert snapshot["run"] == "printf '{}\\n' > .test_durations" + uploads = [ + step["with"] for step in candidate["steps"] if "upload-artifact@" in step.get("uses", "") + ] + assert any( + upload["name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" + and upload["path"] == "release-assets/" + and upload["compression-level"] == 0 + for upload in uploads + ) + assert any( + upload["name"] == "benchmark-arm-cold-timings-${{ github.run_attempt }}" + and upload["path"] == ".test_durations" + and upload["include-hidden-files"] is True + for upload in uploads + ) + + integration = workflow_job(workflow, "integration") + assert integration["needs"] == ["candidate"] + assert integration["uses"] == "./.github/workflows/release-integration.yml" + assert integration["strategy"]["fail-fast"] is False + assert integration["strategy"]["matrix"]["include"] == [ + {"variant": "baseline", "shard": 1, "count": 1, "workers": 4}, + {"variant": "proposed", "shard": 1, "count": 2, "workers": 2}, + {"variant": "proposed", "shard": 2, "count": 2, "workers": 2}, + ] + inputs = integration["with"] + assert inputs["candidate-artifact-name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" + assert inputs["candidate-sha"] == "${{ github.sha }}" + assert ( + inputs["timing-snapshot-artifact"] == "benchmark-arm-cold-timings-${{ github.run_attempt }}" + ) + assert inputs["integration-markers"] == "not live" + assert inputs["splitting-algorithm"] == "least_duration" + assert inputs["shard-count"] == "${{ matrix.count }}" + assert inputs["shard-index"] == "${{ matrix.shard }}" + assert inputs["xdist-workers"] == "${{ matrix.workers }}" + assert inputs["evidence-variant"] == "${{ matrix.variant }}" + assert inputs["evidence-selection"] == "tests/integration/ -m not live" + assert inputs["cache-apm-fetches"] is False + assert inputs["runtime-prerequisites"] == "none" + + comparison = workflow_job(workflow, "compare") + assert comparison["needs"] == ["candidate", "integration"] + assert "always()" in comparison["if"] and "!cancelled()" in comparison["if"] + require = workflow_step(comparison, "Require all benchmark executions succeeded") + assert require["env"] == { + "BUILD_RESULT": "${{ needs.candidate.result }}", + "INTEGRATION_RESULT": "${{ needs.integration.result }}", + } + assert 'test "$BUILD_RESULT" = success' in require["run"] + assert 'test "$INTEGRATION_RESULT" = success' in require["run"] + compare = workflow_step(comparison, "Require identical outcomes and a measured execution gain") + assert "scripts.compare_test_runs compare" in compare["run"] + assert "--minimum-speedup 0.05" in compare["run"] + assert '--expected-sha "$GITHUB_SHA"' in compare["run"] + assert "performance-inputs/baseline/baseline-shard-1.json" in compare["run"] + assert "performance-inputs/proposed-1/proposed-shard-1.json" in compare["run"] + assert "performance-inputs/proposed-2/proposed-shard-2.json" in compare["run"] + + +def test_native_probe_uses_cold_shared_input_and_one_exact_candidate() -> None: + _assert_probe(load_workflow(WORKFLOW)) + + +def test_release_routes_the_same_arm_scheduler_as_the_probe() -> None: + catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text(encoding="utf-8")) + arm = next(row for row in catalog if row["binary_name"] == "apm-darwin-arm64") + caller = workflow_job(load_workflow(ROOT / ".github/workflows/build-release.yml"), "platforms") + assert caller["with"]["integration-shard-count"] == "${{ matrix.integration_shard_count }}" + assert caller["with"]["integration-xdist-workers"] == "${{ matrix.integration_xdist_workers }}" + assert caller["with"]["integration-splitting-algorithm"] == ( + "${{ matrix.integration_splitting_algorithm }}" + ) + assert arm["integration_shard_count"] == 2 + assert arm["integration_xdist_workers"] == 2 + assert arm["integration_splitting_algorithm"] == "least_duration" + + +@pytest.mark.parametrize( + "fault", + ["candidate", "snapshot", "source", "selection", "workers", "missing-shard", "fail-fast"], +) +def test_native_probe_rejects_incomparable_or_incomplete_designs(fault: str) -> None: + workflow = deepcopy(load_workflow(WORKFLOW)) + job = workflow["jobs"]["integration"] + if fault == "candidate": + job["with"]["candidate-artifact-name"] = "candidate-${{ matrix.variant }}" + elif fault == "snapshot": + job["with"]["timing-snapshot-artifact"] = "timings-${{ matrix.variant }}" + elif fault == "source": + job["with"]["candidate-sha"] = "${{ github.event.pull_request.head.sha }}" + elif fault == "selection": + job["with"]["integration-markers"] = "lifecycle_smoke" + elif fault == "workers": + job["with"]["xdist-workers"] = "auto" + elif fault == "missing-shard": + job["strategy"]["matrix"]["include"].pop() + else: + job["strategy"]["fail-fast"] = True + with pytest.raises(AssertionError): + _assert_probe(workflow) + + +def test_no_called_benchmark_workflow_can_request_publishing_permissions() -> None: + pending = [WORKFLOW] + visited = set() + while pending: + path = pending.pop() + if path in visited: + continue + visited.add(path) + workflow = load_workflow(path) + permissions = workflow.get("permissions", {}) + assert isinstance(permissions, dict), path + assert all(value in ("read", "none") for value in permissions.values()), path + for job in workflow["jobs"].values(): + permissions = job.get("permissions", {}) + assert isinstance(permissions, dict), path + assert all(value in ("read", "none") for value in permissions.values()), path + assert "environment" not in job, path + assert "secrets" not in job, path + target = job.get("uses", "") + if target.startswith("./.github/workflows/"): + pending.append(ROOT / target.removeprefix("./")) diff --git a/tests/unit/test_release_publication_overlap.py b/tests/unit/test_release_publication_overlap.py new file mode 100644 index 0000000000..6a26e6627f --- /dev/null +++ b/tests/unit/test_release_publication_overlap.py @@ -0,0 +1,495 @@ +"""Executable contracts for read-only release publication prebuild overlap.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = pytest.mark.component +ROOT = Path(__file__).resolve().parents[2] +BUILD_RELEASE = ROOT / ".github" / "workflows" / "build-release.yml" +DOCS = ROOT / ".github" / "workflows" / "docs.yml" +DOCS_BUILD = ROOT / ".github" / "workflows" / "docs-build.yml" +PYPI_DISTRIBUTIONS = ROOT / ".github" / "workflows" / "pypi-distributions.yml" +DOCS_ARTIFACT = "docs-pages-${{ github.run_attempt }}" +PYPI_ARTIFACT = "pypi-distributions-${{ github.run_attempt }}" +VERIFIED_RELEASE_ARTIFACT = "verified-release-assets-${{ github.run_attempt }}" + + +def _workflow(path: Path) -> dict[str, Any]: + """Load a workflow as a mutable mapping.""" + return load_workflow(path) + + +def _replace_outside_strings(expression: str, pattern: str, replacement: str) -> str: + """Apply a regex replacement only outside single-quoted expression strings.""" + parts = re.split(r"('[^']*')", expression) + for index in range(0, len(parts), 2): + parts[index] = re.sub(pattern, replacement, parts[index]) + return "".join(parts) + + +def _expression_value( + expression: str, + *, + github: dict[str, Any], + needs: dict[str, dict[str, Any]], + inputs: dict[str, Any] | None = None, + cancelled: bool = False, +) -> bool: + """Evaluate the GitHub expression subset used by the release DAG.""" + rendered = " ".join(expression.split()) + if rendered.startswith("${{") and rendered.endswith("}}"): + rendered = rendered.removeprefix("${{").removesuffix("}}").strip() + replacements = { + "github.event_name": github.get("event_name"), + "github.ref_type": github.get("ref_type"), + "github.event.repository.private": github.get("private"), + "github.event.release.prerelease": github.get("release_prerelease"), + } + for token, value in replacements.items(): + rendered = rendered.replace(token, repr(value)) + for match in set(re.findall(r"needs\.([A-Za-z0-9_-]+)\.result", rendered)): + rendered = rendered.replace( + f"needs.{match}.result", + repr(needs.get(match, {}).get("result")), + ) + for job, output in set( + re.findall(r"needs\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9_-]+)", rendered) + ): + value = needs.get(job, {}).get("outputs", {}).get(output) + rendered = rendered.replace(f"needs.{job}.outputs.{output}", repr(value)) + for name in set(re.findall(r"inputs\.([A-Za-z0-9_]+)", rendered)): + rendered = rendered.replace(f"inputs.{name}", repr((inputs or {}).get(name))) + rendered = rendered.replace("always()", "True") + rendered = rendered.replace("!cancelled()", repr(not cancelled)) + rendered = rendered.replace("&&", " and ").replace("||", " or ") + rendered = _replace_outside_strings(rendered, r"\bfalse\b", "False") + rendered = _replace_outside_strings(rendered, r"\btrue\b", "True") + return bool(eval(rendered, {"__builtins__": {}}, {})) # noqa: S307 - test-only DSL. + + +def _job_enabled( + workflow: dict[str, Any], + job_name: str, + *, + github: dict[str, Any], + needs: dict[str, dict[str, Any]], +) -> bool: + """Evaluate one job's conditional against simulated completed needs.""" + expression = workflow_job(workflow, job_name).get("if") + if expression is None: + return True + return _expression_value(expression, github=github, needs=needs) + + +def _ready_jobs( + workflow: dict[str, Any], + *, + github: dict[str, Any], + completed: dict[str, dict[str, Any]], +) -> set[str]: + """Return jobs that can start once their needs are completed and passing their if gate.""" + ready = set() + for name, job in workflow["jobs"].items(): + needs = job.get("needs", []) + if isinstance(needs, str): + needs = [needs] + if all(need in completed for need in needs) and _job_enabled( + workflow, + name, + github=github, + needs=completed, + ): + ready.add(name) + return ready + + +def _stable_public_tag_context() -> dict[str, Any]: + """Return the release event shape for a stable public tag.""" + return { + "event_name": "push", + "ref_type": "tag", + "private": False, + "release_prerelease": None, + } + + +def _successful_plan(**outputs: str) -> dict[str, Any]: + """Return a simulated successful plan job with string outputs.""" + return { + "result": "success", + "outputs": { + "candidate_run_id": "", + "full_validation": "true", + "is_prerelease": "false", + **outputs, + }, + } + + +def _successful_release(*, private: bool = False, prerelease: bool = False) -> dict[str, Any]: + """Return simulated create-release outputs as Actions string values.""" + return { + "result": "success", + "outputs": { + "is_private_repo": "true" if private else "false", + "is_prerelease": "true" if prerelease else "false", + }, + } + + +def _local_workflow_path(uses: str) -> Path | None: + """Resolve a local reusable workflow reference.""" + prefix = "./" + if not uses.startswith(prefix): + return None + path = ROOT / uses.removeprefix(prefix) + if path.parent != ROOT / ".github" / "workflows": + return None + return path + + +def _assert_no_write_or_oidc_permission(node: dict[str, Any], label: str) -> None: + """Reject write or OIDC permissions in a read-only reusable call chain.""" + permissions = node.get("permissions", {}) + if permissions in (None, {}, "read-all"): + return + if not isinstance(permissions, dict): + raise AssertionError(f"{label} must use explicit read-only permissions") + for scope, access in permissions.items(): + if scope == "id-token" or access == "write": + raise AssertionError(f"{label} grants {scope}: {access}") + + +def _assert_reachable_reusable_jobs_are_read_only( + workflows: dict[Path, dict[str, Any]], + workflow_path: Path, + job_name: str, +) -> None: + """Recursively prove a read-only builder cannot reach write/OIDC jobs.""" + seen_workflows: set[Path] = set() + + def check_workflow(path: Path) -> None: + if path in seen_workflows: + return + seen_workflows.add(path) + workflow = workflows[path] + _assert_no_write_or_oidc_permission(workflow, str(path)) + for name, job in workflow["jobs"].items(): + check_job(path, name, job) + + def check_job(path: Path, name: str, job: dict[str, Any]) -> None: + label = f"{path.name}:{name}" + _assert_no_write_or_oidc_permission(job, label) + if target := _local_workflow_path(str(job.get("uses", ""))): + check_workflow(target) + + check_job(workflow_path, job_name, workflows[workflow_path]["jobs"][job_name]) + + +def test_read_only_docs_and_wheels_overlap_fresh_candidate_qualification() -> None: + """Fresh stable tags can prebuild docs and wheels before native qualification is done.""" + workflow = _workflow(BUILD_RELEASE) + completed = {"plan": _successful_plan()} + ready = _ready_jobs(workflow, github=_stable_public_tag_context(), completed=completed) + + assert { + "build-docs-artifact", + "build-pypi-distributions", + "candidate-checks", + "platforms", + } <= ready + assert "candidate-ready" not in ready + assert "verify-candidate" not in ready + assert "create-release" not in ready + assert workflow_job(workflow, "build-docs-artifact")["needs"] == ["plan"] + assert workflow_job(workflow, "build-pypi-distributions")["needs"] == ["plan"] + + +def test_same_sha_candidate_promotion_keeps_prebuilds_but_not_publication_early() -> None: + """Same-SHA promotion still prebuilds read-only artifacts while release waits for verify.""" + workflow = _workflow(BUILD_RELEASE) + completed = {"plan": _successful_plan(candidate_run_id="123456")} + ready = _ready_jobs(workflow, github=_stable_public_tag_context(), completed=completed) + + assert {"build-docs-artifact", "build-pypi-distributions"} <= ready + assert "verify-candidate" not in ready + assert "platforms" not in ready + assert "candidate-checks" not in ready + assert "candidate-ready" not in ready + assert "create-release" not in ready + + completed.update( + { + "platforms": {"result": "skipped"}, + "candidate-checks": {"result": "skipped"}, + "candidate-ready": {"result": "skipped"}, + } + ) + ready = _ready_jobs(workflow, github=_stable_public_tag_context(), completed=completed) + assert "verify-candidate" in ready + assert "create-release" not in ready + + +@pytest.mark.parametrize( + ("completed", "job_name"), + [ + ( + {"plan": {"result": "failure", "outputs": {"is_prerelease": "false"}}}, + "build-docs-artifact", + ), + ( + {"plan": {"result": "failure", "outputs": {"is_prerelease": "false"}}}, + "build-pypi-distributions", + ), + ({"plan": _successful_plan(is_prerelease="true")}, "build-docs-artifact"), + ({"plan": _successful_plan(is_prerelease="true")}, "build-pypi-distributions"), + ({"plan": _successful_plan()}, "create-release"), + ], +) +def test_prebuild_and_publication_conditions_fail_closed( + completed: dict[str, dict[str, Any]], + job_name: str, +) -> None: + """Failure, prerelease, and missing verification states do not pass release gates.""" + workflow = _workflow(BUILD_RELEASE) + + assert not _job_enabled( + workflow, + job_name, + github=_stable_public_tag_context(), + needs=completed, + ) + + +def test_private_stable_release_prebuilds_docs_but_skips_wheel_publication() -> None: + """Private repositories keep docs behavior but never build or publish PyPI artifacts.""" + workflow = _workflow(BUILD_RELEASE) + github = {"event_name": "push", "ref_type": "tag", "private": True} + plan = {"plan": _successful_plan()} + + assert _job_enabled(workflow, "build-docs-artifact", github=github, needs=plan) + assert not _job_enabled(workflow, "build-pypi-distributions", github=github, needs=plan) + assert not _job_enabled( + workflow, + "publish-pypi", + github=github, + needs={ + "create-release": _successful_release(private=True), + "build-pypi-distributions": {"result": "skipped"}, + }, + ) + + +@pytest.mark.parametrize("create_release_result", ["failure", "cancelled", "skipped"]) +def test_privileged_publishers_require_successful_github_release( + create_release_result: str, +) -> None: + """Docs and PyPI publication stay behind the verified GitHub release result.""" + workflow = _workflow(BUILD_RELEASE) + needs = { + "create-release": {"result": create_release_result, "outputs": {}}, + "build-docs-artifact": {"result": "success"}, + "build-pypi-distributions": {"result": "success"}, + } + + assert not _job_enabled( + workflow, "deploy-docs", github=_stable_public_tag_context(), needs=needs + ) + assert not _job_enabled( + workflow, "publish-pypi", github=_stable_public_tag_context(), needs=needs + ) + + +@pytest.mark.parametrize( + ("job_name", "failed_prebuild"), + [ + ("deploy-docs", "build-docs-artifact"), + ("publish-pypi", "build-pypi-distributions"), + ], +) +def test_publication_requires_matching_successful_prebuild( + job_name: str, + failed_prebuild: str, +) -> None: + """A failed read-only artifact build cannot be skipped over by a later publisher.""" + workflow = _workflow(BUILD_RELEASE) + needs = { + "create-release": _successful_release(), + "build-docs-artifact": {"result": "success"}, + "build-pypi-distributions": {"result": "success"}, + } + needs[failed_prebuild]["result"] = "failure" + + assert not _job_enabled(workflow, job_name, github=_stable_public_tag_context(), needs=needs) + + +def test_write_and_oidc_publication_jobs_execute_no_candidate_code() -> None: + """Privileged publication jobs only download or deploy artifacts; they never checkout or run.""" + workflow = _workflow(BUILD_RELEASE) + for job_name in ("create-release", "deploy-docs", "publish-pypi"): + job = workflow_job(workflow, job_name) + steps = job["steps"] + assert all("run" not in step for step in steps) + assert all("actions/checkout" not in step.get("uses", "") for step in steps) + + +def test_attempt_scoped_publication_artifact_names_and_compression() -> None: + """Prebuilt publication artifacts are unique per attempt and skip redundant compression.""" + workflow = _workflow(BUILD_RELEASE) + docs = workflow_job(workflow, "build-docs-artifact") + pypi = workflow_job(workflow, "build-pypi-distributions") + publisher = workflow_job(workflow, "publish-pypi") + verifier_upload = next( + step + for step in workflow_job(workflow, "verify-candidate")["steps"] + if step.get("uses", "").startswith("actions/upload-artifact@") + ) + + assert docs["uses"] == "./.github/workflows/docs-build.yml" + assert docs["with"]["pages_artifact_name"] == DOCS_ARTIFACT + assert pypi["with"]["artifact_name"] == PYPI_ARTIFACT + assert ( + workflow_step(publisher, "Download Python distributions")["with"]["name"] == PYPI_ARTIFACT + ) + assert ( + workflow_step( + workflow_job(workflow, "create-release"), "Download verified release archives" + )["with"]["name"] + == VERIFIED_RELEASE_ARTIFACT + ) + assert verifier_upload["with"]["name"] == VERIFIED_RELEASE_ARTIFACT + assert verifier_upload["with"]["compression-level"] == 0 + + +def test_read_only_docs_builder_reachable_jobs_have_no_write_or_oidc_permissions() -> None: + """The build-release docs prebuild cannot statically reach Pages/OIDC jobs.""" + workflows = { + BUILD_RELEASE: _workflow(BUILD_RELEASE), + DOCS: _workflow(DOCS), + DOCS_BUILD: _workflow(DOCS_BUILD), + } + + _assert_reachable_reusable_jobs_are_read_only( + workflows, + BUILD_RELEASE, + "build-docs-artifact", + ) + + +def test_read_only_docs_builder_permission_mutations_are_rejected() -> None: + """Recursive permission guard fails if a called build workflow gains publish power.""" + workflows = { + BUILD_RELEASE: _workflow(BUILD_RELEASE), + DOCS: _workflow(DOCS), + DOCS_BUILD: _workflow(DOCS_BUILD), + } + workflows[DOCS_BUILD]["jobs"]["build"]["permissions"]["id-token"] = "write" + + with pytest.raises(AssertionError): + _assert_reachable_reusable_jobs_are_read_only( + workflows, + BUILD_RELEASE, + "build-docs-artifact", + ) + + +def test_pypi_distribution_workflow_is_reusable_read_only_and_compression_free() -> None: + """The wheel builder exposes build-only timing inputs without publication privileges.""" + workflow = _workflow(PYPI_DISTRIBUTIONS) + build = workflow_job(workflow, "build") + upload = workflow_step(build, "Upload Python distributions") + + assert "workflow_call" in workflow["on"] + assert "workflow_dispatch" in workflow["on"] + assert workflow["permissions"] == {"contents": "read"} + assert build["permissions"] == {"contents": "read"} + assert workflow_step(build, "Install locked build dependencies")["run"] == ( + "uv sync --frozen --extra dev" + ) + assert workflow_step(build, "Build Python package")["run"] == ( + "uv build --no-build-isolation --no-sources" + ) + assert upload["if"] == "inputs.upload_artifact != false" + assert upload["with"]["name"] == "${{ inputs.artifact_name || 'pypi-distributions' }}" + assert upload["with"]["compression-level"] == 0 + + +def test_docs_workflow_preserves_pr_build_and_release_only_publication() -> None: + """PR docs builds stay build-only; release and reusable callers control upload/deploy.""" + workflow = _workflow(DOCS) + build = workflow_job(workflow, "build") + deploy = workflow_job(workflow, "deploy") + pull_request = { + "event_name": "pull_request", + "ref_type": "branch", + "private": False, + "release_prerelease": None, + } + release = { + "event_name": "release", + "ref_type": "tag", + "private": False, + "release_prerelease": False, + } + manual = { + "event_name": "workflow_dispatch", + "ref_type": "branch", + "private": False, + "release_prerelease": None, + } + + assert build["uses"] == "./.github/workflows/docs-build.yml" + assert build["permissions"] == {"contents": "read"} + assert build["with"]["pages_artifact_name"] == ( + "${{ inputs.pages_artifact_name || 'github-pages' }}" + ) + assert not _expression_value( + build["with"]["upload_pages_artifact"], + github=pull_request, + needs={}, + inputs={}, + ) + assert not _expression_value( + deploy["if"], github=pull_request, needs={"build": {"result": "success"}} + ) + assert _expression_value( + build["with"]["upload_pages_artifact"], github=release, needs={}, inputs={} + ) + assert _expression_value(deploy["if"], github=release, needs={"build": {"result": "success"}}) + assert _expression_value( + build["with"]["upload_pages_artifact"], + github=manual, + needs={}, + inputs={"upload_pages_artifact": True}, + ) + assert not _expression_value( + deploy["if"], + github=manual, + needs={"build": {"result": "success"}}, + inputs={"deploy": False}, + ) + assert deploy["permissions"] == {"pages": "write", "id-token": "write"} + + +def test_docs_build_workflow_exposes_read_only_upload_interface() -> None: + """The canonical docs build can be timed and uploaded without publish permissions.""" + workflow = _workflow(DOCS_BUILD) + build = workflow_job(workflow, "build") + upload = workflow_step(build, "Upload build artifacts") + + assert "workflow_call" in workflow["on"] + assert "workflow_dispatch" in workflow["on"] + assert workflow["permissions"] == {"contents": "read"} + assert build["permissions"] == {"contents": "read"} + assert upload["if"] == "inputs.upload_pages_artifact != false" + assert upload["with"] == { + "name": "${{ inputs.pages_artifact_name || 'github-pages' }}", + "path": "docs/dist", + } diff --git a/tests/unit/test_security_workflow_contracts.py b/tests/unit/test_security_workflow_contracts.py index 587f0379ad..b9c29055f1 100644 --- a/tests/unit/test_security_workflow_contracts.py +++ b/tests/unit/test_security_workflow_contracts.py @@ -60,11 +60,15 @@ def test_pypi_publisher_only_downloads_and_publishes() -> None: workflow = load_workflow(ROOT / ".github" / "workflows" / "build-release.yml") builder = workflow_job(workflow, "build-pypi-distributions") assert builder["permissions"] == {"contents": "read"} - sync = workflow_step(builder, "Install locked build dependencies") + assert builder["uses"] == "./.github/workflows/pypi-distributions.yml" + pypi_workflow = load_workflow(ROOT / ".github" / "workflows" / "pypi-distributions.yml") + pypi_builder = workflow_job(pypi_workflow, "build") + assert pypi_builder["permissions"] == {"contents": "read"} + sync = workflow_step(pypi_builder, "Install locked build dependencies") assert sync["run"] == "uv sync --frozen --extra dev" - build = workflow_step(builder, "Build Python package") + build = workflow_step(pypi_builder, "Build Python package") assert build["run"] == "uv build --no-build-isolation --no-sources" - assert all("uvx" not in str(step.get("run", "")) for step in builder["steps"]) + assert all("uvx" not in str(step.get("run", "")) for step in pypi_builder["steps"]) publisher = workflow_job(workflow, "publish-pypi") assert publisher["permissions"] == {"actions": "read", "id-token": "write"} diff --git a/tests/unit/test_windows_compat_gate_workflow.py b/tests/unit/test_windows_compat_gate_workflow.py index 0c5f747874..64499af789 100644 --- a/tests/unit/test_windows_compat_gate_workflow.py +++ b/tests/unit/test_windows_compat_gate_workflow.py @@ -39,6 +39,7 @@ class (CRLF text-mode writes, backslash path separators leaking into ROOT = Path(__file__).resolve().parents[2] CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +SOURCE_PERFORMANCE_WORKFLOW = ROOT / ".github" / "workflows" / "ci-source-performance.yml" MERGE_GATE_WORKFLOW = ROOT / ".github" / "workflows" / "merge-gate.yml" _COLLECTION_ENV_BASELINE = os.environ.copy() @@ -46,6 +47,11 @@ class (CRLF text-mode writes, backslash path separators leaking into GATE_CHECK_NAME = "Windows Compatibility Gate" GATE_STEP = "Run cross-platform contract family" GATE_MARKER = "windows_compat" +SOURCE_PERFORMANCE_JOB = "windows-compat-performance" +SOURCE_PERFORMANCE_LABEL = "ci-performance" +SOURCE_PERFORMANCE_SELECTION = ( + "-m windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py" +) # The full-suite roots already covered by build-and-test-shard (Linux). # The gate must never invoke these WITHOUT a marker filter -- that @@ -93,7 +99,7 @@ def _gate_pytest_args(step: dict) -> list[str]: # Flags that take a value as the following token -- needed to tell # "-p" and "-m"'s VALUES apart from genuine positional test-path # arguments when computing the selected root(s). -_VALUE_FLAGS = ("-p", "-m") +_VALUE_FLAGS = ("-p", "-m", "-n", "--dist") def _positional_test_paths(args: list[str]) -> list[str]: @@ -139,6 +145,39 @@ def _collect_gate_family(args: list[str]) -> subprocess.CompletedProcess[str]: ) +def _without_execution_only_args(args: list[str]) -> list[str]: + """Strip scheduling flags that are irrelevant to collect-only selection.""" + stripped: list[str] = [] + skip_next = False + for arg in args: + if skip_next: + skip_next = False + continue + if arg in {"-n", "--numprocesses", "--dist"}: + skip_next = True + continue + if arg.startswith(("--numprocesses=", "--dist=")): + continue + stripped.append(arg) + return stripped + + +def _assert_bounded_loadgroup_parallelism(args: list[str]) -> None: + assert "-n" in args, "Windows compatibility gate must declare bounded xdist workers" + worker_index = args.index("-n") + assert args[worker_index : worker_index + 2] == ["-n", "2"], ( + "Windows compatibility gate must use exactly two xdist workers" + ) + assert "--dist" in args, "Windows compatibility gate must declare an xdist scheduler" + dist_index = args.index("--dist") + assert args[dist_index : dist_index + 2] == ["--dist", "loadgroup"], ( + "Windows compatibility gate must use loadgroup so xdist_group fixture affinity is honored" + ) + assert "--numprocesses=auto" not in args and "-nauto" not in args, ( + "Windows compatibility gate must not use unbounded automatic worker selection" + ) + + def test_windows_compat_gate_runs_on_windows_with_bounded_timeout() -> None: job = workflow_job(_ci_workflow(), GATE_JOB) assert job["name"] == GATE_CHECK_NAME @@ -184,6 +223,14 @@ def test_windows_compat_gate_runs_over_narrowest_maintainable_root() -> None: ) +def test_windows_compat_gate_uses_bounded_loadgroup_parallelism() -> None: + """The required gate may parallelize, but only with bounded worker count + and the scheduler that preserves any tests carrying xdist_group affinity.""" + job = workflow_job(_ci_workflow(), GATE_JOB) + step = workflow_step(job, GATE_STEP) + _assert_bounded_loadgroup_parallelism(_gate_pytest_args(step)) + + def test_windows_compat_gate_does_not_duplicate_full_suite() -> None: """The gate must stay a marker-scoped subset, not a second full-suite run. @@ -232,7 +279,9 @@ def test_windows_compat_gate_marker_selects_nonempty_subset() -> None: """ job = workflow_job(_ci_workflow(), GATE_JOB) step = workflow_step(job, GATE_STEP) - _assert_gate_family_collection(_collect_gate_family(_gate_pytest_args(step))) + _assert_gate_family_collection( + _collect_gate_family(_without_execution_only_args(_gate_pytest_args(step))) + ) @pytest.mark.parametrize( @@ -309,6 +358,34 @@ def fake_run( assert captured_env.get("PATH") == expected_path +@pytest.mark.parametrize( + "parallel_args", + [ + "", + "-n 0 --dist loadgroup", + "-n auto --dist loadgroup", + "-n 3 --dist loadgroup", + "-n 2 --dist worksteal", + "-n 2 --dist load", + "-n 2", + "--dist loadgroup", + "-n 2 --dist loadgroup --numprocesses=auto", + ], +) +def test_windows_compat_gate_parallelism_drift_fails(parallel_args: str) -> None: + ci = deepcopy(_ci_workflow()) + job = workflow_job(ci, GATE_JOB) + step = workflow_step(job, GATE_STEP) + step["run"] = re.sub( + r"\s*-n 2 --dist loadgroup\s*", + f"\n {parallel_args}\n", + step["run"], + ) + + with pytest.raises(AssertionError, match=r"xdist|workers|loadgroup|automatic"): + _assert_bounded_loadgroup_parallelism(_gate_pytest_args(step)) + + @pytest.mark.parametrize( "expected_checks_key", ( @@ -382,3 +459,138 @@ def test_narrowing_root_below_windows_compat_root_breaks_contract() -> None: assert positional != ["tests/unit"], ( "mutation setup sanity check failed -- root was not actually narrowed" ) + + +def _source_performance_workflow() -> dict: + return load_workflow(SOURCE_PERFORMANCE_WORKFLOW) + + +def _source_performance_job() -> dict: + return workflow_job(_source_performance_workflow(), SOURCE_PERFORMANCE_JOB) + + +def test_source_performance_probe_is_opt_in_read_only_and_bounded() -> None: + workflow = _source_performance_workflow() + on_block = workflow["on"] + assert on_block["pull_request"]["types"] == [ + "opened", + "reopened", + "synchronize", + "labeled", + ] + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is True + + job = workflow_job(workflow, SOURCE_PERFORMANCE_JOB) + assert SOURCE_PERFORMANCE_LABEL in job["if"] + assert "contains(github.event.pull_request.labels.*.name" in job["if"] + assert job["runs-on"] == "windows-latest" + timeout = job.get("timeout-minutes") + assert isinstance(timeout, int) and 0 < timeout <= 20 + assert "secrets" not in job + + +def test_source_performance_probe_compares_serial_and_parallel_same_selection() -> None: + job = _source_performance_job() + matrix = job["strategy"]["matrix"]["include"] + assert matrix == [ + {"variant": "baseline", "pytest_extra": ""}, + {"variant": "proposed", "pytest_extra": "-n 2 --dist loadgroup"}, + ] + run_step = workflow_step(job, "Run Windows compatibility selection") + run = run_step["run"] + assert SOURCE_PERFORMANCE_SELECTION in " ".join(run.split()) + assert "-p scripts.pytest_performance_evidence" in " ".join(run.split()) + assert "${{ matrix.pytest_extra }}" in run + assert "--junitxml=test-results/${{ matrix.variant }}/windows.xml" in run + assert "--store-durations" in run + + +def test_source_performance_probe_captures_proof_inventory_and_unique_artifacts() -> None: + job = _source_performance_job() + collect_step = workflow_step(job, "Record Windows compatibility selection") + collect_run = collect_step["run"] + assert f"selection={SOURCE_PERFORMANCE_SELECTION.removeprefix('-m ')}" in collect_run + assert "test-results/${{ matrix.variant }}/selection.txt" in collect_run + assert not any("--collect-only" in step.get("run", "") for step in job["steps"]) + run = workflow_step(job, "Run Windows compatibility selection")["run"] + assert "uv run --frozen --no-sync --extra dev python -m pytest" in run + + capture_step = workflow_step(job, "Capture comparable proof") + assert capture_step["if"] == "always()" + capture_run = capture_step["run"] + assert "python -m scripts.compare_test_runs capture" in capture_run + assert "--junit test-results/${{ matrix.variant }}/windows.xml" in capture_run + assert "--output test-results/${{ matrix.variant }}/proof.json" in capture_run + assert '--source-sha "$env:GITHUB_SHA"' in capture_run + assert ( + "--selection 'windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py'" + in capture_run + ) + assert "--variant ${{ matrix.variant }}" in capture_run + assert "--shard 1" in capture_run + assert "--shard-count 1" in capture_run + + upload_step = workflow_step(job, "Upload Windows compatibility performance evidence") + assert upload_step["if"] == "always()" + assert upload_step["uses"] == ( + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" + ) + with_block = upload_step["with"] + assert with_block["name"] == ( + "windows-compat-performance-${{ matrix.variant }}-${{ github.run_attempt }}" + ) + assert with_block["path"] == "test-results/${{ matrix.variant }}/" + assert with_block["if-no-files-found"] == "error" + + +def test_source_performance_probe_compares_proofs_after_download() -> None: + workflow = _source_performance_workflow() + compare_job = workflow_job(workflow, "compare-windows-compat-performance") + assert compare_job["name"] == "Compare Windows Compat Performance" + assert compare_job["needs"] == [SOURCE_PERFORMANCE_JOB] + assert SOURCE_PERFORMANCE_LABEL in compare_job["if"] + assert compare_job["runs-on"] == "ubuntu-24.04" + timeout = compare_job.get("timeout-minutes") + assert isinstance(timeout, int) and 0 < timeout <= 5 + + require_step = workflow_step(compare_job, "Require benchmark variants succeeded") + assert "needs.windows-compat-performance.result" in require_step["run"] + assert '!= "success"' in require_step["run"] + + baseline_download = workflow_step(compare_job, "Download baseline proof") + assert baseline_download["uses"] == ( + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" + ) + assert baseline_download["with"] == { + "name": "windows-compat-performance-baseline-${{ github.run_attempt }}", + "path": "performance-inputs/baseline", + } + proposed_download = workflow_step(compare_job, "Download proposed proof") + assert proposed_download["with"] == { + "name": "windows-compat-performance-proposed-${{ github.run_attempt }}", + "path": "performance-inputs/proposed", + } + + compare_step = workflow_step(compare_job, "Compare equivalent Windows compatibility runs") + compare_run = compare_step["run"] + assert "python -m scripts.compare_test_runs compare" in compare_run + assert "--baseline performance-inputs/baseline/proof.json" in compare_run + assert "--proposed performance-inputs/proposed/proof.json" in compare_run + assert '--expected-sha "$GITHUB_SHA"' in compare_run + assert "--minimum-speedup 0.05" in compare_run + assert "--output performance-comparison/comparison.json" in compare_run + + +def test_source_performance_probe_does_not_write_cache_or_production_evidence() -> None: + text = SOURCE_PERFORMANCE_WORKFLOW.read_text(encoding="utf-8") + forbidden = [ + "actions/cache", + "enable-cache: true", + "release-candidate", + "candidate-", + "GITHUB_APM_PAT", + "ADO_APM_PAT", + "secrets.", + ] + assert [token for token in forbidden if token in text] == [] diff --git a/uv.lock b/uv.lock index 6f9e4f4907..5072faae5e 100644 --- a/uv.lock +++ b/uv.lock @@ -235,6 +235,7 @@ dev = [ { name = "jsonschema" }, { name = "mutmut" }, { name = "mypy" }, + { name = "packaging" }, { name = "pylint" }, { name = "pytest" }, { name = "pytest-cov" }, @@ -255,6 +256,7 @@ requires-dist = [ { name = "llm-github-models", specifier = ">=0.18.0" }, { name = "mutmut", marker = "extra == 'dev'", specifier = "==3.6.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" }, + { name = "packaging", marker = "extra == 'dev'", specifier = ">=24.0" }, { name = "pyinstaller", marker = "extra == 'build'", specifier = ">=6.0.0" }, { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, From e93483c2628dd24a0e2c68bb5d97dba93799b555 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:32:25 +0200 Subject: [PATCH 04/23] fix(ci): preserve candidate directory layout in ARM performance probe Match the native artifact envelope instead of flattening release-assets during upload. Add a regression for the failed hosted probe layout. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-release-performance.yml | 6 ++++- .../unit/test_release_performance_workflow.py | 23 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-release-performance.yml b/.github/workflows/ci-release-performance.yml index 459a026d21..f0fc0731e5 100644 --- a/.github/workflows/ci-release-performance.yml +++ b/.github/workflows/ci-release-performance.yml @@ -48,7 +48,11 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: benchmark-arm-candidate-${{ github.run_attempt }} - path: release-assets/ + # Sibling paths preserve release-assets/ instead of flattening it. + path: | + release-assets/ + scripts/package_release.py + scripts/release-platforms.json compression-level: 0 if-no-files-found: error retention-days: 30 diff --git a/tests/unit/test_release_performance_workflow.py b/tests/unit/test_release_performance_workflow.py index d2b28d325c..524241a72d 100644 --- a/tests/unit/test_release_performance_workflow.py +++ b/tests/unit/test_release_performance_workflow.py @@ -43,7 +43,8 @@ def _assert_probe(workflow: dict) -> None: ] assert any( upload["name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" - and upload["path"] == "release-assets/" + and set(upload["path"].splitlines()) + == {"release-assets/", "scripts/package_release.py", "scripts/release-platforms.json"} and upload["compression-level"] == 0 for upload in uploads ) @@ -118,7 +119,16 @@ def test_release_routes_the_same_arm_scheduler_as_the_probe() -> None: @pytest.mark.parametrize( "fault", - ["candidate", "snapshot", "source", "selection", "workers", "missing-shard", "fail-fast"], + [ + "candidate", + "snapshot", + "source", + "selection", + "workers", + "missing-shard", + "fail-fast", + "flattened-archive", + ], ) def test_native_probe_rejects_incomparable_or_incomplete_designs(fault: str) -> None: workflow = deepcopy(load_workflow(WORKFLOW)) @@ -135,8 +145,15 @@ def test_native_probe_rejects_incomparable_or_incomplete_designs(fault: str) -> job["with"]["xdist-workers"] = "auto" elif fault == "missing-shard": job["strategy"]["matrix"]["include"].pop() - else: + elif fault == "fail-fast": job["strategy"]["fail-fast"] = True + else: + uploads = [ + step + for step in workflow["jobs"]["candidate"]["steps"] + if "upload-artifact@" in step.get("uses", "") + ] + uploads[0]["with"]["path"] = "release-assets/" with pytest.raises(AssertionError): _assert_probe(workflow) From 3aa6eb958156aafb5d66a915e4d3de8ca4856e16 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:56:33 +0200 Subject: [PATCH 05/23] perf(ci): match ARM shard workers to hosted CPU capacity Two-worker cold sharding was not accepted: outcome parity held but hosted images differed and raw timing was below target. Retry three workers per shard without relaxing proof checks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-release-performance.yml | 4 ++-- docs/src/content/docs/contributing/integration-testing.md | 4 ++-- scripts/release-platforms.json | 2 +- tests/unit/test_platform_contract_workflow.py | 2 +- tests/unit/test_release_performance_workflow.py | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-release-performance.yml b/.github/workflows/ci-release-performance.yml index f0fc0731e5..ac5a1550c0 100644 --- a/.github/workflows/ci-release-performance.yml +++ b/.github/workflows/ci-release-performance.yml @@ -89,11 +89,11 @@ jobs: - variant: proposed shard: 1 count: 2 - workers: 2 + workers: 3 - variant: proposed shard: 2 count: 2 - workers: 2 + workers: 3 uses: ./.github/workflows/release-integration.yml permissions: contents: read diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 6a73e37ad1..f3d24f1646 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -404,7 +404,7 @@ the build independently; all remain required at the final gate. and tested. Unix integration uses the shared `release-integration.yml` runner. macOS ARM -uses two outer shards with two workers each; other native Unix lanes use one +uses two outer shards with three workers each; other native Unix lanes use one shard with four workers. ARM uses `least_duration` assignment, preserving relative order within each shard. Every shard must succeed. @@ -562,7 +562,7 @@ compare a later complete release before claiming an end-to-end improvement. Apply the `ci-performance` PR label to run `ci-release-performance.yml` and `ci-source-performance.yml`. The native probe builds one current macOS ARM -candidate, then compares one four-worker run against two two-worker shards. +candidate, then compares one four-worker run against two three-worker shards. All three consume the same verified archive and the same empty timing snapshot, testing cold-history behavior rather than assuming a warm cache. The Windows probe compares serial and two-worker execution of the unchanged diff --git a/scripts/release-platforms.json b/scripts/release-platforms.json index d3aa91fae7..cbf6cf109c 100644 --- a/scripts/release-platforms.json +++ b/scripts/release-platforms.json @@ -45,7 +45,7 @@ "binary_name": "apm-darwin-arm64", "integration_markers": "not live", "integration_shard_count": 2, - "integration_xdist_workers": 2, + "integration_xdist_workers": 3, "integration_splitting_algorithm": "least_duration", "on_main": false } diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index d5bfcf2c63..6fdd7f63cb 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -425,7 +425,7 @@ def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: } assert rows["apm-darwin-x86_64"]["integration_markers"] == "lifecycle_smoke and not live" assert rows["apm-darwin-arm64"]["integration_shard_count"] == 2 - assert rows["apm-darwin-arm64"]["integration_xdist_workers"] == 2 + assert rows["apm-darwin-arm64"]["integration_xdist_workers"] == 3 assert rows["apm-darwin-arm64"]["integration_splitting_algorithm"] == "least_duration" assert all( row["integration_shard_count"] == 1 diff --git a/tests/unit/test_release_performance_workflow.py b/tests/unit/test_release_performance_workflow.py index 524241a72d..7b0876fea5 100644 --- a/tests/unit/test_release_performance_workflow.py +++ b/tests/unit/test_release_performance_workflow.py @@ -61,8 +61,8 @@ def _assert_probe(workflow: dict) -> None: assert integration["strategy"]["fail-fast"] is False assert integration["strategy"]["matrix"]["include"] == [ {"variant": "baseline", "shard": 1, "count": 1, "workers": 4}, - {"variant": "proposed", "shard": 1, "count": 2, "workers": 2}, - {"variant": "proposed", "shard": 2, "count": 2, "workers": 2}, + {"variant": "proposed", "shard": 1, "count": 2, "workers": 3}, + {"variant": "proposed", "shard": 2, "count": 2, "workers": 3}, ] inputs = integration["with"] assert inputs["candidate-artifact-name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" @@ -113,7 +113,7 @@ def test_release_routes_the_same_arm_scheduler_as_the_probe() -> None: "${{ matrix.integration_splitting_algorithm }}" ) assert arm["integration_shard_count"] == 2 - assert arm["integration_xdist_workers"] == 2 + assert arm["integration_xdist_workers"] == 3 assert arm["integration_splitting_algorithm"] == "least_duration" From c41bd4bdfc4bc4a49bc19ce0fd111f33a1731322 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 21:19:56 +0200 Subject: [PATCH 06/23] fix(ci): keep ARM sharding experimental until matched proof passes Both bounded hosted comparisons preserved every case but failed strict runner-image identity. Retain one four-worker release shard and the read-only two-shard experiment; do not claim unaccepted ARM savings. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../content/docs/contributing/integration-testing.md | 10 ++++++---- scripts/release-platforms.json | 6 +++--- tests/unit/test_platform_contract_workflow.py | 6 +----- tests/unit/test_release_performance_workflow.py | 12 ++++++++---- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index f3d24f1646..2858e90f4e 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -403,10 +403,11 @@ the build independently; all remain required at the final gate. 5. Promote only the exact archives that were packaged once, verified, unpacked, and tested. -Unix integration uses the shared `release-integration.yml` runner. macOS ARM -uses two outer shards with three workers each; other native Unix lanes use one -shard with four workers. ARM uses `least_duration` assignment, preserving -relative order within each shard. Every shard must succeed. +Unix integration uses the shared `release-integration.yml` runner. All native +Unix lanes retain one shard with four workers and `duration_based_chunks` +assignment. Multi-runner ARM sharding remains experimental until a matched +performance comparison supports changing the release topology. Every +configured shard must succeed. For stable tags, read-only docs and wheel builds also start after release planning. Their publishers wait for successful GitHub Release creation and @@ -563,6 +564,7 @@ compare a later complete release before claiming an end-to-end improvement. Apply the `ci-performance` PR label to run `ci-release-performance.yml` and `ci-source-performance.yml`. The native probe builds one current macOS ARM candidate, then compares one four-worker run against two three-worker shards. +This proposed topology is probe-only; it does not change native release policy. All three consume the same verified archive and the same empty timing snapshot, testing cold-history behavior rather than assuming a warm cache. The Windows probe compares serial and two-worker execution of the unchanged diff --git a/scripts/release-platforms.json b/scripts/release-platforms.json index cbf6cf109c..6c9bcbbb82 100644 --- a/scripts/release-platforms.json +++ b/scripts/release-platforms.json @@ -44,9 +44,9 @@ "platform": "darwin", "binary_name": "apm-darwin-arm64", "integration_markers": "not live", - "integration_shard_count": 2, - "integration_xdist_workers": 3, - "integration_splitting_algorithm": "least_duration", + "integration_shard_count": 1, + "integration_xdist_workers": 4, + "integration_splitting_algorithm": "duration_based_chunks", "on_main": false } ] diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index 6fdd7f63cb..04d8e17807 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -424,15 +424,11 @@ def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: "apm-windows-x86_64": "windows-latest", } assert rows["apm-darwin-x86_64"]["integration_markers"] == "lifecycle_smoke and not live" - assert rows["apm-darwin-arm64"]["integration_shard_count"] == 2 - assert rows["apm-darwin-arm64"]["integration_xdist_workers"] == 3 - assert rows["apm-darwin-arm64"]["integration_splitting_algorithm"] == "least_duration" assert all( row["integration_shard_count"] == 1 and row["integration_xdist_workers"] == 4 and row["integration_splitting_algorithm"] == "duration_based_chunks" - for name, row in rows.items() - if name != "apm-darwin-arm64" + for row in rows.values() ) assert all( row["integration_markers"] == "not live" diff --git a/tests/unit/test_release_performance_workflow.py b/tests/unit/test_release_performance_workflow.py index 7b0876fea5..410e28a506 100644 --- a/tests/unit/test_release_performance_workflow.py +++ b/tests/unit/test_release_performance_workflow.py @@ -103,7 +103,7 @@ def test_native_probe_uses_cold_shared_input_and_one_exact_candidate() -> None: _assert_probe(load_workflow(WORKFLOW)) -def test_release_routes_the_same_arm_scheduler_as_the_probe() -> None: +def test_release_keeps_baseline_arm_topology_until_sharding_is_accepted() -> None: catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text(encoding="utf-8")) arm = next(row for row in catalog if row["binary_name"] == "apm-darwin-arm64") caller = workflow_job(load_workflow(ROOT / ".github/workflows/build-release.yml"), "platforms") @@ -112,9 +112,13 @@ def test_release_routes_the_same_arm_scheduler_as_the_probe() -> None: assert caller["with"]["integration-splitting-algorithm"] == ( "${{ matrix.integration_splitting_algorithm }}" ) - assert arm["integration_shard_count"] == 2 - assert arm["integration_xdist_workers"] == 3 - assert arm["integration_splitting_algorithm"] == "least_duration" + assert arm["integration_shard_count"] == 1 + assert arm["integration_xdist_workers"] == 4 + assert arm["integration_splitting_algorithm"] == "duration_based_chunks" + variants = workflow_job(load_workflow(WORKFLOW), "integration")["strategy"]["matrix"]["include"] + baseline = next(row for row in variants if row["variant"] == "baseline") + assert baseline["count"] == arm["integration_shard_count"] + assert baseline["workers"] == arm["integration_xdist_workers"] @pytest.mark.parametrize( From 8b752a2fe75667be43b467594528a11f4e8df9b3 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 23:15:55 +0200 Subject: [PATCH 07/23] perf(ci): rehearse controlled whole-pipeline sharding Compare ARM and Linux integration plus Intel units against complete unchanged native/source authorities. Match actual runner identities before pytest and retain exact scenario outcomes and observed native gate delays in the DAG model. Production sharding remains unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/performance-cohort/action.yml | 74 +++ .github/workflows/build-release.yml | 3 + .github/workflows/ci-integration.yml | 3 + .github/workflows/ci-release-performance.yml | 2 + .github/workflows/ci-release-rehearsal.yml | 171 +++++++ .github/workflows/release-integration.yml | 13 + .github/workflows/release-platform.yml | 135 +++-- .github/workflows/release-unit.yml | 226 +++++++++ .../docs/contributing/integration-testing.md | 40 ++ scripts/compare_test_runs.py | 29 +- scripts/performance-cohort.cjs | 35 ++ scripts/performance-cohort.test.cjs | 72 +++ scripts/performance_cohort.py | 150 ++++++ scripts/release-candidate.cjs | 1 + scripts/release_rehearsal.py | 479 ++++++++++++++++++ tests/unit/scripts/test_performance_cohort.py | 179 +++++++ tests/unit/scripts/test_release_rehearsal.py | 331 ++++++++++++ tests/unit/test_ado_live_workflow_contract.py | 2 +- tests/unit/test_platform_contract_workflow.py | 301 ++++++++++- tests/unit/test_pytest_timing_workflows.py | 15 +- tests/unit/test_release_rehearsal_workflow.py | 181 +++++++ tests/unit/test_windows_unit_diagnostics.py | 4 +- 22 files changed, 2381 insertions(+), 65 deletions(-) create mode 100644 .github/actions/performance-cohort/action.yml create mode 100644 .github/workflows/ci-release-rehearsal.yml create mode 100644 .github/workflows/release-unit.yml create mode 100644 scripts/performance-cohort.cjs create mode 100644 scripts/performance-cohort.test.cjs create mode 100644 scripts/performance_cohort.py create mode 100644 scripts/release_rehearsal.py create mode 100644 tests/unit/scripts/test_performance_cohort.py create mode 100644 tests/unit/scripts/test_release_rehearsal.py create mode 100644 tests/unit/test_release_rehearsal_workflow.py diff --git a/.github/actions/performance-cohort/action.yml b/.github/actions/performance-cohort/action.yml new file mode 100644 index 0000000000..34d4bbbb30 --- /dev/null +++ b/.github/actions/performance-cohort/action.yml @@ -0,0 +1,74 @@ +name: Matched performance cohort +description: Require identical identities from all three actual test runners before pytest. +inputs: + cohort: + description: Suite, run attempt and binary identifying this paired cohort. + required: true + member: + description: baseline-1, proposed-1 or proposed-2. + required: true + source-sha: + description: Exact source commit being executed. + required: true + selection: + description: Complete marker and root selection shared by all members. + required: true + candidate-metadata: + description: Native archive metadata for integration; empty for source unit tests. + default: '' +runs: + using: composite + steps: + - name: Record this actual test runner + shell: bash + env: + COHORT: ${{ inputs.cohort }} + MEMBER: ${{ inputs.member }} + SOURCE_SHA: ${{ inputs.source-sha }} + SELECTION: ${{ inputs.selection }} + CANDIDATE_METADATA: ${{ inputs.candidate-metadata }} + run: | + args=() + if [ -n "$CANDIDATE_METADATA" ]; then + args+=(--candidate-metadata "$CANDIDATE_METADATA") + fi + python -m scripts.performance_cohort record \ + --cohort "$COHORT" --member "$MEMBER" \ + --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" \ + --source-sha "$SOURCE_SHA" --selection "$SELECTION" \ + --output ".performance-cohort-out/$MEMBER.json" "${args[@]}" + - name: Publish runner identity + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-cohort-${{ inputs.cohort }}-${{ inputs.member }} + path: .performance-cohort-out/${{ inputs.member }}.json + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Wait for all actual paired runners + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + COHORT: ${{ inputs.cohort }} + with: + script: | + const { waitForCohort } = require('./scripts/performance-cohort.cjs'); + await waitForCohort({ + github, context, cohort: process.env.COHORT, + attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + }); + - name: Download complete cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-cohort-${{ inputs.cohort }}-* + path: .performance-cohort-inputs + merge-multiple: true + - name: Reject different images, dependencies or candidates before execution + shell: bash + env: + COHORT: ${{ inputs.cohort }} + SOURCE_SHA: ${{ inputs.source-sha }} + run: | + python -m scripts.performance_cohort verify \ + --root .performance-cohort-inputs --cohort "$COHORT" \ + --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" \ + --source-sha "$SOURCE_SHA" diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index c37fa17029..741e5b4bd5 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -63,6 +63,9 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} uses: ./.github/workflows/release-platform.yml + permissions: + contents: read + actions: read with: runner: ${{ matrix.runner }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 6c72fdfa4b..eb418b5dd8 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -185,6 +185,9 @@ jobs: # so xdist serializes them on a single worker (race-safe). shard: [1, 2, 3, 4] uses: ./.github/workflows/release-integration.yml + permissions: + contents: read + actions: read with: runner: ubuntu-24.04 platform: linux diff --git a/.github/workflows/ci-release-performance.yml b/.github/workflows/ci-release-performance.yml index ac5a1550c0..96b5d5f20f 100644 --- a/.github/workflows/ci-release-performance.yml +++ b/.github/workflows/ci-release-performance.yml @@ -97,6 +97,7 @@ jobs: uses: ./.github/workflows/release-integration.yml permissions: contents: read + actions: read with: runner: macos-latest platform: darwin @@ -115,6 +116,7 @@ jobs: artifact-prefix: arm-probe-${{ matrix.variant }}-${{ github.run_attempt }} evidence-variant: ${{ matrix.variant }} evidence-selection: tests/integration/ -m not live + performance-cohort: integration-${{ github.run_attempt }}-apm-darwin-arm64 compare: name: Compare ARM Integration Performance diff --git a/.github/workflows/ci-release-rehearsal.yml b/.github/workflows/ci-release-rehearsal.yml new file mode 100644 index 0000000000..b52f33cdda --- /dev/null +++ b/.github/workflows/ci-release-rehearsal.yml @@ -0,0 +1,171 @@ +name: Read-only Release Sharding Rehearsal + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + +permissions: + contents: read + actions: read + +concurrency: + group: ci-release-rehearsal-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + plan: + name: Plan release rehearsal + if: contains(github.event.pull_request.labels.*.name, 'ci-release-rehearsal') + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - id: matrix + name: Derive all five native controls and the three paired surfaces + run: python -m scripts.release_rehearsal matrix >> "$GITHUB_OUTPUT" + + source-checks: + name: Candidate Source Checks + needs: [plan] + uses: ./.github/workflows/ci.yml + permissions: + contents: read + + platforms: + name: ${{ matrix.binary_name }} + needs: [plan] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + uses: ./.github/workflows/release-platform.yml + permissions: + contents: read + actions: read + with: + runner: ${{ matrix.runner }} + platform: ${{ matrix.platform }} + binary-name: ${{ matrix.binary_name }} + integration-markers: ${{ matrix.integration_markers }} + integration-shard-count: ${{ matrix.integration_shard_count }} + integration-xdist-workers: ${{ matrix.integration_xdist_workers }} + integration-splitting-algorithm: ${{ matrix.integration_splitting_algorithm }} + full-validation: true + unit-performance-probe: ${{ matrix.unit_performance_probe }} + integration-performance-probe: ${{ matrix.integration_performance_probe }} + performance-workers: ${{ matrix.performance_workers }} + + docs: + name: Read-only documentation build + needs: [plan] + uses: ./.github/workflows/docs-build.yml + permissions: + contents: read + with: + pages_artifact_name: rehearsal-docs-${{ github.run_attempt }} + upload_pages_artifact: true + + wheels: + name: Read-only Python distribution build + needs: [plan] + uses: ./.github/workflows/pypi-distributions.yml + permissions: + contents: read + with: + artifact_name: rehearsal-wheels-${{ github.run_attempt }} + upload_artifact: true + + compare: + name: Compare whole rehearsal critical paths + needs: [plan, source-checks, platforms, docs, wheels] + if: >- + always() && !cancelled() && needs.plan.result == 'success' && + contains(github.event.pull_request.labels.*.name, 'ci-release-rehearsal') + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + - name: Record current-attempt job timestamps and require production authorities + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RESULTS: ${{ toJSON(needs) }} + with: + script: | + const fs = require('node:fs'); + const { selectRequiredJobs } = require('./scripts/release-candidate.cjs'); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); + const params = {...context.repo, run_id: context.runId, attempt_number: attempt}; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...params, per_page: 100, + }); + const {data: run} = await github.rest.actions.getWorkflowRunAttempt(params); + fs.mkdirSync('rehearsal-results', {recursive: true}); + fs.writeFileSync('rehearsal-results/jobs.json', JSON.stringify({ + schema_version: 1, run_id: String(context.runId), run_attempt: attempt, + source_sha: context.sha, run_started_at: run.run_started_at, + created_at: run.created_at, jobs, + }, null, 2)); + const results = JSON.parse(process.env.RESULTS); + for (const key of ['source-checks', 'platforms', 'docs', 'wheels']) { + if (results[key]?.result !== 'success') throw new Error(`${key} did not succeed`); + } + const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); + selectRequiredJobs(jobs, catalog); + - name: Download current integration proofs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-evidence-${{ github.run_attempt }}-apm-* + path: performance-inputs + - name: Download current Intel unit proofs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-evidence-unit-${{ github.run_attempt }}-apm-* + path: performance-inputs + - name: Download actual ARM runner cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-cohort-integration-${{ github.run_attempt }}-apm-darwin-arm64-* + path: cohort-inputs/arm-integration + merge-multiple: true + - name: Download actual Linux runner cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-cohort-integration-${{ github.run_attempt }}-apm-linux-x86_64-* + path: cohort-inputs/linux-integration + merge-multiple: true + - name: Download actual Intel runner cohort + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: performance-cohort-unit-${{ github.run_attempt }}-apm-darwin-x86_64-* + path: cohort-inputs/intel-units + merge-multiple: true + - name: Require original outcomes and measure the entire critical path + run: >- + python -m scripts.release_rehearsal compare + --jobs rehearsal-results/jobs.json + --reports performance-inputs --cohorts cohort-inputs + --expected-sha "$GITHUB_SHA" + --minimum-lane-gain 0.15 --minimum-overall-gain 0.10 + --output rehearsal-results/comparison.json + - name: Preserve raw timestamps and the comparison, including missed targets + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-rehearsal-comparison-${{ github.run_attempt }} + path: rehearsal-results/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release-integration.yml b/.github/workflows/release-integration.yml index 0c17be8c1f..5d4274768f 100644 --- a/.github/workflows/release-integration.yml +++ b/.github/workflows/release-integration.yml @@ -60,6 +60,9 @@ on: evidence-selection: type: string default: '' + performance-cohort: + type: string + default: '' secrets: GH_CLI_PAT: required: false @@ -68,6 +71,7 @@ on: permissions: contents: read + actions: read env: PYTHON_VERSION: '3.12' @@ -116,6 +120,15 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ inputs.timing-snapshot-artifact }} + - name: Require matched actual test runners before the performance experiment + if: inputs.performance-cohort != '' + uses: ./.github/actions/performance-cohort + with: + cohort: ${{ inputs.performance-cohort }} + member: ${{ inputs.evidence-variant }}-${{ inputs.shard-index }} + source-sha: ${{ inputs.candidate-sha }} + selection: ${{ inputs.evidence-selection || format('tests/integration/ -m {0}', inputs.integration-markers) }} + candidate-metadata: release-assets/${{ inputs.binary-name }}.json - name: Compute cache week bucket if: inputs.cache-apm-fetches id: cache-bucket diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml index 009a6b5106..6dabb05e21 100644 --- a/.github/workflows/release-platform.yml +++ b/.github/workflows/release-platform.yml @@ -24,6 +24,15 @@ on: integration-splitting-algorithm: type: string default: duration_based_chunks + unit-performance-probe: + type: boolean + default: false + integration-performance-probe: + type: boolean + default: false + performance-workers: + type: number + default: 4 full-validation: type: boolean required: true @@ -39,6 +48,7 @@ on: permissions: contents: read + actions: read env: PYTHON_VERSION: '3.12' @@ -48,53 +58,16 @@ jobs: # in the build's needs list: publication, not compilation, consumes its verdict. unit-tests: name: Unit Tests - runs-on: ${{ inputs.runner }} - timeout-minutes: 65 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - name: Install dependencies - run: uv sync --frozen --extra dev - - uses: ./.github/actions/pytest-timing - with: - suite: unit - - name: Run unit tests - if: inputs.platform != 'windows' - env: - GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} - GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} - run: >- - uv run --frozen pytest tests/unit tests/test_console.py - -n auto --dist worksteal --durations=50 --store-durations - --junitxml=test-results/unit.xml - - name: Run unit tests (Windows diagnostics) - if: inputs.platform == 'windows' - timeout-minutes: 60 - env: - PYTHONUNBUFFERED: '1' - GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} - run: >- - uv run --frozen pytest tests/unit tests/test_console.py - -n auto --dist worksteal --durations=50 --store-durations - --junitxml=test-results/unit.xml - -vv --tb=short --show-capture=no --no-showlocals - - name: Upload unit timings and outcomes - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit - path: | - test-results/ - .test_durations - include-hidden-files: true - if-no-files-found: error - retention-days: 30 + uses: ./.github/workflows/release-unit.yml + with: + runner: ${{ inputs.runner }} + platform: ${{ inputs.platform }} + binary-name: ${{ inputs.binary-name }} + performance-probe: ${{ inputs.unit-performance-probe }} + proposed-workers: ${{ inputs.performance-workers }} + secrets: + GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + GH_MODELS_PAT: ${{ secrets.GH_MODELS_PAT }} build: name: Build Candidate @@ -140,11 +113,23 @@ jobs: python scripts/package_release.py pack --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" - name: Freeze integration scheduling hints - if: inputs.full-validation && inputs.platform != 'windows' + if: inputs.full-validation && inputs.platform != 'windows' && !inputs.integration-performance-probe uses: ./.github/actions/pytest-timing with: suite: integration snapshot: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} + - name: Freeze cold integration scheduling hints for performance probe + if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe + run: printf '{}\n' > .test_durations + - name: Upload cold integration scheduling hints for performance probe + if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} + path: .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 - name: Upload binary as workflow artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -193,11 +178,63 @@ jobs: xdist-workers: ${{ inputs.integration-xdist-workers }} splitting-algorithm: ${{ inputs.integration-splitting-algorithm }} timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} - artifact-prefix: ${{ github.run_attempt }}-${{ inputs.binary-name }} + artifact-prefix: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || format('{0}-{1}', github.run_attempt, inputs.binary-name) }} + runtime-prerequisites: none + evidence-variant: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && 'baseline' || '' }} + evidence-selection: tests/integration/ -m ${{ inputs.integration-markers }} + performance-cohort: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }} + secrets: + GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + + integration-tests-proposed-shard: + name: Integration Tests Proposed Shard ${{ matrix.shard }} + needs: [build] + if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe + strategy: + fail-fast: false + matrix: + shard: [1, 2] + uses: ./.github/workflows/release-integration.yml + with: + runner: ${{ inputs.runner }} + platform: ${{ inputs.platform }} + binary-name: ${{ inputs.binary-name }} + candidate-artifact-name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} + candidate-sha: ${{ github.sha }} + timing-snapshot-artifact: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} + integration-markers: ${{ inputs.integration-markers }} + shard-count: 2 + shard-index: ${{ matrix.shard }} + xdist-workers: ${{ inputs.performance-workers }} + splitting-algorithm: least_duration + timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} + artifact-prefix: ${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed runtime-prerequisites: none + evidence-variant: proposed + evidence-selection: tests/integration/ -m ${{ inputs.integration-markers }} + performance-cohort: integration-${{ github.run_attempt }}-${{ inputs.binary-name }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} + integration-performance-probe: + name: Integration Performance Probe + needs: [integration-tests-shard, integration-tests-proposed-shard] + if: always() && inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe + runs-on: ubuntu-24.04 + steps: + - name: Require every integration performance member + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RESULTS: ${{ toJSON(needs) }} + with: + script: | + const results = JSON.parse(process.env.RESULTS); + for (const name of ['integration-tests-shard', 'integration-tests-proposed-shard']) { + if (results[name]?.result !== 'success') { + core.setFailed(`${name} did not succeed`); + } + } + integration-tests: name: Integration Tests needs: [integration-tests-shard] diff --git a/.github/workflows/release-unit.yml b/.github/workflows/release-unit.yml new file mode 100644 index 0000000000..f702dde758 --- /dev/null +++ b/.github/workflows/release-unit.yml @@ -0,0 +1,226 @@ +name: Release unit shard + +on: + workflow_call: + inputs: + runner: + type: string + required: true + platform: + type: string + required: true + binary-name: + type: string + required: true + performance-probe: + type: boolean + default: false + proposed-workers: + type: number + default: 4 + secrets: + GH_CLI_PAT: + required: false + GH_MODELS_PAT: + required: false + +permissions: + contents: read + actions: read + +env: + PYTHON_VERSION: '3.12' + +jobs: + unit-tests: + name: Unit Tests Baseline + runs-on: ${{ inputs.runner }} + timeout-minutes: 65 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev + - name: Restore unit scheduling hints + if: ${{ !inputs.performance-probe }} + uses: ./.github/actions/pytest-timing + with: + suite: unit + - name: Initialize cold unit scheduling hints + if: inputs.performance-probe && inputs.platform != 'windows' + run: printf '{}\n' > .test_durations + - name: Require matched actual test runners before the unit performance experiment + if: inputs.performance-probe && inputs.platform != 'windows' + uses: ./.github/actions/performance-cohort + with: + cohort: unit-${{ github.run_attempt }}-${{ inputs.binary-name }} + member: baseline-1 + source-sha: ${{ github.sha }} + selection: tests/unit tests/test_console.py + - name: Run unit tests + if: inputs.platform != 'windows' && !inputs.performance-probe + env: + GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} + GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} + run: >- + uv run --frozen pytest tests/unit tests/test_console.py + -n auto --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit.xml + - name: Run unit performance baseline + if: inputs.platform != 'windows' && inputs.performance-probe + env: + GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} + GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} + run: >- + uv run --frozen python -m pytest -p scripts.pytest_performance_evidence + tests/unit tests/test_console.py + -n auto --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit-baseline-shard-1.xml + - name: Run unit tests (Windows diagnostics) + if: inputs.platform == 'windows' + timeout-minutes: 60 + env: + PYTHONUNBUFFERED: '1' + GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + run: >- + uv run --frozen pytest tests/unit tests/test_console.py + -n auto --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit.xml + -vv --tb=short --show-capture=no --no-showlocals + - name: Capture unit performance evidence + if: always() && inputs.platform != 'windows' && inputs.performance-probe + run: | + mkdir -p performance-evidence + python -m scripts.compare_test_runs capture \ + --junit test-results/unit-baseline-shard-1.xml \ + --output performance-evidence/baseline-shard-1.json \ + --source-sha "${{ github.sha }}" \ + --selection "tests/unit tests/test_console.py" \ + --variant baseline \ + --shard 1 \ + --shard-count 1 + - name: Upload unit timings and outcomes + if: always() && !inputs.performance-probe + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit + path: | + test-results/ + .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Upload unit performance timings and outcomes + if: always() && inputs.platform != 'windows' && inputs.performance-probe + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-baseline-shard-1 + path: | + test-results/ + .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Upload unit performance evidence + if: always() && inputs.platform != 'windows' && inputs.performance-probe + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-baseline-shard-1 + path: performance-evidence/ + if-no-files-found: error + retention-days: 30 + + unit-tests-proposed: + name: Unit Tests Proposed Shard ${{ matrix.shard }} + if: inputs.performance-probe && inputs.platform != 'windows' + runs-on: ${{ inputs.runner }} + timeout-minutes: 65 + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev + - name: Initialize cold unit scheduling hints + run: printf '{}\n' > .test_durations + - name: Require matched actual test runners before the unit performance experiment + uses: ./.github/actions/performance-cohort + with: + cohort: unit-${{ github.run_attempt }}-${{ inputs.binary-name }} + member: proposed-${{ matrix.shard }} + source-sha: ${{ github.sha }} + selection: tests/unit tests/test_console.py + - name: Run unit performance proposed shard + env: + GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} + GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} + run: >- + uv run --frozen python -m pytest -p scripts.pytest_performance_evidence + tests/unit tests/test_console.py + --splits 2 --group ${{ matrix.shard }} --splitting-algorithm least_duration + -n ${{ inputs.proposed-workers }} --dist worksteal --durations=50 --store-durations + --junitxml=test-results/unit-proposed-shard-${{ matrix.shard }}.xml + - name: Capture unit performance evidence + if: always() + run: | + mkdir -p performance-evidence + python -m scripts.compare_test_runs capture \ + --junit "test-results/unit-proposed-shard-${{ matrix.shard }}.xml" \ + --output "performance-evidence/proposed-shard-${{ matrix.shard }}.json" \ + --source-sha "${{ github.sha }}" \ + --selection "tests/unit tests/test_console.py" \ + --variant proposed \ + --shard "${{ matrix.shard }}" \ + --shard-count 2 + - name: Upload unit performance timings and outcomes + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed-shard-${{ matrix.shard }} + path: | + test-results/ + .test_durations + include-hidden-files: true + if-no-files-found: error + retention-days: 30 + - name: Upload unit performance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed-shard-${{ matrix.shard }} + path: performance-evidence/ + if-no-files-found: error + retention-days: 30 + + unit-performance-probe: + name: Unit Performance Probe + needs: [unit-tests, unit-tests-proposed] + if: always() && inputs.performance-probe && inputs.platform != 'windows' + runs-on: ubuntu-24.04 + steps: + - name: Require every unit performance member + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RESULTS: ${{ toJSON(needs) }} + with: + script: | + const results = JSON.parse(process.env.RESULTS); + for (const name of ['unit-tests', 'unit-tests-proposed']) { + if (results[name]?.result !== 'success') { + core.setFailed(`${name} did not succeed`); + } + } diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 2858e90f4e..19ff9ffbec 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -585,6 +585,46 @@ job timestamps to include setup and queue delays. Remove the opt-in label before unrelated pushes; these measurements are evidence, not required release gates or cached pass results. +### Whole-pipeline rehearsal + +For whole-pipeline release rehearsal, apply the `ci-release-rehearsal` PR label. +`ci-release-rehearsal.yml` reuses the five native release-platform workflows, +canonical source CI, and read-only docs/wheel builds at one current PR merge +SHA, with no PATs, signing secrets, publishing, or production topology change: + +| Surface | Current shape | Rehearsal shape | +| --- | --- | --- | +| macOS ARM integration | 1 shard x 4 workers | 2 shards x 3 workers | +| Linux x64 integration | 1 shard x 4 workers | 2 shards x 4 workers | +| macOS Intel unit | 1 shard x auto workers | 2 shards x 4 workers | + +Each actual runner records image, Python, OS, CPU, lockfile, source, candidate +identity, runner trace, and timestamp proof, then waits up to 15 minutes for +all three matching cohort members before pytest. Missing, stale, or mismatched proof stops before +expensive tests. Integration pairs use exact archive bytes from the same build; +source-unit pairs use the identical SHA; final comparison still requires exact +original case/outcome parity. + +The comparison retains all five production +`Integration Tests` fan-ins and `Native Candidate Gate` jobs, measures each +observed dependency-to-verdict delay, and propagates the selected variant +through the per-platform DAG while unchanged platforms keep their actual gate +completion floor. Proposed native-gate completion timestamps are modeled, not +independently measured; report that assumption. Only experiment-specific +both-world checkers, such as `Intel Unit Performance Probe`, are excluded. +Qualification recording, asset verification, signing, and publishing remain +outside the unprivileged measurement. Read `release-rehearsal-comparison-`, +`performance-evidence*`, and `performance-cohort*`. + +Thresholds are 15% per +paired pytest lane and 10% overall evidence-ready improvement, with 30% as the +target. Do not treat workflow duration as proposed release time: it waits for +both variants, and experimental concurrency/cohort waits can add allocation +cost. Remove the label before unrelated pushes. On failed attempts use **Re-run +all jobs**; artifacts are immutable and attempt-scoped. Limit an evaluation to +three full attempts, including image-inconclusive attempts. Production rollout +requires passing proof plus a maintainer decision. + ## Debugging Test Failures ### Smoke Test Failures diff --git a/scripts/compare_test_runs.py b/scripts/compare_test_runs.py index e20940d9d3..6aa1af08a7 100644 --- a/scripts/compare_test_runs.py +++ b/scripts/compare_test_runs.py @@ -108,6 +108,24 @@ def _identity( } +def capture_identity( + source_sha: str, selection: str, candidate_metadata: Path | None = None +) -> dict[str, object]: + """Capture the same identity before execution and in the final test report.""" + candidate = json.loads(candidate_metadata.read_text("utf-8")) if candidate_metadata else None + return _identity(source_sha, selection, candidate, _environment()) + + +def validate_identity(identity: dict[str, object]) -> dict[str, object]: + """Validate an already captured identity without replacing its environment.""" + return _identity( + identity["source_sha"], + identity["selection"], + identity["candidate"], + identity["environment"], + ) + + def _positive_seconds(value: object) -> float: if ( isinstance(value, bool) @@ -137,8 +155,7 @@ def capture( ) -> Report: """Read one real pytest JUnit report without equating skips with passes.""" _shard_identity(variant, shard, shard_count) - candidate = json.loads(candidate_metadata.read_text("utf-8")) if candidate_metadata else None - identity = _identity(source_sha, selection, candidate, _environment()) + identity = capture_identity(source_sha, selection, candidate_metadata) parser = ET.XMLParser(target=_JUnitTreeBuilder()) # noqa: S314 - rejects all DTD declarations. root = ET.parse(junit, parser=parser).getroot() # noqa: S314 - DTDs rejected before expansion. suites = [root] if root.tag == "testsuite" else list(root) @@ -186,13 +203,7 @@ def read_report(path: Path) -> Report: raise ValueError(f"Invalid report schema: {path}") if data["schema_version"] != 1 or not isinstance(data.get("identity"), dict): raise ValueError(f"Invalid report identity: {path}") - raw_identity = data["identity"] - identity = _identity( - raw_identity["source_sha"], - raw_identity["selection"], - raw_identity["candidate"], - raw_identity["environment"], - ) + identity = validate_identity(data["identity"]) _shard_identity(data["variant"], data["shard"], data["shard_count"]) elapsed = _positive_seconds(data["elapsed_seconds"]) entries = data["cases"] diff --git a/scripts/performance-cohort.cjs b/scripts/performance-cohort.cjs new file mode 100644 index 0000000000..0c9df89391 --- /dev/null +++ b/scripts/performance-cohort.cjs @@ -0,0 +1,35 @@ +'use strict'; + +const MEMBERS = ['baseline-1', 'proposed-1', 'proposed-2']; + +async function waitForCohort({ + github, context, cohort, attempt, + timeoutMs = 900000, intervalMs = 20000, + now = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), +}) { + if (!Number.isSafeInteger(attempt) || attempt < 1 || + !new RegExp(`^(unit|integration)-${attempt}-apm-[a-z0-9]+-[a-z0-9_]+$`).test(cohort) || + !Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 900000 || + !Number.isFinite(intervalMs) || intervalMs <= 0) { + throw new Error('Invalid bounded performance cohort'); + } + const expected = new Set(MEMBERS.map(member => `performance-cohort-${cohort}-${member}`)); + const deadline = now() + timeoutMs; + while (now() < deadline) { + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const members = artifacts.filter(artifact => expected.has(artifact.name)); + if (members.some(artifact => artifact.expired) || + new Set(members.map(artifact => artifact.name)).size !== members.length) { + throw new Error('Expired or duplicate performance cohort artifact'); + } + if (members.length === expected.size) { + return; + } + await sleep(Math.min(intervalMs, Math.max(0, deadline - now()))); + } + throw new Error('Performance cohort timed out before all actual test runners were ready'); +} + +module.exports = {waitForCohort}; diff --git a/scripts/performance-cohort.test.cjs b/scripts/performance-cohort.test.cjs new file mode 100644 index 0000000000..1f1df20859 --- /dev/null +++ b/scripts/performance-cohort.test.cjs @@ -0,0 +1,72 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const {waitForCohort} = require('./performance-cohort.cjs'); + +const cohort = 'integration-2-apm-darwin-arm64'; +const names = ['baseline-1', 'proposed-1', 'proposed-2'] + .map(member => `performance-cohort-${cohort}-${member}`); + +function harness(responses) { + let clock = 0; + let calls = 0; + const github = { + rest: {actions: {listWorkflowRunArtifacts: 'list'}}, + paginate: async (method, params) => { + assert.equal(method, 'list'); + assert.deepEqual(params, {owner: 'owner', repo: 'repo', run_id: 123, per_page: 100}); + return responses[Math.min(calls++, responses.length - 1)]; + }, + }; + return { + options: { + github, context: {repo: {owner: 'owner', repo: 'repo'}, runId: 123}, + cohort, attempt: 2, timeoutMs: 100, intervalMs: 20, + now: () => clock, sleep: async ms => { clock += ms; }, + }, + calls: () => calls, + }; +} + +test('waits for the exact current-attempt trio, ignoring old attempts', async () => { + const current = names.map(name => ({name, expired: false})); + const state = harness([ + [{name: names[0].replace('-2-', '-1-'), expired: false}, current[0]], + current, + ]); + await waitForCohort(state.options); + assert.equal(state.calls(), 2); +}); + +test('missing members time out instead of allowing incomparable tests', async () => { + const state = harness([[{name: names[0], expired: false}]]); + await assert.rejects(waitForCohort(state.options), /timed out/); + assert.equal(state.calls(), 5); +}); + +for (const fault of ['duplicate', 'expired']) { + test(`rejects ${fault} artifact evidence`, async () => { + const members = names.map(name => ({name, expired: fault === 'expired'})); + if (fault === 'duplicate') members.push(members[0]); + const state = harness([members]); + await assert.rejects(waitForCohort(state.options), /Expired or duplicate/); + }); +} + +test('API failures propagate rather than pretending a cohort is ready', async () => { + const state = harness([]); + state.options.github.paginate = async () => { throw new Error('API unavailable'); }; + await assert.rejects(waitForCohort(state.options), /API unavailable/); +}); + +for (const change of [ + {attempt: 1}, {cohort: '../../wrong'}, {timeoutMs: Infinity}, + {timeoutMs: 900001}, {intervalMs: 0}, +]) { + test(`rejects invalid wait input ${JSON.stringify(change)}`, async () => { + const state = harness([]); + await assert.rejects(waitForCohort({...state.options, ...change}), /Invalid bounded/); + assert.equal(state.calls(), 0); + }); +} diff --git a/scripts/performance_cohort.py b/scripts/performance_cohort.py new file mode 100644 index 0000000000..a3f4d689be --- /dev/null +++ b/scripts/performance_cohort.py @@ -0,0 +1,150 @@ +"""Fail before expensive tests unless the actual paired runners have equal identities.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from datetime import UTC, datetime +from pathlib import Path + +from scripts.compare_test_runs import capture_identity, validate_identity +from scripts.package_release import require_sha + +MEMBERS = ("baseline-1", "proposed-1", "proposed-2") + + +def require_context(cohort: str, run_id: str, attempt: int) -> None: + """Restrict artifact names and bind them to one complete workflow attempt.""" + if not isinstance(run_id, str) or re.fullmatch(r"[1-9][0-9]*", run_id) is None: + raise ValueError("A positive workflow run ID is required") + if type(attempt) is not int or attempt < 1: + raise ValueError("A positive workflow attempt is required") + if ( + not isinstance(cohort, str) + or re.fullmatch(rf"(unit|integration)-{attempt}-apm-[a-z0-9]+-[a-z0-9_]+", cohort) is None + ): + raise ValueError("Cohort must identify a suite, current attempt and native binary") + + +def record( + cohort: str, + member: str, + run_id: str, + attempt: int, + source_sha: str, + selection: str, + candidate_metadata: Path | None = None, +) -> dict[str, object]: + """Record identity on the runner that will execute this exact test partition.""" + require_context(cohort, run_id, attempt) + if member not in MEMBERS: + raise ValueError("Unknown cohort member") + identity = capture_identity(source_sha, selection, candidate_metadata) + if not identity["environment"]["runner_image"]: + raise ValueError("Hosted runner ImageVersion is required for a controlled cohort") + if cohort.startswith("integration-") != (identity["candidate"] is not None): + raise ValueError("Only integration cohorts require candidate archive metadata") + runner_name = os.environ.get("RUNNER_NAME", "") + workflow_job = os.environ.get("GITHUB_JOB", "") + if not runner_name or not workflow_job: + raise ValueError("Actual GitHub runner and workflow job identity are required") + return { + "schema_version": 1, + "cohort": cohort, + "member": member, + "run_id": run_id, + "run_attempt": attempt, + "identity": identity, + "execution": { + "runner_name": runner_name, + "workflow_job": workflow_job, + "recorded_at": datetime.now(UTC).isoformat(), + }, + } + + +def verify( + root: Path, cohort: str, run_id: str, attempt: int, source_sha: str +) -> dict[str, object]: + """Reject missing, duplicate, stale, malformed or environmentally different members.""" + require_context(cohort, run_id, attempt) + require_sha(source_sha) + if {path.name for path in root.iterdir()} != {f"{member}.json" for member in MEMBERS}: + raise ValueError("Exactly the three declared cohort member files are required") + identities = [] + for member in MEMBERS: + data = json.loads((root / f"{member}.json").read_text("ascii")) + if ( + not isinstance(data, dict) + or type(data.get("schema_version")) is not int + or data["schema_version"] != 1 + or data.get("cohort") != cohort + or data.get("member") != member + or data.get("run_id") != run_id + or type(data.get("run_attempt")) is not int + or data["run_attempt"] != attempt + or not isinstance(data.get("identity"), dict) + ): + raise ValueError(f"Invalid or stale cohort evidence for {member}") + identity = validate_identity(data["identity"]) + if identity["source_sha"] != source_sha or not identity["environment"]["runner_image"]: + raise ValueError("Cohort source and hosted image must be explicit") + if cohort.startswith("integration-") != (identity["candidate"] is not None): + raise ValueError("Cohort suite does not match its candidate evidence") + execution = data.get("execution") + if ( + not isinstance(execution, dict) + or any( + not isinstance(execution.get(key), str) or not execution[key] + for key in ("runner_name", "workflow_job", "recorded_at") + ) + or datetime.fromisoformat(execution["recorded_at"]).tzinfo is None + ): + raise ValueError("Cohort must bind its actual runner and timestamp to the job timeline") + identities.append(identity) + if any(identity != identities[0] for identity in identities[1:]): + raise ValueError("Cohort identity mismatch; do not run or claim comparable performance") + return identities[0] + + +def main() -> int: + """Capture or validate a pre-execution cohort without producing a pass substitute.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + capture = commands.add_parser("record") + capture.add_argument("--member", choices=MEMBERS, required=True) + capture.add_argument("--selection", required=True) + capture.add_argument("--candidate-metadata", type=Path) + capture.add_argument("--output", type=Path, required=True) + check = commands.add_parser("verify") + check.add_argument("--root", type=Path, required=True) + for command in (capture, check): + command.add_argument("--cohort", required=True) + command.add_argument("--run-id", required=True) + command.add_argument("--attempt", type=int, required=True) + command.add_argument("--source-sha", required=True) + args = parser.parse_args() + try: + if args.operation == "record": + data = record( + args.cohort, + args.member, + args.run_id, + args.attempt, + args.source_sha, + args.selection, + args.candidate_metadata, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", "ascii") + else: + verify(args.root, args.cohort, args.run_id, args.attempt, args.source_sha) + except (ValueError, KeyError, TypeError, OSError) as error: + parser.error(f"Invalid performance cohort: {error}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index 2bfba03844..42eb1f73ed 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -15,6 +15,7 @@ const RERUN_ALL_JOBS_HINT = "Use Re-run all jobs to regenerate complete candidat const CONFIG_HASH_FILES = [ ".github/workflows/build-release.yml", ".github/workflows/release-platform.yml", + ".github/workflows/release-unit.yml", ".github/workflows/release-integration.yml", ".github/workflows/docs-build.yml", ".github/workflows/pypi-distributions.yml", diff --git a/scripts/release_rehearsal.py b/scripts/release_rehearsal.py new file mode 100644 index 0000000000..b350ce7730 --- /dev/null +++ b/scripts/release_rehearsal.py @@ -0,0 +1,479 @@ +"""Compare a complete read-only release rehearsal without adding parallel job savings.""" + +from __future__ import annotations + +import argparse +import json +import math +import shutil +import subprocess +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +from scripts.compare_test_runs import Report, compare, read_report +from scripts.package_release import BINARY_NAMES, require_sha +from scripts.performance_cohort import MEMBERS, verify + +ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class Surface: + """One predeclared paired surface; ordinary native obligations remain unchanged.""" + + key: str + binary: str + suite: str + cpus: str + selection: str + + +SURFACES = ( + Surface( + "arm-integration", "apm-darwin-arm64", "integration", "3", "tests/integration/ -m not live" + ), + Surface( + "linux-integration", + "apm-linux-x86_64", + "integration", + "4", + "tests/integration/ -m not live", + ), + Surface("intel-units", "apm-darwin-x86_64", "unit", "4", "tests/unit tests/test_console.py"), +) + + +def rehearsal_matrix(catalog: list[dict]) -> dict[str, list[dict]]: + """Add opt-in comparison inputs without changing the canonical baseline catalog.""" + if len(catalog) != 5 or {row["binary_name"] for row in catalog} != set(BINARY_NAMES): + raise ValueError("A rehearsal must retain all five unique native platforms") + rows = [] + for row in catalog: + if ( + row["integration_shard_count"] != 1 + or row["integration_xdist_workers"] != 4 + or row["integration_splitting_algorithm"] != "duration_based_chunks" + ): + raise ValueError("The predeclared rehearsal baseline is one four-worker native shard") + binary = row["binary_name"] + rows.append( + { + **row, + "unit_performance_probe": binary == "apm-darwin-x86_64", + "integration_performance_probe": binary in {"apm-darwin-arm64", "apm-linux-x86_64"}, + "performance_workers": 3 if binary == "apm-darwin-arm64" else 4, + } + ) + return {"include": rows} + + +def _timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise ValueError("Explicit timezone-aware job timestamps are required") + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + if result.tzinfo is None: + raise ValueError("Job timestamps must include a timezone") + return result + + +def _job_times(job: dict, origin: datetime) -> tuple[datetime, datetime]: + if job.get("status") != "completed" or job.get("conclusion") != "success": + raise ValueError(f"Required job did not succeed: {job.get('name')}") + start, end = _timestamp(job.get("started_at")), _timestamp(job.get("completed_at")) + if start < origin or end < start: + raise ValueError(f"Invalid required job timeline: {job.get('name')}") + return start, end + + +def _one(jobs: list[dict], name: str, origin: datetime, prefix: str | None = None) -> dict: + matches = [ + job + for job in jobs + if job.get("name") == name + or ( + prefix is not None + and isinstance(job.get("name"), str) + and job["name"].startswith(prefix + " / ") + and job["name"].endswith(" / " + name) + ) + ] + if len(matches) != 1: + raise ValueError(f"Expected exactly one required job: {prefix or ''} / {name}") + _job_times(matches[0], origin) + return matches[0] + + +def source_authorities() -> list[str]: + """Ask the existing release authority for its exact source checks, never fork the list.""" + node = shutil.which("node") + if node is None: + raise OSError("Node.js is required to read the canonical release authorities") + result = subprocess.run( # noqa: S603 - installed Node and a constant repository authority module. + [ + node, + "-e", + "process.stdout.write(JSON.stringify(require('./scripts/release-candidate.cjs').REQUIRED_SOURCE_JOB_NAMES))", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + timeout=30, + ) + names = json.loads(result.stdout) + if not isinstance(names, list) or not names or any(not isinstance(name, str) for name in names): + raise ValueError("Canonical source authorities must be a nonempty name list") + return names + + +def _common_jobs(jobs: list[dict], origin: datetime, source_names: list[str]) -> list[dict]: + common = [_one(jobs, name, origin) for name in source_names] + for binary in BINARY_NAMES: + _one(jobs, f"{binary} / Native Candidate Gate", origin) + common.append(_one(jobs, "Build Candidate", origin, binary)) + common.append(_one(jobs, "Isolated Release Validation", origin, binary)) + if binary != "apm-darwin-x86_64": + common.append(_one(jobs, "Unit Tests Baseline", origin, binary)) + if binary not in {"apm-darwin-arm64", "apm-linux-x86_64"}: + common.append(_one(jobs, "Integration Tests Shard 1", origin, binary)) + if binary == "apm-windows-x86_64": + common.append(_one(jobs, "Candidate Windows Installer", origin, binary)) + common.extend( + [ + _one(jobs, "Read-only documentation build / build", origin), + _one(jobs, "Read-only Python distribution build / Build PyPI Distributions", origin), + ] + ) + if len({job["id"] for job in common}) != len(common): + raise ValueError("A required job was counted twice") + return common + + +def _bind_member( + root: Path, member: str, jobs: list[dict], surface: Surface, origin: datetime +) -> dict: + data = json.loads((root / f"{member}.json").read_text("ascii")) + trace = data["execution"] + runner = trace["runner_name"] + recorded = _timestamp(trace["recorded_at"]) + if not isinstance(runner, str) or not runner: + raise ValueError("Cohort must identify its actual GitHub runner") + matches = [] + for job in jobs: + if ( + job.get("runner_name") == runner + and isinstance(job.get("name"), str) + and job["name"].startswith(surface.binary + " / ") + and job.get("status") == "completed" + ): + start, end = _job_times(job, origin) + if start <= recorded <= end: + matches.append(job) + if len(matches) != 1: + raise ValueError(f"Cannot uniquely bind {surface.key}/{member} to its actual native job") + return matches[0] + + +def _surface_for(report: Report) -> Surface: + candidate = report.identity["candidate"] + for surface in SURFACES: + if report.identity["selection"] == surface.selection and ( + (candidate is None and surface.suite == "unit") + or (isinstance(candidate, dict) and candidate.get("binary_name") == surface.binary) + ): + return surface + raise ValueError("Unexpected performance evidence outside the predeclared three surfaces") + + +def _native_authorities( + jobs: list[dict], pairs: dict, origin: datetime +) -> tuple[list[dict], list[dict]]: + """Retain native gates, propagating their observed dependency-to-verdict delays.""" + authorities, gate_jobs = [], [] + + def end(parts: list[dict]) -> float: + return max((_job_times(job, origin)[1] - origin).total_seconds() for job in parts) + + def delay(gate: dict, prerequisites: list[dict]) -> float: + value = end([gate]) - end(prerequisites) + if value < 0: + raise ValueError(f"Required gate completed before its prerequisites: {gate['name']}") + return value + + for binary in sorted(BINARY_NAMES): + surfaces = {surface.suite: surface.key for surface in SURFACES if surface.binary == binary} + unit_key, integration_key = surfaces.get("unit"), surfaces.get("integration") + unit = ( + pairs[unit_key] + if unit_key + else { + "baseline": [_one(jobs, "Unit Tests Baseline", origin, binary)], + } + ) + integration = ( + pairs[integration_key] + if integration_key + else { + "baseline": [_one(jobs, "Integration Tests Shard 1", origin, binary)], + } + ) + integration_gate = _one(jobs, "Integration Tests", origin, binary) + integration_delay = delay(integration_gate, integration["baseline"]) + observed_unit = unit["baseline"] + if unit_key: + # This extra checker exists only to join both experimental unit worlds. + unit_probe = _one(jobs, "Unit Performance Probe", origin, binary) + delay(unit_probe, [*unit["baseline"], *unit["proposed"]]) + observed_unit = [unit_probe] + independent = [ + _one(jobs, "Build Candidate", origin, binary), + _one(jobs, "Isolated Release Validation", origin, binary), + ] + if binary == "apm-windows-x86_64": + independent.append(_one(jobs, "Candidate Windows Installer", origin, binary)) + gate = _one(jobs, f"{binary} / Native Candidate Gate", origin) + native_delay = delay(gate, [*independent, *observed_unit, integration_gate]) + gate_jobs.extend([integration_gate, gate]) + authorities.append( + { + "name": gate["name"], + "observed_completed_at": gate["completed_at"], + "independent_ready_seconds": end(independent), + "unit_surface": unit_key, + "integration_surface": integration_key, + "unit_ready_seconds": {variant: end(parts) for variant, parts in unit.items()}, + "integration_ready_seconds": { + variant: end(parts) + integration_delay + for variant, parts in integration.items() + }, + "integration_fan_in_delay_seconds": integration_delay, + "native_gate_delay_seconds": native_delay, + } + ) + return authorities, gate_jobs + + +def _scenario( + common: list[dict], + selected: list[dict], + origin: datetime, + authorities: list[dict], + gate_jobs: list[dict], + enabled: set[str], +) -> dict: + jobs = [*common, *selected] + end_job = max(jobs, key=lambda job: _job_times(job, origin)[1]) + ready = (_job_times(end_job, origin)[1] - origin).total_seconds() + limiting = end_job["name"] + native = [] + for authority in authorities: + unit_variant = "proposed" if authority["unit_surface"] in enabled else "baseline" + integration_variant = ( + "proposed" if authority["integration_surface"] in enabled else "baseline" + ) + completed = ( + max( + authority["independent_ready_seconds"], + authority["unit_ready_seconds"][unit_variant], + authority["integration_ready_seconds"][integration_variant], + ) + + authority["native_gate_delay_seconds"] + ) + native.append({**authority, "modeled_ready_seconds": completed}) + if completed >= ready: + ready, limiting = completed, authority["name"] + return { + "evidence_ready_seconds": ready, + "limiting_job": limiting, + "native_authorities": native, + "selected_runner_seconds": sum( + (end - start).total_seconds() + for start, end in (_job_times(job, origin) for job in [*jobs, *gate_jobs]) + ), + } + + +def evaluate( + metadata: dict, + reports: list[Report], + cohorts: Path, + expected_sha: str, + source_names: list[str], + minimum_lane_gain: float, + minimum_overall_gain: float, +) -> dict: + """Reconstruct complete evidence-ready paths, preserving all common and paired work.""" + require_sha(expected_sha) + for threshold in (minimum_lane_gain, minimum_overall_gain): + if not math.isfinite(threshold) or not 0 <= threshold < 1: + raise ValueError("Performance thresholds must be finite fractions in [0, 1)") + if ( + type(metadata.get("schema_version")) is not int + or metadata["schema_version"] != 1 + or metadata.get("source_sha") != expected_sha + or type(metadata.get("run_attempt")) is not int + or metadata["run_attempt"] < 1 + ): + raise ValueError("Current exact-SHA workflow attempt metadata is required") + run_id, attempt = metadata["run_id"], metadata["run_attempt"] + origin = _timestamp(metadata["created_at"] if attempt == 1 else metadata["run_started_at"]) + jobs = metadata["jobs"] + if not isinstance(jobs, list) or not jobs or len({job["id"] for job in jobs}) != len(jobs): + raise ValueError("A complete unique current-attempt job inventory is required") + if any(job.get("run_id") != int(run_id) for job in jobs): + raise ValueError("A job belongs to another workflow run") + if any(job.get("run_attempt") != attempt for job in jobs): + raise ValueError("A job belongs to another workflow attempt") + common = _common_jobs(jobs, origin, source_names) + grouped = {surface.key: [] for surface in SURFACES} + for report in reports: + grouped[_surface_for(report).key].append(report) + pairs, comparisons = {}, {} + bound_ids = set() + for surface in SURFACES: + subset = grouped[surface.key] + baseline = [report for report in subset if report.variant == "baseline"] + proposed = [report for report in subset if report.variant == "proposed"] + if len(baseline) != 1 or baseline[0].shard_count != 1 or len(proposed) != 2: + raise ValueError(f"Expected the exact one-versus-two experiment for {surface.key}") + result = compare(baseline, proposed, expected_sha, minimum_lane_gain) + if result["identity"]["environment"]["cpu_count"] != surface.cpus: + raise ValueError(f"Unexpected runner capacity for {surface.key}") + root = cohorts / surface.key + before = verify( + root, f"{surface.suite}-{attempt}-{surface.binary}", run_id, attempt, expected_sha + ) + if before != result["identity"]: + raise ValueError( + f"Identity drifted between preflight and completed tests: {surface.key}" + ) + bound = [_bind_member(root, member, jobs, surface, origin) for member in MEMBERS] + for member, job in zip(MEMBERS, bound, strict=True): + report = next(item for item in subset if f"{item.variant}-{item.shard}" == member) + start, end = _job_times(job, origin) + if report.elapsed_seconds > (end - start).total_seconds() + 2: + raise ValueError( + f"Pytest timing exceeds its actual job span: {surface.key}/{member}" + ) + if bound_ids & {job["id"] for job in bound} or len({job["id"] for job in bound}) != 3: + raise ValueError("Paired partitions must identify distinct actual execution jobs") + bound_ids.update(job["id"] for job in bound) + pairs[surface.key] = {"baseline": bound[:1], "proposed": bound[1:]} + comparisons[surface.key] = { + **result, + "actual_jobs": [ + { + "id": job["id"], + "name": job["name"], + "runner_name": job["runner_name"], + "started_at": job["started_at"], + "completed_at": job["completed_at"], + "cohort_steps": [ + step + for step in job.get("steps", []) + if "cohort" in step.get("name", "").lower() + or "matched actual" in step.get("name", "").lower() + ], + } + for job in bound + ], + } + if bound_ids & {job["id"] for job in common}: + raise ValueError("A paired execution was also counted as unchanged work") + authorities, gate_jobs = _native_authorities(jobs, pairs, origin) + scenarios = {} + proposals = { + "baseline": set(), + "arm_only": {"arm-integration"}, + "arm_and_linux": {"arm-integration", "linux-integration"}, + "combined": set(pairs), + } + for name, enabled in proposals.items(): + selected = [ + job + for key, pair in pairs.items() + for job in pair["proposed" if key in enabled else "baseline"] + ] + scenarios[name] = _scenario(common, selected, origin, authorities, gate_jobs, enabled) + before = scenarios["baseline"]["evidence_ready_seconds"] + after = scenarios["combined"]["evidence_ready_seconds"] + if before <= 0: + raise ValueError("Baseline must contain positive observed elapsed time") + gain = 1 - after / before + meets_lanes = all(result["meets_threshold"] for result in comparisons.values()) + return { + "schema_version": 1, + "run_id": run_id, + "run_attempt": attempt, + "source_sha": expected_sha, + "scope": "Job-timestamp reconstruction with observed native fan-in delays propagated " + "through each proposed DAG; " + "not actual production release or the all-variants rehearsal workflow duration", + "origin": origin.isoformat(), + "origin_kind": "workflow creation" if attempt == 1 else "attempt start", + "common_job_count": len(common), + "common_jobs": [job["name"] for job in common], + "comparisons": comparisons, + "scenarios": scenarios, + "saved_seconds": before - after, + "overall_gain_fraction": gain, + "minimum_overall_gain": minimum_overall_gain, + "meets_lane_thresholds": meets_lanes, + "meets_minimum": meets_lanes and gain >= minimum_overall_gain, + "reaches_30_percent_target": meets_lanes and gain >= 0.30, + "unchanged_tail_sensitivity": [ + {"tail_seconds": tail, "gain_fraction": (before - after) / (before + tail)} + for tail in (0, 60, 120, 180) + ], + "limitations": [ + "Secret-bearing acceptance and Windows signing are not represented.", + "Native integration/native-gate delays are observed then held constant across modeled worlds; " + "their proposed completion timestamps are not independently measured.", + "Only experiment-specific both-world fan-ins are excluded; production native gates are retained.", + "Recording qualification, verifying publication assets and publishing are excluded.", + "Both worlds share one run; extra experimental concurrency and cohort waits can affect allocation.", + "JUnit case durations include fixtures and waiting, not fixed CPU work.", + ], + } + + +def main() -> int: + """Emit a matrix or durable strict rehearsal result; missed targets remain visible.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + commands.add_parser("matrix") + check = commands.add_parser("compare") + check.add_argument("--jobs", type=Path, required=True) + check.add_argument("--reports", type=Path, required=True) + check.add_argument("--cohorts", type=Path, required=True) + check.add_argument("--expected-sha", required=True) + check.add_argument("--minimum-lane-gain", type=float, default=0.15) + check.add_argument("--minimum-overall-gain", type=float, default=0.10) + check.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.operation == "matrix": + catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) + print("matrix=" + json.dumps(rehearsal_matrix(catalog), separators=(",", ":"))) + return 0 + result = evaluate( + json.loads(args.jobs.read_text("utf-8")), + [read_report(path) for path in sorted(args.reports.rglob("*.json"))], + args.cohorts, + args.expected_sha, + source_authorities(), + args.minimum_lane_gain, + args.minimum_overall_gain, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", "ascii") + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as error: + parser.error(f"Invalid release rehearsal: {error}") + if not result["meets_minimum"]: + parser.exit(1, f"Rehearsal target missed; inspect {args.output}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/scripts/test_performance_cohort.py b/tests/unit/scripts/test_performance_cohort.py new file mode 100644 index 0000000000..82516fc49f --- /dev/null +++ b/tests/unit/scripts/test_performance_cohort.py @@ -0,0 +1,179 @@ +"""Preflight matching must not let a mismatched runner spend time on the full suite.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from scripts import performance_cohort + +pytestmark = pytest.mark.component +SHA = "a" * 40 +COHORT = "unit-2-apm-darwin-x86_64" +ROOT = Path(__file__).resolve().parents[3] + + +@pytest.fixture +def identity(monkeypatch: pytest.MonkeyPatch) -> dict: + value = { + "source_sha": SHA, + "selection": "tests/unit tests/test_console.py", + "candidate": None, + "environment": { + "python": "3.12.10", + "system": "Darwin", + "release": "24.6.0", + "machine": "x86_64", + "cpu_count": "4", + "runner_image": "20260831.1", + "dependency_lock": "f" * 64, + }, + } + monkeypatch.setenv("RUNNER_NAME", "GitHub Actions test") + monkeypatch.setenv("GITHUB_JOB", "unit-tests") + monkeypatch.setattr(performance_cohort, "capture_identity", lambda *args: value) + return value + + +def _members(root: Path) -> None: + for member in performance_cohort.MEMBERS: + data = performance_cohort.record( + COHORT, + member, + "123", + 2, + SHA, + "tests/unit tests/test_console.py", + ) + (root / f"{member}.json").write_text(json.dumps(data), encoding="ascii") + + +def test_matching_actual_runners_produce_the_identity_used_by_final_proof( + tmp_path: Path, + identity: dict, +) -> None: + _members(tmp_path) + assert performance_cohort.verify(tmp_path, COHORT, "123", 2, SHA) == identity + + +@pytest.mark.parametrize( + "fault", + [ + "missing", + "extra", + "member", + "run", + "attempt", + "schema", + "sha", + "image", + "empty-image", + "lock", + "selection", + "candidate", + "malformed", + "missing-runner", + "missing-job", + "invalid-time", + "naive-time", + ], +) +def test_preflight_rejects_incomplete_stale_or_incomparable_cohorts( + tmp_path: Path, + identity: dict, + fault: str, +) -> None: + _members(tmp_path) + path = tmp_path / "proposed-2.json" + data = json.loads(path.read_text("ascii")) + if fault == "missing": + path.unlink() + elif fault == "extra": + (tmp_path / "duplicate.json").write_text(json.dumps(data), encoding="ascii") + elif fault == "malformed": + path.write_text("not json", encoding="ascii") + else: + if fault == "member": + data["member"] = "baseline-1" + elif fault == "run": + data["run_id"] = "122" + elif fault == "attempt": + data["run_attempt"] = 1 + elif fault == "schema": + data["schema_version"] = True + elif fault == "sha": + data["identity"]["source_sha"] = "b" * 40 + elif fault == "image": + data["identity"]["environment"]["runner_image"] = "different-image" + elif fault == "empty-image": + data["identity"]["environment"]["runner_image"] = "" + elif fault == "lock": + data["identity"]["environment"]["dependency_lock"] = "b" * 64 + elif fault == "selection": + data["identity"]["selection"] = "fewer tests" + elif fault == "missing-runner": + data["execution"]["runner_name"] = "" + elif fault == "missing-job": + data["execution"].pop("workflow_job") + elif fault == "invalid-time": + data["execution"]["recorded_at"] = "not a timestamp" + elif fault == "naive-time": + data["execution"]["recorded_at"] = "2026-09-07T12:00:00" + else: + data["identity"]["candidate"] = {"schema_version": 0} + path.write_text(json.dumps(data), encoding="ascii") + with pytest.raises((ValueError, KeyError, TypeError)): + performance_cohort.verify(tmp_path, COHORT, "123", 2, SHA) + + +@pytest.mark.parametrize( + ("cohort", "run_id", "attempt"), + [(COHORT, "123", 1), ("../../escape", "123", 2), (COHORT, "0", 2), (COHORT, "123", True)], +) +def test_rejects_unbounded_or_wrong_attempt_context(cohort: str, run_id: str, attempt: int) -> None: + with pytest.raises(ValueError): + performance_cohort.require_context(cohort, run_id, attempt) + + +def test_capture_rejects_missing_hosted_image(identity: dict) -> None: + identity["environment"]["runner_image"] = "" + with pytest.raises(ValueError, match="ImageVersion"): + performance_cohort.record(COHORT, "baseline-1", "123", 2, SHA, "unit") + + +def test_capture_rejects_unknown_member(identity: dict) -> None: + with pytest.raises(ValueError, match="Unknown"): + performance_cohort.record(COHORT, "proposed-3", "123", 2, SHA, "unit") + + +@pytest.mark.parametrize("variable", ["RUNNER_NAME", "GITHUB_JOB"]) +def test_capture_cannot_emit_unbindable_runner_evidence( + identity: dict, + monkeypatch: pytest.MonkeyPatch, + variable: str, +) -> None: + monkeypatch.delenv(variable) + with pytest.raises(ValueError, match="runner and workflow job identity"): + performance_cohort.record(COHORT, "baseline-1", "123", 2, SHA, "unit") + + +def test_integration_cohort_cannot_omit_the_candidate(identity: dict) -> None: + with pytest.raises(ValueError, match="candidate archive metadata"): + performance_cohort.record( + "integration-2-apm-darwin-arm64", "baseline-1", "123", 2, SHA, "integration" + ) + + +def test_actual_node_artifact_waiter_handles_failures_without_network() -> None: + result = subprocess.run( + ["node", "--test", str(ROOT / "scripts/performance-cohort.test.cjs")], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/scripts/test_release_rehearsal.py b/tests/unit/scripts/test_release_rehearsal.py new file mode 100644 index 0000000000..aa73dcc201 --- /dev/null +++ b/tests/unit/scripts/test_release_rehearsal.py @@ -0,0 +1,331 @@ +"""Whole-path claims must retain native floors and never add parallel savings.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from scripts.compare_test_runs import Report +from scripts.package_release import BINARY_NAMES +from scripts.release_rehearsal import ROOT, SURFACES, evaluate, rehearsal_matrix, source_authorities + +pytestmark = pytest.mark.component +SHA = "a" * 40 +ORIGIN = datetime(2026, 9, 7, tzinfo=timezone.utc) + + +def _at(seconds: float) -> str: + return (ORIGIN + timedelta(seconds=seconds)).isoformat() + + +@pytest.fixture(scope="module") +def source_names() -> list[str]: + return source_authorities() + + +@pytest.fixture +def evidence(tmp_path: Path, source_names: list[str]) -> tuple: + jobs = [] + + def job(name: str, end: float, runner: str = "") -> dict: + entry = { + "id": len(jobs) + 1, + "run_id": 123, + "run_attempt": 1, + "name": name, + "runner_name": runner or f"runner-{len(jobs) + 1}", + "status": "completed", + "conclusion": "success", + "started_at": _at(1), + "completed_at": _at(end), + "steps": [], + } + jobs.append(entry) + return entry + + for name in source_names: + job(name, 100) + for binary in BINARY_NAMES: + integration_end = {"apm-darwin-arm64": 1010, "apm-linux-x86_64": 810}.get(binary, 310) + native_end = { + "apm-darwin-arm64": 1020, + "apm-linux-x86_64": 820, + "apm-darwin-x86_64": 920, + "apm-windows-x86_64": 410, + }.get(binary, 320) + job(f"{binary} / Native Candidate Gate", native_end) + job(f"{binary} / Integration Tests", integration_end) + job(f"{binary} / Build Candidate", 30) + job(f"{binary} / Isolated Release Validation", 50) + if binary != "apm-darwin-x86_64": + job(f"{binary} / Unit Tests / Unit Tests Baseline", 200) + else: + job(f"{binary} / Unit Tests / Unit Performance Probe", 915) + if binary not in {"apm-darwin-arm64", "apm-linux-x86_64"}: + job(f"{binary} / Integration Tests Shard 1 / Integration Tests Shard 1", 300) + if binary == "apm-windows-x86_64": + job(f"{binary} / Candidate Windows Installer", 400) + job("Read-only documentation build / build", 70) + job("Read-only Python distribution build / Build PyPI Distributions", 70) + reports = [] + ends = { + "arm-integration": (1000, 500, 520), + "linux-integration": (800, 450, 470), + "intel-units": (900, 420, 440), + } + for surface in SURFACES: + candidate = ( + None + if surface.suite == "unit" + else { + "schema_version": 1, + "sha": SHA, + "version": "0.30.0", + "binary_name": surface.binary, + "archive": surface.binary + ".tar.gz", + "archive_sha256": "b" * 64, + "executable_sha256": "c" * 64, + } + ) + identity = { + "source_sha": SHA, + "selection": surface.selection, + "candidate": candidate, + "environment": { + "python": "3.12.10", + "system": "Darwin" if "darwin" in surface.binary else "Linux", + "release": "release", + "machine": surface.binary.rsplit("-", 1)[1], + "cpu_count": surface.cpus, + "runner_image": "20260831.1", + "dependency_lock": "f" * 64, + }, + } + cases = {("pytest-nodeid", "test_a"): "passed", ("pytest-nodeid", "test_b"): "skipped"} + root = tmp_path / surface.key + root.mkdir() + for index, member in enumerate(("baseline-1", "proposed-1", "proposed-2")): + variant, shard = member.split("-") + runner = f"{surface.key}-{member}" + job( + f"{surface.binary} / {surface.suite} experiment {member}", + ends[surface.key][index], + runner, + ) + reports.append( + Report( + deepcopy(identity), + variant, + int(shard), + 1 if variant == "baseline" else 2, + ends[surface.key][index] - 40, + cases if index == 0 else dict([list(cases.items())[index - 1]]), + ) + ) + record = { + "schema_version": 1, + "run_id": "123", + "run_attempt": 1, + "cohort": f"{surface.suite}-1-{surface.binary}", + "member": member, + "identity": identity, + "execution": { + "runner_name": runner, + "workflow_job": "tests", + "recorded_at": _at(20), + }, + } + (root / f"{member}.json").write_text(json.dumps(record), encoding="ascii") + metadata = { + "schema_version": 1, + "source_sha": SHA, + "run_id": "123", + "run_attempt": 1, + "created_at": _at(0), + "run_started_at": _at(0), + "jobs": jobs, + } + return metadata, reports, tmp_path + + +def _evaluate(evidence: tuple, source_names: list[str]) -> dict: + metadata, reports, root = evidence + return evaluate(metadata, reports, root, SHA, source_names, 0.15, 0.10) + + +def test_complete_model_uses_maxima_and_reveals_the_next_bottleneck( + evidence: tuple, + source_names: list[str], +) -> None: + result = _evaluate(evidence, source_names) + scenarios = result["scenarios"] + assert scenarios["baseline"]["evidence_ready_seconds"] == 1020 + assert scenarios["arm_only"]["evidence_ready_seconds"] == 905 + assert "apm-darwin-x86_64" in scenarios["arm_only"]["limiting_job"] + assert scenarios["arm_and_linux"]["evidence_ready_seconds"] == 905 + assert scenarios["combined"]["evidence_ready_seconds"] == 540 + assert result["saved_seconds"] == 480 + assert result["overall_gain_fraction"] == pytest.approx(480 / 1020) + assert result["reaches_30_percent_target"] is True + assert result["unchanged_tail_sensitivity"][-1]["gain_fraction"] == pytest.approx(480 / 1200) + authorities = scenarios["combined"]["native_authorities"] + assert len(authorities) == 5 + arm = next(item for item in authorities if item["integration_surface"] == "arm-integration") + assert arm["integration_fan_in_delay_seconds"] == 10 + assert arm["native_gate_delay_seconds"] == 10 + assert arm["modeled_ready_seconds"] == 540 + + +def test_late_required_native_authorities_cannot_be_dropped_to_manufacture_a_win( + evidence: tuple, + source_names: list[str], +) -> None: + metadata, _, _ = evidence + for job in metadata["jobs"]: + if job["name"].endswith("Native Candidate Gate"): + job["completed_at"] = _at(1100) + result = _evaluate(evidence, source_names) + assert result["scenarios"]["baseline"]["evidence_ready_seconds"] == 1100 + assert result["scenarios"]["combined"]["evidence_ready_seconds"] == 1100 + assert result["overall_gain_fraction"] == 0 + assert result["meets_minimum"] is False + + +@pytest.mark.parametrize("fault", ["missing", "failed", "before-prerequisites"]) +def test_native_fan_in_evidence_is_required_and_must_follow_its_inputs( + evidence: tuple, + source_names: list[str], + fault: str, +) -> None: + metadata, _, _ = evidence + gate = next( + job for job in metadata["jobs"] if job["name"] == "apm-darwin-arm64 / Integration Tests" + ) + if fault == "missing": + metadata["jobs"].remove(gate) + elif fault == "failed": + gate["conclusion"] = "failure" + else: + gate["completed_at"] = _at(500) + with pytest.raises(ValueError): + _evaluate(evidence, source_names) + + +def test_fast_shards_do_not_imply_an_overall_gain_when_an_unchanged_job_dominates( + evidence: tuple, + source_names: list[str], +) -> None: + metadata, _, _ = evidence + installer = next( + job for job in metadata["jobs"] if job["name"].endswith("Candidate Windows Installer") + ) + installer["completed_at"] = _at(1200) + gate = next( + job + for job in metadata["jobs"] + if job["name"] == "apm-windows-x86_64 / Native Candidate Gate" + ) + gate["completed_at"] = _at(1210) + result = _evaluate(evidence, source_names) + assert result["meets_lane_thresholds"] is True + assert result["overall_gain_fraction"] == 0 + assert result["meets_minimum"] is False + assert result["scenarios"]["combined"]["limiting_job"] == gate["name"] + + +@pytest.mark.parametrize( + "fault", ["missing", "failed", "skipped", "duplicate", "old-attempt", "old-run", "bad-time"] +) +def test_rejects_incomplete_or_invalid_common_authorities( + evidence: tuple, + source_names: list[str], + fault: str, +) -> None: + metadata, _, _ = evidence + job = metadata["jobs"][0] + if fault == "missing": + metadata["jobs"].pop(0) + elif fault == "duplicate": + metadata["jobs"].append(deepcopy(job)) + elif fault == "old-attempt": + job["run_attempt"] = 2 + elif fault == "old-run": + job["run_id"] = 124 + elif fault == "bad-time": + job["completed_at"] = _at(-1) + else: + job["conclusion"] = fault + with pytest.raises(ValueError): + _evaluate(evidence, source_names) + + +@pytest.mark.parametrize( + "fault", + ["missing-shard", "new-skip", "postflight-drift", "wrong-capacity", "timing-outside-job"], +) +def test_rejects_invalid_paired_proof_without_weakening_the_existing_checker( + evidence: tuple, + source_names: list[str], + fault: str, +) -> None: + _, reports, _ = evidence + if fault == "missing-shard": + reports.pop() + elif fault == "new-skip": + reports[1] = replace(reports[1], cases={("pytest-nodeid", "test_a"): "skipped"}) + elif fault == "timing-outside-job": + reports[1] = replace(reports[1], elapsed_seconds=10000) + else: + for index in range(3): + identity = deepcopy(reports[index].identity) + identity["environment"][ + "runner_image" if fault == "postflight-drift" else "cpu_count" + ] = "different" + reports[index] = replace(reports[index], identity=identity) + with pytest.raises(ValueError): + _evaluate(evidence, source_names) + + +def test_cannot_assign_a_cohort_member_to_a_different_job( + evidence: tuple, + source_names: list[str], +) -> None: + _, _, root = evidence + path = root / "arm-integration/proposed-1.json" + data = json.loads(path.read_text("ascii")) + data["execution"]["runner_name"] = "not-an-observed-runner" + path.write_text(json.dumps(data), encoding="ascii") + with pytest.raises(ValueError, match="uniquely bind"): + _evaluate(evidence, source_names) + + +def test_baseline_native_gate_still_must_succeed( + evidence: tuple, + source_names: list[str], +) -> None: + metadata, _, _ = evidence + gate = next(job for job in metadata["jobs"] if job["name"].endswith("Native Candidate Gate")) + gate["conclusion"] = "failure" + with pytest.raises(ValueError, match="did not succeed"): + _evaluate(evidence, source_names) + + +def test_matrix_preserves_every_platform_and_changes_only_opt_in_inputs() -> None: + catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) + before = deepcopy(catalog) + matrix = rehearsal_matrix(catalog)["include"] + assert catalog == before + assert len(matrix) == 5 + assert {row["binary_name"] for row in matrix if row["unit_performance_probe"]} == { + "apm-darwin-x86_64" + } + assert {row["binary_name"] for row in matrix if row["integration_performance_probe"]} == { + "apm-darwin-arm64", + "apm-linux-x86_64", + } + assert all(row["integration_shard_count"] == 1 for row in matrix) diff --git a/tests/unit/test_ado_live_workflow_contract.py b/tests/unit/test_ado_live_workflow_contract.py index c8be8ff9df..f23f852d8b 100644 --- a/tests/unit/test_ado_live_workflow_contract.py +++ b/tests/unit/test_ado_live_workflow_contract.py @@ -24,7 +24,7 @@ AUTH_WORKFLOW = ROOT / ".github" / "workflows" / "auth-acceptance.yml" INTEGRATION_SCRIPT = ROOT / "scripts" / "test-integration.sh" LIVE_ADO_SELECTOR = "live and requires_ado_pat" -RELEASE_INTEGRATION_JOBS = ("integration-tests-shard",) +RELEASE_INTEGRATION_JOBS = ("integration-tests-shard", "integration-tests-proposed-shard") def _walk_nodes(value: Any) -> list[dict[str, Any]]: diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index 04d8e17807..8ce6ec96bc 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -27,6 +27,7 @@ ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github/workflows/release-platform.yml" +UNIT_WORKFLOW = ROOT / ".github/workflows/release-unit.yml" INTEGRATION_WORKFLOW = ROOT / ".github/workflows/release-integration.yml" RELEASE_WORKFLOW = ROOT / ".github/workflows/build-release.yml" UNIX_TIMEOUT = "${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }}" @@ -55,6 +56,10 @@ def _integration_workflow() -> dict: return load_workflow(INTEGRATION_WORKFLOW) +def _unit_workflow() -> dict: + return load_workflow(UNIT_WORKFLOW) + + def test_candidate_source_authorities_match_the_real_reusable_workflow() -> None: """Reject invented job names and omitted authorities in promotion evidence.""" source = load_workflow(ROOT / ".github/workflows/ci.yml") @@ -134,6 +139,29 @@ def _execute_integration_gate(results: dict) -> list[str]: return json.loads(completed.stdout) +def _execute_integration_probe_gate(results: dict) -> list[str]: + gate = workflow_job(_workflow(), "integration-performance-probe") + script = workflow_step(gate, "Require every integration performance member")["with"]["script"] + harness = ( + "const failures = [];" + "const core = {setFailed: message => failures.push(message)};" + f"(new Function('core', {json.dumps(script)}))(core);" + "process.stdout.write(JSON.stringify(failures));" + ) + completed = subprocess.run( + ["node", "-e", harness], + check=True, + capture_output=True, + text=True, + timeout=15, + env={ + **os.environ, + "RESULTS": json.dumps(results), + }, + ) + return json.loads(completed.stdout) + + @pytest.mark.parametrize( "job_name", ["unit-tests", "build", "integration-tests", "release-validation", "windows-installer"], @@ -174,6 +202,24 @@ def test_native_integration_fan_in_requires_shard_matrix_success(result: str | N assert _execute_integration_gate(results) == ["integration-tests-shard did not succeed"] +@pytest.mark.parametrize( + "job_name", ["integration-tests-shard", "integration-tests-proposed-shard"] +) +@pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "neutral", None]) +def test_integration_probe_fan_in_requires_baseline_and_proposed_success( + job_name: str, result: str | None +) -> None: + results = { + "integration-tests-shard": {"result": "success"}, + "integration-tests-proposed-shard": {"result": "success"}, + } + if result is None: + del results[job_name] + else: + results[job_name]["result"] = result + assert _execute_integration_probe_gate(results) == [f"{job_name} did not succeed"] + + def _assert_native_startup(workflow: dict) -> None: job = workflow_job(workflow, "build") assert_unconditional(job, label="native candidate build") @@ -281,6 +327,152 @@ def test_installer_workflow_inputs_reach_the_actual_candidate_reader(tmp_path: P assert candidate.version == "v9.8.7" +def _assert_unit_call(workflow: dict, unit: dict) -> None: + job = workflow_job(workflow, "unit-tests") + assert "needs" not in job + assert job["uses"] == "./.github/workflows/release-unit.yml" + assert job["with"] == { + "runner": "${{ inputs.runner }}", + "platform": "${{ inputs.platform }}", + "binary-name": "${{ inputs.binary-name }}", + "performance-probe": "${{ inputs.unit-performance-probe }}", + "proposed-workers": "${{ inputs.performance-workers }}", + } + assert job["secrets"] == { + "GH_CLI_PAT": "${{ secrets.GH_CLI_PAT }}", + "GH_MODELS_PAT": "${{ secrets.GH_MODELS_PAT }}", + } + + baseline = workflow_job(unit, "unit-tests") + assert baseline["runs-on"] == "${{ inputs.runner }}" + restore = workflow_step(baseline, "Restore unit scheduling hints") + assert restore["if"] == "${{ !inputs.performance-probe }}" + assert restore["with"] == {"suite": "unit"} + standard = workflow_step(baseline, "Run unit tests") + assert standard["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" + assert_exact_command( + shell_commands(standard), + [ + "uv", + "run", + "--frozen", + "pytest", + "tests/unit", + "tests/test_console.py", + "-n", + "auto", + "--dist", + "worksteal", + "--durations=50", + "--store-durations", + "--junitxml=test-results/unit.xml", + ], + label="standard release unit tests", + ) + upload = workflow_step(baseline, "Upload unit timings and outcomes") + assert upload["if"] == "always() && !inputs.performance-probe" + assert upload["with"]["name"] == ( + "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit" + ) + + +def _assert_unit_probe(unit: dict) -> None: + baseline = workflow_job(unit, "unit-tests") + cold = workflow_step(baseline, "Initialize cold unit scheduling hints") + assert cold["if"] == "inputs.performance-probe && inputs.platform != 'windows'" + assert cold["run"] == "printf '{}\\n' > .test_durations" + cohort = workflow_step( + baseline, "Require matched actual test runners before the unit performance experiment" + ) + assert cohort["if"] == "inputs.performance-probe && inputs.platform != 'windows'" + assert cohort["with"] == { + "cohort": "unit-${{ github.run_attempt }}-${{ inputs.binary-name }}", + "member": "baseline-1", + "source-sha": "${{ github.sha }}", + "selection": "tests/unit tests/test_console.py", + } + run = workflow_step(baseline, "Run unit performance baseline") + assert run["if"] == "inputs.platform != 'windows' && inputs.performance-probe" + assert_exact_command( + shell_commands(run), + [ + "uv", + "run", + "--frozen", + "python", + "-m", + "pytest", + "-p", + "scripts.pytest_performance_evidence", + "tests/unit", + "tests/test_console.py", + "-n", + "auto", + "--dist", + "worksteal", + "--durations=50", + "--store-durations", + "--junitxml=test-results/unit-baseline-shard-1.xml", + ], + label="unit performance baseline", + ) + capture = workflow_step(baseline, "Capture unit performance evidence") + assert "--variant baseline" in capture["run"] + assert "--shard-count 1" in capture["run"] + evidence = workflow_step(baseline, "Upload unit performance evidence") + assert evidence["with"]["name"] == ( + "performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-" + "baseline-shard-1" + ) + + proposed = workflow_job(unit, "unit-tests-proposed") + assert proposed["if"] == "inputs.performance-probe && inputs.platform != 'windows'" + assert proposed["strategy"] == {"fail-fast": False, "matrix": {"shard": [1, 2]}} + proposed_cohort = workflow_step( + proposed, "Require matched actual test runners before the unit performance experiment" + ) + assert proposed_cohort["with"] == { + "cohort": "unit-${{ github.run_attempt }}-${{ inputs.binary-name }}", + "member": "proposed-${{ matrix.shard }}", + "source-sha": "${{ github.sha }}", + "selection": "tests/unit tests/test_console.py", + } + proposed_run = workflow_step(proposed, "Run unit performance proposed shard") + tokens = shell_tokens(proposed_run) + for token in ( + "python", + "-m", + "pytest", + "-p", + "scripts.pytest_performance_evidence", + "tests/unit", + "tests/test_console.py", + "--splits", + "2", + "--group", + "matrix.shard", + "--splitting-algorithm", + "least_duration", + "-n", + "inputs.proposed-workers", + "--dist", + "worksteal", + "--durations=50", + "--store-durations", + ): + assert token in tokens + assert "unit-proposed-shard-${{" in proposed_run["run"] + assert "}}.xml" in proposed_run["run"] + proposed_evidence = workflow_step(proposed, "Upload unit performance evidence") + assert proposed_evidence["with"]["name"] == ( + "performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-" + "proposed-shard-${{ matrix.shard }}" + ) + fan_in = workflow_job(unit, "unit-performance-probe") + assert fan_in["needs"] == ["unit-tests", "unit-tests-proposed"] + assert fan_in["if"] == "always() && inputs.performance-probe && inputs.platform != 'windows'" + + def _assert_parallel_paths(workflow: dict) -> None: for name in ("unit-tests", "build"): assert "needs" not in workflow_job(workflow, name) @@ -323,14 +515,41 @@ def _assert_integration(workflow: dict, integration: dict) -> None: assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" assert job["with"]["timeout-minutes"] == UNIX_TIMEOUT - assert job["with"]["artifact-prefix"] == "${{ github.run_attempt }}-${{ inputs.binary-name }}" + assert job["with"]["artifact-prefix"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || " + "format('{0}-{1}', github.run_attempt, inputs.binary-name) }}" + ) assert job["with"]["runtime-prerequisites"] == "none" + assert job["with"]["evidence-variant"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "'baseline' || '' }}" + ) + assert job["with"]["evidence-selection"] == ( + "tests/integration/ -m ${{ inputs.integration-markers }}" + ) + assert job["with"]["performance-cohort"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }}" + ) build = workflow_job(workflow, "build") snapshot = workflow_step(build, "Freeze integration scheduling hints") - assert snapshot["if"] == "inputs.full-validation && inputs.platform != 'windows'" + assert snapshot["if"] == ( + "inputs.full-validation && inputs.platform != 'windows' && " + "!inputs.integration-performance-probe" + ) assert snapshot["with"]["snapshot"] == ( "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}" ) + cold_upload = workflow_step( + build, "Upload cold integration scheduling hints for performance probe" + ) + assert cold_upload["if"] == ( + "inputs.full-validation && inputs.platform != 'windows' && " + "inputs.integration-performance-probe" + ) + assert cold_upload["with"]["name"] == snapshot["with"]["snapshot"] + assert cold_upload["with"]["path"] == ".test_durations" integration_job = workflow_job(integration, "integration-tests") unpack = workflow_step(integration_job, "Extract checked candidate archive") @@ -396,6 +615,70 @@ def _assert_integration(workflow: dict, integration: dict) -> None: ) +def _assert_integration_probe(workflow: dict) -> None: + inputs = workflow["on"]["workflow_call"]["inputs"] + assert inputs["unit-performance-probe"] == {"type": "boolean", "default": False} + assert inputs["integration-performance-probe"] == {"type": "boolean", "default": False} + assert inputs["performance-workers"] == {"type": "number", "default": 4} + assert workflow["permissions"] == {"contents": "read", "actions": "read"} + + baseline = workflow_job(workflow, "integration-tests-shard") + proposed = workflow_job(workflow, "integration-tests-proposed-shard") + assert proposed["uses"] == "./.github/workflows/release-integration.yml" + assert proposed["needs"] == ["build"] + assert proposed["if"] == ( + "inputs.full-validation && inputs.platform != 'windows' && " + "inputs.integration-performance-probe" + ) + assert proposed["strategy"] == { + "fail-fast": False, + "matrix": {"shard": [1, 2]}, + } + for key in ( + "runner", + "platform", + "binary-name", + "candidate-artifact-name", + "candidate-sha", + "timing-snapshot-artifact", + "integration-markers", + "runtime-prerequisites", + "timeout-minutes", + "evidence-selection", + ): + assert proposed["with"][key] == baseline["with"][key] + assert proposed["with"]["shard-count"] == 2 + assert proposed["with"]["shard-index"] == "${{ matrix.shard }}" + assert proposed["with"]["xdist-workers"] == "${{ inputs.performance-workers }}" + assert proposed["with"]["splitting-algorithm"] == "least_duration" + assert proposed["with"]["artifact-prefix"] == ( + "${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed" + ) + assert proposed["with"]["evidence-variant"] == "proposed" + assert baseline["with"]["evidence-variant"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "'baseline' || '' }}" + ) + assert baseline["with"]["performance-cohort"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }}" + ) + assert proposed["with"]["performance-cohort"] == ( + "integration-${{ github.run_attempt }}-${{ inputs.binary-name }}" + ) + + fan_in = workflow_job(workflow, "integration-performance-probe") + assert fan_in["needs"] == ["integration-tests-shard", "integration-tests-proposed-shard"] + assert ( + fan_in["if"] == "always() && inputs.full-validation && inputs.platform != 'windows' && " + "inputs.integration-performance-probe" + ) + assert ( + "integration-tests-proposed-shard" + in workflow_step(fan_in, "Require every integration performance member")["with"]["script"] + ) + + def test_native_startup_runs_exact_frozen_candidate_before_packaging() -> None: _assert_native_startup(_workflow()) @@ -408,10 +691,22 @@ def test_native_jobs_have_no_unit_or_cross_platform_barriers() -> None: _assert_parallel_paths(_workflow()) +def test_native_unit_runner_is_canonical_and_preserves_standard_abi() -> None: + _assert_unit_call(_workflow(), _unit_workflow()) + + +def test_native_unit_performance_probe_runs_parallel_baseline_and_proposed_halves() -> None: + _assert_unit_probe(_unit_workflow()) + + def test_native_integration_preserves_grouping_bounds_and_artifact_identity() -> None: _assert_integration(_workflow(), _integration_workflow()) +def test_native_integration_performance_probe_uses_same_candidate_and_cold_snapshot() -> None: + _assert_integration_probe(_workflow()) + + def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) rows = {row["binary_name"]: row for row in catalog} @@ -534,7 +829,7 @@ def test_native_artifacts_and_downloads_are_attempt_scoped() -> None: assert upload["with"]["name"] == artifact_name assert upload["with"]["compression-level"] == 0 unit_upload = workflow_step( - workflow_job(workflow, "unit-tests"), "Upload unit timings and outcomes" + workflow_job(_unit_workflow(), "unit-tests"), "Upload unit timings and outcomes" ) assert unit_upload["with"]["name"] == ( "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit" diff --git a/tests/unit/test_pytest_timing_workflows.py b/tests/unit/test_pytest_timing_workflows.py index f7a5b54a77..c8ed0c64e8 100644 --- a/tests/unit/test_pytest_timing_workflows.py +++ b/tests/unit/test_pytest_timing_workflows.py @@ -97,10 +97,19 @@ def assert_release_platform_snapshot(workflow: dict) -> None: build = workflow_job(workflow, "build") snapshot = workflow_step(build, "Freeze integration scheduling hints") assert snapshot["uses"] == ACTION + assert snapshot["if"] == ( + "inputs.full-validation && inputs.platform != 'windows' && " + "!inputs.integration-performance-probe" + ) assert snapshot["with"] == { "suite": "integration", "snapshot": "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}", } + cold_upload = workflow_step( + build, "Upload cold integration scheduling hints for performance probe" + ) + assert cold_upload["with"]["name"] == snapshot["with"]["snapshot"] + assert cold_upload["with"]["path"] == ".test_durations" job = workflow_job(workflow, "integration-tests-shard") assert job["uses"] == RELEASE_INTEGRATION assert job["needs"] == ["build"] @@ -111,7 +120,11 @@ def assert_release_platform_snapshot(workflow: dict) -> None: assert job["with"]["shard-index"] == "${{ matrix.shard }}" assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" - assert job["with"]["artifact-prefix"] == "${{ github.run_attempt }}-${{ inputs.binary-name }}" + assert job["with"]["artifact-prefix"] == ( + "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " + "format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || " + "format('{0}-{1}', github.run_attempt, inputs.binary-name) }}" + ) assert job["with"]["runtime-prerequisites"] == "none" diff --git a/tests/unit/test_release_rehearsal_workflow.py b/tests/unit/test_release_rehearsal_workflow.py new file mode 100644 index 0000000000..d1d995b514 --- /dev/null +++ b/tests/unit/test_release_rehearsal_workflow.py @@ -0,0 +1,181 @@ +"""A whole-release experiment must execute the existing authorities without publication.""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = pytest.mark.component +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ci-release-rehearsal.yml" + + +def _assert_rehearsal(workflow: dict) -> None: + assert workflow["on"] == { + "pull_request": {"types": ["opened", "reopened", "synchronize", "labeled"]}, + } + assert workflow["permissions"] == {"contents": "read", "actions": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is True + assert "ci-release-rehearsal" in workflow["jobs"]["plan"]["if"] + for job in workflow["jobs"].values(): + assert "secrets" not in job + assert "environment" not in job + assert all( + permission in {"read", "none"} for permission in job.get("permissions", {}).values() + ) + + plan = workflow_job(workflow, "plan") + matrix = workflow_step(plan, "Derive all five native controls and the three paired surfaces") + assert matrix["run"] == 'python -m scripts.release_rehearsal matrix >> "$GITHUB_OUTPUT"' + platforms = workflow_job(workflow, "platforms") + assert platforms["uses"] == "./.github/workflows/release-platform.yml" + assert platforms["name"] == "${{ matrix.binary_name }}" + assert platforms["needs"] == ["plan"] + assert platforms["strategy"]["fail-fast"] is False + assert platforms["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.matrix) }}" + assert platforms["with"]["full-validation"] is True + for argument, field in ( + ("unit-performance-probe", "unit_performance_probe"), + ("integration-performance-probe", "integration_performance_probe"), + ("performance-workers", "performance_workers"), + ("integration-markers", "integration_markers"), + ("integration-shard-count", "integration_shard_count"), + ("integration-xdist-workers", "integration_xdist_workers"), + ): + assert platforms["with"][argument] == "${{ matrix." + field + " }}" + + source = workflow_job(workflow, "source-checks") + assert source["name"] == "Candidate Source Checks" + assert source["uses"] == "./.github/workflows/ci.yml" + assert source["needs"] == ["plan"] + for key, builder in (("docs", "docs-build"), ("wheels", "pypi-distributions")): + job = workflow_job(workflow, key) + assert job["uses"] == f"./.github/workflows/{builder}.yml" + assert job["needs"] == ["plan"] + + report = workflow_job(workflow, "compare") + assert report["needs"] == ["plan", "source-checks", "platforms", "docs", "wheels"] + assert "always()" in report["if"] and "!cancelled()" in report["if"] + collect = workflow_step( + report, "Record current-attempt job timestamps and require production authorities" + ) + script = collect["with"]["script"] + assert "listJobsForWorkflowRunAttempt" in script + assert "attempt_number: attempt" in script + assert "selectRequiredJobs(jobs, catalog)" in script + assert "source_sha: context.sha" in script + assert "run_id: String(context.runId), run_attempt: attempt" in script + check = workflow_step(report, "Require original outcomes and measure the entire critical path") + assert "--minimum-lane-gain 0.15 --minimum-overall-gain 0.10" in check["run"] + assert '--expected-sha "$GITHUB_SHA"' in check["run"] + downloads = [ + step["with"]["pattern"] + for step in report["steps"] + if "download-artifact@" in step.get("uses", "") + ] + assert len(downloads) == 5 + assert all("${{ github.run_attempt }}" in pattern for pattern in downloads) + assert downloads[:2] == [ + "performance-evidence-${{ github.run_attempt }}-apm-*", + "performance-evidence-unit-${{ github.run_attempt }}-apm-*", + ] + for surface in ( + "integration-apm-darwin-arm64", + "integration-apm-linux-x86_64", + "unit-apm-darwin-x86_64", + ): + suite, binary = surface.split("-apm-", 1) + assert ( + f"performance-cohort-{suite}-${{{{ github.run_attempt }}}}-apm-{binary}-*" in downloads + ) + + +def test_read_only_rehearsal_reuses_full_native_source_and_publication_build_authorities() -> None: + _assert_rehearsal(load_workflow(WORKFLOW)) + + +@pytest.mark.parametrize( + "fault", + [ + "privileged", + "secrets", + "partial-validation", + "missing-source", + "fail-fast", + "worker-drift", + "old-artifacts", + "missing-lane", + "missing-authority", + "weaker-threshold", + ], +) +def test_rehearsal_cannot_silently_remove_controls_or_accept_weaker_proof(fault: str) -> None: + workflow = deepcopy(load_workflow(WORKFLOW)) + jobs = workflow["jobs"] + if fault == "privileged": + workflow["permissions"]["contents"] = "write" + elif fault == "secrets": + jobs["platforms"]["secrets"] = "inherit" + elif fault == "partial-validation": + jobs["platforms"]["with"]["full-validation"] = False + elif fault == "missing-source": + jobs["source-checks"]["uses"] = "./.github/workflows/ci-source-performance.yml" + elif fault == "fail-fast": + jobs["platforms"]["strategy"]["fail-fast"] = True + elif fault == "worker-drift": + jobs["platforms"]["with"]["performance-workers"] = 99 + elif fault == "missing-lane": + jobs["compare"]["needs"].remove("platforms") + elif fault == "missing-authority": + step = workflow_step( + jobs["compare"], + "Record current-attempt job timestamps and require production authorities", + ) + step["with"]["script"] = step["with"]["script"].replace( + "selectRequiredJobs(jobs, catalog)", "null" + ) + elif fault == "weaker-threshold": + step = workflow_step( + jobs["compare"], "Require original outcomes and measure the entire critical path" + ) + step["run"] = step["run"].replace("--minimum-overall-gain 0.10", "--minimum-overall-gain 0") + else: + step = workflow_step(jobs["compare"], "Download current integration proofs") + step["with"]["pattern"] = "performance-evidence-*" + with pytest.raises(AssertionError): + _assert_rehearsal(workflow) + + +def test_cohort_barrier_is_before_pytest_and_static_callers_grant_read_only_actions() -> None: + integration = load_workflow(ROOT / ".github/workflows/release-integration.yml") + steps = integration["jobs"]["integration-tests"]["steps"] + barrier = next( + index + for index, step in enumerate(steps) + if step.get("uses") == "./.github/actions/performance-cohort" + ) + execution = next( + index + for index, step in enumerate(steps) + if step.get("name") == "Run integration tests (Unix)" + ) + assert barrier < execution + assert steps[barrier]["if"] == "inputs.performance-cohort != ''" + assert integration["permissions"] == {"contents": "read", "actions": "read"} + for file in ("build-release.yml", "ci-integration.yml", "ci-release-performance.yml"): + workflow = load_workflow(ROOT / ".github/workflows" / file) + callers = [ + job + for job in workflow["jobs"].values() + if job.get("uses") + in { + "./.github/workflows/release-integration.yml", + "./.github/workflows/release-platform.yml", + } + ] + assert callers + assert all(job["permissions"] == {"contents": "read", "actions": "read"} for job in callers) diff --git a/tests/unit/test_windows_unit_diagnostics.py b/tests/unit/test_windows_unit_diagnostics.py index 6d7260ba1c..cfe165f1c5 100644 --- a/tests/unit/test_windows_unit_diagnostics.py +++ b/tests/unit/test_windows_unit_diagnostics.py @@ -15,7 +15,7 @@ pytestmark = pytest.mark.component -WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/release-platform.yml" +WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/release-unit.yml" TEST_COMMAND = ["uv", "run", "--frozen", "pytest", "tests/unit", "tests/test_console.py"] PARALLEL_ARGS = [ "-n", @@ -34,7 +34,7 @@ def _assert_windows_diagnostics(workflow: dict) -> None: job = workflow_job(workflow, "unit-tests") linux = workflow_step(job, "Run unit tests") windows = workflow_step(job, "Run unit tests (Windows diagnostics)") - assert linux["if"] == "inputs.platform != 'windows'" + assert linux["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" assert shell_commands(linux) == [TEST_COMMAND + PARALLEL_ARGS] assert windows["if"] == "inputs.platform == 'windows'" assert windows["timeout-minutes"] == 60 From 2b3d7f6ed79b601821c54d8e082a6e8b043563e8 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 23:57:47 +0200 Subject: [PATCH 08/23] fix(ci): retain subtest evidence and repair native rehearsal controls Emit concrete unittest subtest cases in the xdist controller without changing worker scheduling or relaxing JUnit parity. Accept read-only public API credentials without promoting them to PAT/Models aliases. Reserve native DLL path headroom in the installer fixture while retaining adversarial quoting and all original checks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../docs/contributing/integration-testing.md | 9 + scripts/pytest_performance_evidence.py | 61 +++++ scripts/windows/test-release-validation.ps1 | 23 +- .../test_windows_installer_launchers.py | 7 +- tests/unit/scripts/test_compare_test_runs.py | 202 +++++++++++++++++ .../test_release_validation_prerequisites.py | 212 ++++++++++++++++++ .../unit/test_windows_installer_launchers.py | 25 +++ 7 files changed, 534 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_release_validation_prerequisites.py diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 19ff9ffbec..9c1e5e905f 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -577,6 +577,9 @@ fingerprints before accepting a measured execution improvement of at least 5%. New skips, missing tests, failures, or different environments fail the comparison. JUnit controller elapsed time is reported separately from summed runner time; neither includes runner allocation or setup. +The opt-in evidence plugin emits `unittest.subTest` outcomes as distinct +parent-relative ordinal cases, so JUnit summary counts must still exactly +match the concrete inventory; subtests are not discarded to make counts fit. Download `arm-performance-comparison-` or `windows-compat-performance-comparison-` for the JSON verdict, and @@ -605,6 +608,12 @@ expensive tests. Integration pairs use exact archive bytes from the same build; source-unit pairs use the identical SHA; final comparison still requires exact original case/outcome parity. +Windows public release validation accepts `GITHUB_API_TOKEN` without mapping +the read-only job token to PAT/Models variables. Private-module and inference +validation still need their existing credentials. The native installer fixture +uses a short temporary root while retaining spaces and `&` in its install +prefix, leaving room for staged bundle DLL paths. + The comparison retains all five production `Integration Tests` fan-ins and `Native Candidate Gate` jobs, measures each observed dependency-to-verdict delay, and propagates the selected variant diff --git a/scripts/pytest_performance_evidence.py b/scripts/pytest_performance_evidence.py index 413aed1c81..912f016d7b 100644 --- a/scripts/pytest_performance_evidence.py +++ b/scripts/pytest_performance_evidence.py @@ -4,9 +4,70 @@ import pytest +_PARENT_NODEID_PROPERTY = "apm_performance_parent_nodeid" +_ORIGINAL_NODEIDS: set[str] = set() +_SUBTEST_ORDINALS: dict[str, int] = {} +_IS_XDIST_WORKER = False + + +def pytest_configure(config: pytest.Config) -> None: + """Keep nested/in-process pytest sessions from reusing stale identities.""" + global _IS_XDIST_WORKER + _ORIGINAL_NODEIDS.clear() + _SUBTEST_ORDINALS.clear() + _IS_XDIST_WORKER = hasattr(config, "workerinput") + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Record a stable identity without changing selection, grouping, or outcomes.""" for item in items: + _ORIGINAL_NODEIDS.add(item.nodeid) item.user_properties.append(("apm_performance_nodeid", item.nodeid)) + + +def _original_nodeid(nodeid: str) -> str: + """Undo pytest-xdist's loadgroup suffix only when it matches a collected item.""" + if nodeid in _ORIGINAL_NODEIDS: + return nodeid + candidate, separator, _group = nodeid.rpartition("@") + if separator and candidate in _ORIGINAL_NODEIDS: + return candidate + return nodeid + + +def _pop_worker_parent_nodeid(report: pytest.TestReport) -> str | None: + """Read the worker-captured parent nodeid without leaking it to JUnit.""" + parents = [ + value + for name, value in report.user_properties + if name == _PARENT_NODEID_PROPERTY and isinstance(value, str) and value + ] + report.user_properties = [ + (name, value) for name, value in report.user_properties if name != _PARENT_NODEID_PROPERTY + ] + if len(parents) > 1: + raise ValueError("Duplicate performance subtest parent identity") + return parents[0] if parents else None + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Give unittest subTest reports their own JUnit node instead of a summary-only count. + + Pytest 9 emits a passed call report for each ``unittest.TestCase.subTest`` but + its JUnit writer otherwise folds those passed subtests into the parent + ```` while still incrementing ``tests``. Performance evidence + needs the strict XML summary count to equal the concrete scenario inventory, + so split each subtest report into a stable synthetic child node before + junitxml sees it. + """ + if getattr(report, "when", None) != "call" or not hasattr(report, "context"): + return + if _IS_XDIST_WORKER: + report.user_properties.append((_PARENT_NODEID_PROPERTY, _original_nodeid(report.nodeid))) + return + parent_nodeid = _pop_worker_parent_nodeid(report) or _original_nodeid(report.nodeid) + _SUBTEST_ORDINALS[parent_nodeid] = _SUBTEST_ORDINALS.get(parent_nodeid, 0) + 1 + report.nodeid = f"{parent_nodeid}::subTest[{_SUBTEST_ORDINALS[parent_nodeid]}]" + report.user_properties.append(("apm_performance_nodeid", report.nodeid)) diff --git a/scripts/windows/test-release-validation.ps1 b/scripts/windows/test-release-validation.ps1 index 2796d3b642..8023f37604 100644 --- a/scripts/windows/test-release-validation.ps1 +++ b/scripts/windows/test-release-validation.ps1 @@ -106,11 +106,28 @@ function Find-Binary { function Test-Prerequisite { Write-TestHeader "Prerequisites: GitHub token" + $hasReleaseApiToken = [bool]$env:GITHUB_API_TOKEN + $inferenceTestsEnabled = $env:APM_RUN_INFERENCE_TESTS -eq "1" + Initialize-GitHubToken - # Initialize-GitHubToken doesn't return failure — check tokens after setup - if ($env:GITHUB_TOKEN -or $env:GITHUB_APM_PAT) { + + # Initialize-GitHubToken does not return failure; check tokens after setup. + # GITHUB_API_TOKEN is the workflow's read-only GitHub API token for public + # release validation. It is not a PAT/Models credential, so do not copy it + # into GITHUB_TOKEN, GITHUB_APM_PAT, or GITHUB_MODELS_KEY. + $hasPatOrModelsToken = [bool]($env:GITHUB_TOKEN -or $env:GITHUB_APM_PAT) + + if ($inferenceTestsEnabled -and -not $hasPatOrModelsToken) { + Write-ErrorText "Inference tests require GITHUB_TOKEN or GITHUB_APM_PAT; GITHUB_API_TOKEN only covers public release API access" + return $false + } + + if ($hasPatOrModelsToken -or $hasReleaseApiToken) { Write-Success "GitHub tokens configured successfully" + if ($hasReleaseApiToken) { + Write-Success "GITHUB_API_TOKEN is set (public release API access)" + } if ($env:GITHUB_APM_PAT) { Write-Success "GITHUB_APM_PAT is set (APM module access)" } @@ -119,7 +136,7 @@ function Test-Prerequisite { } return $true } else { - Write-ErrorText "GitHub token setup failed" + Write-ErrorText "GitHub token setup failed; set GITHUB_API_TOKEN for public release validation or GITHUB_TOKEN/GITHUB_APM_PAT for inference/private module validation" return $false } } diff --git a/tests/integration/test_windows_installer_launchers.py b/tests/integration/test_windows_installer_launchers.py index c91c5688c4..cf961b2569 100644 --- a/tests/integration/test_windows_installer_launchers.py +++ b/tests/integration/test_windows_installer_launchers.py @@ -31,11 +31,14 @@ ROOT = Path(__file__).resolve().parents[2] -def test_windows_installer_exposes_stable_executable(tmp_path: Path) -> None: +def test_windows_installer_exposes_stable_executable( + tmp_path_factory: pytest.TempPathFactory, +) -> None: """Install APM and launch bare ``apm`` through Windows process boundaries.""" candidate = InstallerArchive.from_environment(os.environ, "CANDIDATE") baseline = InstallerArchive.from_environment(os.environ, "BASELINE") - isolated = IsolatedApmEnvironment.create(tmp_path / "installer", base_env=os.environ) + # Keep the harness prefix short; the PowerShell fixture still adds spaces and "&". + isolated = IsolatedApmEnvironment.create(tmp_path_factory.mktemp("wi"), base_env=os.environ) environment = isolated.subprocess_env() environment["APPDATA"] = str(isolated.home) with ( diff --git a/tests/unit/scripts/test_compare_test_runs.py b/tests/unit/scripts/test_compare_test_runs.py index ab1c116675..579dd5996b 100644 --- a/tests/unit/scripts/test_compare_test_runs.py +++ b/tests/unit/scripts/test_compare_test_runs.py @@ -290,3 +290,205 @@ def test_real_pytest_xdist_reports_preserve_original_node_ids_and_outcomes(tmp_p assert proof["scenario_count"] == 4 assert proof["outcomes"]["passed"] == 2 assert ("pytest-nodeid", "test_example.py::test_case[literal@home]") in baseline.cases + + +def _write_subtest_probe(tmp_path: Path) -> None: + (tmp_path / "pytest.ini").write_text( + "[pytest]\nmarkers = xdist_group: scheduling affinity\n", encoding="ascii" + ) + (tmp_path / "test_subtest_probe.py").write_text( + "import os\n" + "import pytest\n" + "import unittest\n" + "class TestSubtests(unittest.TestCase):\n" + " @pytest.mark.xdist_group('home')\n" + " def test_repeated_context(self):\n" + " for ordinal, label in enumerate(['same', 'same', 'literal@context'], 1):\n" + " with self.subTest(label=label):\n" + " if os.environ.get('FAIL_SECOND_SUBTEST') == '1' and ordinal == 2:\n" + " self.fail('intentional subtest proof failure')\n" + " self.assertTrue(label)\n" + "@pytest.mark.xdist_group('home')\n" + "@pytest.mark.parametrize('value', ['literal@home', 'plain'])\n" + "def test_literal_at_param(value):\n" + " if os.environ.get('SKIP_PLAIN_PARAM') == '1' and value == 'plain':\n" + " pytest.skip('intentional parity regression')\n" + " assert value in {'literal@home', 'plain'}\n", + encoding="ascii", + ) + + +def _run_subtest_probe( + tmp_path: Path, + name: str, + variant: str, + shard: int, + shard_count: int, + workers: str, + dist: str = "worksteal", + *, + fail_second_subtest: bool = False, + skip_plain_param: bool = False, +) -> tuple[subprocess.CompletedProcess[str], Report]: + root = Path(__file__).resolve().parents[3] + (tmp_path / ".test_durations").write_text("{}\n", encoding="ascii") + junit = tmp_path / f"{name}.xml" + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "scripts.pytest_performance_evidence", + "test_subtest_probe.py", + "-n", + workers, + "--dist", + dist, + "--durations=5", + "--store-durations", + f"--junitxml={junit}", + ] + if shard_count > 1: + command.extend( + [ + "--splits", + str(shard_count), + "--group", + str(shard), + "--splitting-algorithm", + "least_duration", + ] + ) + result = subprocess.run( + command, + cwd=tmp_path, + env={ + **os.environ, + "PYTHONPATH": str(root), + "FAIL_SECOND_SUBTEST": "1" if fail_second_subtest else "", + "SKIP_PLAIN_PARAM": "1" if skip_plain_param else "", + }, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + return result, capture(junit, SHA, "subtest probe", variant, shard, shard_count) + + +def test_real_pytest_subtests_compare_under_xdist_baseline_and_two_shards( + tmp_path: Path, +) -> None: + _write_subtest_probe(tmp_path) + result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") + assert result.returncode == 0, result.stdout + result.stderr + proposed = [] + for shard in (1, 2): + result, report = _run_subtest_probe( + tmp_path, f"proposed-{shard}", "proposed", shard, 2, "4" + ) + assert result.returncode == 0, result.stdout + result.stderr + proposed.append(report) + + proof = compare( + [replace(baseline, elapsed_seconds=10.0)], + [replace(report, elapsed_seconds=5.0) for report in proposed], + SHA, + 0.05, + ) + assert proof["scenario_count"] == 6 + assert proof["outcomes"] == { + "passed": 6, + "skipped": 0, + "xfailed": 0, + "failed": 0, + "error": 0, + } + assert ( + "pytest-nodeid", + "test_subtest_probe.py::test_literal_at_param[literal@home]", + ) in baseline.cases + for ordinal in (1, 2, 3): + assert ( + "test_subtest_probe.TestSubtests.test_repeated_context", + f"subTest[{ordinal}]", + ) in baseline.cases + + +def test_real_pytest_subtests_compare_between_serial_and_loadgroup_shards( + tmp_path: Path, +) -> None: + _write_subtest_probe(tmp_path) + result, baseline = _run_subtest_probe( + tmp_path, "baseline-loadgroup", "baseline", 1, 1, "0", "loadgroup" + ) + assert result.returncode == 0, result.stdout + result.stderr + proposed = [] + for shard in (1, 2): + result, report = _run_subtest_probe( + tmp_path, f"proposed-loadgroup-{shard}", "proposed", shard, 2, "2", "loadgroup" + ) + assert result.returncode == 0, result.stdout + result.stderr + proposed.append(report) + + proof = compare( + [replace(baseline, elapsed_seconds=10.0)], + [replace(report, elapsed_seconds=5.0) for report in proposed], + SHA, + 0.05, + ) + assert proof["scenario_count"] == 6 + assert ( + "pytest-nodeid", + "test_subtest_probe.py::test_literal_at_param[literal@home]", + ) in baseline.cases + assert ( + "pytest-nodeid", + "test_subtest_probe.py::test_literal_at_param[literal@home]", + ) in proposed[0].cases | proposed[1].cases + for report in proposed: + assert not any("@home::subTest" in part for case in report.cases for part in case) + + +def test_real_pytest_subtest_failure_remains_a_compare_rejection(tmp_path: Path) -> None: + _write_subtest_probe(tmp_path) + result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") + assert result.returncode == 0, result.stdout + result.stderr + proposed = [] + for shard in (1, 2): + result, report = _run_subtest_probe( + tmp_path, + f"proposed-fail-{shard}", + "proposed", + shard, + 2, + "4", + fail_second_subtest=True, + ) + proposed.append(report) + assert any("failed" in report.cases.values() for report in proposed) + with pytest.raises(ValueError, match="contains failing tests"): + compare([baseline], proposed, SHA, 0.05) + + +def test_real_pytest_subtest_probe_rejects_new_skips(tmp_path: Path) -> None: + _write_subtest_probe(tmp_path) + result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") + assert result.returncode == 0, result.stdout + result.stderr + proposed = [] + for shard in (1, 2): + result, report = _run_subtest_probe( + tmp_path, + f"proposed-skip-{shard}", + "proposed", + shard, + 2, + "4", + skip_plain_param=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + proposed.append(report) + assert any("skipped" in report.cases.values() for report in proposed) + with pytest.raises(ValueError, match="Scenario parity failed"): + compare([baseline], proposed, SHA, 0.05) diff --git a/tests/unit/test_release_validation_prerequisites.py b/tests/unit/test_release_validation_prerequisites.py new file mode 100644 index 0000000000..9e035dc5bc --- /dev/null +++ b/tests/unit/test_release_validation_prerequisites.py @@ -0,0 +1,212 @@ +"""Function-level checks for the Windows release-validation prerequisites.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.component, pytest.mark.windows_compat] + +ROOT = Path(__file__).resolve().parents[2] +PWSH = shutil.which("pwsh") + +_TOKEN_ENV_NAMES = { + "APM_RUN_INFERENCE_TESTS", + "GH_TOKEN", + "GITHUB_API_TOKEN", + "GITHUB_APM_PAT", + "GITHUB_CLI_PAT", + "GITHUB_COPILOT_PAT", + "GITHUB_MODELS_KEY", + "GITHUB_TOKEN", +} + + +def _digest(value: str | None) -> str | None: + if value is None: + return None + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _clean_env(overrides: dict[str, str]) -> dict[str, str]: + """Keep process basics, but never inherit ambient credentials into the fixture.""" + env = { + key: value + for key, value in os.environ.items() + if key not in _TOKEN_ENV_NAMES and not key.startswith("GITHUB_APM_PAT_") + } + env.update(overrides) + return env + + +def _run_test_prerequisite(overrides: dict[str, str]) -> tuple[dict[str, object], str, str]: + command = r""" +$ErrorActionPreference = "Stop" + +. (Join-Path $PWD "scripts/windows/github-token-helper.ps1") + +$script:Messages = [System.Collections.Generic.List[string]]::new() +function Write-Info { param([string]$Message) $script:Messages.Add("INFO:$Message") } +function Write-Success { param([string]$Message) $script:Messages.Add("SUCCESS:$Message") } +function Write-ErrorText { param([string]$Message) $script:Messages.Add("ERROR:$Message") } +function Write-TestHeader { param([string]$Message) $script:Messages.Add("HEADER:$Message") } + +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile( + (Join-Path $PWD "scripts/windows/test-release-validation.ps1"), + [ref]$tokens, + [ref]$errors +) +if ($errors.Count) { throw ($errors | Out-String) } + +$function = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq "Test-Prerequisite" +}, $true) +if (-not $function) { throw "Test-Prerequisite function not found" } + +. ([scriptblock]::Create($function.Extent.Text)) + +function Get-EnvDigest { + param([string]$Name) + + $value = [Environment]::GetEnvironmentVariable($Name, "Process") + if ($null -eq $value) { return $null } + + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($value) + return [System.BitConverter]::ToString($sha.ComputeHash($bytes)).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +$result = Test-Prerequisite +[pscustomobject]@{ + Result = [bool]$result + EnvDigests = [pscustomobject]@{ + GITHUB_API_TOKEN = Get-EnvDigest "GITHUB_API_TOKEN" + GITHUB_APM_PAT = Get-EnvDigest "GITHUB_APM_PAT" + GITHUB_MODELS_KEY = Get-EnvDigest "GITHUB_MODELS_KEY" + GITHUB_TOKEN = Get-EnvDigest "GITHUB_TOKEN" + } + Messages = @($script:Messages) +} | ConvertTo-Json -Compress -Depth 5 +""" + + result = subprocess.run( + [PWSH, "-NoProfile", "-NonInteractive", "-Command", command], + cwd=ROOT, + env=_clean_env(overrides), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + json_lines = [line for line in result.stdout.splitlines() if line.startswith("{")] + assert json_lines, result.stdout + result.stderr + return json.loads(json_lines[-1]), result.stdout, result.stderr + + +def _messages(record: dict[str, object]) -> list[str]: + messages = record["Messages"] + if isinstance(messages, str): + return [messages] + return list(messages) + + +def _assert_sentinels_not_printed(stdout: str, stderr: str, *sentinels: str) -> None: + combined = stdout + stderr + for sentinel in sentinels: + assert sentinel not in combined + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +def test_public_release_api_token_satisfies_prerequisite_when_inference_is_off() -> None: + api_token = "readonly-api-token-sentinel" + + record, stdout, stderr = _run_test_prerequisite({"GITHUB_API_TOKEN": api_token}) + + assert record["Result"] is True + assert record["EnvDigests"] == { + "GITHUB_API_TOKEN": _digest(api_token), + "GITHUB_APM_PAT": None, + "GITHUB_MODELS_KEY": None, + "GITHUB_TOKEN": None, + } + assert any("GITHUB_API_TOKEN is set" in message for message in _messages(record)) + _assert_sentinels_not_printed(stdout, stderr, api_token) + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +def test_missing_release_and_pat_tokens_rejects_prerequisite() -> None: + record, _stdout, _stderr = _run_test_prerequisite({}) + + assert record["Result"] is False + assert any("GitHub token setup failed" in message for message in _messages(record)) + assert record["EnvDigests"] == { + "GITHUB_API_TOKEN": None, + "GITHUB_APM_PAT": None, + "GITHUB_MODELS_KEY": None, + "GITHUB_TOKEN": None, + } + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +def test_api_token_does_not_satisfy_inference_prerequisite_or_seed_pat_aliases() -> None: + api_token = "readonly-api-token-sentinel" + + record, stdout, stderr = _run_test_prerequisite( + {"APM_RUN_INFERENCE_TESTS": "1", "GITHUB_API_TOKEN": api_token} + ) + + assert record["Result"] is False + assert record["EnvDigests"] == { + "GITHUB_API_TOKEN": _digest(api_token), + "GITHUB_APM_PAT": None, + "GITHUB_MODELS_KEY": None, + "GITHUB_TOKEN": None, + } + assert any("Inference tests require" in message for message in _messages(record)) + _assert_sentinels_not_printed(stdout, stderr, api_token) + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +def test_existing_pat_and_models_tokens_are_preserved_and_not_logged() -> None: + api_token = "readonly-api-token-sentinel" + apm_pat = "apm-pat-token-sentinel" + github_token = "models-token-sentinel" + models_key = "models-key-sentinel" + + record, stdout, stderr = _run_test_prerequisite( + { + "APM_RUN_INFERENCE_TESTS": "1", + "GITHUB_API_TOKEN": api_token, + "GITHUB_APM_PAT": apm_pat, + "GITHUB_MODELS_KEY": models_key, + "GITHUB_TOKEN": github_token, + } + ) + + assert record["Result"] is True + assert record["EnvDigests"] == { + "GITHUB_API_TOKEN": _digest(api_token), + "GITHUB_APM_PAT": _digest(apm_pat), + "GITHUB_MODELS_KEY": _digest(models_key), + "GITHUB_TOKEN": _digest(github_token), + } + messages = _messages(record) + assert any("GITHUB_APM_PAT is set" in message for message in messages) + assert any("GITHUB_TOKEN is set" in message for message in messages) + _assert_sentinels_not_printed(stdout, stderr, api_token, apm_pat, github_token, models_key) diff --git a/tests/unit/test_windows_installer_launchers.py b/tests/unit/test_windows_installer_launchers.py index 3e160f2130..fb2f2dca87 100644 --- a/tests/unit/test_windows_installer_launchers.py +++ b/tests/unit/test_windows_installer_launchers.py @@ -2,8 +2,11 @@ from __future__ import annotations +import ast from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[2] @@ -41,6 +44,28 @@ def test_windows_installer_e2e_covers_bare_subprocess_resolution() -> None: assert "Stable executable directory precedes command shim directory in user PATH" in test_script +@pytest.mark.windows_compat +def test_installer_harness_does_not_add_the_test_name_to_native_dll_paths() -> None: + """Reserve loader path space without removing the PowerShell quoting scenarios.""" + source = (ROOT / "tests/integration/test_windows_installer_launchers.py").read_text("utf-8") + tree = ast.parse(source) + create = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "IsolatedApmEnvironment" + and node.func.attr == "create" + ) + root = create.args[0] + assert isinstance(root, ast.Call) and isinstance(root.func, ast.Attribute) + assert isinstance(root.func.value, ast.Name) and root.func.value.id == "tmp_path_factory" + assert root.func.attr == "mktemp" and ast.literal_eval(root.args[0]) == "wi" + test_script = (ROOT / "scripts/windows/test-install-script.ps1").read_text("utf-8") + assert "APM Install Test & Edge" in test_script + + def test_windows_installer_e2e_covers_missing_stable_executable_negative_twin() -> None: """The native-process proof must fail when only the cmd shim remains.""" test_script = (ROOT / "scripts/windows/test-install-script.ps1").read_text(encoding="utf-8") From de21774b4ac28480c0548475ce9ebc7da8dc33c3 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 00:32:01 +0200 Subject: [PATCH 09/23] test(ci): repair rehearsal fixture lifecycle isolation Keep the Windows installer harness short while providing an uncreated isolation root, and clear temporary MCP registry configuration from the process cache on fixture teardown. Preserve native assertions and strict benchmark controls. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../docs/contributing/integration-testing.md | 27 +++++++++++++------ .../test_windows_installer_launchers.py | 4 ++- .../install/test_mcp_registry_config_layer.py | 26 +++++++++++++++++- .../unit/test_windows_installer_launchers.py | 13 +++++++-- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 9c1e5e905f..2c32c0a4b8 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -234,6 +234,10 @@ test-named `tmp_path` roots to avoid Git for Windows metadata path limits. Retain worker-equivalent directory depth in focused gates so they still exercise the paths used by the sharded suite. +When a fixture monkeypatches a temporary config path, reset the config cache at +setup and again in `finally`. Path monkeypatching alone does not invalidate +cached config in shard-exposed fixtures. + It is also not an OS/native-code sandbox: executables found through `PATH` remain trusted, reflective access to CPython internals or native extensions can bypass Python monkey-patches, `file://` access is not confined by the OS, and @@ -495,19 +499,24 @@ serves the verified archives and the current checkout's `install.ps1` bytes on is unchanged. The test also compares the installed candidate executable hash with the executable member inside the candidate ZIP. +The Windows isolation root is `tmp_path_factory.mktemp("wi") / "i"`: a short, +nonexistent child for the shared helper, while the PowerShell install prefix +still includes spaces and `&`. + ### Windows unit hang diagnostics -The Windows full-unit step in `.github/workflows/release-platform.yml` runs: +The Windows full-unit step in `.github/workflows/release-unit.yml` runs: ```sh -uv run pytest tests/unit tests/test_console.py -n auto --dist worksteal -vv --tb=short --show-capture=no --no-showlocals +uv run --frozen pytest tests/unit tests/test_console.py -n auto --dist worksteal --durations=50 --store-durations --junitxml=test-results/unit.xml -vv --tb=short --show-capture=no --no-showlocals ``` `PYTHONUNBUFFERED=1` keeps named test starts and outcomes visible while the suite runs. Captured test output and local-variable dumps remain disabled. These diagnostics identify candidate unfinished tests, not stack frames or the exact blocked phase; an outcome can appear before fixture teardown completes. The 60-minute step limit fails closed on hangs. Test selection and parallelism -are unchanged. The PR-time `windows_compat` gate exercises name visibility -during setup, call, and teardown after a failed call. +are unchanged: Windows unit runs still use xdist `-n auto --dist worksteal`. +The PR-time `windows_compat` gate exercises name visibility during setup, call, +and teardown after a failed call. ### GitHub Actions Authentication @@ -629,10 +638,12 @@ Thresholds are 15% per paired pytest lane and 10% overall evidence-ready improvement, with 30% as the target. Do not treat workflow duration as proposed release time: it waits for both variants, and experimental concurrency/cohort waits can add allocation -cost. Remove the label before unrelated pushes. On failed attempts use **Re-run -all jobs**; artifacts are immutable and attempt-scoped. Limit an evaluation to -three full attempts, including image-inconclusive attempts. Production rollout -requires passing proof plus a maintainer decision. +cost. Remove the label before unrelated pushes. For a new full attempt, remove +and reapply `ci-release-rehearsal` to launch a fresh workflow run; do not rely +on rerunning the same run ID. Rehearsal proofs are attempt-scoped, but canonical +source CI artifacts use bare names. Limit an evaluation to three full attempts, +including invalid or image-inconclusive attempts. Production rollout requires +passing proof plus a maintainer decision. ## Debugging Test Failures diff --git a/tests/integration/test_windows_installer_launchers.py b/tests/integration/test_windows_installer_launchers.py index cf961b2569..7444794878 100644 --- a/tests/integration/test_windows_installer_launchers.py +++ b/tests/integration/test_windows_installer_launchers.py @@ -38,7 +38,9 @@ def test_windows_installer_exposes_stable_executable( candidate = InstallerArchive.from_environment(os.environ, "CANDIDATE") baseline = InstallerArchive.from_environment(os.environ, "BASELINE") # Keep the harness prefix short; the PowerShell fixture still adds spaces and "&". - isolated = IsolatedApmEnvironment.create(tmp_path_factory.mktemp("wi"), base_env=os.environ) + isolated = IsolatedApmEnvironment.create( + tmp_path_factory.mktemp("wi") / "i", base_env=os.environ + ) environment = isolated.subprocess_env() environment["APPDATA"] = str(isolated.home) with ( diff --git a/tests/unit/install/test_mcp_registry_config_layer.py b/tests/unit/install/test_mcp_registry_config_layer.py index fe2abf29ff..1e74c463e7 100644 --- a/tests/unit/install/test_mcp_registry_config_layer.py +++ b/tests/unit/install/test_mcp_registry_config_layer.py @@ -15,6 +15,7 @@ from __future__ import annotations import json +from pathlib import Path from urllib.parse import urlparse import pytest @@ -94,13 +95,36 @@ def project(monkeypatch, tmp_path): monkeypatch.setattr(config_mod, "CONFIG_DIR", str(config_dir)) monkeypatch.setattr(config_mod, "CONFIG_FILE", str(config_dir / "config.json")) config_mod._invalidate_config_cache() - return tmp_path + try: + yield tmp_path + finally: + config_mod._invalidate_config_cache() def _install(args=()): return CliRunner().invoke(cli, ["install", "--no-policy", *args]) +def test_project_fixture_clears_registry_cache_when_its_config_scope_ends( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A temporary config path must not leave its registry in process-global cache.""" + from apm_cli import config as config_mod + + fixture = project.__wrapped__(monkeypatch, tmp_path) + next(fixture) + try: + config_mod.set_mcp_registry_url(CONFIG_REGISTRY) + assert ( + urlparse(config_mod.get_mcp_registry_url()).hostname + == urlparse(CONFIG_REGISTRY).hostname + ) + finally: + fixture.close() + assert config_mod._config_cache is None + + def _printed_hosts(output: str) -> set[str]: """Hostnames of every URL in CLI output, parsed rather than substring-matched.""" return { diff --git a/tests/unit/test_windows_installer_launchers.py b/tests/unit/test_windows_installer_launchers.py index fb2f2dca87..ded1b9231c 100644 --- a/tests/unit/test_windows_installer_launchers.py +++ b/tests/unit/test_windows_installer_launchers.py @@ -7,6 +7,8 @@ import pytest +from tests.utils.isolated_apm_environment import IsolatedApmEnvironment + ROOT = Path(__file__).resolve().parents[2] @@ -45,7 +47,9 @@ def test_windows_installer_e2e_covers_bare_subprocess_resolution() -> None: @pytest.mark.windows_compat -def test_installer_harness_does_not_add_the_test_name_to_native_dll_paths() -> None: +def test_installer_harness_does_not_add_the_test_name_to_native_dll_paths( + tmp_path_factory: pytest.TempPathFactory, +) -> None: """Reserve loader path space without removing the PowerShell quoting scenarios.""" source = (ROOT / "tests/integration/test_windows_installer_launchers.py").read_text("utf-8") tree = ast.parse(source) @@ -58,10 +62,15 @@ def test_installer_harness_does_not_add_the_test_name_to_native_dll_paths() -> N and node.func.value.id == "IsolatedApmEnvironment" and node.func.attr == "create" ) - root = create.args[0] + root_expression = create.args[0] + assert isinstance(root_expression, ast.BinOp) and isinstance(root_expression.op, ast.Div) + assert ast.literal_eval(root_expression.right) == "i" + root = root_expression.left assert isinstance(root, ast.Call) and isinstance(root.func, ast.Attribute) assert isinstance(root.func.value, ast.Name) and root.func.value.id == "tmp_path_factory" assert root.func.attr == "mktemp" and ast.literal_eval(root.args[0]) == "wi" + isolated = IsolatedApmEnvironment.create(tmp_path_factory.mktemp("wi") / "i", base_env={}) + assert isolated.work_root.is_dir() test_script = (ROOT / "scripts/windows/test-install-script.ps1").read_text("utf-8") assert "APM Install Test & Edge" in test_script From 1f9a5295f20688773361b97cfbc92efc93e5898b Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 01:05:28 +0200 Subject: [PATCH 10/23] test(ci): restore console parent module after fallback probes Repair the import-state leak exposed by the last Intel sharding rehearsal. Preserve both sys.modules and parent-package identity, with regressions for rich and colorama fallbacks. Do not relax performance acceptance or production gates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../docs/contributing/integration-testing.md | 2 ++ tests/unit/test_console_utils.py | 25 ++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 2c32c0a4b8..4ec5ad0ff0 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -237,6 +237,8 @@ exercise the paths used by the sharded suite. When a fixture monkeypatches a temporary config path, reset the config cache at setup and again in `finally`. Path monkeypatching alone does not invalidate cached config in shard-exposed fixtures. +Import-fallback fixtures must also restore parent-package attributes, not just +`sys.modules`, so import spellings see the same module after teardown. It is also not an OS/native-code sandbox: executables found through `PATH` remain trusted, reflective access to CPython internals or native extensions can diff --git a/tests/unit/test_console_utils.py b/tests/unit/test_console_utils.py index 8aa6464388..43d87d60f1 100644 --- a/tests/unit/test_console_utils.py +++ b/tests/unit/test_console_utils.py @@ -2,6 +2,7 @@ import importlib import sys +from types import ModuleType from typing import Any from unittest.mock import MagicMock, patch @@ -400,12 +401,14 @@ def test_rich_exception_fallback_yields_none(self, capsys): # --------------------------------------------------------------------------- -def _reimport_console_without(blocked_modules: list[str]): +def _reimport_console_without(blocked_modules: list[str]) -> ModuleType: """Re-import apm_cli.utils.console with ``blocked_modules`` blocked. - Returns the freshly imported module, then restores everything in - sys.modules so other tests are unaffected. + Restore both sys.modules and the parent-package attribute so other + tests cannot import the fallback module through a different spelling. """ + original = importlib.import_module("apm_cli.utils.console") + parent = importlib.import_module("apm_cli.utils") # Save and remove existing entries we will touch. saved: dict[str, object] = {} for key in list(sys.modules.keys()): @@ -433,6 +436,7 @@ def _reimport_console_without(blocked_modules: list[str]): del sys.modules[key] # Restore originals. sys.modules.update(saved) + parent.console = original class TestImportFallbacks: @@ -454,6 +458,21 @@ def test_colorama_unavailable_sets_constants(self): assert fresh.Fore is None assert fresh.Style is None + @pytest.mark.parametrize("blocked_module", ["rich", "colorama"]) + def test_fallback_probe_restores_both_import_paths(self, blocked_module: str) -> None: + """A fallback probe must not strand a fake module on its parent package.""" + original = importlib.import_module("apm_cli.utils.console") + + fresh = _reimport_console_without([blocked_module]) + + import apm_cli.utils.console as qualified + from apm_cli.utils import console as from_parent + + assert fresh is not original + assert sys.modules["apm_cli.utils.console"] is original + assert qualified is original + assert from_parent is original + class TestGetConsoleDoubleCheckLock: """Line 80: inner guard of the double-checked locking pattern.""" From cb0905955e933cd137fb5f42cc1b1c7a4456ec9e Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 11:27:00 +0200 Subject: [PATCH 11/23] ci: add bounded native Windows installer proof Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../workflows/ci-windows-installer-proof.yml | 92 ++++ scripts/windows/run-installer-proof.ps1 | 507 ++++++++++++++++++ .../test_windows_installer_proof_workflow.py | 221 ++++++++ 3 files changed, 820 insertions(+) create mode 100644 .github/workflows/ci-windows-installer-proof.yml create mode 100644 scripts/windows/run-installer-proof.ps1 create mode 100644 tests/unit/test_windows_installer_proof_workflow.py diff --git a/.github/workflows/ci-windows-installer-proof.yml b/.github/workflows/ci-windows-installer-proof.yml new file mode 100644 index 0000000000..3c198944b2 --- /dev/null +++ b/.github/workflows/ci-windows-installer-proof.yml @@ -0,0 +1,92 @@ +name: PR Windows Installer Proof + +on: + pull_request: + types: [labeled] + +permissions: + contents: read + +concurrency: + group: windows-installer-proof-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + PYTHON_VERSION: '3.12' + BASELINE_VERSION: v0.28.0 + +jobs: + windows-installer-proof: + name: Windows installer proof + if: github.event.label.name == 'ci-windows-installer-proof' + runs-on: windows-latest + timeout-minutes: 45 + steps: + - name: Require fresh proof attempt + shell: pwsh + run: | + if ($env:GITHUB_RUN_ATTEMPT -ne '1') { + throw "Use a fresh labelled run, not a partial or mixed-attempt rerun" + } + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --frozen --extra dev --extra build + - name: Build unsigned Windows candidate + shell: pwsh + env: + WINDOWS_CERT_PFX: '' + run: uv run --frozen --no-sync pwsh scripts/windows/build-binary.ps1 + - name: Package exact Windows candidate archive + shell: pwsh + env: + CANDIDATE_SHA: ${{ github.sha }} + BINARY_NAME: apm-windows-x86_64 + run: >- + python scripts/package_release.py pack + --binary-name "$env:BINARY_NAME" --sha "$env:CANDIDATE_SHA" + - name: Upload standalone proof candidate archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-installer-proof-candidate-${{ github.run_attempt }} + path: | + release-assets/apm-windows-x86_64.zip + release-assets/apm-windows-x86_64.zip.sha256 + release-assets/apm-windows-x86_64.json + if-no-files-found: error + compression-level: 0 + retention-days: 30 + - name: Download historical upgrade source once + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download $env:BASELINE_VERSION --repo microsoft/apm ` + --pattern apm-windows-x86_64.zip --pattern apm-windows-x86_64.zip.sha256 ` + --dir installer-baseline + if ($LASTEXITCODE -ne 0) { throw "Could not download the historical upgrade source" } + - name: Diagnose and prove Windows installer root behavior + shell: pwsh + run: >- + scripts/windows/run-installer-proof.ps1 + -CandidateArchive release-assets/apm-windows-x86_64.zip + -CandidateMetadata release-assets/apm-windows-x86_64.json + -BaselineArchive installer-baseline/apm-windows-x86_64.zip + -BaselineChecksum installer-baseline/apm-windows-x86_64.zip.sha256 + -BaselineVersion "$env:BASELINE_VERSION" + -OutputDir windows-installer-proof + - name: Upload Windows installer proof diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-installer-proof-${{ github.run_attempt }} + path: windows-installer-proof/ + if-no-files-found: error + retention-days: 30 diff --git a/scripts/windows/run-installer-proof.ps1 b/scripts/windows/run-installer-proof.ps1 new file mode 100644 index 0000000000..6f94ef40f8 --- /dev/null +++ b/scripts/windows/run-installer-proof.ps1 @@ -0,0 +1,507 @@ +# Focused Windows installer proof for one locally built candidate archive. +# +# The proof runs the unchanged installer assertion harness twice against the +# same archive with explicit pytest basetemps: one long root shaped like the +# failing default runner root, and one short root under RUNNER_TEMP. It also +# records archive path pressure and PowerShell 5.1 native-stderr behavior. + +param( + [string]$CandidateArchive, + [string]$CandidateMetadata, + [string]$BaselineArchive, + [string]$BaselineChecksum, + [string]$BaselineVersion, + [string]$OutputDir = "windows-installer-proof", + [switch]$SelfTestProcessCapture, + [string]$PythonExecutable = "python" +) + +$ErrorActionPreference = "Stop" + +function Resolve-RequiredPath { + param([string]$Path, [string]$Label) + if (-not $Path) { throw "$Label is required" } + $resolved = Resolve-Path -LiteralPath $Path -ErrorAction Stop + if (-not $resolved) { throw "$Label not found: $Path" } + return $resolved.Path +} + +function Get-Sha256Hex { + param([string]$Path) + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Write-JsonFile { + param([object]$Value, [string]$Path) + $directory = Split-Path -Parent $Path + if ($directory) { New-Item -ItemType Directory -Force -Path $directory | Out-Null } + $Value | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $Path -Encoding UTF8 +} + +function Stop-OwnedProcessTree { + param([System.Diagnostics.Process]$Process) + if ($Process.HasExited) { return } + if ($IsWindows) { + & taskkill.exe /PID $Process.Id /T /F | Out-Null + if ($LASTEXITCODE -ne 0 -and -not $Process.HasExited) { + throw "Could not stop owned process tree $($Process.Id)" + } + } else { + $Process.Kill($true) + } +} + +function Invoke-CapturedProcess { + param( + [Parameter(Mandatory = $true)][string]$FilePath, + [Parameter(Mandatory = $true)][string[]]$Arguments, + [Parameter(Mandatory = $true)][string]$Name, + [int]$TimeoutSeconds = 90, + [hashtable]$Environment = @{}, + [string]$WorkingDirectory = (Get-Location).Path + ) + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $FilePath + $psi.WorkingDirectory = $WorkingDirectory + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + foreach ($argument in $Arguments) { + [void]$psi.ArgumentList.Add($argument) + } + foreach ($key in $Environment.Keys) { + $psi.Environment[$key] = [string]$Environment[$key] + } + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $psi + $record = [ordered]@{ + name = $Name + command = @($FilePath) + $Arguments + cwd = $WorkingDirectory + timeout_seconds = $TimeoutSeconds + timed_out = $false + exit_code = $null + stdout = "" + stderr = "" + } + try { + if (-not $process.Start()) { throw "Could not start $Name" } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $record.timed_out = $true + Stop-OwnedProcessTree -Process $process + if (-not $process.WaitForExit(10000)) { + throw "Owned process $($process.Id) did not exit after its timeout" + } + } + $record.exit_code = $process.ExitCode + $record.stdout = $stdoutTask.GetAwaiter().GetResult() + $record.stderr = $stderrTask.GetAwaiter().GetResult() + return $record + } finally { + $process.Dispose() + } +} + +function New-OwnedBaseTemp { + param([string]$Path) + if (Test-Path -LiteralPath $Path) { + throw "Refusing to reuse preexisting proof basetemp: $Path" + } + New-Item -ItemType Directory -Path $Path -ErrorAction Stop | Out-Null + Set-Content -LiteralPath (Join-Path $Path ".apm-windows-installer-proof-owned") ` + -Value "owned by run $env:GITHUB_RUN_ID attempt $env:GITHUB_RUN_ATTEMPT" -Encoding ASCII + return $Path +} + +function Remove-OwnedBaseTemp { + param([string]$Path) + $marker = Join-Path $Path ".apm-windows-installer-proof-owned" + if (-not (Test-Path -LiteralPath $marker)) { + throw "Refusing to remove unmarked proof basetemp: $Path" + } + Remove-Item -Recurse -Force -LiteralPath $Path -ErrorAction Stop +} + +function Read-BaselineSha { + param([string]$Path) + $text = (Get-Content -LiteralPath $Path -Raw).Trim() + if ($text -notmatch '^([a-fA-F0-9]{64})\s+\*?apm-windows-x86_64\.zip$') { + throw "Invalid baseline checksum format: $Path" + } + return $Matches[1].ToLowerInvariant() +} + +function Get-ArchiveInventory { + param( + [string]$Archive, + [string]$LongBaseTemp, + [string]$ShortBaseTemp, + [string]$Version + ) + Add-Type -AssemblyName System.IO.Compression.FileSystem + $zip = [System.IO.Compression.ZipFile]::OpenRead($Archive) + try { + $allEntries = @($zip.Entries) + $files = @($allEntries | Where-Object { -not ($_.FullName.EndsWith("/")) }) + $entries = @($files | ForEach-Object { $_.FullName }) + } finally { + $zip.Dispose() + } + $suffix = "APM Install Test & Edge " + ("a" * 32) + function Convert-MemberPath { + param([string]$Member) + $prefix = "apm-windows-x86_64/" + if ($Member.StartsWith($prefix, [System.StringComparison]::Ordinal)) { + return $Member.Substring($prefix.Length).Replace("/", "\") + } + return $Member.Replace("/", "\") + } + function Measure-BaseTemp { + param([string]$BaseTemp) + $testRoot = Join-Path $BaseTemp "wi0\i\work" + $installRoot = Join-Path $testRoot $suffix + $tempDir = Join-Path (Join-Path $installRoot "tmp") ("apm-install-" + ("b" * 32)) + $stageDir = Join-Path (Join-Path $installRoot "releases") ($Version + ".new-" + ("c" * 32)) + $finalDir = Join-Path (Join-Path $installRoot "releases") $Version + $records = @() + foreach ($entry in $entries) { + $relative = Convert-MemberPath -Member $entry + $records += [ordered]@{ + member = $entry + temp_length = (Join-Path $tempDir ($entry.Replace("/", "\"))).Length + stage_length = (Join-Path $stageDir $relative).Length + final_length = (Join-Path $finalDir $relative).Length + } + } + return [ordered]@{ + base_temp = $BaseTemp + test_root = $testRoot + install_root = $installRoot + max_temp = @($records | Sort-Object temp_length -Descending | Select-Object -First 1)[0] + max_stage = @($records | Sort-Object stage_length -Descending | Select-Object -First 1)[0] + max_final = @($records | Sort-Object final_length -Descending | Select-Object -First 1)[0] + charset = @($records | Where-Object { $_.member -match 'charset_normalizer|chardet' }) + stage_over_260 = @($records | Where-Object { $_.stage_length -ge 260 }).Count + temp_over_260 = @($records | Where-Object { $_.temp_length -ge 260 }).Count + } + } + return [ordered]@{ + file_count = $files.Count + member_count = $allEntries.Count + pyd_count = @($entries | Where-Object { ($_.ToLowerInvariant()).EndsWith(".pyd") }).Count + long_root = Measure-BaseTemp -BaseTemp $LongBaseTemp + short_root = Measure-BaseTemp -BaseTemp $ShortBaseTemp + } +} + +function Invoke-DirectCandidateProbe { + param( + [string]$Name, + [string]$BaseTemp, + [string]$Archive, + [string]$Version, + [string]$OutDir + ) + $testRoot = Join-Path $BaseTemp "wi0\i\work" + $root = Join-Path $testRoot ("APM Install Test & Edge " + [System.Guid]::NewGuid().ToString("N")) + $extractDir = Join-Path $root ("tmp\apm-install-" + [System.Guid]::NewGuid().ToString("N")) + $releaseDir = Join-Path (Join-Path $root "releases") $Version + $stagingDir = "$releaseDir.new-" + [System.Guid]::NewGuid().ToString("N") + $record = [ordered]@{ + name = $Name + powershell_host = [ordered]@{ + edition = $PSVersionTable.PSEdition + version = $PSVersionTable.PSVersion.ToString() + } + base_temp = $BaseTemp + test_root = $testRoot + root = $root + extract_dir = $extractDir + staging_dir = $stagingDir + expanded = $false + moved_to_stage = $false + charset_members = @() + run = $null + error = $null + } + try { + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $releaseDir) | Out-Null + Expand-Archive -LiteralPath $Archive -DestinationPath $extractDir -Force + $record.expanded = $true + $packageDir = Join-Path $extractDir "apm-windows-x86_64" + Get-ChildItem -LiteralPath $packageDir -Recurse -Force -ErrorAction Stop | + Where-Object { $_.FullName -match 'charset_normalizer|chardet' } | + ForEach-Object { + $record.charset_members += [ordered]@{ + path = $_.FullName + length = $_.FullName.Length + exists = $_.Exists + } + } + Move-Item -LiteralPath $packageDir -Destination $stagingDir -Force + $record.moved_to_stage = $true + $exe = Join-Path $stagingDir "apm.exe" + $record.run = Invoke-CapturedProcess -FilePath $exe -Arguments @("--version") ` + -Name "$Name direct apm --version" -TimeoutSeconds 90 + } catch { + $record.error = "$_" + } finally { + Write-JsonFile -Value $record -Path (Join-Path $OutDir "direct-$Name.json") + } + return $record +} + +function Invoke-PowerShellNativeStderrProbe { + param([string]$OutDir) + $child = Join-Path $OutDir "ps5-native-stderr-child.ps1" + @' +$ErrorActionPreference = "Stop" +function Invoke-NativeCase { + param([string]$Name, [int]$ExitCode) + $result = [ordered]@{ + name = $Name + expected_exit_code = $ExitCode + threw = $false + last_exit_code = $null + output = "" + error = "" + } + try { + $output = & cmd.exe /d /c "echo marker-$Name 1>&2 & exit /b $ExitCode" 2>&1 + $result.last_exit_code = $LASTEXITCODE + $result.output = ($output | Out-String) + } catch { + $result.threw = $true + $result.last_exit_code = $LASTEXITCODE + $result.error = "$_" + } + return $result +} +[ordered]@{ + host = [ordered]@{ + edition = $PSVersionTable.PSEdition + version = $PSVersionTable.PSVersion.ToString() + major = $PSVersionTable.PSVersion.Major + minor = $PSVersionTable.PSVersion.Minor + } + cases = @( + (Invoke-NativeCase -Name "exit0-stderr" -ExitCode 0), + (Invoke-NativeCase -Name "exit23-stderr" -ExitCode 23) + ) +} | ConvertTo-Json -Depth 8 +'@ | Set-Content -LiteralPath $child -Encoding UTF8 + $ps5 = Invoke-CapturedProcess -FilePath "powershell.exe" -Arguments @( + "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $child + ) -Name "powershell 5 native stderr probe" -TimeoutSeconds 90 + $exit0 = Invoke-CapturedProcess -FilePath "cmd.exe" -Arguments @( + "/d", "/c", "echo marker-direct-exit0 1>&2 & exit /b 0" + ) -Name "direct native stderr exit0 control" -TimeoutSeconds 90 + $exit23 = Invoke-CapturedProcess -FilePath "cmd.exe" -Arguments @( + "/d", "/c", "echo marker-direct-exit23 1>&2 & exit /b 23" + ) -Name "direct native stderr exit23 control" -TimeoutSeconds 90 + if ($exit0.timed_out -or $exit0.exit_code -ne 0 -or $exit0.stderr -notmatch "marker-direct-exit0") { + throw "Broken direct exit0 native-stderr control" + } + if ($exit23.timed_out -or $exit23.exit_code -ne 23 -or $exit23.stderr -notmatch "marker-direct-exit23") { + throw "Broken direct exit23 native-stderr control" + } + if ($ps5.timed_out -or $ps5.exit_code -ne 0) { + throw "PowerShell 5 native-stderr child failed" + } + try { + $ps5Json = $ps5.stdout | ConvertFrom-Json + } catch { + throw "PowerShell 5 native-stderr child did not emit valid JSON: $_" + } + if ($ps5Json.host.major -ne 5 -or $ps5Json.host.minor -ne 1) { + throw "Expected Windows PowerShell 5.1, got $($ps5Json.host.version)" + } + $caseNames = @($ps5Json.cases | ForEach-Object { $_.name } | Sort-Object) + if ($caseNames.Count -ne 2 -or ($caseNames -join ",") -ne "exit0-stderr,exit23-stderr") { + throw "PowerShell 5 child did not report both native-stderr cases" + } + foreach ($case in $ps5Json.cases) { + # A terminating stderr record can prevent LASTEXITCODE from being updated. + # The independent native controls above establish the real exit codes. + if (-not $case.threw -and $case.name -eq "exit0-stderr" -and $case.last_exit_code -ne 0) { + throw "PowerShell 5 exit0 case lost native exit code" + } + if (-not $case.threw -and $case.name -eq "exit23-stderr" -and $case.last_exit_code -ne 23) { + throw "PowerShell 5 exit23 case lost native exit code" + } + $combined = "$($case.output) $($case.error)" + if ($combined -notmatch ("marker-" + [regex]::Escape($case.name))) { + throw "PowerShell 5 case $($case.name) lost stderr marker" + } + } + $result = [ordered]@{ + name = "deterministic-native-stderr" + powershell5 = $ps5 + powershell5_json = $ps5Json + direct_exit0 = $exit0 + direct_exit23 = $exit23 + } + Write-JsonFile -Value $result -Path (Join-Path $OutDir "native-stderr-behavior.json") + return $result +} + +function Invoke-InstallerPytest { + param( + [string]$Name, + [string]$OutDir, + [string]$BaseTemp, + [hashtable]$InstallerEnvironment + ) + if (Test-Path -LiteralPath $BaseTemp) { + throw "Installer pytest basetemp must be fresh before pytest owns it: $BaseTemp" + } + $arguments = @( + "run", "--frozen", "--no-sync", "pytest", + "tests/integration/test_windows_installer_launchers.py", + "-q", "-s", "--tb=short", + "--junitxml=$OutDir\junit-$Name.xml", + "--basetemp", $BaseTemp + ) + $result = Invoke-CapturedProcess -FilePath "uv" -Arguments $arguments ` + -Name "$Name installer pytest" -TimeoutSeconds 900 -Environment $InstallerEnvironment + $result.stdout | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stdout.log") -Encoding UTF8 + $result.stderr | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stderr.log") -Encoding UTF8 + Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") + Write-Host "[$Name] exit $($result.exit_code) timed_out=$($result.timed_out)" + if ($result.stdout) { + ($result.stdout -split "`n") | Select-Object -Last 30 | ForEach-Object { Write-Host "[$Name stdout] $_" } + } + if ($result.stderr) { + ($result.stderr -split "`n") | Select-Object -Last 30 | ForEach-Object { Write-Host "[$Name stderr] $_" } + } + return $result +} + +function Invoke-ProcessCaptureSelfTest { + param([string]$OutDir, [string]$Python) + New-Item -ItemType Directory -Force -Path $OutDir | Out-Null + $exit0 = Invoke-CapturedProcess -FilePath $Python -Arguments @( + "-c", "import sys; sys.stderr.write('selftest-exit0-stderr\n'); print('selftest-exit0-stdout')" + ) -Name "selftest exit0" -TimeoutSeconds 30 + $exit23 = Invoke-CapturedProcess -FilePath $Python -Arguments @( + "-c", "import sys; sys.stderr.write('selftest-exit23-stderr\n'); sys.exit(23)" + ) -Name "selftest exit23" -TimeoutSeconds 30 + $timeout = Invoke-CapturedProcess -FilePath $Python -Arguments @( + "-c", "import time; time.sleep(10)" + ) -Name "selftest timeout" -TimeoutSeconds 1 + $result = [ordered]@{ exit0 = $exit0; exit23 = $exit23; timeout = $timeout } + Write-JsonFile -Value $result -Path (Join-Path $OutDir "process-capture-selftest.json") + if ($exit0.timed_out -or $exit0.exit_code -ne 0 -or $exit0.stderr -notmatch "selftest-exit0-stderr") { + throw "Process capture self-test exit0 failed" + } + if ($exit23.timed_out -or $exit23.exit_code -ne 23 -or $exit23.stderr -notmatch "selftest-exit23-stderr") { + throw "Process capture self-test exit23 failed" + } + if (-not $timeout.timed_out) { + throw "Process capture self-test timeout did not time out" + } +} + +if ($SelfTestProcessCapture) { + Invoke-ProcessCaptureSelfTest -OutDir $OutputDir -Python $PythonExecutable + exit 0 +} + +$CandidateArchive = Resolve-RequiredPath -Path $CandidateArchive -Label "Candidate archive" +$CandidateMetadata = Resolve-RequiredPath -Path $CandidateMetadata -Label "Candidate metadata" +$BaselineArchive = Resolve-RequiredPath -Path $BaselineArchive -Label "Baseline archive" +$BaselineChecksum = Resolve-RequiredPath -Path $BaselineChecksum -Label "Baseline checksum" +if (-not $BaselineVersion) { throw "BaselineVersion is required" } +if (-not $env:RUNNER_TEMP) { throw "RUNNER_TEMP is required; this proof is CI-only" } +if (-not $env:TEMP) { throw "TEMP is required to reproduce the Windows pytest default root" } +if ([string]$env:GITHUB_RUN_ATTEMPT -ne "1") { + throw "Use a fresh attempt 1 run for Windows installer proof" +} +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +$metadata = Get-Content -LiteralPath $CandidateMetadata -Raw | ConvertFrom-Json +$candidateVersion = "v$($metadata.version)" +$candidateSha = [string]$metadata.archive_sha256 +$candidateExeSha = [string]$metadata.executable_sha256 +if ((Get-Sha256Hex -Path $CandidateArchive) -cne $candidateSha) { + throw "Candidate archive hash does not match metadata" +} +$baselineSha = Read-BaselineSha -Path $BaselineChecksum +if ((Get-Sha256Hex -Path $BaselineArchive) -cne $baselineSha) { + throw "Baseline archive hash does not match sidecar" +} + +$longChild = "wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5) +$longBaseTemp = Join-Path (Join-Path $env:TEMP ("pytest-of-" + $env:USERNAME)) $longChild +$shortBaseTemp = Join-Path $env:RUNNER_TEMP "apm-windows-installer" +Invoke-PowerShellNativeStderrProbe -OutDir $OutputDir | Out-Null +$longOwned = $false +$shortOwned = $false +try { + New-OwnedBaseTemp -Path $longBaseTemp | Out-Null + $longOwned = $true + New-OwnedBaseTemp -Path $shortBaseTemp | Out-Null + $shortOwned = $true + $inventory = Get-ArchiveInventory -Archive $CandidateArchive -LongBaseTemp $longBaseTemp ` + -ShortBaseTemp $shortBaseTemp -Version $candidateVersion + Write-JsonFile -Value $inventory -Path (Join-Path $OutputDir "archive-path-inventory.json") + Invoke-DirectCandidateProbe -Name "long-root" -BaseTemp $longBaseTemp -Archive $CandidateArchive ` + -Version $candidateVersion -OutDir $OutputDir | Out-Null + Invoke-DirectCandidateProbe -Name "short-root" -BaseTemp $shortBaseTemp -Archive $CandidateArchive ` + -Version $candidateVersion -OutDir $OutputDir | Out-Null +} finally { + if ($longOwned -and (Test-Path -LiteralPath $longBaseTemp)) { Remove-OwnedBaseTemp -Path $longBaseTemp } + if ($shortOwned -and (Test-Path -LiteralPath $shortBaseTemp)) { Remove-OwnedBaseTemp -Path $shortBaseTemp } +} + +$installerEnvironment = @{ + "APM_CANDIDATE_ARCHIVE" = $CandidateArchive + "APM_CANDIDATE_VERSION" = $candidateVersion + "APM_CANDIDATE_SHA256" = $candidateSha + "APM_BASELINE_ARCHIVE" = $BaselineArchive + "APM_BASELINE_VERSION" = $BaselineVersion + "APM_BASELINE_SHA256" = $baselineSha +} +$longResult = Invoke-InstallerPytest -Name "long-root" -OutDir $OutputDir -BaseTemp $longBaseTemp ` + -InstallerEnvironment $installerEnvironment +$shortResult = Invoke-InstallerPytest -Name "short-root" -OutDir $OutputDir -BaseTemp $shortBaseTemp ` + -InstallerEnvironment $installerEnvironment + +$summary = [ordered]@{ + candidate = [ordered]@{ + version = $candidateVersion + archive_sha256 = $candidateSha + executable_sha256 = $candidateExeSha + metadata_sha = [string]$metadata.sha + } + baseline = [ordered]@{ + version = $BaselineVersion + archive_sha256 = $baselineSha + } + roots = [ordered]@{ + long_base_temp = $longBaseTemp + short_base_temp = $shortBaseTemp + } + long_root_exit_code = $longResult.exit_code + long_root_timed_out = $longResult.timed_out + short_root_exit_code = $shortResult.exit_code + short_root_timed_out = $shortResult.timed_out + conclusion = if ($shortResult.exit_code -eq 0 -and $longResult.exit_code -ne 0) { + "short-root-passed-long-root-failed" + } elseif ($shortResult.exit_code -eq 0 -and $longResult.exit_code -eq 0) { + "both-roots-passed" + } else { + "short-root-failed" + } +} +Write-JsonFile -Value $summary -Path (Join-Path $OutputDir "summary.json") + +if ($shortResult.timed_out -or $shortResult.exit_code -ne 0) { + Write-Host "Short-root installer proof failed; see $OutputDir" -ForegroundColor Red + if ($shortResult.timed_out) { exit 124 } + exit $shortResult.exit_code +} +Write-Host "Windows installer proof completed: $($summary.conclusion)" diff --git a/tests/unit/test_windows_installer_proof_workflow.py b/tests/unit/test_windows_installer_proof_workflow.py new file mode 100644 index 0000000000..a22261bdb7 --- /dev/null +++ b/tests/unit/test_windows_installer_proof_workflow.py @@ -0,0 +1,221 @@ +"""Contracts for the opt-in Windows installer proof lane.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = pytest.mark.component + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ci-windows-installer-proof.yml" +PROOF_SCRIPT = ROOT / "scripts/windows/run-installer-proof.ps1" + + +def _workflow() -> dict: + """Load the proof workflow.""" + return load_workflow(WORKFLOW) + + +def _script_text() -> str: + """Read the proof script as text for static safety contracts.""" + return PROOF_SCRIPT.read_text(encoding="ascii") + + +def _assert_workflow(workflow: dict) -> None: + """Require a read-only, opt-in diagnostic that builds one unsigned candidate.""" + assert workflow["on"] == {"pull_request": {"types": ["labeled"]}} + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is False + assert workflow["env"]["BASELINE_VERSION"] == "v0.28.0" + assert set(workflow["jobs"]) == {"windows-installer-proof"} + + job = workflow_job(workflow, "windows-installer-proof") + assert job["if"] == "github.event.label.name == 'ci-windows-installer-proof'" + assert job["runs-on"] == "windows-latest" + assert job["timeout-minutes"] == 45 + assert "secrets" not in job + assert "environment" not in job + assert not job.get("continue-on-error", False) + for step in job["steps"]: + assert "secrets." not in str(step) + assert "action-gh-release" not in step.get("uses", "") + assert "deploy-pages" not in step.get("uses", "") + assert "WINDOWS_CERT_PASSWORD" not in str(step) + assert not step.get("continue-on-error", False) + + fresh = workflow_step(job, "Require fresh proof attempt") + assert "GITHUB_RUN_ATTEMPT -ne '1'" in fresh["run"] + + checkout = job["steps"][1] + assert checkout["uses"].startswith("actions/checkout@") + assert checkout["with"]["persist-credentials"] is False + + build = workflow_step(job, "Build unsigned Windows candidate") + assert build["env"] == {"WINDOWS_CERT_PFX": ""} + assert "scripts/windows/build-binary.ps1" in build["run"] + + package = workflow_step(job, "Package exact Windows candidate archive") + assert package["env"] == { + "CANDIDATE_SHA": "${{ github.sha }}", + "BINARY_NAME": "apm-windows-x86_64", + } + assert "scripts/package_release.py pack" in package["run"] + + candidate = workflow_step(job, "Upload standalone proof candidate archive") + assert candidate["uses"].startswith("actions/upload-artifact@") + assert candidate["with"]["name"] == ( + "windows-installer-proof-candidate-${{ github.run_attempt }}" + ) + assert candidate["with"]["path"] == ( + "release-assets/apm-windows-x86_64.zip\n" + "release-assets/apm-windows-x86_64.zip.sha256\n" + "release-assets/apm-windows-x86_64.json\n" + ) + assert candidate["with"]["if-no-files-found"] == "error" + assert candidate["with"]["compression-level"] == 0 + assert candidate["with"]["retention-days"] == 30 + assert "qualified" not in candidate["with"]["name"] + assert "release-evidence" not in candidate["with"]["name"] + + baseline = workflow_step(job, "Download historical upgrade source once") + assert baseline["env"] == {"GH_TOKEN": "${{ github.token }}"} + assert "gh release download $env:BASELINE_VERSION" in baseline["run"] + assert "--repo microsoft/apm" in baseline["run"] + + proof = workflow_step(job, "Diagnose and prove Windows installer root behavior") + assert "scripts/windows/run-installer-proof.ps1" in proof["run"] + assert "-CandidateArchive release-assets/apm-windows-x86_64.zip" in proof["run"] + assert "-CandidateMetadata release-assets/apm-windows-x86_64.json" in proof["run"] + assert "-BaselineArchive installer-baseline/apm-windows-x86_64.zip" in proof["run"] + assert "-BaselineChecksum installer-baseline/apm-windows-x86_64.zip.sha256" in proof["run"] + + upload = workflow_step(job, "Upload Windows installer proof diagnostics") + assert upload["if"] == "always()" + assert upload["with"]["name"] == "windows-installer-proof-${{ github.run_attempt }}" + assert upload["with"]["path"] == "windows-installer-proof/" + assert upload["with"]["if-no-files-found"] == "error" + + +def test_windows_installer_proof_is_read_only_and_opt_in() -> None: + """The focused lane must not publish, sign, or run on every PR update.""" + _assert_workflow(_workflow()) + + +@pytest.mark.parametrize( + "fault", + ["secret", "write-token", "eager-trigger", "signed", "waived-failure", "missing-upload"], +) +def test_windows_installer_proof_rejects_scope_creep(fault: str) -> None: + """Guard against accidentally turning the diagnostic into release machinery.""" + workflow = deepcopy(_workflow()) + job = workflow_job(workflow, "windows-installer-proof") + if fault == "secret": + job["secrets"] = "inherit" + elif fault == "write-token": + workflow["permissions"]["contents"] = "write" + elif fault == "eager-trigger": + workflow["on"]["pull_request"]["types"].append("synchronize") + elif fault == "signed": + workflow_step(job, "Build unsigned Windows candidate")["env"] = { + "WINDOWS_CERT_PFX": "${{ secrets.WINDOWS_CERT_PFX }}" + } + elif fault == "waived-failure": + job["continue-on-error"] = True + else: + workflow_step(job, "Upload Windows installer proof diagnostics")["with"][ + "if-no-files-found" + ] = "ignore" + with pytest.raises(AssertionError): + _assert_workflow(workflow) + + +def test_windows_installer_proof_script_runs_same_archive_through_both_roots() -> None: + """The script must distinguish path roots without changing installer assertions.""" + text = _script_text() + assert "tests/integration/test_windows_installer_launchers.py" in text + assert text.count('"APM_CANDIDATE_ARCHIVE" = $CandidateArchive') == 1 + assert 'Invoke-InstallerPytest -Name "long-root"' in text + assert 'Invoke-InstallerPytest -Name "short-root"' in text + assert '"wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5)' in text + assert 'Join-Path $env:TEMP ("pytest-of-" + $env:USERNAME)' in text + assert 'Join-Path $env:RUNNER_TEMP "apm-windows-installer"' in text + assert 'throw "RUNNER_TEMP is required; this proof is CI-only"' in text + assert 'throw "TEMP is required to reproduce the Windows pytest default root"' in text + assert "Refusing to reuse preexisting proof basetemp" in text + assert "Remove-OwnedBaseTemp -Path $longBaseTemp" in text + assert "Remove-OwnedBaseTemp -Path $shortBaseTemp" in text + assert "--basetemp" in text + assert "if (Test-Path -LiteralPath $BaseTemp)" in text + assert "APM Install Test & Edge " in text + assert "if ($shortResult.timed_out -or $shortResult.exit_code -ne 0)" in text + assert "exit $shortResult.exit_code" in text + + +def test_windows_installer_proof_records_charset_paths_and_ps5_stderr_confound() -> None: + """Diagnostics must capture the observed package and PowerShell behavior.""" + text = _script_text() + assert "archive-path-inventory.json" in text + assert "$allEntries = @($zip.Entries)" in text + assert "member_count = $allEntries.Count" in text + assert "file_count = $files.Count" in text + assert "charset_normalizer|chardet" in text + assert '".pyd"' in text + assert "temp_length" in text + assert "stage_length" in text + assert "final_length" in text + assert "powershell_host" in text + assert "powershell.exe" in text + assert "deterministic-native-stderr" in text + assert "exit23-stderr" in text + assert "Broken direct exit23 native-stderr control" in text + assert "$caseNames.Count -ne 2" in text + assert '-not $case.threw -and $case.name -eq "exit0-stderr"' in text + assert '-not $case.threw -and $case.name -eq "exit23-stderr"' in text + assert "Expected Windows PowerShell 5.1" in text + assert "native-stderr-behavior.json" in text + assert 'Invoke-DirectCandidateProbe -Name "long-root"' in text + assert 'Invoke-DirectCandidateProbe -Name "short-root"' in text + assert '"direct-$Name.json"' in text + + +def test_windows_installer_proof_process_capture_selftest(tmp_path: Path) -> None: + """Exercise timeout, stderr, and nonzero capture through the real PowerShell helper.""" + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("pwsh is not installed") + out_dir = tmp_path / "capture" + completed = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(PROOF_SCRIPT), + "-SelfTestProcessCapture", + "-PythonExecutable", + sys.executable, + "-OutputDir", + str(out_dir), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=45, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + payload = json.loads((out_dir / "process-capture-selftest.json").read_text()) + assert payload["exit0"]["exit_code"] == 0 + assert "selftest-exit0-stderr" in payload["exit0"]["stderr"] + assert payload["exit23"]["exit_code"] == 23 + assert "selftest-exit23-stderr" in payload["exit23"]["stderr"] + assert payload["timeout"]["timed_out"] is True From f8d0a4ab7f8c45534357650240013ec4cc108a3b Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 11:36:38 +0200 Subject: [PATCH 12/23] test(ci): reject skipped Windows installer proof Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/windows/run-installer-proof.ps1 | 36 ++++++++++++++- .../test_windows_installer_proof_workflow.py | 45 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/scripts/windows/run-installer-proof.ps1 b/scripts/windows/run-installer-proof.ps1 index 6f94ef40f8..ac166a8da1 100644 --- a/scripts/windows/run-installer-proof.ps1 +++ b/scripts/windows/run-installer-proof.ps1 @@ -13,6 +13,8 @@ param( [string]$BaselineVersion, [string]$OutputDir = "windows-installer-proof", [switch]$SelfTestProcessCapture, + [string]$ValidateInstallerJUnit, + [int]$InstallerExitCode = 0, [string]$PythonExecutable = "python" ) @@ -348,6 +350,21 @@ function Invoke-NativeCase { return $result } +function Read-InstallerOutcome { + param([string]$JUnitPath, [int]$ExitCode) + [xml]$report = Get-Content -LiteralPath $JUnitPath -Raw -ErrorAction Stop + $cases = @($report.SelectNodes("//testcase")) + $skipped = @($report.SelectNodes("//testcase/skipped")).Count + $failed = @($report.SelectNodes("//testcase/failure | //testcase/error")).Count + if ($cases.Count -eq 0 -or $skipped -ne 0) { + throw "Installer assertions did not execute completely: $($cases.Count) cases, $skipped skipped" + } + if ($ExitCode -eq 0 -and $failed -ne 0) { + throw "Installer exit code claimed success despite $failed failing JUnit cases" + } + return [ordered]@{ cases = $cases.Count; skipped = $skipped; failed = $failed } +} + function Invoke-InstallerPytest { param( [string]$Name, @@ -367,6 +384,8 @@ function Invoke-InstallerPytest { ) $result = Invoke-CapturedProcess -FilePath "uv" -Arguments $arguments ` -Name "$Name installer pytest" -TimeoutSeconds 900 -Environment $InstallerEnvironment + $result.junit = Read-InstallerOutcome -JUnitPath (Join-Path $OutDir "junit-$Name.xml") ` + -ExitCode $result.exit_code $result.stdout | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stdout.log") -Encoding UTF8 $result.stderr | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stderr.log") -Encoding UTF8 Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") @@ -405,6 +424,12 @@ function Invoke-ProcessCaptureSelfTest { } } +if ($ValidateInstallerJUnit) { + Read-InstallerOutcome -JUnitPath $ValidateInstallerJUnit -ExitCode $InstallerExitCode | + ConvertTo-Json + exit 0 +} + if ($SelfTestProcessCapture) { Invoke-ProcessCaptureSelfTest -OutDir $OutputDir -Python $PythonExecutable exit 0 @@ -434,8 +459,14 @@ if ((Get-Sha256Hex -Path $BaselineArchive) -cne $baselineSha) { throw "Baseline archive hash does not match sidecar" } +$resolvedTemp = Invoke-CapturedProcess -FilePath "python" -Arguments @( + "-c", "import os; print(os.path.realpath(os.environ['TEMP']))" +) -Name "Resolve actual long TEMP path" +if ($resolvedTemp.timed_out -or $resolvedTemp.exit_code -ne 0) { + throw "Could not resolve the actual Windows TEMP path" +} $longChild = "wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5) -$longBaseTemp = Join-Path (Join-Path $env:TEMP ("pytest-of-" + $env:USERNAME)) $longChild +$longBaseTemp = Join-Path (Join-Path $resolvedTemp.stdout.Trim() ("pytest-of-" + $env:USERNAME)) $longChild $shortBaseTemp = Join-Path $env:RUNNER_TEMP "apm-windows-installer" Invoke-PowerShellNativeStderrProbe -OutDir $OutputDir | Out-Null $longOwned = $false @@ -458,6 +489,7 @@ try { } $installerEnvironment = @{ + "APM_E2E_TESTS" = "1" "APM_CANDIDATE_ARCHIVE" = $CandidateArchive "APM_CANDIDATE_VERSION" = $candidateVersion "APM_CANDIDATE_SHA256" = $candidateSha @@ -489,6 +521,8 @@ $summary = [ordered]@{ long_root_timed_out = $longResult.timed_out short_root_exit_code = $shortResult.exit_code short_root_timed_out = $shortResult.timed_out + long_root_junit = $longResult.junit + short_root_junit = $shortResult.junit conclusion = if ($shortResult.exit_code -eq 0 -and $longResult.exit_code -ne 0) { "short-root-passed-long-root-failed" } elseif ($shortResult.exit_code -eq 0 -and $longResult.exit_code -eq 0) { diff --git a/tests/unit/test_windows_installer_proof_workflow.py b/tests/unit/test_windows_installer_proof_workflow.py index a22261bdb7..1810eae7b6 100644 --- a/tests/unit/test_windows_installer_proof_workflow.py +++ b/tests/unit/test_windows_installer_proof_workflow.py @@ -146,7 +146,7 @@ def test_windows_installer_proof_script_runs_same_archive_through_both_roots() - assert 'Invoke-InstallerPytest -Name "long-root"' in text assert 'Invoke-InstallerPytest -Name "short-root"' in text assert '"wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5)' in text - assert 'Join-Path $env:TEMP ("pytest-of-" + $env:USERNAME)' in text + assert 'Join-Path $resolvedTemp.stdout.Trim() ("pytest-of-" + $env:USERNAME)' in text assert 'Join-Path $env:RUNNER_TEMP "apm-windows-installer"' in text assert 'throw "RUNNER_TEMP is required; this proof is CI-only"' in text assert 'throw "TEMP is required to reproduce the Windows pytest default root"' in text @@ -158,6 +158,8 @@ def test_windows_installer_proof_script_runs_same_archive_through_both_roots() - assert "APM Install Test & Edge " in text assert "if ($shortResult.timed_out -or $shortResult.exit_code -ne 0)" in text assert "exit $shortResult.exit_code" in text + assert '"APM_E2E_TESTS" = "1"' in text + assert "$result.junit = Read-InstallerOutcome" in text def test_windows_installer_proof_records_charset_paths_and_ps5_stderr_confound() -> None: @@ -219,3 +221,44 @@ def test_windows_installer_proof_process_capture_selftest(tmp_path: Path) -> Non assert payload["exit23"]["exit_code"] == 23 assert "selftest-exit23-stderr" in payload["exit23"]["stderr"] assert payload["timeout"]["timed_out"] is True + + +@pytest.mark.parametrize( + ("cases", "exit_code", "accepted"), + [ + ('', 0, True), + ('', 0, False), + ("", 0, False), + ('', 0, False), + ('', 1, True), + ], +) +def test_installer_outcome_requires_executed_cases( + tmp_path: Path, cases: str, exit_code: int, accepted: bool +) -> None: + """A green pytest exit with skipped assertions is not an installer proof.""" + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("pwsh is not installed") + report = tmp_path / "junit.xml" + report.write_text(f"{cases}") + completed = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(PROOF_SCRIPT), + "-ValidateInstallerJUnit", + str(report), + "-InstallerExitCode", + str(exit_code), + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert (completed.returncode == 0) is accepted, completed.stdout + completed.stderr + if accepted: + assert json.loads(completed.stdout)["skipped"] == 0 From a58042febedd67e7e70a0157fab639dc04036983 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 12:02:01 +0200 Subject: [PATCH 13/23] test(ci): freeze full-release wall-clock comparisons Add read-only, non-promotable full-DAG proof with strict archive, job, source identity, and actual GitHub clock validation. Preserve production job authorities and default test scheduling. Use the empirically proven short Windows installer temp root and reject empty or skipped installer evidence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-release-wallclock.yml | 267 ++++ .github/workflows/ci.yml | 6 +- .github/workflows/release-integration.yml | 14 +- .github/workflows/release-platform.yml | 23 +- .github/workflows/release-unit.yml | 18 +- .../docs/contributing/integration-testing.md | 33 + scripts/release-candidate.cjs | 1 + scripts/release_wallclock.py | 1256 +++++++++++++++++ tests/unit/scripts/test_release_wallclock.py | 1039 ++++++++++++++ tests/unit/test_platform_contract_workflow.py | 29 +- tests/unit/test_release_wallclock_workflow.py | 276 ++++ tests/unit/test_windows_unit_diagnostics.py | 25 +- tests/workflow_contracts.py | 12 + 13 files changed, 2986 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/ci-release-wallclock.yml create mode 100644 scripts/release_wallclock.py create mode 100644 tests/unit/scripts/test_release_wallclock.py create mode 100644 tests/unit/test_release_wallclock_workflow.py diff --git a/.github/workflows/ci-release-wallclock.yml b/.github/workflows/ci-release-wallclock.yml new file mode 100644 index 0000000000..db10e6d0ce --- /dev/null +++ b/.github/workflows/ci-release-wallclock.yml @@ -0,0 +1,267 @@ +name: PR Release Wall-Clock Proof + +on: + pull_request: + types: [labeled] + push: + branches: [danielmeppiel-release-wallclock-proof] + +permissions: + contents: read + actions: read + +concurrency: + group: release-wallclock-proposed-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +jobs: + plan: + name: Plan proposed wall-clock run + if: github.event_name == 'push' || github.event.label.name == 'ci-release-wallclock' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: plan + name: Select all five production catalog platforms + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + if (Number(process.env.GITHUB_RUN_ATTEMPT) !== 1) { + throw new Error('Use a fresh labelled run, not a partial or mixed-attempt rerun'); + } + const fs = require('node:fs'); + const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); + core.setOutput('matrix', JSON.stringify({include: catalog})); + + source-checks: + name: Candidate Source Checks + needs: [plan] + uses: ./.github/workflows/ci.yml + with: + suppress-pr-only-checks: true + permissions: + contents: read + + platforms: + name: ${{ matrix.binary_name }} + needs: [plan] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + uses: ./.github/workflows/release-platform.yml + permissions: + contents: read + actions: read + with: + runner: ${{ matrix.runner }} + platform: ${{ matrix.platform }} + binary-name: ${{ matrix.binary_name }} + integration-markers: ${{ matrix.integration_markers }} + integration-shard-count: ${{ matrix.integration_shard_count }} + integration-xdist-workers: ${{ matrix.integration_xdist_workers }} + integration-splitting-algorithm: ${{ matrix.integration_splitting_algorithm }} + full-validation: true + wallclock-evidence: true + + docs: + name: Read-only documentation build + needs: [plan] + uses: ./.github/workflows/docs-build.yml + permissions: + contents: read + with: + pages_artifact_name: wallclock-proposed-docs-${{ github.run_attempt }} + upload_pages_artifact: true + + wheels: + name: Read-only Python distribution build + needs: [plan] + uses: ./.github/workflows/pypi-distributions.yml + permissions: + contents: read + with: + artifact_name: wallclock-proposed-python-${{ github.run_attempt }} + upload_artifact: true + + candidate-ready: + name: Record non-promotable native evidence + needs: [plan, platforms, source-checks] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + outputs: + native_ids: ${{ steps.record.outputs.native_ids }} + evidence_id: ${{ steps.upload.outputs.artifact-id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: candidate-${{ github.run_attempt }}-apm-* + path: candidate-artifacts + - name: Capture native bytes and successful authorities without qualification + id: record + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('node:fs'); + const { + collectArchiveInventory, candidateVersion, configHashes, selectRequiredJobs, + } = require('./scripts/release-candidate.cjs'); + const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); + const platforms = collectArchiveInventory({ + artifactRoot: 'candidate-artifacts', catalog, sha: context.sha, + }); + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, run_id: context.runId, attempt_number: attempt, per_page: 100, + }); + const required = selectRequiredJobs(jobs, catalog); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const selected = catalog.map(row => { + const name = `candidate-${attempt}-${row.binary_name}`; + const matches = artifacts.filter(artifact => artifact.name === name && !artifact.expired); + if (matches.length !== 1) throw new Error(`Missing or ambiguous native artifact: ${name}`); + return matches[0]; + }); + fs.writeFileSync('wallclock-native-index.json', JSON.stringify({ + kind: 'release-wallclock-native-index', promotable: false, + head_sha: context.sha, run_id: context.runId, run_attempt: attempt, + version: candidateVersion(platforms, context), + config_hashes: configHashes(process.cwd()), platforms, required, artifacts: selected, + }, null, 2)); + core.setOutput('native_ids', selected.map(artifact => artifact.id).join(',')); + - id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wallclock-proposed-native-index-${{ github.run_attempt }} + path: wallclock-native-index.json + if-no-files-found: error + retention-days: 30 + + verify: + name: Verify wall-clock artifacts + needs: [plan, source-checks, platforms, candidate-ready, docs, wheels] + if: always() && !cancelled() && needs.plan.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Require every production authority and record artifact identities + id: authorities + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RESULTS: ${{ toJSON(needs) }} + NATIVE_IDS: ${{ needs.candidate-ready.outputs.native_ids }} + with: + script: | + const fs = require('node:fs'); + const {selectRequiredJobs} = require('./scripts/release-candidate.cjs'); + const results = JSON.parse(process.env.RESULTS); + for (const name of ['plan', 'source-checks', 'platforms', 'candidate-ready', 'docs', 'wheels']) { + if (results[name]?.result !== 'success') throw new Error(`${name} did not succeed`); + } + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, run_id: context.runId, + attempt_number: Number(process.env.GITHUB_RUN_ATTEMPT), per_page: 100, + }); + const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); + const required = selectRequiredJobs(jobs, catalog); + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + const attempt = process.env.GITHUB_RUN_ATTEMPT; + const expected = [ + ...catalog.map(row => `candidate-${attempt}-${row.binary_name}`), + `wallclock-proposed-docs-${attempt}`, `wallclock-proposed-python-${attempt}`, + ]; + const selected = expected.map(name => { + const matches = artifacts.filter(artifact => artifact.name === name && !artifact.expired); + if (matches.length !== 1) throw new Error(`Missing or ambiguous artifact: ${name}`); + return matches[0]; + }); + const nativeIds = selected.slice(0, catalog.length).map(artifact => artifact.id).join(','); + if (nativeIds !== process.env.NATIVE_IDS) { + throw new Error('Native artifact IDs changed after evidence capture'); + } + core.setOutput('docs_id', selected[catalog.length].id); + core.setOutput('python_id', selected[catalog.length + 1].id); + fs.mkdirSync('wallclock-results', {recursive: true}); + fs.writeFileSync('wallclock-results/authorities.json', JSON.stringify({ + promotable: false, required, artifacts: selected, + }, null, 2)); + - name: Download immutable non-promotable native evidence + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ needs.candidate-ready.outputs.evidence_id }} + path: wallclock-native-evidence + - name: Download exact current-run native artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ needs.candidate-ready.outputs.native_ids }} + path: wallclock-inputs/native + - name: Download documentation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ steps.authorities.outputs.docs_id }} + path: wallclock-inputs/docs + - name: Download Python distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ steps.authorities.outputs.python_id }} + path: wallclock-inputs/python + - name: Recheck indexed native bytes and prepare non-promotable assets + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('node:fs'); + const {configHashes, verifyLocalArchiveBytes} = require('./scripts/release-candidate.cjs'); + const evidence = JSON.parse(fs.readFileSync( + 'wallclock-native-evidence/wallclock-native-index.json', 'utf8')); + if (evidence.kind !== 'release-wallclock-native-index' || evidence.promotable !== false || + evidence.run_id !== context.runId || evidence.head_sha !== context.sha || + evidence.run_attempt !== Number(process.env.GITHUB_RUN_ATTEMPT) || + JSON.stringify(evidence.config_hashes) !== JSON.stringify(configHashes(process.cwd()))) { + throw new Error('Non-promotable native index does not match this execution'); + } + const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); + verifyLocalArchiveBytes({ + evidence, catalog, artifactRoot: 'wallclock-inputs/native', + outputRoot: 'wallclock-verified-native', + }); + - name: Verify built bytes without granting release qualification + run: >- + python -m scripts.release_wallclock verify + --side proposed --source-root . --source-sha "$GITHUB_SHA" + --native-root wallclock-inputs/native + --docs-root wallclock-inputs/docs + --python-root wallclock-inputs/python + --output wallclock-results/proof.json + - name: Upload verified native bytes without publishing + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wallclock-proposed-verified-native-${{ github.run_attempt }} + path: wallclock-verified-native/ + compression-level: 0 + if-no-files-found: error + retention-days: 7 + - name: Preserve non-promotable wall-clock evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wallclock-proposed-proof-${{ github.run_attempt }} + path: wallclock-results/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a1f858174..495c85621d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ env: on: workflow_call: + inputs: + suppress-pr-only-checks: + type: boolean + default: false pull_request: branches: [ main ] # Tier 1 also runs in merge queue context so the same unit + build checks @@ -394,7 +398,7 @@ jobs: # and in build-release.yml on release. pr-binary-smoke: name: PR Binary Smoke (Linux) - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && !inputs.suppress-pr-only-checks runs-on: ubuntu-24.04 permissions: contents: read diff --git a/.github/workflows/release-integration.yml b/.github/workflows/release-integration.yml index 5d4274768f..2bcdd32bf3 100644 --- a/.github/workflows/release-integration.yml +++ b/.github/workflows/release-integration.yml @@ -63,6 +63,9 @@ on: performance-cohort: type: string default: '' + wallclock-evidence: + type: boolean + default: false secrets: GH_CLI_PAT: required: false @@ -92,6 +95,13 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Record actual integration runner for wall-clock proof + if: inputs.wallclock-evidence + shell: bash + run: >- + python -m scripts.release_wallclock record-job + --side proposed --source-root . --source-sha "$GITHUB_SHA" + --output test-results/integration-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -154,7 +164,7 @@ jobs: PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} APM_TEST_RUNTIMES: ${{ inputs.runtime-prerequisites }} PYTEST_EXTRA_ARGS: >- - ${{ inputs.evidence-variant != '' && '-p scripts.pytest_performance_evidence' || '' }} + ${{ (inputs.evidence-variant != '' || inputs.wallclock-evidence) && '-p scripts.pytest_performance_evidence' || '' }} --splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} --splitting-algorithm ${{ inputs.splitting-algorithm }} -n ${{ inputs.xdist-workers }} --dist loadgroup @@ -189,7 +199,7 @@ jobs: --shard-count "${{ inputs.shard-count }}" \ --candidate-metadata "release-assets/${{ inputs.binary-name }}.json" - name: Upload integration timings and outcomes - if: always() && inputs.platform != 'windows' + if: always() && (inputs.platform != 'windows' || inputs.wallclock-evidence) uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-results-${{ inputs.artifact-prefix }}-integration-shard-${{ inputs.shard-index }} diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml index 6dabb05e21..e138804dc3 100644 --- a/.github/workflows/release-platform.yml +++ b/.github/workflows/release-platform.yml @@ -33,6 +33,9 @@ on: performance-workers: type: number default: 4 + wallclock-evidence: + type: boolean + default: false full-validation: type: boolean required: true @@ -65,6 +68,7 @@ jobs: binary-name: ${{ inputs.binary-name }} performance-probe: ${{ inputs.unit-performance-probe }} proposed-workers: ${{ inputs.performance-workers }} + wallclock-evidence: ${{ inputs.wallclock-evidence }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} GH_MODELS_PAT: ${{ secrets.GH_MODELS_PAT }} @@ -78,6 +82,13 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Record actual build runner for wall-clock proof + if: inputs.wallclock-evidence + shell: bash + run: >- + python -m scripts.release_wallclock record-job + --side proposed --source-root . --source-sha "$GITHUB_SHA" + --output test-results/build-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -182,6 +193,7 @@ jobs: runtime-prerequisites: none evidence-variant: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && 'baseline' || '' }} evidence-selection: tests/integration/ -m ${{ inputs.integration-markers }} + wallclock-evidence: ${{ inputs.wallclock-evidence }} performance-cohort: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} @@ -293,7 +305,8 @@ jobs: timeout-minutes: 20 env: GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} + # Preserve the legacy public-dependency gate in read-only measurements. + GITHUB_APM_PAT: ${{ inputs.wallclock-evidence && github.token || secrets.GH_CLI_PAT }} APM_E2E_TESTS: '1' run: | $env:PATH = "$PWD\dist\apm-windows-x86_64;$env:PATH" @@ -347,7 +360,13 @@ jobs: $candidate = Get-Content release-assets/apm-windows-x86_64.json -Raw | ConvertFrom-Json $env:APM_CANDIDATE_VERSION = "v$($candidate.version)" $env:APM_CANDIDATE_SHA256 = $candidate.archive_sha256 - uv run --frozen pytest tests/integration/test_windows_installer_launchers.py -vv -ra --tb=short --junitxml=test-results/installer.xml + $installerBaseTemp = Join-Path $env:RUNNER_TEMP "apm-windows-installer" + uv run --frozen pytest tests/integration/test_windows_installer_launchers.py -vv -ra --tb=short --junitxml=test-results/installer.xml --basetemp "$installerBaseTemp" + - name: Require executed installer assertions + shell: pwsh + run: >- + scripts/windows/run-installer-proof.ps1 + -ValidateInstallerJUnit test-results/installer.xml -InstallerExitCode 0 - name: Upload installer outcomes if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/release-unit.yml b/.github/workflows/release-unit.yml index f702dde758..2cca632500 100644 --- a/.github/workflows/release-unit.yml +++ b/.github/workflows/release-unit.yml @@ -18,6 +18,9 @@ on: proposed-workers: type: number default: 4 + wallclock-evidence: + type: boolean + default: false secrets: GH_CLI_PAT: required: false @@ -41,6 +44,13 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Record actual unit runner for wall-clock proof + if: inputs.wallclock-evidence + shell: bash + run: >- + python -m scripts.release_wallclock record-job + --side proposed --source-root . --source-sha "$GITHUB_SHA" + --output test-results/unit-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -68,7 +78,9 @@ jobs: GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} run: >- - uv run --frozen pytest tests/unit tests/test_console.py + uv run --frozen + ${{ inputs.wallclock-evidence && 'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }} + tests/unit tests/test_console.py -n auto --dist worksteal --durations=50 --store-durations --junitxml=test-results/unit.xml - name: Run unit performance baseline @@ -89,7 +101,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} run: >- - uv run --frozen pytest tests/unit tests/test_console.py + uv run --frozen + ${{ inputs.wallclock-evidence && 'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }} + tests/unit tests/test_console.py -n auto --dist worksteal --durations=50 --store-durations --junitxml=test-results/unit.xml -vv --tb=short --show-capture=no --no-showlocals diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 4ec5ad0ff0..a30195b332 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -647,6 +647,39 @@ source CI artifacts use bare names. Limit an evaluation to three full attempts, including invalid or image-inconclusive attempts. Production rollout requires passing proof plus a maintainer decision. +### Actual release wall-clock proof + +Do not use modeled counterfactuals as proof. To measure real GitHub wall time +before merge, apply the `ci-release-wallclock` PR label for an ad hoc run, or +push the reviewed SHA to the workflow's dedicated control branch for a frozen +comparison. The control ref avoids a moving PR merge ref changing the tested +source after the baseline starts. Each trigger runs one proposed +production-default DAG in `ci-release-wallclock.yml`, while a separate frozen +baseline controller branch runs the original dependency ordering from the +selected `main` source. Both sides are non-promotable and use no PATs, signing +secrets, private-service acceptance, publication, or deployment scope. + +Compare only fresh attempt-1 run IDs recorded by the wall-clock protocol. Source +identity and controller identity are recorded separately. The primary clock is +the terminal `Verify wall-clock artifacts` job `completed_at` minus workflow +`created_at`, including queue time; the secondary clock starts at actual +`run_started_at`. Accept no result unless both workflows succeed, every included +gate succeeds, all five native archives plus docs and wheel artifacts verify, +and observer case inventory plus runner environment proof are present. The +proposed wrapper preserves separate candidate-ready native evidence capture and +serial verify / verified-native upload. All `wallclock-*` storage names are +non-promotable. Workflow success alone is not a savings claim; cite two accepted +run IDs and their actual primary and secondary clocks before claiming an +improvement. + +For native Windows installer diagnostics, apply the optional +`ci-windows-installer-proof` PR label. It runs one candidate through the same +assertions at long and short roots, checks native exit/stderr controls, and +retains the archive; this is diagnostic evidence, not whole-pipeline savings +proof. Production Windows installer tests use a short pytest-owned +`RUNNER_TEMP/apm-windows-installer` base temp, keep spaces and `&` fixture +cases, and reject zero/skipped JUnit instead of treating pytest exit 0 as proof. + ## Debugging Test Failures ### Smoke Test Failures diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index 42eb1f73ed..ce88081cd9 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -778,5 +778,6 @@ module.exports = { selectRequiredJobs, candidateArtifactName, validateTrustedReusableRun, + verifyLocalArchiveBytes, verify, }; diff --git a/scripts/release_wallclock.py b/scripts/release_wallclock.py new file mode 100644 index 0000000000..ed9c51d541 --- /dev/null +++ b/scripts/release_wallclock.py @@ -0,0 +1,1256 @@ +"""Capture and compare read-only release wall-clock evidence. + +This module is deliberately not release qualification code. It verifies the +artifacts needed for an apples-to-apples timing comparison, emits a +non-promotable proof, and compares two already-successful GitHub Actions +attempts by their real terminal job timestamps. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import re +import shutil +import stat +import subprocess +import tarfile +import zipfile +from dataclasses import dataclass +from datetime import datetime, timezone +from email.parser import Parser +from pathlib import Path +from typing import Any + +import tomllib + +from scripts.package_release import ( + BINARY_NAMES, + archive_name, + executable_name, + file_digest, + require_member_path, + require_sha, +) + +ABI_VERSION = 1 +KIND_JOB = "release-wallclock-job" +KIND_PROOF = "release-wallclock-proof" +KIND_COMPARISON = "release-wallclock-comparison" +TERMINAL_JOB = "Verify wall-clock artifacts" +WORKFLOW_PATH = ".github/workflows/build-release.yml" +SIDE_CHOICES = ("baseline", "proposed") +BAD_JOB_CONCLUSIONS = {"failure", "cancelled", "timed_out", "action_required"} +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") + + +@dataclass(frozen=True) +class ArchiveEvidence: + """Read-only proof for one native archive.""" + + binary_name: str + archive: str + archive_sha256: str + executable_sha256: str + sidecar: str + candidate_metadata_sha256: str | None + loose_executable_sidecar_sha256: str | None + loose_executable_sha256: str | None + member_count: int + + def payload(self) -> dict[str, object]: + """Return a JSON-safe archive proof.""" + return { + "binary_name": self.binary_name, + "archive": self.archive, + "archive_sha256": self.archive_sha256, + "executable_sha256": self.executable_sha256, + "sidecar": self.sidecar, + "candidate_metadata_sha256": self.candidate_metadata_sha256, + "loose_executable_sidecar_sha256": self.loose_executable_sidecar_sha256, + "loose_executable_sha256": self.loose_executable_sha256, + "member_count": self.member_count, + } + + +def _json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="ascii") + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _timestamp(value: object, label: str) -> datetime: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} timestamp is required") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError(f"{label} timestamp must include a timezone") + return parsed + + +def _seconds_between(start: datetime, end: datetime, label: str) -> float: + value = (end - start).total_seconds() + if not math.isfinite(value) or value < 0: + raise ValueError(f"{label} has an invalid negative timeline") + return value + + +def _positive_int(value: object, label: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{label} must be a positive integer") + try: + result = int(str(value)) + except ValueError as error: + raise ValueError(f"{label} must be a positive integer") from error + if result < 1 or str(result) != str(value): + raise ValueError(f"{label} must be a positive integer") + return result + + +def _full_sha(value: object, label: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{label} must be a full lowercase Git SHA") + require_sha(value) + return value + + +def _side(value: str) -> str: + if value not in SIDE_CHOICES: + raise ValueError("side must be baseline or proposed") + return value + + +def _source_metadata(source_root: Path) -> tuple[str, str]: + data = tomllib.loads((source_root / "pyproject.toml").read_text(encoding="utf-8")) + name = data["project"]["name"] + version = data["project"]["version"] + if not isinstance(name, str) or not name: + raise ValueError("source pyproject.toml must contain project.name") + if not isinstance(version, str) or not version: + raise ValueError("source pyproject.toml must contain project.version") + return name, version + + +def _source_version(source_root: Path) -> str: + return _source_metadata(source_root)[1] + + +def _canonical_project_name(value: str) -> str: + """Normalize distribution names using the wheel/sdist comparison form.""" + if not value: + raise ValueError("Distribution name must be nonempty") + return re.sub(r"[-_.]+", "-", value).lower() + + +def _distribution_token(value: str) -> str: + return _canonical_project_name(value).replace("-", "_") + + +def _is_sha256(value: object) -> bool: + return isinstance(value, str) and SHA256_PATTERN.fullmatch(value) is not None + + +def _source_digests(source_root: Path) -> dict[str, str]: + files = { + "uv.lock": source_root / "uv.lock", + WORKFLOW_PATH: source_root / WORKFLOW_PATH, + } + missing = [name for name, path in files.items() if not path.is_file()] + if missing: + raise FileNotFoundError(f"Missing required source files: {', '.join(missing)}") + return {name: file_digest(path) for name, path in files.items()} + + +def _git_head(source_root: Path) -> str: + git = shutil.which("git") + if git is None: + raise OSError("git executable is required") + result = subprocess.run( # noqa: S603 - constant git executable, no shell. + [git, "-C", str(source_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + encoding="ascii", + timeout=30, + ) + return result.stdout.strip() + + +def _workflow_env(source_sha: str) -> dict[str, object]: + controller_sha = _full_sha(os.environ.get("GITHUB_SHA"), "GITHUB_SHA") + event_name, run_head_sha = _event_run_head_sha(controller_sha) + image = os.environ.get("ImageVersion", "") # noqa: SIM112 - GitHub-hosted image contract. + if not image: + raise ValueError("GitHub hosted runner ImageVersion is required") + runner_name = os.environ.get("RUNNER_NAME", "") + github_job = os.environ.get("GITHUB_JOB", "") + if not runner_name or not github_job: + raise ValueError("GITHUB_JOB and RUNNER_NAME are required") + return { + "source_sha": source_sha, + "controller_sha": controller_sha, + "run_head_sha": run_head_sha, + "event_name": event_name, + "run_id": _positive_int(os.environ.get("GITHUB_RUN_ID"), "GITHUB_RUN_ID"), + "run_attempt": _positive_int(os.environ.get("GITHUB_RUN_ATTEMPT"), "GITHUB_RUN_ATTEMPT"), + "github_job": github_job, + "runner_name": runner_name, + } + + +def _event_payload() -> tuple[str, dict[str, object]]: + event_name = os.environ.get("GITHUB_EVENT_NAME", "") + event_path = os.environ.get("GITHUB_EVENT_PATH", "") + if not event_name or not event_path: + raise ValueError("GITHUB_EVENT_NAME and GITHUB_EVENT_PATH are required") + payload = _json(Path(event_path)) + if not isinstance(payload, dict): + raise ValueError("GitHub event payload must be a JSON object") + return event_name, payload + + +def _event_run_head_sha(controller_sha: str) -> tuple[str, str]: + """Capture immutable event/API run head separately from execution GITHUB_SHA.""" + event_name, payload = _event_payload() + if event_name in {"pull_request", "pull_request_target"}: + pull_request = payload.get("pull_request") + if not isinstance(pull_request, dict): + raise ValueError("pull_request event payload is missing pull_request") + head = pull_request.get("head") + if not isinstance(head, dict): + raise ValueError("pull_request event payload is missing pull_request.head") + return event_name, _full_sha(head.get("sha"), "pull_request.head.sha") + if event_name == "push": + after = _full_sha(payload.get("after"), "push after") + if after != controller_sha: + raise ValueError("push payload after does not match execution GITHUB_SHA") + return event_name, after + # Non-PR, non-push read-only probes do not have a distinct immutable PR + # head. Bind API run.head_sha to the execution SHA for those event types. + return event_name, controller_sha + + +def _host_environment(source_digests: dict[str, str]) -> dict[str, str]: + cpu_count = os.cpu_count() + if cpu_count is None: + raise ValueError("CPU count is required") + cpu = platform.processor() or platform.machine() + if not cpu: + raise ValueError("CPU identity is required") + return { + "system": platform.system(), + "kernel": platform.release(), + "arch": platform.machine(), + "cpu": cpu, + "cpu_count": str(cpu_count), + "image_version": os.environ["ImageVersion"], # noqa: SIM112 - GitHub-hosted image contract. + "python": platform.python_version(), + "source_uv_lock_sha256": source_digests["uv.lock"], + "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], + } + + +def record_job(side: str, source_root: Path, source_sha: str) -> dict[str, object]: + """Capture a minimal hosted-runner identity for one read-only job.""" + side = _side(side) + source_root = source_root.resolve() + source_sha = _full_sha(source_sha, "source_sha") + actual = _git_head(source_root) + if actual != source_sha: + raise ValueError(f"source-root HEAD {actual} does not match expected {source_sha}") + source_digests = _source_digests(source_root) + workflow = _workflow_env(source_sha) + return { + "kind": KIND_JOB, + "schema_version": ABI_VERSION, + "side": side, + "recorded_at": _utc_now(), + "source_root": str(source_root), + "workflow": workflow, + "environment": _host_environment(source_digests), + } + + +def _walk_files(root: Path) -> list[Path]: + if not root.exists(): + raise FileNotFoundError(root) + return sorted(path for path in root.rglob("*") if path.is_file()) + + +def _find_one(root: Path, name: str) -> Path: + matches = [path for path in _walk_files(root) if path.name == name] + if len(matches) != 1: + raise ValueError(f"Expected exactly one {name} under {root}, found {len(matches)}") + return matches[0] + + +def _find_optional(root: Path, name: str) -> Path | None: + matches = [path for path in _walk_files(root) if path.name == name] + if len(matches) > 1: + raise ValueError(f"Expected at most one {name} under {root}, found {len(matches)}") + return matches[0] if matches else None + + +def _read_checksum_sidecar(path: Path, archive: str, digest: str) -> None: + line = path.read_text(encoding="ascii") + match = re.fullmatch(r"([0-9a-fA-F]{64})\s+\*?([A-Za-z0-9_.-]+)\n?", line) + if match is None: + raise ValueError(f"Malformed checksum sidecar: {path.name}") + observed, filename = match.group(1).lower(), match.group(2) + if observed != digest or filename != archive: + raise ValueError(f"Checksum sidecar mismatch: {path.name}") + + +def _parse_loose_checksum(path: Path, binary_name: str) -> tuple[str, str]: + line = path.read_text(encoding="ascii") + match = re.fullmatch(r"([0-9a-fA-F]{64})\s+\*?([A-Za-z0-9_.\-/]+)\n?", line) + if match is None: + raise ValueError(f"Malformed executable checksum sidecar: {path.name}") + digest, member = match.group(1).lower(), match.group(2) + archive_member = member.removeprefix("dist/") + require_member_path(archive_member, binary_name) + if archive_member != f"{binary_name}/{executable_name(binary_name)}": + raise ValueError(f"Executable checksum sidecar targets {member}") + return digest, member + + +def _normal_archive_parts(value: str) -> list[str] | None: + if ( + not value + or value.startswith("/") + or "\\" in value + or ":" in value + or value.startswith("//") + ): + return None + parts: list[str] = [] + for part in value.split("/"): + if part in ("", "."): + continue + if part == "..": + if not parts: + return None + parts.pop() + else: + parts.append(part) + return parts + + +def _link_is_safe(binary_name: str, member: str, target: str, *, relative_to_parent: bool) -> bool: + if target.startswith("/") or "\\" in target or ":" in target: + return False + base = str(Path(member).parent).replace("\\", "/") if relative_to_parent else "" + candidate = f"{base}/{target}" if base else target + parts = _normal_archive_parts(candidate) + if parts is None: + return False + return bool(parts) and parts[0] == binary_name + + +def _zip_mode(item: zipfile.ZipInfo) -> int: + return item.external_attr >> 16 + + +def _is_zip_symlink(item: zipfile.ZipInfo) -> bool: + return stat.S_ISLNK(_zip_mode(item)) + + +def _zip_member_digest(archive: Path, binary_name: str) -> tuple[str, int]: + executable = f"{binary_name}/{executable_name(binary_name)}" + seen: set[str] = set() + executable_digest: str | None = None + with zipfile.ZipFile(archive) as source: + members = source.infolist() + if not members: + raise ValueError(f"{archive.name} is empty") + for item in members: + require_member_path(item.filename, binary_name) + if item.filename in seen: + raise ValueError(f"Duplicate archive member: {item.filename}") + seen.add(item.filename) + if _is_zip_symlink(item): + raise ValueError("ZIP archive contains an unsupported symbolic link") + if item.filename == executable: + mode = _zip_mode(item) + if item.is_dir() or (mode and not stat.S_ISREG(mode)): + raise ValueError(f"{executable} must be a regular file") + if executable_name(binary_name) == "apm" and mode and not (mode & 0o111): + raise ValueError(f"{executable} must be executable") + with source.open(item) as stream: + executable_digest = file_digest_stream(stream) + if executable_digest is None: + raise ValueError(f"{archive.name} must contain exactly one {executable}") + return executable_digest, len(seen) + + +def file_digest_stream(stream: Any) -> str: + """Hash an already-open binary stream without extracting it to disk.""" + import hashlib + + digest = hashlib.sha256() + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _tar_member_digest(archive: Path, binary_name: str) -> tuple[str, int]: + executable = f"{binary_name}/{executable_name(binary_name)}" + seen: set[str] = set() + executable_digest: str | None = None + with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + if not members: + raise ValueError(f"{archive.name} is empty") + for item in members: + require_member_path(item.name, binary_name) + if item.name in seen: + raise ValueError(f"Duplicate archive member: {item.name}") + seen.add(item.name) + if item.issym() and not _link_is_safe( + binary_name, item.name, item.linkname, relative_to_parent=True + ): + raise ValueError("Archive symbolic link escapes its native bundle") + if item.islnk() and not _link_is_safe( + binary_name, item.name, item.linkname, relative_to_parent=False + ): + raise ValueError("Archive hard link escapes its native bundle") + if item.name == executable: + if not item.isfile(): + raise ValueError(f"{executable} must be a regular file") + if not (item.mode & 0o111): + raise ValueError(f"{executable} must be executable") + stream = source.extractfile(item) + if stream is None: + raise ValueError(f"Cannot read {executable}") + with stream: + executable_digest = file_digest_stream(stream) + if executable_digest is None: + raise ValueError(f"{archive.name} must contain exactly one {executable}") + return executable_digest, len(seen) + + +def _candidate_metadata( + metadata_root: Path, + side: str, + binary_name: str, + archive: str, + source_sha: str, + version: str, + archive_digest: str, + executable_digest: str, +) -> str | None: + path = _find_optional(metadata_root, f"{binary_name}.json") + if path is None: + if side == "proposed": + raise ValueError(f"Proposed proof requires candidate metadata for {binary_name}") + return None + data = _json(path) + required = { + "schema_version": 1, + "sha": source_sha, + "version": version, + "binary_name": binary_name, + "archive": archive, + "archive_sha256": archive_digest, + "executable_sha256": executable_digest, + } + for key, expected in required.items(): + if data.get(key) != expected: + raise ValueError(f"Candidate metadata {binary_name} {key} mismatch") + return file_digest(path) + + +def _loose_path(raw_root: Path, sidecar: Path, member: str) -> Path | None: + candidates = [ + raw_root / member, + sidecar.parent / member, + sidecar.parent / member.removeprefix("dist/"), + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + +def _loose_executable_sidecar( + raw_root: Path, side: str, binary_name: str, executable_digest: str +) -> tuple[str | None, str | None]: + path = _find_optional(raw_root, f"{binary_name}.sha256") + if path is None: + if side == "baseline": + raise ValueError(f"Baseline raw executable checksum is missing: {binary_name}.sha256") + return None, None + expected, member = _parse_loose_checksum(path, binary_name) + if expected != executable_digest: + raise ValueError(f"Loose executable checksum mismatch: {path.name}") + loose = _loose_path(raw_root, path, member) + if loose is None: + if side == "baseline": + raise ValueError(f"Baseline raw executable payload is missing: {member}") + return file_digest(path), None + loose_digest = file_digest(loose) + if loose_digest != executable_digest: + raise ValueError(f"Loose executable digest mismatch: {loose}") + return file_digest(path), loose_digest + + +def _verify_native_archives( + side: str, native_root: Path, source_sha: str, version: str +) -> dict[str, dict[str, object]]: + archives_root = native_root / "archives" if (native_root / "archives").is_dir() else native_root + raw_root = native_root / "raw" if (native_root / "raw").is_dir() else native_root + inventory: dict[str, dict[str, object]] = {} + for binary_name in BINARY_NAMES: + archive = archive_name(binary_name) + archive_path = _find_one(archives_root, archive) + sidecar_path = _find_one(archives_root, f"{archive}.sha256") + archive_digest = file_digest(archive_path) + _read_checksum_sidecar(sidecar_path, archive, archive_digest) + if archive.endswith(".zip"): + executable_digest, member_count = _zip_member_digest(archive_path, binary_name) + else: + executable_digest, member_count = _tar_member_digest(archive_path, binary_name) + metadata_digest = _candidate_metadata( + archives_root, + side, + binary_name, + archive, + source_sha, + version, + archive_digest, + executable_digest, + ) + loose_sidecar_digest, loose_executable_digest = _loose_executable_sidecar( + raw_root, side, binary_name, executable_digest + ) + inventory[binary_name] = ArchiveEvidence( + binary_name=binary_name, + archive=archive, + archive_sha256=archive_digest, + executable_sha256=executable_digest, + sidecar=f"{archive}.sha256", + candidate_metadata_sha256=metadata_digest, + loose_executable_sidecar_sha256=loose_sidecar_digest, + loose_executable_sha256=loose_executable_digest, + member_count=member_count, + ).payload() + _reject_unexpected_native_release_files(archives_root, side) + return inventory + + +def _reject_unexpected_native_release_files(native_root: Path, side: str) -> None: + expected: set[str] = set() + for binary_name in BINARY_NAMES: + archive = archive_name(binary_name) + expected.update({archive, f"{archive}.sha256", f"{binary_name}.sha256"}) + if side == "proposed": + expected.add(f"{binary_name}.json") + release_like = [ + path.name + for path in _walk_files(native_root) + if path.name.startswith("apm-") + and re.search(r"\.(tar\.gz|zip|sha256|json)$", path.name) + and path.name not in expected + ] + if release_like: + raise ValueError(f"Unexpected native release files: {', '.join(sorted(release_like))}") + + +def _docs_index(docs_root: Path) -> dict[str, object]: + root_index = docs_root / "index.html" + if root_index.is_file(): + if root_index.stat().st_size <= 0: + raise ValueError("docs index.html is empty") + return {"format": "directory", "index": "index.html", "sha256": file_digest(root_index)} + tars = [ + path + for path in _walk_files(docs_root) + if path.suffix == ".tar" or path.name.endswith(".tar.gz") + ] + if len(tars) != 1: + raise ValueError("Docs artifact must contain index.html or exactly one tar archive") + with tarfile.open(tars[0], "r:*") as source: + matches = [ + item + for item in source.getmembers() + if item.name.removeprefix("./").rstrip("/") == "index.html" + ] + if len(matches) != 1: + raise ValueError("Docs artifact tar must contain exactly one root index.html") + index = matches[0] + if not index.isfile() or index.size <= 0: + raise ValueError("Docs artifact index.html must be a nonempty regular file") + return { + "format": "tar", + "archive": tars[0].name, + "archive_sha256": file_digest(tars[0]), + "index": "index.html", + } + + +def _metadata_value(blob: bytes, key: str) -> str: + parsed = Parser().parsestr(blob.decode("utf-8")) + value = parsed.get(key) + if not value: + raise ValueError(f"Distribution metadata is missing {key}") + return value + + +def _wheel_distribution(path: Path, project_name: str, version: str) -> dict[str, object]: + expected_name = _canonical_project_name(project_name) + expected_token = _distribution_token(project_name) + if not path.name.startswith(f"{expected_token}-{version}-") or not path.name.endswith(".whl"): + raise ValueError(f"Wheel filename does not match source project/version: {path.name}") + with zipfile.ZipFile(path) as wheel: + names = wheel.namelist() + metadata = [ + name for name in names if name.endswith(".dist-info/METADATA") and name.count("/") == 1 + ] + if len(metadata) != 1: + raise ValueError(f"Wheel must contain exactly one METADATA: {path.name}") + dist_info = metadata[0].removesuffix("/METADATA") + required = {f"{dist_info}/METADATA", f"{dist_info}/WHEEL", f"{dist_info}/RECORD"} + if not required.issubset(names): + raise ValueError(f"Wheel is missing METADATA, WHEEL or RECORD: {path.name}") + if not dist_info.startswith(f"{expected_token}-{version}.dist-info"): + raise ValueError(f"Wheel dist-info directory does not match source: {path.name}") + if not any( + name.startswith(f"{expected_token}/") and not name.endswith("/") for name in names + ): + raise ValueError(f"Wheel does not contain source package code: {path.name}") + with wheel.open(f"{dist_info}/METADATA") as stream: + metadata_blob = stream.read() + with wheel.open(f"{dist_info}/WHEEL") as stream: + if not stream.read().strip(): + raise ValueError(f"Wheel WHEEL metadata is empty: {path.name}") + with wheel.open(f"{dist_info}/RECORD") as stream: + if not stream.read().strip(): + raise ValueError(f"Wheel RECORD metadata is empty: {path.name}") + name = _metadata_value(metadata_blob, "Name") + found_version = _metadata_value(metadata_blob, "Version") + if _canonical_project_name(name) != expected_name or found_version != version: + raise ValueError("Wheel metadata name/version does not match source pyproject") + return { + "name": path.name, + "sha256": file_digest(path), + "project_name": name, + "version": found_version, + "dist_info": dist_info, + } + + +def _sdist_distribution(path: Path, project_name: str, version: str) -> dict[str, object]: + expected_name = _canonical_project_name(project_name) + expected_token = _distribution_token(project_name) + expected_root = f"{expected_token}-{version}" + if path.name != f"{expected_root}.tar.gz": + raise ValueError(f"sdist filename does not match source project/version: {path.name}") + with tarfile.open(path, "r:gz") as sdist: + members = sdist.getmembers() + names = [item.name.rstrip("/") for item in members if item.name.rstrip("/")] + roots = {name.split("/", 1)[0] for name in names} + if roots != {expected_root}: + raise ValueError(f"sdist top-level directory does not match source: {path.name}") + metadata = [item for item in members if item.name == f"{expected_root}/PKG-INFO"] + if len(metadata) != 1: + raise ValueError(f"sdist must contain exactly one top-level PKG-INFO: {path.name}") + pyprojects = [item for item in members if item.name == f"{expected_root}/pyproject.toml"] + if len(pyprojects) != 1 or not pyprojects[0].isfile() or pyprojects[0].size <= 0: + raise ValueError(f"sdist must contain top-level pyproject.toml: {path.name}") + if not any( + item.isfile() + and ( + item.name.startswith(f"{expected_root}/src/{expected_token}/") + or item.name.startswith(f"{expected_root}/{expected_token}/") + ) + for item in members + ): + raise ValueError(f"sdist does not contain source package code: {path.name}") + stream = sdist.extractfile(metadata[0]) + if stream is None: + raise ValueError(f"Cannot read sdist metadata: {path.name}") + with stream: + metadata_blob = stream.read() + name = _metadata_value(metadata_blob, "Name") + found_version = _metadata_value(metadata_blob, "Version") + if _canonical_project_name(name) != expected_name or found_version != version: + raise ValueError("sdist metadata name/version does not match source pyproject") + return { + "name": path.name, + "sha256": file_digest(path), + "project_name": name, + "version": found_version, + "root": expected_root, + } + + +def _python_distributions(python_root: Path, project_name: str, version: str) -> dict[str, object]: + wheels = [path for path in _walk_files(python_root) if path.suffix == ".whl"] + sdists = [ + path + for path in _walk_files(python_root) + if path.name.endswith(".tar.gz") and not path.name.startswith("apm-") + ] + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError( + f"Expected exactly one wheel and one sdist, found {len(wheels)} wheel(s) and {len(sdists)} sdist(s)" + ) + return { + "source_project_name": project_name, + "wheel": _wheel_distribution(wheels[0], project_name, version), + "sdist": _sdist_distribution(sdists[0], project_name, version), + } + + +def _job_records( + roots: list[Path], side: str, source_sha: str, workflow: dict[str, object] +) -> list[dict[str, object]]: + records = [] + for root in roots: + for path in _walk_files(root): + if path.suffix != ".json": + continue + data = _json(path) + if not isinstance(data, dict) or data.get("kind") != KIND_JOB: + continue + if data.get("side") != side: + raise ValueError(f"Wall-clock job record side mismatch: {path}") + record_workflow = data.get("workflow") + if not isinstance(record_workflow, dict): + raise ValueError(f"Wall-clock job record workflow malformed: {path}") + for key in ("source_sha", "run_id", "run_attempt", "controller_sha", "run_head_sha"): + if record_workflow.get(key) != workflow[key]: + raise ValueError(f"Wall-clock job record {key} mismatch: {path}") + if record_workflow.get("source_sha") != source_sha: + raise ValueError(f"Wall-clock job record source mismatch: {path}") + records.append( + { + "path": str(path), + "github_job": record_workflow.get("github_job"), + "runner_name": record_workflow.get("runner_name"), + } + ) + names = [(item["github_job"], item["runner_name"]) for item in records] + if len(names) != len(set(names)): + raise ValueError("Duplicate wall-clock job record identity") + return records + + +def verify_artifacts( + side: str, + source_root: Path, + source_sha: str, + native_root: Path, + docs_root: Path, + python_root: Path, +) -> dict[str, object]: + """Verify downloaded read-only artifacts and emit a non-promotable proof.""" + side = _side(side) + source_root = source_root.resolve() + source_sha = _full_sha(source_sha, "source_sha") + actual = _git_head(source_root) + if actual != source_sha: + raise ValueError(f"source-root HEAD {actual} does not match expected {source_sha}") + source_digests = _source_digests(source_root) + source_name, version = _source_metadata(source_root) + workflow = _workflow_env(source_sha) + native = _verify_native_archives(side, native_root, source_sha, version) + docs = _docs_index(docs_root) + python_dists = _python_distributions(python_root, source_name, version) + records = _job_records([native_root, docs_root, python_root], side, source_sha, workflow) + return { + "kind": KIND_PROOF, + "schema_version": ABI_VERSION, + "abi": "scripts.release_wallclock.v1", + "promotable": False, + "side": side, + "source_sha": source_sha, + "controller_sha": workflow["controller_sha"], + "run_head_sha": workflow["run_head_sha"], + "event_name": workflow["event_name"], + "run_id": workflow["run_id"], + "run_attempt": workflow["run_attempt"], + "version": version, + "workflow_digest": source_digests[WORKFLOW_PATH], + "filehashes": { + "source_uv_lock_sha256": source_digests["uv.lock"], + "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], + "native": native, + "docs": docs, + "python": python_dists, + }, + "counts": { + "native_archives": len(native), + "native_sidecars": len(BINARY_NAMES), + "candidate_metadata": sum( + item["candidate_metadata_sha256"] is not None for item in native.values() + ), + "job_records": len(records), + "docs_indexes": 1, + "python_distributions": 2, + }, + "job_records": records, + "captured_at": _utc_now(), + "scope": "read-only wall-clock artifact proof; not release promotion evidence", + } + + +def _require_sha256(value: object, label: str) -> str: + if not _is_sha256(value): + raise ValueError(f"{label} must be a lowercase sha256 hex digest") + return str(value) + + +def _validate_native_proof(native: object, side: str, candidate_metadata_count: object) -> None: + if not isinstance(native, dict) or set(native) != set(BINARY_NAMES): + raise ValueError("Wall-clock proof native inventory must contain exactly five platforms") + expected_metadata = 5 if side == "proposed" else 0 + if candidate_metadata_count != expected_metadata: + raise ValueError(f"{side} candidate metadata count is malformed") + for binary_name in BINARY_NAMES: + item = native.get(binary_name) + if not isinstance(item, dict): + raise ValueError(f"Native proof entry is malformed: {binary_name}") + archive = archive_name(binary_name) + if item.get("binary_name") != binary_name or item.get("archive") != archive: + raise ValueError(f"Native proof identity mismatch: {binary_name}") + if item.get("sidecar") != f"{archive}.sha256": + raise ValueError(f"Native proof sidecar mismatch: {binary_name}") + _require_sha256(item.get("archive_sha256"), f"{binary_name} archive_sha256") + _require_sha256(item.get("executable_sha256"), f"{binary_name} executable_sha256") + if not isinstance(item.get("member_count"), int) or item["member_count"] <= 0: + raise ValueError(f"Native proof member count is malformed: {binary_name}") + metadata_digest = item.get("candidate_metadata_sha256") + loose_sidecar = item.get("loose_executable_sidecar_sha256") + loose_executable = item.get("loose_executable_sha256") + if side == "proposed": + _require_sha256(metadata_digest, f"{binary_name} candidate_metadata_sha256") + elif metadata_digest is not None: + raise ValueError(f"Baseline proof must not carry candidate metadata: {binary_name}") + if side == "baseline": + _require_sha256(loose_sidecar, f"{binary_name} loose_executable_sidecar_sha256") + if loose_executable != item.get("executable_sha256"): + raise ValueError(f"Baseline raw executable digest mismatch: {binary_name}") + elif loose_sidecar is not None: + _require_sha256(loose_sidecar, f"{binary_name} loose_executable_sidecar_sha256") + if loose_executable is not None: + _require_sha256(loose_executable, f"{binary_name} loose_executable_sha256") + + +def _validate_docs_proof(docs: object) -> None: + if not isinstance(docs, dict) or docs.get("index") != "index.html": + raise ValueError("Docs proof is malformed") + if docs.get("format") == "directory": + _require_sha256(docs.get("sha256"), "docs sha256") + elif docs.get("format") == "tar": + if not isinstance(docs.get("archive"), str) or not docs["archive"]: + raise ValueError("Docs proof tar archive name is malformed") + _require_sha256(docs.get("archive_sha256"), "docs archive_sha256") + else: + raise ValueError("Docs proof format is malformed") + + +def _validate_python_proof(python_proof: object) -> None: + if not isinstance(python_proof, dict): + raise ValueError("Python distribution proof is malformed") + source_name = python_proof.get("source_project_name") + if not isinstance(source_name, str) or not source_name: + raise ValueError("Python proof source project name is missing") + canonical = _canonical_project_name(source_name) + wheel = python_proof.get("wheel") + sdist = python_proof.get("sdist") + for label, item in (("wheel", wheel), ("sdist", sdist)): + if not isinstance(item, dict): + raise ValueError(f"Python {label} proof is malformed") + if not isinstance(item.get("name"), str) or not item["name"]: + raise ValueError(f"Python {label} filename is missing") + _require_sha256(item.get("sha256"), f"Python {label} sha256") + if _canonical_project_name(str(item.get("project_name") or "")) != canonical: + raise ValueError(f"Python {label} project name does not match source") + if not isinstance(item.get("version"), str) or not item["version"]: + raise ValueError(f"Python {label} version is missing") + if wheel["version"] != sdist["version"]: + raise ValueError("Python wheel and sdist versions do not match") + if not str(wheel["name"]).endswith(".whl") or not str(sdist["name"]).endswith(".tar.gz"): + raise ValueError("Python distribution filenames are malformed") + + +def _validate_job_records(proof: dict[str, object], counts: dict[str, object]) -> None: + records = proof.get("job_records") + if not isinstance(records, list) or not records: + raise ValueError("Wall-clock proof must contain at least one job record") + if counts.get("job_records") != len(records): + raise ValueError("Wall-clock proof job record count does not match") + seen: set[tuple[str, str]] = set() + for record in records: + if not isinstance(record, dict): + raise ValueError("Wall-clock job record is malformed") + github_job = record.get("github_job") + runner_name = record.get("runner_name") + if not isinstance(github_job, str) or not github_job: + raise ValueError("Wall-clock job record is missing github_job") + if not isinstance(runner_name, str) or not runner_name: + raise ValueError("Wall-clock job record is missing runner_name") + key = (github_job, runner_name) + if key in seen: + raise ValueError("Wall-clock proof contains duplicate job record identity") + seen.add(key) + + +def _read_proof(path: Path, side: str, expected_sha: str) -> dict[str, object]: + proof = _json(path) + if not isinstance(proof, dict): + raise ValueError(f"Malformed proof: {path}") + if proof.get("kind") != KIND_PROOF or proof.get("schema_version") != ABI_VERSION: + raise ValueError(f"Unsupported proof schema: {path}") + if proof.get("promotable") is not False: + raise ValueError(f"Wall-clock proof must be non-promotable: {path}") + if proof.get("side") != side or proof.get("source_sha") != expected_sha: + raise ValueError(f"Proof identity mismatch: {path}") + if proof.get("run_attempt") != 1: + raise ValueError(f"Proof must describe attempt 1: {path}") + _full_sha(proof.get("controller_sha"), "proof controller_sha") + _full_sha(proof.get("source_sha"), "proof source_sha") + _full_sha(proof.get("run_head_sha"), "proof run_head_sha") + counts = proof.get("counts") + if not isinstance(counts, dict) or { + "native_archives": counts.get("native_archives"), + "native_sidecars": counts.get("native_sidecars"), + "candidate_metadata": counts.get("candidate_metadata"), + "docs_indexes": counts.get("docs_indexes"), + "python_distributions": counts.get("python_distributions"), + } != { + "native_archives": len(BINARY_NAMES), + "native_sidecars": len(BINARY_NAMES), + "candidate_metadata": 5 if side == "proposed" else 0, + "docs_indexes": 1, + "python_distributions": 2, + }: + raise ValueError(f"Malformed wall-clock proof counts: {path}") + filehashes = proof.get("filehashes") + if not isinstance(filehashes, dict): + raise ValueError(f"Malformed wall-clock proof file hashes: {path}") + workflow_digest = _require_sha256(proof.get("workflow_digest"), "proof workflow_digest") + if filehashes.get("source_build_release_yml_sha256") != workflow_digest: + raise ValueError("Proof workflow digest does not match source workflow hash") + _require_sha256(filehashes.get("source_uv_lock_sha256"), "source_uv_lock_sha256") + _validate_native_proof(filehashes.get("native"), side, counts.get("candidate_metadata")) + _validate_docs_proof(filehashes.get("docs")) + _validate_python_proof(filehashes.get("python")) + _validate_job_records(proof, counts) + return proof + + +def _read_run(path: Path) -> tuple[dict[str, object], list[dict[str, object]]]: + data = _json(path) + if ( + not isinstance(data, dict) + or not isinstance(data.get("run"), dict) + or not isinstance(data.get("jobs"), list) + or not all(isinstance(job, dict) for job in data["jobs"]) + ): + raise ValueError(f"Run JSON must be {{run, jobs}}: {path}") + return data["run"], data["jobs"] + + +def _run_id(value: object, label: str) -> int: + return _positive_int(value, label) + + +def _validate_run( + run_path: Path, + proof_path: Path, + side: str, + expected_sha: str, + expected_controller_sha: str, + expected_run_head_sha: str, + terminal_name: str, + required_names: tuple[str, ...], +) -> dict[str, object]: + if not required_names or set(required_names) == {terminal_name}: + raise ValueError(f"{side} required job set must contain non-terminal jobs") + if len(set(required_names)) != len(required_names): + raise ValueError(f"{side} required job set contains duplicates") + if terminal_name in required_names: + raise ValueError(f"{side} required job set must not include the terminal job") + run, jobs = _read_run(run_path) + proof = _read_proof(proof_path, side, expected_sha) + run_id = _run_id(run.get("id"), f"{side} run id") + attempt = _positive_int(run.get("run_attempt"), f"{side} run_attempt") + if attempt != 1: + raise ValueError(f"{side} run must be attempt 1") + if proof["run_id"] != run_id or proof["run_attempt"] != attempt: + raise ValueError(f"{side} proof run identity does not match run JSON") + if run.get("status") != "completed" or run.get("conclusion") != "success": + raise ValueError(f"{side} workflow run did not complete successfully") + if proof["controller_sha"] != expected_controller_sha: + raise ValueError(f"{side} proof controller SHA does not match expected execution SHA") + if proof["run_head_sha"] != expected_run_head_sha: + raise ValueError(f"{side} proof run head SHA does not match expected event head SHA") + if run.get("head_sha") != proof["run_head_sha"]: + raise ValueError(f"{side} API run head_sha does not match recorded event head SHA") + created = _timestamp(run.get("created_at"), f"{side} run.created_at") + started = _timestamp(run.get("run_started_at"), f"{side} run.run_started_at") + if started < created: + raise ValueError(f"{side} run_started_at is before created_at") + terminal = _select_terminal(jobs, run_id, terminal_name, required_names, side) + terminal_start = _timestamp(terminal["started_at"], f"{side} terminal.started_at") + terminal_end = _timestamp(terminal["completed_at"], f"{side} terminal.completed_at") + if terminal_end < terminal_start: + raise ValueError(f"{side} terminal job has an invalid timeline") + executed_jobs = [] + runner_seconds = 0.0 + for job in jobs: + if job.get("conclusion") == "skipped": + continue + start = _timestamp(job.get("started_at"), f"{side} job.started_at") + end = _timestamp(job.get("completed_at"), f"{side} job.completed_at") + if end < start or start < created: + raise ValueError(f"{side} job has an invalid timeline: {job.get('name')}") + if job is not terminal and end > terminal_start: + raise ValueError(f"{side} terminal job started before {job.get('name')} completed") + executed_jobs.append(job) + runner_seconds += (end - start).total_seconds() + return { + "side": side, + "run_id": run_id, + "run_attempt": attempt, + "source_sha": expected_sha, + "controller_sha": proof["controller_sha"], + "run_head_sha": proof["run_head_sha"], + "event_name": proof.get("event_name"), + "created_at": run["created_at"], + "run_started_at": run["run_started_at"], + "terminal_job": terminal_name, + "terminal_completed_at": terminal["completed_at"], + "wallclock_seconds_from_created_at": _seconds_between( + created, terminal_end, f"{side} primary wall-clock" + ), + "wallclock_seconds_from_run_started_at": _seconds_between( + started, terminal_end, f"{side} secondary wall-clock" + ), + "unweighted_job_runner_seconds": runner_seconds, + "job_count": len(jobs), + "executed_job_count": len(executed_jobs), + "skipped_job_count": len(jobs) - len(executed_jobs), + "version": proof.get("version"), + } + + +def _select_terminal( + jobs: list[dict[str, object]], + run_id: int, + terminal_name: str, + required_names: tuple[str, ...], + side: str, +) -> dict[str, object]: + required_seen = {name: 0 for name in required_names} + terminals = [] + for job in jobs: + if _run_id(job.get("run_id"), f"{side} job run_id") != run_id: + raise ValueError(f"{side} job belongs to a different run") + if _positive_int(job.get("run_attempt"), f"{side} job run_attempt") != 1: + raise ValueError(f"{side} job belongs to a different attempt") + if job.get("status") != "completed": + raise ValueError(f"{side} job did not complete: {job.get('name')}") + conclusion = job.get("conclusion") + if conclusion in BAD_JOB_CONCLUSIONS or conclusion not in {"success", "skipped"}: + raise ValueError(f"{side} job has rejected conclusion {conclusion}: {job.get('name')}") + if job.get("name") == terminal_name: + terminals.append(job) + if job.get("name") in required_seen: + required_seen[str(job["name"])] += 1 + if conclusion != "success": + raise ValueError(f"{side} required job did not succeed: {job.get('name')}") + if len(terminals) != 1: + raise ValueError(f"{side} expected exactly one terminal job named {terminal_name}") + terminal = terminals[0] + if terminal.get("conclusion") != "success": + raise ValueError(f"{side} terminal job did not succeed") + missing = [name for name, count in required_seen.items() if count != 1] + if missing: + raise ValueError(f"{side} required job missing or duplicated: {', '.join(missing)}") + return terminal + + +def compare_runs( + baseline_run: Path, + proposed_run: Path, + baseline_proof: Path, + proposed_proof: Path, + expected_baseline_sha: str, + expected_proposed_sha: str, + expected_baseline_controller_sha: str | None, + expected_proposed_controller_sha: str | None, + expected_baseline_run_head_sha: str, + expected_proposed_run_head_sha: str, + terminal_name: str, + baseline_required_names: tuple[str, ...], + proposed_required_names: tuple[str, ...], +) -> dict[str, object]: + """Compare two real successful workflow attempts without modeling gate lag.""" + expected_baseline_sha = _full_sha(expected_baseline_sha, "expected_baseline_sha") + expected_proposed_sha = _full_sha(expected_proposed_sha, "expected_proposed_sha") + baseline_controller = _full_sha( + expected_baseline_controller_sha or expected_baseline_sha, + "expected_baseline_controller_sha", + ) + proposed_controller = _full_sha( + expected_proposed_controller_sha or expected_proposed_sha, + "expected_proposed_controller_sha", + ) + expected_baseline_run_head_sha = _full_sha( + expected_baseline_run_head_sha, "expected_baseline_run_head_sha" + ) + expected_proposed_run_head_sha = _full_sha( + expected_proposed_run_head_sha, "expected_proposed_run_head_sha" + ) + baseline = _validate_run( + baseline_run, + baseline_proof, + "baseline", + expected_baseline_sha, + baseline_controller, + expected_baseline_run_head_sha, + terminal_name, + baseline_required_names, + ) + proposed = _validate_run( + proposed_run, + proposed_proof, + "proposed", + expected_proposed_sha, + proposed_controller, + expected_proposed_run_head_sha, + terminal_name, + proposed_required_names, + ) + if baseline["run_id"] == proposed["run_id"]: + raise ValueError("Baseline and proposed run IDs must differ") + if ( + baseline["wallclock_seconds_from_created_at"] <= 0 + or baseline["wallclock_seconds_from_run_started_at"] <= 0 + ): + raise ValueError("Baseline wall-clock denominators must be positive") + primary_gain = ( + baseline["wallclock_seconds_from_created_at"] + - proposed["wallclock_seconds_from_created_at"] + ) + secondary_gain = ( + baseline["wallclock_seconds_from_run_started_at"] + - proposed["wallclock_seconds_from_run_started_at"] + ) + return { + "kind": KIND_COMPARISON, + "schema_version": ABI_VERSION, + "readonly_adapter": True, + "production_qualification": False, + "terminal_job": terminal_name, + "baseline_required_job_names": sorted(baseline_required_names), + "proposed_required_job_names": sorted(proposed_required_names), + "baseline": baseline, + "proposed": proposed, + "primary_metric": "terminal job completed_at minus workflow created_at", + "secondary_metric": "terminal job completed_at minus workflow run_started_at", + "delta_seconds": { + "primary_proposed_minus_baseline": -primary_gain, + "primary_gain_seconds": primary_gain, + "secondary_proposed_minus_baseline": -secondary_gain, + "secondary_gain_seconds": secondary_gain, + "unweighted_runner_seconds_proposed_minus_baseline": proposed[ + "unweighted_job_runner_seconds" + ] + - baseline["unweighted_job_runner_seconds"], + }, + "gain_fraction": { + "primary": primary_gain / baseline["wallclock_seconds_from_created_at"], + "secondary": secondary_gain / baseline["wallclock_seconds_from_run_started_at"], + }, + "gain_percent": { + "primary": 100 * primary_gain / baseline["wallclock_seconds_from_created_at"], + "secondary": 100 * secondary_gain / baseline["wallclock_seconds_from_run_started_at"], + }, + "scope": "actual successful GitHub workflow attempts; no counterfactual gate lag model", + } + + +def main() -> int: + """Command-line entry point for the release wall-clock toolkit.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + + record = commands.add_parser("record-job") + record.add_argument("--side", choices=SIDE_CHOICES, required=True) + record.add_argument("--source-root", type=Path, required=True) + record.add_argument("--source-sha", required=True) + record.add_argument("--output", type=Path, required=True) + + verify = commands.add_parser("verify") + verify.add_argument("--side", choices=SIDE_CHOICES, required=True) + verify.add_argument("--source-root", type=Path, required=True) + verify.add_argument("--source-sha", required=True) + verify.add_argument("--native-root", type=Path, required=True) + verify.add_argument("--docs-root", type=Path, required=True) + verify.add_argument("--python-root", type=Path, required=True) + verify.add_argument("--output", type=Path, required=True) + + compare = commands.add_parser("compare") + compare.add_argument("--baseline-run", type=Path, required=True) + compare.add_argument("--proposed-run", type=Path, required=True) + compare.add_argument("--baseline-proof", type=Path, required=True) + compare.add_argument("--proposed-proof", type=Path, required=True) + compare.add_argument("--expected-baseline-sha", required=True) + compare.add_argument("--expected-proposed-sha", required=True) + compare.add_argument("--expected-baseline-controller-sha") + compare.add_argument("--expected-proposed-controller-sha") + compare.add_argument("--expected-baseline-run-head-sha", required=True) + compare.add_argument("--expected-proposed-run-head-sha", required=True) + compare.add_argument("--terminal-job-name", default=TERMINAL_JOB) + compare.add_argument("--baseline-required-job-name", action="append", required=True) + compare.add_argument("--proposed-required-job-name", action="append", required=True) + compare.add_argument("--output", type=Path, required=True) + + args = parser.parse_args() + if args.operation == "record-job": + result = record_job(args.side, args.source_root, args.source_sha) + elif args.operation == "verify": + result = verify_artifacts( + args.side, + args.source_root, + args.source_sha, + args.native_root, + args.docs_root, + args.python_root, + ) + else: + result = compare_runs( + args.baseline_run, + args.proposed_run, + args.baseline_proof, + args.proposed_proof, + args.expected_baseline_sha, + args.expected_proposed_sha, + args.expected_baseline_controller_sha, + args.expected_proposed_controller_sha, + args.expected_baseline_run_head_sha, + args.expected_proposed_run_head_sha, + args.terminal_job_name, + tuple(args.baseline_required_job_name), + tuple(args.proposed_required_job_name), + ) + _write_json(args.output, result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/scripts/test_release_wallclock.py b/tests/unit/scripts/test_release_wallclock.py new file mode 100644 index 0000000000..73a2438c90 --- /dev/null +++ b/tests/unit/scripts/test_release_wallclock.py @@ -0,0 +1,1039 @@ +"""Read-only release wall-clock evidence rejects modeled or unsafe inputs.""" + +from __future__ import annotations + +import io +import json +import stat +import tarfile +import zipfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from scripts import release_wallclock +from scripts.package_release import BINARY_NAMES, archive_name, executable_name, file_digest +from scripts.release_wallclock import compare_runs, record_job, verify_artifacts + +pytestmark = pytest.mark.component + +BASE_SHA = "a" * 40 +PROPOSED_SHA = "b" * 40 +BASE_CONTROLLER = "c" * 40 +PROPOSED_CONTROLLER = "d" * 40 +BASE_RUN_HEAD = BASE_CONTROLLER +PROPOSED_RUN_HEAD = "e" * 40 +VERSION = "0.30.0" +ORIGIN = datetime(2026, 9, 8, tzinfo=timezone.utc) + + +def _at(seconds: int) -> str: + return (ORIGIN + timedelta(seconds=seconds)).isoformat().replace("+00:00", "Z") + + +def _source_root(tmp_path: Path) -> Path: + root = tmp_path / "source" + (root / ".github" / "workflows").mkdir(parents=True) + (root / "pyproject.toml").write_text( + f'[project]\nname = "apm-cli"\nversion = "{VERSION}"\n', encoding="ascii" + ) + (root / "uv.lock").write_text("lock\n", encoding="ascii") + (root / ".github" / "workflows" / "build-release.yml").write_text( + "name: release\n", encoding="ascii" + ) + return root + + +def _set_workflow_env( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + sha: str, + run_id: int = 101, + *, + event_name: str = "push", + run_head_sha: str | None = None, +) -> None: + run_head_sha = run_head_sha or sha + payload = ( + {"pull_request": {"head": {"sha": run_head_sha}}} + if event_name == "pull_request" + else {"after": run_head_sha} + ) + event_path = tmp_path / f"event-{run_id}.json" + event_path.write_text(json.dumps(payload), encoding="ascii") + monkeypatch.setenv("GITHUB_SHA", sha) + monkeypatch.setenv("GITHUB_RUN_ID", str(run_id)) + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setenv("GITHUB_EVENT_NAME", event_name) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event_path)) + monkeypatch.setenv("GITHUB_JOB", "verify-wallclock") + monkeypatch.setenv("RUNNER_NAME", "runner-1") + monkeypatch.setenv("ImageVersion", "20260901.1") + + +def _tar_archive(path: Path, binary_name: str, payload: bytes = b"native") -> str: + executable = f"{binary_name}/{executable_name(binary_name)}" + with tarfile.open(path, "w:gz") as target: + directory = tarfile.TarInfo(binary_name) + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + target.addfile(directory) + info = tarfile.TarInfo(executable) + info.mode = 0o755 + info.size = len(payload) + target.addfile(info, io.BytesIO(payload)) + hidden = tarfile.TarInfo(f"{binary_name}/_internal/.apm") + hidden.mode = 0o644 + hidden.size = 1 + target.addfile(hidden, io.BytesIO(b"x")) + return file_digest(path) + + +def _zip_archive(path: Path, binary_name: str, payload: bytes = b"native") -> str: + executable = f"{binary_name}/{executable_name(binary_name)}" + with zipfile.ZipFile(path, "w") as target: + info = zipfile.ZipInfo(executable) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + target.writestr(info, payload) + target.writestr(f"{binary_name}/_internal/.apm", b"x") + return file_digest(path) + + +def _native_artifacts(root: Path, side: str, source_sha: str) -> Path: + root.mkdir(parents=True) + for binary_name in BINARY_NAMES: + archive = archive_name(binary_name) + archive_path = root / archive + payload = f"native-{binary_name}".encode("ascii") + if archive.endswith(".zip"): + digest = _zip_archive(archive_path, binary_name, payload) + else: + digest = _tar_archive(archive_path, binary_name, payload) + (root / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") + executable_digest = ( + release_wallclock._zip_member_digest(archive_path, binary_name)[0] + if archive.endswith(".zip") + else release_wallclock._tar_member_digest(archive_path, binary_name)[0] + ) + if side == "proposed": + metadata = { + "schema_version": 1, + "sha": source_sha, + "version": VERSION, + "binary_name": binary_name, + "archive": archive, + "archive_sha256": digest, + "executable_sha256": executable_digest, + } + (root / f"{binary_name}.json").write_text( + json.dumps(metadata, sort_keys=True), encoding="ascii" + ) + elif side == "baseline": + loose = root / binary_name / executable_name(binary_name) + loose.parent.mkdir(parents=True, exist_ok=True) + loose.write_bytes(payload) + (root / f"{binary_name}.sha256").write_text( + f"{executable_digest} {binary_name}/{executable_name(binary_name)}\n", + encoding="ascii", + ) + return root + + +def _docs_artifact(root: Path) -> Path: + root.mkdir() + with tarfile.open(root / "artifact.tar", "w") as target: + directory = tarfile.TarInfo("./") + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + target.addfile(directory) + info = tarfile.TarInfo("./index.html") + body = b"docs" + info.size = len(body) + info.mode = 0o644 + target.addfile(info, io.BytesIO(body)) + return root + + +def _python_distributions(root: Path, version: str = VERSION, name: str = "apm-cli") -> Path: + root.mkdir(exist_ok=True) + token = name.replace("-", "_") + with zipfile.ZipFile(root / f"apm_cli-{version}-py3-none-any.whl", "w") as wheel: + wheel.writestr("apm_cli/__init__.py", "") + wheel.writestr( + f"apm_cli-{version}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n", + ) + wheel.writestr( + f"apm_cli-{version}.dist-info/WHEEL", + "Wheel-Version: 1.0\nGenerator: fixture\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr(f"apm_cli-{version}.dist-info/RECORD", "apm_cli/__init__.py,,\n") + with tarfile.open(root / f"apm_cli-{version}.tar.gz", "w:gz") as sdist: + for member, body in { + f"{token}-{version}/PKG-INFO": ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + ), + f"{token}-{version}/pyproject.toml": b"[project]\nname='apm-cli'\n", + f"{token}-{version}/src/{token}/__init__.py": b"", + }.items(): + info = tarfile.TarInfo(member) + info.size = len(body) + info.mode = 0o644 + sdist.addfile(info, io.BytesIO(body)) + return root + + +def _verified_fixture( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, side: str, source_sha: str, controller: str +) -> tuple[Path, Path, Path, Path]: + source = _source_root(tmp_path) + native = _native_artifacts(tmp_path / "native", side, source_sha) + docs = _docs_artifact(tmp_path / "docs") + python = _python_distributions(tmp_path / "python") + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: source_sha) + _set_workflow_env(monkeypatch, tmp_path, controller) + return source, native, docs, python + + +def test_record_job_captures_minimal_host_identity_without_env_dump( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) + _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) + monkeypatch.setenv("SECRET_TOKEN", "do-not-copy") + + evidence = record_job("baseline", source, BASE_SHA) + + assert evidence["kind"] == "release-wallclock-job" + assert evidence["workflow"]["source_sha"] == BASE_SHA + assert evidence["workflow"]["controller_sha"] == BASE_CONTROLLER + assert evidence["workflow"]["run_head_sha"] == BASE_CONTROLLER + assert evidence["environment"]["image_version"] == "20260901.1" + assert "SECRET_TOKEN" not in json.dumps(evidence) + + +def test_record_job_keeps_pull_request_head_distinct_from_execution_sha( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) + _set_workflow_env( + monkeypatch, + tmp_path, + PROPOSED_CONTROLLER, + event_name="pull_request", + run_head_sha=PROPOSED_RUN_HEAD, + ) + + evidence = record_job("proposed", source, PROPOSED_SHA) + + assert evidence["workflow"]["source_sha"] == PROPOSED_SHA + assert evidence["workflow"]["controller_sha"] == PROPOSED_CONTROLLER + assert evidence["workflow"]["run_head_sha"] == PROPOSED_RUN_HEAD + assert evidence["workflow"]["run_head_sha"] != evidence["workflow"]["controller_sha"] + assert evidence["workflow"]["event_name"] == "pull_request" + + +def test_record_job_fails_bad_source_identity_and_missing_hosted_image( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) + _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) + + with pytest.raises(ValueError, match="does not match expected"): + record_job("baseline", source, BASE_SHA) + + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) + monkeypatch.delenv("ImageVersion") + with pytest.raises(ValueError, match="ImageVersion"): + record_job("baseline", source, BASE_SHA) + + +@pytest.mark.parametrize( + "side,sha,controller,metadata_count", + [ + ("baseline", BASE_SHA, BASE_CONTROLLER, 0), + ("proposed", PROPOSED_SHA, PROPOSED_CONTROLLER, 5), + ], +) +def test_verify_artifacts_emits_nonpromotable_real_archive_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + side: str, + sha: str, + controller: str, + metadata_count: int, +) -> None: + source, native, docs, python = _verified_fixture(tmp_path, monkeypatch, side, sha, controller) + + proof = verify_artifacts(side, source, sha, native, docs, python) + + assert proof["kind"] == "release-wallclock-proof" + assert proof["promotable"] is False + assert "release-candidate-evidence" not in json.dumps(proof) + assert proof["side"] == side + assert proof["source_sha"] == sha + assert proof["controller_sha"] == controller + assert proof["counts"]["native_archives"] == 5 + assert proof["counts"]["candidate_metadata"] == metadata_count + assert proof["filehashes"]["docs"]["format"] == "tar" + assert proof["filehashes"]["python"]["wheel"]["version"] == VERSION + + +def test_verify_proof_keeps_pull_request_head_distinct_from_execution_sha( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + native = _native_artifacts(tmp_path / "native", "proposed", PROPOSED_SHA) + docs = _docs_artifact(tmp_path / "docs") + python = _python_distributions(tmp_path / "python") + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) + _set_workflow_env( + monkeypatch, + tmp_path, + PROPOSED_CONTROLLER, + event_name="pull_request", + run_head_sha=PROPOSED_RUN_HEAD, + ) + + proof = verify_artifacts("proposed", source, PROPOSED_SHA, native, docs, python) + + assert proof["source_sha"] == PROPOSED_SHA + assert proof["controller_sha"] == PROPOSED_CONTROLLER + assert proof["run_head_sha"] == PROPOSED_RUN_HEAD + assert proof["event_name"] == "pull_request" + + +def test_verify_baseline_archives_raw_layout_binds_archives_to_original_executables( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + flat_native = _native_artifacts(tmp_path / "flat-native", "baseline", BASE_SHA) + native = tmp_path / "native-layout" + archives = native / "archives" + raw = native / "raw" / "candidate-1-apm-linux-x86_64" / "dist" + archives.mkdir(parents=True) + raw.mkdir(parents=True) + for artifact in flat_native.iterdir(): + if artifact.is_file() and artifact.name.endswith((".tar.gz", ".zip")): + (archives / artifact.name).write_bytes(artifact.read_bytes()) + for binary_name in BINARY_NAMES: + archive = archive_name(binary_name) + (archives / f"{archive}.sha256").write_bytes( + (flat_native / f"{archive}.sha256").read_bytes() + ) + executable_digest = ( + release_wallclock._zip_member_digest(archives / archive, binary_name)[0] + if archive.endswith(".zip") + else release_wallclock._tar_member_digest(archives / archive, binary_name)[0] + ) + loose = raw / binary_name / executable_name(binary_name) + loose.parent.mkdir() + loose.write_bytes((flat_native / binary_name / executable_name(binary_name)).read_bytes()) + (raw / f"{binary_name}.sha256").write_text( + f"{executable_digest} dist/{binary_name}/{executable_name(binary_name)}\n", + encoding="ascii", + ) + docs = _docs_artifact(tmp_path / "docs") + python = _python_distributions(tmp_path / "python") + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) + _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) + + proof = verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + observed = proof["filehashes"]["native"]["apm-linux-x86_64"] + assert observed["loose_executable_sidecar_sha256"] == file_digest( + raw / "apm-linux-x86_64.sha256" + ) + assert proof["counts"]["candidate_metadata"] == 0 + + +def test_verify_baseline_raw_layout_rejects_original_executable_digest_drift( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_root(tmp_path) + flat_native = _native_artifacts(tmp_path / "flat-native", "baseline", BASE_SHA) + native = tmp_path / "native-layout" + archives = native / "archives" + raw = native / "raw" / "candidate-1-apm-linux-x86_64" / "dist" + archives.mkdir(parents=True) + raw.mkdir(parents=True) + for artifact in flat_native.iterdir(): + if artifact.is_file() and artifact.name.endswith((".tar.gz", ".zip")): + (archives / artifact.name).write_bytes(artifact.read_bytes()) + for binary_name in BINARY_NAMES: + archive = archive_name(binary_name) + (archives / f"{archive}.sha256").write_bytes( + (flat_native / f"{archive}.sha256").read_bytes() + ) + loose = raw / binary_name / executable_name(binary_name) + loose.parent.mkdir() + loose.write_bytes((flat_native / binary_name / executable_name(binary_name)).read_bytes()) + digest = file_digest(loose) + if binary_name == "apm-linux-x86_64": + digest = "1" * 64 + (raw / f"{binary_name}.sha256").write_text( + f"{digest} dist/{binary_name}/{executable_name(binary_name)}\n", + encoding="ascii", + ) + docs = _docs_artifact(tmp_path / "docs") + python = _python_distributions(tmp_path / "python") + monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) + _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) + + with pytest.raises(ValueError, match="Loose executable checksum mismatch"): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +@pytest.mark.parametrize("fault", ["missing-sidecar", "missing-payload"]) +def test_verify_baseline_requires_all_raw_sidecars_and_loose_executables( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + binary_name = "apm-linux-x86_64" + if fault == "missing-sidecar": + (native / f"{binary_name}.sha256").unlink() + match = "raw executable checksum is missing" + else: + (native / binary_name / executable_name(binary_name)).unlink() + match = "raw executable payload is missing" + + with pytest.raises(ValueError, match=match): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +def test_verify_rejects_proposed_archive_without_candidate_json( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "proposed", PROPOSED_SHA, PROPOSED_CONTROLLER + ) + (native / "apm-linux-x86_64.json").unlink() + + with pytest.raises(ValueError, match="requires candidate metadata"): + verify_artifacts("proposed", source, PROPOSED_SHA, native, docs, python) + + +def test_verify_rejects_unsafe_archive_member_even_with_matching_sidecar( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + binary_name = "apm-linux-x86_64" + archive = archive_name(binary_name) + archive_path = native / archive + with tarfile.open(archive_path, "w:gz") as target: + info = tarfile.TarInfo(f"{binary_name}/../escape") + info.size = 4 + target.addfile(info, io.BytesIO(b"nope")) + digest = file_digest(archive_path) + (native / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") + + with pytest.raises(ValueError, match="unsafe member path"): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +def test_verify_rejects_duplicate_executable_members( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + binary_name = "apm-linux-arm64" + archive = archive_name(binary_name) + executable = f"{binary_name}/{executable_name(binary_name)}" + archive_path = native / archive + with tarfile.open(archive_path, "w:gz") as target: + for body in (b"one", b"two"): + info = tarfile.TarInfo(executable) + info.mode = 0o755 + info.size = len(body) + target.addfile(info, io.BytesIO(body)) + digest = file_digest(archive_path) + (native / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") + + with pytest.raises(ValueError, match="Duplicate archive member"): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +def _tar_with_link(path: Path, link_type: bytes, linkname: str) -> None: + binary_name = "apm-darwin-arm64" + with tarfile.open(path, "w:gz") as target: + executable = tarfile.TarInfo(f"{binary_name}/{executable_name(binary_name)}") + executable.mode = 0o755 + executable.size = 6 + target.addfile(executable, io.BytesIO(b"native")) + real = tarfile.TarInfo(f"{binary_name}/_internal/real-target") + real.mode = 0o644 + real.size = 4 + target.addfile(real, io.BytesIO(b"real")) + link = tarfile.TarInfo(f"{binary_name}/_internal/Foo.framework/Versions/Current") + link.type = link_type + link.linkname = linkname + link.mode = 0o777 + target.addfile(link) + + +def test_tar_validator_accepts_real_format_macos_internal_symlinks(tmp_path: Path) -> None: + binary_name = "apm-darwin-arm64" + archive = tmp_path / archive_name(binary_name) + with tarfile.open(archive, "w:gz") as target: + executable = tarfile.TarInfo(f"{binary_name}/{executable_name(binary_name)}") + executable.mode = 0o755 + executable.size = 6 + target.addfile(executable, io.BytesIO(b"native")) + for member, linkname in ( + (f"{binary_name}/_internal/Python", "Python.framework/Versions/3.12/Python"), + (f"{binary_name}/_internal/Python.framework/Python", "Versions/Current/Python"), + ( + f"{binary_name}/_internal/Python.framework/Resources", + "Versions/Current/Resources", + ), + (f"{binary_name}/_internal/Python.framework/Versions/Current", "3.12"), + ): + link = tarfile.TarInfo(member) + link.type = tarfile.SYMTYPE + link.linkname = linkname + target.addfile(link) + + digest, count = release_wallclock._tar_member_digest(archive, binary_name) + + assert digest == release_wallclock.file_digest_stream(io.BytesIO(b"native")) + assert count == 5 + + +def test_tar_validator_rejects_escaping_symlink(tmp_path: Path) -> None: + archive = tmp_path / "escaping-symlink.tar.gz" + _tar_with_link(archive, tarfile.SYMTYPE, "../../../../outside") + + with pytest.raises(ValueError, match="symbolic link escapes"): + release_wallclock._tar_member_digest(archive, "apm-darwin-arm64") + + +def test_tar_validator_uses_archive_root_for_hardlinks(tmp_path: Path) -> None: + archive = tmp_path / "escaping-hardlink.tar.gz" + _tar_with_link(archive, tarfile.LNKTYPE, "outside/target") + + with pytest.raises(ValueError, match="hard link escapes"): + release_wallclock._tar_member_digest(archive, "apm-darwin-arm64") + + safe = tmp_path / "safe-hardlink.tar.gz" + _tar_with_link(safe, tarfile.LNKTYPE, "apm-darwin-arm64/_internal/real-target") + release_wallclock._tar_member_digest(safe, "apm-darwin-arm64") + + +@pytest.mark.parametrize("fault", ["empty-docs-index", "python-version"]) +def test_verify_rejects_mismatched_metadata_or_missing_docs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + if fault == "empty-docs-index": + docs.joinpath("artifact.tar").unlink() + docs.joinpath("index.html").write_text("", encoding="ascii") + match = "docs index.html is empty" + else: + python = _python_distributions(tmp_path / "wrong-python", "9.9.9") + match = "project/version" + + with pytest.raises(ValueError, match=match): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +@pytest.mark.parametrize( + "fault", ["wrong-wheel-name", "missing-wheel-record", "sdist-metadata-only"] +) +def test_verify_rejects_fake_or_wrong_project_python_distributions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + for path in python.iterdir(): + path.unlink() + if fault == "wrong-wheel-name": + _python_distributions(python, VERSION, name="evil-pkg") + match = "Wheel metadata name/version" + elif fault == "missing-wheel-record": + with zipfile.ZipFile(python / f"apm_cli-{VERSION}-py3-none-any.whl", "w") as wheel: + wheel.writestr("apm_cli/__init__.py", "") + wheel.writestr( + f"apm_cli-{VERSION}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n", + ) + wheel.writestr(f"apm_cli-{VERSION}.dist-info/WHEEL", "Wheel-Version: 1.0\n") + with tarfile.open(python / f"apm_cli-{VERSION}.tar.gz", "w:gz") as sdist: + for member, body in { + f"apm_cli-{VERSION}/PKG-INFO": ( + f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n".encode() + ), + f"apm_cli-{VERSION}/pyproject.toml": b"[project]\nname='apm-cli'\n", + f"apm_cli-{VERSION}/src/apm_cli/__init__.py": b"", + }.items(): + info = tarfile.TarInfo(member) + info.size = len(body) + sdist.addfile(info, io.BytesIO(body)) + match = "missing METADATA, WHEEL or RECORD" + else: + with zipfile.ZipFile(python / f"apm_cli-{VERSION}-py3-none-any.whl", "w") as wheel: + wheel.writestr("apm_cli/__init__.py", "") + wheel.writestr( + f"apm_cli-{VERSION}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n", + ) + wheel.writestr(f"apm_cli-{VERSION}.dist-info/WHEEL", "Wheel-Version: 1.0\n") + wheel.writestr(f"apm_cli-{VERSION}.dist-info/RECORD", "apm_cli/__init__.py,,\n") + with tarfile.open(python / f"apm_cli-{VERSION}.tar.gz", "w:gz") as sdist: + body = f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n".encode() + info = tarfile.TarInfo(f"apm_cli-{VERSION}/PKG-INFO") + info.size = len(body) + sdist.addfile(info, io.BytesIO(body)) + match = "top-level pyproject.toml" + + with pytest.raises(ValueError, match=match): + verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + +def _proof( + path: Path, + side: str, + run_id: int, + source_sha: str, + controller_sha: str, + run_head_sha: str, +) -> Path: + native = {} + for index, binary_name in enumerate(BINARY_NAMES, start=1): + archive = archive_name(binary_name) + executable_digest = f"{index + 10:064x}" + native[binary_name] = { + "binary_name": binary_name, + "archive": archive, + "archive_sha256": f"{index:064x}", + "executable_sha256": executable_digest, + "sidecar": f"{archive}.sha256", + "candidate_metadata_sha256": f"{index + 20:064x}" if side == "proposed" else None, + "loose_executable_sidecar_sha256": f"{index + 30:064x}" if side == "baseline" else None, + "loose_executable_sha256": executable_digest if side == "baseline" else None, + "member_count": 2, + } + path.write_text( + json.dumps( + { + "kind": "release-wallclock-proof", + "schema_version": 1, + "promotable": False, + "side": side, + "source_sha": source_sha, + "controller_sha": controller_sha, + "run_head_sha": run_head_sha, + "event_name": "pull_request" if run_head_sha != controller_sha else "push", + "run_id": run_id, + "run_attempt": 1, + "version": VERSION, + "workflow_digest": "e" * 64, + "counts": { + "native_archives": 5, + "native_sidecars": 5, + "candidate_metadata": 5 if side == "proposed" else 0, + "job_records": 1, + "docs_indexes": 1, + "python_distributions": 2, + }, + "filehashes": { + "source_uv_lock_sha256": "1" * 64, + "source_build_release_yml_sha256": "e" * 64, + "native": native, + "docs": { + "format": "tar", + "archive": "artifact.tar", + "archive_sha256": "2" * 64, + "index": "index.html", + }, + "python": { + "source_project_name": "apm-cli", + "wheel": { + "name": f"apm_cli-{VERSION}-py3-none-any.whl", + "sha256": "3" * 64, + "project_name": "apm-cli", + "version": VERSION, + }, + "sdist": { + "name": f"apm_cli-{VERSION}.tar.gz", + "sha256": "4" * 64, + "project_name": "apm-cli", + "version": VERSION, + }, + }, + }, + "job_records": [ + { + "path": "wallclock/job.json", + "github_job": "build", + "runner_name": f"{side}-runner", + } + ], + } + ), + encoding="ascii", + ) + return path + + +def _job(run_id: int, name: str, start: int, end: int, conclusion: str = "success") -> dict: + return { + "id": f"{run_id}-{name}", + "run_id": run_id, + "run_attempt": 1, + "name": name, + "status": "completed", + "conclusion": conclusion, + "started_at": _at(start), + "completed_at": _at(end), + } + + +def _run_json( + path: Path, + run_id: int, + run_head_sha: str, + run_started: int, + terminal_completed: int, +) -> Path: + jobs = [ + _job(run_id, "Build/test/native/docs/wheel proof", run_started + 10, run_started + 20), + { + "id": f"{run_id}-skipped", + "run_id": run_id, + "run_attempt": 1, + "name": "Imported PR-only workflow", + "status": "completed", + "conclusion": "skipped", + }, + _job(run_id, "Verify wall-clock artifacts", terminal_completed - 5, terminal_completed), + ] + payload = { + "run": { + "id": run_id, + "run_attempt": 1, + "status": "completed", + "conclusion": "success", + "head_sha": run_head_sha, + "pull_requests": [{"head": {"sha": "f" * 40}}], + "created_at": _at(0), + "run_started_at": _at(run_started), + }, + "jobs": jobs, + } + path.write_text(json.dumps(payload), encoding="ascii") + return path + + +def _compare_fixture(tmp_path: Path) -> tuple[Path, Path, Path, Path]: + return ( + _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100), + _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90), + _proof( + tmp_path / "baseline-proof.json", + "baseline", + 201, + BASE_SHA, + BASE_CONTROLLER, + BASE_RUN_HEAD, + ), + _proof( + tmp_path / "proposed-proof.json", + "proposed", + 301, + PROPOSED_SHA, + PROPOSED_CONTROLLER, + PROPOSED_RUN_HEAD, + ), + ) + + +def _compare_ok( + baseline_run: Path, + proposed_run: Path, + baseline_proof: Path, + proposed_proof: Path, + baseline_required: tuple[str, ...] = ("Build/test/native/docs/wheel proof",), + proposed_required: tuple[str, ...] = ("Build/test/native/docs/wheel proof",), +) -> dict[str, object]: + return compare_runs( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + PROPOSED_CONTROLLER, + BASE_RUN_HEAD, + PROPOSED_RUN_HEAD, + "Verify wall-clock artifacts", + baseline_required, + proposed_required, + ) + + +@pytest.mark.parametrize( + ("fault", "match"), + [ + ("empty-filehashes", "workflow digest"), + ("missing-native", "native inventory"), + ("missing-proposed-metadata", "candidate_metadata_sha256"), + ("empty-jobs", "at least one job record"), + ], +) +def test_compare_rejects_structurally_fake_or_incomplete_proofs( + tmp_path: Path, fault: str, match: str +) -> None: + baseline_run, proposed_run, baseline_proof, proposed_proof = _compare_fixture(tmp_path) + target = proposed_proof if fault == "missing-proposed-metadata" else baseline_proof + payload = json.loads(target.read_text(encoding="ascii")) + if fault == "empty-filehashes": + payload["filehashes"] = {} + elif fault == "missing-native": + payload["filehashes"]["native"].pop("apm-linux-x86_64") + elif fault == "missing-proposed-metadata": + payload["filehashes"]["native"]["apm-linux-x86_64"]["candidate_metadata_sha256"] = None + else: + payload["counts"]["job_records"] = 0 + payload["job_records"] = [] + target.write_text(json.dumps(payload), encoding="ascii") + + with pytest.raises(ValueError, match=match): + _compare_ok(baseline_run, proposed_run, baseline_proof, proposed_proof) + + +@pytest.mark.parametrize("required", [(), ("Verify wall-clock artifacts",)]) +def test_compare_rejects_empty_or_terminal_only_required_job_sets( + tmp_path: Path, required: tuple[str, ...] +) -> None: + baseline_run, proposed_run, baseline_proof, proposed_proof = _compare_fixture(tmp_path) + + with pytest.raises(ValueError, match="required job set"): + _compare_ok( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + baseline_required=required, + ) + + +def test_compare_uses_actual_terminal_wallclock_not_runner_sum_or_modeled_queue( + tmp_path: Path, +) -> None: + baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 30, 120) + proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 100) + baseline_proof = _proof( + tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD + ) + proposed_proof = _proof( + tmp_path / "proposed-proof.json", + "proposed", + 301, + PROPOSED_SHA, + PROPOSED_CONTROLLER, + PROPOSED_RUN_HEAD, + ) + + result = compare_runs( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + PROPOSED_CONTROLLER, + BASE_RUN_HEAD, + PROPOSED_RUN_HEAD, + "Verify wall-clock artifacts", + ("Build/test/native/docs/wheel proof",), + ("Build/test/native/docs/wheel proof",), + ) + + assert result["readonly_adapter"] is True + assert result["production_qualification"] is False + assert result["baseline"]["wallclock_seconds_from_created_at"] == 120 + assert result["proposed"]["controller_sha"] == PROPOSED_CONTROLLER + assert result["proposed"]["run_head_sha"] == PROPOSED_RUN_HEAD + assert result["baseline"]["wallclock_seconds_from_run_started_at"] == 90 + assert result["proposed"]["wallclock_seconds_from_created_at"] == 100 + assert result["delta_seconds"]["primary_gain_seconds"] == 20 + assert result["delta_seconds"]["secondary_gain_seconds"] == -10 + assert result["delta_seconds"]["unweighted_runner_seconds_proposed_minus_baseline"] == 0 + assert result["delta_seconds"]["unweighted_runner_seconds_proposed_minus_baseline"] != -20 + + +@pytest.mark.parametrize( + ("fault", "match"), + [ + ("api-head", "API run head_sha"), + ("expected-head", "expected event head SHA"), + ("expected-controller", "expected execution SHA"), + ], +) +def test_compare_rejects_unexpected_run_head_or_execution_sha_changes( + tmp_path: Path, fault: str, match: str +) -> None: + baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) + proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90) + baseline_proof = _proof( + tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD + ) + proposed_proof = _proof( + tmp_path / "proposed-proof.json", + "proposed", + 301, + PROPOSED_SHA, + PROPOSED_CONTROLLER, + PROPOSED_RUN_HEAD, + ) + expected_head = PROPOSED_RUN_HEAD + expected_controller = PROPOSED_CONTROLLER + if fault == "api-head": + data = json.loads(proposed_run.read_text(encoding="ascii")) + data["run"]["head_sha"] = "1" * 40 + data["run"]["pull_requests"][0]["head"]["sha"] = PROPOSED_RUN_HEAD + proposed_run.write_text(json.dumps(data), encoding="ascii") + elif fault == "expected-head": + expected_head = "1" * 40 + else: + expected_controller = "1" * 40 + + with pytest.raises(ValueError, match=match): + compare_runs( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + expected_controller, + BASE_RUN_HEAD, + expected_head, + "Verify wall-clock artifacts", + ("Build/test/native/docs/wheel proof",), + ("Build/test/native/docs/wheel proof",), + ) + + +@pytest.mark.parametrize("fault", ["failed-control", "missing-required", "early-terminal"]) +def test_compare_rejects_failed_controls_missing_gates_and_bad_timelines( + tmp_path: Path, fault: str +) -> None: + baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) + proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90) + data = json.loads(proposed_run.read_text(encoding="ascii")) + if fault == "failed-control": + data["jobs"][0]["conclusion"] = "failure" + required = ("Build/test/native/docs/wheel proof",) + match = "rejected conclusion" + elif fault == "missing-required": + required = ("Required native gate",) + match = "required job missing" + else: + data["jobs"][0]["completed_at"] = _at(88) + data["jobs"][2]["started_at"] = _at(87) + required = ("Build/test/native/docs/wheel proof",) + match = "terminal job started before" + proposed_run.write_text(json.dumps(data), encoding="ascii") + + with pytest.raises(ValueError, match=match): + compare_runs( + baseline_run, + proposed_run, + _proof( + tmp_path / "baseline-proof.json", + "baseline", + 201, + BASE_SHA, + BASE_CONTROLLER, + BASE_RUN_HEAD, + ), + _proof( + tmp_path / "proposed-proof.json", + "proposed", + 301, + PROPOSED_SHA, + PROPOSED_CONTROLLER, + PROPOSED_RUN_HEAD, + ), + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + PROPOSED_CONTROLLER, + BASE_RUN_HEAD, + PROPOSED_RUN_HEAD, + "Verify wall-clock artifacts", + ("Build/test/native/docs/wheel proof",), + required, + ) + + +def test_compare_rejects_promotable_or_duplicate_run_proofs(tmp_path: Path) -> None: + baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) + proposed_run = _run_json(tmp_path / "proposed-run.json", 201, PROPOSED_RUN_HEAD, 0, 90) + baseline_proof = _proof( + tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD + ) + proposed_proof = _proof( + tmp_path / "proposed-proof.json", + "proposed", + 201, + PROPOSED_SHA, + PROPOSED_CONTROLLER, + PROPOSED_RUN_HEAD, + ) + payload = json.loads(proposed_proof.read_text(encoding="ascii")) + payload["promotable"] = True + proposed_proof.write_text(json.dumps(payload), encoding="ascii") + + with pytest.raises(ValueError, match="non-promotable"): + compare_runs( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + PROPOSED_CONTROLLER, + BASE_RUN_HEAD, + PROPOSED_RUN_HEAD, + "Verify wall-clock artifacts", + ("Build/test/native/docs/wheel proof",), + ("Build/test/native/docs/wheel proof",), + ) + + payload["promotable"] = False + proposed_proof.write_text(json.dumps(payload), encoding="ascii") + with pytest.raises(ValueError, match="run IDs must differ"): + compare_runs( + baseline_run, + proposed_run, + baseline_proof, + proposed_proof, + BASE_SHA, + PROPOSED_SHA, + BASE_CONTROLLER, + PROPOSED_CONTROLLER, + BASE_RUN_HEAD, + PROPOSED_RUN_HEAD, + "Verify wall-clock artifacts", + ("Build/test/native/docs/wheel proof",), + ("Build/test/native/docs/wheel proof",), + ) diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index 8ce6ec96bc..e5783d278b 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -20,6 +20,7 @@ load_workflow, shell_commands, shell_tokens, + wallclock_unit_commands, workflow_job, workflow_step, workflow_step_index, @@ -32,7 +33,8 @@ RELEASE_WORKFLOW = ROOT / ".github/workflows/build-release.yml" UNIX_TIMEOUT = "${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }}" UNIX_ARGS = ( - "${{ inputs.evidence-variant != '' && '-p scripts.pytest_performance_evidence' || '' }} " + "${{ (inputs.evidence-variant != '' || inputs.wallclock-evidence) " + "&& '-p scripts.pytest_performance_evidence' || '' }} " "--splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} " "--splitting-algorithm ${{ inputs.splitting-algorithm }} " "-n ${{ inputs.xdist-workers }} --dist loadgroup " @@ -278,6 +280,7 @@ def _assert_windows_installer(workflow: dict) -> None: assert env["APM_BASELINE_VERSION"] == "v0.28.0" assert "APM_CANDIDATE_VERSION" in step["run"] assert "APM_CANDIDATE_SHA256" in step["run"] + assert 'Join-Path $env:RUNNER_TEMP "apm-windows-installer"' in step["run"] baseline = workflow_step(job, "Download historical upgrade source once") assert baseline["env"] == {"GH_TOKEN": "${{ github.token }}"} assert "gh release download $env:APM_BASELINE_VERSION" in baseline["run"] @@ -297,9 +300,25 @@ def _assert_windows_installer(workflow: dict) -> None: "-ra", "--tb=short", "--junitxml=test-results/installer.xml", + "--basetemp", + "$installerBaseTemp", ], label="candidate Windows installer", ) + executed = workflow_step(job, "Require executed installer assertions") + assert_exact_command( + shell_commands(executed), + [ + "scripts/windows/run-installer-proof.ps1", + "-ValidateInstallerJUnit", + "test-results/installer.xml", + "-InstallerExitCode", + "0", + ], + label="non-skipped installer assertions", + ) + assert_unconditional(executed, label="installer execution evidence") + assert not executed.get("continue-on-error", False) def test_installer_workflow_inputs_reach_the_actual_candidate_reader(tmp_path: Path) -> None: @@ -337,6 +356,7 @@ def _assert_unit_call(workflow: dict, unit: dict) -> None: "binary-name": "${{ inputs.binary-name }}", "performance-probe": "${{ inputs.unit-performance-probe }}", "proposed-workers": "${{ inputs.performance-workers }}", + "wallclock-evidence": "${{ inputs.wallclock-evidence }}", } assert job["secrets"] == { "GH_CLI_PAT": "${{ secrets.GH_CLI_PAT }}", @@ -351,7 +371,7 @@ def _assert_unit_call(workflow: dict, unit: dict) -> None: standard = workflow_step(baseline, "Run unit tests") assert standard["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" assert_exact_command( - shell_commands(standard), + wallclock_unit_commands(standard, enabled=False), [ "uv", "run", @@ -528,6 +548,7 @@ def _assert_integration(workflow: dict, integration: dict) -> None: assert job["with"]["evidence-selection"] == ( "tests/integration/ -m ${{ inputs.integration-markers }}" ) + assert job["with"]["wallclock-evidence"] == "${{ inputs.wallclock-evidence }}" assert job["with"]["performance-cohort"] == ( "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " "format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }}" @@ -607,7 +628,9 @@ def _assert_integration(workflow: dict, integration: dict) -> None: "${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }}" ) timings_upload = workflow_step(integration_job, "Upload integration timings and outcomes") - assert timings_upload["if"] == "always() && inputs.platform != 'windows'" + assert timings_upload["if"] == ( + "always() && (inputs.platform != 'windows' || inputs.wallclock-evidence)" + ) assert timings_upload["with"]["if-no-files-found"] == "error" coverage_upload = workflow_step(integration_job, "Upload coverage data") assert coverage_upload["with"]["name"] == ( diff --git a/tests/unit/test_release_wallclock_workflow.py b/tests/unit/test_release_wallclock_workflow.py new file mode 100644 index 0000000000..aef7b3e901 --- /dev/null +++ b/tests/unit/test_release_wallclock_workflow.py @@ -0,0 +1,276 @@ +"""The observed PR clock must include one complete, read-only production DAG.""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +import pytest + +from tests.workflow_contracts import load_workflow, workflow_job, workflow_step + +pytestmark = pytest.mark.component +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ci-release-wallclock.yml" + + +def _assert_single_world(workflow: dict) -> None: + assert workflow["on"] == { + "pull_request": {"types": ["labeled"]}, + "push": {"branches": ["danielmeppiel-release-wallclock-proof"]}, + } + assert workflow["permissions"] == {"contents": "read", "actions": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is False + jobs = workflow["jobs"] + assert set(jobs) == { + "plan", + "source-checks", + "platforms", + "candidate-ready", + "docs", + "wheels", + "verify", + } + assert jobs["plan"]["if"] == ( + "github.event_name == 'push' || github.event.label.name == 'ci-release-wallclock'" + ) + for job in jobs.values(): + assert "secrets" not in job + assert "environment" not in job + assert not job.get("continue-on-error", False) + assert set(job.get("permissions", {}).values()) <= {"read", "none"} + for step in job.get("steps", []): + assert "secrets." not in str(step) + assert "action-gh-release" not in step.get("uses", "") + assert "deploy-pages" not in step.get("uses", "") + assert not step.get("continue-on-error", False) + + plan = workflow_step(jobs["plan"], "Select all five production catalog platforms") + assert "GITHUB_RUN_ATTEMPT) !== 1" in plan["with"]["script"] + assert "scripts/release-platforms.json" in plan["with"]["script"] + assert "JSON.stringify({include: catalog})" in plan["with"]["script"] + + platforms = jobs["platforms"] + assert platforms["uses"] == "./.github/workflows/release-platform.yml" + assert platforms["needs"] == ["plan"] + assert platforms["strategy"]["fail-fast"] is False + assert platforms["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.matrix) }}" + assert platforms["with"]["full-validation"] is True + assert platforms["with"]["wallclock-evidence"] is True + assert ( + not { + "unit-performance-probe", + "integration-performance-probe", + "performance-workers", + } + & platforms["with"].keys() + ) + for key in ( + "integration-markers", + "integration-shard-count", + "integration-xdist-workers", + "integration-splitting-algorithm", + ): + assert platforms["with"][key] == "${{ matrix." + key.replace("-", "_") + " }}" + + for name, reusable in ( + ("source-checks", "ci"), + ("docs", "docs-build"), + ("wheels", "pypi-distributions"), + ): + assert jobs[name]["needs"] == ["plan"] + assert jobs[name]["uses"] == f"./.github/workflows/{reusable}.yml" + assert jobs["source-checks"]["with"] == {"suppress-pr-only-checks": True} + + ready = workflow_job(workflow, "candidate-ready") + assert ready["needs"] == ["plan", "platforms", "source-checks"] + record = workflow_step( + ready, "Capture native bytes and successful authorities without qualification" + )["with"]["script"] + assert "collectArchiveInventory({" in record + assert "selectRequiredJobs(jobs, catalog)" in record + assert "kind: 'release-wallclock-native-index', promotable: false" in record + assert "await qualify(" not in record + assert "native_ids" in ready["outputs"] and "evidence_id" in ready["outputs"] + verify = workflow_job(workflow, "verify") + assert verify["name"] == "Verify wall-clock artifacts" + assert set(verify["needs"]) == set(jobs) - {"verify"} + assert "always()" in verify["if"] and "!cancelled()" in verify["if"] + authorities = workflow_step( + verify, "Require every production authority and record artifact identities" + )["with"]["script"] + assert "selectRequiredJobs(jobs, catalog)" in authorities + assert "results[name]?.result !== 'success'" in authorities + assert ( + "['plan', 'source-checks', 'platforms', 'candidate-ready', 'docs', 'wheels']" in authorities + ) + assert "listJobsForWorkflowRunAttempt" in authorities + assert "matches.length !== 1" in authorities + assert "!artifact.expired" in authorities + assert "nativeIds !== process.env.NATIVE_IDS" in authorities + assert ( + workflow_step(verify, "Download exact current-run native artifacts")["with"]["artifact-ids"] + == "${{ needs.candidate-ready.outputs.native_ids }}" + ) + assert ( + "verifyLocalArchiveBytes({" + in workflow_step(verify, "Recheck indexed native bytes and prepare non-promotable assets")[ + "with" + ]["script"] + ) + native_upload = workflow_step(verify, "Upload verified native bytes without publishing") + assert native_upload["with"]["compression-level"] == 0 + assert native_upload["with"]["name"] == ( + "wallclock-proposed-verified-native-${{ github.run_attempt }}" + ) + check = workflow_step(verify, "Verify built bytes without granting release qualification") + assert "scripts.release_wallclock verify" in check["run"] + assert '--side proposed --source-root . --source-sha "$GITHUB_SHA"' in check["run"] + assert "--native-root" in check["run"] + assert "--docs-root" in check["run"] + assert "--python-root" in check["run"] + assert "release_rehearsal compare" not in str(workflow) + upload = workflow_step(verify, "Preserve non-promotable wall-clock evidence") + assert upload["with"]["name"] == "wallclock-proposed-proof-${{ github.run_attempt }}" + + +def test_wallclock_executes_one_full_proposed_dag_before_its_terminal_clock() -> None: + """Real elapsed time ends after every native/source/build-artifact authority.""" + _assert_single_world(load_workflow(WORKFLOW)) + + +def test_pr_binary_smoke_remains_default_but_can_be_excluded_from_release_measurement() -> None: + """A reusable caller must not accidentally inherit the PR-only binary job.""" + ci = load_workflow(ROOT / ".github/workflows/ci.yml") + assert ci["on"]["workflow_call"]["inputs"]["suppress-pr-only-checks"] == { + "type": "boolean", + "default": False, + } + assert ci["jobs"]["pr-binary-smoke"]["if"] == ( + "github.event_name == 'pull_request' && !inputs.suppress-pr-only-checks" + ) + for job_id, job in ci["jobs"].items(): + if job_id != "pr-binary-smoke": + assert "suppress-pr-only-checks" not in job.get("if", "") + + +def test_windows_wallclock_keeps_public_dependency_validation_with_the_job_token() -> None: + """A token-free legacy variable must not silently skip the public dependency gate.""" + native = load_workflow(ROOT / ".github/workflows/release-platform.yml") + step = workflow_step( + workflow_job(native, "release-validation"), "Run release validation tests (Windows)" + ) + assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" + assert step["env"]["GITHUB_APM_PAT"] == ( + "${{ inputs.wallclock-evidence && github.token || secrets.GH_CLI_PAT }}" + ) + assert "GITHUB_TOKEN" not in step["env"] + assert "GH_MODELS_PAT" not in str(step) + assert "APM_RUN_INFERENCE_TESTS" not in step["env"] + + +@pytest.mark.parametrize( + ("filename", "job_id", "step_name", "output"), + [ + ("release-platform", "build", "build", "build"), + ("release-unit", "unit-tests", "unit", "unit"), + ("release-integration", "integration-tests", "integration", "integration"), + ], +) +def test_runner_observation_is_opt_in_and_records_the_executing_source( + filename: str, job_id: str, step_name: str, output: str +) -> None: + """Default release behavior is unchanged; measurements identify actual hosts.""" + workflow = load_workflow(ROOT / f".github/workflows/{filename}.yml") + assert workflow["on"]["workflow_call"]["inputs"]["wallclock-evidence"] == { + "type": "boolean", + "default": False, + } + step = workflow_step( + workflow_job(workflow, job_id), + f"Record actual {step_name} runner for wall-clock proof", + ) + assert step["if"] == "inputs.wallclock-evidence" + assert step["shell"] == "bash" + assert step["run"] == ( + "python -m scripts.release_wallclock record-job " + '--side proposed --source-root . --source-sha "$GITHUB_SHA" ' + f"--output test-results/{output}-environment.json" + ) + assert not step.get("continue-on-error", False) + + +@pytest.mark.parametrize( + "fault", + [ + "publish", + "secret", + "cancel", + "synchronize", + "partial", + "no-observer", + "both-worlds", + "retuned-workers", + "late-docs", + "dropped-docs", + "dropped-source", + "waived-failure", + "bypassed-authorities", + "old-artifacts", + "fake-clock", + "collapsed-native-index", + "pr-only-job", + "broad-push-trigger", + ], +) +def test_wallclock_cannot_silently_change_the_measured_work(fault: str) -> None: + """Reject changes that weaken controls or measure a different pipeline.""" + workflow = deepcopy(load_workflow(WORKFLOW)) + jobs = workflow["jobs"] + if fault == "publish": + workflow["permissions"]["contents"] = "write" + elif fault == "secret": + jobs["platforms"]["secrets"] = "inherit" + elif fault == "cancel": + workflow["concurrency"]["cancel-in-progress"] = True + elif fault == "synchronize": + workflow["on"]["pull_request"]["types"].append("synchronize") + elif fault == "partial": + jobs["platforms"]["with"]["full-validation"] = False + elif fault == "no-observer": + jobs["platforms"]["with"]["wallclock-evidence"] = False + elif fault == "both-worlds": + jobs["platforms"]["with"]["integration-performance-probe"] = True + elif fault == "retuned-workers": + jobs["platforms"]["with"]["integration-xdist-workers"] = 8 + elif fault == "late-docs": + jobs["docs"]["needs"] = ["platforms"] + elif fault == "dropped-docs": + jobs["verify"]["needs"].remove("docs") + elif fault == "dropped-source": + jobs["source-checks"]["uses"] = "./.github/workflows/ci-source-performance.yml" + elif fault == "waived-failure": + jobs["platforms"]["continue-on-error"] = True + elif fault == "bypassed-authorities": + step = workflow_step( + jobs["verify"], "Require every production authority and record artifact identities" + ) + step["with"]["script"] = step["with"]["script"].replace( + "selectRequiredJobs(jobs, catalog)", "[]" + ) + elif fault == "old-artifacts": + step = workflow_step(jobs["verify"], "Preserve non-promotable wall-clock evidence") + step["with"]["name"] = "release-candidate-evidence" + elif fault == "collapsed-native-index": + del jobs["candidate-ready"] + elif fault == "pr-only-job": + jobs["source-checks"]["with"]["suppress-pr-only-checks"] = False + elif fault == "broad-push-trigger": + workflow["on"]["push"]["branches"] = ["**"] + else: + step = workflow_step( + jobs["verify"], "Verify built bytes without granting release qualification" + ) + step["run"] = "python -m scripts.release_rehearsal compare" + with pytest.raises(AssertionError): + _assert_single_world(workflow) diff --git a/tests/unit/test_windows_unit_diagnostics.py b/tests/unit/test_windows_unit_diagnostics.py index cfe165f1c5..db1c63e5f7 100644 --- a/tests/unit/test_windows_unit_diagnostics.py +++ b/tests/unit/test_windows_unit_diagnostics.py @@ -11,7 +11,12 @@ import pytest -from tests.workflow_contracts import load_workflow, shell_commands, workflow_job, workflow_step +from tests.workflow_contracts import ( + load_workflow, + wallclock_unit_commands, + workflow_job, + workflow_step, +) pytestmark = pytest.mark.component @@ -35,10 +40,24 @@ def _assert_windows_diagnostics(workflow: dict) -> None: linux = workflow_step(job, "Run unit tests") windows = workflow_step(job, "Run unit tests (Windows diagnostics)") assert linux["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" - assert shell_commands(linux) == [TEST_COMMAND + PARALLEL_ARGS] assert windows["if"] == "inputs.platform == 'windows'" assert windows["timeout-minutes"] == 60 - assert shell_commands(windows) == [[*TEST_COMMAND, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS]] + for observed in (False, True): + command = TEST_COMMAND + if observed: + command = [ + *TEST_COMMAND[:3], + "python", + "-m", + "pytest", + "-p", + "scripts.pytest_performance_evidence", + *TEST_COMMAND[4:], + ] + assert wallclock_unit_commands(linux, enabled=observed) == [command + PARALLEL_ARGS] + assert wallclock_unit_commands(windows, enabled=observed) == [ + [*command, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS] + ] assert windows["env"] == { "PYTHONUNBUFFERED": "1", "GITHUB_TOKEN": "${{ secrets.GH_MODELS_PAT }}", diff --git a/tests/workflow_contracts.py b/tests/workflow_contracts.py index 9a96f63e6b..4ee2c726ed 100644 --- a/tests/workflow_contracts.py +++ b/tests/workflow_contracts.py @@ -84,6 +84,18 @@ def shell_commands(step: WorkflowNode) -> list[list[str]]: ] +def wallclock_unit_commands(step: WorkflowNode, *, enabled: bool) -> list[list[str]]: + """Resolve only the explicitly supported, observation-only pytest choice.""" + expression = ( + "${{ inputs.wallclock-evidence && " + "'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }}" + ) + command = step["run"] + assert command.count(expression) == 1 + invocation = "python -m pytest -p scripts.pytest_performance_evidence" if enabled else "pytest" + return shell_commands({**step, "run": command.replace(expression, invocation)}) + + def assert_exact_command( commands: list[list[str]], expected: list[str], From 7d32ada472a8305467b1bcc72d00d828cc3564aa Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 14:15:39 +0200 Subject: [PATCH 14/23] fix(release): accept CRLF checksum sidecars Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/release-candidate.cjs | 17 +++++++------ scripts/release-candidate.test.cjs | 39 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index ce88081cd9..972ce1de42 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -176,6 +176,13 @@ function requireFreshArtifact(artifact, label) { requireArtifactDigest(artifact.digest, label); } +function assertChecksumSidecar(sidecarPath, digest, archive) { + const sidecar = fs.readFileSync(sidecarPath, "utf8").trimEnd(); + if (sidecar !== `${digest} ${archive}`) { + throw new Error(`Candidate checksum sidecar mismatch for ${archive}`); + } +} + function findOneArtifactByName(artifacts, name) { const matches = artifacts.filter((artifact) => artifact.name === name); if (matches.length !== 1) { @@ -386,10 +393,7 @@ function collectArchiveInventory({ artifactRoot, catalog, sha }) { if (!/^[0-9a-f]{64}$/.test(String(metadata.executable_sha256 || ""))) { throw new Error(`Candidate ${binaryName} executable digest is missing or malformed`); } - const sidecar = fs.readFileSync(sidecarPath, "utf8"); - if (sidecar !== `${digest} ${archive}\n`) { - throw new Error(`Candidate ${archive} checksum sidecar does not match`); - } + assertChecksumSidecar(sidecarPath, digest, archive); platforms[binaryName] = { runner: row.runner, platform: row.platform, @@ -680,10 +684,7 @@ function verifyLocalArchiveBytes({ evidence, catalog, artifactRoot, outputRoot } if (platform.archive_sha256 !== digest) { throw new Error(`Candidate archive digest mismatch for ${archive}`); } - const sidecar = fs.readFileSync(sidecarPath, "utf8"); - if (sidecar !== `${digest} ${archive}\n`) { - throw new Error(`Candidate checksum sidecar mismatch for ${archive}`); - } + assertChecksumSidecar(sidecarPath, digest, archive); fs.copyFileSync(archivePath, path.join(outputRoot, archive)); fs.copyFileSync(sidecarPath, path.join(outputRoot, `${archive}.sha256`)); } diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs index bcd60b10ed..e100daa2bd 100644 --- a/scripts/release-candidate.test.cjs +++ b/scripts/release-candidate.test.cjs @@ -595,6 +595,45 @@ describe("candidate manifest and publication verification", () => { }); }); + it("accepts Windows CRLF checksum sidecars while preserving digest checks", () => { + const root = makeTemp(); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot); + const archive = rc.archiveName("apm-windows-x86_64"); + const sidecar = path.join( + artifactRoot, + "apm-windows-x86_64", + "release-assets", + `${archive}.sha256`, + ); + const digest = fs.readFileSync(sidecar, "ascii").split(" ")[0]; + fs.writeFileSync(sidecar, `${digest} ${archive}\r\n`, "ascii"); + + const inventory = rc.collectArchiveInventory({ artifactRoot, catalog: catalog(), sha: SHA }); + + assert.equal(inventory["apm-windows-x86_64"].archive, archive); + }); + + it("rejects checksum sidecars with the wrong archive name", () => { + const root = makeTemp(); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot); + const archive = rc.archiveName("apm-windows-x86_64"); + const sidecar = path.join( + artifactRoot, + "apm-windows-x86_64", + "release-assets", + `${archive}.sha256`, + ); + const digest = fs.readFileSync(sidecar, "ascii").split(" ")[0]; + fs.writeFileSync(sidecar, `${digest} wrong.zip\r\n`, "ascii"); + + assert.throws( + () => rc.collectArchiveInventory({ artifactRoot, catalog: catalog(), sha: SHA }), + /Candidate checksum sidecar mismatch for apm-windows-x86_64\.zip/, + ); + }); + it("rejects legacy bare artifact names and tells operators to rerun all jobs", async () => { const root = makeTemp(); writeRequiredConfig(root); From 1687380d74cc255519b2ab93ced5ac8744fdd54b Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 14:54:11 +0200 Subject: [PATCH 15/23] chore(ci): retrigger proposed wall-clock proof Create a fresh push event after the failed mixed-attempt rerun so the proof workflow can run as attempt 1 with matching artifact names. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> From aa850fabc5cbf56d0df6a2b8baedc1a766aa484d Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 8 Sep 2026 20:15:26 +0200 Subject: [PATCH 16/23] fix(ci): require retained runner observations in wall-clock proofs Download exact native observation artifact IDs before terminal verification and use the canonical record validator at proof production. Preserve the measured history rather than accepting green-but-incomplete proof files. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-release-wallclock.yml | 13 ++++++++ .../docs/contributing/integration-testing.md | 6 ++-- scripts/release_wallclock.py | 1 + tests/unit/scripts/test_release_wallclock.py | 31 +++++++++++++++++++ tests/unit/test_release_wallclock_workflow.py | 17 ++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-release-wallclock.yml b/.github/workflows/ci-release-wallclock.yml index db10e6d0ce..ec26f0b70c 100644 --- a/.github/workflows/ci-release-wallclock.yml +++ b/.github/workflows/ci-release-wallclock.yml @@ -186,6 +186,12 @@ jobs: const expected = [ ...catalog.map(row => `candidate-${attempt}-${row.binary_name}`), `wallclock-proposed-docs-${attempt}`, `wallclock-proposed-python-${attempt}`, + ...catalog.flatMap(row => [ + `test-results-${attempt}-${row.binary_name}-core`, + `test-results-${attempt}-${row.binary_name}-unit`, + ...Array.from({length: row.integration_shard_count}, (_, index) => + `test-results-${attempt}-${row.binary_name}-integration-shard-${index + 1}`), + ]), ]; const selected = expected.map(name => { const matches = artifacts.filter(artifact => artifact.name === name && !artifact.expired); @@ -198,6 +204,8 @@ jobs: } core.setOutput('docs_id', selected[catalog.length].id); core.setOutput('python_id', selected[catalog.length + 1].id); + core.setOutput('observation_ids', + selected.slice(catalog.length + 2).map(artifact => artifact.id).join(',')); fs.mkdirSync('wallclock-results', {recursive: true}); fs.writeFileSync('wallclock-results/authorities.json', JSON.stringify({ promotable: false, required, artifacts: selected, @@ -241,6 +249,11 @@ jobs: evidence, catalog, artifactRoot: 'wallclock-inputs/native', outputRoot: 'wallclock-verified-native', }); + - name: Download exact native runner observations + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ steps.authorities.outputs.observation_ids }} + path: wallclock-inputs/native/observations - name: Verify built bytes without granting release qualification run: >- python -m scripts.release_wallclock verify diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index a30195b332..bec75fe9fe 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -666,8 +666,10 @@ the terminal `Verify wall-clock artifacts` job `completed_at` minus workflow `run_started_at`. Accept no result unless both workflows succeed, every included gate succeeds, all five native archives plus docs and wheel artifacts verify, and observer case inventory plus runner environment proof are present. The -proposed wrapper preserves separate candidate-ready native evidence capture and -serial verify / verified-native upload. All `wallclock-*` storage names are +terminal verifier must download native observation artifacts, not only archives, +and rejects missing or malformed runner job records through the shared validator. +The proposed wrapper preserves separate candidate-ready native evidence capture +and serial verify / verified-native upload. All `wallclock-*` storage names are non-promotable. Workflow success alone is not a savings claim; cite two accepted run IDs and their actual primary and secondary clocks before claiming an improvement. diff --git a/scripts/release_wallclock.py b/scripts/release_wallclock.py index ed9c51d541..f2f3553d86 100644 --- a/scripts/release_wallclock.py +++ b/scripts/release_wallclock.py @@ -743,6 +743,7 @@ def _job_records( names = [(item["github_job"], item["runner_name"]) for item in records] if len(names) != len(set(names)): raise ValueError("Duplicate wall-clock job record identity") + _validate_job_records({"job_records": records}, {"job_records": len(records)}) return records diff --git a/tests/unit/scripts/test_release_wallclock.py b/tests/unit/scripts/test_release_wallclock.py index 73a2438c90..f324684798 100644 --- a/tests/unit/scripts/test_release_wallclock.py +++ b/tests/unit/scripts/test_release_wallclock.py @@ -194,6 +194,9 @@ def _verified_fixture( python = _python_distributions(tmp_path / "python") monkeypatch.setattr(release_wallclock, "_git_head", lambda root: source_sha) _set_workflow_env(monkeypatch, tmp_path, controller) + (native / "runner.json").write_text( + json.dumps(record_job(side, source, source_sha)), encoding="ascii" + ) return source, native, docs, python @@ -280,10 +283,32 @@ def test_verify_artifacts_emits_nonpromotable_real_archive_proof( assert proof["controller_sha"] == controller assert proof["counts"]["native_archives"] == 5 assert proof["counts"]["candidate_metadata"] == metadata_count + assert proof["counts"]["job_records"] == 1 assert proof["filehashes"]["docs"]["format"] == "tar" assert proof["filehashes"]["python"]["wheel"]["version"] == VERSION +@pytest.mark.parametrize( + "side,sha,controller", + [ + ("baseline", BASE_SHA, BASE_CONTROLLER), + ("proposed", PROPOSED_SHA, PROPOSED_CONTROLLER), + ], +) +def test_verify_rejects_missing_observations_before_emitting_a_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + side: str, + sha: str, + controller: str, +) -> None: + source, native, docs, python = _verified_fixture(tmp_path, monkeypatch, side, sha, controller) + (native / "runner.json").unlink() + + with pytest.raises(ValueError, match="must contain at least one job record"): + verify_artifacts(side, source, sha, native, docs, python) + + def test_verify_proof_keeps_pull_request_head_distinct_from_execution_sha( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -299,6 +324,9 @@ def test_verify_proof_keeps_pull_request_head_distinct_from_execution_sha( event_name="pull_request", run_head_sha=PROPOSED_RUN_HEAD, ) + (native / "runner.json").write_text( + json.dumps(record_job("proposed", source, PROPOSED_SHA)), encoding="ascii" + ) proof = verify_artifacts("proposed", source, PROPOSED_SHA, native, docs, python) @@ -343,6 +371,9 @@ def test_verify_baseline_archives_raw_layout_binds_archives_to_original_executab monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) + (native / "runner.json").write_text( + json.dumps(record_job("baseline", source, BASE_SHA)), encoding="ascii" + ) proof = verify_artifacts("baseline", source, BASE_SHA, native, docs, python) observed = proof["filehashes"]["native"]["apm-linux-x86_64"] diff --git a/tests/unit/test_release_wallclock_workflow.py b/tests/unit/test_release_wallclock_workflow.py index aef7b3e901..5c546eb343 100644 --- a/tests/unit/test_release_wallclock_workflow.py +++ b/tests/unit/test_release_wallclock_workflow.py @@ -108,6 +108,18 @@ def _assert_single_world(workflow: dict) -> None: assert "matches.length !== 1" in authorities assert "!artifact.expired" in authorities assert "nativeIds !== process.env.NATIVE_IDS" in authorities + for suffix in ("core", "unit", "integration-shard-${index + 1}"): + assert f"test-results-${{attempt}}-${{row.binary_name}}-{suffix}" in authorities + assert "row.integration_shard_count" in authorities + assert "core.setOutput('observation_ids'" in authorities + observations = workflow_step(verify, "Download exact native runner observations") + assert observations["with"] == { + "artifact-ids": "${{ steps.authorities.outputs.observation_ids }}", + "path": "wallclock-inputs/native/observations", + } + assert verify["steps"].index(observations) < verify["steps"].index( + workflow_step(verify, "Verify built bytes without granting release qualification") + ) assert ( workflow_step(verify, "Download exact current-run native artifacts")["with"]["artifact-ids"] == "${{ needs.candidate-ready.outputs.native_ids }}" @@ -221,6 +233,7 @@ def test_runner_observation_is_opt_in_and_records_the_executing_source( "collapsed-native-index", "pr-only-job", "broad-push-trigger", + "missing-observation-download", ], ) def test_wallclock_cannot_silently_change_the_measured_work(fault: str) -> None: @@ -229,6 +242,10 @@ def test_wallclock_cannot_silently_change_the_measured_work(fault: str) -> None: jobs = workflow["jobs"] if fault == "publish": workflow["permissions"]["contents"] = "write" + elif fault == "missing-observation-download": + jobs["verify"]["steps"].remove( + workflow_step(jobs["verify"], "Download exact native runner observations") + ) elif fault == "secret": jobs["platforms"]["secrets"] = "inherit" elif fault == "cancel": From f4a4a02b3f963bdc20b12134f2f83a81f5676406 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 10:38:05 +0200 Subject: [PATCH 17/23] test(release): compare sentinel preservation without password hashes Resolve CodeQL alert 208 by reporting in-process ordinal comparison booleans instead of SHA256 fingerprints of synthetic tokens. Keep absence, preservation, and non-disclosure assertions unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test_release_validation_prerequisites.py | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/tests/unit/test_release_validation_prerequisites.py b/tests/unit/test_release_validation_prerequisites.py index 9e035dc5bc..9b4c83c2dc 100644 --- a/tests/unit/test_release_validation_prerequisites.py +++ b/tests/unit/test_release_validation_prerequisites.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import json import os import shutil @@ -28,12 +27,6 @@ } -def _digest(value: str | None) -> str | None: - if value is None: - return None - return hashlib.sha256(value.encode("utf-8")).hexdigest() - - def _clean_env(overrides: dict[str, str]) -> dict[str, str]: """Keep process basics, but never inherit ambient credentials into the fixture.""" env = { @@ -75,29 +68,28 @@ def _run_test_prerequisite(overrides: dict[str, str]) -> tuple[dict[str, object] . ([scriptblock]::Create($function.Extent.Text)) -function Get-EnvDigest { +$expectedEnvironment = @{} +foreach ($name in @("GITHUB_API_TOKEN", "GITHUB_APM_PAT", "GITHUB_MODELS_KEY", "GITHUB_TOKEN")) { + $expectedEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, "Process") +} + +function Test-EnvPreserved { param([string]$Name) $value = [Environment]::GetEnvironmentVariable($Name, "Process") if ($null -eq $value) { return $null } - $sha = [System.Security.Cryptography.SHA256]::Create() - try { - $bytes = [System.Text.Encoding]::UTF8.GetBytes($value) - return [System.BitConverter]::ToString($sha.ComputeHash($bytes)).Replace("-", "").ToLowerInvariant() - } finally { - $sha.Dispose() - } + return [string]::Equals($value, $expectedEnvironment[$Name], [StringComparison]::Ordinal) } $result = Test-Prerequisite [pscustomobject]@{ Result = [bool]$result - EnvDigests = [pscustomobject]@{ - GITHUB_API_TOKEN = Get-EnvDigest "GITHUB_API_TOKEN" - GITHUB_APM_PAT = Get-EnvDigest "GITHUB_APM_PAT" - GITHUB_MODELS_KEY = Get-EnvDigest "GITHUB_MODELS_KEY" - GITHUB_TOKEN = Get-EnvDigest "GITHUB_TOKEN" + EnvPreserved = [pscustomobject]@{ + GITHUB_API_TOKEN = Test-EnvPreserved "GITHUB_API_TOKEN" + GITHUB_APM_PAT = Test-EnvPreserved "GITHUB_APM_PAT" + GITHUB_MODELS_KEY = Test-EnvPreserved "GITHUB_MODELS_KEY" + GITHUB_TOKEN = Test-EnvPreserved "GITHUB_TOKEN" } Messages = @($script:Messages) } | ConvertTo-Json -Compress -Depth 5 @@ -139,8 +131,8 @@ def test_public_release_api_token_satisfies_prerequisite_when_inference_is_off() record, stdout, stderr = _run_test_prerequisite({"GITHUB_API_TOKEN": api_token}) assert record["Result"] is True - assert record["EnvDigests"] == { - "GITHUB_API_TOKEN": _digest(api_token), + assert record["EnvPreserved"] == { + "GITHUB_API_TOKEN": True, "GITHUB_APM_PAT": None, "GITHUB_MODELS_KEY": None, "GITHUB_TOKEN": None, @@ -155,7 +147,7 @@ def test_missing_release_and_pat_tokens_rejects_prerequisite() -> None: assert record["Result"] is False assert any("GitHub token setup failed" in message for message in _messages(record)) - assert record["EnvDigests"] == { + assert record["EnvPreserved"] == { "GITHUB_API_TOKEN": None, "GITHUB_APM_PAT": None, "GITHUB_MODELS_KEY": None, @@ -172,8 +164,8 @@ def test_api_token_does_not_satisfy_inference_prerequisite_or_seed_pat_aliases() ) assert record["Result"] is False - assert record["EnvDigests"] == { - "GITHUB_API_TOKEN": _digest(api_token), + assert record["EnvPreserved"] == { + "GITHUB_API_TOKEN": True, "GITHUB_APM_PAT": None, "GITHUB_MODELS_KEY": None, "GITHUB_TOKEN": None, @@ -200,11 +192,11 @@ def test_existing_pat_and_models_tokens_are_preserved_and_not_logged() -> None: ) assert record["Result"] is True - assert record["EnvDigests"] == { - "GITHUB_API_TOKEN": _digest(api_token), - "GITHUB_APM_PAT": _digest(apm_pat), - "GITHUB_MODELS_KEY": _digest(models_key), - "GITHUB_TOKEN": _digest(github_token), + assert record["EnvPreserved"] == { + "GITHUB_API_TOKEN": True, + "GITHUB_APM_PAT": True, + "GITHUB_MODELS_KEY": True, + "GITHUB_TOKEN": True, } messages = _messages(record) assert any("GITHUB_APM_PAT is set" in message for message in messages) From 7227d255170f29f01119651b0f17fdc4af45fd0a Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 11:55:22 +0200 Subject: [PATCH 18/23] fix(release): fold qualification and evidence review followups Preserve API-only Windows dependency validation and failure diagnostics; strengthen immutable promotion/signing regressions and runner observation comparability without changing native scheduling or publication authority. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/release-platform.yml | 5 +- .../docs/contributing/development-guide.md | 25 +- .../docs/contributing/integration-testing.md | 88 +++--- scripts/release-candidate.cjs | 18 +- scripts/release-candidate.test.cjs | 201 +++++++++++- scripts/release_wallclock.py | 191 ++++++++++- scripts/windows/run-installer-proof.ps1 | 12 +- .../windows/test-dependency-integration.ps1 | 11 +- scripts/windows/test-release-validation.ps1 | 22 +- tests/unit/scripts/test_compare_test_runs.py | 21 ++ tests/unit/scripts/test_release_wallclock.py | 299 +++++++++++++++++- .../test_release_validation_prerequisites.py | 143 ++++++++- tests/unit/test_release_wallclock_workflow.py | 6 +- .../test_windows_installer_proof_workflow.py | 122 +++++++ tests/unit/test_windows_signing_contract.py | 24 +- 15 files changed, 1063 insertions(+), 125 deletions(-) diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml index e138804dc3..4ee1d6fd93 100644 --- a/.github/workflows/release-platform.yml +++ b/.github/workflows/release-platform.yml @@ -305,12 +305,11 @@ jobs: timeout-minutes: 20 env: GITHUB_API_TOKEN: ${{ github.token }} - # Preserve the legacy public-dependency gate in read-only measurements. - GITHUB_APM_PAT: ${{ inputs.wallclock-evidence && github.token || secrets.GH_CLI_PAT }} + GITHUB_APM_PAT: ${{ !inputs.wallclock-evidence && secrets.GH_CLI_PAT || '' }} APM_E2E_TESTS: '1' run: | $env:PATH = "$PWD\dist\apm-windows-x86_64;$env:PATH" - .\scripts\windows\test-release-validation.ps1 + .\scripts\windows\test-release-validation.ps1 -PublicApiOnly:('${{ inputs.wallclock-evidence }}' -eq 'true') windows-installer: name: Candidate Windows Installer diff --git a/docs/src/content/docs/contributing/development-guide.md b/docs/src/content/docs/contributing/development-guide.md index 522f70b1cd..62c773b430 100644 --- a/docs/src/content/docs/contributing/development-guide.md +++ b/docs/src/content/docs/contributing/development-guide.md @@ -56,24 +56,13 @@ Enhancement suggestions are welcome! Please: ### Required CI and release gates -`Merge Gate / gate` is the required PR and merge-queue check. It accepts only -successful underlying GitHub Actions checks; skipped, neutral, missing, -duplicate, failed, cancelled, or timed-out checks fail closed. It requires Lint -and Test Architecture Ratchets in both contexts. - -Release candidates never reuse PR artifacts. Tags publish only fresh full -qualification or a same-SHA, trusted-main candidate with matching evidence. -Ordinary `main` green status is not enough. See -[Integration Testing](../integration-testing/) for the native platform matrix, -runtime selections, and archive-promotion model. - -Maintainers can apply the `ci-performance` PR label to run read-only native -and Windows before/after comparisons. These probes require identical test -outcomes and report execution time separately from runner setup and queue -delays. They neither publish releases nor replace required qualification. -Remove the label before unrelated follow-up pushes to avoid rerunning them. -See [performance capture](../integration-testing/#timing-and-performance-capture) -for artifact names and acceptance criteria. +`Merge Gate / gate` is the required PR and merge-queue check. +Ordinary `main` success is not release qualification. See the canonical +[CI gates](../integration-testing/#pull-requests-and-merge-queue), +[release qualification](../integration-testing/#release-qualification), +[candidate reuse](../integration-testing/#candidate-reuse-on-tags), +[publication recovery](../integration-testing/#publication-recovery), and +[performance capture](../integration-testing/#timing-and-performance-capture) sections. ### Workflow dependency updates diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index bec75fe9fe..627c0b7a21 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -396,49 +396,65 @@ the build independently; all remain required at the final gate. ### Release qualification -`build-release.yml` qualifies native candidates by platform through -`release-platform.yml`: +`build-release.yml` uses `release-platform.yml` to: -1. Resolve whether to reuse a qualified candidate or build fresh. -2. For fresh full validation, call source CI so lint, architecture ratchets, - coverage, red-team tests, and lifecycle smoke stay in the same authority as - normal CI. +1. Reuse a qualified candidate or build fresh. +2. For fresh full validation, call canonical source CI for lint, architecture + ratchets, coverage, red-team tests, and lifecycle smoke. 3. Run independent native unit and build jobs for each platform. -4. Let platform integration, isolated archive validation, and the Windows - installer job depend only on that platform's build artifact. -5. Promote only the exact archives that were packaged once, verified, unpacked, - and tested. - -Unix integration uses the shared `release-integration.yml` runner. All native -Unix lanes retain one shard with four workers and `duration_based_chunks` -assignment. Multi-runner ARM sharding remains experimental until a matched -performance comparison supports changing the release topology. Every -configured shard must succeed. - -For stable tags, read-only docs and wheel builds also start after release -planning. Their publishers wait for successful GitHub Release creation and -their own build artifacts. Prebuilding does not grant publishing permissions -or make failed qualification deployable. - -The publication path never repacks candidate archives. `scripts/package_release.py` -packages each native archive once, records archive and executable SHA-256 -digests, verifies archive member safety, checks the embedded version/build SHA, -extracts into a fresh destination, and rechecks the extracted executable before -release upload. +4. Run integration, isolated archive validation, and Windows installer tests + depending only on their platform's build artifact. +5. Promote only those exact, verified, unpacked, tested archives. + +Unix integration uses `release-integration.yml`: one shard, four workers, and +`duration_based_chunks` assignment per native lane. Multi-runner ARM sharding +remains experimental pending a matched performance comparison. Every shard +must succeed. + +Stable-tag read-only docs and wheel builds start after planning. Publishers +require successful GitHub Release creation and their own builds; prebuilding +neither grants publishing permissions nor bypasses qualification. + +`scripts/package_release.py` packages each native archive once, records archive +and executable SHA-256 digests, verifies member safety and embedded version/build +SHA, extracts into a fresh destination, and rechecks the executable before upload. +Publication never repacks archives. Candidate artifacts are attempt-scoped (`candidate--`; evidence is `release-candidate-evidence-`), so partial reruns must -not reuse old-attempt artifacts; use **Re-run all jobs** to regenerate complete -qualification. +not reuse old-attempt artifacts. Before publication, use **Re-run all jobs** to +regenerate complete qualification; afterward, follow [publication recovery](#publication-recovery). ### Candidate reuse on tags -A pushed tag may reuse an existing candidate only when release evidence proves -the same commit SHA already completed a successful, trusted-main full -qualification with immutable run identity, artifact identity, and matching -digests. Ordinary `main` green status is not enough, and PR artifacts are never -publication inputs. An absent or expired candidate triggers a fresh full build. -API errors or inconsistent candidate evidence fail the release rather than -silently falling back. +A tag can reuse only successful full qualification from +`build-release.yml` on `main` at the exact tag commit SHA, triggered by +`schedule` or `repository_dispatch` with type `manual-build-release`. +Evidence must match immutable run and artifact identities and digests. +PR, `workflow_dispatch`, ordinary `main` push, and earlier tag-push runs are +not eligible cross-run sources. Fresh tag qualification is usable within its +own run. Absent or expired candidates trigger a fresh full build; API errors +or inconsistent evidence fail closed. + +### Publication recovery + +The human release operator authorizes recovery after publication starts. +Qualification reruns are not publisher retries: +`verified-release-assets-`, `docs-pages-`, and +`pypi-distributions-` are attempt-scoped. **Re-run failed jobs** +increments the attempt without rerunning successful producers, so it cannot +recover their artifacts. **Re-run all jobs** also revisits successful publishers; +it is not a safe delivery-only retry. + +1. Stop retries. Retain exact successful producer and qualification evidence: + run IDs, attempts, SHA/version, artifact IDs/digests, and publisher logs. +2. For failed downloads, verify the expected producer attempt and artifact + availability. For failed Pages deployment, record the existing deployment + and source version. For partial PyPI uploads, inventory accepted and missing + filenames/versions. In every case, verify existing public GitHub/PyPI asset + hashes and versions against retained producer evidence before any retry. +3. Request release-operator authorization with that inventory. + The workflow provides no publisher-only recovery using immutable producer + identities; stop if safe continuation cannot be established. ### Platform matrix diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index 972ce1de42..e23c2cfaca 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -188,7 +188,7 @@ function findOneArtifactByName(artifacts, name) { if (matches.length !== 1) { const ErrorType = matches.length === 0 ? CandidateUnavailableError : Error; throw new ErrorType( - `Expected exactly one workflow artifact named ${name}, found ${matches.length}. ${RERUN_ALL_JOBS_HINT}.`, + `Expected exactly one workflow artifact named ${name}, found ${matches.length}`, ); } return matches[0]; @@ -345,7 +345,9 @@ async function discoverCandidateRun({ github, context, sha, catalog = loadCatalo }); } catch (error) { if (!(error instanceof CandidateUnavailableError)) throw error; - core?.info?.(`Prior candidate unavailable; building a fresh full candidate: ${error.message}`); + core?.info?.( + `Prior candidate run ${sameSha[0].id} attempt ${sameSha[0].run_attempt} unavailable: ${error.message}; building a fresh full candidate automatically. No operator action required.`, + ); return null; } } @@ -356,7 +358,13 @@ async function resolveExplicitCandidateRun({ github, context, runId, sha, catalo throw new Error("candidate_run_id must be a numeric workflow run id"); } const run = await getWorkflowRun(github, { owner, repo, run_id: Number(runId) }); - const candidate = await candidateFromRun({ github, owner, repo, run, expectedSha: sha, catalog }); + let candidate; + try { + candidate = await candidateFromRun({ github, owner, repo, run, expectedSha: sha, catalog }); + } catch (error) { + if (!(error instanceof CandidateUnavailableError)) throw error; + throw addAttemptHint(new Error(`Candidate run ${runId} attempt ${run.run_attempt}: ${error.message}`)); + } if (!candidate) { throw new Error(`Candidate run ${runId} does not contain release-candidate evidence`); } @@ -741,14 +749,18 @@ async function plan({ github, context, core }) { let candidateRunId = ""; let candidateArtifactIds = ""; let candidateEvidenceArtifactId = ""; + let decision = `fresh ${fullValidation ? "full qualification" : "platform validation"}; ${eventName(context)} uses the current run`; if (isTagPush(context)) { const candidate = await discoverCandidateRun({ github, context, sha: context.sha, catalog, core }); + decision = "fresh full qualification; no reusable candidate is available for this SHA"; if (candidate) { candidateRunId = String(candidate.run.id); candidateArtifactIds = candidate.artifacts.map((artifact) => String(artifact.id)).join(","); candidateEvidenceArtifactId = String(candidate.evidence.id); + decision = `reuse; trusted exact-SHA candidate from source run ${candidate.run.id} attempt ${candidate.run.run_attempt} SHA ${candidate.run.head_sha}`; } } + core?.info?.(`Release plan: ${decision}.`); core.setOutput("matrix", JSON.stringify(matrix)); core.setOutput("full_validation", String(fullValidation)); core.setOutput("candidate_run_id", candidateRunId); diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs index e100daa2bd..6e57958881 100644 --- a/scripts/release-candidate.test.cjs +++ b/scripts/release-candidate.test.cjs @@ -3,7 +3,6 @@ const assert = require("node:assert/strict"); const crypto = require("node:crypto"); const fs = require("node:fs"); -const os = require("node:os"); const path = require("node:path"); const { after, describe, it } = require("node:test"); @@ -38,7 +37,7 @@ function digestBytes(bytes) { } function makeTemp() { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "apm-release-candidate-")); + const root = fs.mkdtempSync(path.join(ROOT, ".release-candidate-test-")); after(() => fs.rmSync(root, { recursive: true, force: true })); return root; } @@ -146,7 +145,7 @@ function context(overrides = {}) { }; } -function fakeGithub({ runs = [], artifactsByRun = {}, jobsByRun = {}, runById = {}, calls = [] } = {}) { +function fakeGithub({ runs = [], artifactsByRun = {}, jobsByRunAttempt = {}, runById = {}, calls = [] } = {}) { return { rest: { actions: { @@ -160,11 +159,12 @@ function fakeGithub({ runs = [], artifactsByRun = {}, jobsByRun = {}, runById = }, listJobsForWorkflowRun: async (params) => { calls.push(["listJobsForWorkflowRun", params]); - return { data: { jobs: jobsByRun[Number(params.run_id)] || [] } }; + const attempt = runById[Number(params.run_id)]?.run_attempt || 1; + return { data: { jobs: jobsByRunAttempt[`${params.run_id}:${attempt}`] || [] } }; }, listJobsForWorkflowRunAttempt: async (params) => { calls.push(["listJobsForWorkflowRunAttempt", params]); - return { data: { jobs: jobsByRun[Number(params.run_id)] || [] } }; + return { data: { jobs: jobsByRunAttempt[`${params.run_id}:${params.attempt_number}`] || [] } }; }, getWorkflowRun: async (params) => { calls.push(["getWorkflowRun", params]); @@ -183,6 +183,58 @@ function withCwd(dir, fn) { .finally(() => process.chdir(old)); } +async function withQualifiedReusableCandidate(fn) { + const root = makeTemp(); + writeRequiredConfig(root); + const artifactRoot = path.join(root, "candidate-artifacts"); + writeCandidateFiles(artifactRoot); + const evidencePath = path.join(root, "release-candidate-evidence.json"); + const outputRoot = path.join(root, "release-assets"); + const run = { + id: 123, event: "schedule", path: rc.WORKFLOW_PATH, head_sha: SHA, + head_branch: "main", status: "completed", conclusion: "success", + run_attempt: 1, run_started_at: NOW, + }; + const artifacts = artifactMetadata(); + const calls = []; + const github = fakeGithub({ + artifactsByRun: { 123: artifacts }, + jobsByRunAttempt: { "123:1": requiredJobs(), "123:2": requiredJobs() }, + runById: { 123: run }, + calls, + }); + return withCwd(root, async () => { + const evidence = await rc.qualify({ + github, + context: context({ runId: 123, eventName: "schedule", ref: "refs/heads/main" }), + artifactRoot, + evidencePath, + }); + assert.deepEqual( + calls.filter(([name]) => name.startsWith("listJobs")), + [["listJobsForWorkflowRunAttempt", { + owner: "microsoft", repo: "apm", run_id: 123, attempt_number: 1, per_page: 100, + }]], + ); + assert.equal(evidence.run_id, 123); + assert.equal(evidence.run_attempt, 1); + assert.equal(evidence.event, "schedule"); + assert.equal(fs.existsSync(outputRoot), false); + calls.length = 0; + await fn({ + root, run, artifacts, calls, evidence, + options: { + github, + context: context({ runId: 999, runAttempt: 7 }), + runId: 123, + artifactRoot, + evidencePath, + outputRoot, + }, + }); + }); +} + describe("release candidate planning", () => { for (const unavailable of ["missing evidence", "missing platform", "expired evidence", "expired platform", "old run"]) { it(`builds fresh when automatic discovery finds ${unavailable}`, async () => { @@ -214,6 +266,10 @@ describe("release candidate planning", () => { assert.equal(outputs.candidate_artifact_ids, ""); assert.equal(outputs.candidate_evidence_artifact_id, ""); assert.match(messages[0], /building a fresh full candidate/); + assert.match(messages[0], /Prior candidate run 123 attempt 1 unavailable:/); + assert.match(messages[0], /automatically\. No operator action required\./); + assert.equal(messages.some((message) => /Re-run all jobs/.test(message)), false); + assert.equal(messages[1], "Release plan: fresh full qualification; no reusable candidate is available for this SHA."); }); } @@ -245,6 +301,7 @@ describe("release candidate planning", () => { it("outputs prior candidate run and artifact IDs for exact-SHA tag promotion", async () => { const outputs = {}; + const messages = []; const run = { id: 123, event: "schedule", @@ -264,13 +321,36 @@ describe("release candidate planning", () => { await rc.plan({ github, context: context(), - core: { setOutput(name, value) { outputs[name] = value; } }, + core: { + setOutput(name, value) { outputs[name] = value; }, + info(message) { messages.push(message); }, + }, }); assert.equal(outputs.candidate_run_id, "123"); assert.match(outputs.candidate_evidence_artifact_id, /^[0-9]+$/); assert.equal(outputs.candidate_artifact_ids.split(",").length, 5); + assert.deepEqual(messages, [ + `Release plan: reuse; trusted exact-SHA candidate from source run 123 attempt 1 SHA ${SHA}.`, + ]); }); + + for (const [overrides, message] of [ + [{}, "fresh full qualification; no reusable candidate is available for this SHA"], + [{ eventName: "schedule", ref: "refs/heads/main" }, "fresh full qualification; schedule uses the current run"], + [{ eventName: "repository_dispatch", ref: "refs/heads/main" }, "fresh full qualification; repository_dispatch uses the current run"], + [{ ref: "refs/heads/main" }, "fresh platform validation; push uses the current run"], + ]) { + it(`explains planning when ${message}`, async () => { + const messages = []; + await rc.plan({ + github: fakeGithub(), + context: context(overrides), + core: { setOutput() {}, info(message) { messages.push(message); } }, + }); + assert.deepEqual(messages, [`Release plan: ${message}.`]); + }); + } }); describe("trusted candidate discovery", () => { @@ -469,6 +549,21 @@ describe("trusted candidate discovery", () => { ); }); + it("keeps manual rerun guidance for an explicitly selected unavailable candidate", async () => { + const run = { + id: 123, event: "schedule", path: rc.WORKFLOW_PATH, head_sha: SHA, + head_branch: "main", status: "completed", conclusion: "success", + run_attempt: 1, run_started_at: NOW, + }; + await assert.rejects( + () => rc.resolveExplicitCandidateRun({ + github: fakeGithub({ runById: { 123: run } }), + context: context(), runId: 123, sha: SHA, + }), + /Candidate run 123 attempt 1: Expected exactly one workflow artifact named release-candidate-evidence-1, found 0\. Use Re-run all jobs/, + ); + }); + it("hard-fails invalid explicit candidate_run_id instead of falling back", async () => { const github = fakeGithub({ runById: { @@ -570,6 +665,78 @@ describe("trusted candidate discovery", () => { }); }); +describe("cross-run candidate promotion", () => { + it("promotes schedule run A attempt 1 from tag run B with exact archive and sidecar bytes", async () => { + await withQualifiedReusableCandidate(async ({ calls, evidence, options }) => { + const verified = await rc.verify(options); + + assert.deepEqual(verified, evidence); + assert.deepEqual(calls, [ + ["getWorkflowRun", { owner: "microsoft", repo: "apm", run_id: 123 }], + ["listJobsForWorkflowRunAttempt", { + owner: "microsoft", repo: "apm", run_id: 123, attempt_number: 1, per_page: 100, + }], + ["listWorkflowRunArtifacts", { + owner: "microsoft", repo: "apm", run_id: 123, per_page: 100, + }], + ]); + const expectedFiles = catalog().flatMap((row) => { + const archive = rc.archiveName(row.binary_name); + return [archive, `${archive}.sha256`]; + }); + assert.deepEqual(fs.readdirSync(options.outputRoot).sort(), expectedFiles.sort()); + for (const row of catalog()) { + const archive = rc.archiveName(row.binary_name); + for (const file of [archive, `${archive}.sha256`]) { + assert.deepEqual( + fs.readFileSync(path.join(options.outputRoot, file)), + fs.readFileSync(path.join(options.artifactRoot, row.binary_name, "release-assets", file)), + ); + } + } + }); + }); + + it("rejects stale qualification after the source run advances to another attempt", async () => { + await withQualifiedReusableCandidate(async ({ root, run, calls, options }) => { + await rc.verify({ ...options, outputRoot: path.join(root, "verified-attempt-1") }); + const qualifiedEvidence = fs.readFileSync(options.evidencePath); + run.run_attempt = 2; + calls.length = 0; + + await assert.rejects(() => rc.verify(options), /attempt does not match evidence/); + + assert.equal(fs.existsSync(options.outputRoot), false); + assert.deepEqual(fs.readFileSync(options.evidencePath), qualifiedEvidence); + assert.deepEqual(calls, [ + ["getWorkflowRun", { owner: "microsoft", repo: "apm", run_id: 123 }], + ]); + }); + }); + + for (const [drift, mutate, pattern] of [ + ["artifact ID", ({ artifacts }) => { artifacts[1].id += 1000; }, /Workflow artifact \d+ not found/], + ["artifact digest", ({ artifacts }) => { + artifacts[1].digest = `sha256:${"f".repeat(64)}`; + }, /Workflow artifact digest mismatch/], + ["workflow fingerprint", ({ root }) => { + fs.appendFileSync(path.join(root, rc.WORKFLOW_PATH), "\n# Changed after qualification\n", "ascii"); + }, /Candidate config hash mismatch for \.github\/workflows\/build-release\.yml/], + ["source status", ({ run }) => { run.status = "in_progress"; }, /Candidate run did not complete successfully/], + ["source conclusion", ({ run }) => { run.conclusion = "failure"; }, /Candidate run did not complete successfully/], + ]) { + it(`rejects cross-run ${drift} drift without creating publication assets`, async () => { + await withQualifiedReusableCandidate(async (fixture) => { + mutate(fixture); + + await assert.rejects(() => rc.verify(fixture.options), pattern); + + assert.equal(fs.existsSync(fixture.options.outputRoot), false); + }); + }); + } +}); + describe("candidate manifest and publication verification", () => { it("qualifies a manifest only with full source and five-platform native gate evidence", async () => { const root = makeTemp(); @@ -578,7 +745,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, }); await withCwd(root, async () => { @@ -641,7 +808,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog(), 1, { attemptNames: false }) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, }); await withCwd(root, async () => { @@ -706,7 +873,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot, catalog(), SHA, "1.2.4"); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, }); await withCwd(root, async () => { @@ -725,7 +892,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, runById: { 999: { id: 999, @@ -796,7 +963,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, runById: { 999: { id: 999, @@ -841,7 +1008,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, runById: { 999: { id: 999, @@ -890,7 +1057,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, runById: { 999: { id: 999, @@ -933,12 +1100,12 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const githubForQualify = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, }); const githubForVerify = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { - 999: requiredJobs(catalog(), (jobs) => + jobsByRunAttempt: { + "999:1": requiredJobs(catalog(), (jobs) => jobs.map((job) => job.name.includes("Lint") ? { ...job, conclusion: "failure" } : job, ), @@ -990,7 +1157,7 @@ describe("candidate manifest and publication verification", () => { writeCandidateFiles(artifactRoot); const github = fakeGithub({ artifactsByRun: { 999: artifactMetadata(catalog()) }, - jobsByRun: { 999: requiredJobs() }, + jobsByRunAttempt: { "999:1": requiredJobs() }, runById: { 999: { id: 999, diff --git a/scripts/release_wallclock.py b/scripts/release_wallclock.py index f2f3553d86..9dd3e294bd 100644 --- a/scripts/release_wallclock.py +++ b/scripts/release_wallclock.py @@ -1,9 +1,9 @@ """Capture and compare read-only release wall-clock evidence. -This module is deliberately not release qualification code. It verifies the -artifacts needed for an apples-to-apples timing comparison, emits a -non-promotable proof, and compares two already-successful GitHub Actions -attempts by their real terminal job timestamps. +This module is deliberately not release qualification code. It verifies +artifacts, emits a non-promotable proof, and compares two already-successful +GitHub Actions attempts by their real terminal job timestamps. Native runner +comparability is assessed separately from that clock arithmetic. """ from __future__ import annotations @@ -45,6 +45,17 @@ SIDE_CHOICES = ("baseline", "proposed") BAD_JOB_CONCLUSIONS = {"failure", "cancelled", "timed_out", "action_required"} SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +# Workflow digests are validated against each side, not compared across a DAG change. +RUNNER_COMPARISON_FIELDS = ( + "system", + "kernel", + "arch", + "cpu", + "cpu_count", + "image_version", + "python", + "source_uv_lock_sha256", +) @dataclass(frozen=True) @@ -738,12 +749,12 @@ def _job_records( "path": str(path), "github_job": record_workflow.get("github_job"), "runner_name": record_workflow.get("runner_name"), + "schema_version": data.get("schema_version"), + "recorded_at": data.get("recorded_at"), + "environment": data.get("environment"), } ) - names = [(item["github_job"], item["runner_name"]) for item in records] - if len(names) != len(set(names)): - raise ValueError("Duplicate wall-clock job record identity") - _validate_job_records({"job_records": records}, {"job_records": len(records)}) + _validate_job_records({"side": side, "job_records": records}, {"job_records": len(records)}) return records @@ -801,6 +812,17 @@ def verify_artifacts( "python_distributions": 2, }, "job_records": records, + "observation_assessment": _validate_job_records( + { + "side": side, + "job_records": records, + "filehashes": { + "source_uv_lock_sha256": source_digests["uv.lock"], + "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], + }, + }, + {"job_records": len(records)}, + ), "captured_at": _utc_now(), "scope": "read-only wall-clock artifact proof; not release promotion evidence", } @@ -886,12 +908,59 @@ def _validate_python_proof(python_proof: object) -> None: raise ValueError("Python distribution filenames are malformed") -def _validate_job_records(proof: dict[str, object], counts: dict[str, object]) -> None: +def _observation_roles(side: str) -> dict[tuple[str, str, str], tuple[str, ...]]: + """Index the distinct observer DAGs by job ID and normalized native platform. + + Baseline macOS jobs combine unit/build/integration; other baseline platforms + combine unit/build only. Proposed jobs observe all three roles separately. + Docs, wheels, validation and terminal jobs have no paired observer contract. + """ + roles = {} + for binary_name in BINARY_NAMES: + _, system, arch = binary_name.split("-", 2) + if side == "proposed": + for job, role in ( + ("unit-tests", "unit"), + ("build", "build"), + ("integration-tests", "integration"), + ): + roles[(job, system, arch)] = (f"{role}/{binary_name}",) + elif system == "darwin": + job = ( + "build-and-validate-macos-intel" + if arch == "x86_64" + else "build-and-validate-macos-arm" + ) + roles[(job, system, arch)] = tuple( + f"{role}/{binary_name}" for role in ("unit", "build", "integration") + ) + else: + roles[("build-and-test", system, arch)] = ( + f"unit/{binary_name}", + f"build/{binary_name}", + ) + roles[("integration-tests", system, arch)] = (f"integration/{binary_name}",) + return roles + + +def _validate_job_records(proof: dict[str, object], counts: dict[str, object]) -> dict[str, object]: + """Assess observations in O(N) time using fixed-size role expansion. + + Broken identities remain errors. Missing/invalid measurement metadata leaves + historical clocks readable, but cannot establish runner comparability. + """ records = proof.get("job_records") if not isinstance(records, list) or not records: raise ValueError("Wall-clock proof must contain at least one job record") if counts.get("job_records") != len(records): raise ValueError("Wall-clock proof job record count does not match") + expected = _observation_roles(_side(str(proof.get("side")))) + expected_roles = {role for roles in expected.values() for role in roles} + observed: dict[str, dict[str, str]] = {} + issues: list[str] = [] + unassessed = False + fields = (*RUNNER_COMPARISON_FIELDS, "source_build_release_yml_sha256") + filehashes = proof.get("filehashes", {}) seen: set[tuple[str, str]] = set() for record in records: if not isinstance(record, dict): @@ -906,6 +975,59 @@ def _validate_job_records(proof: dict[str, object], counts: dict[str, object]) - if key in seen: raise ValueError("Wall-clock proof contains duplicate job record identity") seen.add(key) + label = f"{github_job}/{runner_name}" + schema = record.get("schema_version") + environment = record.get("environment") + if isinstance(schema, bool) or not isinstance(schema, int) or schema != ABI_VERSION: + issues.append(f"{label}: missing or unsupported observation schema_version") + unassessed = True + continue + try: + _timestamp(record.get("recorded_at"), "observation recorded_at") + except ValueError: + issues.append(f"{label}: missing or invalid recorded_at") + unassessed = True + continue + if not isinstance(environment, dict) or any( + not isinstance(environment.get(field), str) or not environment[field].strip() + for field in fields + ): + issues.append(f"{label}: missing or invalid environment") + unassessed = True + continue + try: + _positive_int(environment["cpu_count"], "observation cpu_count") + for field in ("source_uv_lock_sha256", "source_build_release_yml_sha256"): + _require_sha256(environment[field], field) + except ValueError: + issues.append(f"{label}: invalid environment CPU count or source digest") + unassessed = True + continue + if isinstance(filehashes, dict): + for field in ("source_uv_lock_sha256", "source_build_release_yml_sha256"): + if field in filehashes and environment[field] != filehashes[field]: + issues.append(f"{label}: environment {field} does not match proof") + system = environment["system"].lower() + arch = environment["arch"].lower() + arch = {"amd64": "x86_64", "aarch64": "arm64"}.get(arch, arch) + roles = expected.get((github_job, system, arch)) + if roles is None: + issues.append(f"{label}: unexpected observer role/platform {system}/{arch}") + continue + for role in roles: + if role in observed: + issues.append(f"{role}: duplicate observer role/platform") + else: + observed[role] = {field: environment[field] for field in RUNNER_COMPARISON_FIELDS} + observed[role].update(system=system, arch=arch) + missing = sorted(expected_roles - observed.keys()) + if missing: + issues.append(f"Missing observer roles: {', '.join(missing)}") + return { + "status": "unassessed" if unassessed else "inconclusive" if issues else "complete", + "issues": issues, + "roles": observed, + } def _read_proof(path: Path, side: str, expected_sha: str) -> dict[str, object]: @@ -948,7 +1070,8 @@ def _read_proof(path: Path, side: str, expected_sha: str) -> dict[str, object]: _validate_native_proof(filehashes.get("native"), side, counts.get("candidate_metadata")) _validate_docs_proof(filehashes.get("docs")) _validate_python_proof(filehashes.get("python")) - _validate_job_records(proof, counts) + # Recompute rather than trusting an assessment saved by an older producer. + proof["observation_assessment"] = _validate_job_records(proof, counts) return proof @@ -1009,6 +1132,16 @@ def _validate_run( terminal_end = _timestamp(terminal["completed_at"], f"{side} terminal.completed_at") if terminal_end < terminal_start: raise ValueError(f"{side} terminal job has an invalid timeline") + assessment = proof["observation_assessment"] + for record in proof["job_records"]: + try: + recorded = _timestamp(record.get("recorded_at"), "observation recorded_at") + except ValueError: + continue # Already unassessed by the metadata validator. + if not created <= recorded <= terminal_end: + if assessment["status"] != "unassessed": + assessment["status"] = "inconclusive" + assessment["issues"].append("Observation recorded_at is outside the run timeline") executed_jobs = [] runner_seconds = 0.0 for job in jobs: @@ -1045,6 +1178,7 @@ def _validate_run( "executed_job_count": len(executed_jobs), "skipped_job_count": len(jobs) - len(executed_jobs), "version": proof.get("version"), + "observation_assessment": assessment, } @@ -1084,6 +1218,41 @@ def _select_terminal( return terminal +def _runner_comparability( + baseline: dict[str, object], proposed: dict[str, object] +) -> dict[str, object]: + """Compare indexed native roles without claiming campaign-level controls.""" + before = baseline["observation_assessment"] + after = proposed["observation_assessment"] + issues = [ + f"{side}: {issue}" + for side, assessment in (("baseline", before), ("proposed", after)) + for issue in assessment["issues"] + ] + for role, environment in before["roles"].items(): + other = after["roles"].get(role) + if other is not None: + for field in RUNNER_COMPARISON_FIELDS: + if environment[field] != other[field]: + issues.append(f"{role}: {field} differs between baseline and proposed") + status = "matched" + if before["status"] == "unassessed" or after["status"] == "unassessed": + status = "unassessed" + elif issues: + status = "inconclusive" + return { + "status": status, + "issues": issues, + "controlled_acceptance": False, + "scope": "native unit/build/integration runner observations only", + "limitations": [ + "Docs, wheels, source checks and validation runners are not assessed", + "Overlap, cache state, retry budget and population variance are not assessed", + "Matching native observations alone do not establish controlled acceptance", + ], + } + + def compare_runs( baseline_run: Path, proposed_run: Path, @@ -1156,6 +1325,8 @@ def compare_runs( "schema_version": ABI_VERSION, "readonly_adapter": True, "production_qualification": False, + "clock_arithmetic": "validated", + "comparability": _runner_comparability(baseline, proposed), "terminal_job": terminal_name, "baseline_required_job_names": sorted(baseline_required_names), "proposed_required_job_names": sorted(proposed_required_names), diff --git a/scripts/windows/run-installer-proof.ps1 b/scripts/windows/run-installer-proof.ps1 index ac166a8da1..7113d9939d 100644 --- a/scripts/windows/run-installer-proof.ps1 +++ b/scripts/windows/run-installer-proof.ps1 @@ -384,8 +384,7 @@ function Invoke-InstallerPytest { ) $result = Invoke-CapturedProcess -FilePath "uv" -Arguments $arguments ` -Name "$Name installer pytest" -TimeoutSeconds 900 -Environment $InstallerEnvironment - $result.junit = Read-InstallerOutcome -JUnitPath (Join-Path $OutDir "junit-$Name.xml") ` - -ExitCode $result.exit_code + # Persist raw diagnostics even when pytest never produced usable JUnit. $result.stdout | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stdout.log") -Encoding UTF8 $result.stderr | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stderr.log") -Encoding UTF8 Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") @@ -396,6 +395,15 @@ function Invoke-InstallerPytest { if ($result.stderr) { ($result.stderr -split "`n") | Select-Object -Last 30 | ForEach-Object { Write-Host "[$Name stderr] $_" } } + try { + $result.junit = Read-InstallerOutcome -JUnitPath (Join-Path $OutDir "junit-$Name.xml") ` + -ExitCode $result.exit_code + } catch { + $result.junit_validation_error = $_.Exception.Message + Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") + throw + } + Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") return $result } diff --git a/scripts/windows/test-dependency-integration.ps1 b/scripts/windows/test-dependency-integration.ps1 index 255cbc16bc..58d27d5962 100644 --- a/scripts/windows/test-dependency-integration.ps1 +++ b/scripts/windows/test-dependency-integration.ps1 @@ -304,7 +304,8 @@ function Test-DependencyCleanup { function Test-DependencyIntegration { param( [Parameter(Mandatory)] - [string]$BinaryPath + [string]$BinaryPath, + [switch]$PublicApiOnly ) Write-DepInfo "=== APM Dependencies Integration Testing ===" @@ -316,8 +317,12 @@ function Test-DependencyIntegration { $testDir = Join-Path $env:TEMP "apm-dep-test-$PID" New-Item -ItemType Directory -Path $testDir -Force | Out-Null - # Check for GitHub token - if (-not $env:GITHUB_CLI_PAT -and -not $env:GITHUB_TOKEN) { + # Public repositories can be tested without treating the API token as a PAT. + if ($PublicApiOnly -and -not $env:GITHUB_API_TOKEN) { + Write-DepError "Public API validation requires GITHUB_API_TOKEN" + return $false + } + if (-not $PublicApiOnly -and -not $env:GITHUB_CLI_PAT -and -not $env:GITHUB_TOKEN) { Write-DepError "GitHub token required for dependency testing" Write-DepInfo "Set GITHUB_CLI_PAT or GITHUB_TOKEN environment variable" return $false diff --git a/scripts/windows/test-release-validation.ps1 b/scripts/windows/test-release-validation.ps1 index 8023f37604..a7fc305481 100644 --- a/scripts/windows/test-release-validation.ps1 +++ b/scripts/windows/test-release-validation.ps1 @@ -12,7 +12,8 @@ # This is the final gate before release - testing the actual product as shipped param( - [string]$BinaryPath + [string]$BinaryPath, + [switch]$PublicApiOnly ) $ErrorActionPreference = "Continue" @@ -109,13 +110,15 @@ function Test-Prerequisite { $hasReleaseApiToken = [bool]$env:GITHUB_API_TOKEN $inferenceTestsEnabled = $env:APM_RUN_INFERENCE_TESTS -eq "1" - Initialize-GitHubToken + if (-not $PublicApiOnly) { + Initialize-GitHubToken + } # Initialize-GitHubToken does not return failure; check tokens after setup. # GITHUB_API_TOKEN is the workflow's read-only GitHub API token for public # release validation. It is not a PAT/Models credential, so do not copy it # into GITHUB_TOKEN, GITHUB_APM_PAT, or GITHUB_MODELS_KEY. - $hasPatOrModelsToken = [bool]($env:GITHUB_TOKEN -or $env:GITHUB_APM_PAT) + $hasPatOrModelsToken = -not $PublicApiOnly -and [bool]($env:GITHUB_TOKEN -or $env:GITHUB_APM_PAT) if ($inferenceTestsEnabled -and -not $hasPatOrModelsToken) { Write-ErrorText "Inference tests require GITHUB_TOKEN or GITHUB_APM_PAT; GITHUB_API_TOKEN only covers public release API access" @@ -425,8 +428,13 @@ function Main { Write-Info "Inference tests decoupled; skipping apm run tests" } - # Add dependency tests to total if available and GITHUB token is present - if ($script:DEPENDENCY_TESTS_AVAILABLE -and ($env:GITHUB_APM_PAT -or $env:GITHUB_TOKEN)) { + # Public API validation selects the same public checks without PAT/Models aliases. + $hasDependencyAccess = if ($PublicApiOnly) { + [bool]$env:GITHUB_API_TOKEN + } else { + [bool]($env:GITHUB_APM_PAT -or $env:GITHUB_TOKEN) + } + if ($script:DEPENDENCY_TESTS_AVAILABLE -and $hasDependencyAccess) { $testsTotal++ $dependencyTestsRun = $true Write-Info "Dependency integration tests will be included" @@ -479,10 +487,10 @@ function Main { Write-ErrorText "Hero scenario 2 (2-min guardrailing) failed" } - # Run dependency integration tests if available and GitHub token is set + # Keep the explicit public-only mode through the helper's credential gate. if ($dependencyTestsRun) { Write-Info "Running dependency integration tests with real GitHub repositories" - if (Test-DependencyIntegration -BinaryPath $script:BINARY_PATH) { + if (Test-DependencyIntegration -BinaryPath $script:BINARY_PATH -PublicApiOnly:$PublicApiOnly) { $testsPassed++ Write-Success "Dependency integration tests passed" } else { diff --git a/tests/unit/scripts/test_compare_test_runs.py b/tests/unit/scripts/test_compare_test_runs.py index 579dd5996b..854379fa2d 100644 --- a/tests/unit/scripts/test_compare_test_runs.py +++ b/tests/unit/scripts/test_compare_test_runs.py @@ -366,6 +366,7 @@ def _run_subtest_probe( env={ **os.environ, "PYTHONPATH": str(root), + "PYTEST_XDIST_AUTO_NUM_WORKERS": "2", "FAIL_SECOND_SUBTEST": "1" if fail_second_subtest else "", "SKIP_PLAIN_PARAM": "1" if skip_plain_param else "", }, @@ -377,6 +378,26 @@ def _run_subtest_probe( return result, capture(junit, SHA, "subtest probe", variant, shard, shard_count) +@pytest.mark.parametrize("detected_cpus", [16, 160]) +def test_nested_probe_auto_workers_are_bounded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, detected_cpus: int +) -> None: + monkeypatch.setattr(os, "cpu_count", lambda: detected_cpus) + monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", str(detected_cpus)) + + def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + env = kwargs["env"] + assert isinstance(env, dict) + assert env["PYTEST_XDIST_AUTO_NUM_WORKERS"] == "2" + assert command[command.index("-n") + 1] == "auto" + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(sys.modules[__name__], "capture", lambda *_: _baseline()) + result, _ = _run_subtest_probe(tmp_path, "budget", "baseline", 1, 1, "auto") + assert result.returncode == 0 + + def test_real_pytest_subtests_compare_under_xdist_baseline_and_two_shards( tmp_path: Path, ) -> None: diff --git a/tests/unit/scripts/test_release_wallclock.py b/tests/unit/scripts/test_release_wallclock.py index f324684798..cf351450a9 100644 --- a/tests/unit/scripts/test_release_wallclock.py +++ b/tests/unit/scripts/test_release_wallclock.py @@ -7,6 +7,7 @@ import stat import tarfile import zipfile +from collections.abc import Iterator from datetime import datetime, timedelta, timezone from pathlib import Path @@ -286,6 +287,11 @@ def test_verify_artifacts_emits_nonpromotable_real_archive_proof( assert proof["counts"]["job_records"] == 1 assert proof["filehashes"]["docs"]["format"] == "tar" assert proof["filehashes"]["python"]["wheel"]["version"] == VERSION + original = json.loads((native / "runner.json").read_text(encoding="ascii")) + for field in ("schema_version", "recorded_at", "environment"): + assert proof["job_records"][0][field] == original[field] + # One terminal observer does not cover either native DAG. + assert proof["observation_assessment"]["status"] == "inconclusive" @pytest.mark.parametrize( @@ -633,6 +639,50 @@ def test_verify_rejects_fake_or_wrong_project_python_distributions( verify_artifacts("baseline", source, BASE_SHA, native, docs, python) +def _observation_records(side: str) -> list[dict[str, object]]: + """Model the real eight-observer baseline and fifteen-observer proposed DAG.""" + records = [] + for system, arch in ( + ("Linux", "x86_64"), + ("Linux", "aarch64"), + ("Windows", "AMD64"), + ("Darwin", "x86_64"), + ("Darwin", "arm64"), + ): + if side == "proposed": + jobs = ("unit-tests", "build", "integration-tests") + elif system == "Darwin": + jobs = ( + "build-and-validate-macos-intel" + if arch == "x86_64" + else "build-and-validate-macos-arm", + ) + else: + jobs = ("build-and-test", "integration-tests") + for job in jobs: + records.append( + { + "path": f"wallclock/{side}-{system}-{arch}-{job}.json", + "github_job": job, + "runner_name": f"{side}-{system}-{arch}-{job}", + "schema_version": 1, + "recorded_at": _at(10), + "environment": { + "system": system, + "kernel": "fixture-kernel", + "arch": arch, + "cpu": f"fixture-{arch}", + "cpu_count": "4", + "image_version": f"fixture-{system}-{arch}-20260901.1", + "python": "3.12.10", + "source_uv_lock_sha256": "1" * 64, + "source_build_release_yml_sha256": "e" * 64, + }, + } + ) + return records + + def _proof( path: Path, side: str, @@ -641,6 +691,7 @@ def _proof( controller_sha: str, run_head_sha: str, ) -> Path: + observations = _observation_records(side) native = {} for index, binary_name in enumerate(BINARY_NAMES, start=1): archive = archive_name(binary_name) @@ -675,7 +726,7 @@ def _proof( "native_archives": 5, "native_sidecars": 5, "candidate_metadata": 5 if side == "proposed" else 0, - "job_records": 1, + "job_records": len(observations), "docs_indexes": 1, "python_distributions": 2, }, @@ -705,13 +756,7 @@ def _proof( }, }, }, - "job_records": [ - { - "path": "wallclock/job.json", - "github_job": "build", - "runner_name": f"{side}-runner", - } - ], + "job_records": observations, } ), encoding="ascii", @@ -816,6 +861,244 @@ def _compare_ok( ) +def test_compare_matches_distinct_native_dags_without_claiming_controlled_acceptance( + tmp_path: Path, +) -> None: + result = _compare_ok(*_compare_fixture(tmp_path)) + + assert len(_observation_records("baseline")) == 8 + assert len(_observation_records("proposed")) == 15 + for side in ("baseline", "proposed"): + assessment = result[side]["observation_assessment"] + assert assessment["status"] == "complete" + assert set(assessment["roles"]) == { + f"{role}/{binary_name}" + for role in ("unit", "build", "integration") + for binary_name in BINARY_NAMES + } + assert result["clock_arithmetic"] == "validated" + assert result["comparability"]["status"] == "matched" + assert result["comparability"]["issues"] == [] + assert result["comparability"]["controlled_acceptance"] is False + assert result["production_qualification"] is False + + +@pytest.mark.parametrize("field", ["schema_version", "recorded_at", "environment"]) +def test_job_record_ingestion_preserves_missing_metadata_as_unassessed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + source, native, docs, python = _verified_fixture( + tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER + ) + path = native / "runner.json" + payload = json.loads(path.read_text(encoding="ascii")) + payload.pop(field) + path.write_text(json.dumps(payload), encoding="ascii") + + proof = verify_artifacts("baseline", source, BASE_SHA, native, docs, python) + + assert proof["job_records"][0][field] is None + assert proof["observation_assessment"]["status"] == "unassessed" + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("schema_version", None), + ("schema_version", True), + ("schema_version", 1.0), + ("schema_version", 2), + ("recorded_at", None), + ("recorded_at", "not-a-time"), + ("recorded_at", "2026-09-08T00:00:10"), + ("environment", None), + ("environment", {}), + ("image_version", ""), + ("cpu_count", "0"), + ("cpu_count", True), + ("source_uv_lock_sha256", "not-a-digest"), + ], +) +def test_compare_incomplete_metadata_cannot_count_as_controlled_gain( + tmp_path: Path, field: str, value: object +) -> None: + paths = _compare_fixture(tmp_path) + payload = json.loads(paths[3].read_text(encoding="ascii")) + record = payload["job_records"][0] + target = ( + record + if field in {"schema_version", "recorded_at", "environment"} + else record["environment"] + ) + if value is None: + target.pop(field) + else: + target[field] = value + # A saved claim must not override recomputation. + payload["observation_assessment"] = {"status": "complete", "issues": []} + paths[3].write_text(json.dumps(payload), encoding="ascii") + + result = _compare_ok(*paths) + + assert result["comparability"]["status"] == "unassessed" + assert result["comparability"]["issues"] + assert result["comparability"]["controlled_acceptance"] is False + assert result["delta_seconds"]["primary_gain_seconds"] == 10 + assert result["clock_arithmetic"] == "validated" + + +def test_compare_legacy_projected_records_keep_raw_clocks_without_inventing_metadata( + tmp_path: Path, +) -> None: + paths = _compare_fixture(tmp_path) + for path in paths[2:]: + payload = json.loads(path.read_text(encoding="ascii")) + payload["job_records"] = [ + {"path": "historical/job.json", "github_job": "build", "runner_name": "legacy"} + ] + payload["counts"]["job_records"] = 1 + path.write_text(json.dumps(payload), encoding="ascii") + + result = _compare_ok(*paths) + + assert result["gain_percent"]["primary"] == 10 + assert result["baseline"]["wallclock_seconds_from_created_at"] == 100 + assert result["proposed"]["wallclock_seconds_from_created_at"] == 90 + assert result["comparability"]["status"] == "unassessed" + assert result["comparability"]["controlled_acceptance"] is False + assert result["baseline"]["observation_assessment"]["roles"] == {} + + +@pytest.mark.parametrize( + ("side", "index"), + [("baseline", index) for index in range(8)] + [("proposed", index) for index in range(15)], +) +def test_compare_requires_every_observer_role_and_platform( + tmp_path: Path, side: str, index: int +) -> None: + paths = _compare_fixture(tmp_path) + path = paths[2] if side == "baseline" else paths[3] + payload = json.loads(path.read_text(encoding="ascii")) + payload["job_records"].pop(index) + payload["counts"]["job_records"] -= 1 + path.write_text(json.dumps(payload), encoding="ascii") + + result = _compare_ok(*paths) + + assert result[side]["observation_assessment"]["status"] == "inconclusive" + assert any("Missing observer roles:" in issue for issue in result["comparability"]["issues"]) + assert result["comparability"]["controlled_acceptance"] is False + assert result["delta_seconds"]["primary_gain_seconds"] == 10 + + +@pytest.mark.parametrize( + ("fault", "message"), + [ + ("image", "image_version differs"), + ("cpu", "cpu_count differs"), + ("duplicate-role", "duplicate observer role/platform"), + ("wrong-platform", "Missing observer roles"), + ("wrong-job", "unexpected observer role/platform"), + ("timestamp", "outside the run timeline"), + ("timestamp-before", "outside the run timeline"), + ("source-digest", "does not match proof"), + ], +) +def test_compare_environment_drift_and_ambiguous_coverage_are_inconclusive( + tmp_path: Path, fault: str, message: str +) -> None: + paths = _compare_fixture(tmp_path) + payload = json.loads(paths[3].read_text(encoding="ascii")) + record = payload["job_records"][0] + if fault == "image": + record["environment"]["image_version"] = "different-image" + elif fault == "cpu": + record["environment"]["cpu_count"] = "8" + elif fault == "duplicate-role": + duplicate = dict(record, runner_name="another-runner") + payload["job_records"].append(duplicate) + payload["counts"]["job_records"] += 1 + elif fault == "wrong-platform": + record["environment"]["arch"] = "aarch64" + elif fault == "wrong-job": + record["github_job"] = "unrecognized-job" + elif fault == "timestamp": + record["recorded_at"] = _at(91) + elif fault == "timestamp-before": + record["recorded_at"] = _at(-1) + else: + record["environment"]["source_build_release_yml_sha256"] = "f" * 64 + paths[3].write_text(json.dumps(payload), encoding="ascii") + + result = _compare_ok(*paths) + + assert result["comparability"]["status"] == "inconclusive" + assert any(message in issue for issue in result["comparability"]["issues"]) + assert result["comparability"]["controlled_acceptance"] is False + assert result["delta_seconds"]["primary_gain_seconds"] == 10 + + +def test_compare_allows_intentional_workflow_digest_change_between_sides(tmp_path: Path) -> None: + paths = _compare_fixture(tmp_path) + payload = json.loads(paths[3].read_text(encoding="ascii")) + payload["workflow_digest"] = "f" * 64 + payload["filehashes"]["source_build_release_yml_sha256"] = "f" * 64 + for record in payload["job_records"]: + record["environment"]["source_build_release_yml_sha256"] = "f" * 64 + paths[3].write_text(json.dumps(payload), encoding="ascii") + + result = _compare_ok(*paths) + + assert result["comparability"]["status"] == "matched" + assert result["comparability"]["controlled_acceptance"] is False + + +def test_observation_validation_record_visits_scale_linearly() -> None: + """Count scans and identity comparisons at N and 10N without wall clocks.""" + + class CountedName(str): + """Expose a quadratic list-based replacement for identity set membership.""" + + comparisons = 0 + + def __eq__(self, other: object) -> bool: + type(self).comparisons += 1 + return super().__eq__(other) + + def __hash__(self) -> int: + return super().__hash__() + + class CountedRecords(list[dict[str, object]]): + """Expose repeated scans, including index-based scans, deterministically.""" + + visits = 0 + + def __iter__(self) -> Iterator[dict[str, object]]: + for record in super().__iter__(): + self.visits += 1 + yield record + + def __getitem__(self, key: int) -> dict[str, object]: + self.visits += 1 + return super().__getitem__(key) + + visits = [] + template = _observation_records("proposed")[0] + for size in (30, 300): + CountedName.comparisons = 0 + records = CountedRecords( + dict(template, runner_name=CountedName(f"runner-{index}")) for index in range(size) + ) + assessment = release_wallclock._validate_job_records( + {"side": "proposed", "job_records": records}, {"job_records": size} + ) + assert assessment["status"] == "inconclusive" # Duplicate roles, not identities. + operations = records.visits + CountedName.comparisons + assert size <= operations <= 3 * size + visits.append(operations) + assert visits[1] <= 12 * visits[0] + + @pytest.mark.parametrize( ("fault", "match"), [ diff --git a/tests/unit/test_release_validation_prerequisites.py b/tests/unit/test_release_validation_prerequisites.py index 9b4c83c2dc..5f9b19c5a2 100644 --- a/tests/unit/test_release_validation_prerequisites.py +++ b/tests/unit/test_release_validation_prerequisites.py @@ -38,7 +38,9 @@ def _clean_env(overrides: dict[str, str]) -> dict[str, str]: return env -def _run_test_prerequisite(overrides: dict[str, str]) -> tuple[dict[str, object], str, str]: +def _run_test_prerequisite( + overrides: dict[str, str], *, public_api_only: bool = False +) -> tuple[dict[str, object], str, str]: command = r""" $ErrorActionPreference = "Stop" @@ -95,6 +97,7 @@ def _run_test_prerequisite(overrides: dict[str, str]) -> tuple[dict[str, object] } | ConvertTo-Json -Compress -Depth 5 """ + command = f"$PublicApiOnly = ${str(public_api_only).lower()}\n" + command result = subprocess.run( [PWSH, "-NoProfile", "-NonInteractive", "-Command", command], cwd=ROOT, @@ -125,10 +128,15 @@ def _assert_sentinels_not_printed(stdout: str, stderr: str, *sentinels: str) -> @pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") -def test_public_release_api_token_satisfies_prerequisite_when_inference_is_off() -> None: +@pytest.mark.parametrize("public_api_only", [False, True]) +def test_public_release_api_token_satisfies_prerequisite_when_inference_is_off( + public_api_only: bool, +) -> None: api_token = "readonly-api-token-sentinel" - record, stdout, stderr = _run_test_prerequisite({"GITHUB_API_TOKEN": api_token}) + record, stdout, stderr = _run_test_prerequisite( + {"GITHUB_API_TOKEN": api_token}, public_api_only=public_api_only + ) assert record["Result"] is True assert record["EnvPreserved"] == { @@ -142,8 +150,9 @@ def test_public_release_api_token_satisfies_prerequisite_when_inference_is_off() @pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") -def test_missing_release_and_pat_tokens_rejects_prerequisite() -> None: - record, _stdout, _stderr = _run_test_prerequisite({}) +@pytest.mark.parametrize("public_api_only", [False, True]) +def test_missing_release_and_pat_tokens_rejects_prerequisite(public_api_only: bool) -> None: + record, _stdout, _stderr = _run_test_prerequisite({}, public_api_only=public_api_only) assert record["Result"] is False assert any("GitHub token setup failed" in message for message in _messages(record)) @@ -156,11 +165,15 @@ def test_missing_release_and_pat_tokens_rejects_prerequisite() -> None: @pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") -def test_api_token_does_not_satisfy_inference_prerequisite_or_seed_pat_aliases() -> None: +@pytest.mark.parametrize("public_api_only", [False, True]) +def test_api_token_does_not_satisfy_inference_prerequisite_or_seed_pat_aliases( + public_api_only: bool, +) -> None: api_token = "readonly-api-token-sentinel" record, stdout, stderr = _run_test_prerequisite( - {"APM_RUN_INFERENCE_TESTS": "1", "GITHUB_API_TOKEN": api_token} + {"APM_RUN_INFERENCE_TESTS": "1", "GITHUB_API_TOKEN": api_token}, + public_api_only=public_api_only, ) assert record["Result"] is False @@ -202,3 +215,119 @@ def test_existing_pat_and_models_tokens_are_preserved_and_not_logged() -> None: assert any("GITHUB_APM_PAT is set" in message for message in messages) assert any("GITHUB_TOKEN is set" in message for message in messages) _assert_sentinels_not_printed(stdout, stderr, api_token, apm_pat, github_token, models_key) + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell parser is not installed") +@pytest.mark.parametrize( + ("public_api_only", "fail_update", "expected_exit"), + [(True, False, 0), (True, True, 1), (False, False, 0)], +) +def test_release_caller_runs_public_dependency_helper_without_api_token_aliases( + tmp_path: Path, public_api_only: bool, fail_update: bool, expected_exit: int +) -> None: + """Run the real caller and helper gates; stub only binary/network leaf checks.""" + scripts = ROOT / "scripts/windows" + caller = (scripts / "test-release-validation.ps1").read_text(encoding="utf-8") + entrypoint = "# Run main function\nMain\n" + assert caller.endswith(entrypoint) + # Inject hermetic leaves immediately before the real Main invocation. Keep + # parameter binding, helper loading, prerequisites and both selection gates. + leaves = r""" +function Test-BasicCommand { return $true } +function Test-RuntimeSetup { return $true } +function Test-HeroZeroConfig { return $true } +function Test-HeroGuardrailing { return $true } +function Test-RealDependencyInstallation { + param([string]$TestDir, [string]$ApmBinary) + $global:SelectedChecks.Add("installation") + return $true +} +function Test-MultiDependencyScenario { + param([string]$TestDir, [string]$ApmBinary) + $global:SelectedChecks.Add("multi-dependency") + return $true +} +function Test-DependencyUpdate { + param([string]$TestDir, [string]$ApmBinary) + $global:SelectedChecks.Add("update") + return $env:FIXTURE_FAIL_UPDATE -ne "1" +} +function Test-DependencyCleanup { + param([string]$TestDir, [string]$ApmBinary) + $global:SelectedChecks.Add("cleanup") + return $true +} +""" + (tmp_path / "caller.ps1").write_text( + caller.removesuffix(entrypoint) + leaves + entrypoint, encoding="utf-8" + ) + for helper in ("github-token-helper.ps1", "test-dependency-integration.ps1"): + shutil.copyfile(scripts / helper, tmp_path / helper) + (tmp_path / "candidate.exe").touch() + command = r""" +$ErrorActionPreference = "Stop" +$global:SelectedChecks = [System.Collections.Generic.List[string]]::new() +$expectedEnvironment = @{} +foreach ($name in @("GITHUB_API_TOKEN", "GITHUB_APM_PAT", "GITHUB_MODELS_KEY", "GITHUB_TOKEN")) { + $expectedEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, "Process") +} +& (Join-Path $PWD "caller.ps1") -BinaryPath (Join-Path $PWD "candidate.exe") ` + -PublicApiOnly:($env:FIXTURE_PUBLIC_API_ONLY -eq "1") +$callerExit = $LASTEXITCODE +$preserved = @{} +foreach ($name in $expectedEnvironment.Keys) { + $value = [Environment]::GetEnvironmentVariable($name, "Process") + $preserved[$name] = if ($null -eq $value) { $null } else { + [string]::Equals($value, $expectedEnvironment[$name], [StringComparison]::Ordinal) + } +} +[pscustomobject]@{ + ExitCode = $callerExit + SelectedChecks = @($global:SelectedChecks) + EnvPreserved = $preserved +} | ConvertTo-Json -Compress -Depth 5 +exit $callerExit +""" + api_token = "caller-readonly-api-sentinel" + pat = "caller-pat-sentinel" + models_token = "caller-models-token-sentinel" + models_key = "caller-models-key-sentinel" + overrides = { + "GITHUB_API_TOKEN": api_token, + "TEMP": str(tmp_path), + "FIXTURE_PUBLIC_API_ONLY": "1" if public_api_only else "0", + "FIXTURE_FAIL_UPDATE": "1" if fail_update else "0", + } + if not public_api_only: + overrides.update( + GITHUB_APM_PAT=pat, GITHUB_TOKEN=models_token, GITHUB_MODELS_KEY=models_key + ) + completed = subprocess.run( + [PWSH, "-NoProfile", "-NonInteractive", "-Command", command], + cwd=tmp_path, + env=_clean_env(overrides), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert completed.returncode == expected_exit, completed.stdout + completed.stderr + records = [line for line in completed.stdout.splitlines() if line.startswith("{")] + assert records, completed.stdout + completed.stderr + record = json.loads(records[-1]) + assert record["ExitCode"] == expected_exit + expected_checks = ["installation", "multi-dependency", "update"] + if not fail_update: + expected_checks.append("cleanup") + assert record["SelectedChecks"] == expected_checks + assert record["EnvPreserved"] == { + "GITHUB_API_TOKEN": True, + "GITHUB_APM_PAT": None if public_api_only else True, + "GITHUB_MODELS_KEY": None if public_api_only else True, + "GITHUB_TOKEN": None if public_api_only else True, + } + assert "Dependency integration tests will be included" in completed.stdout + assert f"Results: {4 if fail_update else 5}/5 tests passed" in completed.stdout + _assert_sentinels_not_printed( + completed.stdout, completed.stderr, api_token, pat, models_token, models_key + ) diff --git a/tests/unit/test_release_wallclock_workflow.py b/tests/unit/test_release_wallclock_workflow.py index 5c546eb343..553469f6b1 100644 --- a/tests/unit/test_release_wallclock_workflow.py +++ b/tests/unit/test_release_wallclock_workflow.py @@ -167,16 +167,18 @@ def test_pr_binary_smoke_remains_default_but_can_be_excluded_from_release_measur def test_windows_wallclock_keeps_public_dependency_validation_with_the_job_token() -> None: - """A token-free legacy variable must not silently skip the public dependency gate.""" + """Select public checks explicitly, without aliasing the job token into a PAT.""" native = load_workflow(ROOT / ".github/workflows/release-platform.yml") step = workflow_step( workflow_job(native, "release-validation"), "Run release validation tests (Windows)" ) assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" assert step["env"]["GITHUB_APM_PAT"] == ( - "${{ inputs.wallclock-evidence && github.token || secrets.GH_CLI_PAT }}" + "${{ !inputs.wallclock-evidence && secrets.GH_CLI_PAT || '' }}" ) + assert "-PublicApiOnly:('${{ inputs.wallclock-evidence }}' -eq 'true')" in step["run"] assert "GITHUB_TOKEN" not in step["env"] + assert "GITHUB_MODELS_KEY" not in step["env"] assert "GH_MODELS_PAT" not in str(step) assert "APM_RUN_INFERENCE_TESTS" not in step["env"] diff --git a/tests/unit/test_windows_installer_proof_workflow.py b/tests/unit/test_windows_installer_proof_workflow.py index 1810eae7b6..4bf6c37d6b 100644 --- a/tests/unit/test_windows_installer_proof_workflow.py +++ b/tests/unit/test_windows_installer_proof_workflow.py @@ -262,3 +262,125 @@ def test_installer_outcome_requires_executed_cases( assert (completed.returncode == 0) is accepted, completed.stdout + completed.stderr if accepted: assert json.loads(completed.stdout)["skipped"] == 0 + + +def test_installer_diagnostics_are_persisted_before_junit_validation() -> None: + """The uploadable raw result and visible summary precede any JUnit rejection.""" + body = _script_text().split("function Invoke-InstallerPytest {", 1)[1] + body = body.split("function Invoke-ProcessCaptureSelfTest {", 1)[0] + validation = body.index("$result.junit = Read-InstallerOutcome") + for diagnostic in ( + '"$Name-stdout.log"', + '"$Name-stderr.log"', + 'Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json")', + 'Write-Host "[$Name] exit $($result.exit_code) timed_out=$($result.timed_out)"', + ): + assert body.index(diagnostic) < validation + + +@pytest.mark.windows_compat +@pytest.mark.parametrize( + ("junit", "exit_code", "timed_out"), + [ + (None, 124, True), + ("", 0, False), + ( + '' + "", + 0, + False, + ), + ( + '' + "", + 0, + False, + ), + ], + ids=["missing-after-timeout", "malformed", "empty", "skipped", "false-success"], +) +def test_installer_rejected_junit_retains_process_diagnostics( + tmp_path: Path, junit: str | None, exit_code: int, timed_out: bool +) -> None: + """Execute the real persistence/validation path with a captured-process stub.""" + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("pwsh is not installed") + out_dir = tmp_path / "diagnostics" + out_dir.mkdir() + if junit is not None: + (out_dir / "junit-fixture.xml").write_text(junit, encoding="ascii") + driver = tmp_path / "invoke-installer.ps1" + driver.write_text( + r""" +param([string]$SourceScript, [string]$OutDir, [int]$FixtureExitCode, [string]$TimedOut) +$ErrorActionPreference = "Stop" +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile( + $SourceScript, [ref]$tokens, [ref]$errors +) +if ($errors.Count) { throw ($errors | Out-String) } +foreach ($name in @("Write-JsonFile", "Read-InstallerOutcome", "Invoke-InstallerPytest")) { + $definition = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq $name + }, $true) + if (-not $definition) { throw "Missing function: $name" } + . ([scriptblock]::Create($definition.Extent.Text)) +} +function Invoke-CapturedProcess { + param($FilePath, $Arguments, $Name, $TimeoutSeconds, $Environment) + return [ordered]@{ + name = $Name + exit_code = $FixtureExitCode + timed_out = $TimedOut -eq "true" + stdout = "fixture captured stdout" + stderr = "fixture captured stderr" + timeout_seconds = $TimeoutSeconds + } +} +Invoke-InstallerPytest -Name "fixture" -OutDir $OutDir ` + -BaseTemp (Join-Path $OutDir "fresh-basetemp") -InstallerEnvironment @{} | + ConvertTo-Json -Depth 8 +""", + encoding="ascii", + ) + completed = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(driver), + "-SourceScript", + str(PROOF_SCRIPT), + "-OutDir", + str(out_dir), + "-FixtureExitCode", + str(exit_code), + "-TimedOut", + str(timed_out).lower(), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert completed.returncode != 0, completed.stdout + completed.stderr + record = json.loads((out_dir / "fixture-result.json").read_text(encoding="utf-8-sig")) + assert record["exit_code"] == exit_code + assert record["timed_out"] is timed_out + assert record["timeout_seconds"] == 900 + assert record["stdout"] == "fixture captured stdout" + assert record["stderr"] == "fixture captured stderr" + assert record["junit_validation_error"] + assert "junit" not in record + for stream in ("stdout", "stderr"): + log = (out_dir / f"fixture-{stream}.log").read_text(encoding="utf-8-sig") + assert log.strip() == f"fixture captured {stream}" + assert f"[fixture {stream}] fixture captured {stream}" in completed.stdout + assert f"[fixture] exit {exit_code} timed_out={timed_out}" in completed.stdout diff --git a/tests/unit/test_windows_signing_contract.py b/tests/unit/test_windows_signing_contract.py index a3e717204e..b233d126e5 100644 --- a/tests/unit/test_windows_signing_contract.py +++ b/tests/unit/test_windows_signing_contract.py @@ -27,6 +27,8 @@ SIGNING_STEP = "Sign Windows binary (Authenticode)" BUILD_STEP = "Build binary (Windows)" +SMOKE_STEP = "Test native binary startup and core contracts" +PACKAGE_STEP = "Package exact candidate archive" UPLOAD_STEP = "Upload binary as workflow artifact" @@ -60,12 +62,14 @@ def _assert_signing_step(workflow: WorkflowNode) -> None: assert "WINDOWS_CERT_PFX" in env, "signing step env must include WINDOWS_CERT_PFX" assert "WINDOWS_CERT_PASSWORD" in env, "signing step env must include WINDOWS_CERT_PASSWORD" - # Ordering: signing must occur after build and before upload. build_idx = workflow_step_index(job, BUILD_STEP) sign_idx = workflow_step_index(job, SIGNING_STEP) + smoke_idx = workflow_step_index(job, SMOKE_STEP) + package_idx = workflow_step_index(job, PACKAGE_STEP) upload_idx = workflow_step_index(job, UPLOAD_STEP) - assert build_idx < sign_idx < upload_idx, ( - f"signing step order wrong: build={build_idx} sign={sign_idx} upload={upload_idx}" + assert build_idx < sign_idx < smoke_idx < package_idx < upload_idx, ( + f"signing step order wrong: build={build_idx} sign={sign_idx} " + f"smoke={smoke_idx} package={package_idx} upload={upload_idx}" ) @@ -104,23 +108,25 @@ def test_signing_step_references_script() -> None: _assert_signing_step(workflow) -def test_signing_step_order() -> None: - """Moving the signing step after upload must fail the contract.""" +@pytest.mark.parametrize( + "after_step", [SMOKE_STEP, PACKAGE_STEP, UPLOAD_STEP], ids=["smoke", "package", "upload"] +) +def test_signing_step_order(after_step: str) -> None: + """Only bytes signed before smoke and packaging can be qualified.""" workflow = deepcopy(_workflow()) job = workflow_job(workflow, "build") steps: list = job["steps"] - # Find and remove the signing step, then insert it after the upload step. sign_step = next( (s for s in steps if isinstance(s, dict) and s.get("name") == SIGNING_STEP), None, ) assert sign_step is not None, "signing step not found for order mutation" steps.remove(sign_step) - upload_idx = next( - i for i, s in enumerate(steps) if isinstance(s, dict) and s.get("name") == UPLOAD_STEP + after_idx = next( + i for i, s in enumerate(steps) if isinstance(s, dict) and s.get("name") == after_step ) - steps.insert(upload_idx + 1, sign_step) + steps.insert(after_idx + 1, sign_step) with pytest.raises(AssertionError, match="order"): _assert_signing_step(workflow) From 2846ed9f166e8232a4b1e0c08d96a82439c4fe71 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 18:04:19 +0200 Subject: [PATCH 19/23] fix(release): preserve prereleases and remove experimental campaigns Retain exact-archive qualification and promotion, independent native lanes, production timing snapshots, and the installer JUnit guard. Remove opt-in measurement workflows and their exclusive machinery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/performance-cohort/action.yml | 74 - .github/workflows/ci-release-performance.yml | 194 --- .github/workflows/ci-release-rehearsal.yml | 171 -- .github/workflows/ci-release-wallclock.yml | 280 ---- .github/workflows/ci-source-performance.yml | 150 -- .../workflows/ci-windows-installer-proof.yml | 92 -- .github/workflows/ci.yml | 6 +- .github/workflows/release-integration.yml | 56 +- .github/workflows/release-platform.yml | 97 +- .github/workflows/release-unit.yml | 170 +- .../docs/contributing/integration-testing.md | 141 +- scripts/compare_test_runs.py | 347 ---- scripts/performance-cohort.cjs | 35 - scripts/performance-cohort.test.cjs | 72 - scripts/performance_cohort.py | 150 -- scripts/pytest_performance_evidence.py | 73 - scripts/release-candidate.cjs | 2 +- scripts/release-candidate.test.cjs | 26 + scripts/release_rehearsal.py | 479 ------ scripts/release_wallclock.py | 1428 ----------------- scripts/test-integration.sh | 3 - scripts/windows/run-installer-proof.ps1 | 549 ------- scripts/windows/validate-installer-result.ps1 | 25 + tests/unit/scripts/test_compare_test_runs.py | 515 ------ tests/unit/scripts/test_performance_cohort.py | 179 --- tests/unit/scripts/test_release_rehearsal.py | 331 ---- tests/unit/scripts/test_release_wallclock.py | 1353 ---------------- tests/unit/test_ado_live_workflow_contract.py | 11 +- tests/unit/test_platform_contract_workflow.py | 309 +--- tests/unit/test_pytest_timing_workflows.py | 25 +- .../unit/test_release_performance_workflow.py | 185 --- tests/unit/test_release_rehearsal_workflow.py | 181 --- tests/unit/test_release_wallclock_workflow.py | 295 ---- .../unit/test_windows_compat_gate_workflow.py | 141 -- .../test_windows_installer_proof_workflow.py | 386 ----- tests/unit/test_windows_installer_result.py | 56 + tests/unit/test_windows_unit_diagnostics.py | 24 +- tests/workflow_contracts.py | 12 - 38 files changed, 180 insertions(+), 8443 deletions(-) delete mode 100644 .github/actions/performance-cohort/action.yml delete mode 100644 .github/workflows/ci-release-performance.yml delete mode 100644 .github/workflows/ci-release-rehearsal.yml delete mode 100644 .github/workflows/ci-release-wallclock.yml delete mode 100644 .github/workflows/ci-source-performance.yml delete mode 100644 .github/workflows/ci-windows-installer-proof.yml delete mode 100644 scripts/compare_test_runs.py delete mode 100644 scripts/performance-cohort.cjs delete mode 100644 scripts/performance-cohort.test.cjs delete mode 100644 scripts/performance_cohort.py delete mode 100644 scripts/pytest_performance_evidence.py delete mode 100644 scripts/release_rehearsal.py delete mode 100644 scripts/release_wallclock.py delete mode 100644 scripts/windows/run-installer-proof.ps1 create mode 100644 scripts/windows/validate-installer-result.ps1 delete mode 100644 tests/unit/scripts/test_compare_test_runs.py delete mode 100644 tests/unit/scripts/test_performance_cohort.py delete mode 100644 tests/unit/scripts/test_release_rehearsal.py delete mode 100644 tests/unit/scripts/test_release_wallclock.py delete mode 100644 tests/unit/test_release_performance_workflow.py delete mode 100644 tests/unit/test_release_rehearsal_workflow.py delete mode 100644 tests/unit/test_release_wallclock_workflow.py delete mode 100644 tests/unit/test_windows_installer_proof_workflow.py create mode 100644 tests/unit/test_windows_installer_result.py diff --git a/.github/actions/performance-cohort/action.yml b/.github/actions/performance-cohort/action.yml deleted file mode 100644 index 34d4bbbb30..0000000000 --- a/.github/actions/performance-cohort/action.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Matched performance cohort -description: Require identical identities from all three actual test runners before pytest. -inputs: - cohort: - description: Suite, run attempt and binary identifying this paired cohort. - required: true - member: - description: baseline-1, proposed-1 or proposed-2. - required: true - source-sha: - description: Exact source commit being executed. - required: true - selection: - description: Complete marker and root selection shared by all members. - required: true - candidate-metadata: - description: Native archive metadata for integration; empty for source unit tests. - default: '' -runs: - using: composite - steps: - - name: Record this actual test runner - shell: bash - env: - COHORT: ${{ inputs.cohort }} - MEMBER: ${{ inputs.member }} - SOURCE_SHA: ${{ inputs.source-sha }} - SELECTION: ${{ inputs.selection }} - CANDIDATE_METADATA: ${{ inputs.candidate-metadata }} - run: | - args=() - if [ -n "$CANDIDATE_METADATA" ]; then - args+=(--candidate-metadata "$CANDIDATE_METADATA") - fi - python -m scripts.performance_cohort record \ - --cohort "$COHORT" --member "$MEMBER" \ - --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" \ - --source-sha "$SOURCE_SHA" --selection "$SELECTION" \ - --output ".performance-cohort-out/$MEMBER.json" "${args[@]}" - - name: Publish runner identity - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: performance-cohort-${{ inputs.cohort }}-${{ inputs.member }} - path: .performance-cohort-out/${{ inputs.member }}.json - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - - name: Wait for all actual paired runners - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - COHORT: ${{ inputs.cohort }} - with: - script: | - const { waitForCohort } = require('./scripts/performance-cohort.cjs'); - await waitForCohort({ - github, context, cohort: process.env.COHORT, - attempt: Number(process.env.GITHUB_RUN_ATTEMPT), - }); - - name: Download complete cohort - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-cohort-${{ inputs.cohort }}-* - path: .performance-cohort-inputs - merge-multiple: true - - name: Reject different images, dependencies or candidates before execution - shell: bash - env: - COHORT: ${{ inputs.cohort }} - SOURCE_SHA: ${{ inputs.source-sha }} - run: | - python -m scripts.performance_cohort verify \ - --root .performance-cohort-inputs --cohort "$COHORT" \ - --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" \ - --source-sha "$SOURCE_SHA" diff --git a/.github/workflows/ci-release-performance.yml b/.github/workflows/ci-release-performance.yml deleted file mode 100644 index 96b5d5f20f..0000000000 --- a/.github/workflows/ci-release-performance.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: CI Release Performance Probe - -on: - pull_request: - types: [opened, reopened, synchronize, labeled] - -permissions: - contents: read - -concurrency: - group: ci-release-performance-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - candidate: - name: Build ARM performance candidate - if: contains(github.event.pull_request.labels.*.name, 'ci-performance') - runs-on: macos-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ github.sha }} - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Install locked build dependencies - run: uv sync --frozen --extra dev --extra build - - name: Build current ARM candidate - env: - UV_FROZEN: 'true' - UV_NO_SYNC: 'true' - run: uv run --frozen --no-sync bash scripts/build-binary.sh - - name: Exercise actual candidate core - env: - APM_E2E_TESTS: '1' - APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-darwin-arm64/apm - run: >- - uv run --frozen --no-sync python -m pytest - tests/integration/test_core_smoke.py -q - --junitxml=test-results/core.xml - - name: Package one immutable candidate for every variant - run: >- - python scripts/package_release.py pack - --binary-name apm-darwin-arm64 --sha "$GITHUB_SHA" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: benchmark-arm-candidate-${{ github.run_attempt }} - # Sibling paths preserve release-assets/ instead of flattening it. - path: | - release-assets/ - scripts/package_release.py - scripts/release-platforms.json - compression-level: 0 - if-no-files-found: error - retention-days: 30 - - name: Freeze identical cold scheduling input - run: printf '{}\n' > .test_durations - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: benchmark-arm-cold-timings-${{ github.run_attempt }} - path: .test_durations - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - - name: Upload candidate startup outcome - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: benchmark-arm-core-${{ github.run_attempt }} - path: test-results/ - if-no-files-found: error - retention-days: 30 - - integration: - name: ARM Integration (${{ matrix.variant }}, shard ${{ matrix.shard }}) - needs: [candidate] - if: contains(github.event.pull_request.labels.*.name, 'ci-performance') - strategy: - fail-fast: false - matrix: - include: - - variant: baseline - shard: 1 - count: 1 - workers: 4 - - variant: proposed - shard: 1 - count: 2 - workers: 3 - - variant: proposed - shard: 2 - count: 2 - workers: 3 - uses: ./.github/workflows/release-integration.yml - permissions: - contents: read - actions: read - with: - runner: macos-latest - platform: darwin - binary-name: apm-darwin-arm64 - candidate-artifact-name: benchmark-arm-candidate-${{ github.run_attempt }} - candidate-sha: ${{ github.sha }} - timing-snapshot-artifact: benchmark-arm-cold-timings-${{ github.run_attempt }} - integration-markers: not live - shard-count: ${{ matrix.count }} - shard-index: ${{ matrix.shard }} - xdist-workers: ${{ matrix.workers }} - splitting-algorithm: least_duration - timeout-minutes: 65 - runtime-prerequisites: none - cache-apm-fetches: false - artifact-prefix: arm-probe-${{ matrix.variant }}-${{ github.run_attempt }} - evidence-variant: ${{ matrix.variant }} - evidence-selection: tests/integration/ -m not live - performance-cohort: integration-${{ github.run_attempt }}-apm-darwin-arm64 - - compare: - name: Compare ARM Integration Performance - needs: [candidate, integration] - if: >- - always() && !cancelled() && - contains(github.event.pull_request.labels.*.name, 'ci-performance') - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Require all benchmark executions succeeded - env: - BUILD_RESULT: ${{ needs.candidate.result }} - INTEGRATION_RESULT: ${{ needs.integration.result }} - run: | - test "$BUILD_RESULT" = success - test "$INTEGRATION_RESULT" = success - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ github.sha }} - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - name: Download baseline evidence - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: performance-evidence-arm-probe-baseline-${{ github.run_attempt }}-baseline-shard-1 - path: performance-inputs/baseline - - name: Download proposed shard one evidence - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: performance-evidence-arm-probe-proposed-${{ github.run_attempt }}-proposed-shard-1 - path: performance-inputs/proposed-1 - - name: Download proposed shard two evidence - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: performance-evidence-arm-probe-proposed-${{ github.run_attempt }}-proposed-shard-2 - path: performance-inputs/proposed-2 - - name: Require identical outcomes and a measured execution gain - run: >- - python -m scripts.compare_test_runs compare - --baseline performance-inputs/baseline/baseline-shard-1.json - --proposed performance-inputs/proposed-1/proposed-shard-1.json - performance-inputs/proposed-2/proposed-shard-2.json - --expected-sha "$GITHUB_SHA" --minimum-speedup 0.05 - --output performance-comparison/arm-cold.json - - name: Upload comparison - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: arm-performance-comparison-${{ github.run_attempt }} - path: performance-comparison/ - if-no-files-found: ignore - retention-days: 30 - - python-distributions: - name: Read-only Python distribution build - if: contains(github.event.pull_request.labels.*.name, 'ci-performance') - uses: ./.github/workflows/pypi-distributions.yml - permissions: - contents: read - with: - artifact_name: benchmark-python-distributions-${{ github.run_attempt }} - upload_artifact: true - - docs: - name: Read-only documentation build - if: contains(github.event.pull_request.labels.*.name, 'ci-performance') - uses: ./.github/workflows/docs-build.yml - permissions: - contents: read - with: - pages_artifact_name: benchmark-docs-${{ github.run_attempt }} - upload_pages_artifact: true diff --git a/.github/workflows/ci-release-rehearsal.yml b/.github/workflows/ci-release-rehearsal.yml deleted file mode 100644 index b52f33cdda..0000000000 --- a/.github/workflows/ci-release-rehearsal.yml +++ /dev/null @@ -1,171 +0,0 @@ -name: Read-only Release Sharding Rehearsal - -on: - pull_request: - types: [opened, reopened, synchronize, labeled] - -permissions: - contents: read - actions: read - -concurrency: - group: ci-release-rehearsal-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - plan: - name: Plan release rehearsal - if: contains(github.event.pull_request.labels.*.name, 'ci-release-rehearsal') - runs-on: ubuntu-24.04 - timeout-minutes: 5 - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - id: matrix - name: Derive all five native controls and the three paired surfaces - run: python -m scripts.release_rehearsal matrix >> "$GITHUB_OUTPUT" - - source-checks: - name: Candidate Source Checks - needs: [plan] - uses: ./.github/workflows/ci.yml - permissions: - contents: read - - platforms: - name: ${{ matrix.binary_name }} - needs: [plan] - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - uses: ./.github/workflows/release-platform.yml - permissions: - contents: read - actions: read - with: - runner: ${{ matrix.runner }} - platform: ${{ matrix.platform }} - binary-name: ${{ matrix.binary_name }} - integration-markers: ${{ matrix.integration_markers }} - integration-shard-count: ${{ matrix.integration_shard_count }} - integration-xdist-workers: ${{ matrix.integration_xdist_workers }} - integration-splitting-algorithm: ${{ matrix.integration_splitting_algorithm }} - full-validation: true - unit-performance-probe: ${{ matrix.unit_performance_probe }} - integration-performance-probe: ${{ matrix.integration_performance_probe }} - performance-workers: ${{ matrix.performance_workers }} - - docs: - name: Read-only documentation build - needs: [plan] - uses: ./.github/workflows/docs-build.yml - permissions: - contents: read - with: - pages_artifact_name: rehearsal-docs-${{ github.run_attempt }} - upload_pages_artifact: true - - wheels: - name: Read-only Python distribution build - needs: [plan] - uses: ./.github/workflows/pypi-distributions.yml - permissions: - contents: read - with: - artifact_name: rehearsal-wheels-${{ github.run_attempt }} - upload_artifact: true - - compare: - name: Compare whole rehearsal critical paths - needs: [plan, source-checks, platforms, docs, wheels] - if: >- - always() && !cancelled() && needs.plan.result == 'success' && - contains(github.event.pull_request.labels.*.name, 'ci-release-rehearsal') - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: '24' - - name: Record current-attempt job timestamps and require production authorities - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - RESULTS: ${{ toJSON(needs) }} - with: - script: | - const fs = require('node:fs'); - const { selectRequiredJobs } = require('./scripts/release-candidate.cjs'); - const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); - const params = {...context.repo, run_id: context.runId, attempt_number: attempt}; - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { - ...params, per_page: 100, - }); - const {data: run} = await github.rest.actions.getWorkflowRunAttempt(params); - fs.mkdirSync('rehearsal-results', {recursive: true}); - fs.writeFileSync('rehearsal-results/jobs.json', JSON.stringify({ - schema_version: 1, run_id: String(context.runId), run_attempt: attempt, - source_sha: context.sha, run_started_at: run.run_started_at, - created_at: run.created_at, jobs, - }, null, 2)); - const results = JSON.parse(process.env.RESULTS); - for (const key of ['source-checks', 'platforms', 'docs', 'wheels']) { - if (results[key]?.result !== 'success') throw new Error(`${key} did not succeed`); - } - const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); - selectRequiredJobs(jobs, catalog); - - name: Download current integration proofs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-evidence-${{ github.run_attempt }}-apm-* - path: performance-inputs - - name: Download current Intel unit proofs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-evidence-unit-${{ github.run_attempt }}-apm-* - path: performance-inputs - - name: Download actual ARM runner cohort - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-cohort-integration-${{ github.run_attempt }}-apm-darwin-arm64-* - path: cohort-inputs/arm-integration - merge-multiple: true - - name: Download actual Linux runner cohort - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-cohort-integration-${{ github.run_attempt }}-apm-linux-x86_64-* - path: cohort-inputs/linux-integration - merge-multiple: true - - name: Download actual Intel runner cohort - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: performance-cohort-unit-${{ github.run_attempt }}-apm-darwin-x86_64-* - path: cohort-inputs/intel-units - merge-multiple: true - - name: Require original outcomes and measure the entire critical path - run: >- - python -m scripts.release_rehearsal compare - --jobs rehearsal-results/jobs.json - --reports performance-inputs --cohorts cohort-inputs - --expected-sha "$GITHUB_SHA" - --minimum-lane-gain 0.15 --minimum-overall-gain 0.10 - --output rehearsal-results/comparison.json - - name: Preserve raw timestamps and the comparison, including missed targets - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-rehearsal-comparison-${{ github.run_attempt }} - path: rehearsal-results/ - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/ci-release-wallclock.yml b/.github/workflows/ci-release-wallclock.yml deleted file mode 100644 index ec26f0b70c..0000000000 --- a/.github/workflows/ci-release-wallclock.yml +++ /dev/null @@ -1,280 +0,0 @@ -name: PR Release Wall-Clock Proof - -on: - pull_request: - types: [labeled] - push: - branches: [danielmeppiel-release-wallclock-proof] - -permissions: - contents: read - actions: read - -concurrency: - group: release-wallclock-proposed-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: false - -jobs: - plan: - name: Plan proposed wall-clock run - if: github.event_name == 'push' || github.event.label.name == 'ci-release-wallclock' - runs-on: ubuntu-24.04 - timeout-minutes: 5 - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - id: plan - name: Select all five production catalog platforms - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - if (Number(process.env.GITHUB_RUN_ATTEMPT) !== 1) { - throw new Error('Use a fresh labelled run, not a partial or mixed-attempt rerun'); - } - const fs = require('node:fs'); - const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); - core.setOutput('matrix', JSON.stringify({include: catalog})); - - source-checks: - name: Candidate Source Checks - needs: [plan] - uses: ./.github/workflows/ci.yml - with: - suppress-pr-only-checks: true - permissions: - contents: read - - platforms: - name: ${{ matrix.binary_name }} - needs: [plan] - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - uses: ./.github/workflows/release-platform.yml - permissions: - contents: read - actions: read - with: - runner: ${{ matrix.runner }} - platform: ${{ matrix.platform }} - binary-name: ${{ matrix.binary_name }} - integration-markers: ${{ matrix.integration_markers }} - integration-shard-count: ${{ matrix.integration_shard_count }} - integration-xdist-workers: ${{ matrix.integration_xdist_workers }} - integration-splitting-algorithm: ${{ matrix.integration_splitting_algorithm }} - full-validation: true - wallclock-evidence: true - - docs: - name: Read-only documentation build - needs: [plan] - uses: ./.github/workflows/docs-build.yml - permissions: - contents: read - with: - pages_artifact_name: wallclock-proposed-docs-${{ github.run_attempt }} - upload_pages_artifact: true - - wheels: - name: Read-only Python distribution build - needs: [plan] - uses: ./.github/workflows/pypi-distributions.yml - permissions: - contents: read - with: - artifact_name: wallclock-proposed-python-${{ github.run_attempt }} - upload_artifact: true - - candidate-ready: - name: Record non-promotable native evidence - needs: [plan, platforms, source-checks] - runs-on: ubuntu-24.04 - timeout-minutes: 20 - outputs: - native_ids: ${{ steps.record.outputs.native_ids }} - evidence_id: ${{ steps.upload.outputs.artifact-id }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: candidate-${{ github.run_attempt }}-apm-* - path: candidate-artifacts - - name: Capture native bytes and successful authorities without qualification - id: record - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const fs = require('node:fs'); - const { - collectArchiveInventory, candidateVersion, configHashes, selectRequiredJobs, - } = require('./scripts/release-candidate.cjs'); - const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); - const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); - const platforms = collectArchiveInventory({ - artifactRoot: 'candidate-artifacts', catalog, sha: context.sha, - }); - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { - ...context.repo, run_id: context.runId, attempt_number: attempt, per_page: 100, - }); - const required = selectRequiredJobs(jobs, catalog); - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - ...context.repo, run_id: context.runId, per_page: 100, - }); - const selected = catalog.map(row => { - const name = `candidate-${attempt}-${row.binary_name}`; - const matches = artifacts.filter(artifact => artifact.name === name && !artifact.expired); - if (matches.length !== 1) throw new Error(`Missing or ambiguous native artifact: ${name}`); - return matches[0]; - }); - fs.writeFileSync('wallclock-native-index.json', JSON.stringify({ - kind: 'release-wallclock-native-index', promotable: false, - head_sha: context.sha, run_id: context.runId, run_attempt: attempt, - version: candidateVersion(platforms, context), - config_hashes: configHashes(process.cwd()), platforms, required, artifacts: selected, - }, null, 2)); - core.setOutput('native_ids', selected.map(artifact => artifact.id).join(',')); - - id: upload - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wallclock-proposed-native-index-${{ github.run_attempt }} - path: wallclock-native-index.json - if-no-files-found: error - retention-days: 30 - - verify: - name: Verify wall-clock artifacts - needs: [plan, source-checks, platforms, candidate-ready, docs, wheels] - if: always() && !cancelled() && needs.plan.result == 'success' - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - name: Require every production authority and record artifact identities - id: authorities - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - RESULTS: ${{ toJSON(needs) }} - NATIVE_IDS: ${{ needs.candidate-ready.outputs.native_ids }} - with: - script: | - const fs = require('node:fs'); - const {selectRequiredJobs} = require('./scripts/release-candidate.cjs'); - const results = JSON.parse(process.env.RESULTS); - for (const name of ['plan', 'source-checks', 'platforms', 'candidate-ready', 'docs', 'wheels']) { - if (results[name]?.result !== 'success') throw new Error(`${name} did not succeed`); - } - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { - ...context.repo, run_id: context.runId, - attempt_number: Number(process.env.GITHUB_RUN_ATTEMPT), per_page: 100, - }); - const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); - const required = selectRequiredJobs(jobs, catalog); - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - ...context.repo, run_id: context.runId, per_page: 100, - }); - const attempt = process.env.GITHUB_RUN_ATTEMPT; - const expected = [ - ...catalog.map(row => `candidate-${attempt}-${row.binary_name}`), - `wallclock-proposed-docs-${attempt}`, `wallclock-proposed-python-${attempt}`, - ...catalog.flatMap(row => [ - `test-results-${attempt}-${row.binary_name}-core`, - `test-results-${attempt}-${row.binary_name}-unit`, - ...Array.from({length: row.integration_shard_count}, (_, index) => - `test-results-${attempt}-${row.binary_name}-integration-shard-${index + 1}`), - ]), - ]; - const selected = expected.map(name => { - const matches = artifacts.filter(artifact => artifact.name === name && !artifact.expired); - if (matches.length !== 1) throw new Error(`Missing or ambiguous artifact: ${name}`); - return matches[0]; - }); - const nativeIds = selected.slice(0, catalog.length).map(artifact => artifact.id).join(','); - if (nativeIds !== process.env.NATIVE_IDS) { - throw new Error('Native artifact IDs changed after evidence capture'); - } - core.setOutput('docs_id', selected[catalog.length].id); - core.setOutput('python_id', selected[catalog.length + 1].id); - core.setOutput('observation_ids', - selected.slice(catalog.length + 2).map(artifact => artifact.id).join(',')); - fs.mkdirSync('wallclock-results', {recursive: true}); - fs.writeFileSync('wallclock-results/authorities.json', JSON.stringify({ - promotable: false, required, artifacts: selected, - }, null, 2)); - - name: Download immutable non-promotable native evidence - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ needs.candidate-ready.outputs.evidence_id }} - path: wallclock-native-evidence - - name: Download exact current-run native artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ needs.candidate-ready.outputs.native_ids }} - path: wallclock-inputs/native - - name: Download documentation artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ steps.authorities.outputs.docs_id }} - path: wallclock-inputs/docs - - name: Download Python distributions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ steps.authorities.outputs.python_id }} - path: wallclock-inputs/python - - name: Recheck indexed native bytes and prepare non-promotable assets - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const fs = require('node:fs'); - const {configHashes, verifyLocalArchiveBytes} = require('./scripts/release-candidate.cjs'); - const evidence = JSON.parse(fs.readFileSync( - 'wallclock-native-evidence/wallclock-native-index.json', 'utf8')); - if (evidence.kind !== 'release-wallclock-native-index' || evidence.promotable !== false || - evidence.run_id !== context.runId || evidence.head_sha !== context.sha || - evidence.run_attempt !== Number(process.env.GITHUB_RUN_ATTEMPT) || - JSON.stringify(evidence.config_hashes) !== JSON.stringify(configHashes(process.cwd()))) { - throw new Error('Non-promotable native index does not match this execution'); - } - const catalog = JSON.parse(fs.readFileSync('scripts/release-platforms.json', 'utf8')); - verifyLocalArchiveBytes({ - evidence, catalog, artifactRoot: 'wallclock-inputs/native', - outputRoot: 'wallclock-verified-native', - }); - - name: Download exact native runner observations - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ steps.authorities.outputs.observation_ids }} - path: wallclock-inputs/native/observations - - name: Verify built bytes without granting release qualification - run: >- - python -m scripts.release_wallclock verify - --side proposed --source-root . --source-sha "$GITHUB_SHA" - --native-root wallclock-inputs/native - --docs-root wallclock-inputs/docs - --python-root wallclock-inputs/python - --output wallclock-results/proof.json - - name: Upload verified native bytes without publishing - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wallclock-proposed-verified-native-${{ github.run_attempt }} - path: wallclock-verified-native/ - compression-level: 0 - if-no-files-found: error - retention-days: 7 - - name: Preserve non-promotable wall-clock evidence - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: wallclock-proposed-proof-${{ github.run_attempt }} - path: wallclock-results/ - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/ci-source-performance.yml b/.github/workflows/ci-source-performance.yml deleted file mode 100644 index c0578dede2..0000000000 --- a/.github/workflows/ci-source-performance.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: CI Source Performance Probe - -on: - pull_request: - types: [opened, reopened, synchronize, labeled] - -permissions: - contents: read - -concurrency: - group: ci-source-performance-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - windows-compat-performance: - name: Windows Compat Performance (${{ matrix.variant }}) - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance') }} - runs-on: windows-latest - timeout-minutes: 20 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - include: - - variant: baseline - pytest_extra: '' - - variant: proposed - pytest_extra: '-n 2 --dist loadgroup' - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - - name: Install dependencies - run: uv sync --frozen --extra dev - - - name: Record Windows compatibility selection - shell: pwsh - run: | - New-Item -ItemType Directory -Force "test-results/${{ matrix.variant }}" | Out-Null - "variant=${{ matrix.variant }}" | Set-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" - "pytest_extra=${{ matrix.pytest_extra }}" | Add-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" - "selection=windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py" | Add-Content -Encoding utf8 "test-results/${{ matrix.variant }}/selection.txt" - - - name: Run Windows compatibility selection - shell: pwsh - run: >- - uv run --frozen --no-sync --extra dev python -m pytest -p no:cacheprovider - -p scripts.pytest_performance_evidence - -v - -m windows_compat - tests/unit - tests/integration/test_lifecycle_workspace_lock.py - ${{ matrix.pytest_extra }} - --durations=50 - --store-durations - --junitxml=test-results/${{ matrix.variant }}/windows.xml - - - name: Capture comparable proof - if: always() - shell: pwsh - run: >- - python -m scripts.compare_test_runs capture - --junit test-results/${{ matrix.variant }}/windows.xml - --output test-results/${{ matrix.variant }}/proof.json - --source-sha "$env:GITHUB_SHA" - --selection 'windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py' - --variant ${{ matrix.variant }} - --shard 1 - --shard-count 1 - - - name: Preserve timing file with mode - if: always() - shell: pwsh - run: | - if (Test-Path ".test_durations") { - Copy-Item ".test_durations" "test-results/${{ matrix.variant }}/.test_durations" - } - - - name: Upload Windows compatibility performance evidence - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: windows-compat-performance-${{ matrix.variant }}-${{ github.run_attempt }} - path: test-results/${{ matrix.variant }}/ - include-hidden-files: true - retention-days: 14 - if-no-files-found: error - - compare-windows-compat-performance: - name: Compare Windows Compat Performance - if: ${{ always() && contains(github.event.pull_request.labels.*.name, 'ci-performance') }} - needs: [windows-compat-performance] - runs-on: ubuntu-24.04 - timeout-minutes: 5 - permissions: - contents: read - steps: - - name: Require benchmark variants succeeded - run: | - if [ "${{ needs.windows-compat-performance.result }}" != "success" ]; then - echo "::error::Windows compatibility benchmark variants did not both succeed: ${{ needs.windows-compat-performance.result }}" - exit 1 - fi - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - - name: Download baseline proof - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: windows-compat-performance-baseline-${{ github.run_attempt }} - path: performance-inputs/baseline - - - name: Download proposed proof - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: windows-compat-performance-proposed-${{ github.run_attempt }} - path: performance-inputs/proposed - - - name: Compare equivalent Windows compatibility runs - run: >- - python -m scripts.compare_test_runs compare - --baseline performance-inputs/baseline/proof.json - --proposed performance-inputs/proposed/proof.json - --expected-sha "$GITHUB_SHA" - --minimum-speedup 0.05 - --output performance-comparison/comparison.json - - - name: Upload Windows compatibility comparison - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: windows-compat-performance-comparison-${{ github.run_attempt }} - path: performance-comparison/ - include-hidden-files: true - retention-days: 14 - if-no-files-found: ignore diff --git a/.github/workflows/ci-windows-installer-proof.yml b/.github/workflows/ci-windows-installer-proof.yml deleted file mode 100644 index 3c198944b2..0000000000 --- a/.github/workflows/ci-windows-installer-proof.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: PR Windows Installer Proof - -on: - pull_request: - types: [labeled] - -permissions: - contents: read - -concurrency: - group: windows-installer-proof-${{ github.event.pull_request.number }} - cancel-in-progress: false - -env: - PYTHON_VERSION: '3.12' - BASELINE_VERSION: v0.28.0 - -jobs: - windows-installer-proof: - name: Windows installer proof - if: github.event.label.name == 'ci-windows-installer-proof' - runs-on: windows-latest - timeout-minutes: 45 - steps: - - name: Require fresh proof attempt - shell: pwsh - run: | - if ($env:GITHUB_RUN_ATTEMPT -ne '1') { - throw "Use a fresh labelled run, not a partial or mixed-attempt rerun" - } - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - name: Install dependencies - run: uv sync --frozen --extra dev --extra build - - name: Build unsigned Windows candidate - shell: pwsh - env: - WINDOWS_CERT_PFX: '' - run: uv run --frozen --no-sync pwsh scripts/windows/build-binary.ps1 - - name: Package exact Windows candidate archive - shell: pwsh - env: - CANDIDATE_SHA: ${{ github.sha }} - BINARY_NAME: apm-windows-x86_64 - run: >- - python scripts/package_release.py pack - --binary-name "$env:BINARY_NAME" --sha "$env:CANDIDATE_SHA" - - name: Upload standalone proof candidate archive - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: windows-installer-proof-candidate-${{ github.run_attempt }} - path: | - release-assets/apm-windows-x86_64.zip - release-assets/apm-windows-x86_64.zip.sha256 - release-assets/apm-windows-x86_64.json - if-no-files-found: error - compression-level: 0 - retention-days: 30 - - name: Download historical upgrade source once - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - run: | - gh release download $env:BASELINE_VERSION --repo microsoft/apm ` - --pattern apm-windows-x86_64.zip --pattern apm-windows-x86_64.zip.sha256 ` - --dir installer-baseline - if ($LASTEXITCODE -ne 0) { throw "Could not download the historical upgrade source" } - - name: Diagnose and prove Windows installer root behavior - shell: pwsh - run: >- - scripts/windows/run-installer-proof.ps1 - -CandidateArchive release-assets/apm-windows-x86_64.zip - -CandidateMetadata release-assets/apm-windows-x86_64.json - -BaselineArchive installer-baseline/apm-windows-x86_64.zip - -BaselineChecksum installer-baseline/apm-windows-x86_64.zip.sha256 - -BaselineVersion "$env:BASELINE_VERSION" - -OutputDir windows-installer-proof - - name: Upload Windows installer proof diagnostics - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: windows-installer-proof-${{ github.run_attempt }} - path: windows-installer-proof/ - if-no-files-found: error - retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 495c85621d..9a1f858174 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,6 @@ env: on: workflow_call: - inputs: - suppress-pr-only-checks: - type: boolean - default: false pull_request: branches: [ main ] # Tier 1 also runs in merge queue context so the same unit + build checks @@ -398,7 +394,7 @@ jobs: # and in build-release.yml on release. pr-binary-smoke: name: PR Binary Smoke (Linux) - if: github.event_name == 'pull_request' && !inputs.suppress-pr-only-checks + if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 permissions: contents: read diff --git a/.github/workflows/release-integration.yml b/.github/workflows/release-integration.yml index 2bcdd32bf3..8634a39e4c 100644 --- a/.github/workflows/release-integration.yml +++ b/.github/workflows/release-integration.yml @@ -54,18 +54,6 @@ on: coverage-enabled: type: boolean default: false - evidence-variant: - type: string - default: '' - evidence-selection: - type: string - default: '' - performance-cohort: - type: string - default: '' - wallclock-evidence: - type: boolean - default: false secrets: GH_CLI_PAT: required: false @@ -95,13 +83,6 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Record actual integration runner for wall-clock proof - if: inputs.wallclock-evidence - shell: bash - run: >- - python -m scripts.release_wallclock record-job - --side proposed --source-root . --source-sha "$GITHUB_SHA" - --output test-results/integration-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -130,15 +111,6 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ inputs.timing-snapshot-artifact }} - - name: Require matched actual test runners before the performance experiment - if: inputs.performance-cohort != '' - uses: ./.github/actions/performance-cohort - with: - cohort: ${{ inputs.performance-cohort }} - member: ${{ inputs.evidence-variant }}-${{ inputs.shard-index }} - source-sha: ${{ inputs.candidate-sha }} - selection: ${{ inputs.evidence-selection || format('tests/integration/ -m {0}', inputs.integration-markers) }} - candidate-metadata: release-assets/${{ inputs.binary-name }}.json - name: Compute cache week bucket if: inputs.cache-apm-fetches id: cache-bucket @@ -164,7 +136,6 @@ jobs: PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} APM_TEST_RUNTIMES: ${{ inputs.runtime-prerequisites }} PYTEST_EXTRA_ARGS: >- - ${{ (inputs.evidence-variant != '' || inputs.wallclock-evidence) && '-p scripts.pytest_performance_evidence' || '' }} --splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} --splitting-algorithm ${{ inputs.splitting-algorithm }} -n ${{ inputs.xdist-workers }} --dist loadgroup @@ -181,25 +152,8 @@ jobs: GITHUB_API_TOKEN: ${{ github.token }} GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} run: uv run --frozen pwsh scripts/windows/test-integration.ps1 -SkipBuild - - name: Capture integration performance evidence - if: always() && inputs.platform != 'windows' && inputs.evidence-variant != '' - env: - EVIDENCE_SELECTION: ${{ inputs.evidence-selection }} - PYTEST_MARK_EXPR: ${{ inputs.integration-markers }} - run: | - mkdir -p performance-evidence - selection="${EVIDENCE_SELECTION:-tests/integration/ -m $PYTEST_MARK_EXPR}" - python -m scripts.compare_test_runs capture \ - --junit "test-results/integration-shard-${{ inputs.shard-index }}.xml" \ - --output "performance-evidence/${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }}.json" \ - --source-sha "${{ inputs.candidate-sha }}" \ - --selection "$selection" \ - --variant "${{ inputs.evidence-variant }}" \ - --shard "${{ inputs.shard-index }}" \ - --shard-count "${{ inputs.shard-count }}" \ - --candidate-metadata "release-assets/${{ inputs.binary-name }}.json" - name: Upload integration timings and outcomes - if: always() && (inputs.platform != 'windows' || inputs.wallclock-evidence) + if: always() && inputs.platform != 'windows' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-results-${{ inputs.artifact-prefix }}-integration-shard-${{ inputs.shard-index }} @@ -209,14 +163,6 @@ jobs: include-hidden-files: true retention-days: 30 if-no-files-found: error - - name: Upload integration performance evidence - if: always() && inputs.platform != 'windows' && inputs.evidence-variant != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: performance-evidence-${{ inputs.artifact-prefix }}-${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }} - path: performance-evidence/ - retention-days: 30 - if-no-files-found: error - name: Rename coverage data for shard if: always() && inputs.coverage-enabled run: | diff --git a/.github/workflows/release-platform.yml b/.github/workflows/release-platform.yml index 4ee1d6fd93..8397aa1007 100644 --- a/.github/workflows/release-platform.yml +++ b/.github/workflows/release-platform.yml @@ -24,18 +24,6 @@ on: integration-splitting-algorithm: type: string default: duration_based_chunks - unit-performance-probe: - type: boolean - default: false - integration-performance-probe: - type: boolean - default: false - performance-workers: - type: number - default: 4 - wallclock-evidence: - type: boolean - default: false full-validation: type: boolean required: true @@ -66,9 +54,6 @@ jobs: runner: ${{ inputs.runner }} platform: ${{ inputs.platform }} binary-name: ${{ inputs.binary-name }} - performance-probe: ${{ inputs.unit-performance-probe }} - proposed-workers: ${{ inputs.performance-workers }} - wallclock-evidence: ${{ inputs.wallclock-evidence }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} GH_MODELS_PAT: ${{ secrets.GH_MODELS_PAT }} @@ -82,13 +67,6 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Record actual build runner for wall-clock proof - if: inputs.wallclock-evidence - shell: bash - run: >- - python -m scripts.release_wallclock record-job - --side proposed --source-root . --source-sha "$GITHUB_SHA" - --output test-results/build-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true @@ -124,23 +102,11 @@ jobs: python scripts/package_release.py pack --binary-name "$BINARY_NAME" --sha "$CANDIDATE_SHA" - name: Freeze integration scheduling hints - if: inputs.full-validation && inputs.platform != 'windows' && !inputs.integration-performance-probe + if: inputs.full-validation && inputs.platform != 'windows' uses: ./.github/actions/pytest-timing with: suite: integration snapshot: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} - - name: Freeze cold integration scheduling hints for performance probe - if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe - run: printf '{}\n' > .test_durations - - name: Upload cold integration scheduling hints for performance probe - if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} - path: .test_durations - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - name: Upload binary as workflow artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -189,64 +155,11 @@ jobs: xdist-workers: ${{ inputs.integration-xdist-workers }} splitting-algorithm: ${{ inputs.integration-splitting-algorithm }} timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} - artifact-prefix: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || format('{0}-{1}', github.run_attempt, inputs.binary-name) }} + artifact-prefix: ${{ github.run_attempt }}-${{ inputs.binary-name }} runtime-prerequisites: none - evidence-variant: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && 'baseline' || '' }} - evidence-selection: tests/integration/ -m ${{ inputs.integration-markers }} - wallclock-evidence: ${{ inputs.wallclock-evidence }} - performance-cohort: ${{ inputs.integration-performance-probe && inputs.platform != 'windows' && format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }} secrets: GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} - integration-tests-proposed-shard: - name: Integration Tests Proposed Shard ${{ matrix.shard }} - needs: [build] - if: inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe - strategy: - fail-fast: false - matrix: - shard: [1, 2] - uses: ./.github/workflows/release-integration.yml - with: - runner: ${{ inputs.runner }} - platform: ${{ inputs.platform }} - binary-name: ${{ inputs.binary-name }} - candidate-artifact-name: candidate-${{ github.run_attempt }}-${{ inputs.binary-name }} - candidate-sha: ${{ github.sha }} - timing-snapshot-artifact: integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }} - integration-markers: ${{ inputs.integration-markers }} - shard-count: 2 - shard-index: ${{ matrix.shard }} - xdist-workers: ${{ inputs.performance-workers }} - splitting-algorithm: least_duration - timeout-minutes: ${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }} - artifact-prefix: ${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed - runtime-prerequisites: none - evidence-variant: proposed - evidence-selection: tests/integration/ -m ${{ inputs.integration-markers }} - performance-cohort: integration-${{ github.run_attempt }}-${{ inputs.binary-name }} - secrets: - GH_CLI_PAT: ${{ secrets.GH_CLI_PAT }} - - integration-performance-probe: - name: Integration Performance Probe - needs: [integration-tests-shard, integration-tests-proposed-shard] - if: always() && inputs.full-validation && inputs.platform != 'windows' && inputs.integration-performance-probe - runs-on: ubuntu-24.04 - steps: - - name: Require every integration performance member - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - RESULTS: ${{ toJSON(needs) }} - with: - script: | - const results = JSON.parse(process.env.RESULTS); - for (const name of ['integration-tests-shard', 'integration-tests-proposed-shard']) { - if (results[name]?.result !== 'success') { - core.setFailed(`${name} did not succeed`); - } - } - integration-tests: name: Integration Tests needs: [integration-tests-shard] @@ -305,11 +218,11 @@ jobs: timeout-minutes: 20 env: GITHUB_API_TOKEN: ${{ github.token }} - GITHUB_APM_PAT: ${{ !inputs.wallclock-evidence && secrets.GH_CLI_PAT || '' }} + GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} APM_E2E_TESTS: '1' run: | $env:PATH = "$PWD\dist\apm-windows-x86_64;$env:PATH" - .\scripts\windows\test-release-validation.ps1 -PublicApiOnly:('${{ inputs.wallclock-evidence }}' -eq 'true') + .\scripts\windows\test-release-validation.ps1 windows-installer: name: Candidate Windows Installer @@ -364,7 +277,7 @@ jobs: - name: Require executed installer assertions shell: pwsh run: >- - scripts/windows/run-installer-proof.ps1 + scripts/windows/validate-installer-result.ps1 -ValidateInstallerJUnit test-results/installer.xml -InstallerExitCode 0 - name: Upload installer outcomes if: always() diff --git a/.github/workflows/release-unit.yml b/.github/workflows/release-unit.yml index 2cca632500..e31fe4da4c 100644 --- a/.github/workflows/release-unit.yml +++ b/.github/workflows/release-unit.yml @@ -12,15 +12,6 @@ on: binary-name: type: string required: true - performance-probe: - type: boolean - default: false - proposed-workers: - type: number - default: 4 - wallclock-evidence: - type: boolean - default: false secrets: GH_CLI_PAT: required: false @@ -36,7 +27,7 @@ env: jobs: unit-tests: - name: Unit Tests Baseline + name: Unit Tests runs-on: ${{ inputs.runner }} timeout-minutes: 65 steps: @@ -44,55 +35,25 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Record actual unit runner for wall-clock proof - if: inputs.wallclock-evidence - shell: bash - run: >- - python -m scripts.release_wallclock record-job - --side proposed --source-root . --source-sha "$GITHUB_SHA" - --output test-results/unit-environment.json - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: true - name: Install dependencies run: uv sync --frozen --extra dev - name: Restore unit scheduling hints - if: ${{ !inputs.performance-probe }} uses: ./.github/actions/pytest-timing with: suite: unit - - name: Initialize cold unit scheduling hints - if: inputs.performance-probe && inputs.platform != 'windows' - run: printf '{}\n' > .test_durations - - name: Require matched actual test runners before the unit performance experiment - if: inputs.performance-probe && inputs.platform != 'windows' - uses: ./.github/actions/performance-cohort - with: - cohort: unit-${{ github.run_attempt }}-${{ inputs.binary-name }} - member: baseline-1 - source-sha: ${{ github.sha }} - selection: tests/unit tests/test_console.py - name: Run unit tests - if: inputs.platform != 'windows' && !inputs.performance-probe + if: inputs.platform != 'windows' env: GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} run: >- - uv run --frozen - ${{ inputs.wallclock-evidence && 'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }} + uv run --frozen pytest tests/unit tests/test_console.py -n auto --dist worksteal --durations=50 --store-durations --junitxml=test-results/unit.xml - - name: Run unit performance baseline - if: inputs.platform != 'windows' && inputs.performance-probe - env: - GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} - GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} - run: >- - uv run --frozen python -m pytest -p scripts.pytest_performance_evidence - tests/unit tests/test_console.py - -n auto --dist worksteal --durations=50 --store-durations - --junitxml=test-results/unit-baseline-shard-1.xml - name: Run unit tests (Windows diagnostics) if: inputs.platform == 'windows' timeout-minutes: 60 @@ -101,140 +62,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_MODELS_PAT }} GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }} run: >- - uv run --frozen - ${{ inputs.wallclock-evidence && 'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }} + uv run --frozen pytest tests/unit tests/test_console.py -n auto --dist worksteal --durations=50 --store-durations --junitxml=test-results/unit.xml -vv --tb=short --show-capture=no --no-showlocals - - name: Capture unit performance evidence - if: always() && inputs.platform != 'windows' && inputs.performance-probe - run: | - mkdir -p performance-evidence - python -m scripts.compare_test_runs capture \ - --junit test-results/unit-baseline-shard-1.xml \ - --output performance-evidence/baseline-shard-1.json \ - --source-sha "${{ github.sha }}" \ - --selection "tests/unit tests/test_console.py" \ - --variant baseline \ - --shard 1 \ - --shard-count 1 - name: Upload unit timings and outcomes - if: always() && !inputs.performance-probe - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit - path: | - test-results/ - .test_durations - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - - name: Upload unit performance timings and outcomes - if: always() && inputs.platform != 'windows' && inputs.performance-probe - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: test-results-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-baseline-shard-1 - path: | - test-results/ - .test_durations - include-hidden-files: true - if-no-files-found: error - retention-days: 30 - - name: Upload unit performance evidence - if: always() && inputs.platform != 'windows' && inputs.performance-probe - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-baseline-shard-1 - path: performance-evidence/ - if-no-files-found: error - retention-days: 30 - - unit-tests-proposed: - name: Unit Tests Proposed Shard ${{ matrix.shard }} - if: inputs.performance-probe && inputs.platform != 'windows' - runs-on: ${{ inputs.runner }} - timeout-minutes: 65 - strategy: - fail-fast: false - matrix: - shard: [1, 2] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - enable-cache: true - - name: Install dependencies - run: uv sync --frozen --extra dev - - name: Initialize cold unit scheduling hints - run: printf '{}\n' > .test_durations - - name: Require matched actual test runners before the unit performance experiment - uses: ./.github/actions/performance-cohort - with: - cohort: unit-${{ github.run_attempt }}-${{ inputs.binary-name }} - member: proposed-${{ matrix.shard }} - source-sha: ${{ github.sha }} - selection: tests/unit tests/test_console.py - - name: Run unit performance proposed shard - env: - GITHUB_TOKEN: ${{ inputs.platform == 'linux' && secrets.GH_MODELS_PAT || '' }} - GITHUB_APM_PAT: ${{ inputs.platform == 'linux' && secrets.GH_CLI_PAT || '' }} - run: >- - uv run --frozen python -m pytest -p scripts.pytest_performance_evidence - tests/unit tests/test_console.py - --splits 2 --group ${{ matrix.shard }} --splitting-algorithm least_duration - -n ${{ inputs.proposed-workers }} --dist worksteal --durations=50 --store-durations - --junitxml=test-results/unit-proposed-shard-${{ matrix.shard }}.xml - - name: Capture unit performance evidence - if: always() - run: | - mkdir -p performance-evidence - python -m scripts.compare_test_runs capture \ - --junit "test-results/unit-proposed-shard-${{ matrix.shard }}.xml" \ - --output "performance-evidence/proposed-shard-${{ matrix.shard }}.json" \ - --source-sha "${{ github.sha }}" \ - --selection "tests/unit tests/test_console.py" \ - --variant proposed \ - --shard "${{ matrix.shard }}" \ - --shard-count 2 - - name: Upload unit performance timings and outcomes if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-results-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed-shard-${{ matrix.shard }} + name: test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit path: | test-results/ .test_durations include-hidden-files: true if-no-files-found: error retention-days: 30 - - name: Upload unit performance evidence - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed-shard-${{ matrix.shard }} - path: performance-evidence/ - if-no-files-found: error - retention-days: 30 - - unit-performance-probe: - name: Unit Performance Probe - needs: [unit-tests, unit-tests-proposed] - if: always() && inputs.performance-probe && inputs.platform != 'windows' - runs-on: ubuntu-24.04 - steps: - - name: Require every unit performance member - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - RESULTS: ${{ toJSON(needs) }} - with: - script: | - const results = JSON.parse(process.env.RESULTS); - for (const name of ['unit-tests', 'unit-tests-proposed']) { - if (results[name]?.result !== 'success') { - core.setFailed(`${name} did not succeed`); - } - } diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 627c0b7a21..b335aac91e 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -407,9 +407,13 @@ the build independently; all remain required at the final gate. 5. Promote only those exact, verified, unpacked, tested archives. Unix integration uses `release-integration.yml`: one shard, four workers, and -`duration_based_chunks` assignment per native lane. Multi-runner ARM sharding -remains experimental pending a matched performance comparison. Every shard -must succeed. +`duration_based_chunks` assignment per native lane. Every shard must succeed. + +For tag pushes, `rc.plan` in `scripts/release-candidate.cjs` treats only names +matching `/^v[0-9]+\.[0-9]+\.[0-9]+$/` as stable. All other `v*` tag pushes, +including PEP 440 `v1.2.3a1`, `v1.2.3b1`, and `v1.2.3rc1`, are classified as +prereleases: they do not build or publish stable docs or PyPI distributions. +Classification does not guarantee successful qualification. Stable-tag read-only docs and wheel builds start after planning. Publishers require successful GitHub Release creation and their own builds; prebuilding @@ -521,6 +525,11 @@ The Windows isolation root is `tmp_path_factory.mktemp("wi") / "i"`: a short, nonexistent child for the shared helper, while the PowerShell install prefix still includes spaces and `&`. +Production uses the short `RUNNER_TEMP/apm-windows-installer` base directory. +`scripts/windows/validate-installer-result.ps1` requires a JUnit report with +at least one concrete test case, no skips, and no failures or errors. Pytest +exit 0 alone is insufficient. + ### Windows unit hang diagnostics The Windows full-unit step in `.github/workflows/release-unit.yml` runs: @@ -574,129 +583,21 @@ They do not restore a rolling cache or present the shards as timing-balanced: the shared planner job required for that can cost more runner allocation time than it saves in balancing. -Native builds also freeze any available timing history through -`.github/actions/pytest-timing`; all shards consume that one immutable -snapshot. Measured shard histories are retained as artifacts, not independently +Native Unix builds also freeze any available timing history through +`.github/actions/pytest-timing`. All shards for that build consume the same +immutable snapshot. Measured shard histories are retained as artifacts, not independently restored or written back by each shard. Hints never replace test execution or act as an allowlist. Grouped tests still use `--dist loadgroup` so HOME-mutating fixtures stay on one worker within each isolated job. Inspect expensive fixtures by opening the slowest `--durations=50` entries and correlating fixture-heavy node IDs with JUnit XML for the same OS, architecture, -suite, and shard. Historical release run `34104956270` completed in 39m16s; -compare a later complete release before claiming an end-to-end improvement. - -#### Opt-in performance comparisons - -Apply the `ci-performance` PR label to run `ci-release-performance.yml` and -`ci-source-performance.yml`. The native probe builds one current macOS ARM -candidate, then compares one four-worker run against two three-worker shards. -This proposed topology is probe-only; it does not change native release policy. -All three consume the same verified archive and the same empty timing -snapshot, testing cold-history behavior rather than assuming a warm cache. -The Windows probe compares serial and two-worker execution of the unchanged -`windows_compat` selection. The probes use read-only permissions and no -production secrets; they cannot publish or qualify a release candidate. - -`scripts.compare_test_runs` checks original pytest node IDs, outcomes, complete -disjoint shard coverage, source/candidate identity, and runtime/lockfile -fingerprints before accepting a measured execution improvement of at least -5%. New skips, missing tests, failures, or different environments fail the -comparison. JUnit controller elapsed time is reported separately from summed -runner time; neither includes runner allocation or setup. -The opt-in evidence plugin emits `unittest.subTest` outcomes as distinct -parent-relative ordinal cases, so JUnit summary counts must still exactly -match the concrete inventory; subtests are not discarded to make counts fit. - -Download `arm-performance-comparison-` or -`windows-compat-performance-comparison-` for the JSON verdict, and -the corresponding per-variant artifacts for raw JUnit and timings. Use Actions -job timestamps to include setup and queue delays. Remove the opt-in label -before unrelated pushes; these measurements are evidence, not required -release gates or cached pass results. - -### Whole-pipeline rehearsal - -For whole-pipeline release rehearsal, apply the `ci-release-rehearsal` PR label. -`ci-release-rehearsal.yml` reuses the five native release-platform workflows, -canonical source CI, and read-only docs/wheel builds at one current PR merge -SHA, with no PATs, signing secrets, publishing, or production topology change: - -| Surface | Current shape | Rehearsal shape | -| --- | --- | --- | -| macOS ARM integration | 1 shard x 4 workers | 2 shards x 3 workers | -| Linux x64 integration | 1 shard x 4 workers | 2 shards x 4 workers | -| macOS Intel unit | 1 shard x auto workers | 2 shards x 4 workers | - -Each actual runner records image, Python, OS, CPU, lockfile, source, candidate -identity, runner trace, and timestamp proof, then waits up to 15 minutes for -all three matching cohort members before pytest. Missing, stale, or mismatched proof stops before -expensive tests. Integration pairs use exact archive bytes from the same build; -source-unit pairs use the identical SHA; final comparison still requires exact -original case/outcome parity. - -Windows public release validation accepts `GITHUB_API_TOKEN` without mapping -the read-only job token to PAT/Models variables. Private-module and inference -validation still need their existing credentials. The native installer fixture -uses a short temporary root while retaining spaces and `&` in its install -prefix, leaving room for staged bundle DLL paths. - -The comparison retains all five production -`Integration Tests` fan-ins and `Native Candidate Gate` jobs, measures each -observed dependency-to-verdict delay, and propagates the selected variant -through the per-platform DAG while unchanged platforms keep their actual gate -completion floor. Proposed native-gate completion timestamps are modeled, not -independently measured; report that assumption. Only experiment-specific -both-world checkers, such as `Intel Unit Performance Probe`, are excluded. -Qualification recording, asset verification, signing, and publishing remain -outside the unprivileged measurement. Read `release-rehearsal-comparison-`, -`performance-evidence*`, and `performance-cohort*`. - -Thresholds are 15% per -paired pytest lane and 10% overall evidence-ready improvement, with 30% as the -target. Do not treat workflow duration as proposed release time: it waits for -both variants, and experimental concurrency/cohort waits can add allocation -cost. Remove the label before unrelated pushes. For a new full attempt, remove -and reapply `ci-release-rehearsal` to launch a fresh workflow run; do not rely -on rerunning the same run ID. Rehearsal proofs are attempt-scoped, but canonical -source CI artifacts use bare names. Limit an evaluation to three full attempts, -including invalid or image-inconclusive attempts. Production rollout requires -passing proof plus a maintainer decision. - -### Actual release wall-clock proof - -Do not use modeled counterfactuals as proof. To measure real GitHub wall time -before merge, apply the `ci-release-wallclock` PR label for an ad hoc run, or -push the reviewed SHA to the workflow's dedicated control branch for a frozen -comparison. The control ref avoids a moving PR merge ref changing the tested -source after the baseline starts. Each trigger runs one proposed -production-default DAG in `ci-release-wallclock.yml`, while a separate frozen -baseline controller branch runs the original dependency ordering from the -selected `main` source. Both sides are non-promotable and use no PATs, signing -secrets, private-service acceptance, publication, or deployment scope. - -Compare only fresh attempt-1 run IDs recorded by the wall-clock protocol. Source -identity and controller identity are recorded separately. The primary clock is -the terminal `Verify wall-clock artifacts` job `completed_at` minus workflow -`created_at`, including queue time; the secondary clock starts at actual -`run_started_at`. Accept no result unless both workflows succeed, every included -gate succeeds, all five native archives plus docs and wheel artifacts verify, -and observer case inventory plus runner environment proof are present. The -terminal verifier must download native observation artifacts, not only archives, -and rejects missing or malformed runner job records through the shared validator. -The proposed wrapper preserves separate candidate-ready native evidence capture -and serial verify / verified-native upload. All `wallclock-*` storage names are -non-promotable. Workflow success alone is not a savings claim; cite two accepted -run IDs and their actual primary and secondary clocks before claiming an -improvement. - -For native Windows installer diagnostics, apply the optional -`ci-windows-installer-proof` PR label. It runs one candidate through the same -assertions at long and short roots, checks native exit/stderr controls, and -retains the archive; this is diagnostic evidence, not whole-pipeline savings -proof. Production Windows installer tests use a short pytest-owned -`RUNNER_TEMP/apm-windows-installer` base temp, keep spaces and `&` fixture -cases, and reject zero/skipped JUnit instead of treating pytest exit 0 as proof. +suite, and shard. + +A historical comparison at source commit `1687380d7` measured 34m41s -> 24m53s +(about 28% lower wall time) with 22.77% more unweighted runner time. The +experiments are retired; these results do not qualify the current code or +guarantee a speedup. ## Debugging Test Failures diff --git a/scripts/compare_test_runs.py b/scripts/compare_test_runs.py deleted file mode 100644 index 6aa1af08a7..0000000000 --- a/scripts/compare_test_runs.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Compare pytest execution time only after proving equivalent test evidence.""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import platform -import re -import xml.etree.ElementTree as ET -from dataclasses import dataclass -from pathlib import Path - -from scripts.package_release import archive_name, file_digest, require_sha - -OUTCOMES = {"passed", "skipped", "xfailed", "failed", "error"} - - -class _JUnitTreeBuilder(ET.TreeBuilder): - def doctype(self, name: str, pubid: str | None, system: str | None) -> None: - """Reject DTDs before the XML parser can expand custom entities.""" - raise ValueError("JUnit document type declarations are forbidden") - - -@dataclass(frozen=True) -class Report: - """One pytest invocation, including its explicit source and shard identity.""" - - identity: dict[str, object] - variant: str - shard: int - shard_count: int - elapsed_seconds: float - cases: dict[tuple[str, str], str] - - def payload(self) -> dict[str, object]: - """Return a deterministic, inspectable evidence document.""" - return { - "schema_version": 1, - "identity": self.identity, - "variant": self.variant, - "shard": self.shard, - "shard_count": self.shard_count, - "elapsed_seconds": self.elapsed_seconds, - "cases": [ - {"id": list(node), "outcome": outcome} - for node, outcome in sorted(self.cases.items()) - ], - } - - -def _environment() -> dict[str, str]: - return { - "python": platform.python_version(), - "system": platform.system(), - "release": platform.release(), - "machine": platform.machine(), - "cpu_count": str(os.cpu_count()), - "runner_image": os.environ.get("ImageVersion", ""), # noqa: SIM112 - runner-images contract. - "dependency_lock": file_digest(Path(__file__).resolve().parents[1] / "uv.lock"), - } - - -def _identity( - source_sha: str, selection: str, candidate: object, environment: object -) -> dict[str, object]: - require_sha(source_sha) - if not isinstance(selection, str) or not selection.strip(): - raise ValueError("A nonempty test selection is required") - if ( - not isinstance(environment, dict) - or set(environment) - != { - "python", - "system", - "release", - "machine", - "cpu_count", - "runner_image", - "dependency_lock", - } - or not all(isinstance(value, str) for value in environment.values()) - or any(not value for key, value in environment.items() if key != "runner_image") - ): - raise ValueError("Explicit runtime/runner environment evidence is required") - if candidate is not None: - if not isinstance(candidate, dict) or type(candidate.get("schema_version")) is not int: - raise ValueError("Invalid candidate metadata schema") - if candidate["schema_version"] != 1 or candidate.get("sha") != source_sha: - raise ValueError("Candidate metadata does not identify the expected source") - binary_name = candidate.get("binary_name") - if not isinstance(binary_name, str) or candidate.get("archive") != archive_name( - binary_name - ): - raise ValueError("Candidate metadata has an invalid archive identity") - if not isinstance(candidate.get("version"), str) or not candidate["version"]: - raise ValueError("Candidate metadata has no version") - for field in ("archive_sha256", "executable_sha256"): - value = candidate.get(field) - if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: - raise ValueError(f"Invalid candidate {field}") - return { - "source_sha": source_sha, - "selection": selection, - "candidate": candidate, - "environment": environment, - } - - -def capture_identity( - source_sha: str, selection: str, candidate_metadata: Path | None = None -) -> dict[str, object]: - """Capture the same identity before execution and in the final test report.""" - candidate = json.loads(candidate_metadata.read_text("utf-8")) if candidate_metadata else None - return _identity(source_sha, selection, candidate, _environment()) - - -def validate_identity(identity: dict[str, object]) -> dict[str, object]: - """Validate an already captured identity without replacing its environment.""" - return _identity( - identity["source_sha"], - identity["selection"], - identity["candidate"], - identity["environment"], - ) - - -def _positive_seconds(value: object) -> float: - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - or value <= 0 - ): - raise ValueError("Elapsed time must be finite and positive") - return float(value) - - -def _shard_identity(variant: str, shard: int, count: int) -> None: - if variant not in ("baseline", "proposed"): - raise ValueError("Variant must be baseline or proposed") - if type(shard) is not int or type(count) is not int or not 1 <= shard <= count <= 16: - raise ValueError("Require a complete bounded shard identity: 1 <= shard <= count <= 16") - - -def capture( - junit: Path, - source_sha: str, - selection: str, - variant: str, - shard: int, - shard_count: int, - candidate_metadata: Path | None = None, -) -> Report: - """Read one real pytest JUnit report without equating skips with passes.""" - _shard_identity(variant, shard, shard_count) - identity = capture_identity(source_sha, selection, candidate_metadata) - parser = ET.XMLParser(target=_JUnitTreeBuilder()) # noqa: S314 - rejects all DTD declarations. - root = ET.parse(junit, parser=parser).getroot() # noqa: S314 - DTDs rejected before expansion. - suites = [root] if root.tag == "testsuite" else list(root) - if len(suites) != 1 or suites[0].tag != "testsuite": - raise ValueError("Expected the single-suite JUnit output of one pytest invocation") - suite = suites[0] - elapsed = _positive_seconds(float(suite.attrib["time"])) - cases = {} - for node in suite.findall("testcase"): - identities = node.findall("./properties/property[@name='apm_performance_nodeid']") - if len(identities) > 1: - raise ValueError("Duplicate original node identity in JUnit") - key = ( - ("pytest-nodeid", identities[0].attrib["value"]) - if identities - else (node.attrib.get("classname", ""), node.attrib["name"]) - ) - if not key[1] or key in cases: - raise ValueError(f"Empty or duplicate JUnit case identity: {key!r}") - failures = [child for child in node if child.tag in ("failure", "error", "skipped")] - if len(failures) > 1: - raise ValueError(f"Ambiguous JUnit outcome: {key!r}") - if not failures: - outcome = "passed" - elif failures[0].tag == "skipped": - outcome = "xfailed" if failures[0].get("type") == "pytest.xfail" else "skipped" - else: - outcome = "failed" if failures[0].tag == "failure" else "error" - cases[key] = outcome - counts = { - "tests": len(cases), - "failures": sum(value == "failed" for value in cases.values()), - "errors": sum(value == "error" for value in cases.values()), - "skipped": sum(value in ("skipped", "xfailed") for value in cases.values()), - } - if not cases or any(int(suite.attrib[field]) != count for field, count in counts.items()): - raise ValueError("JUnit summary counts do not match the nonempty case inventory") - return Report(identity, variant, shard, shard_count, elapsed, cases) - - -def read_report(path: Path) -> Report: - """Reject incomplete or malformed proof rather than silently dropping a shard.""" - data = json.loads(path.read_text("utf-8")) - if not isinstance(data, dict) or type(data.get("schema_version")) is not int: - raise ValueError(f"Invalid report schema: {path}") - if data["schema_version"] != 1 or not isinstance(data.get("identity"), dict): - raise ValueError(f"Invalid report identity: {path}") - identity = validate_identity(data["identity"]) - _shard_identity(data["variant"], data["shard"], data["shard_count"]) - elapsed = _positive_seconds(data["elapsed_seconds"]) - entries = data["cases"] - if not isinstance(entries, list) or not entries: - raise ValueError("Report must contain a nonempty case inventory") - cases = {} - for entry in entries: - if not isinstance(entry, dict): - raise ValueError("Invalid case evidence") - node, outcome = entry.get("id"), entry.get("outcome") - if ( - not isinstance(node, list) - or len(node) != 2 - or not all(isinstance(part, str) for part in node) - or not node[1] - or not isinstance(outcome, str) - or outcome not in OUTCOMES - ): - raise ValueError("Invalid case identity or outcome") - key = (node[0], node[1]) - if key in cases: - raise ValueError(f"Duplicate case evidence: {key!r}") - cases[key] = outcome - return Report(identity, data["variant"], data["shard"], data["shard_count"], elapsed, cases) - - -def _group(reports: list[Report], variant: str) -> dict[tuple[str, str], str]: - if not reports: - raise ValueError(f"No {variant} reports") - count = reports[0].shard_count - if len(reports) != count or {report.shard for report in reports} != set(range(1, count + 1)): - raise ValueError(f"Missing or duplicate {variant} shard") - cases = {} - for report in reports: - if report.variant != variant or report.shard_count != count: - raise ValueError(f"Inconsistent {variant} shard identity") - if set(cases) & report.cases.keys(): - raise ValueError(f"Overlapping {variant} test partitions") - if set(report.cases.values()) & {"failed", "error"}: - raise ValueError(f"{variant} contains failing tests") - cases.update(report.cases) - if "passed" not in cases.values(): - raise ValueError(f"{variant} did not execute any passing tests") - return cases - - -def compare( - baseline: list[Report], proposed: list[Report], expected_sha: str, minimum_speedup: float -) -> dict[str, object]: - """Prove identity, complete selection and outcomes before comparing execution time.""" - require_sha(expected_sha) - if not math.isfinite(minimum_speedup) or not 0 <= minimum_speedup < 1: - raise ValueError("Minimum speedup must be finite and in [0, 1)") - old_cases, new_cases = _group(baseline, "baseline"), _group(proposed, "proposed") - identity = baseline[0].identity - if identity["source_sha"] != expected_sha or any( - report.identity != identity for report in (*baseline, *proposed) - ): - raise ValueError( - "Source, candidate bytes, environment or test selection differ between runs" - ) - if old_cases != new_cases: - missing, extra = old_cases.keys() - new_cases.keys(), new_cases.keys() - old_cases.keys() - changed = sum(old_cases[node] != new_cases[node] for node in old_cases.keys() & new_cases) - raise ValueError( - f"Scenario parity failed: {len(missing)} missing, {len(extra)} extra, " - f"{changed} changed outcomes" - ) - before = max(report.elapsed_seconds for report in baseline) - after = max(report.elapsed_seconds for report in proposed) - speedup = 1 - after / before - return { - "schema_version": 1, - "identity": identity, - "scenario_count": len(old_cases), - "outcomes": { - outcome: list(old_cases.values()).count(outcome) for outcome in sorted(OUTCOMES) - }, - "same_scenarios_and_outcomes": True, - "baseline_pytest_critical_seconds": before, - "proposed_pytest_critical_seconds": after, - "baseline_pytest_runner_seconds": sum(report.elapsed_seconds for report in baseline), - "proposed_pytest_runner_seconds": sum(report.elapsed_seconds for report in proposed), - "speedup_fraction": speedup, - "minimum_speedup_fraction": minimum_speedup, - "meets_threshold": speedup >= minimum_speedup, - "scope": "pytest controller elapsed time; excludes runner setup and queue delays", - } - - -def main() -> int: - """Write durable JSON proof; fail visibly on mismatched evidence or a missed target.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="operation", required=True) - record = commands.add_parser("capture") - record.add_argument("--junit", type=Path, required=True) - record.add_argument("--source-sha", required=True) - record.add_argument("--selection", required=True) - record.add_argument("--variant", choices=("baseline", "proposed"), required=True) - record.add_argument("--shard", type=int, required=True) - record.add_argument("--shard-count", type=int, required=True) - record.add_argument("--candidate-metadata", type=Path) - check = commands.add_parser("compare") - check.add_argument("--baseline", type=Path, nargs="+", required=True) - check.add_argument("--proposed", type=Path, nargs="+", required=True) - check.add_argument("--expected-sha", required=True) - check.add_argument("--minimum-speedup", type=float, default=0.05) - for command in (record, check): - command.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - try: - if args.operation == "capture": - result = capture( - args.junit, - args.source_sha, - args.selection, - args.variant, - args.shard, - args.shard_count, - args.candidate_metadata, - ).payload() - else: - result = compare( - [read_report(path) for path in args.baseline], - [read_report(path) for path in args.proposed], - args.expected_sha, - args.minimum_speedup, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="ascii" - ) - except (ValueError, KeyError, TypeError, OSError, ET.ParseError) as error: - parser.error(f"Invalid performance evidence: {error}") - if args.operation == "compare" and not result["meets_threshold"]: - parser.exit(1, f"Performance target missed; inspect {args.output}\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/performance-cohort.cjs b/scripts/performance-cohort.cjs deleted file mode 100644 index 0c9df89391..0000000000 --- a/scripts/performance-cohort.cjs +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; - -const MEMBERS = ['baseline-1', 'proposed-1', 'proposed-2']; - -async function waitForCohort({ - github, context, cohort, attempt, - timeoutMs = 900000, intervalMs = 20000, - now = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), -}) { - if (!Number.isSafeInteger(attempt) || attempt < 1 || - !new RegExp(`^(unit|integration)-${attempt}-apm-[a-z0-9]+-[a-z0-9_]+$`).test(cohort) || - !Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 900000 || - !Number.isFinite(intervalMs) || intervalMs <= 0) { - throw new Error('Invalid bounded performance cohort'); - } - const expected = new Set(MEMBERS.map(member => `performance-cohort-${cohort}-${member}`)); - const deadline = now() + timeoutMs; - while (now() < deadline) { - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - ...context.repo, run_id: context.runId, per_page: 100, - }); - const members = artifacts.filter(artifact => expected.has(artifact.name)); - if (members.some(artifact => artifact.expired) || - new Set(members.map(artifact => artifact.name)).size !== members.length) { - throw new Error('Expired or duplicate performance cohort artifact'); - } - if (members.length === expected.size) { - return; - } - await sleep(Math.min(intervalMs, Math.max(0, deadline - now()))); - } - throw new Error('Performance cohort timed out before all actual test runners were ready'); -} - -module.exports = {waitForCohort}; diff --git a/scripts/performance-cohort.test.cjs b/scripts/performance-cohort.test.cjs deleted file mode 100644 index 1f1df20859..0000000000 --- a/scripts/performance-cohort.test.cjs +++ /dev/null @@ -1,72 +0,0 @@ -'use strict'; - -const assert = require('node:assert/strict'); -const test = require('node:test'); -const {waitForCohort} = require('./performance-cohort.cjs'); - -const cohort = 'integration-2-apm-darwin-arm64'; -const names = ['baseline-1', 'proposed-1', 'proposed-2'] - .map(member => `performance-cohort-${cohort}-${member}`); - -function harness(responses) { - let clock = 0; - let calls = 0; - const github = { - rest: {actions: {listWorkflowRunArtifacts: 'list'}}, - paginate: async (method, params) => { - assert.equal(method, 'list'); - assert.deepEqual(params, {owner: 'owner', repo: 'repo', run_id: 123, per_page: 100}); - return responses[Math.min(calls++, responses.length - 1)]; - }, - }; - return { - options: { - github, context: {repo: {owner: 'owner', repo: 'repo'}, runId: 123}, - cohort, attempt: 2, timeoutMs: 100, intervalMs: 20, - now: () => clock, sleep: async ms => { clock += ms; }, - }, - calls: () => calls, - }; -} - -test('waits for the exact current-attempt trio, ignoring old attempts', async () => { - const current = names.map(name => ({name, expired: false})); - const state = harness([ - [{name: names[0].replace('-2-', '-1-'), expired: false}, current[0]], - current, - ]); - await waitForCohort(state.options); - assert.equal(state.calls(), 2); -}); - -test('missing members time out instead of allowing incomparable tests', async () => { - const state = harness([[{name: names[0], expired: false}]]); - await assert.rejects(waitForCohort(state.options), /timed out/); - assert.equal(state.calls(), 5); -}); - -for (const fault of ['duplicate', 'expired']) { - test(`rejects ${fault} artifact evidence`, async () => { - const members = names.map(name => ({name, expired: fault === 'expired'})); - if (fault === 'duplicate') members.push(members[0]); - const state = harness([members]); - await assert.rejects(waitForCohort(state.options), /Expired or duplicate/); - }); -} - -test('API failures propagate rather than pretending a cohort is ready', async () => { - const state = harness([]); - state.options.github.paginate = async () => { throw new Error('API unavailable'); }; - await assert.rejects(waitForCohort(state.options), /API unavailable/); -}); - -for (const change of [ - {attempt: 1}, {cohort: '../../wrong'}, {timeoutMs: Infinity}, - {timeoutMs: 900001}, {intervalMs: 0}, -]) { - test(`rejects invalid wait input ${JSON.stringify(change)}`, async () => { - const state = harness([]); - await assert.rejects(waitForCohort({...state.options, ...change}), /Invalid bounded/); - assert.equal(state.calls(), 0); - }); -} diff --git a/scripts/performance_cohort.py b/scripts/performance_cohort.py deleted file mode 100644 index a3f4d689be..0000000000 --- a/scripts/performance_cohort.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Fail before expensive tests unless the actual paired runners have equal identities.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -from datetime import UTC, datetime -from pathlib import Path - -from scripts.compare_test_runs import capture_identity, validate_identity -from scripts.package_release import require_sha - -MEMBERS = ("baseline-1", "proposed-1", "proposed-2") - - -def require_context(cohort: str, run_id: str, attempt: int) -> None: - """Restrict artifact names and bind them to one complete workflow attempt.""" - if not isinstance(run_id, str) or re.fullmatch(r"[1-9][0-9]*", run_id) is None: - raise ValueError("A positive workflow run ID is required") - if type(attempt) is not int or attempt < 1: - raise ValueError("A positive workflow attempt is required") - if ( - not isinstance(cohort, str) - or re.fullmatch(rf"(unit|integration)-{attempt}-apm-[a-z0-9]+-[a-z0-9_]+", cohort) is None - ): - raise ValueError("Cohort must identify a suite, current attempt and native binary") - - -def record( - cohort: str, - member: str, - run_id: str, - attempt: int, - source_sha: str, - selection: str, - candidate_metadata: Path | None = None, -) -> dict[str, object]: - """Record identity on the runner that will execute this exact test partition.""" - require_context(cohort, run_id, attempt) - if member not in MEMBERS: - raise ValueError("Unknown cohort member") - identity = capture_identity(source_sha, selection, candidate_metadata) - if not identity["environment"]["runner_image"]: - raise ValueError("Hosted runner ImageVersion is required for a controlled cohort") - if cohort.startswith("integration-") != (identity["candidate"] is not None): - raise ValueError("Only integration cohorts require candidate archive metadata") - runner_name = os.environ.get("RUNNER_NAME", "") - workflow_job = os.environ.get("GITHUB_JOB", "") - if not runner_name or not workflow_job: - raise ValueError("Actual GitHub runner and workflow job identity are required") - return { - "schema_version": 1, - "cohort": cohort, - "member": member, - "run_id": run_id, - "run_attempt": attempt, - "identity": identity, - "execution": { - "runner_name": runner_name, - "workflow_job": workflow_job, - "recorded_at": datetime.now(UTC).isoformat(), - }, - } - - -def verify( - root: Path, cohort: str, run_id: str, attempt: int, source_sha: str -) -> dict[str, object]: - """Reject missing, duplicate, stale, malformed or environmentally different members.""" - require_context(cohort, run_id, attempt) - require_sha(source_sha) - if {path.name for path in root.iterdir()} != {f"{member}.json" for member in MEMBERS}: - raise ValueError("Exactly the three declared cohort member files are required") - identities = [] - for member in MEMBERS: - data = json.loads((root / f"{member}.json").read_text("ascii")) - if ( - not isinstance(data, dict) - or type(data.get("schema_version")) is not int - or data["schema_version"] != 1 - or data.get("cohort") != cohort - or data.get("member") != member - or data.get("run_id") != run_id - or type(data.get("run_attempt")) is not int - or data["run_attempt"] != attempt - or not isinstance(data.get("identity"), dict) - ): - raise ValueError(f"Invalid or stale cohort evidence for {member}") - identity = validate_identity(data["identity"]) - if identity["source_sha"] != source_sha or not identity["environment"]["runner_image"]: - raise ValueError("Cohort source and hosted image must be explicit") - if cohort.startswith("integration-") != (identity["candidate"] is not None): - raise ValueError("Cohort suite does not match its candidate evidence") - execution = data.get("execution") - if ( - not isinstance(execution, dict) - or any( - not isinstance(execution.get(key), str) or not execution[key] - for key in ("runner_name", "workflow_job", "recorded_at") - ) - or datetime.fromisoformat(execution["recorded_at"]).tzinfo is None - ): - raise ValueError("Cohort must bind its actual runner and timestamp to the job timeline") - identities.append(identity) - if any(identity != identities[0] for identity in identities[1:]): - raise ValueError("Cohort identity mismatch; do not run or claim comparable performance") - return identities[0] - - -def main() -> int: - """Capture or validate a pre-execution cohort without producing a pass substitute.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="operation", required=True) - capture = commands.add_parser("record") - capture.add_argument("--member", choices=MEMBERS, required=True) - capture.add_argument("--selection", required=True) - capture.add_argument("--candidate-metadata", type=Path) - capture.add_argument("--output", type=Path, required=True) - check = commands.add_parser("verify") - check.add_argument("--root", type=Path, required=True) - for command in (capture, check): - command.add_argument("--cohort", required=True) - command.add_argument("--run-id", required=True) - command.add_argument("--attempt", type=int, required=True) - command.add_argument("--source-sha", required=True) - args = parser.parse_args() - try: - if args.operation == "record": - data = record( - args.cohort, - args.member, - args.run_id, - args.attempt, - args.source_sha, - args.selection, - args.candidate_metadata, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", "ascii") - else: - verify(args.root, args.cohort, args.run_id, args.attempt, args.source_sha) - except (ValueError, KeyError, TypeError, OSError) as error: - parser.error(f"Invalid performance cohort: {error}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/pytest_performance_evidence.py b/scripts/pytest_performance_evidence.py deleted file mode 100644 index 912f016d7b..0000000000 --- a/scripts/pytest_performance_evidence.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Preserve original pytest node IDs before xdist appends scheduling groups.""" - -from __future__ import annotations - -import pytest - -_PARENT_NODEID_PROPERTY = "apm_performance_parent_nodeid" -_ORIGINAL_NODEIDS: set[str] = set() -_SUBTEST_ORDINALS: dict[str, int] = {} -_IS_XDIST_WORKER = False - - -def pytest_configure(config: pytest.Config) -> None: - """Keep nested/in-process pytest sessions from reusing stale identities.""" - global _IS_XDIST_WORKER - _ORIGINAL_NODEIDS.clear() - _SUBTEST_ORDINALS.clear() - _IS_XDIST_WORKER = hasattr(config, "workerinput") - - -@pytest.hookimpl(tryfirst=True) -def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - """Record a stable identity without changing selection, grouping, or outcomes.""" - for item in items: - _ORIGINAL_NODEIDS.add(item.nodeid) - item.user_properties.append(("apm_performance_nodeid", item.nodeid)) - - -def _original_nodeid(nodeid: str) -> str: - """Undo pytest-xdist's loadgroup suffix only when it matches a collected item.""" - if nodeid in _ORIGINAL_NODEIDS: - return nodeid - candidate, separator, _group = nodeid.rpartition("@") - if separator and candidate in _ORIGINAL_NODEIDS: - return candidate - return nodeid - - -def _pop_worker_parent_nodeid(report: pytest.TestReport) -> str | None: - """Read the worker-captured parent nodeid without leaking it to JUnit.""" - parents = [ - value - for name, value in report.user_properties - if name == _PARENT_NODEID_PROPERTY and isinstance(value, str) and value - ] - report.user_properties = [ - (name, value) for name, value in report.user_properties if name != _PARENT_NODEID_PROPERTY - ] - if len(parents) > 1: - raise ValueError("Duplicate performance subtest parent identity") - return parents[0] if parents else None - - -@pytest.hookimpl(tryfirst=True) -def pytest_runtest_logreport(report: pytest.TestReport) -> None: - """Give unittest subTest reports their own JUnit node instead of a summary-only count. - - Pytest 9 emits a passed call report for each ``unittest.TestCase.subTest`` but - its JUnit writer otherwise folds those passed subtests into the parent - ```` while still incrementing ``tests``. Performance evidence - needs the strict XML summary count to equal the concrete scenario inventory, - so split each subtest report into a stable synthetic child node before - junitxml sees it. - """ - if getattr(report, "when", None) != "call" or not hasattr(report, "context"): - return - if _IS_XDIST_WORKER: - report.user_properties.append((_PARENT_NODEID_PROPERTY, _original_nodeid(report.nodeid))) - return - parent_nodeid = _pop_worker_parent_nodeid(report) or _original_nodeid(report.nodeid) - _SUBTEST_ORDINALS[parent_nodeid] = _SUBTEST_ORDINALS.get(parent_nodeid, 0) + 1 - report.nodeid = f"{parent_nodeid}::subTest[{_SUBTEST_ORDINALS[parent_nodeid]}]" - report.user_properties.append(("apm_performance_nodeid", report.nodeid)) diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index e23c2cfaca..aba9b08958 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -89,7 +89,7 @@ function isTagPush(context) { } function isPrereleaseTag(context) { - return isTagPush(context) && refName(context.ref).includes("-"); + return isTagPush(context) && !/^v[0-9]+\.[0-9]+\.[0-9]+$/.test(refName(context.ref)); } function tagVersion(context) { diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs index 6e57958881..03ff7a23ff 100644 --- a/scripts/release-candidate.test.cjs +++ b/scripts/release-candidate.test.cjs @@ -236,6 +236,32 @@ async function withQualifiedReusableCandidate(fn) { } describe("release candidate planning", () => { + for (const [tag, prerelease] of [ + ["v1.2.3", false], + ["v0.28.0", false], + ["v1.2.3a1", true], + ["v1.2.3b1", true], + ["v1.2.3rc1", true], + ["v1.2.3-alpha.1", true], + ["v1.2.3-beta.1", true], + ["v1.2.3-rc.1", true], + ["v1.2.3.dev1", true], + ["v1.2.3+build.1", true], + ["v1.2", true], + ["vnext", true], + ]) { + it(`classifies release publication for ${tag} through the planner`, async () => { + const outputs = {}; + await rc.plan({ + github: fakeGithub(), + context: context({ ref: `refs/tags/${tag}` }), + core: { setOutput(name, value) { outputs[name] = value; } }, + }); + assert.equal(outputs.is_prerelease, String(prerelease)); + assert.equal(outputs.full_validation, "true"); + }); + } + for (const unavailable of ["missing evidence", "missing platform", "expired evidence", "expired platform", "old run"]) { it(`builds fresh when automatic discovery finds ${unavailable}`, async () => { const outputs = {}; diff --git a/scripts/release_rehearsal.py b/scripts/release_rehearsal.py deleted file mode 100644 index b350ce7730..0000000000 --- a/scripts/release_rehearsal.py +++ /dev/null @@ -1,479 +0,0 @@ -"""Compare a complete read-only release rehearsal without adding parallel job savings.""" - -from __future__ import annotations - -import argparse -import json -import math -import shutil -import subprocess -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path - -from scripts.compare_test_runs import Report, compare, read_report -from scripts.package_release import BINARY_NAMES, require_sha -from scripts.performance_cohort import MEMBERS, verify - -ROOT = Path(__file__).resolve().parents[1] - - -@dataclass(frozen=True) -class Surface: - """One predeclared paired surface; ordinary native obligations remain unchanged.""" - - key: str - binary: str - suite: str - cpus: str - selection: str - - -SURFACES = ( - Surface( - "arm-integration", "apm-darwin-arm64", "integration", "3", "tests/integration/ -m not live" - ), - Surface( - "linux-integration", - "apm-linux-x86_64", - "integration", - "4", - "tests/integration/ -m not live", - ), - Surface("intel-units", "apm-darwin-x86_64", "unit", "4", "tests/unit tests/test_console.py"), -) - - -def rehearsal_matrix(catalog: list[dict]) -> dict[str, list[dict]]: - """Add opt-in comparison inputs without changing the canonical baseline catalog.""" - if len(catalog) != 5 or {row["binary_name"] for row in catalog} != set(BINARY_NAMES): - raise ValueError("A rehearsal must retain all five unique native platforms") - rows = [] - for row in catalog: - if ( - row["integration_shard_count"] != 1 - or row["integration_xdist_workers"] != 4 - or row["integration_splitting_algorithm"] != "duration_based_chunks" - ): - raise ValueError("The predeclared rehearsal baseline is one four-worker native shard") - binary = row["binary_name"] - rows.append( - { - **row, - "unit_performance_probe": binary == "apm-darwin-x86_64", - "integration_performance_probe": binary in {"apm-darwin-arm64", "apm-linux-x86_64"}, - "performance_workers": 3 if binary == "apm-darwin-arm64" else 4, - } - ) - return {"include": rows} - - -def _timestamp(value: object) -> datetime: - if not isinstance(value, str): - raise ValueError("Explicit timezone-aware job timestamps are required") - result = datetime.fromisoformat(value.replace("Z", "+00:00")) - if result.tzinfo is None: - raise ValueError("Job timestamps must include a timezone") - return result - - -def _job_times(job: dict, origin: datetime) -> tuple[datetime, datetime]: - if job.get("status") != "completed" or job.get("conclusion") != "success": - raise ValueError(f"Required job did not succeed: {job.get('name')}") - start, end = _timestamp(job.get("started_at")), _timestamp(job.get("completed_at")) - if start < origin or end < start: - raise ValueError(f"Invalid required job timeline: {job.get('name')}") - return start, end - - -def _one(jobs: list[dict], name: str, origin: datetime, prefix: str | None = None) -> dict: - matches = [ - job - for job in jobs - if job.get("name") == name - or ( - prefix is not None - and isinstance(job.get("name"), str) - and job["name"].startswith(prefix + " / ") - and job["name"].endswith(" / " + name) - ) - ] - if len(matches) != 1: - raise ValueError(f"Expected exactly one required job: {prefix or ''} / {name}") - _job_times(matches[0], origin) - return matches[0] - - -def source_authorities() -> list[str]: - """Ask the existing release authority for its exact source checks, never fork the list.""" - node = shutil.which("node") - if node is None: - raise OSError("Node.js is required to read the canonical release authorities") - result = subprocess.run( # noqa: S603 - installed Node and a constant repository authority module. - [ - node, - "-e", - "process.stdout.write(JSON.stringify(require('./scripts/release-candidate.cjs').REQUIRED_SOURCE_JOB_NAMES))", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - timeout=30, - ) - names = json.loads(result.stdout) - if not isinstance(names, list) or not names or any(not isinstance(name, str) for name in names): - raise ValueError("Canonical source authorities must be a nonempty name list") - return names - - -def _common_jobs(jobs: list[dict], origin: datetime, source_names: list[str]) -> list[dict]: - common = [_one(jobs, name, origin) for name in source_names] - for binary in BINARY_NAMES: - _one(jobs, f"{binary} / Native Candidate Gate", origin) - common.append(_one(jobs, "Build Candidate", origin, binary)) - common.append(_one(jobs, "Isolated Release Validation", origin, binary)) - if binary != "apm-darwin-x86_64": - common.append(_one(jobs, "Unit Tests Baseline", origin, binary)) - if binary not in {"apm-darwin-arm64", "apm-linux-x86_64"}: - common.append(_one(jobs, "Integration Tests Shard 1", origin, binary)) - if binary == "apm-windows-x86_64": - common.append(_one(jobs, "Candidate Windows Installer", origin, binary)) - common.extend( - [ - _one(jobs, "Read-only documentation build / build", origin), - _one(jobs, "Read-only Python distribution build / Build PyPI Distributions", origin), - ] - ) - if len({job["id"] for job in common}) != len(common): - raise ValueError("A required job was counted twice") - return common - - -def _bind_member( - root: Path, member: str, jobs: list[dict], surface: Surface, origin: datetime -) -> dict: - data = json.loads((root / f"{member}.json").read_text("ascii")) - trace = data["execution"] - runner = trace["runner_name"] - recorded = _timestamp(trace["recorded_at"]) - if not isinstance(runner, str) or not runner: - raise ValueError("Cohort must identify its actual GitHub runner") - matches = [] - for job in jobs: - if ( - job.get("runner_name") == runner - and isinstance(job.get("name"), str) - and job["name"].startswith(surface.binary + " / ") - and job.get("status") == "completed" - ): - start, end = _job_times(job, origin) - if start <= recorded <= end: - matches.append(job) - if len(matches) != 1: - raise ValueError(f"Cannot uniquely bind {surface.key}/{member} to its actual native job") - return matches[0] - - -def _surface_for(report: Report) -> Surface: - candidate = report.identity["candidate"] - for surface in SURFACES: - if report.identity["selection"] == surface.selection and ( - (candidate is None and surface.suite == "unit") - or (isinstance(candidate, dict) and candidate.get("binary_name") == surface.binary) - ): - return surface - raise ValueError("Unexpected performance evidence outside the predeclared three surfaces") - - -def _native_authorities( - jobs: list[dict], pairs: dict, origin: datetime -) -> tuple[list[dict], list[dict]]: - """Retain native gates, propagating their observed dependency-to-verdict delays.""" - authorities, gate_jobs = [], [] - - def end(parts: list[dict]) -> float: - return max((_job_times(job, origin)[1] - origin).total_seconds() for job in parts) - - def delay(gate: dict, prerequisites: list[dict]) -> float: - value = end([gate]) - end(prerequisites) - if value < 0: - raise ValueError(f"Required gate completed before its prerequisites: {gate['name']}") - return value - - for binary in sorted(BINARY_NAMES): - surfaces = {surface.suite: surface.key for surface in SURFACES if surface.binary == binary} - unit_key, integration_key = surfaces.get("unit"), surfaces.get("integration") - unit = ( - pairs[unit_key] - if unit_key - else { - "baseline": [_one(jobs, "Unit Tests Baseline", origin, binary)], - } - ) - integration = ( - pairs[integration_key] - if integration_key - else { - "baseline": [_one(jobs, "Integration Tests Shard 1", origin, binary)], - } - ) - integration_gate = _one(jobs, "Integration Tests", origin, binary) - integration_delay = delay(integration_gate, integration["baseline"]) - observed_unit = unit["baseline"] - if unit_key: - # This extra checker exists only to join both experimental unit worlds. - unit_probe = _one(jobs, "Unit Performance Probe", origin, binary) - delay(unit_probe, [*unit["baseline"], *unit["proposed"]]) - observed_unit = [unit_probe] - independent = [ - _one(jobs, "Build Candidate", origin, binary), - _one(jobs, "Isolated Release Validation", origin, binary), - ] - if binary == "apm-windows-x86_64": - independent.append(_one(jobs, "Candidate Windows Installer", origin, binary)) - gate = _one(jobs, f"{binary} / Native Candidate Gate", origin) - native_delay = delay(gate, [*independent, *observed_unit, integration_gate]) - gate_jobs.extend([integration_gate, gate]) - authorities.append( - { - "name": gate["name"], - "observed_completed_at": gate["completed_at"], - "independent_ready_seconds": end(independent), - "unit_surface": unit_key, - "integration_surface": integration_key, - "unit_ready_seconds": {variant: end(parts) for variant, parts in unit.items()}, - "integration_ready_seconds": { - variant: end(parts) + integration_delay - for variant, parts in integration.items() - }, - "integration_fan_in_delay_seconds": integration_delay, - "native_gate_delay_seconds": native_delay, - } - ) - return authorities, gate_jobs - - -def _scenario( - common: list[dict], - selected: list[dict], - origin: datetime, - authorities: list[dict], - gate_jobs: list[dict], - enabled: set[str], -) -> dict: - jobs = [*common, *selected] - end_job = max(jobs, key=lambda job: _job_times(job, origin)[1]) - ready = (_job_times(end_job, origin)[1] - origin).total_seconds() - limiting = end_job["name"] - native = [] - for authority in authorities: - unit_variant = "proposed" if authority["unit_surface"] in enabled else "baseline" - integration_variant = ( - "proposed" if authority["integration_surface"] in enabled else "baseline" - ) - completed = ( - max( - authority["independent_ready_seconds"], - authority["unit_ready_seconds"][unit_variant], - authority["integration_ready_seconds"][integration_variant], - ) - + authority["native_gate_delay_seconds"] - ) - native.append({**authority, "modeled_ready_seconds": completed}) - if completed >= ready: - ready, limiting = completed, authority["name"] - return { - "evidence_ready_seconds": ready, - "limiting_job": limiting, - "native_authorities": native, - "selected_runner_seconds": sum( - (end - start).total_seconds() - for start, end in (_job_times(job, origin) for job in [*jobs, *gate_jobs]) - ), - } - - -def evaluate( - metadata: dict, - reports: list[Report], - cohorts: Path, - expected_sha: str, - source_names: list[str], - minimum_lane_gain: float, - minimum_overall_gain: float, -) -> dict: - """Reconstruct complete evidence-ready paths, preserving all common and paired work.""" - require_sha(expected_sha) - for threshold in (minimum_lane_gain, minimum_overall_gain): - if not math.isfinite(threshold) or not 0 <= threshold < 1: - raise ValueError("Performance thresholds must be finite fractions in [0, 1)") - if ( - type(metadata.get("schema_version")) is not int - or metadata["schema_version"] != 1 - or metadata.get("source_sha") != expected_sha - or type(metadata.get("run_attempt")) is not int - or metadata["run_attempt"] < 1 - ): - raise ValueError("Current exact-SHA workflow attempt metadata is required") - run_id, attempt = metadata["run_id"], metadata["run_attempt"] - origin = _timestamp(metadata["created_at"] if attempt == 1 else metadata["run_started_at"]) - jobs = metadata["jobs"] - if not isinstance(jobs, list) or not jobs or len({job["id"] for job in jobs}) != len(jobs): - raise ValueError("A complete unique current-attempt job inventory is required") - if any(job.get("run_id") != int(run_id) for job in jobs): - raise ValueError("A job belongs to another workflow run") - if any(job.get("run_attempt") != attempt for job in jobs): - raise ValueError("A job belongs to another workflow attempt") - common = _common_jobs(jobs, origin, source_names) - grouped = {surface.key: [] for surface in SURFACES} - for report in reports: - grouped[_surface_for(report).key].append(report) - pairs, comparisons = {}, {} - bound_ids = set() - for surface in SURFACES: - subset = grouped[surface.key] - baseline = [report for report in subset if report.variant == "baseline"] - proposed = [report for report in subset if report.variant == "proposed"] - if len(baseline) != 1 or baseline[0].shard_count != 1 or len(proposed) != 2: - raise ValueError(f"Expected the exact one-versus-two experiment for {surface.key}") - result = compare(baseline, proposed, expected_sha, minimum_lane_gain) - if result["identity"]["environment"]["cpu_count"] != surface.cpus: - raise ValueError(f"Unexpected runner capacity for {surface.key}") - root = cohorts / surface.key - before = verify( - root, f"{surface.suite}-{attempt}-{surface.binary}", run_id, attempt, expected_sha - ) - if before != result["identity"]: - raise ValueError( - f"Identity drifted between preflight and completed tests: {surface.key}" - ) - bound = [_bind_member(root, member, jobs, surface, origin) for member in MEMBERS] - for member, job in zip(MEMBERS, bound, strict=True): - report = next(item for item in subset if f"{item.variant}-{item.shard}" == member) - start, end = _job_times(job, origin) - if report.elapsed_seconds > (end - start).total_seconds() + 2: - raise ValueError( - f"Pytest timing exceeds its actual job span: {surface.key}/{member}" - ) - if bound_ids & {job["id"] for job in bound} or len({job["id"] for job in bound}) != 3: - raise ValueError("Paired partitions must identify distinct actual execution jobs") - bound_ids.update(job["id"] for job in bound) - pairs[surface.key] = {"baseline": bound[:1], "proposed": bound[1:]} - comparisons[surface.key] = { - **result, - "actual_jobs": [ - { - "id": job["id"], - "name": job["name"], - "runner_name": job["runner_name"], - "started_at": job["started_at"], - "completed_at": job["completed_at"], - "cohort_steps": [ - step - for step in job.get("steps", []) - if "cohort" in step.get("name", "").lower() - or "matched actual" in step.get("name", "").lower() - ], - } - for job in bound - ], - } - if bound_ids & {job["id"] for job in common}: - raise ValueError("A paired execution was also counted as unchanged work") - authorities, gate_jobs = _native_authorities(jobs, pairs, origin) - scenarios = {} - proposals = { - "baseline": set(), - "arm_only": {"arm-integration"}, - "arm_and_linux": {"arm-integration", "linux-integration"}, - "combined": set(pairs), - } - for name, enabled in proposals.items(): - selected = [ - job - for key, pair in pairs.items() - for job in pair["proposed" if key in enabled else "baseline"] - ] - scenarios[name] = _scenario(common, selected, origin, authorities, gate_jobs, enabled) - before = scenarios["baseline"]["evidence_ready_seconds"] - after = scenarios["combined"]["evidence_ready_seconds"] - if before <= 0: - raise ValueError("Baseline must contain positive observed elapsed time") - gain = 1 - after / before - meets_lanes = all(result["meets_threshold"] for result in comparisons.values()) - return { - "schema_version": 1, - "run_id": run_id, - "run_attempt": attempt, - "source_sha": expected_sha, - "scope": "Job-timestamp reconstruction with observed native fan-in delays propagated " - "through each proposed DAG; " - "not actual production release or the all-variants rehearsal workflow duration", - "origin": origin.isoformat(), - "origin_kind": "workflow creation" if attempt == 1 else "attempt start", - "common_job_count": len(common), - "common_jobs": [job["name"] for job in common], - "comparisons": comparisons, - "scenarios": scenarios, - "saved_seconds": before - after, - "overall_gain_fraction": gain, - "minimum_overall_gain": minimum_overall_gain, - "meets_lane_thresholds": meets_lanes, - "meets_minimum": meets_lanes and gain >= minimum_overall_gain, - "reaches_30_percent_target": meets_lanes and gain >= 0.30, - "unchanged_tail_sensitivity": [ - {"tail_seconds": tail, "gain_fraction": (before - after) / (before + tail)} - for tail in (0, 60, 120, 180) - ], - "limitations": [ - "Secret-bearing acceptance and Windows signing are not represented.", - "Native integration/native-gate delays are observed then held constant across modeled worlds; " - "their proposed completion timestamps are not independently measured.", - "Only experiment-specific both-world fan-ins are excluded; production native gates are retained.", - "Recording qualification, verifying publication assets and publishing are excluded.", - "Both worlds share one run; extra experimental concurrency and cohort waits can affect allocation.", - "JUnit case durations include fixtures and waiting, not fixed CPU work.", - ], - } - - -def main() -> int: - """Emit a matrix or durable strict rehearsal result; missed targets remain visible.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="operation", required=True) - commands.add_parser("matrix") - check = commands.add_parser("compare") - check.add_argument("--jobs", type=Path, required=True) - check.add_argument("--reports", type=Path, required=True) - check.add_argument("--cohorts", type=Path, required=True) - check.add_argument("--expected-sha", required=True) - check.add_argument("--minimum-lane-gain", type=float, default=0.15) - check.add_argument("--minimum-overall-gain", type=float, default=0.10) - check.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - try: - if args.operation == "matrix": - catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) - print("matrix=" + json.dumps(rehearsal_matrix(catalog), separators=(",", ":"))) - return 0 - result = evaluate( - json.loads(args.jobs.read_text("utf-8")), - [read_report(path) for path in sorted(args.reports.rglob("*.json"))], - args.cohorts, - args.expected_sha, - source_authorities(), - args.minimum_lane_gain, - args.minimum_overall_gain, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", "ascii") - except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as error: - parser.error(f"Invalid release rehearsal: {error}") - if not result["meets_minimum"]: - parser.exit(1, f"Rehearsal target missed; inspect {args.output}\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/release_wallclock.py b/scripts/release_wallclock.py deleted file mode 100644 index 9dd3e294bd..0000000000 --- a/scripts/release_wallclock.py +++ /dev/null @@ -1,1428 +0,0 @@ -"""Capture and compare read-only release wall-clock evidence. - -This module is deliberately not release qualification code. It verifies -artifacts, emits a non-promotable proof, and compares two already-successful -GitHub Actions attempts by their real terminal job timestamps. Native runner -comparability is assessed separately from that clock arithmetic. -""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import platform -import re -import shutil -import stat -import subprocess -import tarfile -import zipfile -from dataclasses import dataclass -from datetime import datetime, timezone -from email.parser import Parser -from pathlib import Path -from typing import Any - -import tomllib - -from scripts.package_release import ( - BINARY_NAMES, - archive_name, - executable_name, - file_digest, - require_member_path, - require_sha, -) - -ABI_VERSION = 1 -KIND_JOB = "release-wallclock-job" -KIND_PROOF = "release-wallclock-proof" -KIND_COMPARISON = "release-wallclock-comparison" -TERMINAL_JOB = "Verify wall-clock artifacts" -WORKFLOW_PATH = ".github/workflows/build-release.yml" -SIDE_CHOICES = ("baseline", "proposed") -BAD_JOB_CONCLUSIONS = {"failure", "cancelled", "timed_out", "action_required"} -SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") -# Workflow digests are validated against each side, not compared across a DAG change. -RUNNER_COMPARISON_FIELDS = ( - "system", - "kernel", - "arch", - "cpu", - "cpu_count", - "image_version", - "python", - "source_uv_lock_sha256", -) - - -@dataclass(frozen=True) -class ArchiveEvidence: - """Read-only proof for one native archive.""" - - binary_name: str - archive: str - archive_sha256: str - executable_sha256: str - sidecar: str - candidate_metadata_sha256: str | None - loose_executable_sidecar_sha256: str | None - loose_executable_sha256: str | None - member_count: int - - def payload(self) -> dict[str, object]: - """Return a JSON-safe archive proof.""" - return { - "binary_name": self.binary_name, - "archive": self.archive, - "archive_sha256": self.archive_sha256, - "executable_sha256": self.executable_sha256, - "sidecar": self.sidecar, - "candidate_metadata_sha256": self.candidate_metadata_sha256, - "loose_executable_sidecar_sha256": self.loose_executable_sidecar_sha256, - "loose_executable_sha256": self.loose_executable_sha256, - "member_count": self.member_count, - } - - -def _json(path: Path) -> Any: - return json.loads(path.read_text(encoding="utf-8")) - - -def _write_json(path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="ascii") - - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - -def _timestamp(value: object, label: str) -> datetime: - if not isinstance(value, str) or not value: - raise ValueError(f"{label} timestamp is required") - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - if parsed.tzinfo is None: - raise ValueError(f"{label} timestamp must include a timezone") - return parsed - - -def _seconds_between(start: datetime, end: datetime, label: str) -> float: - value = (end - start).total_seconds() - if not math.isfinite(value) or value < 0: - raise ValueError(f"{label} has an invalid negative timeline") - return value - - -def _positive_int(value: object, label: str) -> int: - if isinstance(value, bool): - raise ValueError(f"{label} must be a positive integer") - try: - result = int(str(value)) - except ValueError as error: - raise ValueError(f"{label} must be a positive integer") from error - if result < 1 or str(result) != str(value): - raise ValueError(f"{label} must be a positive integer") - return result - - -def _full_sha(value: object, label: str) -> str: - if not isinstance(value, str): - raise ValueError(f"{label} must be a full lowercase Git SHA") - require_sha(value) - return value - - -def _side(value: str) -> str: - if value not in SIDE_CHOICES: - raise ValueError("side must be baseline or proposed") - return value - - -def _source_metadata(source_root: Path) -> tuple[str, str]: - data = tomllib.loads((source_root / "pyproject.toml").read_text(encoding="utf-8")) - name = data["project"]["name"] - version = data["project"]["version"] - if not isinstance(name, str) or not name: - raise ValueError("source pyproject.toml must contain project.name") - if not isinstance(version, str) or not version: - raise ValueError("source pyproject.toml must contain project.version") - return name, version - - -def _source_version(source_root: Path) -> str: - return _source_metadata(source_root)[1] - - -def _canonical_project_name(value: str) -> str: - """Normalize distribution names using the wheel/sdist comparison form.""" - if not value: - raise ValueError("Distribution name must be nonempty") - return re.sub(r"[-_.]+", "-", value).lower() - - -def _distribution_token(value: str) -> str: - return _canonical_project_name(value).replace("-", "_") - - -def _is_sha256(value: object) -> bool: - return isinstance(value, str) and SHA256_PATTERN.fullmatch(value) is not None - - -def _source_digests(source_root: Path) -> dict[str, str]: - files = { - "uv.lock": source_root / "uv.lock", - WORKFLOW_PATH: source_root / WORKFLOW_PATH, - } - missing = [name for name, path in files.items() if not path.is_file()] - if missing: - raise FileNotFoundError(f"Missing required source files: {', '.join(missing)}") - return {name: file_digest(path) for name, path in files.items()} - - -def _git_head(source_root: Path) -> str: - git = shutil.which("git") - if git is None: - raise OSError("git executable is required") - result = subprocess.run( # noqa: S603 - constant git executable, no shell. - [git, "-C", str(source_root), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - encoding="ascii", - timeout=30, - ) - return result.stdout.strip() - - -def _workflow_env(source_sha: str) -> dict[str, object]: - controller_sha = _full_sha(os.environ.get("GITHUB_SHA"), "GITHUB_SHA") - event_name, run_head_sha = _event_run_head_sha(controller_sha) - image = os.environ.get("ImageVersion", "") # noqa: SIM112 - GitHub-hosted image contract. - if not image: - raise ValueError("GitHub hosted runner ImageVersion is required") - runner_name = os.environ.get("RUNNER_NAME", "") - github_job = os.environ.get("GITHUB_JOB", "") - if not runner_name or not github_job: - raise ValueError("GITHUB_JOB and RUNNER_NAME are required") - return { - "source_sha": source_sha, - "controller_sha": controller_sha, - "run_head_sha": run_head_sha, - "event_name": event_name, - "run_id": _positive_int(os.environ.get("GITHUB_RUN_ID"), "GITHUB_RUN_ID"), - "run_attempt": _positive_int(os.environ.get("GITHUB_RUN_ATTEMPT"), "GITHUB_RUN_ATTEMPT"), - "github_job": github_job, - "runner_name": runner_name, - } - - -def _event_payload() -> tuple[str, dict[str, object]]: - event_name = os.environ.get("GITHUB_EVENT_NAME", "") - event_path = os.environ.get("GITHUB_EVENT_PATH", "") - if not event_name or not event_path: - raise ValueError("GITHUB_EVENT_NAME and GITHUB_EVENT_PATH are required") - payload = _json(Path(event_path)) - if not isinstance(payload, dict): - raise ValueError("GitHub event payload must be a JSON object") - return event_name, payload - - -def _event_run_head_sha(controller_sha: str) -> tuple[str, str]: - """Capture immutable event/API run head separately from execution GITHUB_SHA.""" - event_name, payload = _event_payload() - if event_name in {"pull_request", "pull_request_target"}: - pull_request = payload.get("pull_request") - if not isinstance(pull_request, dict): - raise ValueError("pull_request event payload is missing pull_request") - head = pull_request.get("head") - if not isinstance(head, dict): - raise ValueError("pull_request event payload is missing pull_request.head") - return event_name, _full_sha(head.get("sha"), "pull_request.head.sha") - if event_name == "push": - after = _full_sha(payload.get("after"), "push after") - if after != controller_sha: - raise ValueError("push payload after does not match execution GITHUB_SHA") - return event_name, after - # Non-PR, non-push read-only probes do not have a distinct immutable PR - # head. Bind API run.head_sha to the execution SHA for those event types. - return event_name, controller_sha - - -def _host_environment(source_digests: dict[str, str]) -> dict[str, str]: - cpu_count = os.cpu_count() - if cpu_count is None: - raise ValueError("CPU count is required") - cpu = platform.processor() or platform.machine() - if not cpu: - raise ValueError("CPU identity is required") - return { - "system": platform.system(), - "kernel": platform.release(), - "arch": platform.machine(), - "cpu": cpu, - "cpu_count": str(cpu_count), - "image_version": os.environ["ImageVersion"], # noqa: SIM112 - GitHub-hosted image contract. - "python": platform.python_version(), - "source_uv_lock_sha256": source_digests["uv.lock"], - "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], - } - - -def record_job(side: str, source_root: Path, source_sha: str) -> dict[str, object]: - """Capture a minimal hosted-runner identity for one read-only job.""" - side = _side(side) - source_root = source_root.resolve() - source_sha = _full_sha(source_sha, "source_sha") - actual = _git_head(source_root) - if actual != source_sha: - raise ValueError(f"source-root HEAD {actual} does not match expected {source_sha}") - source_digests = _source_digests(source_root) - workflow = _workflow_env(source_sha) - return { - "kind": KIND_JOB, - "schema_version": ABI_VERSION, - "side": side, - "recorded_at": _utc_now(), - "source_root": str(source_root), - "workflow": workflow, - "environment": _host_environment(source_digests), - } - - -def _walk_files(root: Path) -> list[Path]: - if not root.exists(): - raise FileNotFoundError(root) - return sorted(path for path in root.rglob("*") if path.is_file()) - - -def _find_one(root: Path, name: str) -> Path: - matches = [path for path in _walk_files(root) if path.name == name] - if len(matches) != 1: - raise ValueError(f"Expected exactly one {name} under {root}, found {len(matches)}") - return matches[0] - - -def _find_optional(root: Path, name: str) -> Path | None: - matches = [path for path in _walk_files(root) if path.name == name] - if len(matches) > 1: - raise ValueError(f"Expected at most one {name} under {root}, found {len(matches)}") - return matches[0] if matches else None - - -def _read_checksum_sidecar(path: Path, archive: str, digest: str) -> None: - line = path.read_text(encoding="ascii") - match = re.fullmatch(r"([0-9a-fA-F]{64})\s+\*?([A-Za-z0-9_.-]+)\n?", line) - if match is None: - raise ValueError(f"Malformed checksum sidecar: {path.name}") - observed, filename = match.group(1).lower(), match.group(2) - if observed != digest or filename != archive: - raise ValueError(f"Checksum sidecar mismatch: {path.name}") - - -def _parse_loose_checksum(path: Path, binary_name: str) -> tuple[str, str]: - line = path.read_text(encoding="ascii") - match = re.fullmatch(r"([0-9a-fA-F]{64})\s+\*?([A-Za-z0-9_.\-/]+)\n?", line) - if match is None: - raise ValueError(f"Malformed executable checksum sidecar: {path.name}") - digest, member = match.group(1).lower(), match.group(2) - archive_member = member.removeprefix("dist/") - require_member_path(archive_member, binary_name) - if archive_member != f"{binary_name}/{executable_name(binary_name)}": - raise ValueError(f"Executable checksum sidecar targets {member}") - return digest, member - - -def _normal_archive_parts(value: str) -> list[str] | None: - if ( - not value - or value.startswith("/") - or "\\" in value - or ":" in value - or value.startswith("//") - ): - return None - parts: list[str] = [] - for part in value.split("/"): - if part in ("", "."): - continue - if part == "..": - if not parts: - return None - parts.pop() - else: - parts.append(part) - return parts - - -def _link_is_safe(binary_name: str, member: str, target: str, *, relative_to_parent: bool) -> bool: - if target.startswith("/") or "\\" in target or ":" in target: - return False - base = str(Path(member).parent).replace("\\", "/") if relative_to_parent else "" - candidate = f"{base}/{target}" if base else target - parts = _normal_archive_parts(candidate) - if parts is None: - return False - return bool(parts) and parts[0] == binary_name - - -def _zip_mode(item: zipfile.ZipInfo) -> int: - return item.external_attr >> 16 - - -def _is_zip_symlink(item: zipfile.ZipInfo) -> bool: - return stat.S_ISLNK(_zip_mode(item)) - - -def _zip_member_digest(archive: Path, binary_name: str) -> tuple[str, int]: - executable = f"{binary_name}/{executable_name(binary_name)}" - seen: set[str] = set() - executable_digest: str | None = None - with zipfile.ZipFile(archive) as source: - members = source.infolist() - if not members: - raise ValueError(f"{archive.name} is empty") - for item in members: - require_member_path(item.filename, binary_name) - if item.filename in seen: - raise ValueError(f"Duplicate archive member: {item.filename}") - seen.add(item.filename) - if _is_zip_symlink(item): - raise ValueError("ZIP archive contains an unsupported symbolic link") - if item.filename == executable: - mode = _zip_mode(item) - if item.is_dir() or (mode and not stat.S_ISREG(mode)): - raise ValueError(f"{executable} must be a regular file") - if executable_name(binary_name) == "apm" and mode and not (mode & 0o111): - raise ValueError(f"{executable} must be executable") - with source.open(item) as stream: - executable_digest = file_digest_stream(stream) - if executable_digest is None: - raise ValueError(f"{archive.name} must contain exactly one {executable}") - return executable_digest, len(seen) - - -def file_digest_stream(stream: Any) -> str: - """Hash an already-open binary stream without extracting it to disk.""" - import hashlib - - digest = hashlib.sha256() - while chunk := stream.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - -def _tar_member_digest(archive: Path, binary_name: str) -> tuple[str, int]: - executable = f"{binary_name}/{executable_name(binary_name)}" - seen: set[str] = set() - executable_digest: str | None = None - with tarfile.open(archive, "r:gz") as source: - members = source.getmembers() - if not members: - raise ValueError(f"{archive.name} is empty") - for item in members: - require_member_path(item.name, binary_name) - if item.name in seen: - raise ValueError(f"Duplicate archive member: {item.name}") - seen.add(item.name) - if item.issym() and not _link_is_safe( - binary_name, item.name, item.linkname, relative_to_parent=True - ): - raise ValueError("Archive symbolic link escapes its native bundle") - if item.islnk() and not _link_is_safe( - binary_name, item.name, item.linkname, relative_to_parent=False - ): - raise ValueError("Archive hard link escapes its native bundle") - if item.name == executable: - if not item.isfile(): - raise ValueError(f"{executable} must be a regular file") - if not (item.mode & 0o111): - raise ValueError(f"{executable} must be executable") - stream = source.extractfile(item) - if stream is None: - raise ValueError(f"Cannot read {executable}") - with stream: - executable_digest = file_digest_stream(stream) - if executable_digest is None: - raise ValueError(f"{archive.name} must contain exactly one {executable}") - return executable_digest, len(seen) - - -def _candidate_metadata( - metadata_root: Path, - side: str, - binary_name: str, - archive: str, - source_sha: str, - version: str, - archive_digest: str, - executable_digest: str, -) -> str | None: - path = _find_optional(metadata_root, f"{binary_name}.json") - if path is None: - if side == "proposed": - raise ValueError(f"Proposed proof requires candidate metadata for {binary_name}") - return None - data = _json(path) - required = { - "schema_version": 1, - "sha": source_sha, - "version": version, - "binary_name": binary_name, - "archive": archive, - "archive_sha256": archive_digest, - "executable_sha256": executable_digest, - } - for key, expected in required.items(): - if data.get(key) != expected: - raise ValueError(f"Candidate metadata {binary_name} {key} mismatch") - return file_digest(path) - - -def _loose_path(raw_root: Path, sidecar: Path, member: str) -> Path | None: - candidates = [ - raw_root / member, - sidecar.parent / member, - sidecar.parent / member.removeprefix("dist/"), - ] - for candidate in candidates: - if candidate.is_file(): - return candidate - return None - - -def _loose_executable_sidecar( - raw_root: Path, side: str, binary_name: str, executable_digest: str -) -> tuple[str | None, str | None]: - path = _find_optional(raw_root, f"{binary_name}.sha256") - if path is None: - if side == "baseline": - raise ValueError(f"Baseline raw executable checksum is missing: {binary_name}.sha256") - return None, None - expected, member = _parse_loose_checksum(path, binary_name) - if expected != executable_digest: - raise ValueError(f"Loose executable checksum mismatch: {path.name}") - loose = _loose_path(raw_root, path, member) - if loose is None: - if side == "baseline": - raise ValueError(f"Baseline raw executable payload is missing: {member}") - return file_digest(path), None - loose_digest = file_digest(loose) - if loose_digest != executable_digest: - raise ValueError(f"Loose executable digest mismatch: {loose}") - return file_digest(path), loose_digest - - -def _verify_native_archives( - side: str, native_root: Path, source_sha: str, version: str -) -> dict[str, dict[str, object]]: - archives_root = native_root / "archives" if (native_root / "archives").is_dir() else native_root - raw_root = native_root / "raw" if (native_root / "raw").is_dir() else native_root - inventory: dict[str, dict[str, object]] = {} - for binary_name in BINARY_NAMES: - archive = archive_name(binary_name) - archive_path = _find_one(archives_root, archive) - sidecar_path = _find_one(archives_root, f"{archive}.sha256") - archive_digest = file_digest(archive_path) - _read_checksum_sidecar(sidecar_path, archive, archive_digest) - if archive.endswith(".zip"): - executable_digest, member_count = _zip_member_digest(archive_path, binary_name) - else: - executable_digest, member_count = _tar_member_digest(archive_path, binary_name) - metadata_digest = _candidate_metadata( - archives_root, - side, - binary_name, - archive, - source_sha, - version, - archive_digest, - executable_digest, - ) - loose_sidecar_digest, loose_executable_digest = _loose_executable_sidecar( - raw_root, side, binary_name, executable_digest - ) - inventory[binary_name] = ArchiveEvidence( - binary_name=binary_name, - archive=archive, - archive_sha256=archive_digest, - executable_sha256=executable_digest, - sidecar=f"{archive}.sha256", - candidate_metadata_sha256=metadata_digest, - loose_executable_sidecar_sha256=loose_sidecar_digest, - loose_executable_sha256=loose_executable_digest, - member_count=member_count, - ).payload() - _reject_unexpected_native_release_files(archives_root, side) - return inventory - - -def _reject_unexpected_native_release_files(native_root: Path, side: str) -> None: - expected: set[str] = set() - for binary_name in BINARY_NAMES: - archive = archive_name(binary_name) - expected.update({archive, f"{archive}.sha256", f"{binary_name}.sha256"}) - if side == "proposed": - expected.add(f"{binary_name}.json") - release_like = [ - path.name - for path in _walk_files(native_root) - if path.name.startswith("apm-") - and re.search(r"\.(tar\.gz|zip|sha256|json)$", path.name) - and path.name not in expected - ] - if release_like: - raise ValueError(f"Unexpected native release files: {', '.join(sorted(release_like))}") - - -def _docs_index(docs_root: Path) -> dict[str, object]: - root_index = docs_root / "index.html" - if root_index.is_file(): - if root_index.stat().st_size <= 0: - raise ValueError("docs index.html is empty") - return {"format": "directory", "index": "index.html", "sha256": file_digest(root_index)} - tars = [ - path - for path in _walk_files(docs_root) - if path.suffix == ".tar" or path.name.endswith(".tar.gz") - ] - if len(tars) != 1: - raise ValueError("Docs artifact must contain index.html or exactly one tar archive") - with tarfile.open(tars[0], "r:*") as source: - matches = [ - item - for item in source.getmembers() - if item.name.removeprefix("./").rstrip("/") == "index.html" - ] - if len(matches) != 1: - raise ValueError("Docs artifact tar must contain exactly one root index.html") - index = matches[0] - if not index.isfile() or index.size <= 0: - raise ValueError("Docs artifact index.html must be a nonempty regular file") - return { - "format": "tar", - "archive": tars[0].name, - "archive_sha256": file_digest(tars[0]), - "index": "index.html", - } - - -def _metadata_value(blob: bytes, key: str) -> str: - parsed = Parser().parsestr(blob.decode("utf-8")) - value = parsed.get(key) - if not value: - raise ValueError(f"Distribution metadata is missing {key}") - return value - - -def _wheel_distribution(path: Path, project_name: str, version: str) -> dict[str, object]: - expected_name = _canonical_project_name(project_name) - expected_token = _distribution_token(project_name) - if not path.name.startswith(f"{expected_token}-{version}-") or not path.name.endswith(".whl"): - raise ValueError(f"Wheel filename does not match source project/version: {path.name}") - with zipfile.ZipFile(path) as wheel: - names = wheel.namelist() - metadata = [ - name for name in names if name.endswith(".dist-info/METADATA") and name.count("/") == 1 - ] - if len(metadata) != 1: - raise ValueError(f"Wheel must contain exactly one METADATA: {path.name}") - dist_info = metadata[0].removesuffix("/METADATA") - required = {f"{dist_info}/METADATA", f"{dist_info}/WHEEL", f"{dist_info}/RECORD"} - if not required.issubset(names): - raise ValueError(f"Wheel is missing METADATA, WHEEL or RECORD: {path.name}") - if not dist_info.startswith(f"{expected_token}-{version}.dist-info"): - raise ValueError(f"Wheel dist-info directory does not match source: {path.name}") - if not any( - name.startswith(f"{expected_token}/") and not name.endswith("/") for name in names - ): - raise ValueError(f"Wheel does not contain source package code: {path.name}") - with wheel.open(f"{dist_info}/METADATA") as stream: - metadata_blob = stream.read() - with wheel.open(f"{dist_info}/WHEEL") as stream: - if not stream.read().strip(): - raise ValueError(f"Wheel WHEEL metadata is empty: {path.name}") - with wheel.open(f"{dist_info}/RECORD") as stream: - if not stream.read().strip(): - raise ValueError(f"Wheel RECORD metadata is empty: {path.name}") - name = _metadata_value(metadata_blob, "Name") - found_version = _metadata_value(metadata_blob, "Version") - if _canonical_project_name(name) != expected_name or found_version != version: - raise ValueError("Wheel metadata name/version does not match source pyproject") - return { - "name": path.name, - "sha256": file_digest(path), - "project_name": name, - "version": found_version, - "dist_info": dist_info, - } - - -def _sdist_distribution(path: Path, project_name: str, version: str) -> dict[str, object]: - expected_name = _canonical_project_name(project_name) - expected_token = _distribution_token(project_name) - expected_root = f"{expected_token}-{version}" - if path.name != f"{expected_root}.tar.gz": - raise ValueError(f"sdist filename does not match source project/version: {path.name}") - with tarfile.open(path, "r:gz") as sdist: - members = sdist.getmembers() - names = [item.name.rstrip("/") for item in members if item.name.rstrip("/")] - roots = {name.split("/", 1)[0] for name in names} - if roots != {expected_root}: - raise ValueError(f"sdist top-level directory does not match source: {path.name}") - metadata = [item for item in members if item.name == f"{expected_root}/PKG-INFO"] - if len(metadata) != 1: - raise ValueError(f"sdist must contain exactly one top-level PKG-INFO: {path.name}") - pyprojects = [item for item in members if item.name == f"{expected_root}/pyproject.toml"] - if len(pyprojects) != 1 or not pyprojects[0].isfile() or pyprojects[0].size <= 0: - raise ValueError(f"sdist must contain top-level pyproject.toml: {path.name}") - if not any( - item.isfile() - and ( - item.name.startswith(f"{expected_root}/src/{expected_token}/") - or item.name.startswith(f"{expected_root}/{expected_token}/") - ) - for item in members - ): - raise ValueError(f"sdist does not contain source package code: {path.name}") - stream = sdist.extractfile(metadata[0]) - if stream is None: - raise ValueError(f"Cannot read sdist metadata: {path.name}") - with stream: - metadata_blob = stream.read() - name = _metadata_value(metadata_blob, "Name") - found_version = _metadata_value(metadata_blob, "Version") - if _canonical_project_name(name) != expected_name or found_version != version: - raise ValueError("sdist metadata name/version does not match source pyproject") - return { - "name": path.name, - "sha256": file_digest(path), - "project_name": name, - "version": found_version, - "root": expected_root, - } - - -def _python_distributions(python_root: Path, project_name: str, version: str) -> dict[str, object]: - wheels = [path for path in _walk_files(python_root) if path.suffix == ".whl"] - sdists = [ - path - for path in _walk_files(python_root) - if path.name.endswith(".tar.gz") and not path.name.startswith("apm-") - ] - if len(wheels) != 1 or len(sdists) != 1: - raise ValueError( - f"Expected exactly one wheel and one sdist, found {len(wheels)} wheel(s) and {len(sdists)} sdist(s)" - ) - return { - "source_project_name": project_name, - "wheel": _wheel_distribution(wheels[0], project_name, version), - "sdist": _sdist_distribution(sdists[0], project_name, version), - } - - -def _job_records( - roots: list[Path], side: str, source_sha: str, workflow: dict[str, object] -) -> list[dict[str, object]]: - records = [] - for root in roots: - for path in _walk_files(root): - if path.suffix != ".json": - continue - data = _json(path) - if not isinstance(data, dict) or data.get("kind") != KIND_JOB: - continue - if data.get("side") != side: - raise ValueError(f"Wall-clock job record side mismatch: {path}") - record_workflow = data.get("workflow") - if not isinstance(record_workflow, dict): - raise ValueError(f"Wall-clock job record workflow malformed: {path}") - for key in ("source_sha", "run_id", "run_attempt", "controller_sha", "run_head_sha"): - if record_workflow.get(key) != workflow[key]: - raise ValueError(f"Wall-clock job record {key} mismatch: {path}") - if record_workflow.get("source_sha") != source_sha: - raise ValueError(f"Wall-clock job record source mismatch: {path}") - records.append( - { - "path": str(path), - "github_job": record_workflow.get("github_job"), - "runner_name": record_workflow.get("runner_name"), - "schema_version": data.get("schema_version"), - "recorded_at": data.get("recorded_at"), - "environment": data.get("environment"), - } - ) - _validate_job_records({"side": side, "job_records": records}, {"job_records": len(records)}) - return records - - -def verify_artifacts( - side: str, - source_root: Path, - source_sha: str, - native_root: Path, - docs_root: Path, - python_root: Path, -) -> dict[str, object]: - """Verify downloaded read-only artifacts and emit a non-promotable proof.""" - side = _side(side) - source_root = source_root.resolve() - source_sha = _full_sha(source_sha, "source_sha") - actual = _git_head(source_root) - if actual != source_sha: - raise ValueError(f"source-root HEAD {actual} does not match expected {source_sha}") - source_digests = _source_digests(source_root) - source_name, version = _source_metadata(source_root) - workflow = _workflow_env(source_sha) - native = _verify_native_archives(side, native_root, source_sha, version) - docs = _docs_index(docs_root) - python_dists = _python_distributions(python_root, source_name, version) - records = _job_records([native_root, docs_root, python_root], side, source_sha, workflow) - return { - "kind": KIND_PROOF, - "schema_version": ABI_VERSION, - "abi": "scripts.release_wallclock.v1", - "promotable": False, - "side": side, - "source_sha": source_sha, - "controller_sha": workflow["controller_sha"], - "run_head_sha": workflow["run_head_sha"], - "event_name": workflow["event_name"], - "run_id": workflow["run_id"], - "run_attempt": workflow["run_attempt"], - "version": version, - "workflow_digest": source_digests[WORKFLOW_PATH], - "filehashes": { - "source_uv_lock_sha256": source_digests["uv.lock"], - "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], - "native": native, - "docs": docs, - "python": python_dists, - }, - "counts": { - "native_archives": len(native), - "native_sidecars": len(BINARY_NAMES), - "candidate_metadata": sum( - item["candidate_metadata_sha256"] is not None for item in native.values() - ), - "job_records": len(records), - "docs_indexes": 1, - "python_distributions": 2, - }, - "job_records": records, - "observation_assessment": _validate_job_records( - { - "side": side, - "job_records": records, - "filehashes": { - "source_uv_lock_sha256": source_digests["uv.lock"], - "source_build_release_yml_sha256": source_digests[WORKFLOW_PATH], - }, - }, - {"job_records": len(records)}, - ), - "captured_at": _utc_now(), - "scope": "read-only wall-clock artifact proof; not release promotion evidence", - } - - -def _require_sha256(value: object, label: str) -> str: - if not _is_sha256(value): - raise ValueError(f"{label} must be a lowercase sha256 hex digest") - return str(value) - - -def _validate_native_proof(native: object, side: str, candidate_metadata_count: object) -> None: - if not isinstance(native, dict) or set(native) != set(BINARY_NAMES): - raise ValueError("Wall-clock proof native inventory must contain exactly five platforms") - expected_metadata = 5 if side == "proposed" else 0 - if candidate_metadata_count != expected_metadata: - raise ValueError(f"{side} candidate metadata count is malformed") - for binary_name in BINARY_NAMES: - item = native.get(binary_name) - if not isinstance(item, dict): - raise ValueError(f"Native proof entry is malformed: {binary_name}") - archive = archive_name(binary_name) - if item.get("binary_name") != binary_name or item.get("archive") != archive: - raise ValueError(f"Native proof identity mismatch: {binary_name}") - if item.get("sidecar") != f"{archive}.sha256": - raise ValueError(f"Native proof sidecar mismatch: {binary_name}") - _require_sha256(item.get("archive_sha256"), f"{binary_name} archive_sha256") - _require_sha256(item.get("executable_sha256"), f"{binary_name} executable_sha256") - if not isinstance(item.get("member_count"), int) or item["member_count"] <= 0: - raise ValueError(f"Native proof member count is malformed: {binary_name}") - metadata_digest = item.get("candidate_metadata_sha256") - loose_sidecar = item.get("loose_executable_sidecar_sha256") - loose_executable = item.get("loose_executable_sha256") - if side == "proposed": - _require_sha256(metadata_digest, f"{binary_name} candidate_metadata_sha256") - elif metadata_digest is not None: - raise ValueError(f"Baseline proof must not carry candidate metadata: {binary_name}") - if side == "baseline": - _require_sha256(loose_sidecar, f"{binary_name} loose_executable_sidecar_sha256") - if loose_executable != item.get("executable_sha256"): - raise ValueError(f"Baseline raw executable digest mismatch: {binary_name}") - elif loose_sidecar is not None: - _require_sha256(loose_sidecar, f"{binary_name} loose_executable_sidecar_sha256") - if loose_executable is not None: - _require_sha256(loose_executable, f"{binary_name} loose_executable_sha256") - - -def _validate_docs_proof(docs: object) -> None: - if not isinstance(docs, dict) or docs.get("index") != "index.html": - raise ValueError("Docs proof is malformed") - if docs.get("format") == "directory": - _require_sha256(docs.get("sha256"), "docs sha256") - elif docs.get("format") == "tar": - if not isinstance(docs.get("archive"), str) or not docs["archive"]: - raise ValueError("Docs proof tar archive name is malformed") - _require_sha256(docs.get("archive_sha256"), "docs archive_sha256") - else: - raise ValueError("Docs proof format is malformed") - - -def _validate_python_proof(python_proof: object) -> None: - if not isinstance(python_proof, dict): - raise ValueError("Python distribution proof is malformed") - source_name = python_proof.get("source_project_name") - if not isinstance(source_name, str) or not source_name: - raise ValueError("Python proof source project name is missing") - canonical = _canonical_project_name(source_name) - wheel = python_proof.get("wheel") - sdist = python_proof.get("sdist") - for label, item in (("wheel", wheel), ("sdist", sdist)): - if not isinstance(item, dict): - raise ValueError(f"Python {label} proof is malformed") - if not isinstance(item.get("name"), str) or not item["name"]: - raise ValueError(f"Python {label} filename is missing") - _require_sha256(item.get("sha256"), f"Python {label} sha256") - if _canonical_project_name(str(item.get("project_name") or "")) != canonical: - raise ValueError(f"Python {label} project name does not match source") - if not isinstance(item.get("version"), str) or not item["version"]: - raise ValueError(f"Python {label} version is missing") - if wheel["version"] != sdist["version"]: - raise ValueError("Python wheel and sdist versions do not match") - if not str(wheel["name"]).endswith(".whl") or not str(sdist["name"]).endswith(".tar.gz"): - raise ValueError("Python distribution filenames are malformed") - - -def _observation_roles(side: str) -> dict[tuple[str, str, str], tuple[str, ...]]: - """Index the distinct observer DAGs by job ID and normalized native platform. - - Baseline macOS jobs combine unit/build/integration; other baseline platforms - combine unit/build only. Proposed jobs observe all three roles separately. - Docs, wheels, validation and terminal jobs have no paired observer contract. - """ - roles = {} - for binary_name in BINARY_NAMES: - _, system, arch = binary_name.split("-", 2) - if side == "proposed": - for job, role in ( - ("unit-tests", "unit"), - ("build", "build"), - ("integration-tests", "integration"), - ): - roles[(job, system, arch)] = (f"{role}/{binary_name}",) - elif system == "darwin": - job = ( - "build-and-validate-macos-intel" - if arch == "x86_64" - else "build-and-validate-macos-arm" - ) - roles[(job, system, arch)] = tuple( - f"{role}/{binary_name}" for role in ("unit", "build", "integration") - ) - else: - roles[("build-and-test", system, arch)] = ( - f"unit/{binary_name}", - f"build/{binary_name}", - ) - roles[("integration-tests", system, arch)] = (f"integration/{binary_name}",) - return roles - - -def _validate_job_records(proof: dict[str, object], counts: dict[str, object]) -> dict[str, object]: - """Assess observations in O(N) time using fixed-size role expansion. - - Broken identities remain errors. Missing/invalid measurement metadata leaves - historical clocks readable, but cannot establish runner comparability. - """ - records = proof.get("job_records") - if not isinstance(records, list) or not records: - raise ValueError("Wall-clock proof must contain at least one job record") - if counts.get("job_records") != len(records): - raise ValueError("Wall-clock proof job record count does not match") - expected = _observation_roles(_side(str(proof.get("side")))) - expected_roles = {role for roles in expected.values() for role in roles} - observed: dict[str, dict[str, str]] = {} - issues: list[str] = [] - unassessed = False - fields = (*RUNNER_COMPARISON_FIELDS, "source_build_release_yml_sha256") - filehashes = proof.get("filehashes", {}) - seen: set[tuple[str, str]] = set() - for record in records: - if not isinstance(record, dict): - raise ValueError("Wall-clock job record is malformed") - github_job = record.get("github_job") - runner_name = record.get("runner_name") - if not isinstance(github_job, str) or not github_job: - raise ValueError("Wall-clock job record is missing github_job") - if not isinstance(runner_name, str) or not runner_name: - raise ValueError("Wall-clock job record is missing runner_name") - key = (github_job, runner_name) - if key in seen: - raise ValueError("Wall-clock proof contains duplicate job record identity") - seen.add(key) - label = f"{github_job}/{runner_name}" - schema = record.get("schema_version") - environment = record.get("environment") - if isinstance(schema, bool) or not isinstance(schema, int) or schema != ABI_VERSION: - issues.append(f"{label}: missing or unsupported observation schema_version") - unassessed = True - continue - try: - _timestamp(record.get("recorded_at"), "observation recorded_at") - except ValueError: - issues.append(f"{label}: missing or invalid recorded_at") - unassessed = True - continue - if not isinstance(environment, dict) or any( - not isinstance(environment.get(field), str) or not environment[field].strip() - for field in fields - ): - issues.append(f"{label}: missing or invalid environment") - unassessed = True - continue - try: - _positive_int(environment["cpu_count"], "observation cpu_count") - for field in ("source_uv_lock_sha256", "source_build_release_yml_sha256"): - _require_sha256(environment[field], field) - except ValueError: - issues.append(f"{label}: invalid environment CPU count or source digest") - unassessed = True - continue - if isinstance(filehashes, dict): - for field in ("source_uv_lock_sha256", "source_build_release_yml_sha256"): - if field in filehashes and environment[field] != filehashes[field]: - issues.append(f"{label}: environment {field} does not match proof") - system = environment["system"].lower() - arch = environment["arch"].lower() - arch = {"amd64": "x86_64", "aarch64": "arm64"}.get(arch, arch) - roles = expected.get((github_job, system, arch)) - if roles is None: - issues.append(f"{label}: unexpected observer role/platform {system}/{arch}") - continue - for role in roles: - if role in observed: - issues.append(f"{role}: duplicate observer role/platform") - else: - observed[role] = {field: environment[field] for field in RUNNER_COMPARISON_FIELDS} - observed[role].update(system=system, arch=arch) - missing = sorted(expected_roles - observed.keys()) - if missing: - issues.append(f"Missing observer roles: {', '.join(missing)}") - return { - "status": "unassessed" if unassessed else "inconclusive" if issues else "complete", - "issues": issues, - "roles": observed, - } - - -def _read_proof(path: Path, side: str, expected_sha: str) -> dict[str, object]: - proof = _json(path) - if not isinstance(proof, dict): - raise ValueError(f"Malformed proof: {path}") - if proof.get("kind") != KIND_PROOF or proof.get("schema_version") != ABI_VERSION: - raise ValueError(f"Unsupported proof schema: {path}") - if proof.get("promotable") is not False: - raise ValueError(f"Wall-clock proof must be non-promotable: {path}") - if proof.get("side") != side or proof.get("source_sha") != expected_sha: - raise ValueError(f"Proof identity mismatch: {path}") - if proof.get("run_attempt") != 1: - raise ValueError(f"Proof must describe attempt 1: {path}") - _full_sha(proof.get("controller_sha"), "proof controller_sha") - _full_sha(proof.get("source_sha"), "proof source_sha") - _full_sha(proof.get("run_head_sha"), "proof run_head_sha") - counts = proof.get("counts") - if not isinstance(counts, dict) or { - "native_archives": counts.get("native_archives"), - "native_sidecars": counts.get("native_sidecars"), - "candidate_metadata": counts.get("candidate_metadata"), - "docs_indexes": counts.get("docs_indexes"), - "python_distributions": counts.get("python_distributions"), - } != { - "native_archives": len(BINARY_NAMES), - "native_sidecars": len(BINARY_NAMES), - "candidate_metadata": 5 if side == "proposed" else 0, - "docs_indexes": 1, - "python_distributions": 2, - }: - raise ValueError(f"Malformed wall-clock proof counts: {path}") - filehashes = proof.get("filehashes") - if not isinstance(filehashes, dict): - raise ValueError(f"Malformed wall-clock proof file hashes: {path}") - workflow_digest = _require_sha256(proof.get("workflow_digest"), "proof workflow_digest") - if filehashes.get("source_build_release_yml_sha256") != workflow_digest: - raise ValueError("Proof workflow digest does not match source workflow hash") - _require_sha256(filehashes.get("source_uv_lock_sha256"), "source_uv_lock_sha256") - _validate_native_proof(filehashes.get("native"), side, counts.get("candidate_metadata")) - _validate_docs_proof(filehashes.get("docs")) - _validate_python_proof(filehashes.get("python")) - # Recompute rather than trusting an assessment saved by an older producer. - proof["observation_assessment"] = _validate_job_records(proof, counts) - return proof - - -def _read_run(path: Path) -> tuple[dict[str, object], list[dict[str, object]]]: - data = _json(path) - if ( - not isinstance(data, dict) - or not isinstance(data.get("run"), dict) - or not isinstance(data.get("jobs"), list) - or not all(isinstance(job, dict) for job in data["jobs"]) - ): - raise ValueError(f"Run JSON must be {{run, jobs}}: {path}") - return data["run"], data["jobs"] - - -def _run_id(value: object, label: str) -> int: - return _positive_int(value, label) - - -def _validate_run( - run_path: Path, - proof_path: Path, - side: str, - expected_sha: str, - expected_controller_sha: str, - expected_run_head_sha: str, - terminal_name: str, - required_names: tuple[str, ...], -) -> dict[str, object]: - if not required_names or set(required_names) == {terminal_name}: - raise ValueError(f"{side} required job set must contain non-terminal jobs") - if len(set(required_names)) != len(required_names): - raise ValueError(f"{side} required job set contains duplicates") - if terminal_name in required_names: - raise ValueError(f"{side} required job set must not include the terminal job") - run, jobs = _read_run(run_path) - proof = _read_proof(proof_path, side, expected_sha) - run_id = _run_id(run.get("id"), f"{side} run id") - attempt = _positive_int(run.get("run_attempt"), f"{side} run_attempt") - if attempt != 1: - raise ValueError(f"{side} run must be attempt 1") - if proof["run_id"] != run_id or proof["run_attempt"] != attempt: - raise ValueError(f"{side} proof run identity does not match run JSON") - if run.get("status") != "completed" or run.get("conclusion") != "success": - raise ValueError(f"{side} workflow run did not complete successfully") - if proof["controller_sha"] != expected_controller_sha: - raise ValueError(f"{side} proof controller SHA does not match expected execution SHA") - if proof["run_head_sha"] != expected_run_head_sha: - raise ValueError(f"{side} proof run head SHA does not match expected event head SHA") - if run.get("head_sha") != proof["run_head_sha"]: - raise ValueError(f"{side} API run head_sha does not match recorded event head SHA") - created = _timestamp(run.get("created_at"), f"{side} run.created_at") - started = _timestamp(run.get("run_started_at"), f"{side} run.run_started_at") - if started < created: - raise ValueError(f"{side} run_started_at is before created_at") - terminal = _select_terminal(jobs, run_id, terminal_name, required_names, side) - terminal_start = _timestamp(terminal["started_at"], f"{side} terminal.started_at") - terminal_end = _timestamp(terminal["completed_at"], f"{side} terminal.completed_at") - if terminal_end < terminal_start: - raise ValueError(f"{side} terminal job has an invalid timeline") - assessment = proof["observation_assessment"] - for record in proof["job_records"]: - try: - recorded = _timestamp(record.get("recorded_at"), "observation recorded_at") - except ValueError: - continue # Already unassessed by the metadata validator. - if not created <= recorded <= terminal_end: - if assessment["status"] != "unassessed": - assessment["status"] = "inconclusive" - assessment["issues"].append("Observation recorded_at is outside the run timeline") - executed_jobs = [] - runner_seconds = 0.0 - for job in jobs: - if job.get("conclusion") == "skipped": - continue - start = _timestamp(job.get("started_at"), f"{side} job.started_at") - end = _timestamp(job.get("completed_at"), f"{side} job.completed_at") - if end < start or start < created: - raise ValueError(f"{side} job has an invalid timeline: {job.get('name')}") - if job is not terminal and end > terminal_start: - raise ValueError(f"{side} terminal job started before {job.get('name')} completed") - executed_jobs.append(job) - runner_seconds += (end - start).total_seconds() - return { - "side": side, - "run_id": run_id, - "run_attempt": attempt, - "source_sha": expected_sha, - "controller_sha": proof["controller_sha"], - "run_head_sha": proof["run_head_sha"], - "event_name": proof.get("event_name"), - "created_at": run["created_at"], - "run_started_at": run["run_started_at"], - "terminal_job": terminal_name, - "terminal_completed_at": terminal["completed_at"], - "wallclock_seconds_from_created_at": _seconds_between( - created, terminal_end, f"{side} primary wall-clock" - ), - "wallclock_seconds_from_run_started_at": _seconds_between( - started, terminal_end, f"{side} secondary wall-clock" - ), - "unweighted_job_runner_seconds": runner_seconds, - "job_count": len(jobs), - "executed_job_count": len(executed_jobs), - "skipped_job_count": len(jobs) - len(executed_jobs), - "version": proof.get("version"), - "observation_assessment": assessment, - } - - -def _select_terminal( - jobs: list[dict[str, object]], - run_id: int, - terminal_name: str, - required_names: tuple[str, ...], - side: str, -) -> dict[str, object]: - required_seen = {name: 0 for name in required_names} - terminals = [] - for job in jobs: - if _run_id(job.get("run_id"), f"{side} job run_id") != run_id: - raise ValueError(f"{side} job belongs to a different run") - if _positive_int(job.get("run_attempt"), f"{side} job run_attempt") != 1: - raise ValueError(f"{side} job belongs to a different attempt") - if job.get("status") != "completed": - raise ValueError(f"{side} job did not complete: {job.get('name')}") - conclusion = job.get("conclusion") - if conclusion in BAD_JOB_CONCLUSIONS or conclusion not in {"success", "skipped"}: - raise ValueError(f"{side} job has rejected conclusion {conclusion}: {job.get('name')}") - if job.get("name") == terminal_name: - terminals.append(job) - if job.get("name") in required_seen: - required_seen[str(job["name"])] += 1 - if conclusion != "success": - raise ValueError(f"{side} required job did not succeed: {job.get('name')}") - if len(terminals) != 1: - raise ValueError(f"{side} expected exactly one terminal job named {terminal_name}") - terminal = terminals[0] - if terminal.get("conclusion") != "success": - raise ValueError(f"{side} terminal job did not succeed") - missing = [name for name, count in required_seen.items() if count != 1] - if missing: - raise ValueError(f"{side} required job missing or duplicated: {', '.join(missing)}") - return terminal - - -def _runner_comparability( - baseline: dict[str, object], proposed: dict[str, object] -) -> dict[str, object]: - """Compare indexed native roles without claiming campaign-level controls.""" - before = baseline["observation_assessment"] - after = proposed["observation_assessment"] - issues = [ - f"{side}: {issue}" - for side, assessment in (("baseline", before), ("proposed", after)) - for issue in assessment["issues"] - ] - for role, environment in before["roles"].items(): - other = after["roles"].get(role) - if other is not None: - for field in RUNNER_COMPARISON_FIELDS: - if environment[field] != other[field]: - issues.append(f"{role}: {field} differs between baseline and proposed") - status = "matched" - if before["status"] == "unassessed" or after["status"] == "unassessed": - status = "unassessed" - elif issues: - status = "inconclusive" - return { - "status": status, - "issues": issues, - "controlled_acceptance": False, - "scope": "native unit/build/integration runner observations only", - "limitations": [ - "Docs, wheels, source checks and validation runners are not assessed", - "Overlap, cache state, retry budget and population variance are not assessed", - "Matching native observations alone do not establish controlled acceptance", - ], - } - - -def compare_runs( - baseline_run: Path, - proposed_run: Path, - baseline_proof: Path, - proposed_proof: Path, - expected_baseline_sha: str, - expected_proposed_sha: str, - expected_baseline_controller_sha: str | None, - expected_proposed_controller_sha: str | None, - expected_baseline_run_head_sha: str, - expected_proposed_run_head_sha: str, - terminal_name: str, - baseline_required_names: tuple[str, ...], - proposed_required_names: tuple[str, ...], -) -> dict[str, object]: - """Compare two real successful workflow attempts without modeling gate lag.""" - expected_baseline_sha = _full_sha(expected_baseline_sha, "expected_baseline_sha") - expected_proposed_sha = _full_sha(expected_proposed_sha, "expected_proposed_sha") - baseline_controller = _full_sha( - expected_baseline_controller_sha or expected_baseline_sha, - "expected_baseline_controller_sha", - ) - proposed_controller = _full_sha( - expected_proposed_controller_sha or expected_proposed_sha, - "expected_proposed_controller_sha", - ) - expected_baseline_run_head_sha = _full_sha( - expected_baseline_run_head_sha, "expected_baseline_run_head_sha" - ) - expected_proposed_run_head_sha = _full_sha( - expected_proposed_run_head_sha, "expected_proposed_run_head_sha" - ) - baseline = _validate_run( - baseline_run, - baseline_proof, - "baseline", - expected_baseline_sha, - baseline_controller, - expected_baseline_run_head_sha, - terminal_name, - baseline_required_names, - ) - proposed = _validate_run( - proposed_run, - proposed_proof, - "proposed", - expected_proposed_sha, - proposed_controller, - expected_proposed_run_head_sha, - terminal_name, - proposed_required_names, - ) - if baseline["run_id"] == proposed["run_id"]: - raise ValueError("Baseline and proposed run IDs must differ") - if ( - baseline["wallclock_seconds_from_created_at"] <= 0 - or baseline["wallclock_seconds_from_run_started_at"] <= 0 - ): - raise ValueError("Baseline wall-clock denominators must be positive") - primary_gain = ( - baseline["wallclock_seconds_from_created_at"] - - proposed["wallclock_seconds_from_created_at"] - ) - secondary_gain = ( - baseline["wallclock_seconds_from_run_started_at"] - - proposed["wallclock_seconds_from_run_started_at"] - ) - return { - "kind": KIND_COMPARISON, - "schema_version": ABI_VERSION, - "readonly_adapter": True, - "production_qualification": False, - "clock_arithmetic": "validated", - "comparability": _runner_comparability(baseline, proposed), - "terminal_job": terminal_name, - "baseline_required_job_names": sorted(baseline_required_names), - "proposed_required_job_names": sorted(proposed_required_names), - "baseline": baseline, - "proposed": proposed, - "primary_metric": "terminal job completed_at minus workflow created_at", - "secondary_metric": "terminal job completed_at minus workflow run_started_at", - "delta_seconds": { - "primary_proposed_minus_baseline": -primary_gain, - "primary_gain_seconds": primary_gain, - "secondary_proposed_minus_baseline": -secondary_gain, - "secondary_gain_seconds": secondary_gain, - "unweighted_runner_seconds_proposed_minus_baseline": proposed[ - "unweighted_job_runner_seconds" - ] - - baseline["unweighted_job_runner_seconds"], - }, - "gain_fraction": { - "primary": primary_gain / baseline["wallclock_seconds_from_created_at"], - "secondary": secondary_gain / baseline["wallclock_seconds_from_run_started_at"], - }, - "gain_percent": { - "primary": 100 * primary_gain / baseline["wallclock_seconds_from_created_at"], - "secondary": 100 * secondary_gain / baseline["wallclock_seconds_from_run_started_at"], - }, - "scope": "actual successful GitHub workflow attempts; no counterfactual gate lag model", - } - - -def main() -> int: - """Command-line entry point for the release wall-clock toolkit.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="operation", required=True) - - record = commands.add_parser("record-job") - record.add_argument("--side", choices=SIDE_CHOICES, required=True) - record.add_argument("--source-root", type=Path, required=True) - record.add_argument("--source-sha", required=True) - record.add_argument("--output", type=Path, required=True) - - verify = commands.add_parser("verify") - verify.add_argument("--side", choices=SIDE_CHOICES, required=True) - verify.add_argument("--source-root", type=Path, required=True) - verify.add_argument("--source-sha", required=True) - verify.add_argument("--native-root", type=Path, required=True) - verify.add_argument("--docs-root", type=Path, required=True) - verify.add_argument("--python-root", type=Path, required=True) - verify.add_argument("--output", type=Path, required=True) - - compare = commands.add_parser("compare") - compare.add_argument("--baseline-run", type=Path, required=True) - compare.add_argument("--proposed-run", type=Path, required=True) - compare.add_argument("--baseline-proof", type=Path, required=True) - compare.add_argument("--proposed-proof", type=Path, required=True) - compare.add_argument("--expected-baseline-sha", required=True) - compare.add_argument("--expected-proposed-sha", required=True) - compare.add_argument("--expected-baseline-controller-sha") - compare.add_argument("--expected-proposed-controller-sha") - compare.add_argument("--expected-baseline-run-head-sha", required=True) - compare.add_argument("--expected-proposed-run-head-sha", required=True) - compare.add_argument("--terminal-job-name", default=TERMINAL_JOB) - compare.add_argument("--baseline-required-job-name", action="append", required=True) - compare.add_argument("--proposed-required-job-name", action="append", required=True) - compare.add_argument("--output", type=Path, required=True) - - args = parser.parse_args() - if args.operation == "record-job": - result = record_job(args.side, args.source_root, args.source_sha) - elif args.operation == "verify": - result = verify_artifacts( - args.side, - args.source_root, - args.source_sha, - args.native_root, - args.docs_root, - args.python_root, - ) - else: - result = compare_runs( - args.baseline_run, - args.proposed_run, - args.baseline_proof, - args.proposed_proof, - args.expected_baseline_sha, - args.expected_proposed_sha, - args.expected_baseline_controller_sha, - args.expected_proposed_controller_sha, - args.expected_baseline_run_head_sha, - args.expected_proposed_run_head_sha, - args.terminal_job_name, - tuple(args.baseline_required_job_name), - tuple(args.proposed_required_job_name), - ) - _write_json(args.output, result) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 66562e5e6e..49a5c2609f 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -335,9 +335,6 @@ run_e2e_tests() { # shellcheck disable=SC2206 extra_args=(${PYTEST_EXTRA_ARGS:-}) pytest_command=(pytest) - if [[ " ${extra_args[*]} " == *" scripts.pytest_performance_evidence "* ]]; then - pytest_command=(python -m pytest) - fi marker_args=() if [[ -n "${PYTEST_MARK_EXPR:-}" ]]; then marker_args=(-m "$PYTEST_MARK_EXPR") diff --git a/scripts/windows/run-installer-proof.ps1 b/scripts/windows/run-installer-proof.ps1 deleted file mode 100644 index 7113d9939d..0000000000 --- a/scripts/windows/run-installer-proof.ps1 +++ /dev/null @@ -1,549 +0,0 @@ -# Focused Windows installer proof for one locally built candidate archive. -# -# The proof runs the unchanged installer assertion harness twice against the -# same archive with explicit pytest basetemps: one long root shaped like the -# failing default runner root, and one short root under RUNNER_TEMP. It also -# records archive path pressure and PowerShell 5.1 native-stderr behavior. - -param( - [string]$CandidateArchive, - [string]$CandidateMetadata, - [string]$BaselineArchive, - [string]$BaselineChecksum, - [string]$BaselineVersion, - [string]$OutputDir = "windows-installer-proof", - [switch]$SelfTestProcessCapture, - [string]$ValidateInstallerJUnit, - [int]$InstallerExitCode = 0, - [string]$PythonExecutable = "python" -) - -$ErrorActionPreference = "Stop" - -function Resolve-RequiredPath { - param([string]$Path, [string]$Label) - if (-not $Path) { throw "$Label is required" } - $resolved = Resolve-Path -LiteralPath $Path -ErrorAction Stop - if (-not $resolved) { throw "$Label not found: $Path" } - return $resolved.Path -} - -function Get-Sha256Hex { - param([string]$Path) - return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() -} - -function Write-JsonFile { - param([object]$Value, [string]$Path) - $directory = Split-Path -Parent $Path - if ($directory) { New-Item -ItemType Directory -Force -Path $directory | Out-Null } - $Value | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $Path -Encoding UTF8 -} - -function Stop-OwnedProcessTree { - param([System.Diagnostics.Process]$Process) - if ($Process.HasExited) { return } - if ($IsWindows) { - & taskkill.exe /PID $Process.Id /T /F | Out-Null - if ($LASTEXITCODE -ne 0 -and -not $Process.HasExited) { - throw "Could not stop owned process tree $($Process.Id)" - } - } else { - $Process.Kill($true) - } -} - -function Invoke-CapturedProcess { - param( - [Parameter(Mandatory = $true)][string]$FilePath, - [Parameter(Mandatory = $true)][string[]]$Arguments, - [Parameter(Mandatory = $true)][string]$Name, - [int]$TimeoutSeconds = 90, - [hashtable]$Environment = @{}, - [string]$WorkingDirectory = (Get-Location).Path - ) - $psi = [System.Diagnostics.ProcessStartInfo]::new() - $psi.FileName = $FilePath - $psi.WorkingDirectory = $WorkingDirectory - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - foreach ($argument in $Arguments) { - [void]$psi.ArgumentList.Add($argument) - } - foreach ($key in $Environment.Keys) { - $psi.Environment[$key] = [string]$Environment[$key] - } - $process = [System.Diagnostics.Process]::new() - $process.StartInfo = $psi - $record = [ordered]@{ - name = $Name - command = @($FilePath) + $Arguments - cwd = $WorkingDirectory - timeout_seconds = $TimeoutSeconds - timed_out = $false - exit_code = $null - stdout = "" - stderr = "" - } - try { - if (-not $process.Start()) { throw "Could not start $Name" } - $stdoutTask = $process.StandardOutput.ReadToEndAsync() - $stderrTask = $process.StandardError.ReadToEndAsync() - if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { - $record.timed_out = $true - Stop-OwnedProcessTree -Process $process - if (-not $process.WaitForExit(10000)) { - throw "Owned process $($process.Id) did not exit after its timeout" - } - } - $record.exit_code = $process.ExitCode - $record.stdout = $stdoutTask.GetAwaiter().GetResult() - $record.stderr = $stderrTask.GetAwaiter().GetResult() - return $record - } finally { - $process.Dispose() - } -} - -function New-OwnedBaseTemp { - param([string]$Path) - if (Test-Path -LiteralPath $Path) { - throw "Refusing to reuse preexisting proof basetemp: $Path" - } - New-Item -ItemType Directory -Path $Path -ErrorAction Stop | Out-Null - Set-Content -LiteralPath (Join-Path $Path ".apm-windows-installer-proof-owned") ` - -Value "owned by run $env:GITHUB_RUN_ID attempt $env:GITHUB_RUN_ATTEMPT" -Encoding ASCII - return $Path -} - -function Remove-OwnedBaseTemp { - param([string]$Path) - $marker = Join-Path $Path ".apm-windows-installer-proof-owned" - if (-not (Test-Path -LiteralPath $marker)) { - throw "Refusing to remove unmarked proof basetemp: $Path" - } - Remove-Item -Recurse -Force -LiteralPath $Path -ErrorAction Stop -} - -function Read-BaselineSha { - param([string]$Path) - $text = (Get-Content -LiteralPath $Path -Raw).Trim() - if ($text -notmatch '^([a-fA-F0-9]{64})\s+\*?apm-windows-x86_64\.zip$') { - throw "Invalid baseline checksum format: $Path" - } - return $Matches[1].ToLowerInvariant() -} - -function Get-ArchiveInventory { - param( - [string]$Archive, - [string]$LongBaseTemp, - [string]$ShortBaseTemp, - [string]$Version - ) - Add-Type -AssemblyName System.IO.Compression.FileSystem - $zip = [System.IO.Compression.ZipFile]::OpenRead($Archive) - try { - $allEntries = @($zip.Entries) - $files = @($allEntries | Where-Object { -not ($_.FullName.EndsWith("/")) }) - $entries = @($files | ForEach-Object { $_.FullName }) - } finally { - $zip.Dispose() - } - $suffix = "APM Install Test & Edge " + ("a" * 32) - function Convert-MemberPath { - param([string]$Member) - $prefix = "apm-windows-x86_64/" - if ($Member.StartsWith($prefix, [System.StringComparison]::Ordinal)) { - return $Member.Substring($prefix.Length).Replace("/", "\") - } - return $Member.Replace("/", "\") - } - function Measure-BaseTemp { - param([string]$BaseTemp) - $testRoot = Join-Path $BaseTemp "wi0\i\work" - $installRoot = Join-Path $testRoot $suffix - $tempDir = Join-Path (Join-Path $installRoot "tmp") ("apm-install-" + ("b" * 32)) - $stageDir = Join-Path (Join-Path $installRoot "releases") ($Version + ".new-" + ("c" * 32)) - $finalDir = Join-Path (Join-Path $installRoot "releases") $Version - $records = @() - foreach ($entry in $entries) { - $relative = Convert-MemberPath -Member $entry - $records += [ordered]@{ - member = $entry - temp_length = (Join-Path $tempDir ($entry.Replace("/", "\"))).Length - stage_length = (Join-Path $stageDir $relative).Length - final_length = (Join-Path $finalDir $relative).Length - } - } - return [ordered]@{ - base_temp = $BaseTemp - test_root = $testRoot - install_root = $installRoot - max_temp = @($records | Sort-Object temp_length -Descending | Select-Object -First 1)[0] - max_stage = @($records | Sort-Object stage_length -Descending | Select-Object -First 1)[0] - max_final = @($records | Sort-Object final_length -Descending | Select-Object -First 1)[0] - charset = @($records | Where-Object { $_.member -match 'charset_normalizer|chardet' }) - stage_over_260 = @($records | Where-Object { $_.stage_length -ge 260 }).Count - temp_over_260 = @($records | Where-Object { $_.temp_length -ge 260 }).Count - } - } - return [ordered]@{ - file_count = $files.Count - member_count = $allEntries.Count - pyd_count = @($entries | Where-Object { ($_.ToLowerInvariant()).EndsWith(".pyd") }).Count - long_root = Measure-BaseTemp -BaseTemp $LongBaseTemp - short_root = Measure-BaseTemp -BaseTemp $ShortBaseTemp - } -} - -function Invoke-DirectCandidateProbe { - param( - [string]$Name, - [string]$BaseTemp, - [string]$Archive, - [string]$Version, - [string]$OutDir - ) - $testRoot = Join-Path $BaseTemp "wi0\i\work" - $root = Join-Path $testRoot ("APM Install Test & Edge " + [System.Guid]::NewGuid().ToString("N")) - $extractDir = Join-Path $root ("tmp\apm-install-" + [System.Guid]::NewGuid().ToString("N")) - $releaseDir = Join-Path (Join-Path $root "releases") $Version - $stagingDir = "$releaseDir.new-" + [System.Guid]::NewGuid().ToString("N") - $record = [ordered]@{ - name = $Name - powershell_host = [ordered]@{ - edition = $PSVersionTable.PSEdition - version = $PSVersionTable.PSVersion.ToString() - } - base_temp = $BaseTemp - test_root = $testRoot - root = $root - extract_dir = $extractDir - staging_dir = $stagingDir - expanded = $false - moved_to_stage = $false - charset_members = @() - run = $null - error = $null - } - try { - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null - New-Item -ItemType Directory -Force -Path (Split-Path -Parent $releaseDir) | Out-Null - Expand-Archive -LiteralPath $Archive -DestinationPath $extractDir -Force - $record.expanded = $true - $packageDir = Join-Path $extractDir "apm-windows-x86_64" - Get-ChildItem -LiteralPath $packageDir -Recurse -Force -ErrorAction Stop | - Where-Object { $_.FullName -match 'charset_normalizer|chardet' } | - ForEach-Object { - $record.charset_members += [ordered]@{ - path = $_.FullName - length = $_.FullName.Length - exists = $_.Exists - } - } - Move-Item -LiteralPath $packageDir -Destination $stagingDir -Force - $record.moved_to_stage = $true - $exe = Join-Path $stagingDir "apm.exe" - $record.run = Invoke-CapturedProcess -FilePath $exe -Arguments @("--version") ` - -Name "$Name direct apm --version" -TimeoutSeconds 90 - } catch { - $record.error = "$_" - } finally { - Write-JsonFile -Value $record -Path (Join-Path $OutDir "direct-$Name.json") - } - return $record -} - -function Invoke-PowerShellNativeStderrProbe { - param([string]$OutDir) - $child = Join-Path $OutDir "ps5-native-stderr-child.ps1" - @' -$ErrorActionPreference = "Stop" -function Invoke-NativeCase { - param([string]$Name, [int]$ExitCode) - $result = [ordered]@{ - name = $Name - expected_exit_code = $ExitCode - threw = $false - last_exit_code = $null - output = "" - error = "" - } - try { - $output = & cmd.exe /d /c "echo marker-$Name 1>&2 & exit /b $ExitCode" 2>&1 - $result.last_exit_code = $LASTEXITCODE - $result.output = ($output | Out-String) - } catch { - $result.threw = $true - $result.last_exit_code = $LASTEXITCODE - $result.error = "$_" - } - return $result -} -[ordered]@{ - host = [ordered]@{ - edition = $PSVersionTable.PSEdition - version = $PSVersionTable.PSVersion.ToString() - major = $PSVersionTable.PSVersion.Major - minor = $PSVersionTable.PSVersion.Minor - } - cases = @( - (Invoke-NativeCase -Name "exit0-stderr" -ExitCode 0), - (Invoke-NativeCase -Name "exit23-stderr" -ExitCode 23) - ) -} | ConvertTo-Json -Depth 8 -'@ | Set-Content -LiteralPath $child -Encoding UTF8 - $ps5 = Invoke-CapturedProcess -FilePath "powershell.exe" -Arguments @( - "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $child - ) -Name "powershell 5 native stderr probe" -TimeoutSeconds 90 - $exit0 = Invoke-CapturedProcess -FilePath "cmd.exe" -Arguments @( - "/d", "/c", "echo marker-direct-exit0 1>&2 & exit /b 0" - ) -Name "direct native stderr exit0 control" -TimeoutSeconds 90 - $exit23 = Invoke-CapturedProcess -FilePath "cmd.exe" -Arguments @( - "/d", "/c", "echo marker-direct-exit23 1>&2 & exit /b 23" - ) -Name "direct native stderr exit23 control" -TimeoutSeconds 90 - if ($exit0.timed_out -or $exit0.exit_code -ne 0 -or $exit0.stderr -notmatch "marker-direct-exit0") { - throw "Broken direct exit0 native-stderr control" - } - if ($exit23.timed_out -or $exit23.exit_code -ne 23 -or $exit23.stderr -notmatch "marker-direct-exit23") { - throw "Broken direct exit23 native-stderr control" - } - if ($ps5.timed_out -or $ps5.exit_code -ne 0) { - throw "PowerShell 5 native-stderr child failed" - } - try { - $ps5Json = $ps5.stdout | ConvertFrom-Json - } catch { - throw "PowerShell 5 native-stderr child did not emit valid JSON: $_" - } - if ($ps5Json.host.major -ne 5 -or $ps5Json.host.minor -ne 1) { - throw "Expected Windows PowerShell 5.1, got $($ps5Json.host.version)" - } - $caseNames = @($ps5Json.cases | ForEach-Object { $_.name } | Sort-Object) - if ($caseNames.Count -ne 2 -or ($caseNames -join ",") -ne "exit0-stderr,exit23-stderr") { - throw "PowerShell 5 child did not report both native-stderr cases" - } - foreach ($case in $ps5Json.cases) { - # A terminating stderr record can prevent LASTEXITCODE from being updated. - # The independent native controls above establish the real exit codes. - if (-not $case.threw -and $case.name -eq "exit0-stderr" -and $case.last_exit_code -ne 0) { - throw "PowerShell 5 exit0 case lost native exit code" - } - if (-not $case.threw -and $case.name -eq "exit23-stderr" -and $case.last_exit_code -ne 23) { - throw "PowerShell 5 exit23 case lost native exit code" - } - $combined = "$($case.output) $($case.error)" - if ($combined -notmatch ("marker-" + [regex]::Escape($case.name))) { - throw "PowerShell 5 case $($case.name) lost stderr marker" - } - } - $result = [ordered]@{ - name = "deterministic-native-stderr" - powershell5 = $ps5 - powershell5_json = $ps5Json - direct_exit0 = $exit0 - direct_exit23 = $exit23 - } - Write-JsonFile -Value $result -Path (Join-Path $OutDir "native-stderr-behavior.json") - return $result -} - -function Read-InstallerOutcome { - param([string]$JUnitPath, [int]$ExitCode) - [xml]$report = Get-Content -LiteralPath $JUnitPath -Raw -ErrorAction Stop - $cases = @($report.SelectNodes("//testcase")) - $skipped = @($report.SelectNodes("//testcase/skipped")).Count - $failed = @($report.SelectNodes("//testcase/failure | //testcase/error")).Count - if ($cases.Count -eq 0 -or $skipped -ne 0) { - throw "Installer assertions did not execute completely: $($cases.Count) cases, $skipped skipped" - } - if ($ExitCode -eq 0 -and $failed -ne 0) { - throw "Installer exit code claimed success despite $failed failing JUnit cases" - } - return [ordered]@{ cases = $cases.Count; skipped = $skipped; failed = $failed } -} - -function Invoke-InstallerPytest { - param( - [string]$Name, - [string]$OutDir, - [string]$BaseTemp, - [hashtable]$InstallerEnvironment - ) - if (Test-Path -LiteralPath $BaseTemp) { - throw "Installer pytest basetemp must be fresh before pytest owns it: $BaseTemp" - } - $arguments = @( - "run", "--frozen", "--no-sync", "pytest", - "tests/integration/test_windows_installer_launchers.py", - "-q", "-s", "--tb=short", - "--junitxml=$OutDir\junit-$Name.xml", - "--basetemp", $BaseTemp - ) - $result = Invoke-CapturedProcess -FilePath "uv" -Arguments $arguments ` - -Name "$Name installer pytest" -TimeoutSeconds 900 -Environment $InstallerEnvironment - # Persist raw diagnostics even when pytest never produced usable JUnit. - $result.stdout | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stdout.log") -Encoding UTF8 - $result.stderr | Set-Content -LiteralPath (Join-Path $OutDir "$Name-stderr.log") -Encoding UTF8 - Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") - Write-Host "[$Name] exit $($result.exit_code) timed_out=$($result.timed_out)" - if ($result.stdout) { - ($result.stdout -split "`n") | Select-Object -Last 30 | ForEach-Object { Write-Host "[$Name stdout] $_" } - } - if ($result.stderr) { - ($result.stderr -split "`n") | Select-Object -Last 30 | ForEach-Object { Write-Host "[$Name stderr] $_" } - } - try { - $result.junit = Read-InstallerOutcome -JUnitPath (Join-Path $OutDir "junit-$Name.xml") ` - -ExitCode $result.exit_code - } catch { - $result.junit_validation_error = $_.Exception.Message - Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") - throw - } - Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json") - return $result -} - -function Invoke-ProcessCaptureSelfTest { - param([string]$OutDir, [string]$Python) - New-Item -ItemType Directory -Force -Path $OutDir | Out-Null - $exit0 = Invoke-CapturedProcess -FilePath $Python -Arguments @( - "-c", "import sys; sys.stderr.write('selftest-exit0-stderr\n'); print('selftest-exit0-stdout')" - ) -Name "selftest exit0" -TimeoutSeconds 30 - $exit23 = Invoke-CapturedProcess -FilePath $Python -Arguments @( - "-c", "import sys; sys.stderr.write('selftest-exit23-stderr\n'); sys.exit(23)" - ) -Name "selftest exit23" -TimeoutSeconds 30 - $timeout = Invoke-CapturedProcess -FilePath $Python -Arguments @( - "-c", "import time; time.sleep(10)" - ) -Name "selftest timeout" -TimeoutSeconds 1 - $result = [ordered]@{ exit0 = $exit0; exit23 = $exit23; timeout = $timeout } - Write-JsonFile -Value $result -Path (Join-Path $OutDir "process-capture-selftest.json") - if ($exit0.timed_out -or $exit0.exit_code -ne 0 -or $exit0.stderr -notmatch "selftest-exit0-stderr") { - throw "Process capture self-test exit0 failed" - } - if ($exit23.timed_out -or $exit23.exit_code -ne 23 -or $exit23.stderr -notmatch "selftest-exit23-stderr") { - throw "Process capture self-test exit23 failed" - } - if (-not $timeout.timed_out) { - throw "Process capture self-test timeout did not time out" - } -} - -if ($ValidateInstallerJUnit) { - Read-InstallerOutcome -JUnitPath $ValidateInstallerJUnit -ExitCode $InstallerExitCode | - ConvertTo-Json - exit 0 -} - -if ($SelfTestProcessCapture) { - Invoke-ProcessCaptureSelfTest -OutDir $OutputDir -Python $PythonExecutable - exit 0 -} - -$CandidateArchive = Resolve-RequiredPath -Path $CandidateArchive -Label "Candidate archive" -$CandidateMetadata = Resolve-RequiredPath -Path $CandidateMetadata -Label "Candidate metadata" -$BaselineArchive = Resolve-RequiredPath -Path $BaselineArchive -Label "Baseline archive" -$BaselineChecksum = Resolve-RequiredPath -Path $BaselineChecksum -Label "Baseline checksum" -if (-not $BaselineVersion) { throw "BaselineVersion is required" } -if (-not $env:RUNNER_TEMP) { throw "RUNNER_TEMP is required; this proof is CI-only" } -if (-not $env:TEMP) { throw "TEMP is required to reproduce the Windows pytest default root" } -if ([string]$env:GITHUB_RUN_ATTEMPT -ne "1") { - throw "Use a fresh attempt 1 run for Windows installer proof" -} -New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null - -$metadata = Get-Content -LiteralPath $CandidateMetadata -Raw | ConvertFrom-Json -$candidateVersion = "v$($metadata.version)" -$candidateSha = [string]$metadata.archive_sha256 -$candidateExeSha = [string]$metadata.executable_sha256 -if ((Get-Sha256Hex -Path $CandidateArchive) -cne $candidateSha) { - throw "Candidate archive hash does not match metadata" -} -$baselineSha = Read-BaselineSha -Path $BaselineChecksum -if ((Get-Sha256Hex -Path $BaselineArchive) -cne $baselineSha) { - throw "Baseline archive hash does not match sidecar" -} - -$resolvedTemp = Invoke-CapturedProcess -FilePath "python" -Arguments @( - "-c", "import os; print(os.path.realpath(os.environ['TEMP']))" -) -Name "Resolve actual long TEMP path" -if ($resolvedTemp.timed_out -or $resolvedTemp.exit_code -ne 0) { - throw "Could not resolve the actual Windows TEMP path" -} -$longChild = "wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5) -$longBaseTemp = Join-Path (Join-Path $resolvedTemp.stdout.Trim() ("pytest-of-" + $env:USERNAME)) $longChild -$shortBaseTemp = Join-Path $env:RUNNER_TEMP "apm-windows-installer" -Invoke-PowerShellNativeStderrProbe -OutDir $OutputDir | Out-Null -$longOwned = $false -$shortOwned = $false -try { - New-OwnedBaseTemp -Path $longBaseTemp | Out-Null - $longOwned = $true - New-OwnedBaseTemp -Path $shortBaseTemp | Out-Null - $shortOwned = $true - $inventory = Get-ArchiveInventory -Archive $CandidateArchive -LongBaseTemp $longBaseTemp ` - -ShortBaseTemp $shortBaseTemp -Version $candidateVersion - Write-JsonFile -Value $inventory -Path (Join-Path $OutputDir "archive-path-inventory.json") - Invoke-DirectCandidateProbe -Name "long-root" -BaseTemp $longBaseTemp -Archive $CandidateArchive ` - -Version $candidateVersion -OutDir $OutputDir | Out-Null - Invoke-DirectCandidateProbe -Name "short-root" -BaseTemp $shortBaseTemp -Archive $CandidateArchive ` - -Version $candidateVersion -OutDir $OutputDir | Out-Null -} finally { - if ($longOwned -and (Test-Path -LiteralPath $longBaseTemp)) { Remove-OwnedBaseTemp -Path $longBaseTemp } - if ($shortOwned -and (Test-Path -LiteralPath $shortBaseTemp)) { Remove-OwnedBaseTemp -Path $shortBaseTemp } -} - -$installerEnvironment = @{ - "APM_E2E_TESTS" = "1" - "APM_CANDIDATE_ARCHIVE" = $CandidateArchive - "APM_CANDIDATE_VERSION" = $candidateVersion - "APM_CANDIDATE_SHA256" = $candidateSha - "APM_BASELINE_ARCHIVE" = $BaselineArchive - "APM_BASELINE_VERSION" = $BaselineVersion - "APM_BASELINE_SHA256" = $baselineSha -} -$longResult = Invoke-InstallerPytest -Name "long-root" -OutDir $OutputDir -BaseTemp $longBaseTemp ` - -InstallerEnvironment $installerEnvironment -$shortResult = Invoke-InstallerPytest -Name "short-root" -OutDir $OutputDir -BaseTemp $shortBaseTemp ` - -InstallerEnvironment $installerEnvironment - -$summary = [ordered]@{ - candidate = [ordered]@{ - version = $candidateVersion - archive_sha256 = $candidateSha - executable_sha256 = $candidateExeSha - metadata_sha = [string]$metadata.sha - } - baseline = [ordered]@{ - version = $BaselineVersion - archive_sha256 = $baselineSha - } - roots = [ordered]@{ - long_base_temp = $longBaseTemp - short_base_temp = $shortBaseTemp - } - long_root_exit_code = $longResult.exit_code - long_root_timed_out = $longResult.timed_out - short_root_exit_code = $shortResult.exit_code - short_root_timed_out = $shortResult.timed_out - long_root_junit = $longResult.junit - short_root_junit = $shortResult.junit - conclusion = if ($shortResult.exit_code -eq 0 -and $longResult.exit_code -ne 0) { - "short-root-passed-long-root-failed" - } elseif ($shortResult.exit_code -eq 0 -and $longResult.exit_code -eq 0) { - "both-roots-passed" - } else { - "short-root-failed" - } -} -Write-JsonFile -Value $summary -Path (Join-Path $OutputDir "summary.json") - -if ($shortResult.timed_out -or $shortResult.exit_code -ne 0) { - Write-Host "Short-root installer proof failed; see $OutputDir" -ForegroundColor Red - if ($shortResult.timed_out) { exit 124 } - exit $shortResult.exit_code -} -Write-Host "Windows installer proof completed: $($summary.conclusion)" diff --git a/scripts/windows/validate-installer-result.ps1 b/scripts/windows/validate-installer-result.ps1 new file mode 100644 index 0000000000..04fb74bc72 --- /dev/null +++ b/scripts/windows/validate-installer-result.ps1 @@ -0,0 +1,25 @@ +# Require executed installer assertions before qualifying a candidate. +param( + [Parameter(Mandatory = $true)][string]$ValidateInstallerJUnit, + [int]$InstallerExitCode = 0 +) + +$ErrorActionPreference = "Stop" + +function Read-InstallerOutcome { + param([string]$JUnitPath, [int]$ExitCode) + [xml]$report = Get-Content -LiteralPath $JUnitPath -Raw -ErrorAction Stop + $cases = @($report.SelectNodes("//testcase")) + $skipped = @($report.SelectNodes("//testcase/skipped")).Count + $failed = @($report.SelectNodes("//testcase/failure | //testcase/error")).Count + if ($cases.Count -eq 0 -or $skipped -ne 0) { + throw "Installer assertions did not execute completely: $($cases.Count) cases, $skipped skipped" + } + if ($ExitCode -eq 0 -and $failed -ne 0) { + throw "Installer exit code claimed success despite $failed failing JUnit cases" + } + return [ordered]@{ cases = $cases.Count; skipped = $skipped; failed = $failed } +} + +Read-InstallerOutcome -JUnitPath $ValidateInstallerJUnit -ExitCode $InstallerExitCode | + ConvertTo-Json diff --git a/tests/unit/scripts/test_compare_test_runs.py b/tests/unit/scripts/test_compare_test_runs.py deleted file mode 100644 index 854379fa2d..0000000000 --- a/tests/unit/scripts/test_compare_test_runs.py +++ /dev/null @@ -1,515 +0,0 @@ -"""Performance claims require identical, complete, successful test evidence.""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -from dataclasses import replace -from pathlib import Path - -import pytest - -from scripts.compare_test_runs import Report, capture, compare, read_report - -pytestmark = pytest.mark.component -SHA = "a" * 40 -IDENTITY = { - "source_sha": SHA, - "selection": "not live", - "candidate": None, - "environment": { - "python": "3.12.12", - "system": "Darwin", - "release": "24.6.0", - "machine": "arm64", - "cpu_count": "3", - "runner_image": "20260901", - "dependency_lock": "e" * 64, - }, -} -CASES = {("suite", "test_a"): "passed", ("suite", "test_b"): "skipped"} - - -def _baseline() -> Report: - return Report(IDENTITY, "baseline", 1, 1, 10.0, CASES) - - -def _proposed() -> list[Report]: - return [ - Report(IDENTITY, "proposed", 1, 2, 5.0, {("suite", "test_a"): "passed"}), - Report(IDENTITY, "proposed", 2, 2, 6.0, {("suite", "test_b"): "skipped"}), - ] - - -def test_complete_disjoint_shards_preserve_skips_and_report_wall_separately_from_cost() -> None: - result = compare([_baseline()], _proposed(), SHA, 0.05) - assert result["scenario_count"] == 2 - assert result["outcomes"] == {"passed": 1, "skipped": 1, "xfailed": 0, "failed": 0, "error": 0} - assert result["baseline_pytest_critical_seconds"] == 10.0 - assert result["proposed_pytest_critical_seconds"] == 6.0 - assert result["proposed_pytest_runner_seconds"] == 11.0 - assert result["speedup_fraction"] == pytest.approx(0.4) - assert result["meets_threshold"] is True - - -@pytest.mark.parametrize( - "fault", ["missing", "duplicate", "overlap", "identity", "selection", "sha", "environment"] -) -def test_rejects_missing_overlap_and_identity_drift(fault: str) -> None: - proposed = _proposed() - expected = SHA - if fault == "missing": - proposed.pop() - elif fault == "duplicate": - proposed[1] = proposed[0] - elif fault == "overlap": - proposed[1] = replace(proposed[1], cases=proposed[0].cases) - elif fault == "identity": - proposed[1] = replace( - proposed[1], identity={**IDENTITY, "candidate": {"digest": "different"}} - ) - elif fault == "selection": - proposed[1] = replace(proposed[1], identity={**IDENTITY, "selection": "smoke"}) - elif fault == "environment": - proposed[1] = replace( - proposed[1], - identity={ - **IDENTITY, - "environment": {**IDENTITY["environment"], "runner_image": "new"}, - }, - ) - else: - expected = "b" * 40 - with pytest.raises(ValueError): - compare([_baseline()], proposed, expected, 0.05) - - -@pytest.mark.parametrize("outcome", ["skipped", "xfailed", "failed", "error"]) -def test_cannot_gain_speed_by_skipping_or_failing_a_previously_passing_case(outcome: str) -> None: - proposed = _proposed() - proposed[0] = replace(proposed[0], cases={("suite", "test_a"): outcome}) - with pytest.raises(ValueError): - compare([_baseline()], proposed, SHA, 0.05) - - -@pytest.mark.parametrize("cases", [{("suite", "test_extra"): "passed"}, {}]) -def test_rejects_extra_or_dropped_scenarios(cases: dict[tuple[str, str], str]) -> None: - proposed = _proposed() - proposed[0] = replace(proposed[0], cases=cases) - with pytest.raises(ValueError): - compare([_baseline()], proposed, SHA, 0.05) - - -def test_missed_speed_target_remains_explicit_in_durable_result() -> None: - proposed = [replace(_baseline(), variant="proposed", elapsed_seconds=10.1)] - result = compare([_baseline()], proposed, SHA, 0.05) - assert result["same_scenarios_and_outcomes"] is True - assert result["meets_threshold"] is False - assert result["speedup_fraction"] < 0 - - -@pytest.mark.parametrize("seconds", [float("nan"), float("inf"), 0, -1, True]) -def test_rejects_unusable_timings(tmp_path: Path, seconds: float) -> None: - payload = _baseline().payload() - payload["elapsed_seconds"] = seconds - path = tmp_path / "proof.json" - path.write_text(json.dumps(payload), encoding="ascii") - with pytest.raises(ValueError, match="finite and positive"): - read_report(path) - - -@pytest.mark.parametrize( - "fault", ["duplicate", "unknown-outcome", "empty", "missing-field", "bool-shard"] -) -def test_rejects_malformed_inventory(tmp_path: Path, fault: str) -> None: - payload = _baseline().payload() - if fault == "duplicate": - payload["cases"] = [{"id": ["suite", "a"], "outcome": "passed"}] * 2 - elif fault == "unknown-outcome": - payload["cases"] = [{"id": ["suite", "a"], "outcome": "unknown"}] - elif fault == "empty": - payload["cases"] = [] - elif fault == "missing-field": - del payload["identity"] - else: - payload["shard"] = True - path = tmp_path / "proof.json" - path.write_text(json.dumps(payload), encoding="ascii") - with pytest.raises(ValueError): - read_report(path) - - -def test_actual_junit_capture_roundtrip_keeps_xfail_identity(tmp_path: Path) -> None: - junit = tmp_path / "junit.xml" - junit.write_text( - '' - '' - '' - '' - "", - encoding="ascii", - ) - report = capture(junit, SHA, "not live", "baseline", 1, 1) - assert report.elapsed_seconds == 7.5 - assert report.cases == {("suite", "test_a"): "passed", ("suite", "test_b"): "xfailed"} - path = tmp_path / "proof.json" - path.write_text(json.dumps(report.payload()), encoding="ascii") - assert read_report(path) == report - - -@pytest.mark.parametrize("fault", ["none", "source", "digest", "binary", "version", "schema"]) -def test_candidate_evidence_is_bound_to_the_source_and_archive(tmp_path: Path, fault: str) -> None: - metadata = { - "schema_version": 1, - "sha": SHA, - "version": "0.30.0", - "binary_name": "apm-darwin-arm64", - "archive": "apm-darwin-arm64.tar.gz", - "archive_sha256": "c" * 64, - "executable_sha256": "d" * 64, - } - if fault == "source": - metadata["sha"] = "b" * 40 - elif fault == "digest": - metadata["archive_sha256"] = "not-a-digest" - elif fault == "binary": - metadata["binary_name"] = "apm-windows-x86_64" - elif fault == "version": - metadata["version"] = "" - elif fault == "schema": - metadata["schema_version"] = True - path = tmp_path / "candidate.json" - path.write_text(json.dumps(metadata), encoding="ascii") - junit = tmp_path / "junit.xml" - junit.write_text( - '' - '', - encoding="ascii", - ) - if fault != "none": - with pytest.raises(ValueError): - capture(junit, SHA, "not live", "baseline", 1, 1, path) - else: - report = capture(junit, SHA, "not live", "baseline", 1, 1, path) - assert report.identity["candidate"] == metadata - - -@pytest.mark.parametrize( - "xml", - [ - '', - '' - '', - '' - '', - "", - ], -) -def test_rejects_incomplete_or_ambiguous_junit(tmp_path: Path, xml: str) -> None: - path = tmp_path / "junit.xml" - path.write_text(xml, encoding="ascii") - with pytest.raises(ValueError): - capture(path, SHA, "not live", "baseline", 1, 1) - - -@pytest.mark.parametrize( - "declaration", - [ - ']>', - '', - ], -) -@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) -def test_junit_rejects_dtds_before_expanding_entities( - tmp_path: Path, declaration: str, encoding: str -) -> None: - path = tmp_path / "junit.xml" - path.write_text( - f'{declaration}' - '' - '', - encoding=encoding, - ) - with pytest.raises(ValueError, match="document type declarations are forbidden"): - capture(path, SHA, "not live", "baseline", 1, 1) - - -def test_real_pytest_xdist_reports_preserve_original_node_ids_and_outcomes(tmp_path: Path) -> None: - root = Path(__file__).resolve().parents[3] - (tmp_path / "pytest.ini").write_text( - "[pytest]\nmarkers = xdist_group: scheduling affinity\n", encoding="ascii" - ) - (tmp_path / "test_example.py").write_text( - "import pytest\n" - "@pytest.mark.xdist_group('home')\n" - "@pytest.mark.parametrize('value', ['literal@home', 'other', 'xfail', 'skip'])\n" - "def test_case(value):\n" - " if value == 'skip': pytest.skip('known prerequisite')\n" - " if value == 'xfail': pytest.xfail('known failure')\n" - " assert value in ['literal@home', 'other']\n", - encoding="ascii", - ) - reports = [] - for variant, shard in (("baseline", 1), ("proposed", 1), ("proposed", 2)): - xml = tmp_path / f"{variant}-{shard}.xml" - command = [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "scripts.pytest_performance_evidence", - "--dist", - "loadgroup", - "-n", - "0" if variant == "baseline" else "2", - f"--junitxml={xml}", - "test_example.py", - ] - if variant == "proposed": - command.extend(["--splits", "2", "--group", str(shard)]) - result = subprocess.run( - command, - cwd=tmp_path, - env={**os.environ, "PYTHONPATH": str(root)}, - capture_output=True, - text=True, - timeout=60, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr - reports.append( - capture(xml, SHA, "example", variant, shard, 1 if variant == "baseline" else 2) - ) - # Ignore overhead-dominated toy timings; this case proves actual plugin interoperability. - baseline = replace(reports[0], elapsed_seconds=10) - proposed = [replace(report, elapsed_seconds=5) for report in reports[1:]] - proof = compare([baseline], proposed, SHA, 0.05) - assert proof["scenario_count"] == 4 - assert proof["outcomes"]["passed"] == 2 - assert ("pytest-nodeid", "test_example.py::test_case[literal@home]") in baseline.cases - - -def _write_subtest_probe(tmp_path: Path) -> None: - (tmp_path / "pytest.ini").write_text( - "[pytest]\nmarkers = xdist_group: scheduling affinity\n", encoding="ascii" - ) - (tmp_path / "test_subtest_probe.py").write_text( - "import os\n" - "import pytest\n" - "import unittest\n" - "class TestSubtests(unittest.TestCase):\n" - " @pytest.mark.xdist_group('home')\n" - " def test_repeated_context(self):\n" - " for ordinal, label in enumerate(['same', 'same', 'literal@context'], 1):\n" - " with self.subTest(label=label):\n" - " if os.environ.get('FAIL_SECOND_SUBTEST') == '1' and ordinal == 2:\n" - " self.fail('intentional subtest proof failure')\n" - " self.assertTrue(label)\n" - "@pytest.mark.xdist_group('home')\n" - "@pytest.mark.parametrize('value', ['literal@home', 'plain'])\n" - "def test_literal_at_param(value):\n" - " if os.environ.get('SKIP_PLAIN_PARAM') == '1' and value == 'plain':\n" - " pytest.skip('intentional parity regression')\n" - " assert value in {'literal@home', 'plain'}\n", - encoding="ascii", - ) - - -def _run_subtest_probe( - tmp_path: Path, - name: str, - variant: str, - shard: int, - shard_count: int, - workers: str, - dist: str = "worksteal", - *, - fail_second_subtest: bool = False, - skip_plain_param: bool = False, -) -> tuple[subprocess.CompletedProcess[str], Report]: - root = Path(__file__).resolve().parents[3] - (tmp_path / ".test_durations").write_text("{}\n", encoding="ascii") - junit = tmp_path / f"{name}.xml" - command = [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "scripts.pytest_performance_evidence", - "test_subtest_probe.py", - "-n", - workers, - "--dist", - dist, - "--durations=5", - "--store-durations", - f"--junitxml={junit}", - ] - if shard_count > 1: - command.extend( - [ - "--splits", - str(shard_count), - "--group", - str(shard), - "--splitting-algorithm", - "least_duration", - ] - ) - result = subprocess.run( - command, - cwd=tmp_path, - env={ - **os.environ, - "PYTHONPATH": str(root), - "PYTEST_XDIST_AUTO_NUM_WORKERS": "2", - "FAIL_SECOND_SUBTEST": "1" if fail_second_subtest else "", - "SKIP_PLAIN_PARAM": "1" if skip_plain_param else "", - }, - capture_output=True, - text=True, - timeout=60, - check=False, - ) - return result, capture(junit, SHA, "subtest probe", variant, shard, shard_count) - - -@pytest.mark.parametrize("detected_cpus", [16, 160]) -def test_nested_probe_auto_workers_are_bounded( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, detected_cpus: int -) -> None: - monkeypatch.setattr(os, "cpu_count", lambda: detected_cpus) - monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", str(detected_cpus)) - - def run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: - env = kwargs["env"] - assert isinstance(env, dict) - assert env["PYTEST_XDIST_AUTO_NUM_WORKERS"] == "2" - assert command[command.index("-n") + 1] == "auto" - return subprocess.CompletedProcess(command, 0, "", "") - - monkeypatch.setattr(subprocess, "run", run) - monkeypatch.setattr(sys.modules[__name__], "capture", lambda *_: _baseline()) - result, _ = _run_subtest_probe(tmp_path, "budget", "baseline", 1, 1, "auto") - assert result.returncode == 0 - - -def test_real_pytest_subtests_compare_under_xdist_baseline_and_two_shards( - tmp_path: Path, -) -> None: - _write_subtest_probe(tmp_path) - result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") - assert result.returncode == 0, result.stdout + result.stderr - proposed = [] - for shard in (1, 2): - result, report = _run_subtest_probe( - tmp_path, f"proposed-{shard}", "proposed", shard, 2, "4" - ) - assert result.returncode == 0, result.stdout + result.stderr - proposed.append(report) - - proof = compare( - [replace(baseline, elapsed_seconds=10.0)], - [replace(report, elapsed_seconds=5.0) for report in proposed], - SHA, - 0.05, - ) - assert proof["scenario_count"] == 6 - assert proof["outcomes"] == { - "passed": 6, - "skipped": 0, - "xfailed": 0, - "failed": 0, - "error": 0, - } - assert ( - "pytest-nodeid", - "test_subtest_probe.py::test_literal_at_param[literal@home]", - ) in baseline.cases - for ordinal in (1, 2, 3): - assert ( - "test_subtest_probe.TestSubtests.test_repeated_context", - f"subTest[{ordinal}]", - ) in baseline.cases - - -def test_real_pytest_subtests_compare_between_serial_and_loadgroup_shards( - tmp_path: Path, -) -> None: - _write_subtest_probe(tmp_path) - result, baseline = _run_subtest_probe( - tmp_path, "baseline-loadgroup", "baseline", 1, 1, "0", "loadgroup" - ) - assert result.returncode == 0, result.stdout + result.stderr - proposed = [] - for shard in (1, 2): - result, report = _run_subtest_probe( - tmp_path, f"proposed-loadgroup-{shard}", "proposed", shard, 2, "2", "loadgroup" - ) - assert result.returncode == 0, result.stdout + result.stderr - proposed.append(report) - - proof = compare( - [replace(baseline, elapsed_seconds=10.0)], - [replace(report, elapsed_seconds=5.0) for report in proposed], - SHA, - 0.05, - ) - assert proof["scenario_count"] == 6 - assert ( - "pytest-nodeid", - "test_subtest_probe.py::test_literal_at_param[literal@home]", - ) in baseline.cases - assert ( - "pytest-nodeid", - "test_subtest_probe.py::test_literal_at_param[literal@home]", - ) in proposed[0].cases | proposed[1].cases - for report in proposed: - assert not any("@home::subTest" in part for case in report.cases for part in case) - - -def test_real_pytest_subtest_failure_remains_a_compare_rejection(tmp_path: Path) -> None: - _write_subtest_probe(tmp_path) - result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") - assert result.returncode == 0, result.stdout + result.stderr - proposed = [] - for shard in (1, 2): - result, report = _run_subtest_probe( - tmp_path, - f"proposed-fail-{shard}", - "proposed", - shard, - 2, - "4", - fail_second_subtest=True, - ) - proposed.append(report) - assert any("failed" in report.cases.values() for report in proposed) - with pytest.raises(ValueError, match="contains failing tests"): - compare([baseline], proposed, SHA, 0.05) - - -def test_real_pytest_subtest_probe_rejects_new_skips(tmp_path: Path) -> None: - _write_subtest_probe(tmp_path) - result, baseline = _run_subtest_probe(tmp_path, "baseline", "baseline", 1, 1, "auto") - assert result.returncode == 0, result.stdout + result.stderr - proposed = [] - for shard in (1, 2): - result, report = _run_subtest_probe( - tmp_path, - f"proposed-skip-{shard}", - "proposed", - shard, - 2, - "4", - skip_plain_param=True, - ) - assert result.returncode == 0, result.stdout + result.stderr - proposed.append(report) - assert any("skipped" in report.cases.values() for report in proposed) - with pytest.raises(ValueError, match="Scenario parity failed"): - compare([baseline], proposed, SHA, 0.05) diff --git a/tests/unit/scripts/test_performance_cohort.py b/tests/unit/scripts/test_performance_cohort.py deleted file mode 100644 index 82516fc49f..0000000000 --- a/tests/unit/scripts/test_performance_cohort.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Preflight matching must not let a mismatched runner spend time on the full suite.""" - -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -import pytest - -from scripts import performance_cohort - -pytestmark = pytest.mark.component -SHA = "a" * 40 -COHORT = "unit-2-apm-darwin-x86_64" -ROOT = Path(__file__).resolve().parents[3] - - -@pytest.fixture -def identity(monkeypatch: pytest.MonkeyPatch) -> dict: - value = { - "source_sha": SHA, - "selection": "tests/unit tests/test_console.py", - "candidate": None, - "environment": { - "python": "3.12.10", - "system": "Darwin", - "release": "24.6.0", - "machine": "x86_64", - "cpu_count": "4", - "runner_image": "20260831.1", - "dependency_lock": "f" * 64, - }, - } - monkeypatch.setenv("RUNNER_NAME", "GitHub Actions test") - monkeypatch.setenv("GITHUB_JOB", "unit-tests") - monkeypatch.setattr(performance_cohort, "capture_identity", lambda *args: value) - return value - - -def _members(root: Path) -> None: - for member in performance_cohort.MEMBERS: - data = performance_cohort.record( - COHORT, - member, - "123", - 2, - SHA, - "tests/unit tests/test_console.py", - ) - (root / f"{member}.json").write_text(json.dumps(data), encoding="ascii") - - -def test_matching_actual_runners_produce_the_identity_used_by_final_proof( - tmp_path: Path, - identity: dict, -) -> None: - _members(tmp_path) - assert performance_cohort.verify(tmp_path, COHORT, "123", 2, SHA) == identity - - -@pytest.mark.parametrize( - "fault", - [ - "missing", - "extra", - "member", - "run", - "attempt", - "schema", - "sha", - "image", - "empty-image", - "lock", - "selection", - "candidate", - "malformed", - "missing-runner", - "missing-job", - "invalid-time", - "naive-time", - ], -) -def test_preflight_rejects_incomplete_stale_or_incomparable_cohorts( - tmp_path: Path, - identity: dict, - fault: str, -) -> None: - _members(tmp_path) - path = tmp_path / "proposed-2.json" - data = json.loads(path.read_text("ascii")) - if fault == "missing": - path.unlink() - elif fault == "extra": - (tmp_path / "duplicate.json").write_text(json.dumps(data), encoding="ascii") - elif fault == "malformed": - path.write_text("not json", encoding="ascii") - else: - if fault == "member": - data["member"] = "baseline-1" - elif fault == "run": - data["run_id"] = "122" - elif fault == "attempt": - data["run_attempt"] = 1 - elif fault == "schema": - data["schema_version"] = True - elif fault == "sha": - data["identity"]["source_sha"] = "b" * 40 - elif fault == "image": - data["identity"]["environment"]["runner_image"] = "different-image" - elif fault == "empty-image": - data["identity"]["environment"]["runner_image"] = "" - elif fault == "lock": - data["identity"]["environment"]["dependency_lock"] = "b" * 64 - elif fault == "selection": - data["identity"]["selection"] = "fewer tests" - elif fault == "missing-runner": - data["execution"]["runner_name"] = "" - elif fault == "missing-job": - data["execution"].pop("workflow_job") - elif fault == "invalid-time": - data["execution"]["recorded_at"] = "not a timestamp" - elif fault == "naive-time": - data["execution"]["recorded_at"] = "2026-09-07T12:00:00" - else: - data["identity"]["candidate"] = {"schema_version": 0} - path.write_text(json.dumps(data), encoding="ascii") - with pytest.raises((ValueError, KeyError, TypeError)): - performance_cohort.verify(tmp_path, COHORT, "123", 2, SHA) - - -@pytest.mark.parametrize( - ("cohort", "run_id", "attempt"), - [(COHORT, "123", 1), ("../../escape", "123", 2), (COHORT, "0", 2), (COHORT, "123", True)], -) -def test_rejects_unbounded_or_wrong_attempt_context(cohort: str, run_id: str, attempt: int) -> None: - with pytest.raises(ValueError): - performance_cohort.require_context(cohort, run_id, attempt) - - -def test_capture_rejects_missing_hosted_image(identity: dict) -> None: - identity["environment"]["runner_image"] = "" - with pytest.raises(ValueError, match="ImageVersion"): - performance_cohort.record(COHORT, "baseline-1", "123", 2, SHA, "unit") - - -def test_capture_rejects_unknown_member(identity: dict) -> None: - with pytest.raises(ValueError, match="Unknown"): - performance_cohort.record(COHORT, "proposed-3", "123", 2, SHA, "unit") - - -@pytest.mark.parametrize("variable", ["RUNNER_NAME", "GITHUB_JOB"]) -def test_capture_cannot_emit_unbindable_runner_evidence( - identity: dict, - monkeypatch: pytest.MonkeyPatch, - variable: str, -) -> None: - monkeypatch.delenv(variable) - with pytest.raises(ValueError, match="runner and workflow job identity"): - performance_cohort.record(COHORT, "baseline-1", "123", 2, SHA, "unit") - - -def test_integration_cohort_cannot_omit_the_candidate(identity: dict) -> None: - with pytest.raises(ValueError, match="candidate archive metadata"): - performance_cohort.record( - "integration-2-apm-darwin-arm64", "baseline-1", "123", 2, SHA, "integration" - ) - - -def test_actual_node_artifact_waiter_handles_failures_without_network() -> None: - result = subprocess.run( - ["node", "--test", str(ROOT / "scripts/performance-cohort.test.cjs")], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - timeout=30, - ) - assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/scripts/test_release_rehearsal.py b/tests/unit/scripts/test_release_rehearsal.py deleted file mode 100644 index aa73dcc201..0000000000 --- a/tests/unit/scripts/test_release_rehearsal.py +++ /dev/null @@ -1,331 +0,0 @@ -"""Whole-path claims must retain native floors and never add parallel savings.""" - -from __future__ import annotations - -import json -from copy import deepcopy -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from pathlib import Path - -import pytest - -from scripts.compare_test_runs import Report -from scripts.package_release import BINARY_NAMES -from scripts.release_rehearsal import ROOT, SURFACES, evaluate, rehearsal_matrix, source_authorities - -pytestmark = pytest.mark.component -SHA = "a" * 40 -ORIGIN = datetime(2026, 9, 7, tzinfo=timezone.utc) - - -def _at(seconds: float) -> str: - return (ORIGIN + timedelta(seconds=seconds)).isoformat() - - -@pytest.fixture(scope="module") -def source_names() -> list[str]: - return source_authorities() - - -@pytest.fixture -def evidence(tmp_path: Path, source_names: list[str]) -> tuple: - jobs = [] - - def job(name: str, end: float, runner: str = "") -> dict: - entry = { - "id": len(jobs) + 1, - "run_id": 123, - "run_attempt": 1, - "name": name, - "runner_name": runner or f"runner-{len(jobs) + 1}", - "status": "completed", - "conclusion": "success", - "started_at": _at(1), - "completed_at": _at(end), - "steps": [], - } - jobs.append(entry) - return entry - - for name in source_names: - job(name, 100) - for binary in BINARY_NAMES: - integration_end = {"apm-darwin-arm64": 1010, "apm-linux-x86_64": 810}.get(binary, 310) - native_end = { - "apm-darwin-arm64": 1020, - "apm-linux-x86_64": 820, - "apm-darwin-x86_64": 920, - "apm-windows-x86_64": 410, - }.get(binary, 320) - job(f"{binary} / Native Candidate Gate", native_end) - job(f"{binary} / Integration Tests", integration_end) - job(f"{binary} / Build Candidate", 30) - job(f"{binary} / Isolated Release Validation", 50) - if binary != "apm-darwin-x86_64": - job(f"{binary} / Unit Tests / Unit Tests Baseline", 200) - else: - job(f"{binary} / Unit Tests / Unit Performance Probe", 915) - if binary not in {"apm-darwin-arm64", "apm-linux-x86_64"}: - job(f"{binary} / Integration Tests Shard 1 / Integration Tests Shard 1", 300) - if binary == "apm-windows-x86_64": - job(f"{binary} / Candidate Windows Installer", 400) - job("Read-only documentation build / build", 70) - job("Read-only Python distribution build / Build PyPI Distributions", 70) - reports = [] - ends = { - "arm-integration": (1000, 500, 520), - "linux-integration": (800, 450, 470), - "intel-units": (900, 420, 440), - } - for surface in SURFACES: - candidate = ( - None - if surface.suite == "unit" - else { - "schema_version": 1, - "sha": SHA, - "version": "0.30.0", - "binary_name": surface.binary, - "archive": surface.binary + ".tar.gz", - "archive_sha256": "b" * 64, - "executable_sha256": "c" * 64, - } - ) - identity = { - "source_sha": SHA, - "selection": surface.selection, - "candidate": candidate, - "environment": { - "python": "3.12.10", - "system": "Darwin" if "darwin" in surface.binary else "Linux", - "release": "release", - "machine": surface.binary.rsplit("-", 1)[1], - "cpu_count": surface.cpus, - "runner_image": "20260831.1", - "dependency_lock": "f" * 64, - }, - } - cases = {("pytest-nodeid", "test_a"): "passed", ("pytest-nodeid", "test_b"): "skipped"} - root = tmp_path / surface.key - root.mkdir() - for index, member in enumerate(("baseline-1", "proposed-1", "proposed-2")): - variant, shard = member.split("-") - runner = f"{surface.key}-{member}" - job( - f"{surface.binary} / {surface.suite} experiment {member}", - ends[surface.key][index], - runner, - ) - reports.append( - Report( - deepcopy(identity), - variant, - int(shard), - 1 if variant == "baseline" else 2, - ends[surface.key][index] - 40, - cases if index == 0 else dict([list(cases.items())[index - 1]]), - ) - ) - record = { - "schema_version": 1, - "run_id": "123", - "run_attempt": 1, - "cohort": f"{surface.suite}-1-{surface.binary}", - "member": member, - "identity": identity, - "execution": { - "runner_name": runner, - "workflow_job": "tests", - "recorded_at": _at(20), - }, - } - (root / f"{member}.json").write_text(json.dumps(record), encoding="ascii") - metadata = { - "schema_version": 1, - "source_sha": SHA, - "run_id": "123", - "run_attempt": 1, - "created_at": _at(0), - "run_started_at": _at(0), - "jobs": jobs, - } - return metadata, reports, tmp_path - - -def _evaluate(evidence: tuple, source_names: list[str]) -> dict: - metadata, reports, root = evidence - return evaluate(metadata, reports, root, SHA, source_names, 0.15, 0.10) - - -def test_complete_model_uses_maxima_and_reveals_the_next_bottleneck( - evidence: tuple, - source_names: list[str], -) -> None: - result = _evaluate(evidence, source_names) - scenarios = result["scenarios"] - assert scenarios["baseline"]["evidence_ready_seconds"] == 1020 - assert scenarios["arm_only"]["evidence_ready_seconds"] == 905 - assert "apm-darwin-x86_64" in scenarios["arm_only"]["limiting_job"] - assert scenarios["arm_and_linux"]["evidence_ready_seconds"] == 905 - assert scenarios["combined"]["evidence_ready_seconds"] == 540 - assert result["saved_seconds"] == 480 - assert result["overall_gain_fraction"] == pytest.approx(480 / 1020) - assert result["reaches_30_percent_target"] is True - assert result["unchanged_tail_sensitivity"][-1]["gain_fraction"] == pytest.approx(480 / 1200) - authorities = scenarios["combined"]["native_authorities"] - assert len(authorities) == 5 - arm = next(item for item in authorities if item["integration_surface"] == "arm-integration") - assert arm["integration_fan_in_delay_seconds"] == 10 - assert arm["native_gate_delay_seconds"] == 10 - assert arm["modeled_ready_seconds"] == 540 - - -def test_late_required_native_authorities_cannot_be_dropped_to_manufacture_a_win( - evidence: tuple, - source_names: list[str], -) -> None: - metadata, _, _ = evidence - for job in metadata["jobs"]: - if job["name"].endswith("Native Candidate Gate"): - job["completed_at"] = _at(1100) - result = _evaluate(evidence, source_names) - assert result["scenarios"]["baseline"]["evidence_ready_seconds"] == 1100 - assert result["scenarios"]["combined"]["evidence_ready_seconds"] == 1100 - assert result["overall_gain_fraction"] == 0 - assert result["meets_minimum"] is False - - -@pytest.mark.parametrize("fault", ["missing", "failed", "before-prerequisites"]) -def test_native_fan_in_evidence_is_required_and_must_follow_its_inputs( - evidence: tuple, - source_names: list[str], - fault: str, -) -> None: - metadata, _, _ = evidence - gate = next( - job for job in metadata["jobs"] if job["name"] == "apm-darwin-arm64 / Integration Tests" - ) - if fault == "missing": - metadata["jobs"].remove(gate) - elif fault == "failed": - gate["conclusion"] = "failure" - else: - gate["completed_at"] = _at(500) - with pytest.raises(ValueError): - _evaluate(evidence, source_names) - - -def test_fast_shards_do_not_imply_an_overall_gain_when_an_unchanged_job_dominates( - evidence: tuple, - source_names: list[str], -) -> None: - metadata, _, _ = evidence - installer = next( - job for job in metadata["jobs"] if job["name"].endswith("Candidate Windows Installer") - ) - installer["completed_at"] = _at(1200) - gate = next( - job - for job in metadata["jobs"] - if job["name"] == "apm-windows-x86_64 / Native Candidate Gate" - ) - gate["completed_at"] = _at(1210) - result = _evaluate(evidence, source_names) - assert result["meets_lane_thresholds"] is True - assert result["overall_gain_fraction"] == 0 - assert result["meets_minimum"] is False - assert result["scenarios"]["combined"]["limiting_job"] == gate["name"] - - -@pytest.mark.parametrize( - "fault", ["missing", "failed", "skipped", "duplicate", "old-attempt", "old-run", "bad-time"] -) -def test_rejects_incomplete_or_invalid_common_authorities( - evidence: tuple, - source_names: list[str], - fault: str, -) -> None: - metadata, _, _ = evidence - job = metadata["jobs"][0] - if fault == "missing": - metadata["jobs"].pop(0) - elif fault == "duplicate": - metadata["jobs"].append(deepcopy(job)) - elif fault == "old-attempt": - job["run_attempt"] = 2 - elif fault == "old-run": - job["run_id"] = 124 - elif fault == "bad-time": - job["completed_at"] = _at(-1) - else: - job["conclusion"] = fault - with pytest.raises(ValueError): - _evaluate(evidence, source_names) - - -@pytest.mark.parametrize( - "fault", - ["missing-shard", "new-skip", "postflight-drift", "wrong-capacity", "timing-outside-job"], -) -def test_rejects_invalid_paired_proof_without_weakening_the_existing_checker( - evidence: tuple, - source_names: list[str], - fault: str, -) -> None: - _, reports, _ = evidence - if fault == "missing-shard": - reports.pop() - elif fault == "new-skip": - reports[1] = replace(reports[1], cases={("pytest-nodeid", "test_a"): "skipped"}) - elif fault == "timing-outside-job": - reports[1] = replace(reports[1], elapsed_seconds=10000) - else: - for index in range(3): - identity = deepcopy(reports[index].identity) - identity["environment"][ - "runner_image" if fault == "postflight-drift" else "cpu_count" - ] = "different" - reports[index] = replace(reports[index], identity=identity) - with pytest.raises(ValueError): - _evaluate(evidence, source_names) - - -def test_cannot_assign_a_cohort_member_to_a_different_job( - evidence: tuple, - source_names: list[str], -) -> None: - _, _, root = evidence - path = root / "arm-integration/proposed-1.json" - data = json.loads(path.read_text("ascii")) - data["execution"]["runner_name"] = "not-an-observed-runner" - path.write_text(json.dumps(data), encoding="ascii") - with pytest.raises(ValueError, match="uniquely bind"): - _evaluate(evidence, source_names) - - -def test_baseline_native_gate_still_must_succeed( - evidence: tuple, - source_names: list[str], -) -> None: - metadata, _, _ = evidence - gate = next(job for job in metadata["jobs"] if job["name"].endswith("Native Candidate Gate")) - gate["conclusion"] = "failure" - with pytest.raises(ValueError, match="did not succeed"): - _evaluate(evidence, source_names) - - -def test_matrix_preserves_every_platform_and_changes_only_opt_in_inputs() -> None: - catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) - before = deepcopy(catalog) - matrix = rehearsal_matrix(catalog)["include"] - assert catalog == before - assert len(matrix) == 5 - assert {row["binary_name"] for row in matrix if row["unit_performance_probe"]} == { - "apm-darwin-x86_64" - } - assert {row["binary_name"] for row in matrix if row["integration_performance_probe"]} == { - "apm-darwin-arm64", - "apm-linux-x86_64", - } - assert all(row["integration_shard_count"] == 1 for row in matrix) diff --git a/tests/unit/scripts/test_release_wallclock.py b/tests/unit/scripts/test_release_wallclock.py deleted file mode 100644 index cf351450a9..0000000000 --- a/tests/unit/scripts/test_release_wallclock.py +++ /dev/null @@ -1,1353 +0,0 @@ -"""Read-only release wall-clock evidence rejects modeled or unsafe inputs.""" - -from __future__ import annotations - -import io -import json -import stat -import tarfile -import zipfile -from collections.abc import Iterator -from datetime import datetime, timedelta, timezone -from pathlib import Path - -import pytest - -from scripts import release_wallclock -from scripts.package_release import BINARY_NAMES, archive_name, executable_name, file_digest -from scripts.release_wallclock import compare_runs, record_job, verify_artifacts - -pytestmark = pytest.mark.component - -BASE_SHA = "a" * 40 -PROPOSED_SHA = "b" * 40 -BASE_CONTROLLER = "c" * 40 -PROPOSED_CONTROLLER = "d" * 40 -BASE_RUN_HEAD = BASE_CONTROLLER -PROPOSED_RUN_HEAD = "e" * 40 -VERSION = "0.30.0" -ORIGIN = datetime(2026, 9, 8, tzinfo=timezone.utc) - - -def _at(seconds: int) -> str: - return (ORIGIN + timedelta(seconds=seconds)).isoformat().replace("+00:00", "Z") - - -def _source_root(tmp_path: Path) -> Path: - root = tmp_path / "source" - (root / ".github" / "workflows").mkdir(parents=True) - (root / "pyproject.toml").write_text( - f'[project]\nname = "apm-cli"\nversion = "{VERSION}"\n', encoding="ascii" - ) - (root / "uv.lock").write_text("lock\n", encoding="ascii") - (root / ".github" / "workflows" / "build-release.yml").write_text( - "name: release\n", encoding="ascii" - ) - return root - - -def _set_workflow_env( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - sha: str, - run_id: int = 101, - *, - event_name: str = "push", - run_head_sha: str | None = None, -) -> None: - run_head_sha = run_head_sha or sha - payload = ( - {"pull_request": {"head": {"sha": run_head_sha}}} - if event_name == "pull_request" - else {"after": run_head_sha} - ) - event_path = tmp_path / f"event-{run_id}.json" - event_path.write_text(json.dumps(payload), encoding="ascii") - monkeypatch.setenv("GITHUB_SHA", sha) - monkeypatch.setenv("GITHUB_RUN_ID", str(run_id)) - monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") - monkeypatch.setenv("GITHUB_EVENT_NAME", event_name) - monkeypatch.setenv("GITHUB_EVENT_PATH", str(event_path)) - monkeypatch.setenv("GITHUB_JOB", "verify-wallclock") - monkeypatch.setenv("RUNNER_NAME", "runner-1") - monkeypatch.setenv("ImageVersion", "20260901.1") - - -def _tar_archive(path: Path, binary_name: str, payload: bytes = b"native") -> str: - executable = f"{binary_name}/{executable_name(binary_name)}" - with tarfile.open(path, "w:gz") as target: - directory = tarfile.TarInfo(binary_name) - directory.type = tarfile.DIRTYPE - directory.mode = 0o755 - target.addfile(directory) - info = tarfile.TarInfo(executable) - info.mode = 0o755 - info.size = len(payload) - target.addfile(info, io.BytesIO(payload)) - hidden = tarfile.TarInfo(f"{binary_name}/_internal/.apm") - hidden.mode = 0o644 - hidden.size = 1 - target.addfile(hidden, io.BytesIO(b"x")) - return file_digest(path) - - -def _zip_archive(path: Path, binary_name: str, payload: bytes = b"native") -> str: - executable = f"{binary_name}/{executable_name(binary_name)}" - with zipfile.ZipFile(path, "w") as target: - info = zipfile.ZipInfo(executable) - info.create_system = 3 - info.external_attr = (stat.S_IFREG | 0o644) << 16 - target.writestr(info, payload) - target.writestr(f"{binary_name}/_internal/.apm", b"x") - return file_digest(path) - - -def _native_artifacts(root: Path, side: str, source_sha: str) -> Path: - root.mkdir(parents=True) - for binary_name in BINARY_NAMES: - archive = archive_name(binary_name) - archive_path = root / archive - payload = f"native-{binary_name}".encode("ascii") - if archive.endswith(".zip"): - digest = _zip_archive(archive_path, binary_name, payload) - else: - digest = _tar_archive(archive_path, binary_name, payload) - (root / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") - executable_digest = ( - release_wallclock._zip_member_digest(archive_path, binary_name)[0] - if archive.endswith(".zip") - else release_wallclock._tar_member_digest(archive_path, binary_name)[0] - ) - if side == "proposed": - metadata = { - "schema_version": 1, - "sha": source_sha, - "version": VERSION, - "binary_name": binary_name, - "archive": archive, - "archive_sha256": digest, - "executable_sha256": executable_digest, - } - (root / f"{binary_name}.json").write_text( - json.dumps(metadata, sort_keys=True), encoding="ascii" - ) - elif side == "baseline": - loose = root / binary_name / executable_name(binary_name) - loose.parent.mkdir(parents=True, exist_ok=True) - loose.write_bytes(payload) - (root / f"{binary_name}.sha256").write_text( - f"{executable_digest} {binary_name}/{executable_name(binary_name)}\n", - encoding="ascii", - ) - return root - - -def _docs_artifact(root: Path) -> Path: - root.mkdir() - with tarfile.open(root / "artifact.tar", "w") as target: - directory = tarfile.TarInfo("./") - directory.type = tarfile.DIRTYPE - directory.mode = 0o755 - target.addfile(directory) - info = tarfile.TarInfo("./index.html") - body = b"docs" - info.size = len(body) - info.mode = 0o644 - target.addfile(info, io.BytesIO(body)) - return root - - -def _python_distributions(root: Path, version: str = VERSION, name: str = "apm-cli") -> Path: - root.mkdir(exist_ok=True) - token = name.replace("-", "_") - with zipfile.ZipFile(root / f"apm_cli-{version}-py3-none-any.whl", "w") as wheel: - wheel.writestr("apm_cli/__init__.py", "") - wheel.writestr( - f"apm_cli-{version}.dist-info/METADATA", - f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n", - ) - wheel.writestr( - f"apm_cli-{version}.dist-info/WHEEL", - "Wheel-Version: 1.0\nGenerator: fixture\nRoot-Is-Purelib: true\nTag: py3-none-any\n", - ) - wheel.writestr(f"apm_cli-{version}.dist-info/RECORD", "apm_cli/__init__.py,,\n") - with tarfile.open(root / f"apm_cli-{version}.tar.gz", "w:gz") as sdist: - for member, body in { - f"{token}-{version}/PKG-INFO": ( - f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() - ), - f"{token}-{version}/pyproject.toml": b"[project]\nname='apm-cli'\n", - f"{token}-{version}/src/{token}/__init__.py": b"", - }.items(): - info = tarfile.TarInfo(member) - info.size = len(body) - info.mode = 0o644 - sdist.addfile(info, io.BytesIO(body)) - return root - - -def _verified_fixture( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, side: str, source_sha: str, controller: str -) -> tuple[Path, Path, Path, Path]: - source = _source_root(tmp_path) - native = _native_artifacts(tmp_path / "native", side, source_sha) - docs = _docs_artifact(tmp_path / "docs") - python = _python_distributions(tmp_path / "python") - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: source_sha) - _set_workflow_env(monkeypatch, tmp_path, controller) - (native / "runner.json").write_text( - json.dumps(record_job(side, source, source_sha)), encoding="ascii" - ) - return source, native, docs, python - - -def test_record_job_captures_minimal_host_identity_without_env_dump( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) - _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) - monkeypatch.setenv("SECRET_TOKEN", "do-not-copy") - - evidence = record_job("baseline", source, BASE_SHA) - - assert evidence["kind"] == "release-wallclock-job" - assert evidence["workflow"]["source_sha"] == BASE_SHA - assert evidence["workflow"]["controller_sha"] == BASE_CONTROLLER - assert evidence["workflow"]["run_head_sha"] == BASE_CONTROLLER - assert evidence["environment"]["image_version"] == "20260901.1" - assert "SECRET_TOKEN" not in json.dumps(evidence) - - -def test_record_job_keeps_pull_request_head_distinct_from_execution_sha( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) - _set_workflow_env( - monkeypatch, - tmp_path, - PROPOSED_CONTROLLER, - event_name="pull_request", - run_head_sha=PROPOSED_RUN_HEAD, - ) - - evidence = record_job("proposed", source, PROPOSED_SHA) - - assert evidence["workflow"]["source_sha"] == PROPOSED_SHA - assert evidence["workflow"]["controller_sha"] == PROPOSED_CONTROLLER - assert evidence["workflow"]["run_head_sha"] == PROPOSED_RUN_HEAD - assert evidence["workflow"]["run_head_sha"] != evidence["workflow"]["controller_sha"] - assert evidence["workflow"]["event_name"] == "pull_request" - - -def test_record_job_fails_bad_source_identity_and_missing_hosted_image( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) - _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) - - with pytest.raises(ValueError, match="does not match expected"): - record_job("baseline", source, BASE_SHA) - - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) - monkeypatch.delenv("ImageVersion") - with pytest.raises(ValueError, match="ImageVersion"): - record_job("baseline", source, BASE_SHA) - - -@pytest.mark.parametrize( - "side,sha,controller,metadata_count", - [ - ("baseline", BASE_SHA, BASE_CONTROLLER, 0), - ("proposed", PROPOSED_SHA, PROPOSED_CONTROLLER, 5), - ], -) -def test_verify_artifacts_emits_nonpromotable_real_archive_proof( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - side: str, - sha: str, - controller: str, - metadata_count: int, -) -> None: - source, native, docs, python = _verified_fixture(tmp_path, monkeypatch, side, sha, controller) - - proof = verify_artifacts(side, source, sha, native, docs, python) - - assert proof["kind"] == "release-wallclock-proof" - assert proof["promotable"] is False - assert "release-candidate-evidence" not in json.dumps(proof) - assert proof["side"] == side - assert proof["source_sha"] == sha - assert proof["controller_sha"] == controller - assert proof["counts"]["native_archives"] == 5 - assert proof["counts"]["candidate_metadata"] == metadata_count - assert proof["counts"]["job_records"] == 1 - assert proof["filehashes"]["docs"]["format"] == "tar" - assert proof["filehashes"]["python"]["wheel"]["version"] == VERSION - original = json.loads((native / "runner.json").read_text(encoding="ascii")) - for field in ("schema_version", "recorded_at", "environment"): - assert proof["job_records"][0][field] == original[field] - # One terminal observer does not cover either native DAG. - assert proof["observation_assessment"]["status"] == "inconclusive" - - -@pytest.mark.parametrize( - "side,sha,controller", - [ - ("baseline", BASE_SHA, BASE_CONTROLLER), - ("proposed", PROPOSED_SHA, PROPOSED_CONTROLLER), - ], -) -def test_verify_rejects_missing_observations_before_emitting_a_proof( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - side: str, - sha: str, - controller: str, -) -> None: - source, native, docs, python = _verified_fixture(tmp_path, monkeypatch, side, sha, controller) - (native / "runner.json").unlink() - - with pytest.raises(ValueError, match="must contain at least one job record"): - verify_artifacts(side, source, sha, native, docs, python) - - -def test_verify_proof_keeps_pull_request_head_distinct_from_execution_sha( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - native = _native_artifacts(tmp_path / "native", "proposed", PROPOSED_SHA) - docs = _docs_artifact(tmp_path / "docs") - python = _python_distributions(tmp_path / "python") - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: PROPOSED_SHA) - _set_workflow_env( - monkeypatch, - tmp_path, - PROPOSED_CONTROLLER, - event_name="pull_request", - run_head_sha=PROPOSED_RUN_HEAD, - ) - (native / "runner.json").write_text( - json.dumps(record_job("proposed", source, PROPOSED_SHA)), encoding="ascii" - ) - - proof = verify_artifacts("proposed", source, PROPOSED_SHA, native, docs, python) - - assert proof["source_sha"] == PROPOSED_SHA - assert proof["controller_sha"] == PROPOSED_CONTROLLER - assert proof["run_head_sha"] == PROPOSED_RUN_HEAD - assert proof["event_name"] == "pull_request" - - -def test_verify_baseline_archives_raw_layout_binds_archives_to_original_executables( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - flat_native = _native_artifacts(tmp_path / "flat-native", "baseline", BASE_SHA) - native = tmp_path / "native-layout" - archives = native / "archives" - raw = native / "raw" / "candidate-1-apm-linux-x86_64" / "dist" - archives.mkdir(parents=True) - raw.mkdir(parents=True) - for artifact in flat_native.iterdir(): - if artifact.is_file() and artifact.name.endswith((".tar.gz", ".zip")): - (archives / artifact.name).write_bytes(artifact.read_bytes()) - for binary_name in BINARY_NAMES: - archive = archive_name(binary_name) - (archives / f"{archive}.sha256").write_bytes( - (flat_native / f"{archive}.sha256").read_bytes() - ) - executable_digest = ( - release_wallclock._zip_member_digest(archives / archive, binary_name)[0] - if archive.endswith(".zip") - else release_wallclock._tar_member_digest(archives / archive, binary_name)[0] - ) - loose = raw / binary_name / executable_name(binary_name) - loose.parent.mkdir() - loose.write_bytes((flat_native / binary_name / executable_name(binary_name)).read_bytes()) - (raw / f"{binary_name}.sha256").write_text( - f"{executable_digest} dist/{binary_name}/{executable_name(binary_name)}\n", - encoding="ascii", - ) - docs = _docs_artifact(tmp_path / "docs") - python = _python_distributions(tmp_path / "python") - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) - _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) - - (native / "runner.json").write_text( - json.dumps(record_job("baseline", source, BASE_SHA)), encoding="ascii" - ) - proof = verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - observed = proof["filehashes"]["native"]["apm-linux-x86_64"] - assert observed["loose_executable_sidecar_sha256"] == file_digest( - raw / "apm-linux-x86_64.sha256" - ) - assert proof["counts"]["candidate_metadata"] == 0 - - -def test_verify_baseline_raw_layout_rejects_original_executable_digest_drift( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = _source_root(tmp_path) - flat_native = _native_artifacts(tmp_path / "flat-native", "baseline", BASE_SHA) - native = tmp_path / "native-layout" - archives = native / "archives" - raw = native / "raw" / "candidate-1-apm-linux-x86_64" / "dist" - archives.mkdir(parents=True) - raw.mkdir(parents=True) - for artifact in flat_native.iterdir(): - if artifact.is_file() and artifact.name.endswith((".tar.gz", ".zip")): - (archives / artifact.name).write_bytes(artifact.read_bytes()) - for binary_name in BINARY_NAMES: - archive = archive_name(binary_name) - (archives / f"{archive}.sha256").write_bytes( - (flat_native / f"{archive}.sha256").read_bytes() - ) - loose = raw / binary_name / executable_name(binary_name) - loose.parent.mkdir() - loose.write_bytes((flat_native / binary_name / executable_name(binary_name)).read_bytes()) - digest = file_digest(loose) - if binary_name == "apm-linux-x86_64": - digest = "1" * 64 - (raw / f"{binary_name}.sha256").write_text( - f"{digest} dist/{binary_name}/{executable_name(binary_name)}\n", - encoding="ascii", - ) - docs = _docs_artifact(tmp_path / "docs") - python = _python_distributions(tmp_path / "python") - monkeypatch.setattr(release_wallclock, "_git_head", lambda root: BASE_SHA) - _set_workflow_env(monkeypatch, tmp_path, BASE_CONTROLLER) - - with pytest.raises(ValueError, match="Loose executable checksum mismatch"): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -@pytest.mark.parametrize("fault", ["missing-sidecar", "missing-payload"]) -def test_verify_baseline_requires_all_raw_sidecars_and_loose_executables( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - binary_name = "apm-linux-x86_64" - if fault == "missing-sidecar": - (native / f"{binary_name}.sha256").unlink() - match = "raw executable checksum is missing" - else: - (native / binary_name / executable_name(binary_name)).unlink() - match = "raw executable payload is missing" - - with pytest.raises(ValueError, match=match): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -def test_verify_rejects_proposed_archive_without_candidate_json( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "proposed", PROPOSED_SHA, PROPOSED_CONTROLLER - ) - (native / "apm-linux-x86_64.json").unlink() - - with pytest.raises(ValueError, match="requires candidate metadata"): - verify_artifacts("proposed", source, PROPOSED_SHA, native, docs, python) - - -def test_verify_rejects_unsafe_archive_member_even_with_matching_sidecar( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - binary_name = "apm-linux-x86_64" - archive = archive_name(binary_name) - archive_path = native / archive - with tarfile.open(archive_path, "w:gz") as target: - info = tarfile.TarInfo(f"{binary_name}/../escape") - info.size = 4 - target.addfile(info, io.BytesIO(b"nope")) - digest = file_digest(archive_path) - (native / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") - - with pytest.raises(ValueError, match="unsafe member path"): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -def test_verify_rejects_duplicate_executable_members( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - binary_name = "apm-linux-arm64" - archive = archive_name(binary_name) - executable = f"{binary_name}/{executable_name(binary_name)}" - archive_path = native / archive - with tarfile.open(archive_path, "w:gz") as target: - for body in (b"one", b"two"): - info = tarfile.TarInfo(executable) - info.mode = 0o755 - info.size = len(body) - target.addfile(info, io.BytesIO(body)) - digest = file_digest(archive_path) - (native / f"{archive}.sha256").write_text(f"{digest} {archive}\n", encoding="ascii") - - with pytest.raises(ValueError, match="Duplicate archive member"): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -def _tar_with_link(path: Path, link_type: bytes, linkname: str) -> None: - binary_name = "apm-darwin-arm64" - with tarfile.open(path, "w:gz") as target: - executable = tarfile.TarInfo(f"{binary_name}/{executable_name(binary_name)}") - executable.mode = 0o755 - executable.size = 6 - target.addfile(executable, io.BytesIO(b"native")) - real = tarfile.TarInfo(f"{binary_name}/_internal/real-target") - real.mode = 0o644 - real.size = 4 - target.addfile(real, io.BytesIO(b"real")) - link = tarfile.TarInfo(f"{binary_name}/_internal/Foo.framework/Versions/Current") - link.type = link_type - link.linkname = linkname - link.mode = 0o777 - target.addfile(link) - - -def test_tar_validator_accepts_real_format_macos_internal_symlinks(tmp_path: Path) -> None: - binary_name = "apm-darwin-arm64" - archive = tmp_path / archive_name(binary_name) - with tarfile.open(archive, "w:gz") as target: - executable = tarfile.TarInfo(f"{binary_name}/{executable_name(binary_name)}") - executable.mode = 0o755 - executable.size = 6 - target.addfile(executable, io.BytesIO(b"native")) - for member, linkname in ( - (f"{binary_name}/_internal/Python", "Python.framework/Versions/3.12/Python"), - (f"{binary_name}/_internal/Python.framework/Python", "Versions/Current/Python"), - ( - f"{binary_name}/_internal/Python.framework/Resources", - "Versions/Current/Resources", - ), - (f"{binary_name}/_internal/Python.framework/Versions/Current", "3.12"), - ): - link = tarfile.TarInfo(member) - link.type = tarfile.SYMTYPE - link.linkname = linkname - target.addfile(link) - - digest, count = release_wallclock._tar_member_digest(archive, binary_name) - - assert digest == release_wallclock.file_digest_stream(io.BytesIO(b"native")) - assert count == 5 - - -def test_tar_validator_rejects_escaping_symlink(tmp_path: Path) -> None: - archive = tmp_path / "escaping-symlink.tar.gz" - _tar_with_link(archive, tarfile.SYMTYPE, "../../../../outside") - - with pytest.raises(ValueError, match="symbolic link escapes"): - release_wallclock._tar_member_digest(archive, "apm-darwin-arm64") - - -def test_tar_validator_uses_archive_root_for_hardlinks(tmp_path: Path) -> None: - archive = tmp_path / "escaping-hardlink.tar.gz" - _tar_with_link(archive, tarfile.LNKTYPE, "outside/target") - - with pytest.raises(ValueError, match="hard link escapes"): - release_wallclock._tar_member_digest(archive, "apm-darwin-arm64") - - safe = tmp_path / "safe-hardlink.tar.gz" - _tar_with_link(safe, tarfile.LNKTYPE, "apm-darwin-arm64/_internal/real-target") - release_wallclock._tar_member_digest(safe, "apm-darwin-arm64") - - -@pytest.mark.parametrize("fault", ["empty-docs-index", "python-version"]) -def test_verify_rejects_mismatched_metadata_or_missing_docs( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - if fault == "empty-docs-index": - docs.joinpath("artifact.tar").unlink() - docs.joinpath("index.html").write_text("", encoding="ascii") - match = "docs index.html is empty" - else: - python = _python_distributions(tmp_path / "wrong-python", "9.9.9") - match = "project/version" - - with pytest.raises(ValueError, match=match): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -@pytest.mark.parametrize( - "fault", ["wrong-wheel-name", "missing-wheel-record", "sdist-metadata-only"] -) -def test_verify_rejects_fake_or_wrong_project_python_distributions( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fault: str -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - for path in python.iterdir(): - path.unlink() - if fault == "wrong-wheel-name": - _python_distributions(python, VERSION, name="evil-pkg") - match = "Wheel metadata name/version" - elif fault == "missing-wheel-record": - with zipfile.ZipFile(python / f"apm_cli-{VERSION}-py3-none-any.whl", "w") as wheel: - wheel.writestr("apm_cli/__init__.py", "") - wheel.writestr( - f"apm_cli-{VERSION}.dist-info/METADATA", - f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n", - ) - wheel.writestr(f"apm_cli-{VERSION}.dist-info/WHEEL", "Wheel-Version: 1.0\n") - with tarfile.open(python / f"apm_cli-{VERSION}.tar.gz", "w:gz") as sdist: - for member, body in { - f"apm_cli-{VERSION}/PKG-INFO": ( - f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n".encode() - ), - f"apm_cli-{VERSION}/pyproject.toml": b"[project]\nname='apm-cli'\n", - f"apm_cli-{VERSION}/src/apm_cli/__init__.py": b"", - }.items(): - info = tarfile.TarInfo(member) - info.size = len(body) - sdist.addfile(info, io.BytesIO(body)) - match = "missing METADATA, WHEEL or RECORD" - else: - with zipfile.ZipFile(python / f"apm_cli-{VERSION}-py3-none-any.whl", "w") as wheel: - wheel.writestr("apm_cli/__init__.py", "") - wheel.writestr( - f"apm_cli-{VERSION}.dist-info/METADATA", - f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n", - ) - wheel.writestr(f"apm_cli-{VERSION}.dist-info/WHEEL", "Wheel-Version: 1.0\n") - wheel.writestr(f"apm_cli-{VERSION}.dist-info/RECORD", "apm_cli/__init__.py,,\n") - with tarfile.open(python / f"apm_cli-{VERSION}.tar.gz", "w:gz") as sdist: - body = f"Metadata-Version: 2.1\nName: apm-cli\nVersion: {VERSION}\n".encode() - info = tarfile.TarInfo(f"apm_cli-{VERSION}/PKG-INFO") - info.size = len(body) - sdist.addfile(info, io.BytesIO(body)) - match = "top-level pyproject.toml" - - with pytest.raises(ValueError, match=match): - verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - -def _observation_records(side: str) -> list[dict[str, object]]: - """Model the real eight-observer baseline and fifteen-observer proposed DAG.""" - records = [] - for system, arch in ( - ("Linux", "x86_64"), - ("Linux", "aarch64"), - ("Windows", "AMD64"), - ("Darwin", "x86_64"), - ("Darwin", "arm64"), - ): - if side == "proposed": - jobs = ("unit-tests", "build", "integration-tests") - elif system == "Darwin": - jobs = ( - "build-and-validate-macos-intel" - if arch == "x86_64" - else "build-and-validate-macos-arm", - ) - else: - jobs = ("build-and-test", "integration-tests") - for job in jobs: - records.append( - { - "path": f"wallclock/{side}-{system}-{arch}-{job}.json", - "github_job": job, - "runner_name": f"{side}-{system}-{arch}-{job}", - "schema_version": 1, - "recorded_at": _at(10), - "environment": { - "system": system, - "kernel": "fixture-kernel", - "arch": arch, - "cpu": f"fixture-{arch}", - "cpu_count": "4", - "image_version": f"fixture-{system}-{arch}-20260901.1", - "python": "3.12.10", - "source_uv_lock_sha256": "1" * 64, - "source_build_release_yml_sha256": "e" * 64, - }, - } - ) - return records - - -def _proof( - path: Path, - side: str, - run_id: int, - source_sha: str, - controller_sha: str, - run_head_sha: str, -) -> Path: - observations = _observation_records(side) - native = {} - for index, binary_name in enumerate(BINARY_NAMES, start=1): - archive = archive_name(binary_name) - executable_digest = f"{index + 10:064x}" - native[binary_name] = { - "binary_name": binary_name, - "archive": archive, - "archive_sha256": f"{index:064x}", - "executable_sha256": executable_digest, - "sidecar": f"{archive}.sha256", - "candidate_metadata_sha256": f"{index + 20:064x}" if side == "proposed" else None, - "loose_executable_sidecar_sha256": f"{index + 30:064x}" if side == "baseline" else None, - "loose_executable_sha256": executable_digest if side == "baseline" else None, - "member_count": 2, - } - path.write_text( - json.dumps( - { - "kind": "release-wallclock-proof", - "schema_version": 1, - "promotable": False, - "side": side, - "source_sha": source_sha, - "controller_sha": controller_sha, - "run_head_sha": run_head_sha, - "event_name": "pull_request" if run_head_sha != controller_sha else "push", - "run_id": run_id, - "run_attempt": 1, - "version": VERSION, - "workflow_digest": "e" * 64, - "counts": { - "native_archives": 5, - "native_sidecars": 5, - "candidate_metadata": 5 if side == "proposed" else 0, - "job_records": len(observations), - "docs_indexes": 1, - "python_distributions": 2, - }, - "filehashes": { - "source_uv_lock_sha256": "1" * 64, - "source_build_release_yml_sha256": "e" * 64, - "native": native, - "docs": { - "format": "tar", - "archive": "artifact.tar", - "archive_sha256": "2" * 64, - "index": "index.html", - }, - "python": { - "source_project_name": "apm-cli", - "wheel": { - "name": f"apm_cli-{VERSION}-py3-none-any.whl", - "sha256": "3" * 64, - "project_name": "apm-cli", - "version": VERSION, - }, - "sdist": { - "name": f"apm_cli-{VERSION}.tar.gz", - "sha256": "4" * 64, - "project_name": "apm-cli", - "version": VERSION, - }, - }, - }, - "job_records": observations, - } - ), - encoding="ascii", - ) - return path - - -def _job(run_id: int, name: str, start: int, end: int, conclusion: str = "success") -> dict: - return { - "id": f"{run_id}-{name}", - "run_id": run_id, - "run_attempt": 1, - "name": name, - "status": "completed", - "conclusion": conclusion, - "started_at": _at(start), - "completed_at": _at(end), - } - - -def _run_json( - path: Path, - run_id: int, - run_head_sha: str, - run_started: int, - terminal_completed: int, -) -> Path: - jobs = [ - _job(run_id, "Build/test/native/docs/wheel proof", run_started + 10, run_started + 20), - { - "id": f"{run_id}-skipped", - "run_id": run_id, - "run_attempt": 1, - "name": "Imported PR-only workflow", - "status": "completed", - "conclusion": "skipped", - }, - _job(run_id, "Verify wall-clock artifacts", terminal_completed - 5, terminal_completed), - ] - payload = { - "run": { - "id": run_id, - "run_attempt": 1, - "status": "completed", - "conclusion": "success", - "head_sha": run_head_sha, - "pull_requests": [{"head": {"sha": "f" * 40}}], - "created_at": _at(0), - "run_started_at": _at(run_started), - }, - "jobs": jobs, - } - path.write_text(json.dumps(payload), encoding="ascii") - return path - - -def _compare_fixture(tmp_path: Path) -> tuple[Path, Path, Path, Path]: - return ( - _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100), - _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90), - _proof( - tmp_path / "baseline-proof.json", - "baseline", - 201, - BASE_SHA, - BASE_CONTROLLER, - BASE_RUN_HEAD, - ), - _proof( - tmp_path / "proposed-proof.json", - "proposed", - 301, - PROPOSED_SHA, - PROPOSED_CONTROLLER, - PROPOSED_RUN_HEAD, - ), - ) - - -def _compare_ok( - baseline_run: Path, - proposed_run: Path, - baseline_proof: Path, - proposed_proof: Path, - baseline_required: tuple[str, ...] = ("Build/test/native/docs/wheel proof",), - proposed_required: tuple[str, ...] = ("Build/test/native/docs/wheel proof",), -) -> dict[str, object]: - return compare_runs( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - PROPOSED_CONTROLLER, - BASE_RUN_HEAD, - PROPOSED_RUN_HEAD, - "Verify wall-clock artifacts", - baseline_required, - proposed_required, - ) - - -def test_compare_matches_distinct_native_dags_without_claiming_controlled_acceptance( - tmp_path: Path, -) -> None: - result = _compare_ok(*_compare_fixture(tmp_path)) - - assert len(_observation_records("baseline")) == 8 - assert len(_observation_records("proposed")) == 15 - for side in ("baseline", "proposed"): - assessment = result[side]["observation_assessment"] - assert assessment["status"] == "complete" - assert set(assessment["roles"]) == { - f"{role}/{binary_name}" - for role in ("unit", "build", "integration") - for binary_name in BINARY_NAMES - } - assert result["clock_arithmetic"] == "validated" - assert result["comparability"]["status"] == "matched" - assert result["comparability"]["issues"] == [] - assert result["comparability"]["controlled_acceptance"] is False - assert result["production_qualification"] is False - - -@pytest.mark.parametrize("field", ["schema_version", "recorded_at", "environment"]) -def test_job_record_ingestion_preserves_missing_metadata_as_unassessed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str -) -> None: - source, native, docs, python = _verified_fixture( - tmp_path, monkeypatch, "baseline", BASE_SHA, BASE_CONTROLLER - ) - path = native / "runner.json" - payload = json.loads(path.read_text(encoding="ascii")) - payload.pop(field) - path.write_text(json.dumps(payload), encoding="ascii") - - proof = verify_artifacts("baseline", source, BASE_SHA, native, docs, python) - - assert proof["job_records"][0][field] is None - assert proof["observation_assessment"]["status"] == "unassessed" - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("schema_version", None), - ("schema_version", True), - ("schema_version", 1.0), - ("schema_version", 2), - ("recorded_at", None), - ("recorded_at", "not-a-time"), - ("recorded_at", "2026-09-08T00:00:10"), - ("environment", None), - ("environment", {}), - ("image_version", ""), - ("cpu_count", "0"), - ("cpu_count", True), - ("source_uv_lock_sha256", "not-a-digest"), - ], -) -def test_compare_incomplete_metadata_cannot_count_as_controlled_gain( - tmp_path: Path, field: str, value: object -) -> None: - paths = _compare_fixture(tmp_path) - payload = json.loads(paths[3].read_text(encoding="ascii")) - record = payload["job_records"][0] - target = ( - record - if field in {"schema_version", "recorded_at", "environment"} - else record["environment"] - ) - if value is None: - target.pop(field) - else: - target[field] = value - # A saved claim must not override recomputation. - payload["observation_assessment"] = {"status": "complete", "issues": []} - paths[3].write_text(json.dumps(payload), encoding="ascii") - - result = _compare_ok(*paths) - - assert result["comparability"]["status"] == "unassessed" - assert result["comparability"]["issues"] - assert result["comparability"]["controlled_acceptance"] is False - assert result["delta_seconds"]["primary_gain_seconds"] == 10 - assert result["clock_arithmetic"] == "validated" - - -def test_compare_legacy_projected_records_keep_raw_clocks_without_inventing_metadata( - tmp_path: Path, -) -> None: - paths = _compare_fixture(tmp_path) - for path in paths[2:]: - payload = json.loads(path.read_text(encoding="ascii")) - payload["job_records"] = [ - {"path": "historical/job.json", "github_job": "build", "runner_name": "legacy"} - ] - payload["counts"]["job_records"] = 1 - path.write_text(json.dumps(payload), encoding="ascii") - - result = _compare_ok(*paths) - - assert result["gain_percent"]["primary"] == 10 - assert result["baseline"]["wallclock_seconds_from_created_at"] == 100 - assert result["proposed"]["wallclock_seconds_from_created_at"] == 90 - assert result["comparability"]["status"] == "unassessed" - assert result["comparability"]["controlled_acceptance"] is False - assert result["baseline"]["observation_assessment"]["roles"] == {} - - -@pytest.mark.parametrize( - ("side", "index"), - [("baseline", index) for index in range(8)] + [("proposed", index) for index in range(15)], -) -def test_compare_requires_every_observer_role_and_platform( - tmp_path: Path, side: str, index: int -) -> None: - paths = _compare_fixture(tmp_path) - path = paths[2] if side == "baseline" else paths[3] - payload = json.loads(path.read_text(encoding="ascii")) - payload["job_records"].pop(index) - payload["counts"]["job_records"] -= 1 - path.write_text(json.dumps(payload), encoding="ascii") - - result = _compare_ok(*paths) - - assert result[side]["observation_assessment"]["status"] == "inconclusive" - assert any("Missing observer roles:" in issue for issue in result["comparability"]["issues"]) - assert result["comparability"]["controlled_acceptance"] is False - assert result["delta_seconds"]["primary_gain_seconds"] == 10 - - -@pytest.mark.parametrize( - ("fault", "message"), - [ - ("image", "image_version differs"), - ("cpu", "cpu_count differs"), - ("duplicate-role", "duplicate observer role/platform"), - ("wrong-platform", "Missing observer roles"), - ("wrong-job", "unexpected observer role/platform"), - ("timestamp", "outside the run timeline"), - ("timestamp-before", "outside the run timeline"), - ("source-digest", "does not match proof"), - ], -) -def test_compare_environment_drift_and_ambiguous_coverage_are_inconclusive( - tmp_path: Path, fault: str, message: str -) -> None: - paths = _compare_fixture(tmp_path) - payload = json.loads(paths[3].read_text(encoding="ascii")) - record = payload["job_records"][0] - if fault == "image": - record["environment"]["image_version"] = "different-image" - elif fault == "cpu": - record["environment"]["cpu_count"] = "8" - elif fault == "duplicate-role": - duplicate = dict(record, runner_name="another-runner") - payload["job_records"].append(duplicate) - payload["counts"]["job_records"] += 1 - elif fault == "wrong-platform": - record["environment"]["arch"] = "aarch64" - elif fault == "wrong-job": - record["github_job"] = "unrecognized-job" - elif fault == "timestamp": - record["recorded_at"] = _at(91) - elif fault == "timestamp-before": - record["recorded_at"] = _at(-1) - else: - record["environment"]["source_build_release_yml_sha256"] = "f" * 64 - paths[3].write_text(json.dumps(payload), encoding="ascii") - - result = _compare_ok(*paths) - - assert result["comparability"]["status"] == "inconclusive" - assert any(message in issue for issue in result["comparability"]["issues"]) - assert result["comparability"]["controlled_acceptance"] is False - assert result["delta_seconds"]["primary_gain_seconds"] == 10 - - -def test_compare_allows_intentional_workflow_digest_change_between_sides(tmp_path: Path) -> None: - paths = _compare_fixture(tmp_path) - payload = json.loads(paths[3].read_text(encoding="ascii")) - payload["workflow_digest"] = "f" * 64 - payload["filehashes"]["source_build_release_yml_sha256"] = "f" * 64 - for record in payload["job_records"]: - record["environment"]["source_build_release_yml_sha256"] = "f" * 64 - paths[3].write_text(json.dumps(payload), encoding="ascii") - - result = _compare_ok(*paths) - - assert result["comparability"]["status"] == "matched" - assert result["comparability"]["controlled_acceptance"] is False - - -def test_observation_validation_record_visits_scale_linearly() -> None: - """Count scans and identity comparisons at N and 10N without wall clocks.""" - - class CountedName(str): - """Expose a quadratic list-based replacement for identity set membership.""" - - comparisons = 0 - - def __eq__(self, other: object) -> bool: - type(self).comparisons += 1 - return super().__eq__(other) - - def __hash__(self) -> int: - return super().__hash__() - - class CountedRecords(list[dict[str, object]]): - """Expose repeated scans, including index-based scans, deterministically.""" - - visits = 0 - - def __iter__(self) -> Iterator[dict[str, object]]: - for record in super().__iter__(): - self.visits += 1 - yield record - - def __getitem__(self, key: int) -> dict[str, object]: - self.visits += 1 - return super().__getitem__(key) - - visits = [] - template = _observation_records("proposed")[0] - for size in (30, 300): - CountedName.comparisons = 0 - records = CountedRecords( - dict(template, runner_name=CountedName(f"runner-{index}")) for index in range(size) - ) - assessment = release_wallclock._validate_job_records( - {"side": "proposed", "job_records": records}, {"job_records": size} - ) - assert assessment["status"] == "inconclusive" # Duplicate roles, not identities. - operations = records.visits + CountedName.comparisons - assert size <= operations <= 3 * size - visits.append(operations) - assert visits[1] <= 12 * visits[0] - - -@pytest.mark.parametrize( - ("fault", "match"), - [ - ("empty-filehashes", "workflow digest"), - ("missing-native", "native inventory"), - ("missing-proposed-metadata", "candidate_metadata_sha256"), - ("empty-jobs", "at least one job record"), - ], -) -def test_compare_rejects_structurally_fake_or_incomplete_proofs( - tmp_path: Path, fault: str, match: str -) -> None: - baseline_run, proposed_run, baseline_proof, proposed_proof = _compare_fixture(tmp_path) - target = proposed_proof if fault == "missing-proposed-metadata" else baseline_proof - payload = json.loads(target.read_text(encoding="ascii")) - if fault == "empty-filehashes": - payload["filehashes"] = {} - elif fault == "missing-native": - payload["filehashes"]["native"].pop("apm-linux-x86_64") - elif fault == "missing-proposed-metadata": - payload["filehashes"]["native"]["apm-linux-x86_64"]["candidate_metadata_sha256"] = None - else: - payload["counts"]["job_records"] = 0 - payload["job_records"] = [] - target.write_text(json.dumps(payload), encoding="ascii") - - with pytest.raises(ValueError, match=match): - _compare_ok(baseline_run, proposed_run, baseline_proof, proposed_proof) - - -@pytest.mark.parametrize("required", [(), ("Verify wall-clock artifacts",)]) -def test_compare_rejects_empty_or_terminal_only_required_job_sets( - tmp_path: Path, required: tuple[str, ...] -) -> None: - baseline_run, proposed_run, baseline_proof, proposed_proof = _compare_fixture(tmp_path) - - with pytest.raises(ValueError, match="required job set"): - _compare_ok( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - baseline_required=required, - ) - - -def test_compare_uses_actual_terminal_wallclock_not_runner_sum_or_modeled_queue( - tmp_path: Path, -) -> None: - baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 30, 120) - proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 100) - baseline_proof = _proof( - tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD - ) - proposed_proof = _proof( - tmp_path / "proposed-proof.json", - "proposed", - 301, - PROPOSED_SHA, - PROPOSED_CONTROLLER, - PROPOSED_RUN_HEAD, - ) - - result = compare_runs( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - PROPOSED_CONTROLLER, - BASE_RUN_HEAD, - PROPOSED_RUN_HEAD, - "Verify wall-clock artifacts", - ("Build/test/native/docs/wheel proof",), - ("Build/test/native/docs/wheel proof",), - ) - - assert result["readonly_adapter"] is True - assert result["production_qualification"] is False - assert result["baseline"]["wallclock_seconds_from_created_at"] == 120 - assert result["proposed"]["controller_sha"] == PROPOSED_CONTROLLER - assert result["proposed"]["run_head_sha"] == PROPOSED_RUN_HEAD - assert result["baseline"]["wallclock_seconds_from_run_started_at"] == 90 - assert result["proposed"]["wallclock_seconds_from_created_at"] == 100 - assert result["delta_seconds"]["primary_gain_seconds"] == 20 - assert result["delta_seconds"]["secondary_gain_seconds"] == -10 - assert result["delta_seconds"]["unweighted_runner_seconds_proposed_minus_baseline"] == 0 - assert result["delta_seconds"]["unweighted_runner_seconds_proposed_minus_baseline"] != -20 - - -@pytest.mark.parametrize( - ("fault", "match"), - [ - ("api-head", "API run head_sha"), - ("expected-head", "expected event head SHA"), - ("expected-controller", "expected execution SHA"), - ], -) -def test_compare_rejects_unexpected_run_head_or_execution_sha_changes( - tmp_path: Path, fault: str, match: str -) -> None: - baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) - proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90) - baseline_proof = _proof( - tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD - ) - proposed_proof = _proof( - tmp_path / "proposed-proof.json", - "proposed", - 301, - PROPOSED_SHA, - PROPOSED_CONTROLLER, - PROPOSED_RUN_HEAD, - ) - expected_head = PROPOSED_RUN_HEAD - expected_controller = PROPOSED_CONTROLLER - if fault == "api-head": - data = json.loads(proposed_run.read_text(encoding="ascii")) - data["run"]["head_sha"] = "1" * 40 - data["run"]["pull_requests"][0]["head"]["sha"] = PROPOSED_RUN_HEAD - proposed_run.write_text(json.dumps(data), encoding="ascii") - elif fault == "expected-head": - expected_head = "1" * 40 - else: - expected_controller = "1" * 40 - - with pytest.raises(ValueError, match=match): - compare_runs( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - expected_controller, - BASE_RUN_HEAD, - expected_head, - "Verify wall-clock artifacts", - ("Build/test/native/docs/wheel proof",), - ("Build/test/native/docs/wheel proof",), - ) - - -@pytest.mark.parametrize("fault", ["failed-control", "missing-required", "early-terminal"]) -def test_compare_rejects_failed_controls_missing_gates_and_bad_timelines( - tmp_path: Path, fault: str -) -> None: - baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) - proposed_run = _run_json(tmp_path / "proposed-run.json", 301, PROPOSED_RUN_HEAD, 0, 90) - data = json.loads(proposed_run.read_text(encoding="ascii")) - if fault == "failed-control": - data["jobs"][0]["conclusion"] = "failure" - required = ("Build/test/native/docs/wheel proof",) - match = "rejected conclusion" - elif fault == "missing-required": - required = ("Required native gate",) - match = "required job missing" - else: - data["jobs"][0]["completed_at"] = _at(88) - data["jobs"][2]["started_at"] = _at(87) - required = ("Build/test/native/docs/wheel proof",) - match = "terminal job started before" - proposed_run.write_text(json.dumps(data), encoding="ascii") - - with pytest.raises(ValueError, match=match): - compare_runs( - baseline_run, - proposed_run, - _proof( - tmp_path / "baseline-proof.json", - "baseline", - 201, - BASE_SHA, - BASE_CONTROLLER, - BASE_RUN_HEAD, - ), - _proof( - tmp_path / "proposed-proof.json", - "proposed", - 301, - PROPOSED_SHA, - PROPOSED_CONTROLLER, - PROPOSED_RUN_HEAD, - ), - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - PROPOSED_CONTROLLER, - BASE_RUN_HEAD, - PROPOSED_RUN_HEAD, - "Verify wall-clock artifacts", - ("Build/test/native/docs/wheel proof",), - required, - ) - - -def test_compare_rejects_promotable_or_duplicate_run_proofs(tmp_path: Path) -> None: - baseline_run = _run_json(tmp_path / "baseline-run.json", 201, BASE_RUN_HEAD, 0, 100) - proposed_run = _run_json(tmp_path / "proposed-run.json", 201, PROPOSED_RUN_HEAD, 0, 90) - baseline_proof = _proof( - tmp_path / "baseline-proof.json", "baseline", 201, BASE_SHA, BASE_CONTROLLER, BASE_RUN_HEAD - ) - proposed_proof = _proof( - tmp_path / "proposed-proof.json", - "proposed", - 201, - PROPOSED_SHA, - PROPOSED_CONTROLLER, - PROPOSED_RUN_HEAD, - ) - payload = json.loads(proposed_proof.read_text(encoding="ascii")) - payload["promotable"] = True - proposed_proof.write_text(json.dumps(payload), encoding="ascii") - - with pytest.raises(ValueError, match="non-promotable"): - compare_runs( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - PROPOSED_CONTROLLER, - BASE_RUN_HEAD, - PROPOSED_RUN_HEAD, - "Verify wall-clock artifacts", - ("Build/test/native/docs/wheel proof",), - ("Build/test/native/docs/wheel proof",), - ) - - payload["promotable"] = False - proposed_proof.write_text(json.dumps(payload), encoding="ascii") - with pytest.raises(ValueError, match="run IDs must differ"): - compare_runs( - baseline_run, - proposed_run, - baseline_proof, - proposed_proof, - BASE_SHA, - PROPOSED_SHA, - BASE_CONTROLLER, - PROPOSED_CONTROLLER, - BASE_RUN_HEAD, - PROPOSED_RUN_HEAD, - "Verify wall-clock artifacts", - ("Build/test/native/docs/wheel proof",), - ("Build/test/native/docs/wheel proof",), - ) diff --git a/tests/unit/test_ado_live_workflow_contract.py b/tests/unit/test_ado_live_workflow_contract.py index f23f852d8b..0b7fd2a452 100644 --- a/tests/unit/test_ado_live_workflow_contract.py +++ b/tests/unit/test_ado_live_workflow_contract.py @@ -24,7 +24,7 @@ AUTH_WORKFLOW = ROOT / ".github" / "workflows" / "auth-acceptance.yml" INTEGRATION_SCRIPT = ROOT / "scripts" / "test-integration.sh" LIVE_ADO_SELECTOR = "live and requires_ado_pat" -RELEASE_INTEGRATION_JOBS = ("integration-tests-shard", "integration-tests-proposed-shard") +RELEASE_INTEGRATION_JOBS = ("integration-tests-shard",) def _walk_nodes(value: Any) -> list[dict[str, Any]]: @@ -139,15 +139,6 @@ def test_integration_script_exports_explicit_candidate_binary_path() -> None: assert 'export APM_BINARY_PATH="$(pwd)/dist/$BINARY_NAME/apm"' in script -def test_integration_script_uses_module_pytest_for_performance_plugin() -> None: - """The proof plugin must load through repo-root module imports, not console pytest.""" - script = INTEGRATION_SCRIPT.read_text(encoding="utf-8") - assert "pytest_command=(pytest)" in script - assert 'scripts.pytest_performance_evidence "*' in script - assert "pytest_command=(python -m pytest)" in script - assert 'if "${pytest_command[@]}" tests/integration/ -v --tb=short \\' in script - - def test_auth_acceptance_explicitly_selects_live_ado_nodes() -> None: """The opt-in auth workflow remains the live PAT acceptance owner.""" _assert_auth_acceptance_selects_live_ado(load_workflow(AUTH_WORKFLOW)) diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index e5783d278b..c87fa345de 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -20,7 +20,6 @@ load_workflow, shell_commands, shell_tokens, - wallclock_unit_commands, workflow_job, workflow_step, workflow_step_index, @@ -33,8 +32,6 @@ RELEASE_WORKFLOW = ROOT / ".github/workflows/build-release.yml" UNIX_TIMEOUT = "${{ inputs.integration-markers == 'lifecycle_smoke and not live' && 30 || 60 }}" UNIX_ARGS = ( - "${{ (inputs.evidence-variant != '' || inputs.wallclock-evidence) " - "&& '-p scripts.pytest_performance_evidence' || '' }} " "--splits ${{ inputs.shard-count }} --group ${{ inputs.shard-index }} " "--splitting-algorithm ${{ inputs.splitting-algorithm }} " "-n ${{ inputs.xdist-workers }} --dist loadgroup " @@ -141,29 +138,6 @@ def _execute_integration_gate(results: dict) -> list[str]: return json.loads(completed.stdout) -def _execute_integration_probe_gate(results: dict) -> list[str]: - gate = workflow_job(_workflow(), "integration-performance-probe") - script = workflow_step(gate, "Require every integration performance member")["with"]["script"] - harness = ( - "const failures = [];" - "const core = {setFailed: message => failures.push(message)};" - f"(new Function('core', {json.dumps(script)}))(core);" - "process.stdout.write(JSON.stringify(failures));" - ) - completed = subprocess.run( - ["node", "-e", harness], - check=True, - capture_output=True, - text=True, - timeout=15, - env={ - **os.environ, - "RESULTS": json.dumps(results), - }, - ) - return json.loads(completed.stdout) - - @pytest.mark.parametrize( "job_name", ["unit-tests", "build", "integration-tests", "release-validation", "windows-installer"], @@ -204,24 +178,6 @@ def test_native_integration_fan_in_requires_shard_matrix_success(result: str | N assert _execute_integration_gate(results) == ["integration-tests-shard did not succeed"] -@pytest.mark.parametrize( - "job_name", ["integration-tests-shard", "integration-tests-proposed-shard"] -) -@pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "neutral", None]) -def test_integration_probe_fan_in_requires_baseline_and_proposed_success( - job_name: str, result: str | None -) -> None: - results = { - "integration-tests-shard": {"result": "success"}, - "integration-tests-proposed-shard": {"result": "success"}, - } - if result is None: - del results[job_name] - else: - results[job_name]["result"] = result - assert _execute_integration_probe_gate(results) == [f"{job_name} did not succeed"] - - def _assert_native_startup(workflow: dict) -> None: job = workflow_job(workflow, "build") assert_unconditional(job, label="native candidate build") @@ -309,7 +265,7 @@ def _assert_windows_installer(workflow: dict) -> None: assert_exact_command( shell_commands(executed), [ - "scripts/windows/run-installer-proof.ps1", + "scripts/windows/validate-installer-result.ps1", "-ValidateInstallerJUnit", "test-results/installer.xml", "-InstallerExitCode", @@ -354,9 +310,6 @@ def _assert_unit_call(workflow: dict, unit: dict) -> None: "runner": "${{ inputs.runner }}", "platform": "${{ inputs.platform }}", "binary-name": "${{ inputs.binary-name }}", - "performance-probe": "${{ inputs.unit-performance-probe }}", - "proposed-workers": "${{ inputs.performance-workers }}", - "wallclock-evidence": "${{ inputs.wallclock-evidence }}", } assert job["secrets"] == { "GH_CLI_PAT": "${{ secrets.GH_CLI_PAT }}", @@ -366,12 +319,12 @@ def _assert_unit_call(workflow: dict, unit: dict) -> None: baseline = workflow_job(unit, "unit-tests") assert baseline["runs-on"] == "${{ inputs.runner }}" restore = workflow_step(baseline, "Restore unit scheduling hints") - assert restore["if"] == "${{ !inputs.performance-probe }}" + assert "if" not in restore assert restore["with"] == {"suite": "unit"} standard = workflow_step(baseline, "Run unit tests") - assert standard["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" + assert standard["if"] == "inputs.platform != 'windows'" assert_exact_command( - wallclock_unit_commands(standard, enabled=False), + shell_commands(standard), [ "uv", "run", @@ -390,109 +343,12 @@ def _assert_unit_call(workflow: dict, unit: dict) -> None: label="standard release unit tests", ) upload = workflow_step(baseline, "Upload unit timings and outcomes") - assert upload["if"] == "always() && !inputs.performance-probe" + assert upload["if"] == "always()" assert upload["with"]["name"] == ( "test-results-${{ github.run_attempt }}-${{ inputs.binary-name }}-unit" ) -def _assert_unit_probe(unit: dict) -> None: - baseline = workflow_job(unit, "unit-tests") - cold = workflow_step(baseline, "Initialize cold unit scheduling hints") - assert cold["if"] == "inputs.performance-probe && inputs.platform != 'windows'" - assert cold["run"] == "printf '{}\\n' > .test_durations" - cohort = workflow_step( - baseline, "Require matched actual test runners before the unit performance experiment" - ) - assert cohort["if"] == "inputs.performance-probe && inputs.platform != 'windows'" - assert cohort["with"] == { - "cohort": "unit-${{ github.run_attempt }}-${{ inputs.binary-name }}", - "member": "baseline-1", - "source-sha": "${{ github.sha }}", - "selection": "tests/unit tests/test_console.py", - } - run = workflow_step(baseline, "Run unit performance baseline") - assert run["if"] == "inputs.platform != 'windows' && inputs.performance-probe" - assert_exact_command( - shell_commands(run), - [ - "uv", - "run", - "--frozen", - "python", - "-m", - "pytest", - "-p", - "scripts.pytest_performance_evidence", - "tests/unit", - "tests/test_console.py", - "-n", - "auto", - "--dist", - "worksteal", - "--durations=50", - "--store-durations", - "--junitxml=test-results/unit-baseline-shard-1.xml", - ], - label="unit performance baseline", - ) - capture = workflow_step(baseline, "Capture unit performance evidence") - assert "--variant baseline" in capture["run"] - assert "--shard-count 1" in capture["run"] - evidence = workflow_step(baseline, "Upload unit performance evidence") - assert evidence["with"]["name"] == ( - "performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-" - "baseline-shard-1" - ) - - proposed = workflow_job(unit, "unit-tests-proposed") - assert proposed["if"] == "inputs.performance-probe && inputs.platform != 'windows'" - assert proposed["strategy"] == {"fail-fast": False, "matrix": {"shard": [1, 2]}} - proposed_cohort = workflow_step( - proposed, "Require matched actual test runners before the unit performance experiment" - ) - assert proposed_cohort["with"] == { - "cohort": "unit-${{ github.run_attempt }}-${{ inputs.binary-name }}", - "member": "proposed-${{ matrix.shard }}", - "source-sha": "${{ github.sha }}", - "selection": "tests/unit tests/test_console.py", - } - proposed_run = workflow_step(proposed, "Run unit performance proposed shard") - tokens = shell_tokens(proposed_run) - for token in ( - "python", - "-m", - "pytest", - "-p", - "scripts.pytest_performance_evidence", - "tests/unit", - "tests/test_console.py", - "--splits", - "2", - "--group", - "matrix.shard", - "--splitting-algorithm", - "least_duration", - "-n", - "inputs.proposed-workers", - "--dist", - "worksteal", - "--durations=50", - "--store-durations", - ): - assert token in tokens - assert "unit-proposed-shard-${{" in proposed_run["run"] - assert "}}.xml" in proposed_run["run"] - proposed_evidence = workflow_step(proposed, "Upload unit performance evidence") - assert proposed_evidence["with"]["name"] == ( - "performance-evidence-unit-${{ github.run_attempt }}-${{ inputs.binary-name }}-" - "proposed-shard-${{ matrix.shard }}" - ) - fan_in = workflow_job(unit, "unit-performance-probe") - assert fan_in["needs"] == ["unit-tests", "unit-tests-proposed"] - assert fan_in["if"] == "always() && inputs.performance-probe && inputs.platform != 'windows'" - - def _assert_parallel_paths(workflow: dict) -> None: for name in ("unit-tests", "build"): assert "needs" not in workflow_job(workflow, name) @@ -536,41 +392,24 @@ def _assert_integration(workflow: dict, integration: dict) -> None: assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" assert job["with"]["timeout-minutes"] == UNIX_TIMEOUT assert job["with"]["artifact-prefix"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || " - "format('{0}-{1}', github.run_attempt, inputs.binary-name) }}" + "${{ github.run_attempt }}-${{ inputs.binary-name }}" ) assert job["with"]["runtime-prerequisites"] == "none" - assert job["with"]["evidence-variant"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "'baseline' || '' }}" - ) - assert job["with"]["evidence-selection"] == ( - "tests/integration/ -m ${{ inputs.integration-markers }}" - ) - assert job["with"]["wallclock-evidence"] == "${{ inputs.wallclock-evidence }}" - assert job["with"]["performance-cohort"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }}" - ) + for removed_key in ( + "evidence-variant", + "evidence-selection", + "wallclock-evidence", + "performance-cohort", + ): + assert removed_key not in job["with"], ( + f"experimental input {removed_key!r} must not resurface on integration-tests-shard" + ) build = workflow_job(workflow, "build") snapshot = workflow_step(build, "Freeze integration scheduling hints") - assert snapshot["if"] == ( - "inputs.full-validation && inputs.platform != 'windows' && " - "!inputs.integration-performance-probe" - ) + assert snapshot["if"] == "inputs.full-validation && inputs.platform != 'windows'" assert snapshot["with"]["snapshot"] == ( "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}" ) - cold_upload = workflow_step( - build, "Upload cold integration scheduling hints for performance probe" - ) - assert cold_upload["if"] == ( - "inputs.full-validation && inputs.platform != 'windows' && " - "inputs.integration-performance-probe" - ) - assert cold_upload["with"]["name"] == snapshot["with"]["snapshot"] - assert cold_upload["with"]["path"] == ".test_durations" integration_job = workflow_job(integration, "integration-tests") unpack = workflow_step(integration_job, "Extract checked candidate archive") @@ -597,40 +436,20 @@ def _assert_integration(workflow: dict, integration: dict) -> None: assert step["env"]["GITHUB_APM_PAT"] == "${{ secrets.GH_CLI_PAT }}" assert "APM_RUN_INFERENCE_TESTS" not in effective_env(integration, integration_job, step) assert "-IncludeLiveADO" not in shell_tokens(windows) - capture = workflow_step(integration_job, "Capture integration performance evidence") - assert capture["if"] == ( - "always() && inputs.platform != 'windows' && inputs.evidence-variant != ''" - ) - assert capture["env"] == { - "EVIDENCE_SELECTION": "${{ inputs.evidence-selection }}", - "PYTEST_MARK_EXPR": "${{ inputs.integration-markers }}", - } - capture_tokens = shell_tokens(capture) - for token in ( - "scripts.compare_test_runs", - "capture", - "--junit", - "test-results/integration-shard-${{ inputs.shard-index }}.xml", - "--source-sha", - "${{ inputs.candidate-sha }}", - "--variant", - "${{ inputs.evidence-variant }}", - "--shard-count", - "${{ inputs.shard-count }}", - "--candidate-metadata", - "release-assets/${{ inputs.binary-name }}.json", + for removed_name in ( + "Capture integration performance evidence", + "Upload integration performance evidence", ): - assert token in capture_tokens - proof_upload = workflow_step(integration_job, "Upload integration performance evidence") - assert proof_upload["if"] == capture["if"] - assert proof_upload["with"]["name"] == ( - "performance-evidence-${{ inputs.artifact-prefix }}-" - "${{ inputs.evidence-variant }}-shard-${{ inputs.shard-index }}" - ) + matches = [ + step + for step in integration_job["steps"] + if isinstance(step, dict) and step.get("name") == removed_name + ] + assert matches == [], ( + f"experimental integration step {removed_name!r} must be removed" + ) timings_upload = workflow_step(integration_job, "Upload integration timings and outcomes") - assert timings_upload["if"] == ( - "always() && (inputs.platform != 'windows' || inputs.wallclock-evidence)" - ) + assert timings_upload["if"] == "always() && inputs.platform != 'windows'" assert timings_upload["with"]["if-no-files-found"] == "error" coverage_upload = workflow_step(integration_job, "Upload coverage data") assert coverage_upload["with"]["name"] == ( @@ -638,70 +457,6 @@ def _assert_integration(workflow: dict, integration: dict) -> None: ) -def _assert_integration_probe(workflow: dict) -> None: - inputs = workflow["on"]["workflow_call"]["inputs"] - assert inputs["unit-performance-probe"] == {"type": "boolean", "default": False} - assert inputs["integration-performance-probe"] == {"type": "boolean", "default": False} - assert inputs["performance-workers"] == {"type": "number", "default": 4} - assert workflow["permissions"] == {"contents": "read", "actions": "read"} - - baseline = workflow_job(workflow, "integration-tests-shard") - proposed = workflow_job(workflow, "integration-tests-proposed-shard") - assert proposed["uses"] == "./.github/workflows/release-integration.yml" - assert proposed["needs"] == ["build"] - assert proposed["if"] == ( - "inputs.full-validation && inputs.platform != 'windows' && " - "inputs.integration-performance-probe" - ) - assert proposed["strategy"] == { - "fail-fast": False, - "matrix": {"shard": [1, 2]}, - } - for key in ( - "runner", - "platform", - "binary-name", - "candidate-artifact-name", - "candidate-sha", - "timing-snapshot-artifact", - "integration-markers", - "runtime-prerequisites", - "timeout-minutes", - "evidence-selection", - ): - assert proposed["with"][key] == baseline["with"][key] - assert proposed["with"]["shard-count"] == 2 - assert proposed["with"]["shard-index"] == "${{ matrix.shard }}" - assert proposed["with"]["xdist-workers"] == "${{ inputs.performance-workers }}" - assert proposed["with"]["splitting-algorithm"] == "least_duration" - assert proposed["with"]["artifact-prefix"] == ( - "${{ github.run_attempt }}-${{ inputs.binary-name }}-proposed" - ) - assert proposed["with"]["evidence-variant"] == "proposed" - assert baseline["with"]["evidence-variant"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "'baseline' || '' }}" - ) - assert baseline["with"]["performance-cohort"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "format('integration-{0}-{1}', github.run_attempt, inputs.binary-name) || '' }}" - ) - assert proposed["with"]["performance-cohort"] == ( - "integration-${{ github.run_attempt }}-${{ inputs.binary-name }}" - ) - - fan_in = workflow_job(workflow, "integration-performance-probe") - assert fan_in["needs"] == ["integration-tests-shard", "integration-tests-proposed-shard"] - assert ( - fan_in["if"] == "always() && inputs.full-validation && inputs.platform != 'windows' && " - "inputs.integration-performance-probe" - ) - assert ( - "integration-tests-proposed-shard" - in workflow_step(fan_in, "Require every integration performance member")["with"]["script"] - ) - - def test_native_startup_runs_exact_frozen_candidate_before_packaging() -> None: _assert_native_startup(_workflow()) @@ -718,18 +473,10 @@ def test_native_unit_runner_is_canonical_and_preserves_standard_abi() -> None: _assert_unit_call(_workflow(), _unit_workflow()) -def test_native_unit_performance_probe_runs_parallel_baseline_and_proposed_halves() -> None: - _assert_unit_probe(_unit_workflow()) - - def test_native_integration_preserves_grouping_bounds_and_artifact_identity() -> None: _assert_integration(_workflow(), _integration_workflow()) -def test_native_integration_performance_probe_uses_same_candidate_and_cold_snapshot() -> None: - _assert_integration_probe(_workflow()) - - def test_platform_catalog_retains_all_native_and_non_live_selections() -> None: catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text("ascii")) rows = {row["binary_name"]: row for row in catalog} diff --git a/tests/unit/test_pytest_timing_workflows.py b/tests/unit/test_pytest_timing_workflows.py index c8ed0c64e8..eafa803635 100644 --- a/tests/unit/test_pytest_timing_workflows.py +++ b/tests/unit/test_pytest_timing_workflows.py @@ -97,19 +97,11 @@ def assert_release_platform_snapshot(workflow: dict) -> None: build = workflow_job(workflow, "build") snapshot = workflow_step(build, "Freeze integration scheduling hints") assert snapshot["uses"] == ACTION - assert snapshot["if"] == ( - "inputs.full-validation && inputs.platform != 'windows' && " - "!inputs.integration-performance-probe" - ) + assert snapshot["if"] == "inputs.full-validation && inputs.platform != 'windows'" assert snapshot["with"] == { "suite": "integration", "snapshot": "integration-duration-snapshot-${{ github.run_attempt }}-${{ inputs.binary-name }}", } - cold_upload = workflow_step( - build, "Upload cold integration scheduling hints for performance probe" - ) - assert cold_upload["with"]["name"] == snapshot["with"]["snapshot"] - assert cold_upload["with"]["path"] == ".test_durations" job = workflow_job(workflow, "integration-tests-shard") assert job["uses"] == RELEASE_INTEGRATION assert job["needs"] == ["build"] @@ -120,11 +112,7 @@ def assert_release_platform_snapshot(workflow: dict) -> None: assert job["with"]["shard-index"] == "${{ matrix.shard }}" assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" - assert job["with"]["artifact-prefix"] == ( - "${{ inputs.integration-performance-probe && inputs.platform != 'windows' && " - "format('{0}-{1}-baseline', github.run_attempt, inputs.binary-name) || " - "format('{0}-{1}', github.run_attempt, inputs.binary-name) }}" - ) + assert job["with"]["artifact-prefix"] == "${{ github.run_attempt }}-${{ inputs.binary-name }}" assert job["with"]["runtime-prerequisites"] == "none" @@ -203,8 +191,6 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( sys.executable, "-m", "pytest", - "-p", - "scripts.pytest_performance_evidence", "-c", str(config), "--confcutdir", @@ -236,11 +222,8 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( assert result.returncode == 0, result.stdout + result.stderr cases = list(ET.parse(outcomes).iter("testcase")) # noqa: S314 - generated by this probe. assert cases - nodes = { - identity.attrib["value"] - for case in cases - for identity in case.findall("./properties/property[@name='apm_performance_nodeid']") - } + assert all(case.attrib["classname"] == "test_probe" for case in cases) + nodes = {f"test_probe.py::{case.attrib['name']}" for case in cases} observed.append(nodes) stored = json.loads(durations.read_text("ascii")) assert stored diff --git a/tests/unit/test_release_performance_workflow.py b/tests/unit/test_release_performance_workflow.py deleted file mode 100644 index 410e28a506..0000000000 --- a/tests/unit/test_release_performance_workflow.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Native performance evidence must use the same candidate and complete test selection.""" - -from __future__ import annotations - -import json -from copy import deepcopy -from pathlib import Path - -import pytest - -from tests.workflow_contracts import load_workflow, workflow_job, workflow_step - -pytestmark = pytest.mark.component -ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github/workflows/ci-release-performance.yml" - - -def _assert_probe(workflow: dict) -> None: - assert workflow["on"] == { - "pull_request": {"types": ["opened", "reopened", "synchronize", "labeled"]} - } - assert workflow["permissions"] == {"contents": "read"} - assert workflow["concurrency"]["cancel-in-progress"] is True - for job in workflow["jobs"].values(): - assert "ci-performance" in job["if"] - assert "secrets" not in job - assert "environment" not in job - - candidate = workflow_job(workflow, "candidate") - assert candidate["runs-on"] == "macos-latest" - assert candidate["timeout-minutes"] == 15 - build = workflow_step(candidate, "Build current ARM candidate") - assert "scripts/build-binary.sh" in build["run"] - assert build["env"] == {"UV_FROZEN": "true", "UV_NO_SYNC": "true"} - pack = workflow_step(candidate, "Package one immutable candidate for every variant") - assert 'scripts/package_release.py pack --binary-name apm-darwin-arm64 --sha "$GITHUB_SHA"' in ( - " ".join(pack["run"].split()) - ) - snapshot = workflow_step(candidate, "Freeze identical cold scheduling input") - assert snapshot["run"] == "printf '{}\\n' > .test_durations" - uploads = [ - step["with"] for step in candidate["steps"] if "upload-artifact@" in step.get("uses", "") - ] - assert any( - upload["name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" - and set(upload["path"].splitlines()) - == {"release-assets/", "scripts/package_release.py", "scripts/release-platforms.json"} - and upload["compression-level"] == 0 - for upload in uploads - ) - assert any( - upload["name"] == "benchmark-arm-cold-timings-${{ github.run_attempt }}" - and upload["path"] == ".test_durations" - and upload["include-hidden-files"] is True - for upload in uploads - ) - - integration = workflow_job(workflow, "integration") - assert integration["needs"] == ["candidate"] - assert integration["uses"] == "./.github/workflows/release-integration.yml" - assert integration["strategy"]["fail-fast"] is False - assert integration["strategy"]["matrix"]["include"] == [ - {"variant": "baseline", "shard": 1, "count": 1, "workers": 4}, - {"variant": "proposed", "shard": 1, "count": 2, "workers": 3}, - {"variant": "proposed", "shard": 2, "count": 2, "workers": 3}, - ] - inputs = integration["with"] - assert inputs["candidate-artifact-name"] == "benchmark-arm-candidate-${{ github.run_attempt }}" - assert inputs["candidate-sha"] == "${{ github.sha }}" - assert ( - inputs["timing-snapshot-artifact"] == "benchmark-arm-cold-timings-${{ github.run_attempt }}" - ) - assert inputs["integration-markers"] == "not live" - assert inputs["splitting-algorithm"] == "least_duration" - assert inputs["shard-count"] == "${{ matrix.count }}" - assert inputs["shard-index"] == "${{ matrix.shard }}" - assert inputs["xdist-workers"] == "${{ matrix.workers }}" - assert inputs["evidence-variant"] == "${{ matrix.variant }}" - assert inputs["evidence-selection"] == "tests/integration/ -m not live" - assert inputs["cache-apm-fetches"] is False - assert inputs["runtime-prerequisites"] == "none" - - comparison = workflow_job(workflow, "compare") - assert comparison["needs"] == ["candidate", "integration"] - assert "always()" in comparison["if"] and "!cancelled()" in comparison["if"] - require = workflow_step(comparison, "Require all benchmark executions succeeded") - assert require["env"] == { - "BUILD_RESULT": "${{ needs.candidate.result }}", - "INTEGRATION_RESULT": "${{ needs.integration.result }}", - } - assert 'test "$BUILD_RESULT" = success' in require["run"] - assert 'test "$INTEGRATION_RESULT" = success' in require["run"] - compare = workflow_step(comparison, "Require identical outcomes and a measured execution gain") - assert "scripts.compare_test_runs compare" in compare["run"] - assert "--minimum-speedup 0.05" in compare["run"] - assert '--expected-sha "$GITHUB_SHA"' in compare["run"] - assert "performance-inputs/baseline/baseline-shard-1.json" in compare["run"] - assert "performance-inputs/proposed-1/proposed-shard-1.json" in compare["run"] - assert "performance-inputs/proposed-2/proposed-shard-2.json" in compare["run"] - - -def test_native_probe_uses_cold_shared_input_and_one_exact_candidate() -> None: - _assert_probe(load_workflow(WORKFLOW)) - - -def test_release_keeps_baseline_arm_topology_until_sharding_is_accepted() -> None: - catalog = json.loads((ROOT / "scripts/release-platforms.json").read_text(encoding="utf-8")) - arm = next(row for row in catalog if row["binary_name"] == "apm-darwin-arm64") - caller = workflow_job(load_workflow(ROOT / ".github/workflows/build-release.yml"), "platforms") - assert caller["with"]["integration-shard-count"] == "${{ matrix.integration_shard_count }}" - assert caller["with"]["integration-xdist-workers"] == "${{ matrix.integration_xdist_workers }}" - assert caller["with"]["integration-splitting-algorithm"] == ( - "${{ matrix.integration_splitting_algorithm }}" - ) - assert arm["integration_shard_count"] == 1 - assert arm["integration_xdist_workers"] == 4 - assert arm["integration_splitting_algorithm"] == "duration_based_chunks" - variants = workflow_job(load_workflow(WORKFLOW), "integration")["strategy"]["matrix"]["include"] - baseline = next(row for row in variants if row["variant"] == "baseline") - assert baseline["count"] == arm["integration_shard_count"] - assert baseline["workers"] == arm["integration_xdist_workers"] - - -@pytest.mark.parametrize( - "fault", - [ - "candidate", - "snapshot", - "source", - "selection", - "workers", - "missing-shard", - "fail-fast", - "flattened-archive", - ], -) -def test_native_probe_rejects_incomparable_or_incomplete_designs(fault: str) -> None: - workflow = deepcopy(load_workflow(WORKFLOW)) - job = workflow["jobs"]["integration"] - if fault == "candidate": - job["with"]["candidate-artifact-name"] = "candidate-${{ matrix.variant }}" - elif fault == "snapshot": - job["with"]["timing-snapshot-artifact"] = "timings-${{ matrix.variant }}" - elif fault == "source": - job["with"]["candidate-sha"] = "${{ github.event.pull_request.head.sha }}" - elif fault == "selection": - job["with"]["integration-markers"] = "lifecycle_smoke" - elif fault == "workers": - job["with"]["xdist-workers"] = "auto" - elif fault == "missing-shard": - job["strategy"]["matrix"]["include"].pop() - elif fault == "fail-fast": - job["strategy"]["fail-fast"] = True - else: - uploads = [ - step - for step in workflow["jobs"]["candidate"]["steps"] - if "upload-artifact@" in step.get("uses", "") - ] - uploads[0]["with"]["path"] = "release-assets/" - with pytest.raises(AssertionError): - _assert_probe(workflow) - - -def test_no_called_benchmark_workflow_can_request_publishing_permissions() -> None: - pending = [WORKFLOW] - visited = set() - while pending: - path = pending.pop() - if path in visited: - continue - visited.add(path) - workflow = load_workflow(path) - permissions = workflow.get("permissions", {}) - assert isinstance(permissions, dict), path - assert all(value in ("read", "none") for value in permissions.values()), path - for job in workflow["jobs"].values(): - permissions = job.get("permissions", {}) - assert isinstance(permissions, dict), path - assert all(value in ("read", "none") for value in permissions.values()), path - assert "environment" not in job, path - assert "secrets" not in job, path - target = job.get("uses", "") - if target.startswith("./.github/workflows/"): - pending.append(ROOT / target.removeprefix("./")) diff --git a/tests/unit/test_release_rehearsal_workflow.py b/tests/unit/test_release_rehearsal_workflow.py deleted file mode 100644 index d1d995b514..0000000000 --- a/tests/unit/test_release_rehearsal_workflow.py +++ /dev/null @@ -1,181 +0,0 @@ -"""A whole-release experiment must execute the existing authorities without publication.""" - -from __future__ import annotations - -from copy import deepcopy -from pathlib import Path - -import pytest - -from tests.workflow_contracts import load_workflow, workflow_job, workflow_step - -pytestmark = pytest.mark.component -ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github/workflows/ci-release-rehearsal.yml" - - -def _assert_rehearsal(workflow: dict) -> None: - assert workflow["on"] == { - "pull_request": {"types": ["opened", "reopened", "synchronize", "labeled"]}, - } - assert workflow["permissions"] == {"contents": "read", "actions": "read"} - assert workflow["concurrency"]["cancel-in-progress"] is True - assert "ci-release-rehearsal" in workflow["jobs"]["plan"]["if"] - for job in workflow["jobs"].values(): - assert "secrets" not in job - assert "environment" not in job - assert all( - permission in {"read", "none"} for permission in job.get("permissions", {}).values() - ) - - plan = workflow_job(workflow, "plan") - matrix = workflow_step(plan, "Derive all five native controls and the three paired surfaces") - assert matrix["run"] == 'python -m scripts.release_rehearsal matrix >> "$GITHUB_OUTPUT"' - platforms = workflow_job(workflow, "platforms") - assert platforms["uses"] == "./.github/workflows/release-platform.yml" - assert platforms["name"] == "${{ matrix.binary_name }}" - assert platforms["needs"] == ["plan"] - assert platforms["strategy"]["fail-fast"] is False - assert platforms["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.matrix) }}" - assert platforms["with"]["full-validation"] is True - for argument, field in ( - ("unit-performance-probe", "unit_performance_probe"), - ("integration-performance-probe", "integration_performance_probe"), - ("performance-workers", "performance_workers"), - ("integration-markers", "integration_markers"), - ("integration-shard-count", "integration_shard_count"), - ("integration-xdist-workers", "integration_xdist_workers"), - ): - assert platforms["with"][argument] == "${{ matrix." + field + " }}" - - source = workflow_job(workflow, "source-checks") - assert source["name"] == "Candidate Source Checks" - assert source["uses"] == "./.github/workflows/ci.yml" - assert source["needs"] == ["plan"] - for key, builder in (("docs", "docs-build"), ("wheels", "pypi-distributions")): - job = workflow_job(workflow, key) - assert job["uses"] == f"./.github/workflows/{builder}.yml" - assert job["needs"] == ["plan"] - - report = workflow_job(workflow, "compare") - assert report["needs"] == ["plan", "source-checks", "platforms", "docs", "wheels"] - assert "always()" in report["if"] and "!cancelled()" in report["if"] - collect = workflow_step( - report, "Record current-attempt job timestamps and require production authorities" - ) - script = collect["with"]["script"] - assert "listJobsForWorkflowRunAttempt" in script - assert "attempt_number: attempt" in script - assert "selectRequiredJobs(jobs, catalog)" in script - assert "source_sha: context.sha" in script - assert "run_id: String(context.runId), run_attempt: attempt" in script - check = workflow_step(report, "Require original outcomes and measure the entire critical path") - assert "--minimum-lane-gain 0.15 --minimum-overall-gain 0.10" in check["run"] - assert '--expected-sha "$GITHUB_SHA"' in check["run"] - downloads = [ - step["with"]["pattern"] - for step in report["steps"] - if "download-artifact@" in step.get("uses", "") - ] - assert len(downloads) == 5 - assert all("${{ github.run_attempt }}" in pattern for pattern in downloads) - assert downloads[:2] == [ - "performance-evidence-${{ github.run_attempt }}-apm-*", - "performance-evidence-unit-${{ github.run_attempt }}-apm-*", - ] - for surface in ( - "integration-apm-darwin-arm64", - "integration-apm-linux-x86_64", - "unit-apm-darwin-x86_64", - ): - suite, binary = surface.split("-apm-", 1) - assert ( - f"performance-cohort-{suite}-${{{{ github.run_attempt }}}}-apm-{binary}-*" in downloads - ) - - -def test_read_only_rehearsal_reuses_full_native_source_and_publication_build_authorities() -> None: - _assert_rehearsal(load_workflow(WORKFLOW)) - - -@pytest.mark.parametrize( - "fault", - [ - "privileged", - "secrets", - "partial-validation", - "missing-source", - "fail-fast", - "worker-drift", - "old-artifacts", - "missing-lane", - "missing-authority", - "weaker-threshold", - ], -) -def test_rehearsal_cannot_silently_remove_controls_or_accept_weaker_proof(fault: str) -> None: - workflow = deepcopy(load_workflow(WORKFLOW)) - jobs = workflow["jobs"] - if fault == "privileged": - workflow["permissions"]["contents"] = "write" - elif fault == "secrets": - jobs["platforms"]["secrets"] = "inherit" - elif fault == "partial-validation": - jobs["platforms"]["with"]["full-validation"] = False - elif fault == "missing-source": - jobs["source-checks"]["uses"] = "./.github/workflows/ci-source-performance.yml" - elif fault == "fail-fast": - jobs["platforms"]["strategy"]["fail-fast"] = True - elif fault == "worker-drift": - jobs["platforms"]["with"]["performance-workers"] = 99 - elif fault == "missing-lane": - jobs["compare"]["needs"].remove("platforms") - elif fault == "missing-authority": - step = workflow_step( - jobs["compare"], - "Record current-attempt job timestamps and require production authorities", - ) - step["with"]["script"] = step["with"]["script"].replace( - "selectRequiredJobs(jobs, catalog)", "null" - ) - elif fault == "weaker-threshold": - step = workflow_step( - jobs["compare"], "Require original outcomes and measure the entire critical path" - ) - step["run"] = step["run"].replace("--minimum-overall-gain 0.10", "--minimum-overall-gain 0") - else: - step = workflow_step(jobs["compare"], "Download current integration proofs") - step["with"]["pattern"] = "performance-evidence-*" - with pytest.raises(AssertionError): - _assert_rehearsal(workflow) - - -def test_cohort_barrier_is_before_pytest_and_static_callers_grant_read_only_actions() -> None: - integration = load_workflow(ROOT / ".github/workflows/release-integration.yml") - steps = integration["jobs"]["integration-tests"]["steps"] - barrier = next( - index - for index, step in enumerate(steps) - if step.get("uses") == "./.github/actions/performance-cohort" - ) - execution = next( - index - for index, step in enumerate(steps) - if step.get("name") == "Run integration tests (Unix)" - ) - assert barrier < execution - assert steps[barrier]["if"] == "inputs.performance-cohort != ''" - assert integration["permissions"] == {"contents": "read", "actions": "read"} - for file in ("build-release.yml", "ci-integration.yml", "ci-release-performance.yml"): - workflow = load_workflow(ROOT / ".github/workflows" / file) - callers = [ - job - for job in workflow["jobs"].values() - if job.get("uses") - in { - "./.github/workflows/release-integration.yml", - "./.github/workflows/release-platform.yml", - } - ] - assert callers - assert all(job["permissions"] == {"contents": "read", "actions": "read"} for job in callers) diff --git a/tests/unit/test_release_wallclock_workflow.py b/tests/unit/test_release_wallclock_workflow.py deleted file mode 100644 index 553469f6b1..0000000000 --- a/tests/unit/test_release_wallclock_workflow.py +++ /dev/null @@ -1,295 +0,0 @@ -"""The observed PR clock must include one complete, read-only production DAG.""" - -from __future__ import annotations - -from copy import deepcopy -from pathlib import Path - -import pytest - -from tests.workflow_contracts import load_workflow, workflow_job, workflow_step - -pytestmark = pytest.mark.component -ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github/workflows/ci-release-wallclock.yml" - - -def _assert_single_world(workflow: dict) -> None: - assert workflow["on"] == { - "pull_request": {"types": ["labeled"]}, - "push": {"branches": ["danielmeppiel-release-wallclock-proof"]}, - } - assert workflow["permissions"] == {"contents": "read", "actions": "read"} - assert workflow["concurrency"]["cancel-in-progress"] is False - jobs = workflow["jobs"] - assert set(jobs) == { - "plan", - "source-checks", - "platforms", - "candidate-ready", - "docs", - "wheels", - "verify", - } - assert jobs["plan"]["if"] == ( - "github.event_name == 'push' || github.event.label.name == 'ci-release-wallclock'" - ) - for job in jobs.values(): - assert "secrets" not in job - assert "environment" not in job - assert not job.get("continue-on-error", False) - assert set(job.get("permissions", {}).values()) <= {"read", "none"} - for step in job.get("steps", []): - assert "secrets." not in str(step) - assert "action-gh-release" not in step.get("uses", "") - assert "deploy-pages" not in step.get("uses", "") - assert not step.get("continue-on-error", False) - - plan = workflow_step(jobs["plan"], "Select all five production catalog platforms") - assert "GITHUB_RUN_ATTEMPT) !== 1" in plan["with"]["script"] - assert "scripts/release-platforms.json" in plan["with"]["script"] - assert "JSON.stringify({include: catalog})" in plan["with"]["script"] - - platforms = jobs["platforms"] - assert platforms["uses"] == "./.github/workflows/release-platform.yml" - assert platforms["needs"] == ["plan"] - assert platforms["strategy"]["fail-fast"] is False - assert platforms["strategy"]["matrix"] == "${{ fromJSON(needs.plan.outputs.matrix) }}" - assert platforms["with"]["full-validation"] is True - assert platforms["with"]["wallclock-evidence"] is True - assert ( - not { - "unit-performance-probe", - "integration-performance-probe", - "performance-workers", - } - & platforms["with"].keys() - ) - for key in ( - "integration-markers", - "integration-shard-count", - "integration-xdist-workers", - "integration-splitting-algorithm", - ): - assert platforms["with"][key] == "${{ matrix." + key.replace("-", "_") + " }}" - - for name, reusable in ( - ("source-checks", "ci"), - ("docs", "docs-build"), - ("wheels", "pypi-distributions"), - ): - assert jobs[name]["needs"] == ["plan"] - assert jobs[name]["uses"] == f"./.github/workflows/{reusable}.yml" - assert jobs["source-checks"]["with"] == {"suppress-pr-only-checks": True} - - ready = workflow_job(workflow, "candidate-ready") - assert ready["needs"] == ["plan", "platforms", "source-checks"] - record = workflow_step( - ready, "Capture native bytes and successful authorities without qualification" - )["with"]["script"] - assert "collectArchiveInventory({" in record - assert "selectRequiredJobs(jobs, catalog)" in record - assert "kind: 'release-wallclock-native-index', promotable: false" in record - assert "await qualify(" not in record - assert "native_ids" in ready["outputs"] and "evidence_id" in ready["outputs"] - verify = workflow_job(workflow, "verify") - assert verify["name"] == "Verify wall-clock artifacts" - assert set(verify["needs"]) == set(jobs) - {"verify"} - assert "always()" in verify["if"] and "!cancelled()" in verify["if"] - authorities = workflow_step( - verify, "Require every production authority and record artifact identities" - )["with"]["script"] - assert "selectRequiredJobs(jobs, catalog)" in authorities - assert "results[name]?.result !== 'success'" in authorities - assert ( - "['plan', 'source-checks', 'platforms', 'candidate-ready', 'docs', 'wheels']" in authorities - ) - assert "listJobsForWorkflowRunAttempt" in authorities - assert "matches.length !== 1" in authorities - assert "!artifact.expired" in authorities - assert "nativeIds !== process.env.NATIVE_IDS" in authorities - for suffix in ("core", "unit", "integration-shard-${index + 1}"): - assert f"test-results-${{attempt}}-${{row.binary_name}}-{suffix}" in authorities - assert "row.integration_shard_count" in authorities - assert "core.setOutput('observation_ids'" in authorities - observations = workflow_step(verify, "Download exact native runner observations") - assert observations["with"] == { - "artifact-ids": "${{ steps.authorities.outputs.observation_ids }}", - "path": "wallclock-inputs/native/observations", - } - assert verify["steps"].index(observations) < verify["steps"].index( - workflow_step(verify, "Verify built bytes without granting release qualification") - ) - assert ( - workflow_step(verify, "Download exact current-run native artifacts")["with"]["artifact-ids"] - == "${{ needs.candidate-ready.outputs.native_ids }}" - ) - assert ( - "verifyLocalArchiveBytes({" - in workflow_step(verify, "Recheck indexed native bytes and prepare non-promotable assets")[ - "with" - ]["script"] - ) - native_upload = workflow_step(verify, "Upload verified native bytes without publishing") - assert native_upload["with"]["compression-level"] == 0 - assert native_upload["with"]["name"] == ( - "wallclock-proposed-verified-native-${{ github.run_attempt }}" - ) - check = workflow_step(verify, "Verify built bytes without granting release qualification") - assert "scripts.release_wallclock verify" in check["run"] - assert '--side proposed --source-root . --source-sha "$GITHUB_SHA"' in check["run"] - assert "--native-root" in check["run"] - assert "--docs-root" in check["run"] - assert "--python-root" in check["run"] - assert "release_rehearsal compare" not in str(workflow) - upload = workflow_step(verify, "Preserve non-promotable wall-clock evidence") - assert upload["with"]["name"] == "wallclock-proposed-proof-${{ github.run_attempt }}" - - -def test_wallclock_executes_one_full_proposed_dag_before_its_terminal_clock() -> None: - """Real elapsed time ends after every native/source/build-artifact authority.""" - _assert_single_world(load_workflow(WORKFLOW)) - - -def test_pr_binary_smoke_remains_default_but_can_be_excluded_from_release_measurement() -> None: - """A reusable caller must not accidentally inherit the PR-only binary job.""" - ci = load_workflow(ROOT / ".github/workflows/ci.yml") - assert ci["on"]["workflow_call"]["inputs"]["suppress-pr-only-checks"] == { - "type": "boolean", - "default": False, - } - assert ci["jobs"]["pr-binary-smoke"]["if"] == ( - "github.event_name == 'pull_request' && !inputs.suppress-pr-only-checks" - ) - for job_id, job in ci["jobs"].items(): - if job_id != "pr-binary-smoke": - assert "suppress-pr-only-checks" not in job.get("if", "") - - -def test_windows_wallclock_keeps_public_dependency_validation_with_the_job_token() -> None: - """Select public checks explicitly, without aliasing the job token into a PAT.""" - native = load_workflow(ROOT / ".github/workflows/release-platform.yml") - step = workflow_step( - workflow_job(native, "release-validation"), "Run release validation tests (Windows)" - ) - assert step["env"]["GITHUB_API_TOKEN"] == "${{ github.token }}" - assert step["env"]["GITHUB_APM_PAT"] == ( - "${{ !inputs.wallclock-evidence && secrets.GH_CLI_PAT || '' }}" - ) - assert "-PublicApiOnly:('${{ inputs.wallclock-evidence }}' -eq 'true')" in step["run"] - assert "GITHUB_TOKEN" not in step["env"] - assert "GITHUB_MODELS_KEY" not in step["env"] - assert "GH_MODELS_PAT" not in str(step) - assert "APM_RUN_INFERENCE_TESTS" not in step["env"] - - -@pytest.mark.parametrize( - ("filename", "job_id", "step_name", "output"), - [ - ("release-platform", "build", "build", "build"), - ("release-unit", "unit-tests", "unit", "unit"), - ("release-integration", "integration-tests", "integration", "integration"), - ], -) -def test_runner_observation_is_opt_in_and_records_the_executing_source( - filename: str, job_id: str, step_name: str, output: str -) -> None: - """Default release behavior is unchanged; measurements identify actual hosts.""" - workflow = load_workflow(ROOT / f".github/workflows/{filename}.yml") - assert workflow["on"]["workflow_call"]["inputs"]["wallclock-evidence"] == { - "type": "boolean", - "default": False, - } - step = workflow_step( - workflow_job(workflow, job_id), - f"Record actual {step_name} runner for wall-clock proof", - ) - assert step["if"] == "inputs.wallclock-evidence" - assert step["shell"] == "bash" - assert step["run"] == ( - "python -m scripts.release_wallclock record-job " - '--side proposed --source-root . --source-sha "$GITHUB_SHA" ' - f"--output test-results/{output}-environment.json" - ) - assert not step.get("continue-on-error", False) - - -@pytest.mark.parametrize( - "fault", - [ - "publish", - "secret", - "cancel", - "synchronize", - "partial", - "no-observer", - "both-worlds", - "retuned-workers", - "late-docs", - "dropped-docs", - "dropped-source", - "waived-failure", - "bypassed-authorities", - "old-artifacts", - "fake-clock", - "collapsed-native-index", - "pr-only-job", - "broad-push-trigger", - "missing-observation-download", - ], -) -def test_wallclock_cannot_silently_change_the_measured_work(fault: str) -> None: - """Reject changes that weaken controls or measure a different pipeline.""" - workflow = deepcopy(load_workflow(WORKFLOW)) - jobs = workflow["jobs"] - if fault == "publish": - workflow["permissions"]["contents"] = "write" - elif fault == "missing-observation-download": - jobs["verify"]["steps"].remove( - workflow_step(jobs["verify"], "Download exact native runner observations") - ) - elif fault == "secret": - jobs["platforms"]["secrets"] = "inherit" - elif fault == "cancel": - workflow["concurrency"]["cancel-in-progress"] = True - elif fault == "synchronize": - workflow["on"]["pull_request"]["types"].append("synchronize") - elif fault == "partial": - jobs["platforms"]["with"]["full-validation"] = False - elif fault == "no-observer": - jobs["platforms"]["with"]["wallclock-evidence"] = False - elif fault == "both-worlds": - jobs["platforms"]["with"]["integration-performance-probe"] = True - elif fault == "retuned-workers": - jobs["platforms"]["with"]["integration-xdist-workers"] = 8 - elif fault == "late-docs": - jobs["docs"]["needs"] = ["platforms"] - elif fault == "dropped-docs": - jobs["verify"]["needs"].remove("docs") - elif fault == "dropped-source": - jobs["source-checks"]["uses"] = "./.github/workflows/ci-source-performance.yml" - elif fault == "waived-failure": - jobs["platforms"]["continue-on-error"] = True - elif fault == "bypassed-authorities": - step = workflow_step( - jobs["verify"], "Require every production authority and record artifact identities" - ) - step["with"]["script"] = step["with"]["script"].replace( - "selectRequiredJobs(jobs, catalog)", "[]" - ) - elif fault == "old-artifacts": - step = workflow_step(jobs["verify"], "Preserve non-promotable wall-clock evidence") - step["with"]["name"] = "release-candidate-evidence" - elif fault == "collapsed-native-index": - del jobs["candidate-ready"] - elif fault == "pr-only-job": - jobs["source-checks"]["with"]["suppress-pr-only-checks"] = False - elif fault == "broad-push-trigger": - workflow["on"]["push"]["branches"] = ["**"] - else: - step = workflow_step( - jobs["verify"], "Verify built bytes without granting release qualification" - ) - step["run"] = "python -m scripts.release_rehearsal compare" - with pytest.raises(AssertionError): - _assert_single_world(workflow) diff --git a/tests/unit/test_windows_compat_gate_workflow.py b/tests/unit/test_windows_compat_gate_workflow.py index 64499af789..a64cf52f0f 100644 --- a/tests/unit/test_windows_compat_gate_workflow.py +++ b/tests/unit/test_windows_compat_gate_workflow.py @@ -39,7 +39,6 @@ class (CRLF text-mode writes, backslash path separators leaking into ROOT = Path(__file__).resolve().parents[2] CI_WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" -SOURCE_PERFORMANCE_WORKFLOW = ROOT / ".github" / "workflows" / "ci-source-performance.yml" MERGE_GATE_WORKFLOW = ROOT / ".github" / "workflows" / "merge-gate.yml" _COLLECTION_ENV_BASELINE = os.environ.copy() @@ -47,11 +46,6 @@ class (CRLF text-mode writes, backslash path separators leaking into GATE_CHECK_NAME = "Windows Compatibility Gate" GATE_STEP = "Run cross-platform contract family" GATE_MARKER = "windows_compat" -SOURCE_PERFORMANCE_JOB = "windows-compat-performance" -SOURCE_PERFORMANCE_LABEL = "ci-performance" -SOURCE_PERFORMANCE_SELECTION = ( - "-m windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py" -) # The full-suite roots already covered by build-and-test-shard (Linux). # The gate must never invoke these WITHOUT a marker filter -- that @@ -459,138 +453,3 @@ def test_narrowing_root_below_windows_compat_root_breaks_contract() -> None: assert positional != ["tests/unit"], ( "mutation setup sanity check failed -- root was not actually narrowed" ) - - -def _source_performance_workflow() -> dict: - return load_workflow(SOURCE_PERFORMANCE_WORKFLOW) - - -def _source_performance_job() -> dict: - return workflow_job(_source_performance_workflow(), SOURCE_PERFORMANCE_JOB) - - -def test_source_performance_probe_is_opt_in_read_only_and_bounded() -> None: - workflow = _source_performance_workflow() - on_block = workflow["on"] - assert on_block["pull_request"]["types"] == [ - "opened", - "reopened", - "synchronize", - "labeled", - ] - assert workflow["permissions"] == {"contents": "read"} - assert workflow["concurrency"]["cancel-in-progress"] is True - - job = workflow_job(workflow, SOURCE_PERFORMANCE_JOB) - assert SOURCE_PERFORMANCE_LABEL in job["if"] - assert "contains(github.event.pull_request.labels.*.name" in job["if"] - assert job["runs-on"] == "windows-latest" - timeout = job.get("timeout-minutes") - assert isinstance(timeout, int) and 0 < timeout <= 20 - assert "secrets" not in job - - -def test_source_performance_probe_compares_serial_and_parallel_same_selection() -> None: - job = _source_performance_job() - matrix = job["strategy"]["matrix"]["include"] - assert matrix == [ - {"variant": "baseline", "pytest_extra": ""}, - {"variant": "proposed", "pytest_extra": "-n 2 --dist loadgroup"}, - ] - run_step = workflow_step(job, "Run Windows compatibility selection") - run = run_step["run"] - assert SOURCE_PERFORMANCE_SELECTION in " ".join(run.split()) - assert "-p scripts.pytest_performance_evidence" in " ".join(run.split()) - assert "${{ matrix.pytest_extra }}" in run - assert "--junitxml=test-results/${{ matrix.variant }}/windows.xml" in run - assert "--store-durations" in run - - -def test_source_performance_probe_captures_proof_inventory_and_unique_artifacts() -> None: - job = _source_performance_job() - collect_step = workflow_step(job, "Record Windows compatibility selection") - collect_run = collect_step["run"] - assert f"selection={SOURCE_PERFORMANCE_SELECTION.removeprefix('-m ')}" in collect_run - assert "test-results/${{ matrix.variant }}/selection.txt" in collect_run - assert not any("--collect-only" in step.get("run", "") for step in job["steps"]) - run = workflow_step(job, "Run Windows compatibility selection")["run"] - assert "uv run --frozen --no-sync --extra dev python -m pytest" in run - - capture_step = workflow_step(job, "Capture comparable proof") - assert capture_step["if"] == "always()" - capture_run = capture_step["run"] - assert "python -m scripts.compare_test_runs capture" in capture_run - assert "--junit test-results/${{ matrix.variant }}/windows.xml" in capture_run - assert "--output test-results/${{ matrix.variant }}/proof.json" in capture_run - assert '--source-sha "$env:GITHUB_SHA"' in capture_run - assert ( - "--selection 'windows_compat tests/unit tests/integration/test_lifecycle_workspace_lock.py'" - in capture_run - ) - assert "--variant ${{ matrix.variant }}" in capture_run - assert "--shard 1" in capture_run - assert "--shard-count 1" in capture_run - - upload_step = workflow_step(job, "Upload Windows compatibility performance evidence") - assert upload_step["if"] == "always()" - assert upload_step["uses"] == ( - "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" - ) - with_block = upload_step["with"] - assert with_block["name"] == ( - "windows-compat-performance-${{ matrix.variant }}-${{ github.run_attempt }}" - ) - assert with_block["path"] == "test-results/${{ matrix.variant }}/" - assert with_block["if-no-files-found"] == "error" - - -def test_source_performance_probe_compares_proofs_after_download() -> None: - workflow = _source_performance_workflow() - compare_job = workflow_job(workflow, "compare-windows-compat-performance") - assert compare_job["name"] == "Compare Windows Compat Performance" - assert compare_job["needs"] == [SOURCE_PERFORMANCE_JOB] - assert SOURCE_PERFORMANCE_LABEL in compare_job["if"] - assert compare_job["runs-on"] == "ubuntu-24.04" - timeout = compare_job.get("timeout-minutes") - assert isinstance(timeout, int) and 0 < timeout <= 5 - - require_step = workflow_step(compare_job, "Require benchmark variants succeeded") - assert "needs.windows-compat-performance.result" in require_step["run"] - assert '!= "success"' in require_step["run"] - - baseline_download = workflow_step(compare_job, "Download baseline proof") - assert baseline_download["uses"] == ( - "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" - ) - assert baseline_download["with"] == { - "name": "windows-compat-performance-baseline-${{ github.run_attempt }}", - "path": "performance-inputs/baseline", - } - proposed_download = workflow_step(compare_job, "Download proposed proof") - assert proposed_download["with"] == { - "name": "windows-compat-performance-proposed-${{ github.run_attempt }}", - "path": "performance-inputs/proposed", - } - - compare_step = workflow_step(compare_job, "Compare equivalent Windows compatibility runs") - compare_run = compare_step["run"] - assert "python -m scripts.compare_test_runs compare" in compare_run - assert "--baseline performance-inputs/baseline/proof.json" in compare_run - assert "--proposed performance-inputs/proposed/proof.json" in compare_run - assert '--expected-sha "$GITHUB_SHA"' in compare_run - assert "--minimum-speedup 0.05" in compare_run - assert "--output performance-comparison/comparison.json" in compare_run - - -def test_source_performance_probe_does_not_write_cache_or_production_evidence() -> None: - text = SOURCE_PERFORMANCE_WORKFLOW.read_text(encoding="utf-8") - forbidden = [ - "actions/cache", - "enable-cache: true", - "release-candidate", - "candidate-", - "GITHUB_APM_PAT", - "ADO_APM_PAT", - "secrets.", - ] - assert [token for token in forbidden if token in text] == [] diff --git a/tests/unit/test_windows_installer_proof_workflow.py b/tests/unit/test_windows_installer_proof_workflow.py deleted file mode 100644 index 4bf6c37d6b..0000000000 --- a/tests/unit/test_windows_installer_proof_workflow.py +++ /dev/null @@ -1,386 +0,0 @@ -"""Contracts for the opt-in Windows installer proof lane.""" - -from __future__ import annotations - -import json -import shutil -import subprocess -import sys -from copy import deepcopy -from pathlib import Path - -import pytest - -from tests.workflow_contracts import load_workflow, workflow_job, workflow_step - -pytestmark = pytest.mark.component - -ROOT = Path(__file__).resolve().parents[2] -WORKFLOW = ROOT / ".github/workflows/ci-windows-installer-proof.yml" -PROOF_SCRIPT = ROOT / "scripts/windows/run-installer-proof.ps1" - - -def _workflow() -> dict: - """Load the proof workflow.""" - return load_workflow(WORKFLOW) - - -def _script_text() -> str: - """Read the proof script as text for static safety contracts.""" - return PROOF_SCRIPT.read_text(encoding="ascii") - - -def _assert_workflow(workflow: dict) -> None: - """Require a read-only, opt-in diagnostic that builds one unsigned candidate.""" - assert workflow["on"] == {"pull_request": {"types": ["labeled"]}} - assert workflow["permissions"] == {"contents": "read"} - assert workflow["concurrency"]["cancel-in-progress"] is False - assert workflow["env"]["BASELINE_VERSION"] == "v0.28.0" - assert set(workflow["jobs"]) == {"windows-installer-proof"} - - job = workflow_job(workflow, "windows-installer-proof") - assert job["if"] == "github.event.label.name == 'ci-windows-installer-proof'" - assert job["runs-on"] == "windows-latest" - assert job["timeout-minutes"] == 45 - assert "secrets" not in job - assert "environment" not in job - assert not job.get("continue-on-error", False) - for step in job["steps"]: - assert "secrets." not in str(step) - assert "action-gh-release" not in step.get("uses", "") - assert "deploy-pages" not in step.get("uses", "") - assert "WINDOWS_CERT_PASSWORD" not in str(step) - assert not step.get("continue-on-error", False) - - fresh = workflow_step(job, "Require fresh proof attempt") - assert "GITHUB_RUN_ATTEMPT -ne '1'" in fresh["run"] - - checkout = job["steps"][1] - assert checkout["uses"].startswith("actions/checkout@") - assert checkout["with"]["persist-credentials"] is False - - build = workflow_step(job, "Build unsigned Windows candidate") - assert build["env"] == {"WINDOWS_CERT_PFX": ""} - assert "scripts/windows/build-binary.ps1" in build["run"] - - package = workflow_step(job, "Package exact Windows candidate archive") - assert package["env"] == { - "CANDIDATE_SHA": "${{ github.sha }}", - "BINARY_NAME": "apm-windows-x86_64", - } - assert "scripts/package_release.py pack" in package["run"] - - candidate = workflow_step(job, "Upload standalone proof candidate archive") - assert candidate["uses"].startswith("actions/upload-artifact@") - assert candidate["with"]["name"] == ( - "windows-installer-proof-candidate-${{ github.run_attempt }}" - ) - assert candidate["with"]["path"] == ( - "release-assets/apm-windows-x86_64.zip\n" - "release-assets/apm-windows-x86_64.zip.sha256\n" - "release-assets/apm-windows-x86_64.json\n" - ) - assert candidate["with"]["if-no-files-found"] == "error" - assert candidate["with"]["compression-level"] == 0 - assert candidate["with"]["retention-days"] == 30 - assert "qualified" not in candidate["with"]["name"] - assert "release-evidence" not in candidate["with"]["name"] - - baseline = workflow_step(job, "Download historical upgrade source once") - assert baseline["env"] == {"GH_TOKEN": "${{ github.token }}"} - assert "gh release download $env:BASELINE_VERSION" in baseline["run"] - assert "--repo microsoft/apm" in baseline["run"] - - proof = workflow_step(job, "Diagnose and prove Windows installer root behavior") - assert "scripts/windows/run-installer-proof.ps1" in proof["run"] - assert "-CandidateArchive release-assets/apm-windows-x86_64.zip" in proof["run"] - assert "-CandidateMetadata release-assets/apm-windows-x86_64.json" in proof["run"] - assert "-BaselineArchive installer-baseline/apm-windows-x86_64.zip" in proof["run"] - assert "-BaselineChecksum installer-baseline/apm-windows-x86_64.zip.sha256" in proof["run"] - - upload = workflow_step(job, "Upload Windows installer proof diagnostics") - assert upload["if"] == "always()" - assert upload["with"]["name"] == "windows-installer-proof-${{ github.run_attempt }}" - assert upload["with"]["path"] == "windows-installer-proof/" - assert upload["with"]["if-no-files-found"] == "error" - - -def test_windows_installer_proof_is_read_only_and_opt_in() -> None: - """The focused lane must not publish, sign, or run on every PR update.""" - _assert_workflow(_workflow()) - - -@pytest.mark.parametrize( - "fault", - ["secret", "write-token", "eager-trigger", "signed", "waived-failure", "missing-upload"], -) -def test_windows_installer_proof_rejects_scope_creep(fault: str) -> None: - """Guard against accidentally turning the diagnostic into release machinery.""" - workflow = deepcopy(_workflow()) - job = workflow_job(workflow, "windows-installer-proof") - if fault == "secret": - job["secrets"] = "inherit" - elif fault == "write-token": - workflow["permissions"]["contents"] = "write" - elif fault == "eager-trigger": - workflow["on"]["pull_request"]["types"].append("synchronize") - elif fault == "signed": - workflow_step(job, "Build unsigned Windows candidate")["env"] = { - "WINDOWS_CERT_PFX": "${{ secrets.WINDOWS_CERT_PFX }}" - } - elif fault == "waived-failure": - job["continue-on-error"] = True - else: - workflow_step(job, "Upload Windows installer proof diagnostics")["with"][ - "if-no-files-found" - ] = "ignore" - with pytest.raises(AssertionError): - _assert_workflow(workflow) - - -def test_windows_installer_proof_script_runs_same_archive_through_both_roots() -> None: - """The script must distinguish path roots without changing installer assertions.""" - text = _script_text() - assert "tests/integration/test_windows_installer_launchers.py" in text - assert text.count('"APM_CANDIDATE_ARCHIVE" = $CandidateArchive') == 1 - assert 'Invoke-InstallerPytest -Name "long-root"' in text - assert 'Invoke-InstallerPytest -Name "short-root"' in text - assert '"wp-" + [System.Guid]::NewGuid().ToString("N").Substring(0, 5)' in text - assert 'Join-Path $resolvedTemp.stdout.Trim() ("pytest-of-" + $env:USERNAME)' in text - assert 'Join-Path $env:RUNNER_TEMP "apm-windows-installer"' in text - assert 'throw "RUNNER_TEMP is required; this proof is CI-only"' in text - assert 'throw "TEMP is required to reproduce the Windows pytest default root"' in text - assert "Refusing to reuse preexisting proof basetemp" in text - assert "Remove-OwnedBaseTemp -Path $longBaseTemp" in text - assert "Remove-OwnedBaseTemp -Path $shortBaseTemp" in text - assert "--basetemp" in text - assert "if (Test-Path -LiteralPath $BaseTemp)" in text - assert "APM Install Test & Edge " in text - assert "if ($shortResult.timed_out -or $shortResult.exit_code -ne 0)" in text - assert "exit $shortResult.exit_code" in text - assert '"APM_E2E_TESTS" = "1"' in text - assert "$result.junit = Read-InstallerOutcome" in text - - -def test_windows_installer_proof_records_charset_paths_and_ps5_stderr_confound() -> None: - """Diagnostics must capture the observed package and PowerShell behavior.""" - text = _script_text() - assert "archive-path-inventory.json" in text - assert "$allEntries = @($zip.Entries)" in text - assert "member_count = $allEntries.Count" in text - assert "file_count = $files.Count" in text - assert "charset_normalizer|chardet" in text - assert '".pyd"' in text - assert "temp_length" in text - assert "stage_length" in text - assert "final_length" in text - assert "powershell_host" in text - assert "powershell.exe" in text - assert "deterministic-native-stderr" in text - assert "exit23-stderr" in text - assert "Broken direct exit23 native-stderr control" in text - assert "$caseNames.Count -ne 2" in text - assert '-not $case.threw -and $case.name -eq "exit0-stderr"' in text - assert '-not $case.threw -and $case.name -eq "exit23-stderr"' in text - assert "Expected Windows PowerShell 5.1" in text - assert "native-stderr-behavior.json" in text - assert 'Invoke-DirectCandidateProbe -Name "long-root"' in text - assert 'Invoke-DirectCandidateProbe -Name "short-root"' in text - assert '"direct-$Name.json"' in text - - -def test_windows_installer_proof_process_capture_selftest(tmp_path: Path) -> None: - """Exercise timeout, stderr, and nonzero capture through the real PowerShell helper.""" - pwsh = shutil.which("pwsh") - if pwsh is None: - pytest.skip("pwsh is not installed") - out_dir = tmp_path / "capture" - completed = subprocess.run( - [ - pwsh, - "-NoProfile", - "-NonInteractive", - "-File", - str(PROOF_SCRIPT), - "-SelfTestProcessCapture", - "-PythonExecutable", - sys.executable, - "-OutputDir", - str(out_dir), - ], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - timeout=45, - ) - assert completed.returncode == 0, completed.stdout + completed.stderr - payload = json.loads((out_dir / "process-capture-selftest.json").read_text()) - assert payload["exit0"]["exit_code"] == 0 - assert "selftest-exit0-stderr" in payload["exit0"]["stderr"] - assert payload["exit23"]["exit_code"] == 23 - assert "selftest-exit23-stderr" in payload["exit23"]["stderr"] - assert payload["timeout"]["timed_out"] is True - - -@pytest.mark.parametrize( - ("cases", "exit_code", "accepted"), - [ - ('', 0, True), - ('', 0, False), - ("", 0, False), - ('', 0, False), - ('', 1, True), - ], -) -def test_installer_outcome_requires_executed_cases( - tmp_path: Path, cases: str, exit_code: int, accepted: bool -) -> None: - """A green pytest exit with skipped assertions is not an installer proof.""" - pwsh = shutil.which("pwsh") - if pwsh is None: - pytest.skip("pwsh is not installed") - report = tmp_path / "junit.xml" - report.write_text(f"{cases}") - completed = subprocess.run( - [ - pwsh, - "-NoProfile", - "-NonInteractive", - "-File", - str(PROOF_SCRIPT), - "-ValidateInstallerJUnit", - str(report), - "-InstallerExitCode", - str(exit_code), - ], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - assert (completed.returncode == 0) is accepted, completed.stdout + completed.stderr - if accepted: - assert json.loads(completed.stdout)["skipped"] == 0 - - -def test_installer_diagnostics_are_persisted_before_junit_validation() -> None: - """The uploadable raw result and visible summary precede any JUnit rejection.""" - body = _script_text().split("function Invoke-InstallerPytest {", 1)[1] - body = body.split("function Invoke-ProcessCaptureSelfTest {", 1)[0] - validation = body.index("$result.junit = Read-InstallerOutcome") - for diagnostic in ( - '"$Name-stdout.log"', - '"$Name-stderr.log"', - 'Write-JsonFile -Value $result -Path (Join-Path $OutDir "$Name-result.json")', - 'Write-Host "[$Name] exit $($result.exit_code) timed_out=$($result.timed_out)"', - ): - assert body.index(diagnostic) < validation - - -@pytest.mark.windows_compat -@pytest.mark.parametrize( - ("junit", "exit_code", "timed_out"), - [ - (None, 124, True), - ("", 0, False), - ( - '' - "", - 0, - False, - ), - ( - '' - "", - 0, - False, - ), - ], - ids=["missing-after-timeout", "malformed", "empty", "skipped", "false-success"], -) -def test_installer_rejected_junit_retains_process_diagnostics( - tmp_path: Path, junit: str | None, exit_code: int, timed_out: bool -) -> None: - """Execute the real persistence/validation path with a captured-process stub.""" - pwsh = shutil.which("pwsh") - if pwsh is None: - pytest.skip("pwsh is not installed") - out_dir = tmp_path / "diagnostics" - out_dir.mkdir() - if junit is not None: - (out_dir / "junit-fixture.xml").write_text(junit, encoding="ascii") - driver = tmp_path / "invoke-installer.ps1" - driver.write_text( - r""" -param([string]$SourceScript, [string]$OutDir, [int]$FixtureExitCode, [string]$TimedOut) -$ErrorActionPreference = "Stop" -$tokens = $null -$errors = $null -$ast = [System.Management.Automation.Language.Parser]::ParseFile( - $SourceScript, [ref]$tokens, [ref]$errors -) -if ($errors.Count) { throw ($errors | Out-String) } -foreach ($name in @("Write-JsonFile", "Read-InstallerOutcome", "Invoke-InstallerPytest")) { - $definition = $ast.Find({ - param($node) - $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and - $node.Name -eq $name - }, $true) - if (-not $definition) { throw "Missing function: $name" } - . ([scriptblock]::Create($definition.Extent.Text)) -} -function Invoke-CapturedProcess { - param($FilePath, $Arguments, $Name, $TimeoutSeconds, $Environment) - return [ordered]@{ - name = $Name - exit_code = $FixtureExitCode - timed_out = $TimedOut -eq "true" - stdout = "fixture captured stdout" - stderr = "fixture captured stderr" - timeout_seconds = $TimeoutSeconds - } -} -Invoke-InstallerPytest -Name "fixture" -OutDir $OutDir ` - -BaseTemp (Join-Path $OutDir "fresh-basetemp") -InstallerEnvironment @{} | - ConvertTo-Json -Depth 8 -""", - encoding="ascii", - ) - completed = subprocess.run( - [ - pwsh, - "-NoProfile", - "-NonInteractive", - "-File", - str(driver), - "-SourceScript", - str(PROOF_SCRIPT), - "-OutDir", - str(out_dir), - "-FixtureExitCode", - str(exit_code), - "-TimedOut", - str(timed_out).lower(), - ], - cwd=tmp_path, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - assert completed.returncode != 0, completed.stdout + completed.stderr - record = json.loads((out_dir / "fixture-result.json").read_text(encoding="utf-8-sig")) - assert record["exit_code"] == exit_code - assert record["timed_out"] is timed_out - assert record["timeout_seconds"] == 900 - assert record["stdout"] == "fixture captured stdout" - assert record["stderr"] == "fixture captured stderr" - assert record["junit_validation_error"] - assert "junit" not in record - for stream in ("stdout", "stderr"): - log = (out_dir / f"fixture-{stream}.log").read_text(encoding="utf-8-sig") - assert log.strip() == f"fixture captured {stream}" - assert f"[fixture {stream}] fixture captured {stream}" in completed.stdout - assert f"[fixture] exit {exit_code} timed_out={timed_out}" in completed.stdout diff --git a/tests/unit/test_windows_installer_result.py b/tests/unit/test_windows_installer_result.py new file mode 100644 index 0000000000..efe27fd5c2 --- /dev/null +++ b/tests/unit/test_windows_installer_result.py @@ -0,0 +1,56 @@ +"""Executable contracts for production installer JUnit validation.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.component + +ROOT = Path(__file__).resolve().parents[2] +RESULT_SCRIPT = ROOT / "scripts/windows/validate-installer-result.ps1" + + +@pytest.mark.parametrize( + ("cases", "exit_code", "accepted"), + [ + ('', 0, True), + ('', 0, False), + ("", 0, False), + ('', 0, False), + ('', 1, True), + ], +) +def test_installer_outcome_requires_executed_cases( + tmp_path: Path, cases: str, exit_code: int, accepted: bool +) -> None: + """A green pytest exit with skipped assertions is not installer evidence.""" + pwsh = shutil.which("pwsh") + if pwsh is None: + pytest.skip("pwsh is not installed") + report = tmp_path / "junit.xml" + report.write_text(f"{cases}") + completed = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(RESULT_SCRIPT), + "-ValidateInstallerJUnit", + str(report), + "-InstallerExitCode", + str(exit_code), + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert (completed.returncode == 0) is accepted, completed.stdout + completed.stderr + if accepted: + assert json.loads(completed.stdout)["skipped"] == 0 diff --git a/tests/unit/test_windows_unit_diagnostics.py b/tests/unit/test_windows_unit_diagnostics.py index db1c63e5f7..fe6ee57f7c 100644 --- a/tests/unit/test_windows_unit_diagnostics.py +++ b/tests/unit/test_windows_unit_diagnostics.py @@ -13,7 +13,7 @@ from tests.workflow_contracts import ( load_workflow, - wallclock_unit_commands, + shell_commands, workflow_job, workflow_step, ) @@ -39,25 +39,13 @@ def _assert_windows_diagnostics(workflow: dict) -> None: job = workflow_job(workflow, "unit-tests") linux = workflow_step(job, "Run unit tests") windows = workflow_step(job, "Run unit tests (Windows diagnostics)") - assert linux["if"] == "inputs.platform != 'windows' && !inputs.performance-probe" + assert linux["if"] == "inputs.platform != 'windows'" assert windows["if"] == "inputs.platform == 'windows'" assert windows["timeout-minutes"] == 60 - for observed in (False, True): - command = TEST_COMMAND - if observed: - command = [ - *TEST_COMMAND[:3], - "python", - "-m", - "pytest", - "-p", - "scripts.pytest_performance_evidence", - *TEST_COMMAND[4:], - ] - assert wallclock_unit_commands(linux, enabled=observed) == [command + PARALLEL_ARGS] - assert wallclock_unit_commands(windows, enabled=observed) == [ - [*command, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS] - ] + assert shell_commands(linux) == [TEST_COMMAND + PARALLEL_ARGS] + assert shell_commands(windows) == [ + [*TEST_COMMAND, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS] + ] assert windows["env"] == { "PYTHONUNBUFFERED": "1", "GITHUB_TOKEN": "${{ secrets.GH_MODELS_PAT }}", diff --git a/tests/workflow_contracts.py b/tests/workflow_contracts.py index 4ee2c726ed..9a96f63e6b 100644 --- a/tests/workflow_contracts.py +++ b/tests/workflow_contracts.py @@ -84,18 +84,6 @@ def shell_commands(step: WorkflowNode) -> list[list[str]]: ] -def wallclock_unit_commands(step: WorkflowNode, *, enabled: bool) -> list[list[str]]: - """Resolve only the explicitly supported, observation-only pytest choice.""" - expression = ( - "${{ inputs.wallclock-evidence && " - "'python -m pytest -p scripts.pytest_performance_evidence' || 'pytest' }}" - ) - command = step["run"] - assert command.count(expression) == 1 - invocation = "python -m pytest -p scripts.pytest_performance_evidence" if enabled else "pytest" - return shell_commands({**step, "run": command.replace(expression, invocation)}) - - def assert_exact_command( commands: list[list[str]], expected: list[str], From c1af195b48cc3954dda71a98b91c614f16058b5a Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 19:25:17 +0200 Subject: [PATCH 20/23] fix(release): explain prerelease publication decisions Keep the planner output and operator diagnostic on the same classification result. Explain why non-stable tags skip docs and PyPI publication. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/release-candidate.cjs | 8 ++++++-- scripts/release-candidate.test.cjs | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/release-candidate.cjs b/scripts/release-candidate.cjs index aba9b08958..8003191863 100644 --- a/scripts/release-candidate.cjs +++ b/scripts/release-candidate.cjs @@ -746,6 +746,7 @@ async function verify({ async function plan({ github, context, core }) { const catalog = loadCatalog(); const { matrix, fullValidation } = matrixForContext(context, catalog); + const prerelease = isPrereleaseTag(context); let candidateRunId = ""; let candidateArtifactIds = ""; let candidateEvidenceArtifactId = ""; @@ -760,13 +761,16 @@ async function plan({ github, context, core }) { decision = `reuse; trusted exact-SHA candidate from source run ${candidate.run.id} attempt ${candidate.run.run_attempt} SHA ${candidate.run.head_sha}`; } } - core?.info?.(`Release plan: ${decision}.`); + const publication = !isTagPush(context) ? "" : prerelease + ? " Prerelease tag; stable docs and PyPI publication will be skipped. Use an exact vN.N.N tag for stable publication." + : " Stable tag."; + core?.info?.(`Release plan: ${decision}.${publication}`); core.setOutput("matrix", JSON.stringify(matrix)); core.setOutput("full_validation", String(fullValidation)); core.setOutput("candidate_run_id", candidateRunId); core.setOutput("candidate_artifact_ids", candidateArtifactIds); core.setOutput("candidate_evidence_artifact_id", candidateEvidenceArtifactId); - core.setOutput("is_prerelease", String(isPrereleaseTag(context))); + core.setOutput("is_prerelease", String(prerelease)); } module.exports = { diff --git a/scripts/release-candidate.test.cjs b/scripts/release-candidate.test.cjs index 03ff7a23ff..e70631ba8b 100644 --- a/scripts/release-candidate.test.cjs +++ b/scripts/release-candidate.test.cjs @@ -252,13 +252,23 @@ describe("release candidate planning", () => { ]) { it(`classifies release publication for ${tag} through the planner`, async () => { const outputs = {}; + const messages = []; await rc.plan({ github: fakeGithub(), context: context({ ref: `refs/tags/${tag}` }), - core: { setOutput(name, value) { outputs[name] = value; } }, + core: { + setOutput(name, value) { outputs[name] = value; }, + info(message) { messages.push(message); }, + }, }); assert.equal(outputs.is_prerelease, String(prerelease)); assert.equal(outputs.full_validation, "true"); + const publication = prerelease + ? " Prerelease tag; stable docs and PyPI publication will be skipped. Use an exact vN.N.N tag for stable publication." + : " Stable tag."; + assert.deepEqual(messages, [ + `Release plan: fresh full qualification; no reusable candidate is available for this SHA.${publication}`, + ]); }); } @@ -295,7 +305,7 @@ describe("release candidate planning", () => { assert.match(messages[0], /Prior candidate run 123 attempt 1 unavailable:/); assert.match(messages[0], /automatically\. No operator action required\./); assert.equal(messages.some((message) => /Re-run all jobs/.test(message)), false); - assert.equal(messages[1], "Release plan: fresh full qualification; no reusable candidate is available for this SHA."); + assert.equal(messages[1], "Release plan: fresh full qualification; no reusable candidate is available for this SHA. Stable tag."); }); } @@ -357,7 +367,7 @@ describe("release candidate planning", () => { assert.match(outputs.candidate_evidence_artifact_id, /^[0-9]+$/); assert.equal(outputs.candidate_artifact_ids.split(",").length, 5); assert.deepEqual(messages, [ - `Release plan: reuse; trusted exact-SHA candidate from source run 123 attempt 1 SHA ${SHA}.`, + `Release plan: reuse; trusted exact-SHA candidate from source run 123 attempt 1 SHA ${SHA}. Stable tag.`, ]); }); @@ -374,7 +384,8 @@ describe("release candidate planning", () => { context: context(overrides), core: { setOutput() {}, info(message) { messages.push(message); } }, }); - assert.deepEqual(messages, [`Release plan: ${message}.`]); + const publication = Object.keys(overrides).length === 0 ? " Stable tag." : ""; + assert.deepEqual(messages, [`Release plan: ${message}.${publication}`]); }); } }); From 7045750abdb1796a0146f37b29294cf2f28fc59d Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 19:52:10 +0200 Subject: [PATCH 21/23] fix(release): report completed candidate artifact paths Keep the isolated packaging helper standard-library only while reporting the resolved archive or verified executable after success. Cover both CLI operations with fixture archives and document the output. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../docs/contributing/integration-testing.md | 2 + scripts/package_release.py | 6 ++- tests/unit/scripts/test_package_release.py | 45 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index b335aac91e..855f654fb8 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -423,6 +423,8 @@ neither grants publishing permissions nor bypasses qualification. and executable SHA-256 digests, verifies member safety and embedded version/build SHA, extracts into a fresh destination, and rechecks the executable before upload. Publication never repacks archives. +For local runs, successful `pack` and `verify-extract` commands print the resolved +archive and extracted executable paths, respectively. Candidate artifacts are attempt-scoped (`candidate--`; evidence is `release-candidate-evidence-`), so partial reruns must not reuse old-attempt artifacts. Before publication, use **Re-run all jobs** to diff --git a/scripts/package_release.py b/scripts/package_release.py index 1d9a821359..24712f4a4d 100644 --- a/scripts/package_release.py +++ b/scripts/package_release.py @@ -177,8 +177,12 @@ def main() -> None: args = parser.parse_args() if args.operation == "pack": package(args.binary_name, args.dist, args.archives, args.sha, args.pyproject) + artifact = args.archives / archive_name(args.binary_name) + action = "Packaged" else: - verify_extract(args.binary_name, args.archives, args.dist, args.sha) + artifact = verify_extract(args.binary_name, args.archives, args.dist, args.sha) + action = "Verified" + print(f"[+] {action} {args.binary_name}: {artifact.resolve()}") if __name__ == "__main__": diff --git a/tests/unit/scripts/test_package_release.py b/tests/unit/scripts/test_package_release.py index e52b06edd3..56d2994dc1 100644 --- a/tests/unit/scripts/test_package_release.py +++ b/tests/unit/scripts/test_package_release.py @@ -17,6 +17,7 @@ BINARY_NAMES, archive_name, file_digest, + main, package, require_binary_identity, verify_extract, @@ -42,6 +43,50 @@ def candidate(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Pa return dist, archives, pyproject +@pytest.mark.parametrize("operation", ["pack", "verify-extract"]) +def test_cli_reports_the_completed_candidate_path( + candidate: tuple[Path, Path, Path], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + operation: str, +) -> None: + dist, archives, pyproject = candidate + binary_name = "apm-linux-x86_64" + bundle = dist / binary_name + bundle.mkdir(parents=True) + (bundle / "apm").write_bytes(b"candidate") + expected = archives / archive_name(binary_name) + action = "Packaged" + if operation == "verify-extract": + package(binary_name, dist, archives, SHA, pyproject) + dist = tmp_path / "isolated" + expected = dist / binary_name / "apm" + action = "Verified" + monkeypatch.setattr( + "sys.argv", + [ + "package_release.py", + operation, + "--binary-name", + binary_name, + "--sha", + SHA, + "--dist", + str(dist), + "--archives", + str(archives), + "--pyproject", + str(pyproject), + ], + ) + + main() + + assert capsys.readouterr().out == f"[+] {action} {binary_name}: {expected.resolve()}\n" + assert expected.is_file() + + @pytest.mark.parametrize("binary_name", BINARY_NAMES) def test_archive_round_trip_keeps_candidate_bytes_and_hidden_files( candidate: tuple[Path, Path, Path], tmp_path: Path, binary_name: str From f1b1a2283b52cc77f629860aaf97655699eb8c95 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 19:58:50 +0200 Subject: [PATCH 22/23] test(ci): preserve xdist affinity in JUnit identity assertions Compare the real reported identities including their known loadgroup suffix, while keeping scheduling hints keyed by collected node IDs. Repeating one shard still fails the disjoint-coverage guard. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/unit/test_pytest_timing_workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_pytest_timing_workflows.py b/tests/unit/test_pytest_timing_workflows.py index eafa803635..e3be6adc3d 100644 --- a/tests/unit/test_pytest_timing_workflows.py +++ b/tests/unit/test_pytest_timing_workflows.py @@ -229,4 +229,5 @@ def test_real_split_and_xdist_preserve_complete_disjoint_coverage( assert stored assert all(duration >= 0 for duration in stored.values()) assert observed[0].isdisjoint(observed[1]) - assert observed[0] | observed[1] == expected + # xdist loadgroup retains the fixture-affinity suffix in standard JUnit names. + assert observed[0] | observed[1] == {f"{node}@shared_home" for node in expected} From 6ce3be12bba2eb118a01d012d1dfd6e69d2b1ff5 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Wed, 9 Sep 2026 20:03:27 +0200 Subject: [PATCH 23/23] style(tests): format retained release contracts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/unit/test_platform_contract_workflow.py | 8 ++------ tests/unit/test_windows_unit_diagnostics.py | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_platform_contract_workflow.py b/tests/unit/test_platform_contract_workflow.py index c87fa345de..3f02176ee8 100644 --- a/tests/unit/test_platform_contract_workflow.py +++ b/tests/unit/test_platform_contract_workflow.py @@ -391,9 +391,7 @@ def _assert_integration(workflow: dict, integration: dict) -> None: assert job["with"]["xdist-workers"] == "${{ inputs.integration-xdist-workers }}" assert job["with"]["splitting-algorithm"] == "${{ inputs.integration-splitting-algorithm }}" assert job["with"]["timeout-minutes"] == UNIX_TIMEOUT - assert job["with"]["artifact-prefix"] == ( - "${{ github.run_attempt }}-${{ inputs.binary-name }}" - ) + assert job["with"]["artifact-prefix"] == ("${{ github.run_attempt }}-${{ inputs.binary-name }}") assert job["with"]["runtime-prerequisites"] == "none" for removed_key in ( "evidence-variant", @@ -445,9 +443,7 @@ def _assert_integration(workflow: dict, integration: dict) -> None: for step in integration_job["steps"] if isinstance(step, dict) and step.get("name") == removed_name ] - assert matches == [], ( - f"experimental integration step {removed_name!r} must be removed" - ) + assert matches == [], f"experimental integration step {removed_name!r} must be removed" timings_upload = workflow_step(integration_job, "Upload integration timings and outcomes") assert timings_upload["if"] == "always() && inputs.platform != 'windows'" assert timings_upload["with"]["if-no-files-found"] == "error" diff --git a/tests/unit/test_windows_unit_diagnostics.py b/tests/unit/test_windows_unit_diagnostics.py index fe6ee57f7c..6ac17c3f17 100644 --- a/tests/unit/test_windows_unit_diagnostics.py +++ b/tests/unit/test_windows_unit_diagnostics.py @@ -43,9 +43,7 @@ def _assert_windows_diagnostics(workflow: dict) -> None: assert windows["if"] == "inputs.platform == 'windows'" assert windows["timeout-minutes"] == 60 assert shell_commands(linux) == [TEST_COMMAND + PARALLEL_ARGS] - assert shell_commands(windows) == [ - [*TEST_COMMAND, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS] - ] + assert shell_commands(windows) == [[*TEST_COMMAND, *PARALLEL_ARGS, *DIAGNOSTIC_ARGS]] assert windows["env"] == { "PYTHONUNBUFFERED": "1", "GITHUB_TOKEN": "${{ secrets.GH_MODELS_PAT }}",