diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5d224b..4fc73a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,6 +255,11 @@ jobs: - name: Fast-path contract tests run: python3 scripts/test_fast_path.py + # E2E off the pull request: the `CI_E2E_IN_CI` switch and `mode: e2e` + # (the post-merge / nightly run), over the real job graph. + - name: E2E mode contract tests + run: python3 scripts/test_e2e_mode.py + # The pull-request E2E subset (`e2e-pr-shards`/`e2e-pr-args`). Two # halves: that leaving both unset is byte-for-byte today's behaviour on # every event — every pinned adopter passes neither — and that `e2e` diff --git a/.github/workflows/nuxt-cloudflare.yml b/.github/workflows/nuxt-cloudflare.yml index 9c8a325..6edcbd0 100644 --- a/.github/workflows/nuxt-cloudflare.yml +++ b/.github/workflows/nuxt-cloudflare.yml @@ -112,6 +112,29 @@ name: Reusable Nuxt + Cloudflare CI # alongside its push/pull_request triggers, with the same `ci:` job. The # quarantine lane (`e2e-quarantine-args`) runs there too. # +# E2E OFF THE PULL REQUEST (fast-CI program, Logan 2026-09-29: "skip on PRs; +# run after each merge, newest wins, plus nightly; one org switch"). Two +# additive pieces, both no-ops until used: +# vars.CI_E2E_IN_CI == 'false' -> `E2E plan`, `E2E` and `E2E quarantine` are +# skipped in an ordinary CI run on any event +# and `Required` reads that skip as success. +# It never overrides `e2e-full-paths` +# (protected-path escalation) or +# `expected-candidate-sha` (explicit release +# validation), both promises to run browsers. +# mode: e2e -> the post-merge / nightly run: Build -> +# E2E plan -> E2E and a `Required` verdict, +# every other lane skipped, the variable +# ignored. It lives in its own caller +# workflow (`E2E`) because an app's promote +# workflow fires on completion of the whole +# CI workflow. See README "E2E off the pull +# request". +# The effective switch is one expression, repeated wherever `inputs.run-e2e` +# used to gate a lane; scripts/test_e2e_mode.py evaluates all of them. +# `pnpm audit` is a warning on a pull request and blocks on the default branch, +# schedule and workflow_dispatch (see the `Dependency audit` step). +# # THE QUALITY LEVEL (coding-standards deep dive, 2026-09-27). The estate has # web quality tools that gate nothing on most apps because each one is an # opt-in nobody opted into: the foundation check, the per-route performance @@ -420,6 +443,20 @@ on: required: false type: boolean default: false + mode: + description: >- + `ci` (the default) is the ordinary gate. `e2e` is the post-merge and + nightly Playwright run and nothing else: Build -> E2E plan -> E2E and a + final `Required` verdict, every other lane skipped, and the + `CI_E2E_IN_CI` variable ignored. It requires `run-e2e: true` (the + verdict fails otherwise) and is meant for a caller workflow named + `E2E` triggered by push to the default branch, a nightly schedule and + workflow_dispatch, with a workflow-level concurrency group keyed on the + repository and `cancel-in-progress: true` so the newest merge wins. See README + "E2E off the pull request". Any other value fails `Required`. + required: false + type: string + default: "ci" e2e-script: description: >- package.json script for the e2e suite. When run-e2e is true this @@ -1055,7 +1092,8 @@ jobs: reuse-plan: name: Reuse plan if: >- - inputs.required-reuse-pr-results + inputs.mode != 'e2e' + && inputs.required-reuse-pr-results && inputs.journey-smoke-url == '' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) @@ -1604,6 +1642,9 @@ jobs: const workflow = process.env.CALLER_WORKFLOW_REF.split('@')[0]; const workflowPath = workflow.slice(`${repo.owner}/${repo.repo}/`.length); const inputs = JSON.parse(process.env.PROOF_INPUTS); + // Added after proofs already existed; absent from the key at its + // default so those proofs still match (the E2E key drops it always). + if (inputs.mode === 'ci') delete inputs.mode; const canonicalInputs = Object.fromEntries(Object.entries(inputs).sort(([a], [b]) => a.localeCompare(b))); const { data: commit } = await github.rest.repos.getCommit({ ...repo, ref: context.sha, request: { timeout: 10000 } }); const tree = commit.commit.tree.sha; @@ -1631,7 +1672,7 @@ jobs: build: # Named `Required` when it is the only lane for this event (REQUIRED # FOLDED INTO BUILD in the header); `Build` otherwise, as before. - name: ${{ (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results) && 'Required' || 'Build' }} + name: ${{ (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results) && 'Required' || 'Build' }} # P3-C1: skipped only when `Reuse plan` proved this exact tree, or in # journey-smoke mode. `!cancelled()` keeps it running when `Reuse plan` # was skipped (the default) or could not decide. @@ -2171,12 +2212,28 @@ jobs: # audit itself takes. `Required` covers it through `build`, which it # already demands success from. - name: Dependency audit - if: inputs.dependency-audit + if: inputs.dependency-audit && inputs.mode != 'e2e' env: PM: ${{ inputs.package-manager }} AUDIT_IGNORE: ${{ inputs.audit-ignore }} + EVENT_NAME: ${{ github.event_name }} run: | set -euo pipefail + # WARN ON A PULL REQUEST, BLOCK ON MAIN. An advisory is published on + # its own schedule, not the pull request's, so a red `pnpm audit` on + # an unrelated PR blocks work the PR did not cause (Logan 2026-09-29, + # "Main + nightly"). On `pull_request` / `pull_request_target` every + # finding below is re-labelled `::warning::` (the advisory list is + # still printed and summarised) and the step passes. A push to the + # default branch, a schedule and a manual dispatch keep the + # fail-closed behaviour: "the audit did not run" is still a failure. + warn_only=false + case "${EVENT_NAME:-}" in + pull_request|pull_request_target) warn_only=true ;; + esac + as_annotation() { + if [ "$warn_only" = true ]; then sed 's/^::error::/::warning::/'; else cat; fi + } tmpdir="${RUNNER_TEMP:-/tmp}" report="$tmpdir/dependency-audit.json" errlog="$tmpdir/dependency-audit.err" @@ -2188,11 +2245,16 @@ jobs: audit_status=$? set -e if [ ! -s "$report" ]; then - echo "::error::dependency audit produced no report ($PM audit exited $audit_status) -- the audit did not run, which is a failure, not a pass" + echo "::error::dependency audit produced no report ($PM audit exited $audit_status) -- the audit did not run, which is a failure, not a pass" | as_annotation sed -n '1,20p' "$errlog" >&2 || true + if [ "$warn_only" = true ]; then + echo "::warning::dependency audit did not run on this pull request; it blocks on the default branch, schedule and manual runs" + exit 0 + fi exit 1 fi - AUDIT_REPORT="$report" AUDIT_STATUS="$audit_status" python3 - <<'PYEOF_AUDIT' + audit_rc=0 + AUDIT_REPORT="$report" AUDIT_STATUS="$audit_status" python3 - <<'PYEOF_AUDIT' | as_annotation || audit_rc=$? import json import os import re @@ -2375,6 +2437,11 @@ jobs: ) raise SystemExit(1) PYEOF_AUDIT + if [ "$audit_rc" -ne 0 ] && [ "$warn_only" = true ]; then + echo "::warning::dependency audit found blocking findings (listed above as warnings). They do not fail this pull request; they will fail the default branch, the nightly run and a manual run until fixed or given an audit-ignore reason." + exit 0 + fi + exit "$audit_rc" # OPT-IN web-foundation conformance gate (company-hq # docs/WEB-FOUNDATION-CHECK.md §4/§5, D-WEBFOUND-2 Q5/Q9 (a), D-WEBFOUND-3). @@ -2401,7 +2468,7 @@ jobs: # check" below prints it to the log and applies the decided blocking # rule. - name: Run web-foundation conformance check - if: steps.quality.outputs.foundation-check == 'true' + if: steps.quality.outputs.foundation-check == 'true' && inputs.mode != 'e2e' env: PM: ${{ inputs.package-manager }} TOOL_VERSION: ${{ inputs.foundation-check-tool-version }} @@ -2520,7 +2587,7 @@ jobs: # a check that cannot see cannot pass (WEB-FOUNDATION-CHECK.md §5, "the # app's CI must treat [UNKNOWN] as a failure too"). - name: Evaluate web-foundation conformance check - if: always() && steps.quality.outputs.foundation-check == 'true' + if: always() && steps.quality.outputs.foundation-check == 'true' && inputs.mode != 'e2e' run: | set -uo pipefail echo "## Web foundation conformance check" >> "$GITHUB_STEP_SUMMARY" @@ -2577,7 +2644,7 @@ jobs: # copied, so the two places cannot drift. Before `build-script`, so a # type error fails the job before the build spends its time. - name: Typecheck Worker - if: inputs.checks-in-build + if: inputs.checks-in-build && inputs.mode != 'e2e' env: &checks_tw_env GATE: Typecheck Worker SCRIPT: ${{ inputs.typecheck-worker-script }} @@ -2609,7 +2676,7 @@ jobs: fi if [ "$PM" = "pnpm" ]; then pnpm run "$SCRIPT"; else npm run "$SCRIPT"; fi - name: Typecheck Nuxt - if: inputs.checks-in-build + if: inputs.checks-in-build && inputs.mode != 'e2e' env: &checks_tn_env GATE: Typecheck Nuxt SCRIPT: ${{ inputs.typecheck-web-script }} @@ -2641,7 +2708,7 @@ jobs: fi if [ "$PM" = "pnpm" ]; then pnpm run "$SCRIPT"; else npm run "$SCRIPT"; fi - name: Unit tests - if: inputs.checks-in-build && inputs.run-tests + if: inputs.checks-in-build && inputs.run-tests && inputs.mode != 'e2e' env: &checks_ut_env GATE: Unit tests SCRIPT: ${{ inputs.test-script }} @@ -2683,7 +2750,7 @@ jobs: # hooks (see the input's BUILD OUTPUTS caveat). - name: Start concurrent scripts id: concurrent-start - if: inputs.concurrent-scripts != '' + if: inputs.concurrent-scripts != '' && inputs.mode != 'e2e' env: GATE: Concurrent scripts SCRIPTS: ${{ inputs.concurrent-scripts }} @@ -2769,7 +2836,7 @@ jobs: # extra-scripts entry is always something the caller wrote down by name, # so a typo in one of them is precisely the silent-loss case. - name: Extra scripts - if: inputs.extra-scripts != '' + if: inputs.extra-scripts != '' && inputs.mode != 'e2e' env: GATE: Extra scripts SCRIPTS: ${{ inputs.extra-scripts }} @@ -2838,7 +2905,7 @@ jobs: # this step refuses a report whose app has no build output, and treats # an unreadable report as a failure rather than as "no violations". - name: Performance budget - if: steps.quality.outputs.performance-budget == 'true' + if: steps.quality.outputs.performance-budget == 'true' && inputs.mode != 'e2e' env: PM: ${{ inputs.package-manager }} PERF_ARGS: ${{ inputs.performance-budget-args }} @@ -2901,7 +2968,7 @@ jobs: id: e2e-build-path env: BUILD_PATH: ${{ inputs.e2e-build-artifact-path }} - RUN_E2E: ${{ inputs.run-e2e }} + RUN_E2E: ${{ (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} ARTIFACT_SCOPE: ${{ inputs.artifact-scope }} run: | set -euo pipefail @@ -2979,7 +3046,7 @@ jobs: # remove a required parser (mirrors ci.yml's own "Ensure PyYAML is # available" step). - name: Ensure PyYAML is available - if: "!cancelled() && (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" + if: "!cancelled() && (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" working-directory: ${{ github.workspace }} run: &caller_lint_pyyaml | set -euo pipefail @@ -2992,7 +3059,7 @@ jobs: fi - name: Install actionlint - if: "!cancelled() && (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" + if: "!cancelled() && (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" working-directory: ${{ github.workspace }} env: &caller_lint_actionlint_env ACTIONLINT_VERSION: 1.7.12 @@ -3010,7 +3077,7 @@ jobs: ./actionlint -version - name: actionlint (caller's own workflows) - if: "!cancelled() && (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" + if: "!cancelled() && (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" working-directory: ${{ github.workspace }} run: &caller_lint_actionlint | set -euo pipefail @@ -3023,7 +3090,7 @@ jobs: ./actionlint -no-color "${files[@]}" - name: Caller workflow hygiene audit (concurrency, timeouts, SHA pins, permissions) - if: "!cancelled() && (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" + if: "!cancelled() && (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" working-directory: ${{ github.workspace }} run: &caller_lint_audit | set -euo pipefail @@ -3144,7 +3211,7 @@ jobs: # P3-C1: same reuse/smoke gating as `Build`. `checks-in-build` runs these # steps in `Build` instead, and this job is skipped. needs: [reuse-plan] - if: "!cancelled() && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !inputs.checks-in-build" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !inputs.checks-in-build" # Blacksmith overflow (see the block above this file's first routed job) -- D-CI-CAP-1 (c)/D-BLACKSMITH-2, fleet#337. runs-on: ${{ fromJSON(vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"')) }} timeout-minutes: 30 @@ -3299,7 +3366,7 @@ jobs: name: Extra gate # P3-C1: same reuse/smoke gating as `Build`. needs: [reuse-plan] - if: "!cancelled() && inputs.extra-gate-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.extra-gate-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" # Blacksmith overflow (see the block above this file's first routed job) -- D-CI-CAP-1 (c)/D-BLACKSMITH-2, fleet#337. runs-on: ${{ fromJSON(vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"')) }} timeout-minutes: 30 @@ -3437,7 +3504,7 @@ jobs: name: E2E plan # P3-C1: same reuse/smoke gating as `Build`. needs: [reuse-plan] - if: "!cancelled() && inputs.run-e2e && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" + if: "!cancelled() && (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" # CI lightweight route (company-hq CI policy); caller override takes precedence. runs-on: ${{ fromJSON(github.event.repository.private == false && '"ubuntu-latest"' || inputs.lightweight-runner || vars.CI_LIGHTWEIGHT_RUNNER || (vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"'))) }} # Headroom for the linux-ci capacity-hook slot wait (up to 900s, counts against timeout-minutes) -- fleet#276. @@ -3489,6 +3556,10 @@ jobs: const workflow = process.env.CALLER_WORKFLOW_REF.split('@')[0]; const workflowPath = workflow.slice(`${repo.owner}/${repo.repo}/`.length); const inputs = JSON.parse(process.env.E2E_INPUTS); + // `mode` is not part of the key (it says which caller ran this, not + // what was tested). The workflow path IS, so a ci.yml proof and an + // E2E-workflow proof never cross. + delete inputs.mode; // Fast-path inputs (P3-C1) and the quality-level inputs stay out of // the key at their defaults, so proof minted before they existed // still matches. @@ -3533,6 +3604,7 @@ jobs: REUSED: ${{ steps.proof.outputs.reused }} SKIP_PATTERNS: ${{ inputs.e2e-skip-paths }} FULL_PATTERNS: ${{ inputs.e2e-full-paths }} + MODE: ${{ inputs.mode }} run: | set -euo pipefail # `set -f`: the pattern lists below are split with unquoted `$SKIP_PATTERNS` @@ -3553,6 +3625,17 @@ jobs: echo "the same Git tree and full E2E configuration already passed on the pull request" exit 0 fi + # THE POST-MERGE RUN NEVER PATH-SKIPS. Its concurrency group cancels + # the run in flight when a newer merge lands, so the newest push's + # diff no longer covers the commits whose run was cancelled: a + # docs-only push after a code push would skip E2E while the code push + # was never tested. Running the whole suite is the only sound answer. + if [ "${MODE:-ci}" = e2e ]; then + echo "skipped=false" >> "$GITHUB_OUTPUT" + echo "full=true" >> "$GITHUB_OUTPUT" + echo "mode e2e runs the full suite: a newer push cancels this run, so no diff can prove a skip" + exit 0 + fi # One changed-file read serves both path inputs. `full` answers # "does this PR need the full configuration?" (e2e-full-paths). When # the files cannot be determined it is `true` for a caller that @@ -3837,7 +3920,7 @@ jobs: # opt-in and off by default. name: E2E # P3-C1: explicit status guard (see SKIP PROPAGATION at `reuse-plan`). - if: "!cancelled() && needs.build.result == 'success' && needs.e2e-plan.result == 'success' && inputs.run-e2e && needs.e2e-plan.outputs.skipped != 'true'" + if: "!cancelled() && needs.build.result == 'success' && needs.e2e-plan.result == 'success' && (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && needs.e2e-plan.outputs.skipped != 'true'" needs: [build, e2e-plan] # `e2e-runner` when the caller routes browsers to an isolated pool, # otherwise `runner` — same guest as the build, the previous behaviour. @@ -4508,7 +4591,7 @@ jobs: e2e-quarantine: name: E2E # P3-C1: explicit status guard (see SKIP PROPAGATION at `reuse-plan`). - if: "!cancelled() && needs.build.result == 'success' && needs.e2e-plan.result == 'success' && inputs.run-e2e && inputs.e2e-quarantine-args != '' && needs.e2e-plan.outputs.skipped != 'true'" + if: "!cancelled() && needs.build.result == 'success' && needs.e2e-plan.result == 'success' && (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.e2e-quarantine-args != '' && needs.e2e-plan.outputs.skipped != 'true'" needs: [build, e2e-plan] runs-on: ${{ fromJSON(inputs.e2e-runner != '' && inputs.e2e-runner || (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"')) }} timeout-minutes: 30 @@ -4570,6 +4653,7 @@ jobs: # P3-C1: explicit status guard (see SKIP PROPAGATION at `reuse-plan`). if: >- !cancelled() && needs.build.result == 'success' + && inputs.mode != 'e2e' && inputs.preview-checks != 'none' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') needs: build @@ -5172,7 +5256,7 @@ jobs: deploy-dry-run: name: Deploy dry run # P3-C1: explicit status guard (see SKIP PROPAGATION at `reuse-plan`). - if: "!cancelled() && needs.build.result == 'success' && inputs.wrangler-dry-run" + if: "!cancelled() && needs.build.result == 'success' && inputs.mode != 'e2e' && inputs.wrangler-dry-run" needs: build # Blacksmith overflow (see the block above this file's first routed job) -- D-CI-CAP-1 (c)/D-BLACKSMITH-2, fleet#337. runs-on: ${{ fromJSON(vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (github.event.repository.private == true && vars.CI_LINUX_RUNNER || inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"')) }} @@ -5408,7 +5492,7 @@ jobs: # plan's `full` never applies to Fast, and `Required` still demands the # plan succeed. fast: - name: ${{ (inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != ''))) && 'Fast' || 'Fast (not run)' }} + name: ${{ (inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != ''))) && 'Fast' || 'Fast (not run)' }} needs: [reuse-plan] # !cancelled() runs Fast after a failed or skipped need, and never after # the run is cancelled: GitHub reports a job whose `if:` is false in a @@ -5417,9 +5501,9 @@ jobs: # caller's concurrency group while it queued for a seat (operator-portal # run 36494070028, ~14 min). The rest is FAST_ENABLED, so a disabled Fast # takes no runner at all (see NAMING IS THE GATE above). - if: "!cancelled() && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != ''))" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != ''))" env: - FAST_ENABLED: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} + FAST_ENABLED: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} # Read by the shared steps: this job never escalates and never reads # `E2E plan`. FAST_ESCALABLE: false @@ -5564,7 +5648,7 @@ jobs: # Exact-path cleanup only: persistent self-hosted workspaces must not keep # registry credentials after install (success, failure, or cancel). - name: Remove package auth materialization - if: always() && inputs.install-script == '' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' + if: always() && inputs.install-script == '' && inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' run: rm -f "${GITHUB_WORKSPACE}/${{ inputs.working-directory }}/.npmrc.auth" - name: Install dependencies (caller script) if: env.FAST_ENABLED == 'true' && (inputs.install-script != '') @@ -5659,11 +5743,11 @@ jobs: # it keeps that `needs:` and today's behaviour exactly: named `Fast`, or # `Fast lanes (escalated)` when `fast-escalated` takes the name. fast-escalable: - name: ${{ !(inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && 'Fast (not run)' || ((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true') && 'Fast lanes (escalated)' || 'Fast' }} + name: ${{ !(inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && 'Fast (not run)' || ((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true') && 'Fast lanes (escalated)' || 'Fast' }} needs: [reuse-plan, e2e-plan] - if: "!cancelled() && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')" env: - FAST_ENABLED: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') }} + FAST_ENABLED: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') }} FAST_ESCALABLE: true E2E_PLAN_RESULT: ${{ needs.e2e-plan.result }} ESCALATED: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true' }} @@ -5696,7 +5780,7 @@ jobs: # deploy back. journey-smoke: name: Journey smoke - if: inputs.journey-smoke-url != '' + if: inputs.journey-smoke-url != '' && inputs.mode != 'e2e' runs-on: ${{ fromJSON(inputs.e2e-runner != '' && inputs.e2e-runner || (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"')) }} timeout-minutes: 15 permissions: @@ -5993,12 +6077,12 @@ jobs: # evaluated this name is not `Required` then, so it can never produce a # second, skipped (passing) `ci / Required`. The condition is the one in # `build`'s name, verbatim (R5 in scripts/lint_callables.py). - name: ${{ (inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results) && 'Required (folded into Build)' || 'Required' }} + name: ${{ (inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results) && 'Required (folded into Build)' || 'Required' }} # !cancelled(), not always(): it still runs after any failed or skipped # need, and a cancelled run reports it CANCELLED -- a non-passing check -- # without queueing for a runner that would hold the caller's concurrency # group (operator-portal run 36494070028 waited ~14 min for one). - if: "!cancelled() && !(inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !inputs.run-e2e && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" + if: "!cancelled() && !(inputs.mode != 'e2e' && inputs.checks-in-build && inputs.extra-gate-scripts == '' && inputs.fast-scripts == '' && !(inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) && inputs.preview-checks == 'none' && !inputs.wrangler-dry-run && inputs.journey-smoke-url == '' && !inputs.required-reuse-pr-results)" needs: [reuse-plan, build, checks, extra-gate, e2e-plan, e2e, preview, deploy-dry-run, fast, fast-escalable, fast-escalated, journey-smoke] # Organization-wide lightweight route; unset preserves the caller fallback. runs-on: ${{ fromJSON(github.event.repository.private == false && '"ubuntu-latest"' || inputs.lightweight-runner || vars.CI_LIGHTWEIGHT_RUNNER || (vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"'))) }} @@ -6023,7 +6107,10 @@ jobs: PREVIEW_CHECKS: ${{ inputs.preview-checks }} # The preview lane exists only where a pull-request preview does. EVENT_NAME: ${{ github.event_name }} - RUN_E2E: ${{ inputs.run-e2e }} + RUN_E2E: ${{ (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' || inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} + # `ci` (default) or `e2e`: the post-merge / nightly Playwright run. + MODE: ${{ inputs.mode }} + REQUESTED_E2E: ${{ inputs.run-e2e }} # The plan job's EFFECTIVE shard count, falling back to the input # when the plan produced no output at all (it was skipped, or it # failed before writing one). The fallback keeps this aggregation @@ -6054,6 +6141,44 @@ jobs: status=1 fi } + case "${MODE:-ci}" in + ci|e2e) ;; + *) + echo "::error::mode must be 'ci' or 'e2e' (got '${MODE:-}')" + exit 1 + ;; + esac + # MODE E2E (post-merge / nightly). Only Build -> E2E plan -> E2E ran: + # every other lane must be SKIPPED, and the E2E lanes are judged + # exactly as in the ordinary gate below. `CI_E2E_IN_CI` is ignored + # here. A caller that asks for this mode without `run-e2e` gets a red + # verdict, never a green run that tested nothing. + if [ "${MODE:-ci}" = e2e ]; then + if [ "${REQUESTED_E2E:-}" != true ]; then + echo "::error::mode e2e requires run-e2e: true; nothing was tested" + status=1 + fi + if [ -n "${JOURNEY_SMOKE_URL:-}" ]; then + echo "::error::mode e2e cannot be combined with journey-smoke-url" + status=1 + fi + for lane in checks:"$CHECKS_RESULT" extra-gate:"$EXTRA_GATE_RESULT" \ + preview:"$PREVIEW_RESULT" deploy-dry-run:"$DEPLOY_DRY_RUN_RESULT"; do + require_skipped "${lane%%:*}" "${lane#*:}" + done + require_success build "$BUILD_RESULT" + require_success e2e-plan "$E2E_PLAN_RESULT" + if [ "$E2E_PLAN_SKIPPED" = "true" ]; then + require_skipped e2e "$E2E_RESULT" + else + require_success e2e "$E2E_RESULT" + if ! [ "$E2E_SHARDS" -ge 1 ] 2>/dev/null; then + echo "::error::effective e2e shard count is not a positive integer (got '$E2E_SHARDS')" + status=1 + fi + fi + exit "$status" + fi # P3-C1: when `Reuse plan` proved this exact tree (or this call is a # journey smoke), every CI lane must be SKIPPED -- a lane that ran # anyway, or failed, still fails the gate. Unset (the default) falls @@ -6131,7 +6256,7 @@ jobs: exit "$status" - name: Require fast-path and journey-smoke lanes env: - REUSE_APPLIES: ${{ inputs.required-reuse-pr-results && inputs.journey-smoke-url == '' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + REUSE_APPLIES: ${{ inputs.mode != 'e2e' && inputs.required-reuse-pr-results && inputs.journey-smoke-url == '' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} REUSE_PLAN_RESULT: ${{ needs.reuse-plan.result }} FAST_RESULT: ${{ needs.fast.result }} FAST_ESCALABLE_RESULT: ${{ needs.fast-escalable.result }} @@ -6140,9 +6265,9 @@ jobs: # applies and be skipped when it does not (a disabled Fast job takes # no runner, so `skipped` is its only correct result). `fast` and # `fast-escalable` split one condition into two disjoint domains. - FAST_APPLIES: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} - FAST_ESCALABLE_APPLIES: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') }} - FAST_ESCALATION_APPLIES: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true' }} + FAST_APPLIES: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && !((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) }} + FAST_ESCALABLE_APPLIES: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') }} + FAST_ESCALATION_APPLIES: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true' }} JOURNEY_SMOKE_URL: ${{ inputs.journey-smoke-url }} JOURNEY_SMOKE_RESULT: ${{ needs.journey-smoke.result }} run: | @@ -6205,7 +6330,7 @@ jobs: # `Required` before any proof-publishing step (those need `success()`). # `fast-escalated` runs the same anchored scripts when it holds `Fast`. - name: Check out the caller for Caller lint - if: "!cancelled()" + if: "!cancelled() && inputs.mode != 'e2e'" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -6221,7 +6346,7 @@ jobs: sparse-checkout-cone-mode: false - name: Verify exact validation candidate - if: "!cancelled() && inputs.expected-candidate-sha != ''" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.expected-candidate-sha != ''" working-directory: ${{ github.workspace }} shell: bash env: &caller_lint_candidate_env @@ -6244,20 +6369,20 @@ jobs: # This step imports PyYAML (the anchored scripts are defined in `build`, # which runs them when Required is folded into it). - name: Ensure PyYAML is available - if: "!cancelled()" + if: "!cancelled() && inputs.mode != 'e2e'" run: *caller_lint_pyyaml - name: Install actionlint - if: "!cancelled()" + if: "!cancelled() && inputs.mode != 'e2e'" env: *caller_lint_actionlint_env run: *caller_lint_install_actionlint - name: actionlint (caller's own workflows) - if: "!cancelled()" + if: "!cancelled() && inputs.mode != 'e2e'" run: *caller_lint_actionlint - name: Caller workflow hygiene audit (concurrency, timeouts, SHA pins, permissions) - if: "!cancelled()" + if: "!cancelled() && inputs.mode != 'e2e'" run: *caller_lint_audit # `required-reuse-pr-results` proof: minted only on a same-repository # pull request whose Required passed with every enabled lane RUN, and @@ -6266,6 +6391,10 @@ jobs: # script `Reuse plan` runs on the push. - name: Compute Required proof key id: required-key + # RAW `run-e2e`, not the effective switch: a pull request that skipped + # E2E because `CI_E2E_IN_CI` was false must mint no proof, or the + # variable's later removal (rollback) or a repo override would let the + # push reuse it and turn main green without E2E ever having run. if: >- success() && inputs.required-reuse-pr-results && inputs.journey-smoke-url == '' && github.event_name == 'pull_request' @@ -6331,15 +6460,15 @@ jobs: # A job waiting on its needs has no check run yet, so the `Fast` context stays pending, which blocks the # merge, until the full gate has finished. fast-escalated: - name: ${{ (inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true') && ((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true') && 'Fast' || 'Fast (escalation not needed)' }} + name: ${{ (inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true') && ((github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true') && 'Fast' || 'Fast (escalation not needed)' }} needs: [reuse-plan, fast-escalable, build, checks, extra-gate, e2e-plan, e2e, preview, deploy-dry-run] # Starts only when this run escalates (the same condition as FAST_ENABLED); # !cancelled() keeps a failed or skipped lane from skipping it, because it # is then the check named `Fast` and must report that failure. A cancelled # run reports it CANCELLED without a runner (see `fast`). - if: "!cancelled() && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true'" + if: "!cancelled() && inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true'" env: - FAST_ENABLED: ${{ inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true' }} + FAST_ENABLED: ${{ inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && (inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '') && needs.e2e-plan.outputs.full == 'true' }} # Organization-wide lightweight route; unset preserves the caller fallback. runs-on: ${{ fromJSON(github.event.repository.private == false && '"ubuntu-latest"' || inputs.lightweight-runner || vars.CI_LIGHTWEIGHT_RUNNER || (vars.BLACKSMITH_RUNNERS_ENABLED == 'true' && (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"') != '"ubuntu-latest"' && format('"{0}"', vars.BLACKSMITH_LINUX_LABEL || 'blacksmith-2vcpu-ubuntu-2404') || (inputs.runner || github.event.repository.private == true && '{"group":"linux-ci","labels":["self-hosted","Linux","X64","proxmox","linux-ci"]}' || '"ubuntu-latest"'))) }} timeout-minutes: 20 diff --git a/.github/workflows/red-main-listener.yml b/.github/workflows/red-main-listener.yml index 471b5d3..38955f6 100644 --- a/.github/workflows/red-main-listener.yml +++ b/.github/workflows/red-main-listener.yml @@ -34,16 +34,22 @@ name: Reusable Red-Main Listener # types: [completed] # jobs: # red-main: -# # `event == 'push' || event == 'workflow_dispatch'` matters: CI also +# # `event == 'push' || event == 'workflow_dispatch' || event == +# # 'schedule'` matters: CI also # # runs on `pull_request`, whose `head_branch` is the PR's OWN # # branch, not `main` -- a branch that happens to be named `main` # # would otherwise pass the filter below. `workflow_dispatch` is # # included because a manually-dispatched CI run against the default # # branch is just as real a "main is red" signal as a push; drop it # # from this list if a given caller's `workflow_dispatch` runs never -# # target the default branch as CI. +# # target the default branch as CI. `schedule` is included because a +# # scheduled run always runs the default branch, so a red nightly +# # (the `E2E` workflow's safety net, see README "E2E off the pull +# # request") must open or refresh the issue too. List the post-merge +# # workflow beside CI: `workflows: ["CI", "E2E"]`; each workflow gets +# # its own "main is red: " issue. # if: | -# (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') && +# (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule') && # github.event.workflow_run.head_branch == github.event.repository.default_branch # uses: narduk-enterprises/workflows/.github/workflows/red-main-listener.yml@ # v1 # permissions: diff --git a/.github/workflows/red-main-self.yml b/.github/workflows/red-main-self.yml index 2b39a2e..b32db98 100644 --- a/.github/workflows/red-main-self.yml +++ b/.github/workflows/red-main-self.yml @@ -25,8 +25,13 @@ jobs: # is exactly as real a "main is red" signal as a red push -- the # `head_branch == default_branch` check still excludes a manual dispatch # against any other branch or ref. + # `schedule` is included (fast-CI program, 2026-09-29): a nightly run -- + # the post-merge `E2E` workflow's safety net -- runs the default branch, so + # a red nightly is a "main is red" signal too. This repository's own CI has + # no schedule trigger; the clause keeps the reference adopter identical to + # the caller shape README documents for adopters that do. if: | - (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule') && github.event.workflow_run.head_branch == github.event.repository.default_branch uses: ./.github/workflows/red-main-listener.yml permissions: diff --git a/README.md b/README.md index 75cac88..0a7dd74 100644 --- a/README.md +++ b/README.md @@ -1038,6 +1038,20 @@ That last row is the same fail-closed rule `require-scripts` exists for: "the audit did not run" must never be indistinguishable from "the audit found nothing". +**A pull request only warns (Logan, 2026-09-29, "Main + nightly").** An +advisory is published on its own schedule, not the pull request's, so a fixable +high/critical finding used to block an unrelated PR the day it appeared. On +`pull_request` and `pull_request_target` every `::error::` row above is +re-labelled `::warning::` (the same advisory list, still in the job summary) +and the step passes; the closing line says the finding will fail the default +branch. On a `push` to the default branch, a `schedule` run and a +`workflow_dispatch` the step fails exactly as before, so a red audit is caught +by the merge that follows and by the nightly run (see +[E2E off the pull request](#e2e-off-the-pull-request-mode-e2e-and-ci_e2e_in_ci), +whose nightly caller is not the place for it: the audit runs in the `ci` +workflow, so add `schedule:` to that caller too if the audit should be checked +nightly). + Both report shapes are parsed, and the `package-manager` input selects the *command*, never the parser — npm changed this format once already, and a parser keyed on the input would silently read zero advisories the next time it @@ -1089,6 +1103,105 @@ it is a security bar rather than an optional lane. It is still a new gate that can turn an existing adopter red, so the `v1` tag must not move onto it until the adopters have been checked — see [Versioning policy](#versioning-policy). +#### E2E off the pull request (`mode: e2e` and `CI_E2E_IN_CI`) + +Playwright is the slowest lane and it almost never catches a bug that the +merge-time run would not (Logan, 2026-09-29: "skip on PRs; run after each merge, +newest wins, plus nightly; one org switch"). Two pieces, both additive: with +neither in play a caller behaves exactly as before. + +**1. The org switch: `vars.CI_E2E_IN_CI`.** When the organization (or one repo, +which overrides the org value) sets `CI_E2E_IN_CI` to `false` (GitHub compares strings case-insensitively, so `False` and `FALSE` switch it off too), `E2E plan`, `E2E` +and `E2E quarantine` are skipped in an ordinary CI run on **every** event, and +`Required` reads that skip as success (a lane that runs anyway still fails +it). Build then skips packing and uploading the prebuilt application for the E2E +jobs, and a `checks-in-build` caller with no other lane folds `Required` into +`Build` (one job, no extra queue hop). A pull request that skipped E2E this way mints no `Required` proof (the proof key does not +include the variable, so a reusable proof would let a later push, after the variable is removed +or overridden, go green without E2E ever having run). Unset, empty or any other value changes +nothing. The switch deliberately does **not** override two promises to run +browsers: `e2e-full-paths` (a caller that sets it wants protected-path escalation, +which needs `E2E plan` to decide, so acre-oracle keeps browsers on auth/payment +changes) and `expected-candidate-sha` (explicit release validation cannot skip +configured browser coverage). + +**2. The post-merge and nightly run: `mode: e2e`.** With `mode: e2e` the +callable runs only Build -> E2E plan -> E2E (and the quarantine lane) and a +`Required` verdict. Every other lane, the Caller lint and the dependency audit +are skipped, the `CI_E2E_IN_CI` variable is ignored, and the run never +path-skips (a newer push cancels this run, so no diff can prove a skip). +`Required` is red if the mode is not `ci` or `e2e`, if `run-e2e` is not `true`, +if `journey-smoke-url` is set, or if any E2E lane fails, is cancelled, or was +skipped without the plan's say-so. The mode has its own workflow because an +app's `promote.yml` fires on `workflow_run` completion of the **whole** CI +workflow: E2E left in `ci.yml` would delay every promotion. + +Name the caller workflow **`E2E`** (the reaper and the red-main listener look for +that name) and stamp it exactly, with the same `with:` values as the app's `ci` +job for the E2E inputs and the same runner inputs: + +```yaml +name: E2E + +# Post-merge and nightly Playwright. The newest merge wins: a push cancels the +# run still in flight, and every run tests the whole suite. +on: + push: + branches: [main] + schedule: + - cron: '17 8 * * *' # 08:17 UTC = 3:17 AM CT + workflow_dispatch: + +concurrency: + group: e2e-${{ github.repository }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ci: + permissions: + contents: read + packages: read + actions: read + pull-requests: write + uses: narduk-enterprises/workflows/.github/workflows/nuxt-cloudflare.yml@ # v2 + secrets: inherit + with: + mode: e2e + run-e2e: true + # ...the app's ci.yml E2E and runner inputs, verbatim (e2e-runner, + # e2e-shards, e2e-args, e2e-build-artifact-path, e2e-quarantine-args, + # install-script, node-version, working-directory, ...) +``` + +Format the stamped file with the app's own Prettier config: apps that run `format:check` over `.github/workflows` (cloudflarestat-us, single quotes) fail a double-quoted cron. + +The calling job id stays `ci` so the composed context reads `E2E / ci / Required`. +`concurrency` sits in the caller, never in the callable (R6). Pass the same +`permissions:` block the app's `ci` job grants: a job in the callable may only +use what its caller granted (R12). + +A red post-merge or nightly run reaches the same `red-main` issue flow as CI: +add the workflow to the app's red-main listener, and accept `schedule`. + +```yaml +on: + workflow_run: + workflows: ["CI", "E2E"] + types: [completed] +jobs: + red-main: + if: | + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule') && + github.event.workflow_run.head_branch == github.event.repository.default_branch +``` + +Each workflow keeps its own "main is red: ``" issue, opened by the first +red run and closed by the next green one. A run cancelled by a newer merge is +not a verdict and does nothing. + #### Quality level (`quality-level`, `quality-opt-out`) The estate has web quality tools that gated nothing on most apps, because each @@ -2070,6 +2183,13 @@ P3-C2 / O-D8 rather than kept as a dead compatibility surface. callers who never asked for one, which is a breaking change dressed as an additive input. Getting the *default* wrong is how an "additive" change breaks people. +- `nuxt-cloudflare.yml`'s `mode` input and `vars.CI_E2E_IN_CI` switch + ([E2E off the pull request](#e2e-off-the-pull-request-mode-e2e-and-ci_e2e_in_ci)) + are within-major on the same rule: one optional input defaulting to `ci`, no + new job, no new permission, and an unset variable reproduces every + adopter's behaviour exactly. The `Dependency audit` step's pull-request + downgrade (warn on `pull_request`, block on push, schedule and dispatch) only + *loosens* a gate on one event, so no adopter turns red because of it. - `nuxt-cloudflare.yml`'s `foundation-check` / `foundation-check-tool-version` (company-hq docs/WEB-FOUNDATION-CHECK.md, D-WEBFOUND-2 Q5/Q9 (a), D-WEBFOUND-3) are within-major on the same rule — two optional inputs, no diff --git a/scripts/test_dependency_audit.py b/scripts/test_dependency_audit.py index 57c842e..4cbe866 100755 --- a/scripts/test_dependency_audit.py +++ b/scripts/test_dependency_audit.py @@ -20,6 +20,10 @@ * a stale `audit-ignore` entry warns so it gets removed * a missing, empty, non-JSON or unrecognised report is a HARD FAILURE -- "the audit did not run" must never look like "the audit found nothing" + * on `pull_request` / `pull_request_target` EVERY finding above becomes a + `::warning::` carrying the same advisory list and the step passes; the + default branch (`push`), `schedule` and `workflow_dispatch` keep failing + (fast-CI program, Logan 2026-09-29 "Main + nightly") Run: python3 scripts/test_dependency_audit.py """ @@ -139,7 +143,8 @@ def npm7_vuln(name: str, severity: str, fix_available, ghsa: str, source: int = # --- harness -------------------------------------------------------------- -def run_case(script: str, *, pm: str, stdout: str, exit_code: int, audit_ignore: str = "") -> tuple[int, str, str]: +def run_case(script: str, *, pm: str, stdout: str, exit_code: int, audit_ignore: str = "", + event_name: str | None = None) -> tuple[int, str, str]: """Execute the shipped gate text with a fake package manager on PATH.""" tmp = tempfile.mkdtemp(prefix="dependency-audit-") try: @@ -164,6 +169,8 @@ def run_case(script: str, *, pm: str, stdout: str, exit_code: int, audit_ignore: "GITHUB_STEP_SUMMARY": str(summary), "HOME": tmp, } + if event_name is not None: + env["EVENT_NAME"] = event_name proc = subprocess.run( ["bash", "-c", script], cwd=tmp, env=env, capture_output=True, text=True ) @@ -320,6 +327,97 @@ def run_case(script: str, *, pm: str, stdout: str, exit_code: int, audit_ignore: ] +# The same findings, on the event that decides whether they block. A pull +# request only warns; every other event (unset included) fails as before. +PR_EVENTS = ("pull_request", "pull_request_target") +BLOCKING_EVENTS = ("push", "schedule", "workflow_dispatch") +for _event in PR_EVENTS: + CASES += [ + ( + f"{_event}: fixable high warns and passes, listing the advisory", + {"pm": "pnpm", "stdout": FIXABLE_HIGH, "exit_code": 1, "event_name": _event}, + 0, + ["::warning::high in left-pad", "FIX AVAILABLE (>=1.2.3)", "GHSA-AAAA-BBBB-CCCC", + "do not fail this pull request"], + ["::error::"], + ), + ( + f"{_event}: mixed tree warns on both and passes", + {"pm": "pnpm", "stdout": MIXED, "exit_code": 1, "event_name": _event}, + 0, + ["::warning::high in left-pad", "::warning::critical in tar", "1 fixable, 1 unfixable"], + ["::error::"], + ), + ( + f"{_event}: an audit-ignore entry without a reason warns and passes", + {"pm": "pnpm", "stdout": FIXABLE_HIGH, "exit_code": 1, "audit_ignore": "GHSA-aaaa-bbbb-cccc", + "event_name": _event}, + 0, + ["::warning::audit-ignore entry", "has no reason"], + ["::error::"], + ), + ( + f"{_event}: an empty report (audit did not run) warns and passes", + {"pm": "pnpm", "stdout": "", "exit_code": 0, "event_name": _event}, + 0, + ["::warning::dependency audit produced no report", "did not run on this pull request"], + ["::error::"], + ), + ( + f"{_event}: a non-JSON report warns and passes", + {"pm": "pnpm", "stdout": "ERR_PNPM_AUDIT_ENDPOINT_UNAVAILABLE\n", "exit_code": 1, "event_name": _event}, + 0, + ["::warning::dependency audit report is not JSON"], + ["::error::"], + ), + ( + f"{_event}: an unrecognised report shape warns and passes", + {"pm": "pnpm", "stdout": UNKNOWN_SHAPE, "exit_code": 0, "event_name": _event}, + 0, + ["::warning::dependency audit report has neither"], + ["::error::"], + ), + ( + f"{_event}: npm 7+ fixable high warns and passes", + {"pm": "npm", "stdout": NPM7_FIXABLE_HIGH, "exit_code": 1, "event_name": _event}, + 0, + ["::warning::high in axios", "FIX AVAILABLE (axios@1.7.4)"], + ["::error::"], + ), + ( + f"{_event}: a clean report still says nothing alarming", + {"pm": "pnpm", "stdout": CLEAN_PNPM, "exit_code": 0, "event_name": _event}, + 0, + ["0 fixable, 0 unfixable, 0 suppressed"], + ["::error::", "::warning::"], + ), + ] +for _event in BLOCKING_EVENTS: + CASES += [ + ( + f"{_event}: fixable high still fails", + {"pm": "pnpm", "stdout": FIXABLE_HIGH, "exit_code": 1, "event_name": _event}, + 1, + ["::error::high in left-pad", "FIX AVAILABLE (>=1.2.3)"], + ["do not fail this pull request"], + ), + ( + f"{_event}: an empty report (audit did not run) still fails", + {"pm": "pnpm", "stdout": "", "exit_code": 0, "event_name": _event}, + 1, + ["::error::dependency audit produced no report"], + [], + ), + ( + f"{_event}: a non-JSON report still fails", + {"pm": "pnpm", "stdout": "ERR_PNPM_AUDIT_ENDPOINT_UNAVAILABLE\n", "exit_code": 1, "event_name": _event}, + 1, + ["::error::dependency audit report is not JSON"], + [], + ), + ] + + def check_shipped_flags(script: str) -> list[str]: """The gate must ask for JSON and must not let the tool's own exit status decide.""" problems = [] diff --git a/scripts/test_e2e_mode.py b/scripts/test_e2e_mode.py new file mode 100644 index 0000000..e4f290a --- /dev/null +++ b/scripts/test_e2e_mode.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Contract tests for taking E2E off the pull request (fast-CI program, 2026-09-29). + +Two additions to nuxt-cloudflare.yml, both proven by EVALUATING the shipped job +conditions and names over the real job graph (test_fast_path.py's simulator) and +by executing the shipped gate text: + + 1. The org switch. `vars.CI_E2E_IN_CI == 'false'` turns `E2E plan`, `E2E` and + `E2E quarantine` off in an ordinary CI run on ANY event, and `Required` + reads that skip as success. Unset, empty or any other value changes + nothing. Two things deliberately outlive the switch, because both are + promises to run browsers that a variable must not be able to break: + `e2e-full-paths` (a protected-path pull request escalates to the full + suite, which needs `E2E plan` to decide) and `expected-candidate-sha` + (explicit release validation "cannot skip configured browser coverage"). + + 2. `mode: e2e`, the post-merge / nightly run. Only Build -> E2E plan -> E2E + (and the quarantine lane) run, `Required` is the verdict, every other lane + is skipped, and the variable is ignored. Asking for the mode without + `run-e2e`, with journey-smoke-url, or with an unknown mode is a red + verdict, never a green run that tested nothing. The mode never path-skips + (a newer push cancels the run, so no diff can prove a skip), and its proof + key equals the pull-request run's so a reused proof is still honoured. + +Run: python3 scripts/test_e2e_mode.py +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import test_fast_path as tfp # noqa: E402 +import test_e2e_skip_paths as skip # noqa: E402 + +JOBS = tfp.JOBS +INPUTS = tfp.INPUTS +EVENTS = tfp.EVENTS +E2E_LANES = ("e2e-plan", "e2e", "e2e-quarantine") +NON_E2E_LANES = ("reuse-plan", "checks", "extra-gate", "preview", "deploy-dry-run", "fast", + "fast-escalable", "fast-escalated", "journey-smoke") + +_real_context = tfp.context + + +def context_with_vars(event, ref="refs/heads/main", inputs=None, needs=None, status="success", + variables=None): + ctx = _real_context(event, ref, inputs, needs, status) + ctx["vars"] = dict(variables or {}) + return ctx + + +CURRENT_VARS: dict = {} + + +def simulate(inputs: dict, event: str, ref: str, variables: dict | None = None, + outputs: dict | None = None, results: dict | None = None) -> dict: + """The real graph with a repo/org variable set (tfp.context has no `vars`).""" + global CURRENT_VARS + CURRENT_VARS = variables or {} + tfp.context = lambda e, r="refs/heads/main", i=None, n=None, s="success": context_with_vars( + e, r, i, n, s, CURRENT_VARS) + try: + return tfp.simulate(JOBS, inputs, event, ref, outputs, results) + finally: + tfp.context = _real_context + + +def gates(state: dict) -> list: + global CURRENT_VARS + tfp.context = _real_context + return tfp.run_gates("required", state) + + +def plan(skipped: str = "false", shards: str = "1") -> dict: + return tfp.plan_outputs(skipped, shards) + + +PR_EVENTS = [(e, r) for e, r in EVENTS if e in ("pull_request", "pull_request_target")] +OTHER_EVENTS = [(e, r) for e, r in EVENTS if e not in ("pull_request", "pull_request_target")] +BASE = {"run-e2e": True, "e2e-quarantine-args": "--grep=@flaky"} + + +def ran(state: dict) -> set[str]: + return {job for job, value in state.items() if value["result"] != "skipped"} + + +def assert_required_passes(state: dict, label: str) -> None: + for name, code, out in gates(state): + assert code == 0, (label, name, out) + + +def test_mode_input() -> None: + spec = INPUTS["mode"] + assert spec["default"] == "ci" and spec["type"] == "string" and spec["required"] is False, spec + print("PASS mode input: string, default 'ci', optional") + + +def test_switch_defaults_change_nothing() -> None: + """Unset, empty or any value but 'false' is today's behaviour, on every event.""" + for variables in ({}, {"CI_E2E_IN_CI": ""}, {"CI_E2E_IN_CI": "true"}, {"CI_E2E_IN_CI": "0"}, + {"CI_E2E_IN_CI": "no"}): + for event, ref in EVENTS: + state = simulate(BASE, event, ref, variables, plan()) + assert {"build", "e2e-plan", "e2e", "e2e-quarantine", "required"} <= ran(state), (variables, event, ran(state)) + assert_required_passes(state, f"{variables} {event}") + print("PASS CI_E2E_IN_CI unset / empty / true / other values: E2E runs on every event, as before") + + +def test_switch_off_skips_the_e2e_lanes() -> None: + off = {"CI_E2E_IN_CI": "false"} + for event, ref in EVENTS: + state = simulate(BASE, event, ref, off, plan()) + for lane in E2E_LANES: + assert state[lane]["result"] == "skipped", (event, lane) + assert state["build"]["result"] == "success" and state["required"]["result"] == "success", event + assert_required_passes(state, f"off {event}") + # The skip is not a waiver: an E2E lane that ran anyway fails Required, + # and so does a failed Build. + for lane, result in (("e2e", "success"), ("e2e-plan", "success")): + bad = tfp.with_need(state, "required", lane, result) + assert gates(bad)[0][1] == 1, (event, lane) + assert gates(tfp.with_need(state, "required", "build", "failure"))[0][1] == 1, event + # Case-insensitive like every GitHub string comparison. + upper = simulate(BASE, event, ref, {"CI_E2E_IN_CI": "False"}, plan()) + assert upper["e2e"]["result"] == "skipped", event + print("PASS CI_E2E_IN_CI=false: E2E plan, E2E and quarantine skip on every event; Required passes and still catches a stray lane") + + +def test_switch_leaves_promises_to_run_browsers() -> None: + off = {"CI_E2E_IN_CI": "false"} + promises = { + "e2e-full-paths": {"e2e-full-paths": "**/*auth*/**"}, + "expected-candidate-sha": {"expected-candidate-sha": "a" * 40}, + } + for label, extra in promises.items(): + for event, ref in EVENTS: + state = simulate({**BASE, **extra}, event, ref, off, plan()) + assert {"e2e-plan", "e2e"} <= ran(state), (label, event) + # And the required gate reads it the same way: the lanes are enabled. + state = simulate({**BASE, "e2e-full-paths": "**/*auth*/**"}, "pull_request", "refs/pull/1/merge", off, plan()) + assert state["required"]["ctx"]["needs"]["e2e"]["result"] == "success" + assert_required_passes(state, "full-paths") + print("PASS the switch never overrides e2e-full-paths (protected-path escalation) or expected-candidate-sha") + + +def test_switch_off_without_run_e2e_is_unchanged() -> None: + for variables in ({}, {"CI_E2E_IN_CI": "false"}): + state = simulate({}, "pull_request", "refs/pull/1/merge", variables, plan()) + assert not (set(E2E_LANES) & ran(state)), variables + assert_required_passes(state, f"no e2e {variables}") + print("PASS a caller without run-e2e is unaffected by the variable") + + +def test_fold_reads_the_effective_switch() -> None: + """With checks-in-build and E2E off, Build is the only lane, so it becomes + `Required` and the aggregator steps aside (the folded gate). Any E2E lane + that is really on, or mode e2e, keeps the aggregator.""" + folded = {**tfp.FOLD_INPUTS, "run-e2e": True} + cases = [ + ({"CI_E2E_IN_CI": "false"}, folded, "Required", "skipped"), + ({}, folded, "Build", "success"), + ({"CI_E2E_IN_CI": "true"}, folded, "Build", "success"), + ({"CI_E2E_IN_CI": "false"}, {**folded, "e2e-full-paths": "**/auth/**"}, "Build", "success"), + ({"CI_E2E_IN_CI": "false"}, {**folded, "mode": "e2e"}, "Build", "success"), + ({"CI_E2E_IN_CI": "false"}, {**tfp.FOLD_INPUTS, "mode": "e2e"}, "Build", "success"), + ] + for variables, inputs, build_name, required_result in cases: + state = simulate(inputs, "pull_request", "refs/pull/1/merge", variables, plan()) + ctx = {**state["build"]["ctx"], "vars": variables} + tfp.context = _real_context + name = tfp.render(JOBS["build"]["name"], ctx) + assert name == build_name, (variables, inputs, name) + assert state["required"]["result"] == required_result, (variables, inputs, state["required"]["result"]) + # Exactly one job reports the check named Required. + required_name = tfp.render(JOBS["required"]["name"], {**state["required"]["ctx"], "vars": variables}) + reporters = [n for n, jn in (("build", name), ("required", required_name)) + if jn == "Required" and state[n]["result"] != "skipped"] + if build_name == "Required": + assert reporters == ["build"], (variables, inputs, reporters) + else: + assert reporters == ["required"], (variables, inputs, reporters) + print("PASS the folded Required gate follows the effective switch and never folds in mode e2e") + + +def test_mode_e2e_runs_only_the_e2e_chain() -> None: + loud = {**BASE, "mode": "e2e", "extra-gate-scripts": "check:extra", "fast-scripts": "fast", + "preview-checks": "og", "wrangler-dry-run": True, "required-reuse-pr-results": True, + "checks-in-build": False} + for variables in ({}, {"CI_E2E_IN_CI": "false"}, {"CI_E2E_IN_CI": "true"}): + for event, ref in EVENTS: + state = simulate(loud, event, ref, variables, plan()) + assert ran(state) == {"build", "e2e-plan", "e2e", "e2e-quarantine", "required"}, (variables, event, ran(state)) + assert_required_passes(state, f"mode e2e {variables} {event}") + # Escalation inputs cannot wake a fast lane in this mode either. + escalating = {**loud, "e2e-full-paths": "**/auth/**"} + for event, ref in EVENTS: + assert ran(simulate(escalating, event, ref, {}, plan())) == \ + {"build", "e2e-plan", "e2e", "e2e-quarantine", "required"}, event + print("PASS mode e2e: only Build, E2E plan, E2E (+quarantine) and Required run, on every event, whatever the variable says") + + +def test_mode_e2e_verdict() -> None: + inputs = {**BASE, "mode": "e2e"} + state = simulate(inputs, "push", "refs/heads/main", {}, plan()) + for lane, result in (("build", "failure"), ("e2e", "failure"), ("e2e-plan", "failure"), + ("e2e", "skipped"), ("e2e-plan", "skipped"), ("e2e", "cancelled")): + assert gates(tfp.with_need(state, "required", lane, result))[0][1] == 1, (lane, result) + # A lane that must be skipped but ran fails the verdict. + for lane in ("checks", "extra-gate", "preview", "deploy-dry-run"): + assert gates(tfp.with_need(state, "required", lane, "success"))[0][1] == 1, lane + for lane in ("reuse-plan", "fast", "fast-escalable", "fast-escalated", "journey-smoke"): + results = [r for r in (gates(tfp.with_need(state, "required", lane, "success"))) if r[1] == 1] + assert results, lane + # A reused PR proof skips E2E: still a pass, and E2E must then be skipped. + proven = simulate(inputs, "push", "refs/heads/main", {}, plan("true")) + assert proven["e2e"]["result"] == "skipped" + assert_required_passes(proven, "proof reuse") + assert gates(tfp.with_need(proven, "required", "e2e", "success"))[0][1] == 1 + # The verdict cannot pass a run that tested nothing. + for bad, expect in (({"run-e2e": False, "mode": "e2e"}, "requires run-e2e"), + ({**inputs, "journey-smoke-url": "https://example.test"}, "journey-smoke-url"), + ({**inputs, "mode": "nightly"}, "mode must be")): + broken = simulate(bad, "push", "refs/heads/main", {}, plan()) + out = " ".join(o for _, code, o in gates(broken) if code) + assert any(code == 1 for _, code, _ in gates(broken)), bad + assert expect in out, (bad, out) + zero = simulate(inputs, "push", "refs/heads/main", {}, plan(shards="0")) + assert gates(zero)[0][1] == 1 + print("PASS mode e2e verdict: red for any failed/skipped/cancelled E2E lane, a stray lane, no run-e2e, a bad mode or zero shards") + + +def test_mode_e2e_skips_caller_lint_and_lint_steps() -> None: + required = {s.get("name"): s for s in JOBS["required"]["steps"]} + lint = ["Check out the caller for Caller lint", "Ensure PyYAML is available", "Install actionlint", + "actionlint (caller's own workflows)", + "Caller workflow hygiene audit (concurrency, timeouts, SHA pins, permissions)", + "Verify exact validation candidate"] + for mode, want in (("ci", True), ("e2e", False)): + ctx = tfp.context("push", inputs={"mode": mode, "expected-candidate-sha": "a" * 40}) + for name in lint: + assert tfp.ev(tfp.job_condition(required[name]), ctx) is want, (mode, name) + build = {s.get("name"): s for s in JOBS["build"]["steps"]} + for name in ("Dependency audit", "Typecheck Worker", "Typecheck Nuxt", "Unit tests", "Extra scripts", + "Start concurrent scripts"): + for mode, want in (("ci", True), ("e2e", False)): + ctx = tfp.context("push", inputs={"mode": mode, "dependency-audit": True, "checks-in-build": True, + "run-tests": True, "extra-scripts": "x", "concurrent-scripts": "y"}) + assert tfp.ev(tfp.job_condition(build[name]), ctx) is want, (mode, name) + print("PASS mode e2e: Build runs no audit, typecheck, unit test or extra script; Required runs no Caller lint") + + +def test_prebuilt_handoff_follows_the_switch() -> None: + """Build packs and uploads the app for the E2E jobs only when they will run.""" + step = next(s for s in JOBS["build"]["steps"] if s.get("id") == "e2e-build-path") + for variables, inputs, want in (({"CI_E2E_IN_CI": "false"}, BASE, "false"), ({}, BASE, "true"), + ({"CI_E2E_IN_CI": "false"}, {**BASE, "mode": "e2e"}, "true")): + ctx = context_with_vars("pull_request", "refs/pull/1/merge", inputs, variables=variables) + assert tfp.gh_str(tfp.render(step["env"]["RUN_E2E"], ctx)) == want, (variables, inputs) + print("PASS the prebuilt E2E application is packed and published only when E2E will run") + + +def run_skip(mode: str, event: str, files: list[str], *, skip_patterns="**/*.md", full_patterns="") -> tuple[str, str]: + """Execute the shipped `Decide whether E2E can be skipped` text with MODE.""" + base, head = "a" * 40, "b" * 40 + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bin_dir = root / "bin" + bin_dir.mkdir() + skip.make_stub_gh(bin_dir, files) + output = root / "out" + output.write_text("") + env = dict(os.environ, PATH=f"{bin_dir}:{os.environ['PATH']}", GH_TOKEN="x", REPO="o/r", + EVENT_NAME=event, REUSED="false", BASE_SHA=base, HEAD_SHA=head, + SKIP_PATTERNS=skip_patterns, FULL_PATTERNS=full_patterns, + GITHUB_OUTPUT=str(output), GITHUB_STEP_SUMMARY=str(root / "summary")) + if mode: + env["MODE"] = mode + (root / "workspace").mkdir() + result = subprocess.run(["bash", "-c", skip.skip_step_script()], capture_output=True, text=True, + env=env, cwd=root / "workspace") + assert result.returncode == 0, result.stdout + result.stderr + values = dict(line.split("=", 1) for line in output.read_text().split()) + return values["skipped"], values["full"] + + +def test_mode_e2e_never_path_skips() -> None: + docs_only = ["README.md", "docs/a.md"] + assert run_skip("", "push", docs_only)[0] == "true", "control: a docs-only push path-skips in ci mode" + assert run_skip("ci", "push", docs_only)[0] == "true" + for event in ("push", "schedule", "workflow_dispatch", "pull_request"): + skipped, full = run_skip("e2e", event, docs_only) + assert (skipped, full) == ("false", "true"), (event, skipped, full) + print("PASS mode e2e never path-skips: a docs-only push after a cancelled code push still runs the full suite") + + +def test_proof_key_ignores_mode() -> None: + script = next(s for s in JOBS["e2e-plan"]["steps"] if s.get("id") == "proof")["with"]["script"] + legacy = {"e2e-args": "--project=web", "e2e-shards": 3} + ci = tfp.run_script(script, {}, {**legacy, "mode": "ci"}, "E2E_INPUTS")["outputs"]["key"] + post_merge = tfp.run_script(script, {}, {**legacy, "mode": "e2e"}, "E2E_INPUTS")["outputs"]["key"] + absent = tfp.run_script(script, {}, legacy, "E2E_INPUTS")["outputs"]["key"] + assert ci == post_merge == absent, "the mode input changed the E2E proof key" + required = JOBS["reuse-plan"]["steps"] + lookup = next(s for s in required if s.get("id") == "proof")["with"]["script"] + with_ci = tfp.run_script(lookup, {}, {**tfp.defaults(), "required-reuse-pr-results": True}, "PROOF_INPUTS") + without = {k: v for k, v in {**tfp.defaults(), "required-reuse-pr-results": True}.items() if k != "mode"} + assert with_ci["outputs"]["key"] == tfp.run_script(lookup, {}, without, "PROOF_INPUTS")["outputs"]["key"], \ + "the default mode changed the Required proof key" + print("PASS the mode input never changes a proof key (E2E: always dropped; Required: dropped at its default)") + + +def test_switch_off_pull_request_mints_no_required_proof() -> None: + """A PR that skipped E2E because the variable was false must not mint a Required proof. + + The proof key does not include the variable. If it minted, unsetting the + variable (the rollback) or a repo override would let the push's Reuse plan + reuse the proof and turn main green without E2E ever having run. + """ + key_if = tfp.step("required", "Compute Required proof key")["if"] + inputs = {"run-e2e": True, "required-reuse-pr-results": True, "e2e-args": "--project=web", "e2e-shards": 3} + + def mint(variables: dict, e2e: dict, plan_outputs: dict) -> bool: + ctx = context_with_vars("pull_request", "refs/pull/1/merge", inputs, { + "build": {"result": "success", "outputs": {}}, "checks": {"result": "success", "outputs": {}}, + "e2e": {"result": e2e["result"], "outputs": {}}, + "e2e-plan": {"result": "success" if plan_outputs else "skipped", "outputs": plan_outputs}, + }, variables=variables) + ctx["github"]["repository"] = "o/r" + ctx["github"]["event"]["pull_request"] = {"head": {"repo": {"full_name": "o/r"}}} + return tfp.ev(key_if, ctx) + + ran_plan = {"e2e-args": "--project=web", "shard-total": "3"} + for variables in ({"CI_E2E_IN_CI": "false"}, {"CI_E2E_IN_CI": "False"}): + assert mint(variables, {"result": "skipped"}, {}) is False, variables + # Unchanged when E2E really ran and passed, with the variable off or not. + for variables in ({}, {"CI_E2E_IN_CI": "true"}, {"CI_E2E_IN_CI": "false"}): + assert mint(variables, {"result": "success"}, ran_plan) is True, variables + # And still no proof for an unset variable when E2E was skipped or failed. + assert mint({}, {"result": "skipped"}, {}) is False + assert mint({}, {"result": "failure"}, ran_plan) is False + print("PASS CI_E2E_IN_CI=false: a pull request that skipped E2E mints no Required proof (rollback cannot reuse it)") + + +def main() -> None: + for test in (test_mode_input, test_switch_defaults_change_nothing, test_switch_off_skips_the_e2e_lanes, + test_switch_leaves_promises_to_run_browsers, test_switch_off_without_run_e2e_is_unchanged, + test_fold_reads_the_effective_switch, test_mode_e2e_runs_only_the_e2e_chain, + test_mode_e2e_verdict, test_mode_e2e_skips_caller_lint_and_lint_steps, + test_prebuilt_handoff_follows_the_switch, test_mode_e2e_never_path_skips, + test_proof_key_ignores_mode, test_switch_off_pull_request_mints_no_required_proof): + test() + print("e2e mode contract passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_e2e_skip_paths.py b/scripts/test_e2e_skip_paths.py index 451c0a0..0cbc36d 100644 --- a/scripts/test_e2e_skip_paths.py +++ b/scripts/test_e2e_skip_paths.py @@ -84,7 +84,9 @@ def check_job_wiring() -> None: # `Reuse plan` ancestor would turn false (scripts/test_fast_path.py). assert e2e["if"] == ( "!cancelled() && needs.build.result == 'success' && needs.e2e-plan.result == 'success' " - "&& inputs.run-e2e && needs.e2e-plan.outputs.skipped != 'true'" + "&& (inputs.run-e2e && (vars.CI_E2E_IN_CI != 'false' || inputs.mode == 'e2e' " + "|| inputs.e2e-full-paths != '' || inputs.expected-candidate-sha != '')) " + "&& needs.e2e-plan.outputs.skipped != 'true'" ) assert "e2e-plan" in e2e["needs"] diff --git a/scripts/test_exact_candidate.py b/scripts/test_exact_candidate.py index 3d20e71..e47117a 100644 --- a/scripts/test_exact_candidate.py +++ b/scripts/test_exact_candidate.py @@ -38,7 +38,7 @@ def test_every_source_checkout_is_pinned_and_verified(self): # Caller lint runs inside Required (and escalated Fast) # after the gate steps, under !cancelled(). if name == 'required': - condition = f"!cancelled() && {condition}" + condition = f"!cancelled() && inputs.mode != 'e2e' && {condition}" if name == 'fast-escalated': condition = f"env.FAST_ENABLED == 'true' && !cancelled() && {condition}" self.assertEqual(guard['if'], condition, name) @@ -104,7 +104,7 @@ def test_required_does_not_waive_failed_candidate_jobs(self): 'Install actionlint', "actionlint (caller's own workflows)", 'Caller workflow hygiene audit (concurrency, timeouts, SHA pins, permissions)'] escalated = self.workflow['jobs']['fast-escalated'] - for job, expected in [(required, '!cancelled()'), + for job, expected in [(required, "!cancelled() && inputs.mode != 'e2e'"), (escalated, "env.FAST_ENABLED == 'true' && !cancelled()")]: steps = {step.get('name'): step for step in job['steps']} for name in caller_lint: diff --git a/scripts/test_fast_path.py b/scripts/test_fast_path.py index 5a94c53..99a4590 100644 --- a/scripts/test_fast_path.py +++ b/scripts/test_fast_path.py @@ -463,8 +463,9 @@ def test_checks_in_build() -> None: names = ["Typecheck Worker", "Typecheck Nuxt", "Unit tests"] for name in names: assert steps[name]["run"] == checks[name]["run"] and steps[name]["env"] == checks[name]["env"], name - assert steps["Typecheck Worker"]["if"] == steps["Typecheck Nuxt"]["if"] == "inputs.checks-in-build" - assert steps["Unit tests"]["if"] == "inputs.checks-in-build && inputs.run-tests" + # `mode: e2e` (the post-merge / nightly Playwright run) never typechecks or tests. + assert steps["Typecheck Worker"]["if"] == steps["Typecheck Nuxt"]["if"] == "inputs.checks-in-build && inputs.mode != 'e2e'" + assert steps["Unit tests"]["if"] == "inputs.checks-in-build && inputs.run-tests && inputs.mode != 'e2e'" assert "if" not in checks["Typecheck Worker"] and checks["Unit tests"]["if"] == "inputs.run-tests" order = [s.get("name") for s in JOBS["build"]["steps"]] assert order.index("Unit tests") < order.index("Build"), order @@ -945,7 +946,7 @@ def test_concurrent_scripts() -> None: a failed, missing, hooked or killed script fails Build.""" start = step("build", "Start concurrent scripts") wait = step("build", "Await concurrent scripts") - assert INPUTS["concurrent-scripts"]["default"] == "" and start["if"] == "inputs.concurrent-scripts != ''" + assert INPUTS["concurrent-scripts"]["default"] == "" and start["if"] == "inputs.concurrent-scripts != '' && inputs.mode != 'e2e'" assert wait["if"] == "!cancelled() && steps.concurrent-start.outcome == 'success'" order = [x.get("name") for x in JOBS["build"]["steps"]] assert order.index("Start concurrent scripts") + 1 == order.index("Build") < order.index("Await concurrent scripts") diff --git a/scripts/test_package_auth_cleanup.py b/scripts/test_package_auth_cleanup.py index 2690d52..f23b392 100644 --- a/scripts/test_package_auth_cleanup.py +++ b/scripts/test_package_auth_cleanup.py @@ -123,7 +123,7 @@ def auth_jobs(document: dict) -> dict[str, dict]: def validate_cleanup_step(job_id: str, step: dict, condition: str = "always() && inputs.install-script == ''") -> None: if job_id in ("fast", "fast-escalable"): - condition += " && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" + condition += " && inputs.mode != 'e2e' && inputs.fast-scripts != '' && inputs.journey-smoke-url == '' && needs.reuse-plan.outputs.reused != 'true'" assert step.get("if") == condition, ( f"{job_id}: cleanup must always run for the legacy materialization path, " f"got {step.get('if')!r}" diff --git a/scripts/test_quality_gates.py b/scripts/test_quality_gates.py index 7569dbc..b31b6e9 100644 --- a/scripts/test_quality_gates.py +++ b/scripts/test_quality_gates.py @@ -122,9 +122,9 @@ def gates(outputs: dict[str, str]) -> tuple[str, str, str]: check(PREVIEW_PROBE["run"] == SMOKE_PROBE["run"], "both security-headers probes must run the anchored text") conditions = { - ("build", "Run web-foundation conformance check"): "steps.quality.outputs.foundation-check == 'true'", - ("build", "Evaluate web-foundation conformance check"): "always() && steps.quality.outputs.foundation-check == 'true'", - ("build", "Performance budget"): "steps.quality.outputs.performance-budget == 'true'", + ("build", "Run web-foundation conformance check"): "steps.quality.outputs.foundation-check == 'true' && inputs.mode != 'e2e'", + ("build", "Evaluate web-foundation conformance check"): "always() && steps.quality.outputs.foundation-check == 'true' && inputs.mode != 'e2e'", + ("build", "Performance budget"): "steps.quality.outputs.performance-budget == 'true' && inputs.mode != 'e2e'", } for (job, name), expected in conditions.items(): check(str(step(job, name).get("if", "")).strip() == expected, f"{job}/{name} condition drifted") diff --git a/scripts/test_red_main_listener.py b/scripts/test_red_main_listener.py index e1755a4..ac95226 100644 --- a/scripts/test_red_main_listener.py +++ b/scripts/test_red_main_listener.py @@ -228,6 +228,18 @@ def check_self_adopter_allows_workflow_dispatch() -> tuple[bool, str]: return ok, f"red-main-self.yml job `if:` was: {condition!r}" +def check_self_adopter_allows_schedule() -> tuple[bool, str]: + """A nightly `schedule` run always runs the default branch, so a red one is + a "main is red" signal (the post-merge `E2E` workflow's nightly). The job + `if:` must accept it, still gated by `head_branch == default_branch`; the + pull_request exclusion above is untouched.""" + doc = yaml.safe_load(SELF_ADOPTER.read_text()) + condition = doc["jobs"]["red-main"].get("if") or "" + ok = ("workflow_run.event == 'schedule'" in condition and "head_branch" in condition + and "pull_request" not in condition) + return ok, f"red-main-self.yml job `if:` was: {condition!r}" + + def main() -> int: script = extract_step_script() total = 0 @@ -243,6 +255,11 @@ def main() -> int: if not check("self-adopter if: also allows event == 'workflow_dispatch'", ok, detail): failures += 1 + total += 1 + ok, detail = check_self_adopter_allows_schedule() + if not check("self-adopter if: also allows event == 'schedule' (nightly E2E)", ok, detail): + failures += 1 + with tempfile.TemporaryDirectory() as tmp_str: tmp = Path(tmp_str) h = Harness(tmp)