diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml index bf102b78c4..07a176dc79 100644 --- a/.github/workflows/engine-verify.yml +++ b/.github/workflows/engine-verify.yml @@ -4,10 +4,11 @@ name: Engine verify # and captions engines (crates/verify, crates/verify-expect, the decoder model # fuzzer in cap-rendering). See crates/verify/README.md. # -# Tiers: FULL for every pull request that changes an engine crate -# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the +# Tiers: FULL for every ready (non-draft) pull request that changes an engine +# crate (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the # `verify-full` label (engine-verify-label.yml), in the merge queue and -# nightly; FAST for other pull requests. Only suites whose +# nightly; FAST for drafts and other pull requests. Marking a draft ready +# runs the FULL tier on the head that will merge. Only suites whose # code a change can reach run (scripts/verify/plan.py maps changed paths to # suites through the workspace dependency graph), and only on the platforms # where they test platform code. The "Engine verify" job is the one to require. @@ -169,11 +170,16 @@ jobs: ENGINE: ${{ steps.risk.outputs.engine }} EVENT: ${{ github.event_name }} BASE: ${{ inputs.base }} + DRAFT: ${{ github.event.pull_request.draft }} run: | tier="$TIER" if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then - tier=full - echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + if [[ "$DRAFT" == true ]]; then + echo "Engine or high-risk code changed in a draft: FAST tier; marking the pull request ready runs the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + else + tier=full + echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + fi fi echo "tier=$tier" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index 2ab1e6cfee..28a0af8872 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -9,18 +9,23 @@ name: Verify recording (real capture) # - Scope decides from the diff whether the change reaches the recorder's or # the stimulus's dependency closure or the harness (scripts/verify/plan.py # recording); a pull request that does not is green without running; -# - one that does runs the FAST tier on macOS 15, or the FULL tier on macOS -# 15 and 26 when it touches a high-risk region or an engine crate; nightly -# and dispatch run FULL on both. +# - one that does runs the FAST tier on Blacksmith macOS 26, or, once it is +# ready for review (not a draft), the FULL tier there when it touches a +# high-risk region or an engine crate. Blacksmith macOS 15 runs nightly +# only: its Instant recordings with system audio lose a microphone marker +# in every run (on main too), which is under investigation; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 +# and 26, dispatch FULL on the chosen runners. # -# Runners: GitHub-hosted macOS by default, because real Macs record through -# the VideoToolbox hardware H.264 encoder and Blacksmith macOS VMs have none +# Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, no concurrency cap) for pull +# requests, pushes and dispatch: jobs start at once and every run gets the +# same machine. Blacksmith VMs have no VideoToolbox hardware H.264 encoder # (VTCompressionSessionCreate with RequireHardwareAcceleratedVideoEncoder -# returns -12903, probe run 38847791287), so every recording there takes the -# software (libx264) path. Blacksmith (M4 Pro VM, 6 vCPU, no concurrency cap) -# is a dispatch option (runners=blacksmith or both) that covers that path; it -# needs Screen Recording and Microphone granted in TCC, its display woken, -# and an audio device (BlackHole), which the setup step below does. +# returns -12903, probe run 38847791287), so recordings there take the +# software (libx264) path; the nightly run adds GitHub-hosted macOS 15 and +# 26, which record through the hardware encoder real Macs use (dispatch +# runners=hosted or both for the same). Blacksmith needs Screen Recording and +# Microphone granted in TCC, its display woken, and an audio device +# (BlackHole), which the setup step below does. on: workflow_dispatch: inputs: @@ -42,10 +47,10 @@ on: type: boolean default: false runners: - description: "hosted (default), blacksmith (software encoder path) or both" + description: "blacksmith (default, software encoder path), hosted (VideoToolbox hardware encoder) or both" type: choice - options: [hosted, blacksmith, both] - default: hosted + options: [blacksmith, hosted, both] + default: blacksmith schedule: - cron: "41 3 * * *" pull_request: @@ -83,6 +88,7 @@ jobs: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} PR: ${{ github.event.pull_request.number }} + DRAFT: ${{ github.event.pull_request.draft }} BEFORE: ${{ github.event.before }} INPUT_TIER: ${{ inputs.tier }} INPUT_RUNNERS: ${{ inputs.runners }} @@ -91,15 +97,17 @@ jobs: set -euo pipefail files="$RUNNER_TEMP/changed.txt" hosted='["macos-15","macos-26"]' + blacksmith='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' + both='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' case "$EVENT" in - schedule) run=true; runners="$hosted"; tier=full ;; + schedule) run=true; runners="$both"; tier=full ;; workflow_dispatch) run=true tier="${INPUT_TIER:-full}" - case "${INPUT_RUNNERS:-hosted}" in - blacksmith) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' ;; - both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;; - *) runners="$hosted" ;; + case "${INPUT_RUNNERS:-blacksmith}" in + hosted) runners="$hosted" ;; + both) runners="$both" ;; + *) runners="$blacksmith" ;; esac ;; *) if [[ "$EVENT" == pull_request ]]; then @@ -116,10 +124,10 @@ jobs: cat "$RUNNER_TEMP/scope.txt" run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")" python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt" - if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then - runners="$hosted"; tier=full + if [[ "${DRAFT:-false}" != true ]] && { grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; }; then + runners='["blacksmith-6vcpu-macos-26"]'; tier=full else - runners='["macos-15"]'; tier=fast + runners='["blacksmith-6vcpu-macos-26"]'; tier=fast fi ;; esac {