ci: verify real capture on Blacksmith and run the FULL tier once a PR is ready

.github/workflows/engine-verify.yml

@@ -4,10 +4,11 @@ name: Engine verify
 # and captions engines (crates/verify, crates/verify-expect, the decoder model
 # fuzzer in cap-rendering). See crates/verify/README.md.
 #
-# Tiers: FULL for every pull request that changes an engine crate
-# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the
+# Tiers: FULL for every ready (non-draft) pull request that changes an engine
+# crate (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the
 # `verify-full` label (engine-verify-label.yml), in the merge queue and
-# nightly; FAST for other pull requests. Only suites whose
+# nightly; FAST for drafts and other pull requests. Marking a draft ready
+# runs the FULL tier on the head that will merge. Only suites whose
 # code a change can reach run (scripts/verify/plan.py maps changed paths to
 # suites through the workspace dependency graph), and only on the platforms
 # where they test platform code. The "Engine verify" job is the one to require.
@@ -169,11 +170,16 @@ jobs:
           ENGINE: ${{ steps.risk.outputs.engine }}
           EVENT: ${{ github.event_name }}
           BASE: ${{ inputs.base }}
+          DRAFT: ${{ github.event.pull_request.draft }}
         run: |
           tier="$TIER"
           if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then
-            tier=full
-            echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"
+            if [[ "$DRAFT" == true ]]; then
+              echo "Engine or high-risk code changed in a draft: FAST tier; marking the pull request ready runs the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"
+            else
+              tier=full
+              echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"
+            fi
           fi
           echo "tier=$tier" >> "$GITHUB_OUTPUT"
 

.github/workflows/verify-recording.yml

@@ -9,18 +9,23 @@ name: Verify recording (real capture)
 # - Scope decides from the diff whether the change reaches the recorder's or
 #   the stimulus's dependency closure or the harness (scripts/verify/plan.py
 #   recording); a pull request that does not is green without running;
-# - one that does runs the FAST tier on macOS 15, or the FULL tier on macOS
-#   15 and 26 when it touches a high-risk region or an engine crate; nightly
-#   and dispatch run FULL on both.
+# - one that does runs the FAST tier on Blacksmith macOS 26, or, once it is
+#   ready for review (not a draft), the FULL tier there when it touches a
+#   high-risk region or an engine crate. Blacksmith macOS 15 runs nightly
+#   only: its Instant recordings with system audio lose a microphone marker
+#   in every run (on main too), which is under investigation; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15
+#   and 26, dispatch FULL on the chosen runners.
 #
-# Runners: GitHub-hosted macOS by default, because real Macs record through
-# the VideoToolbox hardware H.264 encoder and Blacksmith macOS VMs have none
+# Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, no concurrency cap) for pull
+# requests, pushes and dispatch: jobs start at once and every run gets the
+# same machine. Blacksmith VMs have no VideoToolbox hardware H.264 encoder
 # (VTCompressionSessionCreate with RequireHardwareAcceleratedVideoEncoder
-# returns -12903, probe run 38847791287), so every recording there takes the
-# software (libx264) path. Blacksmith (M4 Pro VM, 6 vCPU, no concurrency cap)
-# is a dispatch option (runners=blacksmith or both) that covers that path; it
-# needs Screen Recording and Microphone granted in TCC, its display woken,
-# and an audio device (BlackHole), which the setup step below does.
+# returns -12903, probe run 38847791287), so recordings there take the
+# software (libx264) path; the nightly run adds GitHub-hosted macOS 15 and
+# 26, which record through the hardware encoder real Macs use (dispatch
+# runners=hosted or both for the same). Blacksmith needs Screen Recording and
+# Microphone granted in TCC, its display woken, and an audio device
+# (BlackHole), which the setup step below does.
 on:
   workflow_dispatch:
     inputs:
@@ -42,10 +47,10 @@ on:
         type: boolean
         default: false
       runners:
-        description: "hosted (default), blacksmith (software encoder path) or both"
+        description: "blacksmith (default, software encoder path), hosted (VideoToolbox hardware encoder) or both"
         type: choice
-        options: [hosted, blacksmith, both]
-        default: hosted
+        options: [blacksmith, hosted, both]
+        default: blacksmith
   schedule:
     - cron: "41 3 * * *"
   pull_request:
@@ -83,6 +88,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           EVENT: ${{ github.event_name }}
           PR: ${{ github.event.pull_request.number }}
+          DRAFT: ${{ github.event.pull_request.draft }}
           BEFORE: ${{ github.event.before }}
           INPUT_TIER: ${{ inputs.tier }}
           INPUT_RUNNERS: ${{ inputs.runners }}
@@ -91,15 +97,17 @@ jobs:
           set -euo pipefail
           files="$RUNNER_TEMP/changed.txt"
           hosted='["macos-15","macos-26"]'
+          blacksmith='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]'
+          both='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]'
           case "$EVENT" in
-            schedule) run=true; runners="$hosted"; tier=full ;;
+            schedule) run=true; runners="$both"; tier=full ;;
             workflow_dispatch)
               run=true
               tier="${INPUT_TIER:-full}"
-              case "${INPUT_RUNNERS:-hosted}" in
-                blacksmith) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' ;;
-                both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;;
-                *) runners="$hosted" ;;
+              case "${INPUT_RUNNERS:-blacksmith}" in
+                hosted) runners="$hosted" ;;
+                both) runners="$both" ;;
+                *) runners="$blacksmith" ;;
               esac ;;
             *)
               if [[ "$EVENT" == pull_request ]]; then
@@ -116,10 +124,10 @@ jobs:
               cat "$RUNNER_TEMP/scope.txt"
               run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")"
               python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt"
-              if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then
-                runners="$hosted"; tier=full
+              if [[ "${DRAFT:-false}" != true ]] && { grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; }; then
+                runners='["blacksmith-6vcpu-macos-26"]'; tier=full
               else
-                runners='["macos-15"]'; tier=fast
+                runners='["blacksmith-6vcpu-macos-26"]'; tier=fast
               fi ;;
           esac
           {