From db17691aeef42965cb0af57d3e36f53f6a8b25f5 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:24:52 +0900 Subject: [PATCH 1/3] ci: run real-capture verification on Blacksmith macOS Pull requests, pushes and dispatches now record on Blacksmith macOS 15 and 26: jobs start at once with no concurrency cap and every run gets the same machine. GitHub-hosted runs added rare failures with four different signatures (start frames, a camera frame, a VideoToolbox encode error, a self-check). Blacksmith has no hardware H.264 encoder, so the nightly run also records on GitHub-hosted macOS 15 and 26 to keep the VideoToolbox path covered. --- .github/workflows/verify-recording.yml | 45 ++++++++++++++------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index 2ab1e6cfee2..e0a96fb9119 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -9,18 +9,21 @@ name: Verify recording (real capture) # - Scope decides from the diff whether the change reaches the recorder's or # the stimulus's dependency closure or the harness (scripts/verify/plan.py # recording); a pull request that does not is green without running; -# - one that does runs the FAST tier on macOS 15, or the FULL tier on macOS -# 15 and 26 when it touches a high-risk region or an engine crate; nightly -# and dispatch run FULL on both. +# - one that does runs the FAST tier on Blacksmith macOS 15, or the FULL tier +# on Blacksmith macOS 15 and 26 when it touches a high-risk region or an +# engine crate; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 +# and 26, dispatch FULL on the chosen runners. # -# Runners: GitHub-hosted macOS by default, because real Macs record through -# the VideoToolbox hardware H.264 encoder and Blacksmith macOS VMs have none +# Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, no concurrency cap) for pull +# requests, pushes and dispatch: jobs start at once and every run gets the +# same machine. Blacksmith VMs have no VideoToolbox hardware H.264 encoder # (VTCompressionSessionCreate with RequireHardwareAcceleratedVideoEncoder -# returns -12903, probe run 38847791287), so every recording there takes the -# software (libx264) path. Blacksmith (M4 Pro VM, 6 vCPU, no concurrency cap) -# is a dispatch option (runners=blacksmith or both) that covers that path; it -# needs Screen Recording and Microphone granted in TCC, its display woken, -# and an audio device (BlackHole), which the setup step below does. +# returns -12903, probe run 38847791287), so recordings there take the +# software (libx264) path; the nightly run adds GitHub-hosted macOS 15 and +# 26, which record through the hardware encoder real Macs use (dispatch +# runners=hosted or both for the same). Blacksmith needs Screen Recording and +# Microphone granted in TCC, its display woken, and an audio device +# (BlackHole), which the setup step below does. on: workflow_dispatch: inputs: @@ -42,10 +45,10 @@ on: type: boolean default: false runners: - description: "hosted (default), blacksmith (software encoder path) or both" + description: "blacksmith (default, software encoder path), hosted (VideoToolbox hardware encoder) or both" type: choice - options: [hosted, blacksmith, both] - default: hosted + options: [blacksmith, hosted, both] + default: blacksmith schedule: - cron: "41 3 * * *" pull_request: @@ -91,15 +94,17 @@ jobs: set -euo pipefail files="$RUNNER_TEMP/changed.txt" hosted='["macos-15","macos-26"]' + blacksmith='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' + both='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' case "$EVENT" in - schedule) run=true; runners="$hosted"; tier=full ;; + schedule) run=true; runners="$both"; tier=full ;; workflow_dispatch) run=true tier="${INPUT_TIER:-full}" - case "${INPUT_RUNNERS:-hosted}" in - blacksmith) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' ;; - both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;; - *) runners="$hosted" ;; + case "${INPUT_RUNNERS:-blacksmith}" in + hosted) runners="$hosted" ;; + both) runners="$both" ;; + *) runners="$blacksmith" ;; esac ;; *) if [[ "$EVENT" == pull_request ]]; then @@ -117,9 +122,9 @@ jobs: run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")" python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt" if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then - runners="$hosted"; tier=full + runners="$blacksmith"; tier=full else - runners='["macos-15"]'; tier=fast + runners='["blacksmith-6vcpu-macos-15"]'; tier=fast fi ;; esac { From ff6f2e766d7879c459d5b2c8cf0141e6b7c9c2c4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:33:03 +0900 Subject: [PATCH 2/3] ci: run the FULL verification tier once a pull request is ready Draft pull requests that change engine or high-risk code now run the FAST tier on every push; marking the pull request ready (and any push after that) runs the FULL tier on the head that will merge. The verify-full label still forces FULL on a draft. --- .github/workflows/engine-verify.yml | 16 +++++++++++----- .github/workflows/verify-recording.yml | 9 +++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml index bf102b78c41..07a176dc79a 100644 --- a/.github/workflows/engine-verify.yml +++ b/.github/workflows/engine-verify.yml @@ -4,10 +4,11 @@ name: Engine verify # and captions engines (crates/verify, crates/verify-expect, the decoder model # fuzzer in cap-rendering). See crates/verify/README.md. # -# Tiers: FULL for every pull request that changes an engine crate -# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the +# Tiers: FULL for every ready (non-draft) pull request that changes an engine +# crate (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the # `verify-full` label (engine-verify-label.yml), in the merge queue and -# nightly; FAST for other pull requests. Only suites whose +# nightly; FAST for drafts and other pull requests. Marking a draft ready +# runs the FULL tier on the head that will merge. Only suites whose # code a change can reach run (scripts/verify/plan.py maps changed paths to # suites through the workspace dependency graph), and only on the platforms # where they test platform code. The "Engine verify" job is the one to require. @@ -169,11 +170,16 @@ jobs: ENGINE: ${{ steps.risk.outputs.engine }} EVENT: ${{ github.event_name }} BASE: ${{ inputs.base }} + DRAFT: ${{ github.event.pull_request.draft }} run: | tier="$TIER" if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then - tier=full - echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + if [[ "$DRAFT" == true ]]; then + echo "Engine or high-risk code changed in a draft: FAST tier; marking the pull request ready runs the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + else + tier=full + echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + fi fi echo "tier=$tier" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index e0a96fb9119..c4b36a57718 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -9,9 +9,9 @@ name: Verify recording (real capture) # - Scope decides from the diff whether the change reaches the recorder's or # the stimulus's dependency closure or the harness (scripts/verify/plan.py # recording); a pull request that does not is green without running; -# - one that does runs the FAST tier on Blacksmith macOS 15, or the FULL tier -# on Blacksmith macOS 15 and 26 when it touches a high-risk region or an -# engine crate; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 +# - one that does runs the FAST tier on Blacksmith macOS 15, or, once it is +# ready for review (not a draft), the FULL tier on Blacksmith macOS 15 and +# 26 when it touches a high-risk region or an engine crate; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 # and 26, dispatch FULL on the chosen runners. # # Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, no concurrency cap) for pull @@ -86,6 +86,7 @@ jobs: GH_TOKEN: ${{ github.token }} EVENT: ${{ github.event_name }} PR: ${{ github.event.pull_request.number }} + DRAFT: ${{ github.event.pull_request.draft }} BEFORE: ${{ github.event.before }} INPUT_TIER: ${{ inputs.tier }} INPUT_RUNNERS: ${{ inputs.runners }} @@ -121,7 +122,7 @@ jobs: cat "$RUNNER_TEMP/scope.txt" run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")" python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt" - if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then + if [[ "${DRAFT:-false}" != true ]] && { grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; }; then runners="$blacksmith"; tier=full else runners='["blacksmith-6vcpu-macos-15"]'; tier=fast From 462334b58e8f9ffc5004739158af5bcf02517a34 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 17 Oct 2026 02:04:14 +0900 Subject: [PATCH 3/3] ci: verify pull requests on Blacksmith macOS 26 until the macOS 15 Instant finding is fixed Five repeats on main: Blacksmith macOS 26 recorded every case as expected (65 of 65); Blacksmith macOS 15 recorded every Studio case as expected but its Instant recordings with system audio lost one to three microphone markers in every run. Pull requests run on macOS 26; the nightly keeps Blacksmith and GitHub-hosted macOS 15 and 26. --- .github/workflows/verify-recording.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml index c4b36a57718..28a0af88722 100644 --- a/.github/workflows/verify-recording.yml +++ b/.github/workflows/verify-recording.yml @@ -9,9 +9,11 @@ name: Verify recording (real capture) # - Scope decides from the diff whether the change reaches the recorder's or # the stimulus's dependency closure or the harness (scripts/verify/plan.py # recording); a pull request that does not is green without running; -# - one that does runs the FAST tier on Blacksmith macOS 15, or, once it is -# ready for review (not a draft), the FULL tier on Blacksmith macOS 15 and -# 26 when it touches a high-risk region or an engine crate; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 +# - one that does runs the FAST tier on Blacksmith macOS 26, or, once it is +# ready for review (not a draft), the FULL tier there when it touches a +# high-risk region or an engine crate. Blacksmith macOS 15 runs nightly +# only: its Instant recordings with system audio lose a microphone marker +# in every run (on main too), which is under investigation; nightly runs FULL on Blacksmith and GitHub-hosted macOS 15 # and 26, dispatch FULL on the chosen runners. # # Runners: Blacksmith macOS (M4 Pro VM, 6 vCPU, no concurrency cap) for pull @@ -123,9 +125,9 @@ jobs: run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")" python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt" if [[ "${DRAFT:-false}" != true ]] && { grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; }; then - runners="$blacksmith"; tier=full + runners='["blacksmith-6vcpu-macos-26"]'; tier=full else - runners='["blacksmith-6vcpu-macos-15"]'; tier=fast + runners='["blacksmith-6vcpu-macos-26"]'; tier=fast fi ;; esac {