diff --git a/.config/hakari.toml b/.config/hakari.toml index 3b2ff13795f..03c9ecf86a0 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -35,7 +35,7 @@ platforms = [ # unified tree into every `cargo build -p cap-muxer`. Any drift it causes is # bounded by its own tiny graph. [traversal-excludes] -workspace-members = ["cap-muxer", "cap-muxer-protocol"] +workspace-members = ["cap-muxer", "cap-muxer-protocol", "cap-verify-expect"] # hakari unifies dev-dependencies as well, so left alone it pins tauri with its # `test` feature and tokio `test-util` into every release binary and pulls the diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..a9f409ed444 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,41 @@ +# cargo-nextest settings. The engine verification suite (scripts/verify/run.sh) +# runs with the `default` profile locally and `ci` in engine-verify.yml. + +[profile.default] +retries = 0 +fail-fast = false +slow-timeout = { period = "60s" } + +# The decoder model fuzzers run 3 to 4 worker threads each. +[[profile.default.overrides]] +filter = 'binary_id(cap-rendering) and test(/^decoder::model::/)' +threads-required = 3 + +[profile.ci] +# No retries: a flaky case is a bug. run.sh reruns failures once afterwards +# only to label them flaky in the job summary; the job still fails. +retries = 0 +fail-fast = false +failure-output = "immediate-final" +success-output = "never" +status-level = "pass" +final-status-level = "slow" +slow-timeout = { period = "60s", terminate-after = 15 } + +[profile.ci.junit] +path = "junit.xml" +store-success-output = false +store-failure-output = true + +[[profile.ci.overrides]] +filter = 'binary_id(cap-rendering) and test(/^decoder::model::/)' +threads-required = 3 + + +# The FULL-tier seek storms (3000 seeks per decoder) are the slowest tests: +# on the macos-15 runner one storm fixture takes 240 to 570 s per decoder, +# up to about 900 s for both decoders on one shard. Twice that, so a real +# hang still fails. +[[profile.ci.overrides]] +filter = 'binary_id(cap-verify::seek) and test(/^storm_/)' +slow-timeout = { period = "60s", terminate-after = 30 } diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000..42d3a07b80c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +self-hosted-runner: + labels: + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-6vcpu-macos-15 + - blacksmith-6vcpu-macos-26 + - blacksmith-8vcpu-windows-2025 diff --git a/.github/actions/engine-verify-setup/action.yml b/.github/actions/engine-verify-setup/action.yml new file mode 100644 index 00000000000..1cfedb12b1c --- /dev/null +++ b/.github/actions/engine-verify-setup/action.yml @@ -0,0 +1,168 @@ +name: Engine verify setup +description: Toolchain, cargo-nextest, Rust cache, native deps and fixture cache for the engine verification suite +inputs: + target: + description: Rust target triple + required: true + tier: + description: fast or full (fixture cache key) + required: true + rust-cache: + description: Restore the Rust build cache (build jobs only) + default: "true" +outputs: + build-cache-key: + description: Key to save the build cache under + value: ${{ steps.build-key.outputs.key }} + build-cache-hit: + description: Whether the build cache was restored exactly + value: ${{ steps.build-cache.outputs.cache-hit }} + fixtures-hit: + description: Whether the fixture cache was restored exactly + value: ${{ steps.fixtures.outputs.cache-hit }} + fixtures-key: + description: Fixture cache key + value: ${{ steps.fixtures.outputs.cache-primary-key }} +runs: + using: composite + steps: + # Only what the suite's dependency closure links against (no webkit/gtk), + # plus lavapipe for software Vulkan rendering. + - name: Linux packages + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + libasound2-dev libasound2-plugins libpipewire-0.3-dev libspa-0.2-dev libclang-dev \ + libx11-dev libxrandr-dev libxcb1-dev libxcb-randr0-dev libxcb-shm0-dev \ + libxcb-xfixes0-dev libxfixes-dev libwayland-dev libxkbcommon-dev \ + libdbus-1-dev libudev-dev libva-dev mesa-vulkan-drivers libvulkan1 >/dev/null + + - name: Rust toolchain + if: inputs.rust-cache == 'true' + uses: dtolnay/rust-toolchain@1.88.0 + with: + targets: ${{ inputs.target }} + + - name: cargo-nextest + uses: taiki-e/install-action@v2 + with: + tool: cargo-nextest@0.9.148 + + # One cache entry per OS per day (saved by main, nightly and the CI dev + # branches), restored by PRs. Unlike swatinem/rust-cache it includes the + # workspace crates and is refreshed daily even when Cargo.lock is + # unchanged, so a PR only rebuilds what changed since the last save. The + # chain restarts every ISO week so stale artifacts cannot pile up. + - name: Build cache key + id: build-key + if: inputs.rust-cache == 'true' + shell: bash + run: | + # runner.environment separates GitHub-hosted from self-hosted + # (Blacksmith) builds: their paths and toolchains differ, so one's + # target directory would only force a rebuild on the other. + # A new week starts a new chain, seeded from last week's newest cache + # of the same provider so the first build of the week stays warm. + py="$(command -v python3 || command -v python)" + week="$("$py" -c 'import datetime as d; y, w, _ = d.datetime.now(d.timezone.utc).isocalendar(); print(f"{y}-W{w:02d}")')" + last_week="$("$py" -c 'import datetime as d; y, w, _ = (d.datetime.now(d.timezone.utc) - d.timedelta(days=7)).isocalendar(); print(f"{y}-W{w:02d}")')" + provider="verify-build-v3-${{ inputs.target }}-${{ runner.environment }}" + base="$provider-$week" + echo "base=$base" >> "$GITHUB_OUTPUT" + echo "previous=$provider-$last_week" >> "$GITHUB_OUTPUT" + echo "key=$base-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml') }}-$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Restore build cache + id: build-cache + if: inputs.rust-cache == 'true' + uses: actions/cache/restore@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ci-verify/ + key: ${{ steps.build-key.outputs.key }} + restore-keys: | + ${{ steps.build-key.outputs.base }}-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'rust-toolchain.toml') }}- + ${{ steps.build-key.outputs.base }}- + ${{ steps.build-key.outputs.previous }}- + + # Unchanged workspace crates are reused from the cache instead of being + # rebuilt because the checkout touched every file (scripts/verify/mtime.py). + - name: Reuse unchanged workspace crates + if: inputs.rust-cache == 'true' + shell: bash + run: | + "$(command -v python3 || command -v python)" scripts/verify/mtime.py apply target/ci-verify/verify-src-manifest.txt + + - name: Native dependencies + shell: bash + env: + RUST_TARGET_TRIPLE: ${{ inputs.target }} + run: node scripts/setup.js + + - name: FFmpeg libraries on the loader path (Linux) + if: runner.os == 'Linux' + shell: bash + run: echo "LD_LIBRARY_PATH=${{ github.workspace }}/target/native-deps/lib" >> "$GITHUB_ENV" + + # The app ships DXC next to its executable and cap-rendering looks for it + # next to the test binary or in its profile dir; setup.js only fills + # target/debug and target/release. Without it wgpu falls back to FXC. + - name: DXC for the ci-verify profile (Windows) + if: runner.os == 'Windows' + shell: bash + run: | + mkdir -p target/ci-verify + cp target/native-deps/dxc/dxcompiler.dll target/native-deps/dxc/dxil.dll target/ci-verify/ + + # Debug builds of wgpu (the ci-verify profile keeps debug assertions) + # enable the D3D12 debug layer whenever d3d12SDKLayers.dll exists. When + # that DLL comes from an unserviced Graphics Tools feature older than the + # D3D12 runtime (blacksmith-8vcpu-windows-2025: layers 10.0.26100.1591, + # runtime 10.0.26100.33296), D3D12CreateDevice fails with E_INVALIDARG on + # every adapter, WARP included, and every case fails "No GPU adapter + # found". Hiding the mismatched layer makes D3D12GetDebugInterface report + # DXGI_ERROR_SDK_COMPONENT_MISSING, which wgpu treats as "no debug layer". + - name: Hide a D3D12 debug layer that does not match the runtime (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $sys = Join-Path $env:WINDIR 'System32' + $layers = Join-Path $sys 'd3d12SDKLayers.dll' + $core = Join-Path $sys 'D3D12Core.dll' + if (-not (Test-Path $layers)) { Write-Host 'no D3D12 debug layer installed'; exit 0 } + $lv = (Get-Item $layers).VersionInfo + $cv = (Get-Item $core).VersionInfo + $lver = "$($lv.FileMajorPart).$($lv.FileMinorPart).$($lv.FileBuildPart).$($lv.FilePrivatePart)" + $cver = "$($cv.FileMajorPart).$($cv.FileMinorPart).$($cv.FileBuildPart).$($cv.FilePrivatePart)" + Write-Host "D3D12 debug layer $lver, runtime $cver" + if ($lver -eq $cver) { exit 0 } + takeown /f $layers | Out-Null + icacls $layers /grant "*S-1-5-32-544:F" | Out-Null + Rename-Item $layers 'd3d12SDKLayers.dll.mismatched' + Write-Host "hid the mismatched debug layer" + + - name: FFmpeg DLLs on PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Add-Content -Path $env:GITHUB_PATH -Value "${{ github.workspace }}\target\ffmpeg\bin" + + - name: Fixture location + shell: bash + run: echo "CAP_VERIFY_FIXTURE_DIR=${{ runner.temp }}/verify-fixtures" >> "$GITHUB_ENV" + + # Fixtures are generated by code (including the real-pipeline Studio and + # Instant recordings, ~1 min of real-time capture); the key is every + # crate's source and build script, the vendored crates (wgpu-hal, tao), + # the workspace manifest, lockfile and toolchain, plus the native deps. Exact key only: a fixture made + # by an older generator is never reused. + - name: Restore fixtures + id: fixtures + uses: actions/cache/restore@v4 + with: + path: ${{ runner.temp }}/verify-fixtures + key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/build.rs', 'crates/**/Cargo.toml', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'vendor/**', 'scripts/setup.js') }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43d79ddaa7a..cea83ac2a45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ concurrency: jobs: changes: name: Detect Changes - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: rust: ${{ steps.filter.outputs.rust }} gpui: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.gpui == 'true' || steps.desktop-resources.outputs.desktop-resources == 'true' }} @@ -88,7 +88,7 @@ jobs: typecheck: name: Typecheck - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Checkout uses: actions/checkout@v4 @@ -133,7 +133,7 @@ jobs: mobile: name: Mobile - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Checkout uses: actions/checkout@v4 @@ -159,7 +159,7 @@ jobs: format-biome: name: Format (Biome) - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Checkout uses: actions/checkout@v4 @@ -176,7 +176,7 @@ jobs: format-rust: name: Format (Cargo) - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Checkout uses: actions/checkout@v4 @@ -198,9 +198,9 @@ jobs: matrix: settings: - target: aarch64-apple-darwin - runner: macos-latest + runner: blacksmith-6vcpu-macos-15 - target: x86_64-pc-windows-msvc - runner: windows-2022 + runner: blacksmith-8vcpu-windows-2025 runs-on: ${{ matrix.settings.runner }} steps: - name: Checkout repository @@ -235,7 +235,7 @@ jobs: lint-biome: name: Lint (Biome) - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Checkout uses: actions/checkout@v4 @@ -253,7 +253,7 @@ jobs: needs: changes if: needs.changes.outputs.rust == 'true' timeout-minutes: 60 - runs-on: ubuntu-24.04 + runs-on: blacksmith-8vcpu-ubuntu-2404 env: RUST_TARGET_TRIPLE: x86_64-unknown-linux-gnu LD_LIBRARY_PATH: ${{ github.workspace }}/target/native-deps/lib:${{ github.workspace }}/target/debug @@ -293,11 +293,11 @@ jobs: matrix: settings: - target: aarch64-apple-darwin - runner: macos-latest + runner: blacksmith-6vcpu-macos-15 - target: x86_64-pc-windows-msvc - runner: windows-2022 + runner: blacksmith-8vcpu-windows-2025 - target: x86_64-unknown-linux-gnu - runner: ubuntu-24.04 + runner: blacksmith-8vcpu-ubuntu-2404 runs-on: ${{ matrix.settings.runner }} env: # Full debuginfo for the opt-level 2 dependency tree is most of the @@ -349,7 +349,8 @@ jobs: shell: bash run: | if [[ "$RUNNER_OS" == "Windows" ]]; then - export PATH="$(cygpath "$GITHUB_WORKSPACE/target/debug"):$PATH" + debug_path="$(cygpath "$GITHUB_WORKSPACE/target/debug")" + export PATH="$debug_path:$PATH" elif [[ "$RUNNER_OS" == "macOS" ]]; then frameworks="$GITHUB_WORKSPACE/apps/desktop-gpui/target/${{ matrix.settings.target }}/debug/Frameworks" mkdir -p "$frameworks" @@ -367,7 +368,7 @@ jobs: tauri-plugins: name: Verify Tauri plugin versions - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 needs: changes if: needs.changes.outputs.tauri-plugins == 'true' steps: diff --git a/.github/workflows/engine-verify-label.yml b/.github/workflows/engine-verify-label.yml new file mode 100644 index 00000000000..fc1cfbba0d9 --- /dev/null +++ b/.github/workflows/engine-verify-label.yml @@ -0,0 +1,20 @@ +name: Engine verify (full) + +# Adding the `verify-full` label to a PR runs the FULL tier once. A separate +# workflow so that adding any other label never produces a skipped (and +# therefore passing) "Engine verify" check that would hide a failed FAST run. + +on: + pull_request: + types: [labeled] + +permissions: + contents: read + pull-requests: read + +jobs: + full: + if: github.event.label.name == 'verify-full' + uses: ./.github/workflows/engine-verify.yml + with: + tier: full diff --git a/.github/workflows/engine-verify-os.yml b/.github/workflows/engine-verify-os.yml new file mode 100644 index 00000000000..95da0616218 --- /dev/null +++ b/.github/workflows/engine-verify-os.yml @@ -0,0 +1,234 @@ +name: Engine verify (one platform) + +# Called by engine-verify.yml once per platform. Cases are split across +# `shards` jobs by a stable hash of the case id (CAP_VERIFY_SHARD), so a suite +# whose test functions loop over many cases still spreads evenly. +# +# By default every shard builds for itself: with the Rust cache and +# scripts/verify/mtime.py a warm build only compiles what changed, which is +# cheaper than an artifact round trip and a second runner queue. With +# `archive: true` one job builds a nextest archive and the shards run it. + +on: + workflow_call: + inputs: + platform: + type: string + required: true + runner: + type: string + required: true + target: + type: string + required: true + tier: + type: string + required: true + suites: + type: string + required: true + shards: + type: number + default: 1 + shard-list: + type: string + default: "[1]" + archive: + type: boolean + default: false + save-cache: + type: boolean + default: false + cargo-config: + type: string + default: "" + cargo-log: + type: string + default: "" + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + RUST_BACKTRACE: "1" + CAP_VERIFY_CARGO_PROFILE: ci-verify + CAP_VERIFY_NEXTEST_PROFILE: ci + CAP_VERIFY_LOCKED: "1" + CAP_VERIFY_CARGO_CONFIG: ${{ inputs.cargo-config }} + # A case this runner cannot run fails unless scripts/verify/suites.json + # allows it as unsupported on this platform (with the reason). + CAP_VERIFY_REQUIRE_SUPPORTED: "1" + # Machine class for host-scoped expectations (crates/verify-expect hosts). + CAP_VERIFY_HOST: ${{ startsWith(inputs.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }} + +jobs: + build: + name: build + if: inputs.archive + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup + id: setup + uses: ./.github/actions/engine-verify-setup + with: + target: ${{ inputs.target }} + tier: ${{ inputs.tier }} + + - name: Build archive + shell: bash + run: scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --archive "$RUNNER_TEMP/verify.tar.zst" + + - name: Record sources of the cached build + if: inputs.save-cache && steps.setup.outputs.build-cache-hit != 'true' + shell: bash + run: | + "$(command -v python3 || command -v python)" scripts/verify/mtime.py record target/ci-verify/verify-src-manifest.txt + + - name: Save build cache + if: inputs.save-cache && steps.setup.outputs.build-cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ci-verify/ + key: ${{ steps.setup.outputs.build-cache-key }} + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: verify-archive-${{ inputs.platform }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/verify.tar.zst + retention-days: 1 + compression-level: 0 + + test: + name: ${{ inputs.shards > 1 && format('test {0}/{1}', matrix.shard, inputs.shards) || 'build and test' }} + needs: build + if: always() && (needs.build.result == 'success' || needs.build.result == 'skipped') + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(inputs.shard-list) }} + env: + CAP_VERIFY_SHARD: ${{ inputs.shards > 1 && format('{0}/{1}', matrix.shard, inputs.shards) || '' }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup + id: setup + uses: ./.github/actions/engine-verify-setup + with: + target: ${{ inputs.target }} + tier: ${{ inputs.tier }} + rust-cache: ${{ !inputs.archive }} + + - name: Report location + shell: bash + run: | + echo "CAP_VERIFY_REPORT_DIR=$RUNNER_TEMP/verify-report" >> "$GITHUB_ENV" + echo "CAP_VERIFY_FLAKY_FILE=$RUNNER_TEMP/verify-report/flaky.txt" >> "$GITHUB_ENV" + + - name: Build + if: ${{ !inputs.archive }} + shell: bash + env: + CARGO_LOG: ${{ inputs.cargo-log }} + run: scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --no-run + + - name: Record sources of the cached build + if: ${{ !inputs.archive && inputs.save-cache && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' }} + shell: bash + run: | + "$(command -v python3 || command -v python)" scripts/verify/mtime.py record target/ci-verify/verify-src-manifest.txt + + - name: Save build cache + if: ${{ !inputs.archive && inputs.save-cache && matrix.shard == 1 && steps.setup.outputs.build-cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ci-verify/ + key: ${{ steps.setup.outputs.build-cache-key }} + + # The retry and flaky reruns read test names back from Python (CRLF on + # Windows): check that such a name selects exactly its test. + - name: Rerun selection self-check + if: ${{ !inputs.archive }} + shell: bash + run: scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --check-rerun-selection + + - name: Download archive + if: inputs.archive + uses: actions/download-artifact@v4 + with: + name: verify-archive-${{ inputs.platform }}-${{ github.run_attempt }} + path: ${{ runner.temp }} + + - name: Prepare recorded fixtures without concurrent rendering + if: contains(format(',{0},', inputs.suites), ',export,') + shell: bash + env: + CAP_VERIFY_TIER: ${{ inputs.tier }} + run: | + if [[ "${{ inputs.archive }}" == true ]]; then + scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --prepare-fixtures --from-archive "$RUNNER_TEMP/verify.tar.zst" + else + scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --prepare-fixtures + fi + + - name: Test + id: test + shell: bash + run: | + if [[ "${{ inputs.archive }}" == true ]]; then + scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --from-archive "$RUNNER_TEMP/verify.tar.zst" + else + scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" + fi + + # Equivalent-mutant guards (scripts/verify/guards.toml): a failing + # sentinel is a warning for review, never a failure. + - name: Guard sentinels + if: always() && inputs.platform == 'linux' && matrix.shard == 1 && steps.test.outcome != 'skipped' + continue-on-error: true + shell: bash + run: python3 scripts/verify/guards.py --profile "$CAP_VERIFY_CARGO_PROFILE" + + - name: Summary + if: always() && steps.test.outcome != 'skipped' + shell: bash + run: >- + "$(command -v python3 || command -v python)" scripts/verify/summary.py + --title "${{ inputs.platform }} ${{ inputs.tier }}${{ inputs.shards > 1 && format(' shard {0}/{1}', matrix.shard, inputs.shards) || '' }}" + --junit "$CAP_VERIFY_REPORT_DIR/junit.xml" --flaky "$CAP_VERIFY_FLAKY_FILE" + "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload report + if: always() && steps.test.outcome != 'skipped' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: verify-report-${{ inputs.platform }}-${{ inputs.tier }}-${{ matrix.shard }} + path: ${{ runner.temp }}/verify-report + if-no-files-found: ignore + retention-days: 14 + + - name: Save fixtures + if: always() && inputs.save-cache && steps.setup.outputs.fixtures-hit != 'true' && matrix.shard == 1 + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/verify-fixtures + key: ${{ steps.setup.outputs.fixtures-key }} diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml new file mode 100644 index 00000000000..bf102b78c41 --- /dev/null +++ b/.github/workflows/engine-verify.yml @@ -0,0 +1,306 @@ +name: Engine verify + +# Oracle-based regression suite for the playback, export, recording, editing +# and captions engines (crates/verify, crates/verify-expect, the decoder model +# fuzzer in cap-rendering). See crates/verify/README.md. +# +# Tiers: FULL for every pull request that changes an engine crate +# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the +# `verify-full` label (engine-verify-label.yml), in the merge queue and +# nightly; FAST for other pull requests. Only suites whose +# code a change can reach run (scripts/verify/plan.py maps changed paths to +# suites through the workspace dependency graph), and only on the platforms +# where they test platform code. The "Engine verify" job is the one to require. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + schedule: + - cron: "23 3 * * *" + push: + branches: + - main + - ci/engine-verify + - ci/verify-ci + workflow_dispatch: + inputs: + tier: + description: Tier + type: choice + options: [fast, full] + default: full + suites: + description: "Comma-separated suites (default all, see scripts/verify/suites.json)" + type: string + default: "" + cargo-config: + description: "Extra cargo --config overrides, separated by ; (profile experiments)" + type: string + default: "" + cargo-log: + description: "CARGO_LOG for the build (for example cargo::core::compiler::fingerprint=info)" + type: string + default: "" + base: + description: "Plan from the diff against this ref, like a pull request (scope, risk escalation)" + type: string + default: "" + workflow_call: + inputs: + tier: + type: string + default: full + +permissions: + contents: read + pull-requests: read + +concurrency: + # Manual dispatches each get their own group, so repeats of the same ref + # run side by side instead of cancelling each other. + group: engine-verify-${{ github.event.pull_request.number || github.ref }}-${{ inputs.tier || 'auto' }}${{ github.event_name == 'workflow_dispatch' && format('-{0}', github.run_id) || '' }} + # Superseded PR pushes are cancelled; main and nightly runs finish so their + # caches are saved. + cancel-in-progress: ${{ github.event_name != 'schedule' && github.ref != 'refs/heads/main' }} + +jobs: + plan: + name: Plan + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.plan.outputs.run }} + tier: ${{ steps.escalated.outputs.tier }} + escalate: ${{ steps.risk.outputs.escalate }} + engine: ${{ steps.risk.outputs.engine }} + matrix: ${{ steps.plan.outputs.matrix }} + save-cache: ${{ steps.tier.outputs.save-cache }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check the planner and expectations files + env: + RUSTUP_TOOLCHAIN: stable + run: | + python3 scripts/verify/plan.py test > /dev/null + python3 scripts/verify/test_diff.py + if grep -rn --include='expectations*.toml' '^reason = "TODO' crates; then + echo "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging" + exit 1 + fi + + - name: Tier + id: tier + env: + INPUT_TIER: ${{ inputs.tier }} + EVENT: ${{ github.event_name }} + FULL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'verify-full') }} + run: | + tier=fast + if [[ -n "$INPUT_TIER" ]]; then + tier="$INPUT_TIER" + elif [[ "$EVENT" == schedule || "$EVENT" == merge_group || "$FULL_LABEL" == true ]]; then + tier=full + fi + save=false + # Pushes and manual runs on the suite branches save caches too, so a + # proof batch never runs cold twice (pull requests only restore). + if [[ "$EVENT" == schedule || ( ( "$EVENT" == push || "$EVENT" == workflow_dispatch ) && "$GITHUB_REF" =~ ^refs/heads/(main|ci/engine-verify|ci/verify-ci)$ ) ]]; then + save=true + fi + echo "tier=$tier" >> "$GITHUB_OUTPUT" + echo "save-cache=$save" >> "$GITHUB_OUTPUT" + + - name: Changed files + id: files + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + PR: ${{ github.event.pull_request.number }} + BEFORE: ${{ github.event.before }} + MQ_BASE: ${{ github.event.merge_group.base_sha }} + MQ_HEAD: ${{ github.event.merge_group.head_sha }} + BASE: ${{ inputs.base }} + SUITES: ${{ inputs.suites }} + run: | + set -euo pipefail + files="$RUNNER_TEMP/changed.txt" + scope=files + max=300 + case "$EVENT" in + pull_request) + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$files" + max=3000 ;; + merge_group) + gh api "repos/$GITHUB_REPOSITORY/compare/$MQ_BASE...$MQ_HEAD" --jq '.files[].filename' > "$files" ;; + push) + if [[ "$BEFORE" =~ ^0+$ ]]; then + scope=all + else + gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$GITHUB_SHA" --jq '.files[].filename' > "$files" + fi ;; + *) + if [[ -n "${BASE:-}" ]]; then + gh api "repos/$GITHUB_REPOSITORY/compare/$BASE...$GITHUB_SHA" --jq '.files[].filename' > "$files" + max=3000 + else + scope=all + fi ;; + esac + echo "scope=$scope" >> "$GITHUB_OUTPUT" + echo "max=$max" >> "$GITHUB_OUTPUT" + [[ -f "$files" ]] && { echo "$(wc -l < "$files") changed files"; head -50 "$files"; } || true + + # High-risk code (scripts/verify/guards.toml): warn, and run a pull + # request that touches it at the FULL tier. + - name: Risk + id: risk + if: steps.files.outputs.scope == 'files' + run: python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --summary "$GITHUB_STEP_SUMMARY" --output "$GITHUB_OUTPUT" + + - name: Escalate tier + id: escalated + env: + TIER: ${{ steps.tier.outputs.tier }} + ESCALATE: ${{ steps.risk.outputs.escalate }} + ENGINE: ${{ steps.risk.outputs.engine }} + EVENT: ${{ github.event_name }} + BASE: ${{ inputs.base }} + run: | + tier="$TIER" + if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then + tier=full + echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY" + fi + echo "tier=$tier" >> "$GITHUB_OUTPUT" + + - name: Plan + id: plan + env: + TIER: ${{ steps.escalated.outputs.tier }} + SCOPE: ${{ steps.files.outputs.scope }} + MAX: ${{ steps.files.outputs.max }} + SUITES: ${{ inputs.suites }} + RUSTUP_TOOLCHAIN: stable + # Runner labels live in scripts/verify/suites.json (macOS: + # Blacksmith). A repository variable overrides one without a code + # change, e.g. ENGINE_VERIFY_MACOS_RUNNER=macos-15 for GitHub-hosted. + ENGINE_VERIFY_MACOS_RUNNER: ${{ vars.ENGINE_VERIFY_MACOS_RUNNER }} + ENGINE_VERIFY_LINUX_RUNNER: ${{ vars.ENGINE_VERIFY_LINUX_RUNNER }} + ENGINE_VERIFY_WINDOWS_RUNNER: ${{ vars.ENGINE_VERIFY_WINDOWS_RUNNER }} + run: | + set -euo pipefail + if [[ "$SCOPE" == all ]]; then + python3 scripts/verify/plan.py changes --tier "$TIER" --all ${SUITES:+--suites "$SUITES"} > "$RUNNER_TEMP/plan.txt" + else + python3 scripts/verify/plan.py changes --tier "$TIER" --files "$RUNNER_TEMP/changed.txt" --max-files "$MAX" ${SUITES:+--suites "$SUITES"} > "$RUNNER_TEMP/plan.txt" + fi + cat "$RUNNER_TEMP/plan.txt" + cat "$RUNNER_TEMP/plan.txt" >> "$GITHUB_OUTPUT" + { + echo "## Engine verify plan ($TIER tier)" + grep -E '^(reason|suites)=' "$RUNNER_TEMP/plan.txt" | sed 's/^/- /' + } >> "$GITHUB_STEP_SUMMARY" + + # Labels a risky pull request needs-deep-review. Kept apart from the plan + # job so no step that runs repository code holds a write token; fork pull + # requests get a read-only token and only keep the summary and annotations. + deep-review-label: + name: Deep review label + needs: plan + if: github.event_name == 'pull_request' && needs.plan.outputs.escalate == 'true' + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - name: Label + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" -f "labels[]=needs-deep-review" > /dev/null \ + || echo "::notice::could not add the needs-deep-review label with this token; the summary and annotations still flag the change" + + verify: + name: ${{ matrix.platform }} + needs: plan + if: needs.plan.outputs.run == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + uses: ./.github/workflows/engine-verify-os.yml + with: + platform: ${{ matrix.platform }} + runner: ${{ matrix.runner }} + target: ${{ matrix.target }} + tier: ${{ needs.plan.outputs.tier }} + suites: ${{ matrix.suites }} + shards: ${{ matrix.shards }} + shard-list: ${{ matrix.shard_list }} + archive: ${{ matrix.archive }} + save-cache: ${{ needs.plan.outputs.save-cache == 'true' }} + cargo-config: ${{ inputs.cargo-config || '' }} + cargo-log: ${{ inputs.cargo-log || '' }} + + gate: + name: Engine verify + needs: [plan, verify] + if: always() + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + sparse-checkout: scripts/verify + + - uses: actions/download-artifact@v4 + if: needs.plan.outputs.run == 'true' + with: + pattern: verify-report-* + path: ${{ runner.temp }}/reports + + - name: Verdict + env: + PLAN: ${{ needs.plan.result }} + RUN: ${{ needs.plan.outputs.run }} + VERIFY: ${{ needs.verify.result }} + TIER: ${{ needs.plan.outputs.tier }} + MATRIX: ${{ needs.plan.outputs.matrix }} + run: | + if [[ "$PLAN" != success ]]; then + echo "planning failed" >&2 + exit 1 + fi + if [[ "$RUN" != true ]]; then + echo "No engine code changed: nothing to verify." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if [[ -d "$RUNNER_TEMP/reports" ]]; then + for dir in "$RUNNER_TEMP"/reports/*/; do + name="$(basename "$dir")" + flaky=() + [[ -f "$dir/flaky.txt" ]] && flaky=(--flaky "$dir/flaky.txt") + python3 scripts/verify/summary.py --title "${name#verify-report-}" --junit "$dir/junit.xml" "${flaky[@]}" "$dir" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + done + fi + if [[ "$VERIFY" != success ]]; then + echo "engine verification $VERIFY ($TIER tier): see the platform jobs" >&2 + exit 1 + fi + # Executed-case floor: every planned suite ran at least its known + # number of cases on every platform, and every named test ran. + floor=0 + python3 scripts/verify/floor.py --matrix "$MATRIX" --tier "$TIER" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$? + cat "$RUNNER_TEMP/floor.txt" + { echo "### Executed cases"; echo '```text'; cat "$RUNNER_TEMP/floor.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + if [[ $floor != 0 ]]; then + echo "::error::fewer cases or tests ran than planned (see Executed cases)" + exit 1 + fi + echo "engine verification passed ($TIER tier)" diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 25560b00842..1b6aa450898 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -147,8 +147,10 @@ jobs: cargo test --locked -p cap-camera-effects --lib cargo test --locked -p cap-recording --lib -- --test-threads=1 # --nocapture so a WARP-adapter notch skip prints instead of - # looking identical to a pass in the CI log. - cargo test --locked -p cap-rendering -- --nocapture --skip zoom_spring::tests::precompute_cost_is_bounded_for_long_projects + # looking identical to a pass in the CI log. The decoder model + # fuzzing runs in Engine verify (scripts/verify/suites.json); its + # 2 s decode deadline is not meant for a loaded hosted runner. + cargo test --locked -p cap-rendering -- --nocapture --skip zoom_spring::tests::precompute_cost_is_bounded_for_long_projects --skip decoder::model:: cargo test --locked -p cap-rendering --lib zoom_spring::tests::precompute_cost_is_bounded_for_long_projects -- --exact --nocapture --test-threads=1 - name: Editor audio playback and export regressions diff --git a/.github/workflows/verify-crash.yml b/.github/workflows/verify-crash.yml new file mode 100644 index 00000000000..302f109a1b1 --- /dev/null +++ b/.github/workflows/verify-crash.yml @@ -0,0 +1,47 @@ +name: Verify crash safety + +# The `crash` engine-verify suite (crates/verify/tests/crash.rs) on all three +# platforms: in-process vs out-of-process muxer equality and recorder, muxer +# and finalize kills against what is physically on disk. engine-verify.yml +# runs it like every other suite; this workflow runs it alone on Blacksmith +# runners for crash-safety proof branches. + +on: + push: + branches: + - ci/fixproof-crash* + - scratch/crash-* + workflow_dispatch: + inputs: + tier: + description: Tier + type: choice + options: [fast, full] + default: full + +permissions: + contents: read + +jobs: + verify: + name: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - platform: macos + runner: blacksmith-6vcpu-macos-15 + target: aarch64-apple-darwin + - platform: linux + runner: blacksmith-8vcpu-ubuntu-2404 + target: x86_64-unknown-linux-gnu + - platform: windows + runner: blacksmith-8vcpu-windows-2025 + target: x86_64-pc-windows-msvc + uses: ./.github/workflows/engine-verify-os.yml + with: + platform: ${{ matrix.platform }} + runner: ${{ matrix.runner }} + target: ${{ matrix.target }} + tier: ${{ inputs.tier || 'full' }} + suites: crash diff --git a/.github/workflows/verify-recording.yml b/.github/workflows/verify-recording.yml new file mode 100644 index 00000000000..2ab1e6cfee2 --- /dev/null +++ b/.github/workflows/verify-recording.yml @@ -0,0 +1,374 @@ +name: Verify recording (real capture) + +# Records a marker stimulus through Cap's real Studio and Instant pipelines +# (ScreenCaptureKit + VideoToolbox) on macOS runners and checks every frame +# and audio marker against the host-clock truth. See +# crates/verify/tests/recording.rs. +# +# Every pull request gets a "Verify recording" verdict (the job to require): +# - Scope decides from the diff whether the change reaches the recorder's or +# the stimulus's dependency closure or the harness (scripts/verify/plan.py +# recording); a pull request that does not is green without running; +# - one that does runs the FAST tier on macOS 15, or the FULL tier on macOS +# 15 and 26 when it touches a high-risk region or an engine crate; nightly +# and dispatch run FULL on both. +# +# Runners: GitHub-hosted macOS by default, because real Macs record through +# the VideoToolbox hardware H.264 encoder and Blacksmith macOS VMs have none +# (VTCompressionSessionCreate with RequireHardwareAcceleratedVideoEncoder +# returns -12903, probe run 38847791287), so every recording there takes the +# software (libx264) path. Blacksmith (M4 Pro VM, 6 vCPU, no concurrency cap) +# is a dispatch option (runners=blacksmith or both) that covers that path; it +# needs Screen Recording and Microphone granted in TCC, its display woken, +# and an audio device (BlackHole), which the setup step below does. +on: + workflow_dispatch: + inputs: + repeat: + description: "Runs of each case" + default: "1" + tier: + description: "fast or full" + default: "full" + case_filter: + description: "Case globs, comma-separated (empty runs all)" + default: "" + keep_artifacts: + description: "Preserve recordings and raw crash snapshots for passing cases" + type: boolean + default: false + system_audio: + description: "Blacksmith only: play and require system audio through BlackHole (not yet characterised)" + type: boolean + default: false + runners: + description: "hosted (default), blacksmith (software encoder path) or both" + type: choice + options: [hosted, blacksmith, both] + default: hosted + schedule: + - cron: "41 3 * * *" + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: + - ci/verify-ci + - ci/verify-recording + +concurrency: + group: verify-recording-${{ github.event.pull_request.number || github.ref_name }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && format('-{0}', github.run_id) || '' }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + +permissions: + contents: read + pull-requests: read + +jobs: + scope: + name: Scope + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.scope.outputs.run }} + runners: ${{ steps.scope.outputs.runners }} + tier: ${{ steps.scope.outputs.tier }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Scope + id: scope + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + PR: ${{ github.event.pull_request.number }} + BEFORE: ${{ github.event.before }} + INPUT_TIER: ${{ inputs.tier }} + INPUT_RUNNERS: ${{ inputs.runners }} + RUSTUP_TOOLCHAIN: stable + run: | + set -euo pipefail + files="$RUNNER_TEMP/changed.txt" + hosted='["macos-15","macos-26"]' + case "$EVENT" in + schedule) run=true; runners="$hosted"; tier=full ;; + workflow_dispatch) + run=true + tier="${INPUT_TIER:-full}" + case "${INPUT_RUNNERS:-hosted}" in + blacksmith) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26"]' ;; + both) runners='["blacksmith-6vcpu-macos-15","blacksmith-6vcpu-macos-26","macos-15","macos-26"]' ;; + *) runners="$hosted" ;; + esac ;; + *) + if [[ "$EVENT" == pull_request ]]; then + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$files" + max=3000 + elif [[ "$BEFORE" =~ ^0+$ ]]; then + : > "$files"; echo Cargo.toml >> "$files" + max=300 + else + gh api "repos/$GITHUB_REPOSITORY/compare/$BEFORE...$GITHUB_SHA" --jq '.files[].filename' > "$files" + max=300 + fi + python3 scripts/verify/plan.py recording --files "$files" --max-files "$max" > "$RUNNER_TEMP/scope.txt" + cat "$RUNNER_TEMP/scope.txt" + run="$(sed -n 's/^run=//p' "$RUNNER_TEMP/scope.txt")" + python3 scripts/verify/plan.py risk --files "$files" --summary "$GITHUB_STEP_SUMMARY" --output "$RUNNER_TEMP/risk.txt" + if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt"; then + runners="$hosted"; tier=full + else + runners='["macos-15"]'; tier=fast + fi ;; + esac + { + echo "run=$run" + echo "runners=$runners" + echo "tier=$tier" + } >> "$GITHUB_OUTPUT" + if [[ "$run" == true ]]; then + echo "real capture: $tier tier on $runners" | tee -a "$GITHUB_STEP_SUMMARY" + else + echo "real capture: no recording code changed, nothing to run" | tee -a "$GITHUB_STEP_SUMMARY" + fi + + real-capture: + needs: scope + if: needs.scope.outputs.run == 'true' + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(needs.scope.outputs.runners) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + env: + CAP_VERIFY_CAPTURE: display + CAP_VERIFY_REQUIRE_CAPTURE: "1" + CAP_VERIFY_TIER: ${{ needs.scope.outputs.tier }} + CAP_VERIFY_REPEAT: ${{ inputs.repeat || (github.event_name == 'schedule' && '3') || '1' }} + CAP_VERIFY_CASE: ${{ inputs.case_filter || '' }} + CAP_VERIFY_KEEP: ${{ inputs.keep_artifacts && '1' || '' }} + CAP_VERIFY_SCRATCH: ${{ github.workspace }}/verify-scratch + CAP_VERIFY_REPORT_DIR: ${{ github.workspace }}/verify-report + CAP_VERIFY_DUMP: ${{ github.workspace }}/verify-dump + CAP_VERIFY_HOST: ${{ startsWith(matrix.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }} + # A case this runner cannot run fails, except system audio as the only + # audio while the stimulus stays silent (see below). + CAP_VERIFY_REQUIRE_SUPPORTED: "1" + CAP_VERIFY_UNSUPPORTED_ALLOW: ${{ !(startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio) && 'recording/instant/system-only' || '' }} + # Hosted runners' output devices give ScreenCaptureKit only silence, so + # system audio is not checked there. Blacksmith gets BlackHole (set up + # below), which does carry the stimulus's audio on macOS 15 (not on + # 26), but its system-audio timing is not characterised yet (run + # 38843005408: system audio +31 to +44 ms against the mic, BlackHole + # latency not in the truth), so by default the stimulus stays silent + # there too; dispatch system_audio=true plays and requires it. + CAP_VERIFY_AUDIO_OUTPUT: ${{ startsWith(matrix.runner, 'blacksmith-') && 'BlackHole 2ch' || matrix.runner == 'macos-26' && 'Null Audio Device' || '' }} + CAP_VERIFY_STIMULUS_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && !inputs.system_audio && '0' || '' }} + CAP_VERIFY_REQUIRE_SYSTEM_AUDIO: ${{ startsWith(matrix.runner, 'blacksmith-') && inputs.system_audio && '1' || '' }} + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.88.0 + + - uses: ./.github/actions/setup-rust-cache + with: + target: aarch64-apple-darwin + prefix-key: v1-rust-verify-recording-${{ matrix.runner }} + save-cache: "true" + + - uses: ./.github/actions/install-desktop-deps + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Native dependencies + env: + RUST_TARGET_TRIPLE: aarch64-apple-darwin + run: node scripts/setup.js + + - name: Runner devices + run: | + sw_vers + system_profiler SPDisplaysDataType SPAudioDataType 2>/dev/null | sed -n 1,80p + sysctl hw.memsize hw.ncpu + + # Spotlight indexes the build and every recording the suite writes: + # mds (priority 50) plus a dozen mdworkers averaged 1 to 1.8 of the 3 + # cores during the tests (ps samples, run 38736744971). + - name: Disable Spotlight indexing + run: | + sudo mdutil -a -i off || true + sudo mdutil -as || true + + - name: Build + run: | + start=$(date +%s) + cargo build --locked -p cap-verify-stimulus + cargo test --locked -p cap-verify --test recording --no-run + echo "build seconds: $(( $(date +%s) - start ))" + + # Blacksmith runs a logged-in GUI session with SIP disabled, but: + # - ScreenCaptureKit is denied until TCC grants the responsible process + # (Runner.Worker), so Screen Recording and Microphone are granted to + # every binary that captures plus the whole runner ancestry, in the + # system and the user TCC database (Microphone is a per-user + # service: without it CoreAudio blocks the stimulus forever in + # AudioDeviceCreateIOProcID waiting for coreaudiod), then tccd + # restarts; + # - the display starts asleep (CGDisplayIsAsleep, no active display), + # so ScreenCaptureKit lists windows but no display: wake it and keep + # it awake for the job; + # - there is no audio device: BlackHole provides input and output. + - name: Blacksmith capture permissions and audio device + if: startsWith(matrix.runner, 'blacksmith-') + run: | + set -euo pipefail + sudo pmset -a displaysleep 0 sleep 0 disksleep 0 + caffeinate -u -t 2 + nohup caffeinate -dimsu > /dev/null 2>&1 & + brew install --cask blackhole-2ch + brew install switchaudio-osx + sudo killall coreaudiod || true + for _ in $(seq 1 30); do + SwitchAudioSource -a -t output | grep -qx "BlackHole 2ch" && break + sleep 1 + done + SwitchAudioSource -s "BlackHole 2ch" -t output + SwitchAudioSource -s "BlackHole 2ch" -t input + echo "outputs:"; SwitchAudioSource -a -t output + echo "default output: $(SwitchAudioSource -c -t output)" + + { cargo build --locked -p cap-verify-stimulus --message-format=json 2>/dev/null; \ + cargo test --locked -p cap-verify --test recording --no-run --message-format=json 2>/dev/null; } \ + | python3 -c 'import json, sys + for line in sys.stdin: + m = json.loads(line) + if m.get("reason") == "compiler-artifact" and m.get("executable") and m["target"]["name"] in ("recording", "cap-verify-stimulus"): + print(m["executable"])' > "$RUNNER_TEMP/capture-binaries" + test "$(wc -l < "$RUNNER_TEMP/capture-binaries")" -ge 2 + p=$$ + while [ "$p" -gt 1 ]; do + ps -o comm= -p "$p" + p=$(ps -o ppid= -p "$p" | tr -d ' ') + done >> "$RUNNER_TEMP/capture-binaries" + for c in "$(rustup which cargo)" "$HOME/.cargo/bin/cargo"; do echo "$c"; done >> "$RUNNER_TEMP/capture-binaries" + + db="/Library/Application Support/com.apple.TCC/TCC.db" + user_db="$HOME/Library/Application Support/com.apple.TCC/TCC.db" + approvals="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist" + mkdir -p "$(dirname "$approvals")" + now=$(date +%s) + until=$(date -u -v+1y '+%a %b %d %H:%M:%S GMT %Y') + sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do + for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do + row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" + sudo sqlite3 "$db" "$row" + sqlite3 "$user_db" "$row" + done + /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true + echo "granted $client" + done + killall tccd || true + sudo killall tccd || true + sleep 2 + cat > "$RUNNER_TEMP/display.swift" <<'SWIFT' + import CoreGraphics + let main = CGMainDisplayID() + print("display \(main): asleep \(CGDisplayIsAsleep(main)), active \(CGDisplayIsActive(main)), \(CGDisplayBounds(main)), screen capture \(CGPreflightScreenCaptureAccess())") + SWIFT + swiftc "$RUNNER_TEMP/display.swift" -o "$RUNNER_TEMP/display" + "$RUNNER_TEMP/display" + + - name: Real-capture recording oracle + run: | + mkdir -p "$CAP_VERIFY_DUMP" + # One-second process samples (CPU, scheduler priority) so an + # environment failure can be attributed. + (while sleep 1; do + date -u +%H:%M:%S + ps -A -o pid,pri,%cpu,comm -r | head -14 + for p in $(pgrep -f cap-verify-stimulus | head -1) $(pgrep -f 'deps/recording-' | head -1); do + ps -M -p "$p" | awk 'NR == 1 || $4 > 0.5 || NR < 4' | cut -c1-90 + done + done) > "$CAP_VERIFY_DUMP/ps.log" 2>&1 & + top_pid=$! + start=$(date +%s) + set +e + cargo test --locked -p cap-verify --test recording -- --nocapture + status=$? + set -e + kill "$top_pid" || true + echo "test seconds: $(( $(date +%s) - start ))" + exit "$status" + + - name: Summary + if: always() + run: | + python3 scripts/verify/summary.py --title "real capture ${{ matrix.runner }} ${{ needs.scope.outputs.tier }}" "$CAP_VERIFY_REPORT_DIR" >> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: verify-recording-${{ matrix.runner }} + path: | + ${{ github.workspace }}/verify-report + ${{ github.workspace }}/verify-dump + ${{ github.workspace }}/verify-scratch + if-no-files-found: ignore + + # The check to require: green when the change is out of scope or every + # planned runner passed with at least the tier's case count. + gate: + name: Verify recording + needs: [scope, real-capture] + if: always() + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + sparse-checkout: scripts/verify + + - uses: actions/download-artifact@v4 + if: needs.scope.outputs.run == 'true' + with: + pattern: verify-recording-* + path: ${{ runner.temp }}/reports + + - name: Verdict + env: + SCOPE: ${{ needs.scope.result }} + RUN: ${{ needs.scope.outputs.run }} + CAPTURE: ${{ needs.real-capture.result }} + TIER: ${{ needs.scope.outputs.tier }} + RUNNERS: ${{ needs.scope.outputs.runners }} + CASE_FILTER: ${{ inputs.case_filter }} + run: | + if [[ "$SCOPE" != success ]]; then + echo "scoping failed" >&2 + exit 1 + fi + if [[ "$RUN" != true ]]; then + echo "No recording code changed: nothing to verify." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if [[ "$CAPTURE" != success ]]; then + echo "real capture $CAPTURE ($TIER tier): see the runner jobs" >&2 + exit 1 + fi + if [[ -n "$CASE_FILTER" ]]; then + echo "case filter set: executed-case floor not applied" + exit 0 + fi + floor=0 + python3 scripts/verify/floor.py --recording --tier "$TIER" --runners "$RUNNERS" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$? + cat "$RUNNER_TEMP/floor.txt" | tee -a "$GITHUB_STEP_SUMMARY" + if [[ $floor != 0 ]]; then + echo "::error::fewer real-capture cases ran than planned" + exit 1 + fi diff --git a/Cargo.lock b/Cargo.lock index f059b7eff1e..2e6a9faee11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2019,6 +2019,7 @@ dependencies = [ "cap-flags", "cap-media-info", "cap-project", + "cap-verify-expect", "cap-video-decode", "cidre", "clap", @@ -2150,6 +2151,62 @@ dependencies = [ "zbus", ] +[[package]] +name = "cap-verify" +version = "0.1.0" +dependencies = [ + "anyhow", + "cap-audio", + "cap-editor", + "cap-enc-avfoundation", + "cap-export", + "cap-media-info", + "cap-project", + "cap-recording", + "cap-rendering", + "cap-timestamp", + "cap-verify-expect", + "cidre", + "ffmpeg-next", + "flume", + "futures", + "relative-path", + "rustfft", + "scap-targets", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing-subscriber", + "workspace-hack", +] + +[[package]] +name = "cap-verify-expect" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "toml 0.8.2", +] + +[[package]] +name = "cap-verify-stimulus" +version = "0.1.0" +dependencies = [ + "anyhow", + "cap-verify", + "cidre", + "cpal 0.15.3 (git+https://github.com/CapSoftware/cpal?rev=6013cb5f8bd3)", + "futures", + "objc2-foundation 0.3.1", + "scap-screencapturekit", + "serde_json", + "softbuffer", + "winit", + "workspace-hack", +] + [[package]] name = "cap-video-decode" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 8a9be0ffb64..a65433c7cc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,26 @@ opt-level = 2 [profile.dev.package.cap-audio] opt-level = 2 +# The verification suite decodes markers from every output pixel it reads; +# unoptimised it spends minutes in its own loops. +[profile.dev.package.cap-verify] +opt-level = 3 + +# Engine verification suite in CI (.github/workflows/engine-verify.yml). +# Dependencies come from the Rust cache, so optimising them costs nothing on +# a warm run and speeds up the cases; workspace crates rebuild on every run +# and stay cheap to compile. No debuginfo or incremental state to cache. +[profile.ci-verify] +inherits = "dev" +debug = 0 +incremental = false + +[profile.ci-verify.package."*"] +opt-level = 1 + +[profile.ci-verify.package.cap-verify] +opt-level = 3 + # Optimize for smaller binary size [profile.release] panic = "unwind" diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index 4d2ac0436be..469e8a14692 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -1547,6 +1547,7 @@ dependencies = [ "cap-rendering", "cap-timestamp", "cap-utils", + "cap-verify", "chrono", "cidre", "clipboard-rs", @@ -1584,6 +1585,7 @@ dependencies = [ "sha2", "smallvec", "tauri-winres", + "tempfile", "tiny-skia", "tokio", "tracing", @@ -2001,6 +2003,38 @@ dependencies = [ "zbus", ] +[[package]] +name = "cap-verify" +version = "0.1.0" +dependencies = [ + "anyhow", + "cap-enc-avfoundation", + "cap-media-info", + "cap-project", + "cap-recording", + "cap-timestamp", + "cap-verify-expect", + "cidre", + "ffmpeg-next", + "flume 0.11.1", + "futures", + "relative-path", + "rustfft", + "serde", + "serde_json", + "tokio", + "workspace-hack", +] + +[[package]] +name = "cap-verify-expect" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "toml 0.8.23", +] + [[package]] name = "cap-video-decode" version = "0.1.0" diff --git a/apps/desktop-gpui/Cargo.toml b/apps/desktop-gpui/Cargo.toml index c803d72db27..50fccce29ad 100644 --- a/apps/desktop-gpui/Cargo.toml +++ b/apps/desktop-gpui/Cargo.toml @@ -169,7 +169,11 @@ tauri-winres = "0.3.5" # decoders, renderer and preview tasks are all tokio-spawned -- a current-thread # runtime would deadlock the moment the test thread blocks on a frame. Kept in # dev-dependencies so the shipped binary's tokio features are unchanged. -tokio = { version = "1", features = ["rt-multi-thread", "time", "test-util"] } +tokio = { version = "1", features = ["rt-multi-thread", "time", "test-util", "net", "io-util", "macros"] } +# Caption oracles (`src/transcription/verify_tests.rs`): fixture projects and +# audio markers for the fake transcriber. +cap-verify = { path = "../../crates/verify" } +tempfile = "3" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/apps/desktop-gpui/src/store.rs b/apps/desktop-gpui/src/store.rs index 52dc02a8b4d..9c75aec9d12 100644 --- a/apps/desktop-gpui/src/store.rs +++ b/apps/desktop-gpui/src/store.rs @@ -338,6 +338,11 @@ thread_local! { }; } +#[cfg(test)] +pub(crate) fn set_test_tauri_store_path(path: Option) { + TEST_TAURI_STORE_PATH.with(|current| *current.borrow_mut() = path); +} + /// The tauri-plugin-store file, shared with the shipping app. /// /// `CAP_GPUI_TAURI_STORE` redirects it at a copy, which is how the tests -- diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index f75b189e1d5..3240295a91b 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -1613,10 +1613,22 @@ fn process_with_parakeet( tracing::info!("Parakeet produced {} timed tokens", result.tokens.len()); + caption_segments_from_tokens(result.tokens.iter().map(|token| CaptionWord { + text: token.text.clone(), + start: token.start, + end: token.end, + })) +} + +/// Parakeet's timed word tokens -> caption segments: drop empty tokens, +/// normalize the words and chunk them. +#[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))] +fn caption_segments_from_tokens( + tokens: impl IntoIterator, +) -> Result, String> { let words = normalize_caption_words( - result - .tokens - .iter() + tokens + .into_iter() .filter(|token| !token.text.trim().is_empty()) .map(|token| CaptionWord { text: token.text.trim().to_string(), @@ -1679,32 +1691,42 @@ pub fn transcribe_blocking( )); } - let temp_dir = std::env::temp_dir().join(format!( - "cap-gpui-transcription-{}-{}", - std::process::id(), - TEMP_COUNTER.fetch_add(1, Ordering::Relaxed) - )); - std::fs::create_dir_all(&temp_dir) - .map_err(|e| format!("Failed to create temporary directory: {e}"))?; - let audio_path = temp_dir.join("audio.wav"); - - let result = (|| { - extract_audio_from_video(project_path, &audio_path)?; - + transcribe_project_with(project_path, |audio_path| { let _guard = lock_transcription_worker_slot(); if is_parakeet_model(model) { tracing::info!("Using Parakeet TDT engine"); - let result = process_with_parakeet(&audio_path, model_file.to_string_lossy().as_ref()); + let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()); schedule_parakeet_release(); result } else { tracing::info!("Using Whisper engine"); let hints = transcription_hints(); let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; - let result = process_with_whisper(&audio_path, context, language, &hints); + let result = process_with_whisper(audio_path, context, language, &hints); release_whisper_context_after_transcription(); result } + }) +} + +/// Extracts the project's transcription audio to a temporary 16 kHz mono WAV +/// and runs `engine` on it. +fn transcribe_project_with( + project_path: &Path, + engine: impl FnOnce(&Path) -> Result, String>, +) -> Result, String> { + let temp_dir = std::env::temp_dir().join(format!( + "cap-gpui-transcription-{}-{}", + std::process::id(), + TEMP_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&temp_dir) + .map_err(|e| format!("Failed to create temporary directory: {e}"))?; + let audio_path = temp_dir.join("audio.wav"); + + let result = (|| { + extract_audio_from_video(project_path, &audio_path)?; + engine(&audio_path) })(); let _ = std::fs::remove_dir_all(&temp_dir); @@ -2175,6 +2197,9 @@ pub fn caption_generation_error_message(message: &str) -> String { message.to_string() } +#[cfg(test)] +mod verify_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rs new file mode 100644 index 00000000000..39a2af3bec2 --- /dev/null +++ b/apps/desktop-gpui/src/transcription/verify_tests.rs @@ -0,0 +1,652 @@ +//! cap-verify oracles for caption generation, with a fake transcriber in +//! place of Whisper/Parakeet (no model download): +//! +//! - `fake_transcriber_*`: the real project audio extraction +//! ([`transcribe_project_with`]) feeds a fake engine that hears the +//! fixture's audio markers in the 16 kHz WAV and returns Parakeet-style +//! timed tokens (one word per marker, some followed by a punctuation token +//! and empty tokens). Every word must sit at its source time on the +//! concatenated recordings, and the caption segments built by the Parakeet +//! path ([`caption_segments_from_tokens`]) must match the reference +//! segmentation below. Multi-segment Studio projects with per-track start +//! offsets, legacy single-segment Studio projects, and Instant projects +//! (refused: Instant recordings are transcribed by the web app). +//! - `segmentation_*`: random token streams through the Parakeet path +//! against the reference segmentation. +//! +//! Reference segmentation (the documented caption rules, written for the +//! test): +//! - Empty tokens are dropped; a token starting with attaching punctuation +//! (`,.!?;:%)]}'’` and their CJK forms) joins the previous word and extends +//! its end. Word ends are clamped to `start + 2.5 s`. +//! - Words are grouped six to a caption; a caption whose last word is weak +//! (one letter, or a short function word such as "of", "to", "is") takes +//! more words, up to eight; when one or two words would be left over and +//! the first of them is weak, they join the last caption. Pauses do not +//! split captions. +//! - A caption spans its first word's start to its last word's end; its text +//! joins the words with spaces, except before attaching punctuation; ids +//! are `segment-`. + +use cap_verify::{ + audio_code::{self, AudioSource, MARKER_SECS}, + catalog, + fixture::{self, AudioTrackSpec, FixtureSpec, Rng, SegmentSpec, Timing, VideoTrackSpec}, + media, + report::{self, Case, Tier}, +}; + +use super::*; + +const MAX_WORD_SECS: f32 = 2.5; +const WORD_TOLERANCE_SECS: f64 = 0.0015; +const MARKER_RATE: u32 = 16_000; + +fn finish(case: Case) { + for note in &case.notes { + eprintln!("[cap-verify] {}: {note}", case.id); + } + if let Err(message) = report::settle(case) { + panic!("{message}"); + } +} + +const ATTACHING: &str = ",.!?;:%)]}'’、。！？；：，"; +const WEAK: [&str; 20] = [ + "an", "as", "at", "be", "by", "do", "he", "if", "in", "is", "it", "me", "my", "of", "on", "or", + "so", "to", "up", "we", +]; + +fn attaches(text: &str) -> bool { + text.trim() + .chars() + .next() + .is_some_and(|c| ATTACHING.contains(c)) +} + +fn weak(text: &str) -> bool { + let letters: String = text + .chars() + .filter(|c| c.is_alphanumeric()) + .collect::() + .to_lowercase(); + letters.chars().count() <= 1 || WEAK.contains(&letters.as_str()) +} + +fn reference_words(tokens: &[CaptionWord]) -> Vec { + let mut words: Vec = Vec::new(); + for token in tokens { + let text = token.text.trim(); + if text.is_empty() { + continue; + } + match words.last_mut() { + Some(previous) if attaches(text) => { + previous.text.push_str(text); + previous.end = token.end; + } + _ => words.push(CaptionWord { + text: text.to_string(), + start: token.start, + end: token.end, + }), + } + } + for word in &mut words { + word.end = word.end.min(word.start + MAX_WORD_SECS); + } + words +} + +fn reference_text(words: &[CaptionWord]) -> String { + let mut text = String::new(); + for word in words { + if !text.is_empty() && !attaches(&word.text) { + text.push(' '); + } + text.push_str(word.text.trim()); + } + text +} + +fn reference_segments(tokens: &[CaptionWord]) -> Vec { + let words = reference_words(tokens); + let mut groups: Vec<&[CaptionWord]> = Vec::new(); + let mut rest = &words[..]; + while !rest.is_empty() { + if rest.len() <= 6 { + groups.push(rest); + break; + } + let mut take = 6; + while take < 8 && take < rest.len() && weak(&rest[take - 1].text) { + take += 1; + } + let left = rest.len() - take; + if (1..3).contains(&left) && weak(&rest[take].text) { + take = rest.len(); + } + groups.push(&rest[..take]); + rest = &rest[take..]; + } + groups + .into_iter() + .enumerate() + .map(|(i, group)| CaptionSegment { + id: format!("segment-{i}"), + start: group[0].start, + end: group[group.len() - 1].end, + text: reference_text(group), + words: group.to_vec(), + }) + .collect() +} + +fn describe(segment: &CaptionSegment) -> String { + format!( + "{} {:.3}-{:.3} \"{}\"", + segment.id, segment.start, segment.end, segment.text + ) +} + +/// Differences between the engine's caption segments and the reference's, +/// with times compared to `tolerance` seconds. +fn compare_segments( + product: &[CaptionSegment], + reference: &[CaptionSegment], + tolerance: f64, +) -> Vec { + let close = |a: f32, b: f32| (f64::from(a) - f64::from(b)).abs() <= tolerance; + let mut problems = Vec::new(); + if product.len() != reference.len() { + problems.push(format!( + "{} caption segments, the reference has {}", + product.len(), + reference.len() + )); + } + for (p, r) in product.iter().zip(reference) { + let same = p.id == r.id + && p.text == r.text + && close(p.start, r.start) + && close(p.end, r.end) + && p.words.len() == r.words.len() + && p.words + .iter() + .zip(&r.words) + .all(|(a, b)| a.text == b.text && close(a.start, b.start) && close(a.end, b.end)); + if !same { + problems.push(format!("engine {}, reference {}", describe(p), describe(r))); + } + } + problems +} + +fn marker_word(source: AudioSource, index: u32) -> String { + format!("{}{index}", source.name()) +} + +/// Parakeet-style tokens for the markers heard in `audio_path`: one word per +/// marker; every third word is followed by a "," token and an empty token. +fn fake_tokens(audio_path: &Path) -> Result, String> { + let audio = media::decode_audio(audio_path, Some(MARKER_RATE)) + .map_err(|e| format!("fake transcriber could not read the WAV: {e}"))?; + let mono = audio.mono(); + let mut heard: Vec<(f64, String)> = [AudioSource::Mic, AudioSource::System] + .into_iter() + .flat_map(|source| { + audio_code::locate_markers(&mono, MARKER_RATE, source) + .into_iter() + .filter_map(move |m| { + Some(( + audio.start_secs + m.time_secs, + marker_word(source, m.index?), + )) + }) + }) + .collect(); + heard.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut tokens = Vec::new(); + for (i, (time, text)) in heard.into_iter().enumerate() { + let end = time + MARKER_SECS; + tokens.push(CaptionWord { + text: format!(" {text}"), + start: time as f32, + end: end as f32, + }); + if i % 3 == 2 { + tokens.push(CaptionWord { + text: ",".into(), + start: end as f32, + end: (end + 0.01) as f32, + }); + tokens.push(CaptionWord { + text: " ".into(), + start: (end + 0.01) as f32, + end: (end + 0.02) as f32, + }); + } + } + Ok(tokens) +} + +/// Expected takes of a fixture project: display duration, (audio file, +/// source, start time) per track, latest track start. +type Takes = Vec<(f64, Vec<(PathBuf, AudioSource, f64)>, f64)>; + +/// Where each marker must be heard: a track sample at file time `m` +/// happened at `track start + m` on the recording clock; a take's +/// transcription time 0 is its latest track start and takes follow each +/// other by display duration (as the container reports it). Returns +/// `(word, time)` sorted by time. +fn expected_marker_words(project: &Path, takes: &Takes) -> Vec<(String, f64)> { + let mut out = Vec::new(); + let mut take_start = 0.0; + for (display_duration, tracks, latest) in takes { + for (path, source, start_time) in tracks { + let decoded = media::decode_audio(&project.join(path), Some(48_000)) + .expect("decode source audio"); + for marker in audio_code::locate_markers(&decoded.mono(), 48_000, *source) { + let Some(index) = marker.index else { continue }; + let t = start_time + decoded.start_secs + marker.time_secs - latest; + if t >= 0.0 && t + MARKER_SECS <= *display_duration { + out.push((marker_word(*source, index), take_start + t)); + } + } + } + take_start += display_duration; + } + out.sort_by(|a, b| a.1.total_cmp(&b.1)); + out +} + +fn multi_segment_takes(project: &Path, spec: &FixtureSpec) -> Takes { + let meta = RecordingMeta::load_for_project(project).expect("meta"); + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + panic!("multi-segment fixture"); + }; + inner + .segments + .iter() + .zip(&spec.segments) + .map(|(segment, seg_spec)| { + let display = Video::new(segment.display.path.to_path(project), 0.0) + .expect("display") + .duration; + let mut tracks = Vec::new(); + let mut starts = vec![seg_spec.display.start_time]; + for (meta, spec) in [ + (&segment.mic, &seg_spec.mic), + (&segment.system_audio, &seg_spec.system_audio), + ] { + if let (Some(meta), Some(spec)) = (meta, spec) { + tracks.push(( + PathBuf::from(meta.path.as_str()), + spec.source, + spec.start_time, + )); + starts.push(spec.start_time); + } + } + let latest = starts.into_iter().fold(f64::NEG_INFINITY, f64::max); + (display, tracks, latest) + }) + .collect() +} + +fn takes_fixture() -> FixtureSpec { + let screen = |duration: f64, start_time: f64| { + let mut v = VideoTrackSpec::new( + catalog::SCREEN_W, + catalog::SCREEN_H, + 30, + Timing::Constant { duration }, + ); + v.start_time = start_time; + v + }; + let audio = |source: AudioSource, duration: f64, start_time: f64| AudioTrackSpec { + source, + sample_rate: 48_000, + channels: if source == AudioSource::System { 2 } else { 1 }, + duration, + start_time, + }; + FixtureSpec { + name: "captions-takes".into(), + segments: vec![ + SegmentSpec { + display: screen(5.0, 0.05), + camera: None, + mic: Some(audio(AudioSource::Mic, 5.0, 0.25)), + system_audio: Some(audio(AudioSource::System, 5.2, -0.1)), + }, + SegmentSpec { + display: screen(4.0, 0.0), + camera: None, + mic: Some(audio(AudioSource::Mic, 4.0, 0.0)), + system_audio: None, + }, + ], + } +} + +/// Runs the real extraction with the fake engine and checks words and +/// segments. +fn check_project(case: &mut Case, project: &Path, takes: &Takes) { + let mut tokens = Vec::new(); + let result = transcribe_project_with(project, |audio_path| { + tokens = fake_tokens(audio_path)?; + caption_segments_from_tokens(tokens.clone()) + }); + let segments = match result { + Ok(segments) => segments, + Err(e) => { + case.problem("error", format!("transcription failed: {e}")); + return; + } + }; + let expected = expected_marker_words(project, takes); + let words: Vec<&CaptionWord> = segments.iter().flat_map(|s| &s.words).collect(); + let mut problems = Vec::new(); + let mut worst = 0.0f64; + for (text, time) in &expected { + let hits: Vec<&&CaptionWord> = words + .iter() + .filter(|w| { + w.text.trim_end_matches(',') == text.as_str() + && (f64::from(w.start) - time).abs() < 0.05 + }) + .collect(); + match hits.as_slice() { + [w] => { + let error = f64::from(w.start) - time; + worst = worst.max(error.abs()); + if error.abs() > WORD_TOLERANCE_SECS { + problems.push(format!("{text} at {:.4}s, expected {time:.4}s", w.start)); + } + } + [] => problems.push(format!("{text} missing (expected at {time:.4}s)")), + _ => problems.push(format!("{text} heard {} times", hits.len())), + } + } + if words.len() != expected.len() { + problems.push(format!( + "{} words, {} expected", + words.len(), + expected.len() + )); + } + if expected.len() < 8 { + problems.push(format!("only {} reference words", expected.len())); + } + if problems.is_empty() { + case.note(format!( + "{} words at their source times (worst {:.3} ms)", + words.len(), + worst * 1000.0 + )); + } else { + case.problem( + "caption-words", + format!( + "{} word problems (first: {})", + problems.len(), + problems + .iter() + .take(4) + .cloned() + .collect::>() + .join("; ") + ), + ); + } + let segment_problems = compare_segments(&segments, &reference_segments(&tokens), 1e-6); + if segment_problems.is_empty() { + case.note(format!( + "{} caption segments match the reference", + segments.len() + )); + } else { + case.problem( + "caption-segments", + format!( + "{} segment problems (first: {})", + segment_problems.len(), + segment_problems[0] + ), + ); + } +} + +#[test] +fn fake_transcriber_multi_segment_studio() { + let id = "captions/app/transcribe/studio-takes"; + if !report::selected(id) { + return; + } + let spec = takes_fixture(); + let project = fixture::ensure_fixture(&spec, fixture::write_fixture).expect("fixture"); + let mut case = Case::new(id); + let takes = multi_segment_takes(&project, &spec); + check_project(&mut case, &project, &takes); + finish(case); +} + +#[test] +fn fake_transcriber_single_segment_studio() { + let id = "captions/app/transcribe/studio-single-segment"; + if !report::selected(id) { + return; + } + let spec = catalog::basic(report::tier()); + let source = fixture::ensure_fixture(&spec, fixture::write_fixture).expect("fixture"); + let dir = tempfile::tempdir().expect("temp dir"); + let project = dir.path().join("single.cap"); + let segment_dir = project.join("content"); + std::fs::create_dir_all(&segment_dir).expect("mkdir"); + let meta = RecordingMeta::load_for_project(&source).expect("meta"); + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + panic!("multi-segment fixture"); + }; + let segment = &inner.segments[0]; + let mic = segment.mic.as_ref().expect("mic"); + for (from, to) in [ + (segment.display.path.as_str(), "content/display.mp4"), + (mic.path.as_str(), "content/audio-input.ogg"), + ] { + std::fs::copy(source.join(from), project.join(to)).expect("copy track"); + } + let single = serde_json::json!({ + "pretty_name": "single", + "display": { "path": "content/display.mp4", "fps": 30 }, + "audio": { "path": "content/audio-input.ogg" } + }); + std::fs::write( + project.join("recording-meta.json"), + serde_json::to_string(&single).expect("meta json"), + ) + .expect("write meta"); + let mut case = Case::new(id); + match RecordingMeta::load_for_project(&project) { + Ok(meta) + if matches!( + meta.studio_meta(), + Some(StudioRecordingMeta::SingleSegment { .. }) + ) => {} + other => case.problem( + "error", + format!( + "the single-segment project did not load as one: {:?}", + other.err() + ), + ), + } + let display = Video::new(project.join("content/display.mp4"), 0.0) + .expect("display") + .duration; + let takes = vec![( + display, + vec![( + PathBuf::from("content/audio-input.ogg"), + AudioSource::Mic, + 0.0, + )], + 0.0, + )]; + check_project(&mut case, &project, &takes); + finish(case); +} + +#[test] +fn instant_project_is_refused() { + let id = "captions/app/transcribe/instant"; + if !report::selected(id) { + return; + } + let dir = tempfile::tempdir().expect("temp dir"); + let project = dir.path().join("instant.cap"); + std::fs::create_dir_all(project.join("content")).expect("mkdir"); + let meta = serde_json::json!({ + "pretty_name": "instant", + "fps": 30, + "sample_rate": 48000 + }); + std::fs::write( + project.join("recording-meta.json"), + serde_json::to_string(&meta).expect("meta json"), + ) + .expect("write meta"); + let mut case = Case::new(id); + match RecordingMeta::load_for_project(&project) { + Ok(meta) if matches!(meta.inner, cap_project::RecordingMetaInner::Instant(_)) => {} + other => case.problem( + "error", + format!("the Instant project did not load as one: {:?}", other.err()), + ), + } + let mut called = false; + let result = transcribe_project_with(&project, |_| { + called = true; + Ok(Vec::new()) + }); + match result { + Err(message) if !called && message.contains("Only studio recordings") => { + case.note(format!("refused before the engine ran: {message}")); + } + other => case.problem( + "error", + format!("Instant project: engine called {called}, result {other:?}"), + ), + } + finish(case); +} + +const ASCII_WORDS: [&str; 22] = [ + "hello", "world", "is", "a", "river", "cloud", "it", "to", "so", "screen", "record", "of", + "we", "on", "capture", "edit", "timeline", "in", "up", "my", "an", "I", +]; +/// Accented and CJK words, and "à", a one-letter (weak) French preposition +/// outside ASCII. +const UNICODE_WORDS: [&str; 5] = ["über", "café", "日本", "à", "ça"]; +const PUNCTUATION: [&str; 9] = [",", ".", "?", "!", "'s", "%", "。", "、", ")"]; + +fn random_tokens(rng: &mut Rng, unicode: bool) -> Vec { + let mut tokens = Vec::new(); + let mut t = rng.f64() * 0.5; + for _ in 0..rng.range(1, 60) { + let length = if rng.range(0, 14) == 0 { + 3.0 + rng.f64() * 5.0 + } else { + 0.05 + rng.f64() * 0.6 + }; + let text = match rng.range(0, 9) { + 0 | 1 => PUNCTUATION[rng.range(0, PUNCTUATION.len() as u64 - 1) as usize].to_string(), + 2 => [" ", "", " "][rng.range(0, 2) as usize].to_string(), + 3 if unicode => { + format!( + " {}", + UNICODE_WORDS[rng.range(0, UNICODE_WORDS.len() as u64 - 1) as usize] + ) + } + _ => format!( + " {}", + ASCII_WORDS[rng.range(0, ASCII_WORDS.len() as u64 - 1) as usize] + ), + }; + tokens.push(CaptionWord { + text, + start: t as f32, + end: (t + length) as f32, + }); + t += length + + match rng.range(0, 3) { + 0 => 0.0, + 1 => rng.f64() * 0.3, + 2 => rng.f64() * 1.0, + _ => rng.f64() * 8.0, + }; + } + tokens +} + +fn segmentation_case(id: &str, unicode: bool) { + if !report::selected(id) { + return; + } + let mut case = Case::new(id); + let seeds = match report::tier() { + Tier::Fast => 2000, + Tier::Full => 50_000, + }; + let (mut failed, mut segments, mut empty) = (0, 0usize, 0); + for seed in 0..seeds { + let tokens = random_tokens( + &mut Rng::new(0x5E6_0000 ^ (seed << 1) ^ u64::from(unicode)), + unicode, + ); + let reference = reference_segments(&tokens); + match caption_segments_from_tokens(tokens.clone()) { + Err(e) if reference.is_empty() && e.contains("No speech") => empty += 1, + Err(e) => { + failed += 1; + case.problem( + "caption-segments", + format!("seed {seed}: unexpected error {e}"), + ); + } + Ok(product) => { + segments += product.len(); + let problems = compare_segments(&product, &reference, 0.0); + if !problems.is_empty() { + failed += 1; + if failed <= 3 { + case.problem( + "caption-segments", + format!( + "seed {seed}: {} problems (first: {}); tokens {:?}", + problems.len(), + problems[0], + tokens.iter().map(|t| t.text.as_str()).collect::>() + ), + ); + } + } + } + } + } + case.note(format!( + "{seeds} token streams, {segments} segments, {empty} without speech, {failed} failed" + )); + finish(case); +} + +#[test] +fn segmentation_matches_reference() { + segmentation_case("captions/app/segmentation/ascii", false); +} + +#[test] +fn segmentation_matches_reference_unicode() { + segmentation_case("captions/app/segmentation/unicode", true); +} diff --git a/apps/desktop-gpui/src/upload.rs b/apps/desktop-gpui/src/upload.rs index 58a36374aed..025f464ab8f 100644 --- a/apps/desktop-gpui/src/upload.rs +++ b/apps/desktop-gpui/src/upload.rs @@ -18,6 +18,8 @@ use crate::auth::{self, AuthApiError}; #[cfg(test)] mod multipart_tests; pub(crate) mod queue; +#[cfg(test)] +mod verify_tests; const MIN_CHUNK_SIZE: u64 = 5 * 1024 * 1024; const MAX_CHUNK_SIZE: u64 = 15 * 1024 * 1024; @@ -698,7 +700,7 @@ async fn upload_segments( let mut manifest = SegmentUploadManifest::default(); let mut uploads = FuturesUnordered::new(); let mut events_closed = false; - let mut last_manifest_upload: Option = None; + let mut last_manifest_upload: Option = None; let mut preparation = Preparation::default(); let mut preparation_enabled = true; let mut preparation_batch = Vec::new(); @@ -853,7 +855,7 @@ async fn upload_segments( if let Err(error) = uploaded { tracing::warn!("Instant progress manifest upload failed; retrying at the next interval: {error}"); } - last_manifest_upload = Some(Instant::now()); + last_manifest_upload = Some(tokio::time::Instant::now()); } } else => break, diff --git a/apps/desktop-gpui/src/upload/queue.rs b/apps/desktop-gpui/src/upload/queue.rs index 40733877c28..6cf371979e4 100644 --- a/apps/desktop-gpui/src/upload/queue.rs +++ b/apps/desktop-gpui/src/upload/queue.rs @@ -1110,6 +1110,9 @@ pub(crate) async fn delete_recording(project: PathBuf) -> Result<(), String> { crate::library::delete_recording_directory(&project) } +#[cfg(test)] +mod verify_resume_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/apps/desktop-gpui/src/upload/queue/verify_resume_tests.rs b/apps/desktop-gpui/src/upload/queue/verify_resume_tests.rs new file mode 100644 index 00000000000..5b2e8de53ca --- /dev/null +++ b/apps/desktop-gpui/src/upload/queue/verify_resume_tests.rs @@ -0,0 +1,209 @@ +//! Quit and relaunch through the upload queue's persisted state, against the +//! mock Cap API from `upload::verify_tests` on a virtual clock. + +use super::*; +use crate::upload::verify_tests::{ + Fault, MockCap, OWNER, Route, SAMPLE_MP4, Scratch, assert_segments_stored, budget, + check_budget, instant_tracks, rule, virtual_time, write_thumbnail, +}; + +const VIDEO_ID: &str = "resumeverify01"; +const PAIRS: u32 = 6; + +fn saved_instant_project( + root: &Path, + server: &str, +) -> ( + PathBuf, + Vec, + Vec, +) { + let project = root.join("instant.cap"); + std::fs::create_dir_all(project.join("content")).unwrap(); + std::fs::write(project.join("content/output.mp4"), SAMPLE_MP4).unwrap(); + let (video, audio) = instant_tracks(&project, PAIRS); + write_thumbnail(&project.join("screenshots/display.jpg")); + let video_info = VideoUploadInfo { + id: VIDEO_ID.into(), + link: format!("{server}/s/{VIDEO_ID}"), + config: S3UploadMeta { + id: VIDEO_ID.into(), + }, + }; + RecordingMeta { + platform: None, + project_path: project.clone(), + pretty_name: "Resume verify".into(), + sharing: Some(SharingMeta { + id: VIDEO_ID.into(), + link: video_info.link.clone(), + content_hash: None, + }), + inner: RecordingMetaInner::Instant(InstantRecordingMeta::Complete { + fps: 30, + sample_rate: Some(48_000), + }), + upload: Some(UploadMeta::SegmentUpload { + video_id: VIDEO_ID.into(), + pre_created_video: video_info, + recording_dir: project.clone(), + }), + } + .save_for_project() + .unwrap(); + write_state( + &project, + &UploadState { + version: 1, + video_id: VIDEO_ID.into(), + server_url: server.into(), + owner_id: Some(OWNER.into()), + kind: UploadKind::Segments, + requested_audio: Some(true), + preserve_local: false, + phase: UploadPhase::Pending, + attempt_count: 0, + last_error: None, + next_retry_at: None, + verification: None, + receipt: None, + }, + ) + .unwrap(); + (project, video, audio) +} + +/// What the queue does for a pending recording at launch. +async fn launch(manager: &Arc>, project: &Path) { + let ownership = UploadLock::acquire(project).unwrap(); + let project = ownership.project_path().to_path_buf(); + let state = read_state(&project).unwrap().unwrap(); + assert!(state.pending(now(), false), "not resumable: {state:?}"); + manager.resume(project, state, ownership).await.unwrap(); +} + +async fn settle(manager: &Manager) { + loop { + if manager + .jobs + .lock() + .await + .values() + .all(|job| job.task.is_finished()) + { + manager.reap().await; + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +fn assert_verified( + mock: &MockCap, + project: &Path, + video: &[SegmentCompletedEvent], + audio: &[SegmentCompletedEvent], +) { + let state = read_state(project).unwrap().unwrap(); + assert_eq!(state.phase, UploadPhase::Verified, "{state:?}"); + assert!(state.receipt.is_some() && state.last_error.is_none()); + assert!(matches!( + RecordingMeta::load_for_project(project).unwrap().upload, + Some(UploadMeta::Complete) + )); + assert_segments_stored(mock, VIDEO_ID, video, audio); + assert!( + mock.object(&format!("{VIDEO_ID}/screenshot/screen-capture.jpg")) + .is_some() + ); + assert!(project.join("content/output.mp4").exists()); + mock.assert_clean(); +} + +/// The app quits while a saved Instant recording is half uploaded by the +/// queue, then relaunches. The relaunch resumes from `instant-upload.json` +/// with the same video id and verifies; it sends each object at most once. +#[test] +fn saved_instant_upload_resumes_after_quit_mid_transfer() { + let case = "queue_resume_after_quit_mid_transfer"; + let scratch = Scratch::new(case); + virtual_time(async { + let mock = MockCap::start(vec![rule( + Route::Object("segments/video/segment_002"), + Fault::Trip(0.5), + )]) + .await; + scratch.sign_in(&mock.url); + let (project, video, audio) = saved_instant_project(&scratch.0, &mock.url); + + let first = Arc::new(Manager::new(LiveBackend, CONFIRMATION_TIMEOUT)); + launch(&first, &project).await; + mock.tripped().await; + first.shutdown().await; + let state = read_state(&project).unwrap().unwrap(); + assert_eq!(state.phase, UploadPhase::Uploading, "{state:?}"); + assert!(mock.confirmations().is_empty()); + assert_eq!(mock.deletes(), 0); + + mock.drain().await; + mock.new_session(); + let relaunched = Arc::new(Manager::new(LiveBackend, CONFIRMATION_TIMEOUT)); + launch(&relaunched, &project).await; + settle(&relaunched).await; + assert_verified(&mock, &project, &video, &audio); + assert_eq!(mock.confirmations().len(), 1); + + let stats = mock.stats(); + let media: u64 = video + .iter() + .chain(&audio) + .map(|event| event.file_size) + .sum(); + let thumbnail = mock + .object(&format!("{VIDEO_ID}/screenshot/screen-capture.jpg")) + .unwrap() + .len() as u64; + assert!( + stats.media_bytes <= media + thumbnail, + "relaunch sent {} media bytes for {} bytes of recording", + stats.media_bytes, + media + thumbnail + ); + check_budget(case, &mock, budget::QUEUE_RESUME_AFTER_QUIT_MID_TRANSFER); + }); +} + +/// The app quits while the queue waits for cloud verification. The relaunch +/// confirms the already uploaded recording without sending it again. +#[test] +fn saved_instant_upload_quit_during_verification_confirms_without_reupload() { + let case = "queue_resume_after_quit_during_verification"; + let scratch = Scratch::new(case); + virtual_time(async { + let mock = MockCap::start(vec![rule(Route::Api("recording-complete"), Fault::Hang)]).await; + scratch.sign_in(&mock.url); + let (project, video, audio) = saved_instant_project(&scratch.0, &mock.url); + + let first = Arc::new(Manager::new(LiveBackend, CONFIRMATION_TIMEOUT)); + launch(&first, &project).await; + while mock.count("POST", "recording-complete") == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + first.shutdown().await; + let state = read_state(&project).unwrap().unwrap(); + assert_eq!(state.phase, UploadPhase::Processing, "{state:?}"); + assert!(state.verification.is_some()); + + mock.drain().await; + mock.new_session(); + let relaunched = Arc::new(Manager::new(LiveBackend, CONFIRMATION_TIMEOUT)); + launch(&relaunched, &project).await; + settle(&relaunched).await; + assert_verified(&mock, &project, &video, &audio); + check_budget( + case, + &mock, + budget::QUEUE_RESUME_AFTER_QUIT_DURING_VERIFICATION, + ); + }); +} diff --git a/apps/desktop-gpui/src/upload/verify_tests.rs b/apps/desktop-gpui/src/upload/verify_tests.rs new file mode 100644 index 00000000000..7d13956beb8 --- /dev/null +++ b/apps/desktop-gpui/src/upload/verify_tests.rs @@ -0,0 +1,1813 @@ +//! Deterministic fault-injection tests for the share and Instant upload paths. +//! +//! The real upload code runs against `MockCap`, an in-process HTTP/1.1 mock of +//! the Cap upload API and its object store, on a paused tokio clock. Retry +//! backoff, outages and request timeouts therefore cost virtual time only. +//! The clock advances in 1 ms steps, and freezes while the mock is serving a +//! request, and for a few real milliseconds after the mock or any other task +//! last did work. Bytes still crossing the loopback socket therefore never +//! see virtual time move, and a far timer (a 5 minute request timeout, say) +//! never fires early. +//! +//! Every case checks the stored objects against the local files byte for byte +//! and holds request counts and re-uploaded bytes to budgets taken from the +//! behaviour of main when the case was written. Lower is allowed; higher fails. + +use super::*; +use cap_recording::upload_verification::UploadVerification; +use sha2::{Digest as _, Sha256}; +use std::collections::BTreeMap; +use std::sync::atomic::AtomicBool; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; + +pub(super) const API_KEY: &str = "verify-api-key"; +pub(super) const OWNER: &str = "verify-owner"; +pub(super) const SAMPLE_MP4: &[u8] = + include_bytes!("../../../media-server/src/__tests__/fixtures/test-with-audio.mp4"); + +const MIB: usize = 1024 * 1024; +const PART: u64 = 5 * MIB as u64; +const STUDIO_SIZE: usize = 5 * 5 * MIB + MIB + 12_345; +const STUDIO_LAST_PART: u64 = STUDIO_SIZE as u64 - 5 * PART; +const VIDEO_SEGMENT: usize = 256 * 1024; +const AUDIO_SEGMENT: usize = 32 * 1024; +const INSTANT_PAIRS: u32 = 8; +const INSTANT_VIDEO_ID: &str = "instantverify01"; +const RETRY_WINDOW: Duration = Duration::from_secs(5 * 60); + +const SETTLE: Duration = Duration::from_millis(50); +const WORKED: Duration = Duration::from_micros(200); + +#[derive(Default)] +struct Activity { + in_flight: std::sync::atomic::AtomicUsize, + last: Mutex>, +} + +impl Activity { + fn touch(&self) { + *self.last.lock().unwrap_or_else(|error| error.into_inner()) = + Some(std::time::Instant::now()); + } + + fn busy(&self) -> bool { + self.in_flight.load(Ordering::Acquire) > 0 + || self + .last + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_some_and(|at| at.elapsed() < SETTLE) + } +} + +struct InFlight(Arc); + +impl InFlight { + fn new(activity: &Arc) -> Self { + activity.in_flight.fetch_add(1, Ordering::AcqRel); + activity.touch(); + Self(activity.clone()) + } +} + +impl Drop for InFlight { + fn drop(&mut self) { + self.0.in_flight.fetch_sub(1, Ordering::AcqRel); + self.0.touch(); + } +} + +thread_local! { + static ACTIVITY: std::cell::RefCell>> = const { + std::cell::RefCell::new(None) + }; +} + +pub(super) fn virtual_time(future: F) -> F::Output { + let activity = Arc::new(Activity::default()); + ACTIVITY.with(|current| *current.borrow_mut() = Some(activity.clone())); + let output = tokio::runtime::Builder::new_current_thread() + .enable_all() + .start_paused(true) + .build() + .unwrap() + .block_on(async { + let ticker = tokio::spawn(async move { + loop { + if activity.busy() { + std::thread::sleep(Duration::from_micros(50)); + tokio::task::yield_now().await; + } else { + let step = std::time::Instant::now(); + tokio::time::sleep(Duration::from_millis(1)).await; + if step.elapsed() > WORKED { + activity.touch(); + } + } + } + }); + let output = future.await; + ticker.abort(); + output + }); + ACTIVITY.with(|current| *current.borrow_mut() = None); + output +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Route { + Part(u32), + Object(&'static str), + Api(&'static str), +} + +impl Route { + fn matches(self, method: &str, path: &str) -> bool { + match self { + Route::Part(number) => { + method == "PUT" + && path.starts_with("/s3mp/") + && path.rsplit('/').next() == Some(number.to_string().as_str()) + } + Route::Object(fragment) => { + method == "PUT" && path.starts_with("/s3/") && path.contains(fragment) + } + Route::Api(fragment) => path.starts_with("/api/") && path.contains(fragment), + } + } +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum Fault { + /// Read the whole body, then answer with this status. + Status(u16), + /// Read this fraction of the body, then reset the connection. + Reset(f64), + /// Read this fraction of the body, then take the network down: every open + /// connection is reset and new ones are reset on accept until it ends. + Outage(f64, Duration), + /// Read the body, wait, then answer normally. + Delay(Duration), + /// Read the body and never answer. + Hang, + /// Read this fraction of the body, raise the test's cancel flag (or quit + /// signal), then finish the request normally. + Trip(f64), +} + +pub(super) struct Rule { + route: Route, + fault: Fault, + skip: u32, + count: u32, + seen: u32, + fired: u32, +} + +pub(super) fn rule(route: Route, fault: Fault) -> Rule { + Rule { + route, + fault, + skip: 0, + count: 1, + seen: 0, + fired: 0, + } +} + +impl Rule { + pub(super) fn times(mut self, count: u32) -> Self { + self.count = count; + self + } +} + +#[derive(Default)] +struct Multipart { + video_id: String, + parts: BTreeMap)>, + completed: bool, + aborted: bool, +} + +#[derive(Default)] +struct MockState { + rules: Vec, + requests: Vec<(String, String, u16, Duration)>, + epoch: Option, + media_put_bytes: u64, + useful_offset: u64, + objects: BTreeMap>, + uploads: BTreeMap, + outage_until: Option, + reset_on_accept: u32, + accepted: usize, + connections: Vec, + next_id: u32, + confirmations: Vec, + deletes: u32, + problems: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Stats { + /// HTTP requests that reached the mock (API calls and object PUTs). + pub(super) requests: usize, + /// PUTs of media objects and multipart parts (manifests excluded). + pub(super) media_puts: usize, + /// PUTs of `segments/manifest.json`. + pub(super) manifest_puts: usize, + /// Media body bytes the server received, whole or cut off. + pub(super) media_bytes: u64, + /// Media body bytes the server received beyond what it finally kept: + /// failed or reset bodies, rewritten objects and parts of multipart + /// uploads that were never completed. + pub(super) reupload_bytes: u64, + /// Multipart uploads initiated and never completed or aborted. + pub(super) orphans: usize, + /// Connections the mock accepted and served. + pub(super) connections: usize, + /// Connections reset on accept during an outage. + pub(super) reset_on_accept: u32, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct Budget { + pub(super) requests: usize, + pub(super) media_puts: usize, + pub(super) manifest_puts: usize, + pub(super) reupload_bytes: u64, + pub(super) orphans: usize, + pub(super) connections: usize, +} + +/// Every budget, in one table: main's measured behaviour at 83237e994 (#2522 +/// merged: multipart parts go one at a time over per-upload clients). A +/// change that should move a number, like a retry fix, updates it here and +/// says why; anything that goes over fails. +/// +/// Studio: a 26 MiB export in six parts (five of 5 MiB, one of 1 MiB + 12345 +/// bytes), each part a presign and a PUT. Instant: 2 init segments and 8 +/// audio/video pairs on URLs from one batch presign, progress manifests (a +/// presign and a PUT each) and the completion signal. +pub(super) mod budget { + use super::{Budget, PART, STUDIO_LAST_PART, STUDIO_SIZE, VIDEO_SEGMENT}; + + const fn budget( + requests: usize, + media_puts: usize, + manifest_puts: usize, + reupload_bytes: u64, + orphans: usize, + connections: usize, + ) -> Budget { + Budget { + requests, + media_puts, + manifest_puts, + reupload_bytes, + orphans, + connections, + } + } + + // budget(requests, media_puts, manifest_puts, reupload_bytes, orphans, connections) + const SEGMENT_3: u64 = VIDEO_SEGMENT as u64 + 21; + /// Instant segments share one pooled client, and whether a finished + /// request's connection is back in the pool when the next one starts is + /// a race: under heavy load main was seen opening up to 2 more + /// connections than its usual count. Studio connection counts are exact. + const POOL_RACE: usize = 3; + const SEGMENT_4: u64 = VIDEO_SEGMENT as u64 + 28; + + pub(in crate::upload) const STUDIO_SHARE_CLEAN: Budget = budget(18, 7, 0, 0, 0, 8); + pub(in crate::upload) const STUDIO_SHARE_PART_503: Budget = budget(20, 8, 0, PART, 0, 8); + pub(in crate::upload) const STUDIO_SHARE_RESET_MID_PART: Budget = + budget(19, 8, 0, PART / 2, 0, 9); + pub(in crate::upload) const STUDIO_SHARE_OUTAGE_MID_PART: Budget = + budget(19, 8, 0, PART / 2, 0, 10); + pub(in crate::upload) const STUDIO_SHARE_PRESIGN_500: Budget = budget(19, 7, 0, 0, 0, 8); + pub(in crate::upload) const STUDIO_SHARE_HUNG_PART: Budget = budget(19, 8, 0, PART, 0, 10); + pub(in crate::upload) const STUDIO_SHARE_SLOW_PART: Budget = budget(18, 7, 0, 0, 0, 8); + pub(in crate::upload) const STUDIO_SHARE_PERMANENT_403: Budget = + budget(17, 7, 0, 6 * PART + STUDIO_LAST_PART, 1, 5); + pub(in crate::upload) const STUDIO_SHARE_COMPLETE_500: Budget = + budget(16, 6, 0, STUDIO_SIZE as u64, 1, 6); + pub(in crate::upload) const STUDIO_SHARE_CANCEL_THEN_RESUME: Budget = + budget(26, 10, 0, 3 * PART, 1, 12); + + pub(in crate::upload) const INSTANT_LIVE_CLEAN: Budget = + budget(38, 18, 9, 0, 0, 13 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503: Budget = + budget(42, 19, 10, SEGMENT_3, 0, 15 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_SEGMENT_503X3: Budget = + budget(44, 21, 9, 3 * SEGMENT_3, 0, 16 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_SEGMENT_RESET: Budget = + budget(44, 21, 9, 3 * (SEGMENT_4 / 2), 0, 18 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_OUTAGE: Budget = + budget(41, 19, 7, SEGMENT_3 / 2, 0, 22 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_SLOW_SEGMENT: Budget = + budget(38, 18, 9, 0, 0, 13 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_OUTAGE_PAST_WINDOW: Budget = + budget(15, 8, 3, SEGMENT_3 / 2, 0, 6 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_PERMANENT_403: Budget = + budget(22, 11, 4, 3 * SEGMENT_3, 0, 9 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_CANCEL: Budget = + budget(19, 10, 4, 0, 0, 7 + POOL_RACE); + pub(in crate::upload) const INSTANT_LIVE_COMPLETION_500: Budget = + budget(40, 18, 9, 0, 0, 15 + POOL_RACE); + + /// The relaunch session only: every segment and the thumbnail once. + pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_MID_TRANSFER: Budget = + budget(22, 15, 2, 1_795_340, 0, 12 + POOL_RACE); + pub(in crate::upload) const QUEUE_RESUME_AFTER_QUIT_DURING_VERIFICATION: Budget = + budget(1, 0, 0, 0, 0, 1); +} + +pub(super) fn check_budget(case: &str, mock: &MockCap, budget: Budget) { + let stats = mock.stats(); + eprintln!("upload-verify {case}: {stats:?}"); + let over = [ + ("requests", stats.requests as u64, budget.requests as u64), + ( + "media_puts", + stats.media_puts as u64, + budget.media_puts as u64, + ), + ( + "manifest_puts", + stats.manifest_puts as u64, + budget.manifest_puts as u64, + ), + ( + "reupload_bytes", + stats.reupload_bytes, + budget.reupload_bytes, + ), + ("orphans", stats.orphans as u64, budget.orphans as u64), + ( + "connections", + stats.connections as u64, + budget.connections as u64, + ), + ] + .into_iter() + .filter(|(_, actual, limit)| actual > limit) + .map(|(name, actual, limit)| format!("{name} {actual} > budget {limit}")) + .collect::>(); + assert!( + over.is_empty(), + "{case} is over budget: {over:?} ({stats:?})\nrequests (virtual time):\n{}", + mock.timeline() + ); +} + +pub(super) struct MockCap { + pub(super) url: String, + activity: Arc, + state: Arc>, + cancel: Arc, + tripped: Arc, + accept: tokio::task::JoinHandle<()>, +} + +impl Drop for MockCap { + fn drop(&mut self) { + self.accept.abort(); + for connection in self.lock().connections.drain(..) { + connection.abort(); + } + } +} + +impl MockCap { + pub(super) async fn start(rules: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(MockState { + rules, + epoch: Some(tokio::time::Instant::now()), + ..Default::default() + })); + let cancel = Arc::new(AtomicBool::new(false)); + let tripped = Arc::new(tokio::sync::Notify::new()); + let activity: Arc = ACTIVITY + .with(|current| current.borrow().clone()) + .unwrap_or_default(); + let server = Server { + activity: activity.clone(), + url: url.clone(), + state: state.clone(), + cancel: cancel.clone(), + tripped: tripped.clone(), + }; + let accept = tokio::spawn(server.accept(listener)); + Self { + url, + activity, + state, + cancel, + tripped, + accept, + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, MockState> { + self.state.lock().unwrap_or_else(|error| error.into_inner()) + } + + pub(super) fn cancel_flag(&self) -> Arc { + self.cancel.clone() + } + + pub(super) async fn tripped(&self) { + self.tripped.notified().await; + } + + pub(super) fn object(&self, key: &str) -> Option> { + self.lock().objects.get(key).cloned() + } + + pub(super) fn confirmations(&self) -> Vec { + self.lock().confirmations.clone() + } + + pub(super) fn deletes(&self) -> u32 { + self.lock().deletes + } + + pub(super) fn count(&self, method: &str, fragment: &str) -> usize { + self.lock() + .requests + .iter() + .filter(|(m, path, ..)| m == method && path.contains(fragment)) + .count() + } + + pub(super) fn timeline(&self) -> String { + self.lock() + .requests + .iter() + .map(|(method, path, status, at)| { + format!("{:>10.3}s {status:>3} {method} {path}\n", at.as_secs_f64()) + }) + .collect() + } + + pub(super) fn assert_clean(&self) { + let problems = self.lock().problems.clone(); + assert!( + problems.is_empty(), + "upload protocol problems: {problems:?}" + ); + } + + /// Waits until the mock has been idle for the settle window, so requests + /// a quit app already handed to the kernel are served before the next + /// session starts counting. + pub(super) async fn drain(&self) { + while self.activity.busy() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + + /// Forget every request so far and drop every open connection, as a quit + /// does. Stored objects survive, so a relaunch can be measured on its own. + pub(super) fn new_session(&self) { + let mut state = self.lock(); + for connection in state.connections.drain(..) { + connection.abort(); + } + state.requests.clear(); + state.media_put_bytes = 0; + state.reset_on_accept = 0; + state.accepted = 0; + state.useful_offset = state.useful_bytes(); + } + + pub(super) fn stats(&self) -> Stats { + let state = self.lock(); + let is_media = |path: &str| { + path.starts_with("/s3mp/") + || (path.starts_with("/s3/") && !path.ends_with("/manifest.json")) + }; + Stats { + requests: state.requests.len(), + media_puts: state + .requests + .iter() + .filter(|(method, path, ..)| method == "PUT" && is_media(path)) + .count(), + manifest_puts: state + .requests + .iter() + .filter(|(method, path, ..)| method == "PUT" && path.ends_with("/manifest.json")) + .count(), + media_bytes: state.media_put_bytes, + reupload_bytes: state + .media_put_bytes + .saturating_sub(state.useful_bytes().saturating_sub(state.useful_offset)), + orphans: state + .uploads + .values() + .filter(|upload| !upload.completed && !upload.aborted) + .count(), + connections: state.accepted, + reset_on_accept: state.reset_on_accept, + } + } +} + +#[derive(Clone)] +struct Server { + activity: Arc, + url: String, + state: Arc>, + cancel: Arc, + tripped: Arc, +} + +struct Head { + method: String, + path: String, + query: String, + headers: HashMap, +} + +impl Server { + fn lock(&self) -> std::sync::MutexGuard<'_, MockState> { + self.state.lock().unwrap_or_else(|error| error.into_inner()) + } + + async fn accept(self, listener: TcpListener) { + loop { + let Ok((stream, _)) = listener.accept().await else { + continue; + }; + self.activity.touch(); + let _ = stream.set_zero_linger(); + let _ = stream.set_nodelay(true); + let mut state = self.lock(); + if state + .outage_until + .is_some_and(|until| tokio::time::Instant::now() < until) + { + state.reset_on_accept += 1; + continue; + } + state.accepted += 1; + let connection = tokio::spawn(self.clone().connection(stream)); + state + .connections + .retain(|connection| !connection.is_finished()); + state.connections.push(connection.abort_handle()); + } + } + + fn take_fault(&self, method: &str, path: &str) -> Option { + let mut state = self.lock(); + for rule in &mut state.rules { + if rule.route.matches(method, path) { + rule.seen += 1; + if rule.seen > rule.skip && rule.fired < rule.count { + rule.fired += 1; + return Some(rule.fault); + } + } + } + None + } + + fn record(&self, head: &Head, status: u16) { + let mut state = self.lock(); + let at = state.epoch.map_or(Duration::ZERO, |epoch| epoch.elapsed()); + state + .requests + .push((head.method.clone(), head.path.clone(), status, at)); + } + + fn outage(&self, lasts: Duration) { + let mut state = self.lock(); + state.outage_until = Some(tokio::time::Instant::now() + lasts); + for connection in state.connections.drain(..) { + connection.abort(); + } + } + + async fn connection(self, mut stream: TcpStream) { + let mut pending = Vec::new(); + while let Some(head) = read_head(&mut stream, &mut pending).await { + let mut in_flight = Some(InFlight::new(&self.activity)); + let length = head + .headers + .get("content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + let media = head.method == "PUT" + && (head.path.starts_with("/s3mp/") + || (head.path.starts_with("/s3/") && !head.path.ends_with("/manifest.json"))); + let fault = self.take_fault(&head.method, &head.path); + let cut = match fault { + Some(Fault::Reset(at) | Fault::Outage(at, _) | Fault::Trip(at)) => { + ((length as f64) * at) as usize + } + _ => length, + }; + let mut body = Vec::with_capacity(length); + if !self + .read_body(&mut stream, &mut pending, &mut body, cut, media) + .await + { + self.record(&head, 0); + return; + } + match fault { + Some(Fault::Reset(_)) => { + self.record(&head, 0); + return; + } + Some(Fault::Outage(_, lasts)) => { + self.record(&head, 0); + self.outage(lasts); + return; + } + Some(Fault::Trip(_)) => { + self.cancel.store(true, Ordering::Release); + self.tripped.notify_one(); + if !self + .read_body(&mut stream, &mut pending, &mut body, length, media) + .await + { + self.record(&head, 0); + return; + } + } + Some(Fault::Delay(delay)) => { + drop(in_flight.take()); + tokio::time::sleep(delay).await; + in_flight = Some(InFlight::new(&self.activity)); + } + Some(Fault::Hang) => { + self.record(&head, 0); + drop(in_flight.take()); + std::future::pending::<()>().await; + } + Some(Fault::Status(status)) => { + self.record(&head, status); + if respond(&mut stream, status, &[], br#"{"error":"injected"}"#) + .await + .is_err() + { + return; + } + continue; + } + None => {} + } + let (status, headers, payload) = self.handle(&head, body); + self.record(&head, status); + if respond(&mut stream, status, &headers, &payload) + .await + .is_err() + { + return; + } + drop(in_flight); + } + } + + async fn read_body( + &self, + stream: &mut TcpStream, + pending: &mut Vec, + body: &mut Vec, + until: usize, + media: bool, + ) -> bool { + let mut buffer = vec![0u8; 64 * 1024]; + while body.len() < until { + let wanted = until - body.len(); + let read = if pending.is_empty() { + match stream.read(&mut buffer[..wanted.min(64 * 1024)]).await { + Ok(0) | Err(_) => return false, + Ok(read) => { + body.extend_from_slice(&buffer[..read]); + read + } + } + } else { + let read = wanted.min(pending.len()); + body.extend(pending.drain(..read)); + read + }; + self.activity.touch(); + if media { + self.lock().media_put_bytes += read as u64; + } + } + true + } + + fn handle(&self, head: &Head, body: Vec) -> (u16, Vec<(String, String)>, Vec) { + let mut guard = self.lock(); + let state = &mut *guard; + let path = head.path.as_str(); + if head.method == "PUT" && path.starts_with("/s3mp/") { + let mut parts = path["/s3mp/".len()..].splitn(2, '/'); + let upload_id = parts.next().unwrap_or_default().to_string(); + let number = parts + .next() + .and_then(|n| n.parse::().ok()) + .unwrap_or(0); + let etag = to_hex(&cap_gpui_kernels::codec::md5(&body)); + if let Some(md5) = head.headers.get("content-md5") { + let actual = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + cap_gpui_kernels::codec::md5(&body), + ); + if md5 != &actual { + state + .problems + .push(format!("Content-MD5 mismatch on {path}")); + return (400, Vec::new(), b"{}".to_vec()); + } + } + let Some(upload) = state.uploads.get_mut(&upload_id) else { + state + .problems + .push(format!("part for unknown upload {path}")); + return (404, Vec::new(), b"{}".to_vec()); + }; + if upload.completed || upload.aborted { + state.problems.push(format!("part after completion {path}")); + return (404, Vec::new(), b"{}".to_vec()); + } + upload.parts.insert(number, (etag.clone(), body)); + return ( + 200, + vec![("ETag".into(), format!("\"{etag}\""))], + Vec::new(), + ); + } + if head.method == "PUT" && path.starts_with("/s3/") { + state.objects.insert(path["/s3/".len()..].to_string(), body); + return (200, Vec::new(), Vec::new()); + } + + if head.headers.get("authorization").map(String::as_str) + != Some(format!("Bearer {API_KEY}").as_str()) + { + state + .problems + .push(format!("unauthenticated {} {path}", head.method)); + return (401, Vec::new(), b"{}".to_vec()); + } + let request: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + let query = head + .query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>(); + let text = |value: &Value, key: &str| { + value + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + let reply = + |status: u16, value: Value| (status, Vec::new(), value.to_string().into_bytes()); + match (head.method.as_str(), path) { + ("GET", "/api/desktop/video/new-id") => { + state.next_id += 1; + reply( + 200, + json!({ "id": format!("verifyvideo{:04}", state.next_id) }), + ) + } + ("GET", "/api/desktop/video/create") => { + let id = query.get("videoId").cloned().unwrap_or_else(|| { + state.next_id += 1; + format!("verifyvideo{:04}", state.next_id) + }); + reply(200, json!({ "id": id })) + } + ("POST", "/api/upload/multipart/initiate") => { + state.next_id += 1; + let upload_id = format!("upload{:04}", state.next_id); + state.uploads.insert( + upload_id.clone(), + Multipart { + video_id: text(&request, "videoId"), + ..Default::default() + }, + ); + reply(200, json!({ "uploadId": upload_id })) + } + ("POST", "/api/upload/multipart/presign-part") => { + let upload_id = text(&request, "uploadId"); + let number = request.get("partNumber").and_then(Value::as_u64); + if !state.uploads.contains_key(&upload_id) || number.is_none() { + state.problems.push(format!("bad presign {request}")); + return reply(404, json!({})); + } + reply( + 200, + json!({ "presignedUrl": format!("{}/s3mp/{upload_id}/{}", self.url, number.unwrap()) }), + ) + } + ("POST", "/api/upload/multipart/complete") => { + let upload_id = text(&request, "uploadId"); + let listed = request + .get("parts") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let Some(upload) = state.uploads.get(&upload_id) else { + state + .problems + .push(format!("complete for unknown upload {upload_id}")); + return reply(404, json!({})); + }; + let mut problems = Vec::new(); + let mut object = Vec::new(); + for (position, part) in listed.iter().enumerate() { + let number = part.get("partNumber").and_then(Value::as_u64).unwrap_or(0); + if number != position as u64 + 1 { + problems.push(format!("part {number} listed at {position}")); + } + match upload.parts.get(&(number as u32)) { + Some((etag, bytes)) + if Some(etag.as_str()) == part.get("etag").and_then(Value::as_str) => + { + if part.get("size").and_then(Value::as_u64) != Some(bytes.len() as u64) + { + problems.push(format!("part {number} size mismatch")); + } + object.extend_from_slice(bytes); + } + Some(_) => problems.push(format!("part {number} etag mismatch")), + None => problems.push(format!("part {number} missing")), + } + } + if upload.parts.len() != listed.len() { + problems.push(format!( + "{} parts stored, {} listed", + upload.parts.len(), + listed.len() + )); + } + if !problems.is_empty() { + state.problems.extend(problems); + return reply(400, json!({ "error": "invalid parts" })); + } + state + .objects + .insert(format!("{}/result.mp4", upload.video_id), object); + state.uploads.get_mut(&upload_id).unwrap().completed = true; + reply(200, json!({ "success": true })) + } + ("POST", "/api/upload/multipart/abort") => { + if let Some(upload) = state.uploads.get_mut(&text(&request, "uploadId")) { + upload.aborted = true; + } + reply(200, json!({ "success": true })) + } + ("POST", "/api/upload/signed") => { + let url = format!( + "{}/s3/{}/{}", + self.url, + text(&request, "videoId"), + text(&request, "subpath") + ); + reply( + 200, + json!({ "presignedPutData": { "url": url, "headers": {} } }), + ) + } + ("POST", "/api/upload/signed/batch") => { + let video_id = text(&request, "videoId"); + let urls = request + .get("subpaths") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(|subpath| { + ( + subpath.to_string(), + Value::String(format!("{}/s3/{video_id}/{subpath}", self.url)), + ) + }) + .collect::>(); + reply(200, json!({ "urls": urls })) + } + ("POST", "/api/recording/prepare") => reply(404, json!({ "error": "not found" })), + ("POST", "/api/upload/recording-complete") => { + state.confirmations.push(request.clone()); + let video_id = text(&request, "videoId"); + let Some(verification) = request.get("verification") else { + return reply(200, json!({ "success": true })); + }; + match state.verify_segments(&video_id, verification) { + Ok(receipt) => reply( + 200, + json!({ "success": true, "status": "verified", "verification": receipt }), + ), + Err(problem) => { + state.problems.push(problem); + reply(409, json!({ "status": "reupload-required" })) + } + } + } + ("DELETE", "/api/desktop/video/delete") => { + state.deletes += 1; + reply(200, json!({ "success": true })) + } + _ => { + state + .problems + .push(format!("unknown route {} {path}", head.method)); + reply(404, json!({})) + } + } + } +} + +impl MockState { + fn useful_bytes(&self) -> u64 { + self.objects + .iter() + .filter(|(key, _)| !key.ends_with("/manifest.json")) + .map(|(_, object)| object.len() as u64) + .sum() + } + + fn verify_segments(&self, video_id: &str, verification: &Value) -> Result { + let claimed: UploadVerification = serde_json::from_value(verification.clone()) + .map_err(|error| format!("unreadable verification: {error}"))?; + let stored = self + .objects + .get(&format!("{video_id}/segments/manifest.json")) + .ok_or("verification without a stored manifest")?; + if UploadVerification::segments(stored, claimed.required_audio) != claimed { + return Err("verification does not match the stored manifest".into()); + } + let manifest: SegmentUploadManifest = + serde_json::from_slice(stored).map_err(|error| error.to_string())?; + if !manifest.is_complete || !manifest.video_init_uploaded { + return Err("stored manifest is not complete".into()); + } + let mut keys = vec!["segments/video/init.mp4".to_string()]; + if manifest.audio_init_uploaded { + keys.push("segments/audio/init.mp4".into()); + } + keys.extend( + manifest + .video_segments + .iter() + .map(|s| format!("segments/video/segment_{:03}.m4s", s.index)), + ); + keys.extend( + manifest + .audio_segments + .iter() + .map(|s| format!("segments/audio/segment_{:03}.m4s", s.index)), + ); + let mut size = 0; + for key in keys { + size += self + .objects + .get(&format!("{video_id}/{key}")) + .ok_or_else(|| format!("manifest lists missing object {key}"))? + .len() as u64; + } + Ok(json!({ + "version": 1, + "videoId": video_id, + "artifact": claimed.artifact, + "fileSize": size, + "duration": manifest.video_segments.iter().map(|s| s.duration).sum::(), + "hasAudio": !manifest.audio_segments.is_empty(), + "fullDecode": true, + "requiredAudioVerified": claimed.required_audio && !manifest.audio_segments.is_empty(), + })) + } +} + +async fn read_head(stream: &mut TcpStream, pending: &mut Vec) -> Option { + let mut buffer = [0u8; 16 * 1024]; + let end = loop { + if let Some(end) = pending.windows(4).position(|window| window == b"\r\n\r\n") { + break end; + } + match stream.read(&mut buffer).await { + Ok(0) | Err(_) => return None, + Ok(read) => pending.extend_from_slice(&buffer[..read]), + } + }; + let text = String::from_utf8_lossy(&pending[..end]).into_owned(); + pending.drain(..end + 4); + let mut lines = text.split("\r\n"); + let mut request = lines.next()?.split(' '); + let method = request.next()?.to_string(); + let target = request.next()?; + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let headers = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_ascii_lowercase(), value.trim().to_string())) + .collect(); + Some(Head { + method, + path: path.to_string(), + query: query.to_string(), + headers, + }) +} + +async fn respond( + stream: &mut TcpStream, + status: u16, + headers: &[(String, String)], + body: &[u8], +) -> std::io::Result<()> { + let mut response = format!( + "HTTP/1.1 {status} Mock\r\nContent-Type: application/json\r\nContent-Length: {}\r\n", + body.len() + ); + for (name, value) in headers { + response.push_str(&format!("{name}: {value}\r\n")); + } + response.push_str("\r\n"); + let mut bytes = response.into_bytes(); + bytes.extend_from_slice(body); + stream.write_all(&bytes).await +} + +fn to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub(super) fn noise(seed: u64, len: usize) -> Vec { + let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut bytes = Vec::with_capacity(len + 8); + while bytes.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + bytes.extend_from_slice(&state.to_le_bytes()); + } + bytes.truncate(len); + bytes +} + +pub(super) struct Scratch(pub(super) PathBuf); + +impl Scratch { + pub(super) fn new(tag: &str) -> Self { + let root = std::env::temp_dir().join(format!( + "cap-upload-verify-{tag}-{}", + crate::store::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + Self(root) + } + + /// Signs the test thread in to `server` through the thread-local store. + pub(super) fn sign_in(&self, server: &str) { + let path = self.0.join("store"); + std::fs::write( + &path, + serde_json::to_vec(&json!({ + "general_settings": { + "serverUrl": server, + "enableNotifications": false, + "hasCompletedOnboarding": true, + }, + "auth": { + "secret": { "api_key": API_KEY }, + "user_id": OWNER, + "plan": { "upgraded": true, "manual": true }, + "organizations": [], + }, + })) + .unwrap(), + ) + .unwrap(); + crate::store::set_test_tauri_store_path(Some(path)); + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + crate::store::set_test_tauri_store_path(None); + let _ = std::fs::remove_dir_all(&self.0); + } +} + +pub(super) fn write_thumbnail(path: &Path) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + image::RgbImage::from_fn(64, 36, |x, y| { + image::Rgb([(x * 4) as u8, (y * 7) as u8, 90]) + }) + .save(path) + .unwrap(); +} + +fn studio_project(root: &Path) -> PathBuf { + let project = root.join("share.cap"); + std::fs::create_dir_all(project.join("output")).unwrap(); + std::fs::write( + project.join("recording-meta.json"), + r#"{"pretty_name":"Upload verify","sharing":null,"segments":[{"display":{"path":"content/segments/segment-0/display.mp4","fps":30}}]}"#, + ) + .unwrap(); + let mut file = SAMPLE_MP4.to_vec(); + let pad = STUDIO_SIZE - file.len(); + file.extend_from_slice(&(pad as u32).to_be_bytes()); + file.extend_from_slice(b"free"); + file.extend(noise(0x5743_4150, pad - 8)); + std::fs::write(project.join("output/result.mp4"), file).unwrap(); + write_thumbnail(&project.join("screenshots/display.jpg")); + project +} + +/// A fragmented-MP4-shaped segment: a `moof` box then an `mdat` box, sized +/// and filled deterministically. +fn fragment(seed: u64, size: usize) -> Vec { + let mut bytes = Vec::with_capacity(size); + bytes.extend_from_slice(&32u32.to_be_bytes()); + bytes.extend_from_slice(b"moof"); + bytes.extend(noise(seed, 24)); + bytes.extend_from_slice(&((size - 32) as u32).to_be_bytes()); + bytes.extend_from_slice(b"mdat"); + bytes.extend(noise(seed ^ 0xA5A5, size - 40)); + bytes +} + +/// Writes `content/display` and `content/audio` the way the Instant recorder +/// leaves them and returns their segment events, init first. +pub(super) fn instant_tracks( + project: &Path, + pairs: u32, +) -> (Vec, Vec) { + let track = |name: &str, media: SegmentMediaType, size: usize, version: u32, kind: &str| { + let dir = project.join("content").join(name); + std::fs::create_dir_all(&dir).unwrap(); + let init = dir.join("init.mp4"); + std::fs::write(&init, SAMPLE_MP4).unwrap(); + let mut events = vec![SegmentCompletedEvent { + path: init, + index: 0, + duration: 0.0, + file_size: SAMPLE_MP4.len() as u64, + is_init: true, + media_type: media, + }]; + let mut listed = Vec::new(); + for index in 1..=pairs { + let bytes = fragment( + u64::from(index) * 2 + (media == SegmentMediaType::Audio) as u64, + size + index as usize * 7, + ); + let path = dir.join(format!("segment_{index:03}.m4s")); + std::fs::write(&path, &bytes).unwrap(); + listed.push(json!({ + "path": format!("segment_{index:03}.m4s"), + "index": index, + "duration": 2.0, + "is_complete": true, + "file_size": bytes.len(), + })); + events.push(SegmentCompletedEvent { + path, + index, + duration: 2.0, + file_size: bytes.len() as u64, + is_init: false, + media_type: media, + }); + } + std::fs::write( + dir.join("manifest.json"), + serde_json::to_vec(&json!({ + "version": version, + "type": kind, + "init_segment": "init.mp4", + "segments": listed, + "is_complete": true, + })) + .unwrap(), + ) + .unwrap(); + events + }; + ( + track( + "display", + SegmentMediaType::Video, + VIDEO_SEGMENT, + 5, + "m4s_segments", + ), + track( + "audio", + SegmentMediaType::Audio, + AUDIO_SEGMENT, + 2, + "m4s_audio_segments", + ), + ) +} + +pub(super) fn expected_manifest( + video: &[SegmentCompletedEvent], + audio: &[SegmentCompletedEvent], +) -> SegmentUploadManifest { + let mut manifest = SegmentUploadManifest::default(); + for event in video.iter().chain(audio) { + manifest.record(event); + } + manifest.is_complete = true; + manifest +} + +/// Every local segment is stored byte for byte and the final manifest lists +/// exactly the local recording. +pub(super) fn assert_segments_stored( + mock: &MockCap, + video_id: &str, + video: &[SegmentCompletedEvent], + audio: &[SegmentCompletedEvent], +) { + for event in video.iter().chain(audio) { + let key = format!("{video_id}/{}", segment_subpath(event)); + let stored = mock.object(&key); + assert!( + stored.as_deref() == Some(std::fs::read(&event.path).unwrap().as_slice()), + "{key} is missing or differs from the local file" + ); + } + let manifest: SegmentUploadManifest = serde_json::from_slice( + &mock + .object(&format!("{video_id}/segments/manifest.json")) + .expect("no manifest uploaded"), + ) + .unwrap(); + assert_eq!(manifest, expected_manifest(video, audio)); +} + +struct StudioRun { + result: Result, + progress: Vec, +} + +async fn share(project: &Path, cancel: Arc) -> StudioRun { + let progress = Arc::new(Mutex::new(Vec::new())); + let sink = progress.clone(); + let result = upload_exported_video( + project.to_path_buf(), + None, + move |value| sink.lock().unwrap().push(value), + cancel, + ) + .await; + let progress = progress.lock().unwrap().clone(); + assert!( + progress.first() == Some(&0.0) + && progress.windows(2).all(|pair| pair[1] >= pair[0]) + && progress.iter().all(|value| (0.0..=1.0).contains(value)), + "progress is not monotonic within 0..=1: {progress:?}" + ); + // 100% means shared: only the final report after completion may be 1.0. + assert!( + progress + .split_last() + .is_none_or(|(_, earlier)| earlier.iter().all(|value| *value < 1.0)) + && (result.is_ok() || progress.iter().all(|value| *value < 1.0)), + "progress reached 100% before the upload completed: {progress:?}" + ); + StudioRun { result, progress } +} + +fn assert_shared(mock: &MockCap, project: &Path, run: &StudioRun) { + let meta = RecordingMeta::load_for_project(project).unwrap(); + let sharing = meta.sharing.as_ref().expect("no sharing state persisted"); + assert_eq!( + run.result, + Ok(UploadResult::Success(format!( + "{}/s/{}", + mock.url, sharing.id + ))) + ); + assert_eq!(run.progress.last(), Some(&1.0)); + assert!(matches!(meta.upload, Some(UploadMeta::Complete))); + let local = std::fs::read(project.join("output/result.mp4")).unwrap(); + let stored = mock + .object(&format!("{}/result.mp4", sharing.id)) + .expect("no object stored"); + assert!( + stored == local, + "stored object ({} bytes, sha256 {}) differs from the local file ({} bytes, sha256 {})", + stored.len(), + to_hex(&Sha256::digest(&stored)), + local.len(), + to_hex(&Sha256::digest(&local)), + ); + assert!( + mock.object(&format!("{}/screenshot/screen-capture.jpg", sharing.id)) + .is_some_and(|thumbnail| thumbnail.starts_with(&[0xFF, 0xD8])), + "no JPEG thumbnail stored" + ); + mock.assert_clean(); +} + +fn assert_share_failed(project: &Path, run: &StudioRun, error: &str) { + let failure = run + .result + .as_ref() + .expect_err("share unexpectedly succeeded"); + assert!(failure.contains(error), "unexpected error: {failure}"); + let meta = RecordingMeta::load_for_project(project).unwrap(); + assert!( + matches!(meta.upload, Some(UploadMeta::SinglePartUpload { .. })), + "the reserved video id must stay persisted for a retry: {:?}", + meta.upload + ); + assert!(meta.sharing.is_none()); +} + +fn studio_case( + case: &str, + rules: Vec, + budget: Budget, + check: impl FnOnce(&MockCap, &Path, StudioRun), +) { + ffmpeg::init().unwrap(); + let scratch = Scratch::new(case); + virtual_time(async { + let mock = MockCap::start(rules).await; + scratch.sign_in(&mock.url); + let project = studio_project(&scratch.0); + let started = tokio::time::Instant::now(); + let run = share(&project, Arc::new(AtomicBool::new(false))).await; + eprintln!("upload-verify {case}: virtual {:?}", started.elapsed()); + check(&mock, &project, run); + check_budget(case, &mock, budget); + }); +} + +#[test] +fn studio_share_clean() { + studio_case( + "studio_share_clean", + Vec::new(), + budget::STUDIO_SHARE_CLEAN, + |mock, project, run| { + assert_shared(mock, project, &run); + assert_eq!(mock.count("PUT", "/s3mp/"), 6); + }, + ); +} + +#[test] +fn studio_share_part_503_retries_the_part_at_the_end() { + studio_case( + "studio_share_part_503", + vec![rule(Route::Part(3), Fault::Status(503))], + budget::STUDIO_SHARE_PART_503, + |mock, project, run| assert_shared(mock, project, &run), + ); +} + +#[test] +fn studio_share_reset_mid_part_retries_in_place() { + studio_case( + "studio_share_reset_mid_part", + vec![rule(Route::Part(4), Fault::Reset(0.5))], + budget::STUDIO_SHARE_RESET_MID_PART, + |mock, project, run| assert_shared(mock, project, &run), + ); +} + +/// The network drops while part 2 is half sent and stays down for 5 s. Main +/// has only that part in flight, so it loses half a part and retries it after +/// the outage (2 s, then 4 s backoff). +#[test] +fn studio_share_outage_mid_part() { + studio_case( + "studio_share_outage_mid_part", + vec![rule( + Route::Part(2), + Fault::Outage(0.5, Duration::from_secs(5)), + )], + budget::STUDIO_SHARE_OUTAGE_MID_PART, + |mock, project, run| { + assert_shared(mock, project, &run); + assert!(mock.stats().reset_on_accept >= 1); + }, + ); +} + +#[test] +fn studio_share_presign_500_retries_the_part_at_the_end() { + studio_case( + "studio_share_presign_500", + vec![Rule { + skip: 3, + ..rule(Route::Api("multipart/presign-part"), Fault::Status(500)) + }], + budget::STUDIO_SHARE_PRESIGN_500, + |mock, project, run| assert_shared(mock, project, &run), + ); +} + +/// A part whose response never arrives hits the 5 minute request timeout +/// (virtual) and is retried in place. +#[test] +fn studio_share_hung_part_times_out_and_retries() { + studio_case( + "studio_share_hung_part", + vec![rule(Route::Part(5), Fault::Hang)], + budget::STUDIO_SHARE_HUNG_PART, + |mock, project, run| assert_shared(mock, project, &run), + ); +} + +#[test] +fn studio_share_slow_part_costs_nothing_extra() { + studio_case( + "studio_share_slow_part", + vec![rule(Route::Part(3), Fault::Delay(Duration::from_secs(40)))], + budget::STUDIO_SHARE_SLOW_PART, + |mock, project, run| assert_shared(mock, project, &run), + ); +} + +#[test] +fn studio_share_permanent_403_fails_with_the_part_error() { + studio_case( + "studio_share_permanent_403", + vec![rule(Route::Part(2), Fault::Status(403)).times(u32::MAX)], + budget::STUDIO_SHARE_PERMANENT_403, + |mock, project, run| { + assert_share_failed(project, &run, "uploader/part/2/status/403"); + assert_eq!(mock.count("POST", "multipart/complete"), 0); + mock.assert_clean(); + }, + ); +} + +#[test] +fn studio_share_complete_500_fails_and_keeps_the_video_id() { + studio_case( + "studio_share_complete_500", + vec![rule(Route::Api("multipart/complete"), Fault::Status(500))], + budget::STUDIO_SHARE_COMPLETE_500, + |mock, project, run| { + assert_share_failed(project, &run, "upload_multipart_complete/500"); + mock.assert_clean(); + }, + ); +} + +/// Cancel (or quit) while part 3 is half sent, then share again. The second +/// attempt reuses the persisted video id and starts a new multipart upload. +#[test] +fn studio_share_cancel_then_resume_reuses_the_persisted_video() { + ffmpeg::init().unwrap(); + let case = "studio_share_cancel_then_resume"; + let scratch = Scratch::new(case); + virtual_time(async { + let mock = MockCap::start(vec![rule(Route::Part(3), Fault::Trip(0.5))]).await; + scratch.sign_in(&mock.url); + let project = studio_project(&scratch.0); + let first = share(&project, mock.cancel_flag()).await; + assert_share_failed(&project, &first, "Export cancelled"); + let reserved = match RecordingMeta::load_for_project(&project).unwrap().upload { + Some(UploadMeta::SinglePartUpload { video_id, .. }) => video_id, + other => panic!("unexpected upload state {other:?}"), + }; + let second = share(&project, Arc::new(AtomicBool::new(false))).await; + assert_shared(&mock, &project, &second); + assert_eq!( + RecordingMeta::load_for_project(&project) + .unwrap() + .sharing + .unwrap() + .id, + reserved + ); + assert_eq!(mock.count("GET", "video/new-id"), 1); + check_budget(case, &mock, budget::STUDIO_SHARE_CANCEL_THEN_RESUME); + }); +} + +/// Feeds the recorder's segment events on a fixed virtual schedule: both init +/// segments at 0 s, then audio segment k at 2k s and video segment k at +/// 2k + 0.6 s, so no two segments are in flight at the same moment unless a +/// retry puts them there. +async fn feed( + video: Vec, + audio: Vec, + events: flume::Sender, +) { + let start = tokio::time::Instant::now(); + let _ = events.send(video[0].clone()); + let _ = events.send(audio[0].clone()); + for k in 1..video.len() { + let at = start + Duration::from_secs(2 * k as u64); + tokio::time::sleep_until(at).await; + let _ = events.send(audio[k].clone()); + tokio::time::sleep_until(at + Duration::from_millis(600)).await; + let _ = events.send(video[k].clone()); + } +} + +struct InstantRun { + result: Result<(), String>, + elapsed: Duration, + video: Vec, + audio: Vec, +} + +fn instant_case( + case: &str, + rules: Vec, + budget: Budget, + check: impl FnOnce(&MockCap, InstantRun), +) { + let scratch = Scratch::new(case); + virtual_time(async { + let mock = MockCap::start(rules).await; + scratch.sign_in(&mock.url); + let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS); + let (sender, events) = flume::unbounded(); + let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender)); + let started = tokio::time::Instant::now(); + let result = upload_segments( + &LiveSegmentTransport, + INSTANT_VIDEO_ID, + events, + mock.cancel_flag(), + None, + ) + .await; + let elapsed = started.elapsed(); + eprintln!("upload-verify {case}: virtual {elapsed:?}"); + feeder.abort(); + check( + &mock, + InstantRun { + result, + elapsed, + video, + audio, + }, + ); + check_budget(case, &mock, budget); + }); +} + +fn assert_instant_complete(mock: &MockCap, run: &InstantRun) { + assert_eq!(run.result, Ok(())); + assert_segments_stored(mock, INSTANT_VIDEO_ID, &run.video, &run.audio); + let confirmations = mock.confirmations(); + assert_eq!(confirmations.len(), 1, "{confirmations:?}"); + assert_eq!(confirmations[0], json!({ "videoId": INSTANT_VIDEO_ID })); + mock.assert_clean(); +} + +#[test] +fn instant_live_clean() { + instant_case( + "instant_live_clean", + Vec::new(), + budget::INSTANT_LIVE_CLEAN, + |mock, run| { + assert_instant_complete(mock, &run); + }, + ); +} + +#[test] +fn instant_live_segment_503_retries_in_call() { + instant_case( + "instant_live_segment_503", + vec![rule( + Route::Object("segments/video/segment_003"), + Fault::Status(503), + )], + budget::INSTANT_LIVE_SEGMENT_503, + |mock, run| assert_instant_complete(mock, &run), + ); +} + +/// Three 503s exhaust the in-call attempts; the segment is deferred and +/// retried in place instead of failing the live upload. +#[test] +fn instant_live_segment_503_three_times_is_deferred_not_fatal() { + instant_case( + "instant_live_segment_503x3", + vec![ + rule( + Route::Object("segments/video/segment_003"), + Fault::Status(503), + ) + .times(3), + ], + budget::INSTANT_LIVE_SEGMENT_503X3, + |mock, run| assert_instant_complete(mock, &run), + ); +} + +#[test] +fn instant_live_segment_reset_mid_body() { + instant_case( + "instant_live_segment_reset", + vec![ + rule( + Route::Object("segments/video/segment_004"), + Fault::Reset(0.5), + ) + .times(3), + ], + budget::INSTANT_LIVE_SEGMENT_RESET, + |mock, run| assert_instant_complete(mock, &run), + ); +} + +#[test] +fn instant_live_deferred_retries_back_off_after_recording_ends() { + let scratch = Scratch::new("instant_live_deferred_retries_back_off"); + virtual_time(async { + let mock = MockCap::start(vec![ + rule( + Route::Object("segments/video/segment_008"), + Fault::Status(503), + ) + .times(12), + ]) + .await; + scratch.sign_in(&mock.url); + let (video, audio) = instant_tracks(&scratch.0, INSTANT_PAIRS); + let (sender, events) = flume::unbounded(); + let feeder = tokio::spawn(feed(video.clone(), audio.clone(), sender)); + let result = upload_segments( + &LiveSegmentTransport, + INSTANT_VIDEO_ID, + events, + mock.cancel_flag(), + None, + ) + .await; + feeder.await.unwrap(); + assert_eq!(result, Ok(())); + assert_segments_stored(&mock, INSTANT_VIDEO_ID, &video, &audio); + assert_eq!(mock.confirmations().len(), 1); + let attempts = mock + .lock() + .requests + .iter() + .filter(|(method, path, ..)| { + method == "PUT" && path.ends_with("segments/video/segment_008.m4s") + }) + .map(|(_, _, status, at)| (*status, *at)) + .collect::>(); + assert_eq!(attempts.len(), 13, "{}", mock.timeline()); + assert!(attempts[..12].iter().all(|(status, _)| *status == 503)); + assert_eq!(attempts[12].0, 200); + for (round, seconds) in [2, 4, 8, 16].into_iter().enumerate() { + let last_failure = attempts[round * 3 + 2].1; + let next_attempt = attempts[round * 3 + 3].1; + assert!( + next_attempt.saturating_sub(last_failure) >= Duration::from_secs(seconds), + "deferred retry round {round} did not wait {seconds}s:\n{}", + mock.timeline() + ); + } + mock.assert_clean(); + }); +} + +/// The network drops while video segment 3 is half sent and stays down for +/// 5.7 s, past the in-call attempts. The live upload survives it. +#[test] +fn instant_live_outage_mid_segment() { + instant_case( + "instant_live_outage", + vec![rule( + Route::Object("segments/video/segment_003"), + Fault::Outage(0.5, Duration::from_millis(5_700)), + )], + budget::INSTANT_LIVE_OUTAGE, + |mock, run| { + assert_instant_complete(mock, &run); + assert!(mock.stats().reset_on_accept >= 1); + }, + ); +} + +#[test] +fn instant_live_slow_segment() { + instant_case( + "instant_live_slow_segment", + vec![rule( + Route::Object("segments/video/segment_002"), + Fault::Delay(Duration::from_millis(9_300)), + )], + budget::INSTANT_LIVE_SLOW_SEGMENT, + |mock, run| assert_instant_complete(mock, &run), + ); +} + +/// An outage that outlasts the 5 minute retry window fails the live upload +/// with the first failure once the window has passed, leaving the rest to the +/// queue. +#[test] +fn instant_live_outage_past_the_retry_window_fails() { + instant_case( + "instant_live_outage_past_window", + vec![rule( + Route::Object("segments/video/segment_003"), + Fault::Outage(0.5, Duration::from_secs(6 * 60)), + )], + budget::INSTANT_LIVE_OUTAGE_PAST_WINDOW, + |mock, run| { + let error = run.result.expect_err("upload outlived its retry window"); + assert!( + error.contains("segments/video/segment_003.m4s"), + "unexpected error: {error}" + ); + assert!( + (RETRY_WINDOW..RETRY_WINDOW + Duration::from_secs(30)).contains(&run.elapsed), + "gave up after {:?}, not once the retry window had passed", + run.elapsed + ); + assert!(mock.confirmations().is_empty()); + mock.assert_clean(); + }, + ); +} + +#[test] +fn instant_live_permanent_403_is_fatal() { + instant_case( + "instant_live_permanent_403", + vec![ + rule( + Route::Object("segments/video/segment_003"), + Fault::Status(403), + ) + .times(u32::MAX), + ], + budget::INSTANT_LIVE_PERMANENT_403, + |mock, run| { + let error = run.result.expect_err("a 403 must not be retried forever"); + assert!( + error.contains("segment_003.m4s") && error.contains("403"), + "unexpected error: {error}" + ); + assert!(mock.confirmations().is_empty()); + mock.assert_clean(); + }, + ); +} + +#[test] +fn instant_live_cancel_stops_without_completing() { + instant_case( + "instant_live_cancel", + vec![rule( + Route::Object("segments/video/segment_004"), + Fault::Trip(0.5), + )], + budget::INSTANT_LIVE_CANCEL, + |mock, run| { + assert_eq!( + run.result, + Err("Instant recording upload cancelled".to_string()) + ); + assert!(mock.confirmations().is_empty()); + assert_eq!(mock.count("PUT", "segments/video/segment_005"), 0); + mock.assert_clean(); + }, + ); +} + +#[test] +fn instant_live_completion_signal_retries() { + instant_case( + "instant_live_completion_500", + vec![rule(Route::Api("recording-complete"), Fault::Status(500)).times(2)], + budget::INSTANT_LIVE_COMPLETION_500, + |mock, run| { + assert_eq!(run.result, Ok(())); + assert_segments_stored(mock, INSTANT_VIDEO_ID, &run.video, &run.audio); + assert_eq!(mock.count("POST", "recording-complete"), 3); + mock.assert_clean(); + }, + ); +} diff --git a/crates/audio/src/voice.rs b/crates/audio/src/voice.rs index 32e786a3761..f4013c4189e 100644 --- a/crates/audio/src/voice.rs +++ b/crates/audio/src/voice.rs @@ -287,6 +287,38 @@ mod tests { } } + /// Guard sentinel A09 (scripts/verify/guards.toml): a render that does + /// not continue the previous one must sound exactly like a render by a + /// new enhancer (the reset in `render`, and the editor cache that never + /// reuses an enhancer across a seek). + #[test] + fn guard_a09_render_after_a_seek_equals_a_fresh_enhancer() { + ffmpeg::init().unwrap(); + // Voiced, syllable-modulated harmonics plus noise, so the render + // is not silence. + let mut source = signal(96_000 * 2, 2); + for (index, sample) in source.data.iter_mut().enumerate() { + let t = (index / 2) as f32 / 48_000.0; + let pitch = 140.0 + 30.0 * (t * 1.3).sin(); + let voice: f32 = (1..6) + .map(|h| (std::f32::consts::TAU * pitch * h as f32 * t).sin() / h as f32) + .sum(); + let syllables = (std::f32::consts::TAU * 4.0 * t).sin().max(0.0); + *sample += 0.25 * voice * syllables; + } + let mut reused = VoiceEnhancer::with_settings(2, 1.0, VoiceProfile::default()); + let _ = reused.render(&source, 0, 9_600); + let _ = reused.render(&source, 9_600, 9_600); + let after_seek = reused.render(&source, 120_000, 9_600); + let fresh = VoiceEnhancer::with_settings(2, 1.0, VoiceProfile::default()) + .render(&source, 120_000, 9_600); + assert!( + fresh.samples().iter().any(|v| v.abs() > 0.01), + "silent render" + ); + assert_eq!(after_seek.samples(), fresh.samples()); + } + #[test] fn leveling_preserves_chunk_boundaries_timing_and_peak_headroom() { ffmpeg::init().unwrap(); diff --git a/crates/editor/Cargo.toml b/crates/editor/Cargo.toml index e0bc091d4f6..5091c32fcf2 100644 --- a/crates/editor/Cargo.toml +++ b/crates/editor/Cargo.toml @@ -26,6 +26,10 @@ path = "examples/audio-output-idle-benchmark.rs" name = "editor-startup-benchmark" path = "examples/editor-startup-benchmark.rs" +[features] +# Offline audio output for the cap-verify suite. Never enabled by the app. +test-hooks = [] + [dependencies] cap-utils = { path = "../utils" } cap-media = { path = "../media" } diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index f15188a1df4..3204f89249b 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -1687,6 +1687,26 @@ impl> PrerenderedAudioBuffer { } } + /// Blocks until the next `samples` reads are rendered (or the render + /// finished or failed). Returns false on timeout or failure. + #[cfg(feature = "test-hooks")] + pub(crate) fn wait_until_rendered(&self, samples: usize, timeout: std::time::Duration) -> bool { + let deadline = std::time::Instant::now() + timeout; + loop { + let ready = match &self.mode { + PrerenderedAudioBufferMode::Progressive(buffer) => buffer.rendered(samples), + PrerenderedAudioBufferMode::Streaming(buffer) => buffer.rendered(samples), + }; + match ready { + Some(ready) if !ready && std::time::Instant::now() < deadline => { + std::thread::sleep(std::time::Duration::from_micros(200)); + } + Some(ready) => return ready, + None => return false, + } + } + } + pub fn set_playhead(&mut self, playhead_secs: f64) { match &mut self.mode { PrerenderedAudioBufferMode::Progressive(buffer) => buffer.set_playhead(playhead_secs), @@ -1845,6 +1865,35 @@ impl> ProgressiveAudioBuffer { } } + /// Whether the next `samples` reads are rendered; `None` once the + /// render can no longer produce them. + #[cfg(feature = "test-hooks")] + fn rendered(&self, samples: usize) -> Option { + let timeline = &self.timeline; + let start = self.read_position.min(timeline.samples.len()); + let end = (self.read_position + samples).min(timeline.samples.len()); + if start >= end || timeline.complete.load(Ordering::Acquire) { + return Some(true); + } + let primary_end = timeline.primary_end.load(Ordering::Acquire); + let wrap_end = timeline.wrap_end.load(Ordering::Acquire); + let primary = timeline.primary_start; + let covered = if start >= primary { + primary_end >= end + } else if end <= primary { + wrap_end >= end + } else { + wrap_end >= primary && primary_end >= end + }; + if covered { + Some(true) + } else if timeline.stop.load(Ordering::Acquire) { + None + } else { + Some(false) + } + } + pub fn set_playhead(&mut self, playhead_secs: f64) { let sample_position = (playhead_secs * self.sample_rate as f64) as usize * self.channels; self.read_position = sample_position.min(self.timeline.samples.len()); @@ -2056,6 +2105,18 @@ impl> StreamingAudioBuffer { } } + #[cfg(feature = "test-hooks")] + fn rendered(&self, samples: usize) -> Option { + if self.timeline.failed.load(Ordering::Acquire) { + return None; + } + Some( + self.timeline.published_generation.load(Ordering::Acquire) == self.generation + && self.timeline.published_end.load(Ordering::Acquire) + >= self.read_position + samples, + ) + } + fn set_playhead(&mut self, playhead_secs: f64) { let position = output_sample_index(playhead_secs, self.sample_rate, self.channels, usize::MAX); diff --git a/crates/editor/src/audio_output.rs b/crates/editor/src/audio_output.rs index f93c3958cc9..d7d5f7218f5 100644 --- a/crates/editor/src/audio_output.rs +++ b/crates/editor/src/audio_output.rs @@ -875,6 +875,23 @@ fn drain_source_commands>( } } +/// Re-anchors the audio on the video playhead when they drift apart. +fn sync_video_playhead>( + audio: &mut PrerenderedAudioBuffer, + playhead_rx: &mut watch::Receiver, + latency_secs: f64, +) { + if playhead_rx.has_changed().unwrap_or(false) { + let video_playhead = *playhead_rx.borrow_and_update(); + let audible_playhead = audio.current_audible_playhead(latency_secs); + let drift = (video_playhead - audible_playhead).abs(); + + if drift > 0.04 { + audio.set_playhead(video_playhead + latency_secs); + } + } +} + /// Renders one output block from the active source: applies the video /// playhead sync policy, fills the buffer and acknowledges the first /// consumed block. Shared by the live cpal callback and the headless sink so @@ -894,15 +911,7 @@ fn render_source_block>( } match &mut source.buffer { ActiveSourceBuffer::Ordinary(audio) => { - if source.playhead_rx.has_changed().unwrap_or(false) { - let video_playhead = *source.playhead_rx.borrow_and_update(); - let audible_playhead = audio.current_audible_playhead(latency_secs); - let drift = (video_playhead - audible_playhead).abs(); - - if drift > 0.04 { - audio.set_playhead(video_playhead + latency_secs); - } - } + sync_video_playhead(audio, &mut source.playhead_rx, latency_secs); audio.fill(buffer); } ActiveSourceBuffer::Preparing(audio) => { @@ -1274,6 +1283,129 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: info!("Headless audio output thread finished"); } +/// A sink with no device and no clock, for verification harnesses: the +/// caller pulls every block, and each pull first waits until the samples it +/// reads are rendered, so the PCM depends only on the sequence of plays, +/// seeks and pulls, never on render speed. Sources are built by the same +/// `install_source` as the device and headless sinks. +#[cfg(feature = "test-hooks")] +pub struct OfflineAudioSink { + source_rx: std_mpsc::Receiver>, + active: Option>, +} + +#[cfg(feature = "test-hooks")] +impl AudioOutput { + /// Output at [`HEADLESS_SAMPLE_RATE`] / [`HEADLESS_CHANNELS`] whose blocks + /// are pulled through the returned sink. `play` returns once the source + /// is installed. + pub fn new_offline() -> (Self, OfflineAudioSink) { + let (control_tx, control_rx) = std_mpsc::channel(); + let (source_tx, source_rx) = std_mpsc::channel(); + if let Err(e) = std::thread::Builder::new() + .name("cap-audio-offline".into()) + .spawn(move || control_thread_offline(control_rx, source_tx)) + { + error!("Failed to spawn offline audio output thread: {e}"); + } + ( + Self { + control_tx, + next_generation: AtomicU64::new(0), + }, + OfflineAudioSink { + source_rx, + active: None, + }, + ) + } +} + +#[cfg(feature = "test-hooks")] +impl OfflineAudioSink { + /// Renders the next interleaved block. Returns false (and silence) when + /// no source is installed or its samples did not render within + /// `timeout`. + pub fn pull(&mut self, block: &mut [f32], timeout: Duration) -> bool { + drain_source_commands(&mut self.active, &self.source_rx); + let Some(source) = self.active.as_mut() else { + block.fill(0.0); + return false; + }; + let mut rendered = true; + if let ActiveSourceBuffer::Ordinary(audio) = &mut source.buffer { + sync_video_playhead(audio, &mut source.playhead_rx, 0.0); + rendered = audio.wait_until_rendered(block.len(), timeout); + } + render_source_block(source, block, 0.0); + rendered + } +} + +#[cfg(feature = "test-hooks")] +fn control_thread_offline( + control_rx: std_mpsc::Receiver, + source_tx: std_mpsc::Sender>, +) { + let output_info = AudioInfo::new_raw( + AudioData::SAMPLE_FORMAT, + HEADLESS_SAMPLE_RATE, + HEADLESS_CHANNELS, + ); + while let Ok(msg) = control_rx.recv() { + match msg { + ControlMsg::Refresh { + spec, + generation, + revision, + retire_tx, + } => { + if let Err(error) = install_source::( + spec, + generation, + SourceAcknowledgement::Refresh(retire_tx, revision), + output_info, + false, + &source_tx, + ) { + error!(%error, "Could not update offline audio playback"); + } + } + ControlMsg::Play { + spec, + generation, + result_tx, + } => { + let (ack_tx, _ack_rx) = std_mpsc::channel(); + let ok = install_source::( + spec, + generation, + SourceAcknowledgement::Ordinary(ack_tx), + output_info, + false, + &source_tx, + ) + .inspect_err(|e| error!("Failed to install offline audio source: {e}")) + .is_ok(); + let _ = result_tx.send(ok); + } + ControlMsg::PreparePlayback { request, .. } => request.complete(false), + ControlMsg::PrepareProgressivePlayback { installation, .. } => { + installation.request.complete(false) + } + ControlMsg::StopPlayback { generation } => { + let _ = source_tx.send(SourceCommand::Remove { + generation: Some(generation), + }); + } + ControlMsg::EnsureStream | ControlMsg::HoldAwake | ControlMsg::ReleaseAwake => {} + ControlMsg::Retire(source) => drop(source), + ControlMsg::Realign(realign) => realign(), + ControlMsg::Shutdown => break, + } + } +} + fn handle_play(state: &mut Option, spec: Box, generation: u64) -> bool { if !ensure_stream(state) { return false; diff --git a/crates/editor/src/lib.rs b/crates/editor/src/lib.rs index 612c25cb4c9..16a618e214d 100644 --- a/crates/editor/src/lib.rs +++ b/crates/editor/src/lib.rs @@ -15,6 +15,8 @@ mod telemetry; mod thumbnail; pub use audio::{AudioRenderer, AudioSegment, MusicTracks}; +#[cfg(feature = "test-hooks")] +pub use audio_output::OfflineAudioSink; pub use audio_output::{ AudioOutput, HEADLESS_BLOCK_FRAMES, HEADLESS_CHANNELS, HEADLESS_SAMPLE_RATE, HeadlessAudioTap, PlaySpec, diff --git a/crates/enc-ffmpeg/src/base.rs b/crates/enc-ffmpeg/src/base.rs index 41b34700dce..c2a5626df20 100644 --- a/crates/enc-ffmpeg/src/base.rs +++ b/crates/enc-ffmpeg/src/base.rs @@ -328,6 +328,40 @@ fn normalize_input_pts(pts: i64, last_pts: Option) -> i64 { mod tests { use super::*; + /// Guard sentinel E02 (scripts/verify/guards.toml): a stream whose first + /// frame carries a non-zero pts is rebased to start at 0 and keeps its + /// frame spacing. Exports start at 0 anyway, so removing the rebase + /// changes no export output today; this checks the rebase itself. + #[test] + fn guard_e02_rebases_a_stream_whose_first_pts_is_not_zero() { + ffmpeg::init().unwrap(); + let mut encoder = ffmpeg::codec::context::Context::new().encoder(); + let mut base = EncoderBase::new(0); + let mut seen = Vec::new(); + for pts in [3_000_i64, 3_512, 4_024, 4_536] { + let mut frame = frame::Video::new(ffmpeg::format::Pixel::YUV420P, 2, 2); + frame.set_pts(Some(pts)); + base.update_pts(&mut frame, Duration::MAX, &mut encoder); + seen.push(frame.pts()); + } + assert_eq!(seen, [Some(0), Some(512), Some(1_024), Some(1_536)]); + + // Capture timestamps (time base 1/90000) starting 2.5 s in. + let mut video = ffmpeg::codec::context::Context::new() + .encoder() + .video() + .unwrap(); + video.set_time_base((1, 90_000)); + let mut base = EncoderBase::new(0); + let mut seen = Vec::new(); + for ms in [2_500_u64, 2_533, 2_567, 2_600] { + let mut frame = frame::Video::new(ffmpeg::format::Pixel::YUV420P, 2, 2); + base.update_pts(&mut frame, Duration::from_millis(ms), &mut video); + seen.push(frame.pts()); + } + assert_eq!(seen, [Some(0), Some(2_970), Some(6_030), Some(9_000)]); + } + #[test] fn normalize_input_pts_passes_monotonic_input_through() { // Any strictly-forward delta passes untouched, even sub-frame ones diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index d9417d61c1f..5a8304307e2 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -2003,6 +2003,91 @@ impl Message for CameraFeed { } } +/// Test-only camera feeds driven by a synthetic device instead of hardware, +/// so recording tests exercise the real feed, lock and pipeline paths. +#[cfg(feature = "test-utils")] +pub mod synthetic { + use super::*; + + impl Message for CameraFeed { + type Reply = (); + + async fn handle( + &mut self, + msg: crate::feeds::Stall, + _: &mut Context, + ) -> Self::Reply { + tokio::time::sleep(msg.0).await; + } + } + + pub struct SyntheticCamera { + feed: ActorRef, + _done_rx: std::sync::Mutex>, + } + + impl SyntheticCamera { + pub fn spawn(name: &str, video_info: VideoInfo) -> Self { + let (done_tx, done_rx) = mpsc::sync_channel(1); + let id = DeviceOrModelID::DeviceID(name.to_string()); + let camera_info = serde_json::from_value(serde_json::json!({ + "device_id": name, + "model_id": null, + "display_name": name + })) + .expect("synthetic camera info"); + let attached = AttachedState::new( + id.clone(), + InputConnected { + generation: 0, + id, + done_tx, + camera_info, + video_info, + settings: None, + }, + ); + let feed = CameraFeed::spawn(CameraFeed { + state: State::Open(OpenState { + connecting: None, + attached: Some(attached), + }), + ..CameraFeed::default() + }); + Self { + feed, + _done_rx: std::sync::Mutex::new(done_rx), + } + } + + pub async fn lock(&self) -> anyhow::Result { + self.feed + .ask(Lock) + .await + .map_err(|error| anyhow::anyhow!("synthetic camera lock: {error}")) + } + + pub async fn push_native(&self, frame: NativeCameraFrame) { + let _ = self.feed.tell(NewNativeFrame(frame)).await; + } + + /// Hands a frame over the way the macOS capture callback does + /// (`try_send`, never waiting for mailbox capacity). + pub fn try_push_native(&self, frame: NativeCameraFrame) -> crate::feeds::Admission { + crate::feeds::Admission::of(self.feed.tell(NewNativeFrame(frame)).try_send()) + } + + /// Blocks the feed actor for `duration` (scripted slow consumer). + pub fn stall(&self, duration: Duration) -> crate::feeds::Admission { + crate::feeds::Admission::of(self.feed.tell(crate::feeds::Stall(duration)).try_send()) + } + + pub async fn push(&self, frame: FFmpegVideoFrame) { + let _ = self.feed.tell(NewFrame(frame)).await; + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/recording/src/feeds/microphone.rs b/crates/recording/src/feeds/microphone.rs index 73cea2dd92d..df542c01fbf 100644 --- a/crates/recording/src/feeds/microphone.rs +++ b/crates/recording/src/feeds/microphone.rs @@ -3076,6 +3076,126 @@ impl Message for MicrophoneFeed { } } +/// Test-only microphone feeds driven by a synthetic device instead of +/// hardware, so recording tests exercise the real feed, lock and pipeline +/// paths with samples whose capture times are known exactly. +#[cfg(feature = "test-utils")] +pub mod synthetic { + use super::*; + + const STREAM_ID: u32 = 1; + + /// cpal has no public constructor for `InputCallbackInfo`, and a + /// synthetic device has no device callback. Consumers time samples from + /// `MicrophoneSamples::timestamp`; this info reads as stream instant 0. + fn zero_callback_info() -> InputCallbackInfo { + // SAFETY: `InputCallbackInfo` (cpal pinned by git rev) holds two + // `StreamInstant { secs: i64, nanos: u32 }` values and nothing else, + // so the all-zero bit pattern is a valid value (instant 0) whatever + // the field layout. + unsafe { std::mem::zeroed() } + } + + impl Message for MicrophoneFeed { + type Reply = (); + + async fn handle( + &mut self, + msg: crate::feeds::Stall, + _: &mut Context, + ) -> Self::Reply { + tokio::time::sleep(msg.0).await; + } + } + + pub struct SyntheticMicrophone { + feed: ActorRef, + sample_rate: u32, + channels: u16, + _done_rx: std::sync::Mutex>, + } + + impl SyntheticMicrophone { + /// A feed attached to an f32 input device named `name`. + pub fn spawn(name: &str, sample_rate: u32, channels: u16) -> Self { + let (done_tx, done_rx) = mpsc::sync_channel(1); + let (error_tx, _) = flume::unbounded(); + let mut feed = MicrophoneFeed::new(error_tx); + feed.input_id_counter = STREAM_ID; + feed.state = State::Open(OpenState { + connecting: None, + attached: Some(AttachedState { + id: STREAM_ID, + label: name.to_string(), + config: SupportedStreamConfig::new( + channels, + cpal::SampleRate(sample_rate), + cpal::SupportedBufferSize::Unknown, + SampleFormat::F32, + ), + buffer_size_frames: None, + done_tx, + }), + }); + Self { + feed: MicrophoneFeed::spawn(feed), + sample_rate, + channels, + _done_rx: std::sync::Mutex::new(done_rx), + } + } + + pub async fn lock(&self) -> anyhow::Result { + self.feed + .ask(Lock) + .await + .map_err(|error| anyhow::anyhow!("synthetic microphone lock: {error}")) + } + + /// Delivers interleaved f32 samples whose first frame was captured at + /// `timestamp`. + pub async fn push(&self, samples: &[f32], timestamp: Timestamp) { + let data = samples.iter().flat_map(|s| s.to_ne_bytes()).collect(); + let _ = self + .feed + .tell(MicrophoneSamples { + stream_id: STREAM_ID, + data, + format: SampleFormat::F32, + sample_rate: self.sample_rate, + channels: self.channels, + info: zero_callback_info(), + timestamp, + }) + .await; + } + + /// Hands samples over the way the capture callback does + /// (`enqueue_microphone_samples`: `try_send`, never waiting). + pub fn try_push(&self, samples: &[f32], timestamp: Timestamp) -> crate::feeds::Admission { + let data = samples.iter().flat_map(|s| s.to_ne_bytes()).collect(); + crate::feeds::Admission::of( + self.feed + .tell(MicrophoneSamples { + stream_id: STREAM_ID, + data, + format: SampleFormat::F32, + sample_rate: self.sample_rate, + channels: self.channels, + info: zero_callback_info(), + timestamp, + }) + .try_send(), + ) + } + + /// Blocks the feed actor for `duration` (scripted slow consumer). + pub fn stall(&self, duration: Duration) -> crate::feeds::Admission { + crate::feeds::Admission::of(self.feed.tell(crate::feeds::Stall(duration)).try_send()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/recording/src/feeds/mod.rs b/crates/recording/src/feeds/mod.rs index 3bf6a9a9cfe..09263bf3b28 100644 --- a/crates/recording/src/feeds/mod.rs +++ b/crates/recording/src/feeds/mod.rs @@ -1,2 +1,31 @@ pub mod camera; pub mod microphone; + +/// Outcome of a synthetic device's hand-off. Device callbacks never wait for +/// a feed: they `try_send` and the frame is lost when the feed's mailbox is +/// full or the feed has stopped. Test-only sources hand over the same way so +/// a feed that cannot keep up is visible as a rejection, not as a late +/// source. +#[cfg(feature = "test-utils")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Admission { + Accepted, + Full, + Closed, +} + +#[cfg(feature = "test-utils")] +impl Admission { + fn of(result: Result<(), kameo::error::SendError>) -> Self { + match result { + Ok(()) => Self::Accepted, + Err(kameo::error::SendError::MailboxFull(_)) => Self::Full, + Err(_) => Self::Closed, + } + } +} + +/// Test-only slow consumer: the feed actor handles nothing else for the +/// given time, as a feed blocked on a downstream consumer would. +#[cfg(feature = "test-utils")] +pub struct Stall(pub std::time::Duration); diff --git a/crates/rendering/Cargo.toml b/crates/rendering/Cargo.toml index cce5010de20..90557746a46 100644 --- a/crates/rendering/Cargo.toml +++ b/crates/rendering/Cargo.toml @@ -70,6 +70,8 @@ windows_0_58 = { package = "windows", version = "0.58", features = [ [dev-dependencies] cap-media-info = { path = "../media-info" } +cap-video-decode = { path = "../video-decode", features = ["scripted"] } +cap-verify-expect = { path = "../verify-expect" } tempfile = "3" pollster = "0.4.0" pretty_assertions = "1.4.1" diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 0703321ca98..d666771eaff 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -644,6 +644,10 @@ impl AVAssetReaderDecoder { sequential_access: Arc, ) { let dimensions = (|| { + #[cfg(test)] + if let Some((width, height)) = cap_video_decode::scripted::dimensions(&path) { + return Ok((width, height, YuvMatrix::default())); + } let input = format::input(&path).map_err(|error| error.to_string())?; let stream = input .streams() diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 67ed38fef7b..fdabc7e2d5f 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -23,6 +23,8 @@ mod ffmpeg; mod frame_converter; #[cfg(target_os = "windows")] mod media_foundation; +#[cfg(test)] +mod model; #[cfg(target_os = "macos")] pub mod multi_position; diff --git a/crates/rendering/src/decoder/model/expectations.toml b/crates/rendering/src/decoder/model/expectations.toml new file mode 100644 index 00000000000..80a6f350a0f --- /dev/null +++ b/crates/rendering/src/decoder/model/expectations.toml @@ -0,0 +1,1698 @@ +# Strict expected failures for the decoder model tests +# (cap-rendering decoder::model: `cargo test -p cap-rendering --lib decoder::model`). +# +# Format: the shared engine verification format (crates/verify-expect): +# [[xfail]] +# case = "" +# class = "" +# platforms = ["macos"] (optional; absent = every platform) +# seeds = "" (optional; absent = any seed) +# reason = "" +# link = "" +# +# Every listed seed must still fail with that class: a seed that passes fails +# the run with "remove this xfail", so fixes stay locked in. Any failure not +# listed here fails the run. After a deliberate change, rewrite this file from +# a run with `scripts/verify/bless.sh` (locally, or from a CI run's report +# artifacts with `--run `). Reasons and links of existing entries are +# kept; review the difference. +# +# Entries below were recorded on origin/main 5705ee851. "NEW:" marks bugs the +# model found that no open PR fixes. + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "future" +seeds = "0 7 16 49" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "15 19" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "grid-rounding" +seeds = "8 50" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "overread" +seeds = "1 2 4 5 7-17 19-25 27-29 31-40 42-44 46-52 54 56 58 59" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "0 3 30 57" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "stale" +seeds = "1 3 6 8 10 15 18-20 24 26 28-30 47 50 52-54 57-59" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/random-seeks/playback" +class = "stale" +platforms = ["macos"] +seeds = "7 49" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future; aVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "future" +seeds = "0 1 4 6 8 9 11-13 15-18 20 22 23 25-28" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "grid-rounding" +seeds = "0" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "grid-rounding" +platforms = ["macos"] +seeds = "23" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "overread" +seeds = "0-29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "resets" +platforms = ["macos"] +seeds = "0 1 4 6 8 11-13 15-18 20 22 23 25-27" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): reader starts and seeks exceed the budget of two per jump in the request stream later in the sequence (seed index 0 request 1815). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/export" +class = "stale" +platforms = ["macos"] +seeds = "15" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): a later request is answered with an older sample than the right one (seed index 15 request 3029). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/playback" +class = "future" +seeds = "1 4 10 11 14 17 19 24 25 27" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "19" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): an answer is the sample on screen at the exact request time instead of the grid frame (seed index 19 request 3265). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/playback" +class = "overread" +seeds = "0-4 6-29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/playback" +class = "resets" +seeds = "8 28" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera-fast/sequential/playback" +class = "resets" +platforms = ["macos"] +seeds = "1 4 10 11 14 17 19 24 25 27" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/export" +class = "future" +seeds = "0-2" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/export" +class = "grid-rounding" +platforms = ["macos"] +seeds = "0" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): an answer is the sample on screen at the exact request time instead of the grid frame (seed index 0 request 8094). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/export" +class = "overread" +seeds = "0-3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/export" +class = "resets" +platforms = ["macos"] +seeds = "0-2" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): reader starts and seeks exceed the budget of two per jump in the request stream later in the sequence (seed index 0 request 4610). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/playback" +class = "future" +seeds = "2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/playback" +class = "overread" +seeds = "0 2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/camera/sequential/playback" +class = "resets" +platforms = ["macos"] +seeds = "2 3" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): reader starts and seeks exceed the budget of two per jump in the request stream later in the sequence (seed index 2 request 7107). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "future" +seeds = "10 42 59 85 87 205 214 219 232 234 240 250 258 260" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "3 13 16 34 37 65 71 72 75 81 83 92 94 100-102 109 110 119 139 146 162 171 179 197 210 212 218 221 225 228 230 239 246 274 282 283 293 297" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "25 32 42 55 73 76 81 88 94 113 122 124 132 152 160 185 228 229 243 280 290" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "1-3 16 20 37 38 43 46 48 71 87 90 100 101 107 110 116 119 125 133 137 139 142 154 169 178 181 196 206 209 210 212 214 218 225 231-234 240 246 249 251 256 262 267 271 274 282 284 293 296 297" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "3 6 9-11 16 18 20-22 25 32 33 37 40 42 43 46 48 55 58 65 71 75 83 87-90 94 95 100 101 106-108 110-114 116 119 120 122 125 127 132 138-140 142 146 150 160 170 178 180 181 185 188 197 201 205 207-210 214 218 219 221 226 228-231 234 236 238 241 243 246 253 256 260 262 267 270 271 274 280 282 287 290 293 296 297" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 3 request 8). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "stale" +seeds = "1-3 6-9 11 13 15-22 25 27 28 32-35 37 38 40 41 43-51 53-55 58 65 66 68-73 75 76 79 81 83 84 86 88-92 94-96 100-103 106-114 116-122 124-129 131-133 136-140 142 145-147 149 150 153 154 156 157 160 162 164 169-173 175 176 178-181 185-188 191 196 197 201 204 206-210 212 218 221-231 233 235-239 241-244 246-249 251-257 262 264 265 267 268 270-275 277 278 280-284 287 289-291 293 295-299" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/keyframe-seeks/playback" +class = "stale" +platforms = ["macos"] +seeds = "42 87 214 232 234 240 258" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "future" +seeds = "52 221" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "0 33 224 238" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "grid-rounding" +seeds = "10 26 190 208 212 218 238 271 274" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "6 11 19 29 33 46 50 86 111 112 124 131 132 150 161 169 188 191 203 241 261 299" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "overread" +seeds = "54 79 146 184 190 197" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "6 8 10 11 15 19 20 26 29 33 35 36 38 41 42 46-48 55 58 62 64 70 71 75 76 83-86 92 93 96 97 102 110-112 118-120 122 124 126 131 132 138 143-145 150 151 154 155 160 161 167 169 170 174 181 188 189 191 193 196 202 203 216 218 226 230 232 238 241 261 262 271 272 274 275 281 283 299" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/random-seeks/playback" +class = "stale" +seeds = "0 2 6 8 10 11 14 15 19-27 29-36 38 41-43 45-48 50 55 57 58 62 64 67 69-71 74-77 80 82-87 91-93 95-97 99 100 102 107 110-112 114 115 118-124 126 127 129 131-133 136 138 140 141 143-148 150 151 154 155 157-161 163 165-167 169 170 172 174-176 179-181 183 185 188-193 196 201-203 205 207 212 214-216 218 220 224-226 229-236 238 240-243 245 248-251 254-259 261 262 264 269 271 272 274 275 277 281-283 286 291 296 299" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/scrub/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "83 177" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): an answer is the sample on screen at the exact request time instead of the grid frame (seed index 83 request 167). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/cfr/scrub/playback" +class = "overread" +seeds = "41 89 96" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/scrub/playback" +class = "overread" +platforms = ["macos"] +seeds = "15 17 65 70 71 76 83 154 168 169 181" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/scrub/playback" +class = "stale" +seeds = "1 2 5 7-9 13-15 17 19 22-24 27-29 34 36 37 40 43 48 53 55 56 58 59 61 62 65 66 70-73 76 78-81 83 84 86-88 91 93 99-101 105-107 109 110 113 115 120 123 128 129 131-137 142-144 148 149 151 152 154 155 157-160 162 166-169 171 173-175 177 180-182 184 185 189 190 196-199" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/sequential/playback" +class = "stale" +seeds = "23 115" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/tail/export" +class = "stale" +seeds = "1 7 54 60 69" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/tail/playback" +class = "stale" +seeds = "12 34 40 73 93" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/timeline/export" +class = "grid-rounding" +seeds = "16 28 45 57 93 128 137" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/timeline/export" +class = "grid-rounding" +platforms = ["macos"] +seeds = "36 40 52 54 58 60 70 75 117 142 148" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/timeline/export" +class = "overread" +seeds = "56 131" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/timeline/export" +class = "overread" +platforms = ["macos"] +seeds = "2 7-9 11 19 24 31 35-37 40 45 52-54 57 60 63 66 70 71 75 76 81 86 93 95 101 102 106 112 119 121 128-130 132-134 137 142 143 147 148" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/timeline/export" +class = "stale" +seeds = "2 7-12 14-17 19 23 24 26 28 30-33 35-40 42 44 45 47 49 52-55 57 58 60-63 66 69-71 75 76 81-83 86 88 90 93-95 97 98 101-103 106 108 112 113 117-119 121 122 125 126 128-130 132-134 137 142 143 147 148" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "future" +seeds = "17 51 64" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "future" +platforms = ["macos"] +seeds = "19" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "grid-rounding" +seeds = "16 32" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "overread" +platforms = ["macos"] +seeds = "6 12 16 28 38 42 48 65 75 78 79" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 6 request 225). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "stale" +seeds = "0 5 6 8 10 12 16 19 22-25 28 33 38 40 42 48 50 56 59 61 65 66 68 73 75 78 79" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/export" +class = "stale" +platforms = ["macos"] +seeds = "17" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/playback" +class = "future" +seeds = "45" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/playback" +class = "grid-rounding" +seeds = "40" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/playback" +class = "overread" +platforms = ["macos"] +seeds = "4 8 11 42 66 73 76" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 4 request 189). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/cfr/transition/playback" +class = "stale" +seeds = "4 6 8 11 14-16 20 21 23 25 29 35 36 41 42 58 60 64 66 68-70 73 74 76" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/dense/random-seeks/playback" +class = "future" +seeds = "132" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/dense/random-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "1 46 53 56 58 65 73 81 96 110 118 126 134 137 138 149" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 1 request 323). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/dense/random-seeks/playback" +class = "stale" +seeds = "0 1 3 5 9 13 15 18 25 28 30 32 34 36 37 39 43-47 49 51 53 56-59 61 62 64-66 69 73 74 76 79-82 87 89 91 96 97 100-102 105 107 109 110 114 115 117-119 122 126 127 131 134-140 143 149" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/export" +class = "future" +platforms = ["macos"] +seeds = "3 15 50" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): a later request is answered with a sample ahead of the right one (seed index 3 request 1622). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/export" +class = "overread" +seeds = "18 30 54 55" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/export" +class = "overread" +platforms = ["macos"] +seeds = "2 3 8 15 17 20 22 45 50 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/export" +class = "stale" +seeds = "2 3 8 15 17 19 20 22 30 37 39 42-45 50 54 55 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "future" +seeds = "30 33 38" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "future" +platforms = ["macos"] +seeds = "34 58 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "grid-rounding" +seeds = "7" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "25" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "overread" +seeds = "7 37 55" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "overread" +platforms = ["macos"] +seeds = "2 18 22-27 30 33 34 38 41 42 58 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/halfgrid/back-into-old-gap/playback" +class = "stale" +seeds = "2 4 7 18 20 22-27 29 34 36 37 39 41 42 45 47 55 58 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "future" +seeds = "36 40 61 67 79" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "28 39 74" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "grid-rounding" +seeds = "12 31 37 43" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "11 14 38 47 50 74" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "overread" +seeds = "0-7 10 13 16-21 23-29 32-37 39 40 42 44-49 51-60 64-75 77-79" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "11 12 14 15 22 38 41 43 50 61-63" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "stale" +seeds = "4 8 11-16 18 22 24 26-28 31 32 37-39 41-44 47 50 57 62-64 74" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/random-seeks/playback" +class = "stale" +platforms = ["macos"] +seeds = "61 67 79" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "future" +seeds = "2 5 20 21 23 24 28 38 41-43 45 52 55 57 63 72 77 78" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "overread" +seeds = "0-2 5-10 14 15 19-29 31 32 34 37 38 40-47 49 51-59 66 70-78" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "overread" +platforms = ["macos"] +seeds = "63" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "resets" +seeds = "29 75" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "resets" +platforms = ["macos"] +seeds = "42" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "stale" +seeds = "50 60 64 79" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/late-start/sequential/playback" +class = "stale" +platforms = ["macos"] +seeds = "24" +reason = "AVAssetReader requests before a late-starting stream's first frame read the reader one frame further ahead per request, and seeks into the stream start show the wrong frame" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/syn/edlist/export" +class = "future" +seeds = "0-7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/syn/edlist/export" +class = "overread" +seeds = "0-7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/syn/edlist/playback" +class = "future" +seeds = "2" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/syn/edlist/playback" +class = "overread" +seeds = "0-7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "future" +seeds = "51 56 60 61 63 86" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "future" +seeds = "28 53 59" +flaky = true +reason = "Seeds 28, 53 and 59 fail in every run; which wrong answer comes first depends on the wall clock. Without a stall the first wrong answer is a future one at a seek (seed 28 request 61, seed 53 request 8, seed 59 request 9: the next sample, e.g. sample 68 for 67): pool position capping, the reader's pool position is capped to the newest request (avassetreader.rs update_decoder_position(decoder_idx, capped)) so a reader that already decoded past the sample is reused without a reset. When the test thread stalls for 250 ms or more, the decoder's idle timeout (IDLE_READER_TIMEOUT, avassetreader.rs) clears the cache and reader state, the later requests take another path and the first wrong answer is a stale one from the wrong-GOP seek (reader started on the keyframe time rounded to 10 ms). Measured on Apple silicon: future 3 of 3 per seed without a stall, stale for all three seeds with a 300 ms stall before every request, and for seed 53 with a single 300 ms stall before request 6 (a 240 ms stall keeps future). macos-15: future in 6 of 7 runs, stale in 1 (38743345280). With #2595's decoder changes seed 28 passes and seeds 53 and 59 keep only the stale (wrong-GOP) answer" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "0 3 6 7 9 14 19 24 26 29 34 36 41 45 47 49 55 62 65 70 76 83 85 88 95 97" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "5 25 27 32 48 52 56 65 76 86 90 98" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "0 3 7 14 17 21 24 26 34 35 37 40 41 49 51 59 61 68 69 75 83 88 97 99" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "overread" +seeds = "65 76 86 98" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "0 6 7 14 17 19 21 25 28 29 32 34 37 39-41 47 48 51-53 56 60 61 63 64 68-71 75 78 83-85 87-89 94 95 97" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "stale" +seeds = "28 53 59" +flaky = true +reason = "Seeds 28, 53 and 59 fail in every run; which wrong answer comes first depends on the wall clock. Without a stall the first wrong answer is a future one at a seek (seed 28 request 61, seed 53 request 8, seed 59 request 9: the next sample, e.g. sample 68 for 67): pool position capping, the reader's pool position is capped to the newest request (avassetreader.rs update_decoder_position(decoder_idx, capped)) so a reader that already decoded past the sample is reused without a reset. When the test thread stalls for 250 ms or more, the decoder's idle timeout (IDLE_READER_TIMEOUT, avassetreader.rs) clears the cache and reader state, the later requests take another path and the first wrong answer is a stale one from the wrong-GOP seek (reader started on the keyframe time rounded to 10 ms). Measured on Apple silicon: future 3 of 3 per seed without a stall, stale for all three seeds with a 300 ms stall before every request, and for seed 53 with a single 300 ms stall before request 6 (a 240 ms stall keeps future). macos-15: future in 6 of 7 runs, stale in 1 (38743345280). With #2595's decoder changes seed 28 passes and seeds 53 and 59 keep only the stale (wrong-GOP) answer" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "stale" +seeds = "0-3 6 7 9 10 12-22 24-26 29-37 39-42 44-50 52 54 55 57 58 62 64-76 78-85 87-91 93-97 99" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/keyframe-seeks/playback" +class = "stale" +platforms = ["macos"] +seeds = "51 56 60 61 63" +reason = "Seeds 28, 53 and 59 fail in every run; which wrong answer comes first depends on the wall clock. Without a stall the first wrong answer is a future one at a seek (seed 28 request 61, seed 53 request 8, seed 59 request 9: the next sample, e.g. sample 68 for 67): pool position capping, the reader's pool position is capped to the newest request (avassetreader.rs update_decoder_position(decoder_idx, capped)) so a reader that already decoded past the sample is reused without a reset. When the test thread stalls for 250 ms or more, the decoder's idle timeout (IDLE_READER_TIMEOUT, avassetreader.rs) clears the cache and reader state, the later requests take another path and the first wrong answer is a stale one from the wrong-GOP seek (reader started on the keyframe time rounded to 10 ms). Measured on Apple silicon: future 3 of 3 per seed without a stall, stale for all three seeds with a 300 ms stall before every request, and for seed 53 with a single 300 ms stall before request 6 (a 240 ms stall keeps future). macos-15: future in 6 of 7 runs, stale in 1 (38743345280). With #2595's decoder changes seed 28 passes and seeds 53 and 59 keep only the stale (wrong-GOP) answer" +link = "https://github.com/CapSoftware/Cap/pull/2595 https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "future" +seeds = "7 17 23 76 84 97 100 105 109 114 115 119 135 136 148 174" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "future" +platforms = ["macos"] +seeds = "5 18 24 41 44 83 103 110 126 142 153" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "grid-rounding" +seeds = "77 87 114 133 156" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "97" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "overread" +seeds = "58 171 180 192" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "overread" +platforms = ["macos"] +seeds = "5 7 11 12 17 18 23 24 31 41 43 44 63 66 73 75-77 79 83 84 86 87 90 96 97 100 102-105 107 109 110 112 114-116 120 135-137 142 146 148 153 156 162 163 165 172 174 176 181 189 191 193" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "stale" +seeds = "5 6 10-12 18 20-22 24 25 31 35 36 41-44 48-50 58 61 63 66 69 72-75 77 79 81-83 86 87 89 90 96 99 101-104 107 110 112 116 120 122 123 125-127 131 132 137 142 143 146 151 153 156 157 159 162 163 165 171 172 176 179-181 189 191-194" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/random-seeks/playback" +class = "stale" +platforms = ["macos"] +seeds = "17 97 109 119" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "future" +seeds = "6 47 68 94" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "future" +platforms = ["macos"] +seeds = "16 22 24 26 34 45 57 97 101 110" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "68 70" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): an answer is the sample on screen at the exact request time instead of the grid frame (seed index 68 request 147). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "overread" +seeds = "28 36 48 66 77 114" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "overread" +platforms = ["macos"] +seeds = "7 9 16 22 24-26 34 37 40 45 50 62 68 69 72 74 84 87 89 90 101 110 118" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "stale" +seeds = "1 2 7 9 10 12 14 16 17 21-26 29 30 34 37 40 41 43-45 49 50 53 54 56-58 61 62 65 66 69 70 72-74 77-92 97 100-104 108-110 113 114 116 118 119" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/scrub/playback" +class = "stale" +platforms = ["macos"] +seeds = "94" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/sequential/export" +class = "stale" +seeds = "34 40 73 99" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/sequential/playback" +class = "stale" +seeds = "6 15 22 29 49 69 85" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/timeline/export" +class = "future" +seeds = "13 44 59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/timeline/export" +class = "grid-rounding" +seeds = "43 77 91 93 96" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/timeline/export" +class = "overread" +seeds = "59" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/timeline/export" +class = "overread" +platforms = ["macos"] +seeds = "12 13 15 18 27 37 43 44 51 72 82 91 96 102" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/timeline/export" +class = "stale" +seeds = "12 15 17 18 20 23 27 31 34 37 43 50 51 54 61 72 73 76 77 82 85 86 90 91 93 96 102 103 108 112 115" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "future" +seeds = "5 14 15 17 20 22 23 27 28 30 31 33 36 40 43 48 51 54 63-65 76-78" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "future" +platforms = ["macos"] +seeds = "18 35 41 42 44 50 56 58 62 68-70 74" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "grid-rounding" +seeds = "8" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "grid-rounding" +platforms = ["macos"] +seeds = "43 44" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "overread" +platforms = ["macos"] +seeds = "5 10 13 14 17 18 20 22 23 27 28 30 31 33 35-37 40-44 48 50 51 54 56 58 62 63 65 67-70 74 76-79" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 5 request 157). AVAssetReader decoder request handling, the mechanisms of this decoder's other entries (pool position capping, reader reuse after a reset, pts holes), which #2638 targets" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "stale" +seeds = "0 4 10 11 13 16 18 29 35 37 39 41 42 44 47 50 56-60 62 67-70 74 79" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/avf/vfr/transition/playback" +class = "stale" +platforms = ["macos"] +seeds = "17 28 31 51 63 65" +reason = "AVAssetReader seeks: the reader starts on the keyframe time rounded to 10 ms, so it can return the previous GOP's last frame under the keyframe's number, and a reused pooled reader answers from frames cached by an earlier, unrelated seek" +link = "https://github.com/CapSoftware/Cap/pull/2603" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/random-seeks/playback" +class = "future" +seeds = "7" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/random-seeks/playback" +class = "grid-rounding" +seeds = "2 11-13 20 23 26 33 47" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/random-seeks/playback" +class = "overread" +seeds = "7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/random-seeks/playback" +class = "overread" +seeds = "0-4 6 8-12 14-16 18 19 22-32 34-51 54-59" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/random-seeks/playback" +class = "stale" +seeds = "47" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/export" +class = "grid-rounding" +seeds = "4 11 22 27 29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/export" +class = "grid-rounding" +seeds = "3 6 28" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/export" +class = "overread" +seeds = "0-29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/export" +class = "stale" +seeds = "0 3-6 11 15 22 27-29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/playback" +class = "grid-rounding" +seeds = "2 6 24" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/playback" +class = "overread" +seeds = "0-8 10-14 16 17 19-21 23-29" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera-fast/sequential/playback" +class = "stale" +seeds = "0 2 6 11 24" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/export" +class = "grid-rounding" +seeds = "0 2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/export" +class = "grid-rounding" +seeds = "1" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/export" +class = "overread" +seeds = "0-3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/export" +class = "resets" +seeds = "0-3" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): reader starts and seeks exceed the budget of two per jump in the request stream later in the sequence (seed index 0 request 6042). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/export" +class = "stale" +seeds = "0-3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/playback" +class = "grid-rounding" +seeds = "2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/playback" +class = "overread" +seeds = "2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/playback" +class = "resets" +seeds = "2 3" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): reader starts and seeks exceed the budget of two per jump in the request stream later in the sequence (seed index 2 request 6210). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/camera/sequential/playback" +class = "stale" +seeds = "2 3" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "future" +seeds = "0 38 46 56 62 69 83 90 93 104 110 133 145 169 181 183 186 205 207 239 253 256 260 299" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "future" +seeds = "1 4 6 17 18 20 23 41 42 48 50 53 63 65 101 108 115 118 119 122 124 130 136 140 142 147 149-151 154 161 167 188 192 199-201 203 209 212 216 219 223 235 237 245 248 250 261 263 268 270 271 275 285 288 290 291" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "0 4 5 8 20 29 48 53 62 63 69 115 121 130 136 142 147 151 154 161 167 173 188 200 201 207 212 216 219 223 225 226 234 235 237-239 245 248 254 255 263 268 270 275 278 285 291 299" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "2 6 9 12 17 19 23 27 28 30 31 33 38 39 41 42 46 50 52 54 56 60 61 65 68 71 72 74 75 79 82-84 86 90 92 93 96 98 99 101-104 106-108 110 111 119 122 124 125 129 131 133 135 137 138 140 141 143 145 148-150 152 156 164 165 168-172 177-179 181-183 186 189-192 194 195 197-199 203 205 206 209-211 222 227 236 241 243 250 253 256 258 265 266 273 274 283 284 287 290 294 295 297" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "overread" +seeds = "0 4-8 12 15 16 19 20 26 31 37 46 47 51 53 62 63 69 70 72 80 88 91 95 100 101 111 112 115 117 121 125 127 130 136 138 142 147 153 155 161 167 170 172 173 175 183 196 200 202-204 207-212 216 219 225-227 230 234 235 237-241 244 245 253-256 263 265 275 278 285 286 291 292 294 297 299" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "overread" +seeds = "2 11 14 27 29 30 33 38 39 50 52 54 56 59-61 67 68 71 74 82 83 86 90 92 93 96 98 99 102-104 106 108 110 113 119 126 129 131 133 135 137 139 144 145 148 152 156 162 164 165 168 169 171 177-179 181 182 185 186 189 190 197 198 205 222 229 236 243 247 258 260 266 274 287 295" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "stale" +seeds = "4 6 20 53 63 101 108 115 119 130 136 147 154 161 200 203 209 212 216 219 235 237 245 275 291" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/keyframe-seeks/playback" +class = "stale" +seeds = "0 2 5 7 8 12 14-16 19 26 27 29-31 33 37-39 46 47 51 52 54 56 59-62 67-72 74 80 82 83 86 88 90-93 95 96 98-100 102-104 106 110-113 117 121 125-127 129 131 133 135 137-139 144 145 148 152 153 155 156 162 164 165 168-173 175 177-179 181-183 186 189 190 196-198 202 204 205 207 208 210 211 215 222 225-227 229 230 234 236 238-241 243 244 247 253-256 258-260 265 269 274 278 286 287 292 294 295 297 299" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "future" +seeds = "60" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "grid-rounding" +seeds = "67 100 254 279" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "grid-rounding" +seeds = "13 31 40 52 57 61 63 69 74 80 93 117 120 126 140 153 166 180 182 188 196 251 258 276 299" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "overread" +seeds = "0 4 10-12 16 18 25 27 30 32 42 50 55 67 73 82 89 100 108 126 128 129 134 137-139 146 147 151 164 166 171 174 178-181 207 217 220 228 232 246 247 250 254 262 273 279 281 289 290 297" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "overread" +seeds = "3 9 13 64 95 101 106 140 148 163 187 204 227 287 299" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/random-seeks/playback" +class = "stale" +seeds = "0 4 10-13 16 18 20 25 27 30 32 42 43 50 55 61 62 67 73 82 89 100 108 126 128 129 134 137-140 146-148 151 164 166 171 173 174 178-181 184 198 207 217 220 228 232 233 243 246 247 250 254 258 262 267 273 279 281 285 289 290 297" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "future" +seeds = "88" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): a later request is answered with a sample ahead of the right one (seed index 88 request 220). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "grid-rounding" +seeds = "27 33 39 41 50 68 86 90 93 101 104 109 121 123 124 133 144 149 155 160 165 170 173 179 180 183 186 192 199" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "grid-rounding" +seeds = "30 62 63 84 106 122 126 143 176 187" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "overread" +seeds = "1 4 7 8 11 14 16-20 26 27 36-38 40-43 46 52 53 55 56 58 64 68 70 71 73 74 77 78 80 82 83 85 88 90-94 98-101 105 107 110 113 114 117 121 124 127 128 131-133 135 137 141 142 145 148 149 151 152 154 155 157 162 163 167 173-175 177 179 181 183 187 189 192 193 198" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "overread" +seeds = "0 3 13 21 31 33-35 39 44 50 51 54 67 75 84 86 97 103 104 106 108 109 111 115 118 119 122 123 134 138-140 144 147 158-161 165 168 170 176 180 185 186 197" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/scrub/playback" +class = "stale" +seeds = "0 1 3 4 7 8 11 13-22 26 27 33 35-43 46 49-56 58 60 63 64 66-68 70-75 77 78 80-86 88 90-95 97-101 104-110 113-115 117-119 121-125 127 128 131-135 137 139 141 142 144 145 147-152 154 155 157 160-165 167 168 170 173-177 179-181 183 185-189 191-194 197-199" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/tail/export" +class = "grid-rounding" +seeds = "26" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/timeline/export" +class = "future" +seeds = "29 50" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/timeline/export" +class = "grid-rounding" +seeds = "16 26 32 50 65 75 95 102 133 145" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/timeline/export" +class = "overread" +seeds = "12 42 43 64 133" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/timeline/export" +class = "overread" +seeds = "53 66 103 107 132" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/timeline/export" +class = "stale" +seeds = "12 35 42 43 64 133 135" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "future" +seeds = "73" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "future" +seeds = "6 66" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "grid-rounding" +seeds = "0 15 22 23 25 29 66 73" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "grid-rounding" +seeds = "5-7 38 41 45 53 56 61 62 72" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "overread" +seeds = "4 5 19 22 23 26 29 52 60 73" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 4 request 108). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/export" +class = "stale" +seeds = "0-2 4 5 7 15 19 22-26 29 33-36 38 40 41 52 55 60 62 67 73-75 79" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/playback" +class = "future" +seeds = "69" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/playback" +class = "grid-rounding" +seeds = "14 20 21 25 31 53 54 57 61 66" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/playback" +class = "grid-rounding" +seeds = "9 19 29 42 51 58 64 65" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/playback" +class = "overread" +seeds = "6 25 27 39 50 57 70" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): the reader decodes past the sample after the answer, running ahead of the requests (seed index 6 request 48). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/cfr/transition/playback" +class = "stale" +seeds = "3 6 7 9 14 20 21 25 27 31 36 37 39 43-45 50 53 54 57 59 61 64-66 70 77" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "future" +seeds = "90 121" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "grid-rounding" +seeds = "17 22 33" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "grid-rounding" +seeds = "1 7 15 21 25 32 35 36 45 47 48 55 64 70 82 86 90 91 100 103 114 115 121 126 147" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "overread" +seeds = "0 4 14 15 17 20 22 25 27 29 31 32 38 43 50 54 56 57 61 65 67 71 84 91 92 97 99 102 114 116 123 126 128 130 135 137 140 144" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "overread" +seeds = "33 86" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/dense/random-seeks/playback" +class = "stale" +seeds = "0 4 14 15 17 20 22 25 27 29 31-33 38 39 43 50 54 56 57 61 63 65 67 71 84 86 91-93 97 99 102 114 116 123 126 128 130 135 137 140 144" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/halfgrid/back-into-old-gap/export" +class = "overread" +seeds = "23 41" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/halfgrid/back-into-old-gap/playback" +class = "overread" +seeds = "20" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/syn/edlist/export" +class = "overread" +seeds = "0-7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/syn/edlist/playback" +class = "overread" +seeds = "0-7" +reason = "Camera runaway: when the pts-hole map is full it evicts the narrowest hole, so with two requests per source frame the reader reads one frame further ahead per hole until answers come from the future" +link = "https://github.com/CapSoftware/Cap/pull/2595" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "future" +seeds = "3 8 10 46 61 68 69 86 89 99" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "future" +seeds = "2 4 5 9 11 14 20-22 24-27 29 31 32 34 36 37 39-41 48 50 53-56 58 59 62-65 67 70 71 73 76 79 82 84 88 90 91 93-98" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "2 10 13 21 22 24 28 32 34 36 37 40 42 46 48 53-55 63-65 68 69 73 76 88 90 91 93-95" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "grid-rounding" +seeds = "1 3-5 8 9 11 15 16 18 23 25 27 29-31 41 43 44 47 50 51 56-62 66 70-72 74 75 79-87 89 92 96 97 99" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "overread" +seeds = "1-5 7 8 10-14 19-21 24 25 29 34 36 40-43 48 51 53 55 59 62 64 68 73 79 81 86 91 94 96" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "overread" +seeds = "6 9 15 17 18 23 28 30 31 44 46 47 50 52 57 58 60 61 66 69 72 75 76 78 83 84 87 89 97 99" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "stale" +seeds = "2 4 9 11 14 20 21 24 25 29 31 34 40 41 48 55 59 62 64 73 76 79 84 91 93 94 96" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/keyframe-seeks/playback" +class = "stale" +seeds = "1 3 7 8 10 12 13 15 17-19 23 28 30 42-44 46 47 51 52 57 60 61 66 68 69 72 75 78 81 83 86 87 89 99" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/random-seeks/playback" +class = "future" +seeds = "62" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/random-seeks/playback" +class = "grid-rounding" +seeds = "3 8 10 12 16 20 34 46 52 58 59 64 77 78 86 95 117 137 144 160 168 185 196" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/random-seeks/playback" +class = "overread" +seeds = "3 12 18 20 29 53 55 67 79 84 93 120 148 167 177" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/random-seeks/playback" +class = "overread" +seeds = "11 80 101 102 104 110 130 157 171 180" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/random-seeks/playback" +class = "stale" +seeds = "3 12 18 20 29 53 55 67 79 80 84 93 120 148 167 177" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "future" +seeds = "114" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "grid-rounding" +seeds = "2 37 59 67 81 111" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "grid-rounding" +seeds = "23 30 46 48 78 87 114" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "overread" +seeds = "1 7 12 18 33 34 36 37 43 45 46 49 56 67 70-72 81 101 103 105 111 112" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "overread" +seeds = "2 4 10 15 19 23 25 28 38 53 54 59 69 75 80 88 91 95 98 107 110 114 119" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/scrub/playback" +class = "stale" +seeds = "0-2 7 8 12 15 17-19 21 23 33 34 36 37 43 45 46 49 53 54 56 59 66 67 69-72 75 80 81 86 88 91 95 101-103 105 107 108 110-112 119" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/sequential/playback" +class = "grid-rounding" +seeds = "25" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/timeline/export" +class = "future" +seeds = "25" +reason = "NEW: the FFmpeg decoder seeks with the request time rounded to the stream tick, so a request a fraction of a tick before a keyframe lands on that keyframe and is answered with a later frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/timeline/export" +class = "grid-rounding" +seeds = "20" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/timeline/export" +class = "grid-rounding" +seeds = "77 110" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/timeline/export" +class = "overread" +seeds = "0 20 47 56 62 88 107 110 119" +reason = "NEW: after a backward request served from the FFmpeg decoder's cache, the reader is left far ahead and the next request reads past the frame it needs" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/timeline/export" +class = "stale" +seeds = "20 56 119" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/transition/playback" +class = "future" +seeds = "66 75 77" +reason = "Found once each sequence keeps running after its first wrong answer (full failure sets, runs 38851482697 and 38851482675): a later request is answered with a sample ahead of the right one (seed index 66 request 206). FFmpeg decoder request handling, the mechanisms of this decoder's other entries (backward lookback tolerance, first-frame shortcut, reset decisions)" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/transition/playback" +class = "grid-rounding" +seeds = "10 12 13 66 71 75" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/transition/playback" +class = "grid-rounding" +seeds = "15 16 19 20 28 33 34 43 47 48 56 59 62 68 70 77" +reason = "NEW: the FFmpeg decoder seeks to the exact request time, so after a reset it answers with the frame on screen at that time even when its rounded grid label is after the request; sequential playback shows the previous frame for the same request" +link = "" + +[[xfail]] +case = "decoder-model/ffmpeg/vfr/transition/playback" +class = "stale" +seeds = "4-6 8 10 12 13 18 19 21 28 31 33 38 39 41 45-47 49 50 52 55 56 58 59 61 66 68 71 75-77 79" +reason = "NEW: the FFmpeg decoder answers backward requests with a cached frame up to 2 grid frames older than the requested one, answers grid frames 0 to 2 with the first frame, and after a backward request served from the cache it reads forward from a reader that is still far ahead, answering with an older cached frame" +link = "" + diff --git a/crates/rendering/src/decoder/model/generate.rs b/crates/rendering/src/decoder/model/generate.rs new file mode 100644 index 00000000000..1a00a6b6881 --- /dev/null +++ b/crates/rendering/src/decoder/model/generate.rs @@ -0,0 +1,552 @@ +use cap_video_decode::scripted::{Script, ScriptFrame}; + +use super::{Mode, Request}; + +/// SplitMix64: small, fast and stable across platforms and Rust versions. +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Self(seed) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + + pub fn unit(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + + pub fn range(&mut self, low: f64, high: f64) -> f64 { + low + (high - low) * self.unit() + } + + pub fn chance(&mut self, p: f64) -> bool { + self.unit() < p + } + + pub fn pick(&mut self, items: &[T]) -> T { + items[self.below(items.len())] + } +} + +pub fn family_seed(family: &str, index: u64) -> u64 { + let mut hash = 0xCBF2_9CE4_8422_2325u64; + for byte in family.bytes() { + hash = (hash ^ byte as u64).wrapping_mul(0x0100_0000_01B3); + } + hash ^ index.wrapping_mul(0x9E37_79B9_7F4A_7C15) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SourceKind { + /// Constant rate with optional jitter, at or below the decoder rate. + Cfr, + /// Source faster than the decoder grid: several samples share a label. + Dense, + /// Low-light webcam: nominal 30 fps, delivers about 15 with jitter, long. + Camera, + /// Like `Camera` with many 3 and 4 frame gaps and a shorter duration. + CameraFast, + /// Screen recording with static holds of random length. + Vfr, + /// 30 fps screen read on a 60 fps grid (every gap is a pts hole) with long + /// holds early in the file. + HalfGrid, + /// The first sample starts well after zero. + LateStart, + /// The synthetic screen behind the #2595 backward-jump report: 60 fps + /// grid, gaps of 1 to 20 grid frames, a 10 s hold at 30 s and a 2.5 s + /// hold at 95 s, 120 s long. + Syn, +} + +impl SourceKind { + pub fn name(self) -> &'static str { + match self { + Self::Cfr => "cfr", + Self::Dense => "dense", + Self::Camera => "camera", + Self::CameraFast => "camera-fast", + Self::Vfr => "vfr", + Self::HalfGrid => "halfgrid", + Self::LateStart => "late-start", + Self::Syn => "syn", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PatternKind { + Sequential, + Timeline, + Transition, + RandomSeeks, + KeyframeSeeks, + Scrub, + BackIntoOldGap, + Tail, + /// The editor list from the #2595 report: play 5 to 35 s, jump to 100 to + /// 110 s, jump back to 28 to 45 s, at 60 fps. + Edlist, +} + +impl PatternKind { + pub fn name(self) -> &'static str { + match self { + Self::Sequential => "sequential", + Self::Timeline => "timeline", + Self::Transition => "transition", + Self::RandomSeeks => "random-seeks", + Self::KeyframeSeeks => "keyframe-seeks", + Self::Scrub => "scrub", + Self::BackIntoOldGap => "back-into-old-gap", + Self::Tail => "tail", + Self::Edlist => "edlist", + } + } +} + +pub struct Source { + pub script: Script, + /// Decoder grid rate (the recording metadata fps). + pub fps: u32, + pub max_fallback_distance: u32, +} + +const TIME_BASES: [i32; 6] = [600, 1000, 15360, 30000, 90000, 1_000_000]; +const GOPS: [usize; 7] = [1, 10, 30, 60, 120, 240, 600]; + +fn script_from_times(rng: &mut Rng, times: &[f64], nominal_interval: f64) -> Script { + let den = rng.pick(&TIME_BASES); + let gop = rng.pick(&GOPS); + let key_after_gap = rng.chance(0.5); + script_with(rng, times, nominal_interval, den, gop, key_after_gap) +} + +fn script_with( + rng: &mut Rng, + times: &[f64], + nominal_interval: f64, + den: i32, + gop: usize, + key_after_gap: bool, +) -> Script { + let mut frames: Vec = Vec::with_capacity(times.len()); + let mut since_key = 0usize; + for (index, &time) in times.iter().enumerate() { + let mut pts = (time * den as f64).round() as i64; + if let Some(previous) = frames.last() { + pts = pts.max(previous.pts + 1); + } + let gap = index + .checked_sub(1) + .map_or(0.0, |previous| time - times[previous]); + let keyframe = + index == 0 || since_key + 1 >= gop || (key_after_gap && gap > 0.5 && rng.chance(0.7)); + since_key = if keyframe { 0 } else { since_key + 1 }; + frames.push(ScriptFrame { pts, keyframe }); + } + Script { + time_base: (1, den), + frames, + last_duration: ((nominal_interval * den as f64).round() as i64).max(1), + } +} + +fn steady_times( + rng: &mut Rng, + start: f64, + duration: f64, + interval: impl Fn(&mut Rng) -> f64, +) -> Vec { + let mut times = Vec::new(); + let mut time = start; + while time < start + duration { + times.push(time); + time += interval(rng).max(0.001); + } + times +} + +pub fn source(rng: &mut Rng, kind: SourceKind) -> Source { + let screen = 4; + let camera = 2; + match kind { + SourceKind::Cfr => { + let (source_fps, fps) = rng.pick(&[(30.0, 30), (60.0, 60), (24.0, 30), (25.0, 30)]); + let jitter = rng.pick(&[0.0, 0.0, 0.1, 0.3]); + let duration = rng.range(2.0, 20.0); + let interval = 1.0 / source_fps; + let times = steady_times(rng, 0.0, duration, |rng| { + interval * (1.0 + jitter * rng.range(-1.0, 1.0)) + }); + Source { + script: script_from_times(rng, ×, interval), + fps, + max_fallback_distance: screen, + } + } + SourceKind::Dense => { + let source_fps = rng.pick(&[50.0, 60.0]); + let duration = rng.range(2.0, 15.0); + let interval = 1.0 / source_fps; + let times = steady_times(rng, 0.0, duration, |rng| { + interval * (1.0 + 0.1 * rng.range(-1.0, 1.0)) + }); + Source { + script: script_from_times(rng, ×, interval), + fps: 30, + max_fallback_distance: screen, + } + } + SourceKind::Camera | SourceKind::CameraFast => { + let (p1, p3, p4, duration) = if kind == SourceKind::Camera { + ( + rng.range(0.0, 0.05), + rng.range(0.05, 0.15), + rng.range(0.0, 0.02), + rng.range(120.0, 200.0), + ) + } else { + ( + rng.range(0.0, 0.05), + rng.range(0.25, 0.45), + rng.range(0.02, 0.1), + rng.range(40.0, 80.0), + ) + }; + let times = steady_times(rng, 0.0, duration, |rng| { + let roll = rng.unit(); + let frames = if roll < p1 { + 1.0 + } else if roll < p1 + p3 { + 3.0 + } else if roll < p1 + p3 + p4 { + 4.0 + } else { + 2.0 + }; + (frames + rng.range(-0.15, 0.15)) / 30.0 + }); + Source { + script: script_from_times(rng, ×, 2.0 / 30.0), + fps: 30, + max_fallback_distance: camera, + } + } + SourceKind::Vfr => { + let fps = rng.pick(&[30, 60]); + let interval = if fps == 60 && rng.chance(0.5) { + 1.0 / 30.0 + } else { + 1.0 / fps as f64 + }; + let hold_chance = rng.range(0.002, 0.02); + let duration = rng.range(10.0, 90.0); + let times = steady_times(rng, 0.0, duration, |rng| { + if rng.chance(hold_chance) { + let hold = rng.pick(&[0.1, 0.4, 1.0, 2.5, 5.0, 12.0]); + hold * rng.range(0.5, 1.5) + } else { + interval * (1.0 + 0.05 * rng.range(-1.0, 1.0)) + } + }); + Source { + script: script_from_times(rng, ×, interval), + fps, + max_fallback_distance: screen, + } + } + SourceKind::HalfGrid => { + let duration = rng.range(40.0, 120.0); + let hold_count = 1 + rng.below(3); + let mut holds: Vec<(f64, f64)> = (0..hold_count) + .map(|_| (rng.range(1.0, duration * 0.3), rng.range(2.0, 12.0))) + .collect(); + holds.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut times = Vec::new(); + let mut time = 0.0; + let mut next_hold = 0; + while time < duration { + times.push(time); + if next_hold < holds.len() && time >= holds[next_hold].0 { + time += holds[next_hold].1; + next_hold += 1; + } else { + time += (1.0 + 0.05 * rng.range(-1.0, 1.0)) / 30.0; + } + } + Source { + script: script_from_times(rng, ×, 1.0 / 30.0), + fps: 60, + max_fallback_distance: screen, + } + } + SourceKind::Syn => { + let mut times = Vec::new(); + let mut time = 0.0; + let holds = [(29.95, 10.08), (94.98, 2.53)]; + let mut next_hold = 0; + while time < 120.0 { + times.push(time); + if next_hold < holds.len() && time >= holds[next_hold].0 { + time += holds[next_hold].1; + next_hold += 1; + continue; + } + let mut grid_frames = 1.0; + while grid_frames < 20.0 && rng.chance(0.64) { + grid_frames += 1.0; + } + time += grid_frames / 60.0; + } + let gop = 20 + rng.below(20); + Source { + script: script_with(rng, ×, 1.0 / 60.0, 600, gop, false), + fps: 60, + max_fallback_distance: screen, + } + } + SourceKind::LateStart => { + let start = rng.range(0.3, 4.0); + let interval = rng.pick(&[1.0 / 30.0, 2.0 / 30.0]); + let duration = rng.range(3.0, 15.0); + let times = steady_times(rng, start, duration, |rng| { + interval * (1.0 + 0.1 * rng.range(-1.0, 1.0)) + }); + Source { + script: script_from_times(rng, ×, interval), + fps: 30, + max_fallback_distance: camera, + } + } + } +} + +fn secs(script: &Script, ticks: i64) -> f64 { + script.secs(ticks) +} + +fn at(time: f64) -> f32 { + time as f32 +} + +struct Builder { + requests: Vec, + playback: bool, +} + +impl Builder { + fn seek(&mut self, time: f64) { + self.requests.push(Request { + time: at(time.max(0.0)), + initial: self.playback, + }); + } + + fn play(&mut self, from: f64, frames: usize, out_fps: f64) { + for index in 0..frames { + self.requests.push(Request { + time: at((from + index as f64 / out_fps).max(0.0)), + initial: false, + }); + } + } + + fn seek_and_play(&mut self, from: f64, frames: usize, out_fps: f64) { + self.seek(from); + self.play(from + 1.0 / out_fps, frames, out_fps); + } +} + +/// Source time requests for one sequence. Times stay inside the stream except +/// for `Tail`, which plays past the last sample. +pub fn requests(rng: &mut Rng, source: &Source, kind: PatternKind, mode: Mode) -> Vec { + let script = &source.script; + let end = secs(script, script.end_pts()); + let first = secs(script, script.frames[0].pts); + let fps = source.fps as f64; + let out_fps = rng.pick(&[30.0, 60.0, 60.0]); + let mut builder = Builder { + requests: Vec::new(), + playback: mode == Mode::Playback, + }; + let frames_until = + |from: f64, to: f64, out_fps: f64| ((to - from) * out_fps).floor().max(0.0) as usize; + + match kind { + PatternKind::Sequential => { + let from = if mode == Mode::Playback && rng.chance(0.5) { + rng.range(0.0, end * 0.5) + } else { + 0.0 + }; + builder.seek(from); + builder.play( + from + 1.0 / out_fps, + frames_until(from + 1.0 / out_fps, end, out_fps), + out_fps, + ); + } + PatternKind::Tail => { + let from = (end - rng.range(0.5, 3.0)).max(0.0); + builder.seek(from); + builder.play( + from, + frames_until(from, end + rng.range(0.2, 1.0), out_fps), + out_fps, + ); + } + PatternKind::Timeline => { + for _ in 0..2 + rng.below(7) { + let start = rng.range(0.0, end); + let length = rng.range(0.2, 6.0).min(end - start); + let speed = rng.pick(&[0.5, 1.0, 1.0, 1.0, 1.5, 2.0]); + let count = (length / speed * out_fps).floor() as usize; + for index in 0..count { + builder.requests.push(Request { + time: at(start + index as f64 * speed / out_fps), + initial: false, + }); + } + } + } + PatternKind::Transition => { + for _ in 0..1 + rng.below(3) { + let a = rng.range(0.0, end * 0.9); + let b = rng.range(0.0, end * 0.9); + let overlap = 10 + rng.below(50); + for index in 0..overlap { + let step = index as f64 / out_fps; + if a + step < end { + builder.requests.push(Request { + time: at(a + step), + initial: false, + }); + } + if b + step < end { + builder.requests.push(Request { + time: at(b + step), + initial: false, + }); + } + } + let after = b + overlap as f64 / out_fps; + if after < end { + builder.play( + after, + frames_until(after, end, out_fps).min(rng.below(60)), + out_fps, + ); + } + } + } + PatternKind::RandomSeeks => { + for _ in 0..5 + rng.below(36) { + let from = rng.range(0.0, end); + let frames = frames_until(from, end, out_fps).min(rng.below(41)); + builder.seek_and_play(from, frames, out_fps); + } + } + PatternKind::KeyframeSeeks => { + let keyframes: Vec = script + .frames + .iter() + .filter(|frame| frame.keyframe) + .map(|frame| secs(script, frame.pts)) + .collect(); + for _ in 0..10 + rng.below(51) { + let keyframe = rng.pick(&keyframes); + let from = match rng.below(4) { + 0 => keyframe, + 1 => (keyframe * fps).round() / fps, + 2 => ((keyframe * fps).round() + rng.pick(&[-1.0, 1.0])) / fps, + _ => keyframe + rng.range(-0.5, 0.5) / fps, + } + .clamp(0.0, end - 0.001); + let frames = frames_until(from, end, out_fps).min(rng.below(6)); + builder.seek_and_play(from, frames, out_fps); + } + } + PatternKind::Scrub => { + let mut time = rng.range(0.0, end); + builder.seek(time); + for _ in 0..50 + rng.below(251) { + let magnitude = if rng.chance(0.6) { + rng.range(1.0, 6.0) + } else { + rng.range(6.0, 90.0) + }; + let step = magnitude.round() / fps * if rng.chance(0.5) { 1.0 } else { -1.0 }; + time = (time + step).clamp(0.0, end - 0.001); + if step.abs() * fps > 1.5 { + builder.seek(time); + } else { + builder.requests.push(Request { + time: at(time), + initial: false, + }); + } + } + } + PatternKind::Edlist => { + for (from, to) in [(300, 2100), (6000, 6600), (1680, 2700)] { + for frame in from..to { + let time = frame as f64 / 60.0; + if time < end { + builder.requests.push(Request { + time: at(time), + initial: frame == from && mode == Mode::Playback, + }); + } + } + } + } + PatternKind::BackIntoOldGap => { + let holds: Vec<(f64, f64)> = script + .frames + .windows(2) + .map(|pair| (secs(script, pair[0].pts), secs(script, pair[1].pts))) + .filter(|(start, next)| next - start > 1.5) + .collect(); + let (hold_start, hold_end) = + holds.first().copied().unwrap_or((first, first + 1.0 / fps)); + let mut position = 0.0; + for _ in 0..1 + rng.below(3) { + let target = (hold_end + rng.range(3.0, 30.0)).min(end - 0.01); + if target > position { + builder.seek(position); + builder.play( + position + 1.0 / out_fps, + frames_until(position + 1.0 / out_fps, target, out_fps), + out_fps, + ); + position = target; + } + let inside = rng.range(hold_start, hold_end); + let frames = frames_until(inside, end, out_fps).min(30 + rng.below(90)); + builder.seek_and_play(inside, frames, out_fps); + if rng.chance(0.5) { + position = inside + rng.range(0.0, 2.0); + } + } + } + } + + if mode == Mode::Export { + for request in &mut builder.requests { + request.initial = false; + } + } + builder.requests +} diff --git a/crates/rendering/src/decoder/model/mod.rs b/crates/rendering/src/decoder/model/mod.rs new file mode 100644 index 00000000000..b29dac20e08 --- /dev/null +++ b/crates/rendering/src/decoder/model/mod.rs @@ -0,0 +1,1119 @@ +//! Model-based tests of the decoder request logic. +//! +//! The real decoders (AVAssetReader pool on macOS, FFmpeg everywhere) are +//! spawned through `spawn_decoder` on scripted media +//! (`cap_video_decode::scripted`): sample lists with exact presentation times +//! and sync flags. Seeded request sequences (export and playback order, double +//! requests at 60 fps over 30 fps grids, timelines with cuts, speed changes +//! and backward jumps, transitions, random seeks, keyframe seeks, scrubs, +//! jumps back into old pts holes, reads past the end) run against them, and +//! every answer is checked against the reference rule below. +//! +//! Reference rule (the decoder contract documented next to `pts_to_frame`): +//! a sample is labelled `round(fps * pts_seconds)` on the decoder grid, where +//! FFmpeg measures pts from the stream start time and AVAssetReader from +//! zero. A request for time `t` asks for grid index `floor(t * fps)`. The +//! correct answer is the newest sample whose label is at or before that index +//! (the frame on screen at `t`); when several samples share that label any of +//! them is accepted. A request before the first sample is answered with the +//! first sample. +//! +//! An answer that is not the grid frame but is the sample on screen at the +//! exact request time is `grid-rounding`: correct for a seek straight to `t`, +//! but different from what sequential playback shows for the same request. +//! Other wrong answers are `future` or `stale`. The sequence continues after +//! any wrong answer, so a seed's recorded classes are its full failure set +//! (an expected first failure cannot hide a later, different one). Every +//! request also checks that the reader did not decode past the sample after +//! the answer (`overread`: the reader running ahead of the requests) and that +//! reader starts and seeks stay within a budget of two per jump in the request +//! stream (`resets`); these, like `grid-rounding`, are recorded once and the +//! sequence continues. A request that gets no frame is `dropped`, and the +//! macOS AVAssetReader decoder falling back to FFmpeg is `fallback`. +//! +//! Known failures are strict expected failures, per seed, in +//! `expectations.toml`. +//! +//! Environment: +//! - `CAP_DECODER_MODEL_SCALE`: multiplies every family's seed count. +//! - `CAP_DECODER_MODEL_FILTER`: only run cases whose name contains this. +//! - `CAP_DECODER_MODEL_CASE` + `CAP_DECODER_MODEL_SEED_INDEX`: replay one +//! sequence and print every request. +//! - `CAP_DECODER_MODEL_SHRINK`: `none`, `new` (default, unexpected failures +//! only) or `all`. +//! - `CAP_DECODER_MODEL_THREADS`: worker threads (at most 3 for +//! AVAssetReader, 4 for FFmpeg). +//! - `CAP_DECODER_MODEL_REPORT=`: also write the report to +//! `..txt`; `CAP_DECODER_MODEL_SEEDS_OUT=` writes +//! every seed's outcome to `..tsv`. +//! - `CAP_VERIFY_REPORT_DIR=`: write one JSON record per case for the +//! CI summary and for `cap-verify-expect bless` (see +//! `crates/verify-expect`), which rewrites `expectations.toml` to match. +//! +//! Wall-clock inputs of the decoders (the AVAssetReader idle timeout of +//! 250 ms and the scrub detector's 100 ms cooldown) only matter when a +//! request is delayed by that long, so outcomes are deterministic unless the +//! machine stalls the test threads. + +mod generate; + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Write as _, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; + +use cap_video_decode::scripted::{self, Script}; + +use generate::{PatternKind, Rng, SourceKind}; + +#[cfg(target_os = "macos")] +use super::DecoderType; +use super::spawn_decoder; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Decoder { + #[cfg_attr(not(target_os = "macos"), allow(dead_code))] + Avf, + Ffmpeg, +} + +impl Decoder { + fn name(self) -> &'static str { + match self { + Self::Avf => "avf", + Self::Ffmpeg => "ffmpeg", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + /// `prefer_sequential_access`, like export. + Export, + /// Editor playback: seeks use the initial request path. + Playback, +} + +impl Mode { + fn name(self) -> &'static str { + match self { + Self::Export => "export", + Self::Playback => "playback", + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Request { + pub time: f32, + pub initial: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum FailureClass { + GridRounding, + Future, + Stale, + Dropped, + Fallback, + Unidentified, + Overread, + Resets, +} + +impl FailureClass { + fn name(self) -> &'static str { + match self { + Self::GridRounding => "grid-rounding", + Self::Future => "future", + Self::Stale => "stale", + Self::Dropped => "dropped", + Self::Fallback => "fallback", + Self::Unidentified => "unidentified", + Self::Overread => "overread", + Self::Resets => "resets", + } + } +} + +#[derive(Clone, Debug)] +struct Failure { + class: FailureClass, + request: usize, + detail: String, +} + +#[derive(Clone)] +struct Case { + name: String, + seed_index: u64, + seed: u64, + decoder: Decoder, + mode: Mode, + fps: u32, + max_fallback_distance: u32, + script: Script, + requests: Vec, +} + +struct Outcome { + /// The first occurrence of each class over the whole sequence + /// (`Future`, `Stale`, `GridRounding`, `Overread`, `Resets`), then at + /// most one terminal failure (`Dropped`, `Fallback`, `Unidentified`) + /// that ended it. + failures: Vec, + requests: usize, + starts: u64, +} + +struct Oracle { + labels: Vec, + secs: Vec, + fps: u32, + decoder: Decoder, +} + +fn label(pts: i64, time_base: (i32, i32), fps: u32) -> u32 { + (fps as f64 * ((pts as f64 * time_base.0 as f64) / time_base.1 as f64)).round() as u32 +} + +impl Oracle { + fn new(script: &Script, decoder: Decoder, fps: u32) -> Self { + let origin = match decoder { + Decoder::Avf => 0, + Decoder::Ffmpeg => script.frames[0].pts, + }; + Self { + labels: script + .frames + .iter() + .map(|frame| label(frame.pts - origin, script.time_base, fps)) + .collect(), + secs: script + .frames + .iter() + .map(|frame| script.secs(frame.pts - origin)) + .collect(), + fps, + decoder, + } + } + + /// The newest sample presented at or before `time`, ignoring the grid: + /// what a seek straight to `time` shows. + fn on_screen(&self, time: f32) -> usize { + let time = match self.decoder { + Decoder::Avf => time, + Decoder::Ffmpeg => time.max(0.0), + } as f64; + self.secs + .partition_point(|&secs| secs <= time) + .saturating_sub(1) + } + + fn index(&self, time: f32) -> u32 { + let time = match self.decoder { + Decoder::Avf => time, + Decoder::Ffmpeg => time.max(0.0), + }; + (time * self.fps as f32).floor() as u32 + } + + /// Inclusive range of acceptable sample indices for a request. + fn expect(&self, index: u32) -> (usize, usize) { + let after = self.labels.partition_point(|&label| label <= index); + let newest = after.saturating_sub(1); + let label = self.labels[newest]; + let oldest = self.labels.partition_point(|&other| other < label); + (oldest, newest) + } + + fn last_label(&self) -> u32 { + *self.labels.last().unwrap() + } +} + +/// Records the first occurrence of a failure class that does not end the +/// sequence. +fn note(failures: &mut Vec, class: FailureClass, request: usize, detail: String) { + if !failures.iter().any(|failure| failure.class == class) { + failures.push(Failure { + class, + request, + detail, + }); + } +} + +fn frame_identity(frame: &crate::DecodedFrame) -> Option { + let data = frame.data(); + scripted::frame_id(&data[..data.len().min(16)]) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +fn run_case(case: &Case, runtime: &tokio::runtime::Runtime, trace: bool) -> Outcome { + let registration = scripted::register(case.script.clone()); + let oracle = Oracle::new(&case.script, case.decoder, case.fps); + let path = registration.path().to_path_buf(); + + runtime.block_on(async { + let mut failures: Vec = Vec::new(); + let fail = |mut failures: Vec, class, request, detail: String| { + failures.push(Failure { + class, + request, + detail, + }); + Outcome { + failures, + requests: request + 1, + starts: registration.reader_starts() + registration.seeks(), + } + }; + let handle = match spawn_decoder( + "model", + path, + case.fps, + 0.0, + case.decoder == Decoder::Ffmpeg, + ) + .await + { + Ok(handle) => handle.with_max_fallback_distance(case.max_fallback_distance), + Err(error) => { + return fail( + failures, + FailureClass::Dropped, + 0, + format!("spawn failed: {error}"), + ); + } + }; + if case.mode == Mode::Export { + handle.prefer_sequential_access(); + } + let decoder_type = handle.decoder_type(); + #[cfg(target_os = "macos")] + if case.decoder == Decoder::Avf { + assert_eq!(decoder_type, DecoderType::AVAssetReader); + } + registration.take_vended(); + + let mut previous: Option<(u32, usize)> = None; + let mut discontinuities = 0u64; + let mut past_end = 0u64; + for (position, request) in case.requests.iter().enumerate() { + let index = oracle.index(request.time); + let (oldest, newest) = oracle.expect(index); + if previous.is_none_or(|(previous_index, previous_sample)| { + index < previous_index + || index > previous_index + 2 + || case.script.secs(case.script.frames[newest].pts) + - case.script.secs(case.script.frames[previous_sample].pts) + > LONG_HOLD_SECS + }) { + discontinuities += 1; + } + if index > oracle.last_label() { + past_end += 1; + } + previous = Some((index, newest)); + + let frame = if request.initial { + handle.get_frame_initial(request.time).await + } else { + handle.get_frame(request.time).await + }; + let vended = registration.take_vended(); + let starts = registration.reader_starts() + registration.seeks(); + let describe = |got: Option| { + let mut text = format!( + "request {position}: t={:?} ({}) -> grid {index}, expected sample {}..={} (label {}), got {}", + request.time, + if request.initial { "initial" } else { "normal" }, + oldest, + newest, + oracle.labels[newest], + got.map_or("nothing".to_string(), |got| format!( + "sample {got} (label {})", + oracle.labels.get(got as usize).copied().unwrap_or(u32::MAX) + )), + ); + let shown: Vec = vended.iter().take(12).map(u32::to_string).collect(); + let _ = write!( + text, + "; vended [{}{}], reader starts+seeks {starts}", + shown.join(", "), + if vended.len() > 12 { ", ..." } else { "" } + ); + text + }; + + if handle.decoder_type() != decoder_type { + return fail( + failures, + FailureClass::Fallback, + position, + format!("decoder switched to {}; {}", handle.decoder_type(), describe(None)), + ); + } + let Some(frame) = frame else { + return fail(failures, FailureClass::Dropped, position, describe(None)); + }; + let Some(got) = frame_identity(&frame) else { + return fail( + failures, + FailureClass::Unidentified, + position, + describe(None), + ); + }; + if trace { + eprintln!("{}", describe(Some(got))); + } + let got_index = got as usize; + if !(oldest..=newest).contains(&got_index) { + if got_index == oracle.on_screen(request.time) { + note( + &mut failures, + FailureClass::GridRounding, + position, + describe(Some(got)), + ); + } else if got_index > newest { + note( + &mut failures, + FailureClass::Future, + position, + describe(Some(got)), + ); + } else { + note( + &mut failures, + FailureClass::Stale, + position, + describe(Some(got)), + ); + } + } + if let Some(&furthest) = vended.iter().max() + && furthest as usize > newest.max(got_index) + 1 + { + note( + &mut failures, + FailureClass::Overread, + position, + describe(Some(got)), + ); + } + let allowed = 2 + 2 * discontinuities + past_end / 10; + if starts > allowed { + note( + &mut failures, + FailureClass::Resets, + position, + format!( + "{} (allowed {allowed} for {discontinuities} discontinuities)", + describe(Some(got)) + ), + ); + } + } + Outcome { + failures, + requests: case.requests.len(), + starts: registration.reader_starts() + registration.seeks(), + } + }) +} + +/// Sequential requests that cross a hold this long count as a jump: the +/// AVAssetReader pool repositions a reader once a request is 5 s past it. +const LONG_HOLD_SECS: f64 = 4.0; + +struct Family { + source: SourceKind, + pattern: PatternKind, + mode: Mode, + seeds: usize, + avf_only: bool, +} + +const fn family(source: SourceKind, pattern: PatternKind, mode: Mode, seeds: usize) -> Family { + Family { + source, + pattern, + mode, + seeds, + avf_only: false, + } +} + +const FAMILIES: &[Family] = { + use Mode::*; + use PatternKind::*; + use SourceKind::*; + &[ + family(Cfr, Sequential, Export, 120), + family(Cfr, Sequential, Playback, 120), + family(Cfr, Timeline, Export, 150), + family(Cfr, Transition, Export, 80), + family(Cfr, Transition, Playback, 80), + family(Cfr, RandomSeeks, Playback, 300), + family(Cfr, KeyframeSeeks, Playback, 300), + family(Cfr, Scrub, Playback, 200), + family(Cfr, Tail, Export, 100), + family(Cfr, Tail, Playback, 100), + family(Dense, Sequential, Export, 60), + family(Dense, RandomSeeks, Playback, 150), + family(Camera, Sequential, Export, 4), + family(Camera, Sequential, Playback, 4), + family(CameraFast, Sequential, Export, 30), + family(CameraFast, Sequential, Playback, 30), + family(CameraFast, RandomSeeks, Playback, 60), + family(Vfr, Sequential, Export, 100), + family(Vfr, Sequential, Playback, 100), + family(Vfr, Timeline, Export, 120), + family(Vfr, RandomSeeks, Playback, 200), + family(Vfr, KeyframeSeeks, Playback, 100), + family(Vfr, Scrub, Playback, 120), + family(Vfr, Transition, Playback, 80), + family(HalfGrid, Sequential, Export, 20), + family(HalfGrid, BackIntoOldGap, Playback, 60), + family(HalfGrid, BackIntoOldGap, Export, 60), + family(Syn, Edlist, Playback, 8), + family(Syn, Edlist, Export, 8), + Family { + source: LateStart, + pattern: Sequential, + mode: Playback, + seeds: 80, + avf_only: true, + }, + Family { + source: LateStart, + pattern: RandomSeeks, + mode: Playback, + seeds: 80, + avf_only: true, + }, + ] +}; + +fn case_name(decoder: Decoder, family: &Family) -> String { + format!( + "decoder-model/{}/{}/{}/{}", + decoder.name(), + family.source.name(), + family.pattern.name(), + family.mode.name() + ) +} + +fn build_case(decoder: Decoder, family: &Family, seed_index: u64) -> Case { + let name = case_name(decoder, family); + let seed = generate::family_seed(&name, seed_index); + let mut rng = Rng::new(seed); + let source = generate::source(&mut rng, family.source); + let requests = generate::requests(&mut rng, &source, family.pattern, family.mode); + Case { + name, + seed_index, + seed, + decoder, + mode: family.mode, + fps: source.fps, + max_fallback_distance: source.max_fallback_distance, + script: source.script, + requests, + } +} + +fn fails_with(case: &Case, class: FailureClass, runtime: &tokio::runtime::Runtime) -> bool { + run_case(case, runtime, false) + .failures + .iter() + .any(|failure| failure.class == class) +} + +/// Delta-debugging over the request list, then trims the script tail. +fn shrink(case: &Case, failure: &Failure, runtime: &tokio::runtime::Runtime) -> Case { + let deadline = Instant::now() + Duration::from_secs(20); + let class = failure.class; + let mut best = case.clone(); + best.requests.truncate(failure.request + 1); + + let mut chunks = 2usize; + while best.requests.len() >= 2 && Instant::now() < deadline { + let size = best.requests.len().div_ceil(chunks); + let mut reduced = false; + for start in (0..best.requests.len()).step_by(size) { + let mut candidate = best.clone(); + candidate + .requests + .drain(start..(start + size).min(candidate.requests.len())); + if !candidate.requests.is_empty() && fails_with(&candidate, class, runtime) { + best = candidate; + chunks = (chunks - 1).max(2); + reduced = true; + break; + } + if Instant::now() >= deadline { + break; + } + } + if !reduced { + if chunks >= best.requests.len() { + break; + } + chunks = (chunks * 2).min(best.requests.len()); + } + } + + let oracle = Oracle::new(&best.script, best.decoder, best.fps); + let needed = best + .requests + .iter() + .map(|request| oracle.expect(oracle.index(request.time)).1 + 2) + .max() + .unwrap_or(1); + if needed < best.script.frames.len() { + let mut candidate = best.clone(); + candidate.script.frames.truncate(needed); + if fails_with(&candidate, class, runtime) { + best = candidate; + } + } + best +} + +fn describe_script(script: &Script) -> String { + let mut runs: Vec<(i64, usize)> = Vec::new(); + for pair in script.frames.windows(2) { + let delta = pair[1].pts - pair[0].pts; + match runs.last_mut() { + Some((last, count)) if *last == delta => *count += 1, + _ => runs.push((delta, 1)), + } + } + let deltas: Vec = runs + .iter() + .map(|(delta, count)| { + if *count == 1 { + delta.to_string() + } else { + format!("{delta}x{count}") + } + }) + .collect(); + let keyframes: Vec = script + .frames + .iter() + .enumerate() + .filter(|(_, frame)| frame.keyframe) + .map(|(index, _)| index.to_string()) + .collect(); + format!( + "time_base {}/{}, {} samples, first pts {}, pts deltas [{}], sync samples [{}], last duration {}", + script.time_base.0, + script.time_base.1, + script.frames.len(), + script.frames[0].pts, + deltas.join(" "), + keyframes.join(" "), + script.last_duration + ) +} + +fn describe_requests(requests: &[Request]) -> String { + requests + .iter() + .map(|request| { + if request.initial { + format!("{:?}i", request.time) + } else { + format!("{:?}", request.time) + } + }) + .collect::>() + .join(" ") +} + +struct FamilyResult { + name: String, + seeds: usize, + requests: usize, + starts: u64, + elapsed: Duration, + failures: Vec<(Case, Failure)>, + outcomes: Vec<(u64, Vec<(FailureClass, usize)>)>, +} + +fn env_usize(name: &str) -> Option { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) +} + +const EXPECTATIONS: &str = "crates/rendering/src/decoder/model/expectations.toml"; + +fn write_records( + decoder: Decoder, + results: &[FamilyResult], + ran: &cap_verify_expect::Ran, + verdict: &cap_verify_expect::Verdict, + platform: &str, +) { + use cap_verify_expect::{Outcome, Record}; + if cap_verify_expect::report_dir().is_none() { + return; + } + let records: Vec<(Record, serde_json::Value)> = results + .iter() + .map(|result| { + let mut failures: BTreeMap> = BTreeMap::new(); + let mut problems: BTreeMap = BTreeMap::new(); + for (case, failure) in &result.failures { + let class = failure.class.name().to_string(); + failures + .entry(class.clone()) + .or_default() + .insert(case.seed_index); + problems.entry(class).or_insert_with(|| { + format!( + "seed index {} request {}: {}", + case.seed_index, failure.request, failure.detail + ) + }); + } + let new: Vec<&cap_verify_expect::Unexpected> = verdict + .unexpected + .iter() + .filter(|found| found.case == result.name) + .collect(); + let fixed: Vec<&cap_verify_expect::Fixed> = verdict + .fixed + .iter() + .filter(|fixed| fixed.case == result.name) + .collect(); + let outcome = if !new.is_empty() { + Outcome::Fail + } else if !fixed.is_empty() { + Outcome::Xpass + } else if failures.is_empty() { + Outcome::Pass + } else { + Outcome::Xfail + }; + let mut summary: Vec = new + .iter() + .map(|found| { + format!( + "{} on seeds {}: {}", + found.class, + cap_verify_expect::format_seeds(&found.seeds), + problems.get(&found.class).map_or("", String::as_str) + ) + }) + .collect(); + summary.extend(fixed.iter().map(|fixed| fixed.message(EXPECTATIONS))); + if summary.is_empty() { + summary.push(format!( + "{} seeds, {} requests{}", + result.seeds, + result.requests, + if failures.is_empty() { + String::new() + } else { + format!( + ", expected failures: {}", + failures.keys().cloned().collect::>().join(", ") + ) + } + )); + } + let replay_seed = new + .first() + .and_then(|found| found.seeds.first().copied()) + .or_else(|| { + fixed + .first() + .and_then(|fixed| fixed.seeds.as_ref()?.first().copied()) + }); + let test = match decoder { + Decoder::Avf => "avassetreader_answers_match_the_reference_model", + Decoder::Ffmpeg => "ffmpeg_answers_match_the_reference_model", + }; + let repro = match replay_seed { + Some(seed) => format!( + "CAP_DECODER_MODEL_CASE={} CAP_DECODER_MODEL_SEED_INDEX={seed} cargo test -p cap-rendering --lib decoder::model::{test} -- --nocapture", + result.name + ), + None => format!( + "CAP_DECODER_MODEL_FILTER={} cargo test -p cap-rendering --lib decoder::model::{test} -- --nocapture", + result.name + ), + }; + let record = Record { + suite: "decoder-model".into(), + expectations: EXPECTATIONS.into(), + case: result.name.clone(), + platform: platform.into(), + tier: std::env::var("CAP_VERIFY_TIER").unwrap_or_else(|_| "fast".into()), + outcome, + failures, + seeds_ran: ran.get(&result.name).cloned().flatten(), + unobserved: Default::default(), + problems, + measures: Default::default(), + excused: Default::default(), + starved: false, + summary: summary.join("\n"), + repro, + seconds: result.elapsed.as_secs_f64(), + }; + (record, serde_json::Value::Null) + }) + .collect(); + cap_verify_expect::write_records(&records); +} + +fn run_suite(decoder: Decoder) { + let xfails = cap_verify_expect::parse(include_str!("expectations.toml")) + .unwrap_or_else(|e| panic!("{EXPECTATIONS}: {e}")); + let filter = std::env::var("CAP_DECODER_MODEL_FILTER").ok(); + let scale: f64 = std::env::var("CAP_DECODER_MODEL_SCALE") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(1.0); + + if let (Ok(name), Some(seed_index)) = ( + std::env::var("CAP_DECODER_MODEL_CASE"), + env_usize("CAP_DECODER_MODEL_SEED_INDEX"), + ) { + let Some(family) = FAMILIES + .iter() + .find(|family| case_name(decoder, family) == name) + else { + return; + }; + let case = build_case(decoder, family, seed_index as u64); + eprintln!( + "replaying {} seed index {} (seed {:#x}): {}", + case.name, + seed_index, + case.seed, + describe_script(&case.script) + ); + let outcome = run_case(&case, &runtime(), true); + for failure in &outcome.failures { + eprintln!("{} {}", failure.class.name(), failure.detail); + } + assert!(outcome.failures.is_empty()); + return; + } + + let families: Vec<&Family> = FAMILIES + .iter() + .filter(|family| decoder == Decoder::Avf || !family.avf_only) + .filter(|family| { + filter + .as_ref() + .is_none_or(|filter| case_name(decoder, family).contains(filter.as_str())) + }) + .filter(|family| cap_verify_expect::case_selected(&case_name(decoder, family))) + .collect(); + let jobs: Vec<(usize, u64)> = families + .iter() + .enumerate() + .flat_map(|(family_index, family)| { + let seeds = ((family.seeds as f64 * scale).ceil() as u64).max(1); + (0..seeds).map(move |seed| (family_index, seed)) + }) + .collect(); + + let started = Instant::now(); + let next = AtomicUsize::new(0); + let results = Mutex::new( + families + .iter() + .map(|family| FamilyResult { + name: case_name(decoder, family), + seeds: 0, + requests: 0, + starts: 0, + elapsed: Duration::ZERO, + failures: Vec::new(), + outcomes: Vec::new(), + }) + .collect::>(), + ); + // AVAssetReader pools share 16 reader permits per process and grow to 5 + // readers each, so more than 3 concurrent sequences would let one + // sequence's pool growth depend on another's and make outcomes vary. + let max_threads = match decoder { + Decoder::Avf => 3, + Decoder::Ffmpeg => 4, + }; + let threads = env_usize("CAP_DECODER_MODEL_THREADS") + .unwrap_or_else(|| { + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(2) + }) + .clamp(1, max_threads); + std::thread::scope(|scope| { + for _ in 0..threads { + scope.spawn(|| { + let runtime = runtime(); + loop { + let job = next.fetch_add(1, Ordering::Relaxed); + let Some(&(family_index, seed_index)) = jobs.get(job) else { + break; + }; + let case = build_case(decoder, families[family_index], seed_index); + let case_started = Instant::now(); + let outcome = run_case(&case, &runtime, false); + let mut results = results.lock().unwrap(); + let result = &mut results[family_index]; + result.seeds += 1; + result.requests += outcome.requests; + result.starts += outcome.starts; + result.elapsed += case_started.elapsed(); + result.outcomes.push(( + seed_index, + outcome + .failures + .iter() + .map(|failure| (failure.class, failure.request)) + .collect(), + )); + for failure in outcome.failures { + result.failures.push((case.clone(), failure)); + } + } + }); + } + }); + let mut results = results.into_inner().unwrap(); + for result in &mut results { + result + .failures + .sort_by_key(|(case, failure)| (failure.class, case.seed_index)); + } + + if let Ok(path) = std::env::var("CAP_DECODER_MODEL_SEEDS_OUT") { + let mut lines = String::new(); + for result in &mut results { + result.outcomes.sort_by_key(|(seed_index, _)| *seed_index); + for (seed_index, outcome) in &result.outcomes { + let _ = writeln!( + lines, + "{}\t{}\t{}", + result.name, + seed_index, + if outcome.is_empty() { + "ok".to_string() + } else { + outcome + .iter() + .map(|(class, request)| format!("{} {}", class.name(), request)) + .collect::>() + .join(", ") + } + ); + } + } + let _ = std::fs::write(format!("{path}.{}.tsv", decoder.name()), lines); + } + + let ran: cap_verify_expect::Ran = results + .iter() + .map(|result| { + let seeds = result.outcomes.iter().map(|(seed, _)| *seed).collect(); + (result.name.clone(), Some(seeds)) + }) + .collect(); + let mut observed = cap_verify_expect::Observed::new(); + for result in &results { + for (case, failure) in &result.failures { + observed + .entry((result.name.clone(), failure.class.name().to_string())) + .or_default() + .insert(case.seed_index); + } + } + let platform = cap_verify_expect::platform(); + let verdict = cap_verify_expect::judge(&xfails, platform, &ran, &observed); + let unexpected: BTreeSet<(String, String, u64)> = verdict + .unexpected + .iter() + .flat_map(|found| { + found + .seeds + .iter() + .map(move |&seed| (found.case.clone(), found.class.clone(), seed)) + }) + .collect(); + write_records(decoder, &results, &ran, &verdict, platform); + + let mut report = String::new(); + let total_requests: usize = results.iter().map(|result| result.requests).sum(); + let total_seeds: usize = results.iter().map(|result| result.seeds).sum(); + let _ = writeln!( + report, + "{} decoder model: {} sequences, {} requests, {:.1} s wall on {} threads", + decoder.name(), + total_seeds, + total_requests, + started.elapsed().as_secs_f64(), + threads + ); + for result in &results { + let mut classes: BTreeMap = BTreeMap::new(); + for (case, failure) in &result.failures { + let counts = classes.entry(failure.class).or_default(); + counts.0 += 1; + if unexpected.contains(&( + result.name.clone(), + failure.class.name().to_string(), + case.seed_index, + )) { + counts.1 += 1; + } + } + let summary = if classes.is_empty() { + "ok".to_string() + } else { + classes + .iter() + .map(|(class, (count, new))| { + let status = if *new == 0 { + "xfail".to_string() + } else { + format!("{new} NEW") + }; + format!("{} {}/{} ({status})", class.name(), count, result.seeds) + }) + .collect::>() + .join(", ") + }; + let _ = writeln!( + report, + " {:<58} seeds {:>4} requests {:>7} starts {:>6} {:>6.1}s {}", + result.name, + result.seeds, + result.requests, + result.starts, + result.elapsed.as_secs_f64(), + summary + ); + } + + let shrink_mode = std::env::var("CAP_DECODER_MODEL_SHRINK").unwrap_or_else(|_| "new".into()); + let runtime = runtime(); + for result in &results { + let mut shown: BTreeMap = BTreeMap::new(); + for (case, failure) in &result.failures { + let is_new = unexpected.contains(&( + result.name.clone(), + failure.class.name().to_string(), + case.seed_index, + )); + match shown.get(&failure.class) { + Some(true) => continue, + Some(false) if !is_new => continue, + _ => {} + } + shown.insert(failure.class, is_new); + let _ = writeln!( + report, + "\n{} {} {}: seed index {} (seed {:#x}), first failure at request {} of {}\n {}", + if is_new { "FAIL" } else { "xfail" }, + result.name, + failure.class.name(), + case.seed_index, + case.seed, + failure.request, + case.requests.len(), + failure.detail + ); + let _ = writeln!( + report, + " replay: CAP_DECODER_MODEL_CASE={} CAP_DECODER_MODEL_SEED_INDEX={}", + result.name, case.seed_index + ); + let wants_shrink = match shrink_mode.as_str() { + "all" => true, + "none" => false, + _ => is_new, + }; + if wants_shrink { + let minimal = shrink(case, failure, &runtime); + let outcome = run_case(&minimal, &runtime, false); + let _ = writeln!( + report, + " minimal repro: fps {}, max fallback {}, mode {}\n script: {}\n requests ({}): {}\n {}", + minimal.fps, + minimal.max_fallback_distance, + minimal.mode.name(), + describe_script(&minimal.script), + minimal.requests.len(), + describe_requests(&minimal.requests), + outcome + .failures + .iter() + .find(|found| found.class == failure.class) + .map_or("(did not reproduce)".to_string(), |found| found + .detail + .clone()) + ); + } + } + } + + for found in &verdict.unexpected { + let _ = writeln!( + report, + "\nUNEXPECTED FAILURE {} {} on seed indices {}", + found.case, + found.class, + cap_verify_expect::format_seeds(&found.seeds) + ); + } + for fixed in &verdict.fixed { + let _ = writeln!(report, "\nUNEXPECTED PASS {}", fixed.message(EXPECTATIONS)); + } + eprintln!("{report}"); + if let Ok(path) = std::env::var("CAP_DECODER_MODEL_REPORT") { + let _ = std::fs::write(format!("{path}.{}.txt", decoder.name()), report.as_bytes()); + } + assert!( + verdict.ok(), + "decoder model: {} unexpected failures, {} unexpected passes (see report above)", + verdict.unexpected.len(), + verdict.fixed.len() + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn avassetreader_answers_match_the_reference_model() { + run_suite(Decoder::Avf); +} + +#[test] +fn ffmpeg_answers_match_the_reference_model() { + run_suite(Decoder::Ffmpeg); +} diff --git a/crates/rendering/tests/expectations.toml b/crates/rendering/tests/expectations.toml new file mode 100644 index 00000000000..403a7e54ad6 --- /dev/null +++ b/crates/rendering/tests/expectations.toml @@ -0,0 +1,5 @@ +# Strict expected failures of the native rendering tests in this directory +# (format: crates/verify-expect). A listed failure that no longer occurs fails +# the run: remove the entry together with the fix. +# +# No known failures: the IOSurface source lifetime bug was fixed by #2625. diff --git a/crates/rendering/tests/iosurface-source-lifetime.rs b/crates/rendering/tests/iosurface-source-lifetime.rs index 11eda551838..c38abb4919f 100644 --- a/crates/rendering/tests/iosurface-source-lifetime.rs +++ b/crates/rendering/tests/iosurface-source-lifetime.rs @@ -126,6 +126,45 @@ fn assert_frames(device: &wgpu::Device, output: &wgpu::Buffer, lumas: &[u8]) { output.unmap(); } +/// Sources whose picture is not their own grey (the pool repainted them +/// before the GPU sampled them). +fn wrong_frames(device: &wgpu::Device, output: &wgpu::Buffer, lumas: &[u8]) -> Vec { + let (sender, receiver) = std::sync::mpsc::channel(); + output + .slice(..) + .map_async(wgpu::MapMode::Read, move |result| { + sender.send(result).unwrap(); + }); + device.poll(wgpu::PollType::Wait).unwrap(); + receiver.recv().unwrap().unwrap(); + let mut wrong = Vec::new(); + { + let bytes = output.slice(..).get_mapped_range(); + for (index, &luma) in lumas.iter().enumerate() { + let expected = (f32::from(luma) - 16.0) * 255.0 / 219.0; + let start = index * FRAME_BYTES as usize; + let bad = (0..SIZE as usize) + .flat_map(|row| { + let row = start + row * ROW_BYTES as usize; + bytes[row..row + SIZE as usize * 4].chunks_exact(4) + }) + .find(|pixel| { + pixel[3] != 255 + || pixel[..3] + .iter() + .any(|&channel| (f32::from(channel) - expected).abs() >= 3.0) + }); + if let Some(pixel) = bad { + wrong.push(format!( + "source {index} (luma {luma}, expected grey {expected:.1}) rendered {pixel:?}" + )); + } + } + } + output.unmap(); + wrong +} + #[test] fn conversions_keep_recycled_pixel_buffers_until_submission() { let (device, queue) = gpu(); @@ -145,7 +184,7 @@ fn conversions_keep_recycled_pixel_buffers_until_submission() { let mut next = pool.pixel_buf().unwrap(); fill(&mut next, 235); queue.submit([encoder.finish()]); - assert_frames(&device, &output, &lumas); + settle(&wrong_frames(&device, &output, &lumas)); } /// The converter holds only its most recent sources. With the pool capped one @@ -181,3 +220,74 @@ fn conversions_release_sources_once_the_ring_moves_on() { assert_frames(&device, &output, &[luma]); } } + +const CASE: &str = "native/rendering/iosurface-source-lifetime/recycled-pool"; +const CLASS: &str = "recycled-source"; +const EXPECTATIONS: &str = "crates/rendering/tests/expectations.toml"; + +/// Judges the result against the strict expectations file and writes the +/// case record, like every engine verification suite. +fn settle(wrong: &[String]) { + use cap_verify_expect::{Observed, Outcome, Ran, Record, judge, load, platform}; + let xfails = + load(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/expectations.toml")); + let ran = Ran::from([(CASE.to_string(), None)]); + let mut observed = Observed::new(); + if !wrong.is_empty() { + observed.insert((CASE.to_string(), CLASS.to_string()), Default::default()); + } + let verdict = judge(&xfails, platform(), &ran, &observed); + let outcome = if !verdict.unexpected.is_empty() { + Outcome::Fail + } else if !verdict.fixed.is_empty() { + Outcome::Xpass + } else if !wrong.is_empty() { + Outcome::Xfail + } else { + Outcome::Pass + }; + let summary = if wrong.is_empty() { + "every source rendered its own picture after the pool recycled it".to_string() + } else { + format!( + "pool recycled a source before the GPU sampled it: {}", + wrong.join("; ") + ) + }; + let repro = + "cargo test -p cap-rendering --test iosurface-source-lifetime -- --nocapture".to_string(); + let record = Record { + suite: "cap-rendering-native".into(), + expectations: EXPECTATIONS.into(), + case: CASE.into(), + platform: platform().into(), + tier: std::env::var("CAP_VERIFY_TIER").unwrap_or_else(|_| "fast".into()), + outcome, + failures: observed + .keys() + .map(|(_, class)| (class.clone(), Default::default())) + .collect(), + seeds_ran: None, + unobserved: Default::default(), + problems: if wrong.is_empty() { + Default::default() + } else { + [(CLASS.to_string(), summary.clone())].into() + }, + measures: Default::default(), + excused: Default::default(), + starved: false, + summary: summary.clone(), + repro: repro.clone(), + seconds: 0.0, + }; + cap_verify_expect::write_records(&[(record, serde_json::json!({}))]); + eprintln!("[cap-verify] {outcome:?} {CASE}: {summary}"); + if let Some(fixed) = verdict.fixed.first() { + panic!("{}\n repro: {repro}", fixed.message(EXPECTATIONS)); + } + assert!( + verdict.unexpected.is_empty(), + "{CASE} failed ({CLASS}): {summary}\n repro: {repro}" + ); +} diff --git a/crates/verify-expect/Cargo.toml b/crates/verify-expect/Cargo.toml new file mode 100644 index 00000000000..468e2c0bc8d --- /dev/null +++ b/crates/verify-expect/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "cap-verify-expect" +version = "0.1.0" +edition = "2024" +publish = false + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +toml = "0.8" diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rs new file mode 100644 index 00000000000..33030b5cf3e --- /dev/null +++ b/crates/verify-expect/src/lib.rs @@ -0,0 +1,1617 @@ +//! Strict expected failures and case records shared by every engine +//! verification suite (the `cap-verify` oracle tests, the decoder model +//! fuzzer, the upload fault suite). +//! +//! Each suite keeps one TOML file of `[[xfail]]` entries: +//! +//! ```toml +//! [[xfail]] +//! case = "export/early-camera/full/*" # case id, `*` globs allowed +//! class = "future" # failure class that must still occur +//! platforms = ["macos"] # optional, absent = every platform +//! seeds = "0 3-7" # optional, seeded suites only +//! flaky = true # optional, may pass (state the rate) +//! starved = true # optional, flaky only: applies only +//! # when the case's starvation sentinel +//! # saw the machine stall +//! bound = { frames = 12, delta = 49 } # the mechanism's signature: each +//! # metric's measured range must lie in +//! # [min, max] (a number is a max) +//! reason = "what is wrong" +//! link = "https://github.com/CapSoftware/Cap/pull/2595" # empty = no fix yet +//! ``` +//! +//! The verdict is strict both ways. A failure class (or seed) that is not +//! listed fails the run, and a listed, non-flaky class (or seed) that no +//! longer fails also fails the run ("remove this xfail"), so fixes stay +//! locked in. +//! +//! An entry excuses a failure only inside its `bound`: the suite measures +//! each failing class (frames affected, source-frame error, milliseconds of +//! offset, ...), and a failure whose measurement falls outside the bound, or +//! that lacks a bounded metric, is a new failure. A file that starts with +//! `require_bound = true` rejects unseeded entries without a bound. +//! +//! Every suite writes one JSON line per case ([`Record`]) to +//! `CAP_VERIFY_REPORT_DIR` when it is set. The `cap-verify-expect` binary +//! rewrites the expectations files from those records (`bless`), and +//! `scripts/verify/summary.py` turns them into the CI job summary. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Write as _, + io::Write as _, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; + +pub const PLATFORMS: [&str; 3] = ["linux", "macos", "windows"]; + +/// The machine class from `CAP_VERIFY_HOST` (see [`Xfail::hosts`]); `local` +/// when unset (a developer machine). +pub fn host() -> Option { + Some( + std::env::var("CAP_VERIFY_HOST") + .ok() + .map(|h| h.trim().to_string()) + .filter(|h| !h.is_empty()) + .unwrap_or_else(|| "local".to_string()), + ) +} + +pub fn platform() -> &'static str { + if cfg!(target_os = "macos") { + "macos" + } else if cfg!(target_os = "windows") { + "windows" + } else { + "linux" + } +} + +/// A closed range of measured values. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Range { + pub min: f64, + pub max: f64, +} + +impl Range { + pub fn at(value: f64) -> Self { + Self { + min: value, + max: value, + } + } + + pub fn include(&mut self, value: f64) { + self.min = self.min.min(value); + self.max = self.max.max(value); + } + + fn render(&self) -> String { + let number = |v: f64| { + if v.fract() == 0.0 && v.abs() < 1e15 { + format!("{}", v as i64) + } else { + format!("{v}") + } + }; + if self.min == f64::NEG_INFINITY { + number(self.max) + } else { + format!("[{}, {}]", number(self.min), number(self.max)) + } + } + + fn describe(&self) -> String { + if self.min == self.max { + format!("{}", self.min) + } else { + format!("{}..{}", self.min, self.max) + } + } +} + +impl Serialize for Range { + fn serialize(&self, serializer: S) -> Result { + [self.min, self.max].serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Range { + fn deserialize>(deserializer: D) -> Result { + let [min, max] = <[f64; 2]>::deserialize(deserializer)?; + Ok(Self { min, max }) + } +} + +/// An entry's mechanism bound: metric -> allowed range of its measurement. +pub type Bound = BTreeMap; + +/// Measured metrics of one failing class: metric -> observed range. +pub type Metrics = BTreeMap; + +/// Checks a class's measurements against a bound. The bound is a complete +/// signature: every bounded metric must have been measured with its whole +/// observed range inside, and a measured metric the bound does not name must +/// be exactly zero (a new symptom in the same class is a new failure). +pub fn within(bound: &Bound, measured: Option<&Metrics>) -> Result<(), String> { + let mut outside = Vec::new(); + for (metric, range) in measured.into_iter().flatten() { + if !bound.contains_key(metric) && (range.min != 0.0 || range.max != 0.0) { + outside.push(format!( + "{metric} {} measured but not bounded", + range.describe() + )); + } + } + for (metric, allowed) in bound { + match measured.and_then(|m| m.get(metric)) { + None => outside.push(format!("{metric} not measured")), + Some(seen) => { + let eps = 1e-9; + if seen.min < allowed.min - eps || seen.max > allowed.max + eps { + outside.push(format!( + "{metric} {} outside {}", + seen.describe(), + if allowed.min == f64::NEG_INFINITY { + format!("<= {}", allowed.max) + } else { + format!("{}..{}", allowed.min, allowed.max) + } + )); + } + } + } + } + if outside.is_empty() { + Ok(()) + } else { + Err(outside.join(", ")) + } +} + +/// One strict expected failure. +#[derive(Clone, Debug, PartialEq)] +pub struct Xfail { + pub case: String, + pub class: String, + /// Empty means every platform. + pub platforms: Vec, + /// `None` means the class must occur (on any seed, for seeded suites). + pub seeds: Option>, + pub flaky: bool, + /// Applies only to cases that ran while the starvation sentinel saw the + /// machine stall (flaky entries only). + pub starved: bool, + /// Machine classes the entry applies to, matched against + /// `CAP_VERIFY_HOST` (CI sets `github-hosted` or `blacksmith`; `local` + /// when unset); empty means every host. For a defect only one class of machine reaches + /// (for example the software encoder path of VMs without a hardware + /// encoder), with the reason in `reason`. + pub hosts: Vec, + /// The documented mechanism's bound (empty = unbounded). + pub bound: Bound, + pub reason: String, + pub link: String, +} + +impl Xfail { + pub fn applies_on(&self, platform: &str) -> bool { + self.applies_on_host(platform, host().as_deref()) + } + + fn applies_on_host(&self, platform: &str, source_host: Option<&str>) -> bool { + (self.platforms.is_empty() || self.platforms.iter().any(|p| p == platform)) + && (self.hosts.is_empty() + || source_host.is_some_and(|h| self.hosts.iter().any(|candidate| candidate == h))) + } + + pub fn matches(&self, case: &str, platform: &str) -> bool { + self.applies_on(platform) && glob_match(&self.case, case) + } + + fn effective_platforms(&self) -> BTreeSet { + if self.platforms.is_empty() { + PLATFORMS.iter().map(|p| p.to_string()).collect() + } else { + self.platforms.iter().cloned().collect() + } + } + + fn with_platforms(&self, platforms: BTreeSet) -> Self { + let all = platforms.len() == PLATFORMS.len(); + Self { + platforms: if all { + Vec::new() + } else { + platforms.into_iter().collect() + }, + ..self.clone() + } + } +} + +/// `*` matches any run of characters (including `/`). +pub fn glob_match(pattern: &str, text: &str) -> bool { + let parts: Vec<&str> = pattern.split('*').collect(); + if parts.len() == 1 { + return pattern == text; + } + let Some(mut rest) = text.strip_prefix(parts[0]) else { + return false; + }; + let last = parts.len() - 1; + for part in &parts[1..last] { + let Some(at) = rest.find(part) else { + return false; + }; + rest = &rest[at + part.len()..]; + } + rest.ends_with(parts[last]) +} + +/// Case selection shared by every suite: +/// - `CAP_VERIFY_CASE`: comma-separated globs; only matching cases run. +/// - `CAP_VERIFY_SKIP`: comma-separated globs; matching cases are skipped +/// (the per-platform FAST tier trims in `scripts/verify/suites.json`). +/// - `CAP_VERIFY_SHARD=i/n`: only cases whose id hashes to shard `i` (1-based) +/// of `n` run, so one suite can be split across CI jobs case by case. +pub fn case_selected(id: &str) -> bool { + let env = |name: &str| std::env::var(name).ok().filter(|v| !v.trim().is_empty()); + let globs = |value: String| -> Vec { + value + .split(',') + .map(|p| p.trim().to_string()) + .filter(|p| !p.is_empty()) + .collect() + }; + if let Some(only) = env("CAP_VERIFY_CASE") + && !globs(only).iter().any(|p| glob_match(p, id)) + { + return false; + } + if let Some(skip) = env("CAP_VERIFY_SKIP") + && globs(skip).iter().any(|p| glob_match(p, id)) + { + return false; + } + match env("CAP_VERIFY_SHARD").and_then(|v| parse_shard(&v)) { + Some((index, count)) => shard_of(id, count) == index, + None => true, + } +} + +fn parse_shard(value: &str) -> Option<(u64, u64)> { + let (index, count) = value.trim().split_once('/')?; + let (index, count) = (index.parse().ok()?, count.parse().ok()?); + (count >= 1 && (1..=count).contains(&index)).then_some((index, count)) +} + +/// 1-based shard of a case id (FNV-1a, stable across platforms and runs). +pub fn shard_of(id: &str, count: u64) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in id.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + hash % count.max(1) + 1 +} + +pub fn parse_seeds(text: &str) -> Result, String> { + let mut seeds = BTreeSet::new(); + for item in text.split_whitespace() { + let parse = |value: &str| { + value + .parse::() + .map_err(|_| format!("bad seed {value:?}")) + }; + match item.split_once('-') { + Some((first, last)) => seeds.extend(parse(first)?..=parse(last)?), + None => { + seeds.insert(parse(item)?); + } + } + } + Ok(seeds) +} + +pub fn format_seeds(seeds: &BTreeSet) -> String { + let mut runs: Vec<(u64, u64)> = Vec::new(); + for &seed in seeds { + match runs.last_mut() { + Some((_, last)) if *last + 1 == seed => *last = seed, + _ => runs.push((seed, seed)), + } + } + runs.iter() + .map(|&(first, last)| match last - first { + 0 => first.to_string(), + 1 => format!("{first} {last}"), + _ => format!("{first}-{last}"), + }) + .collect::>() + .join(" ") +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawXfail { + case: String, + class: String, + #[serde(default)] + platforms: Vec, + seeds: Option, + #[serde(default)] + flaky: bool, + #[serde(default)] + starved: bool, + #[serde(default)] + hosts: Vec, + #[serde(default)] + bound: Option, + reason: String, + #[serde(default)] + link: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawFile { + #[serde(default)] + require_bound: bool, + #[serde(default)] + xfail: Vec, +} + +fn parse_bound(case: &str, table: Option) -> Result { + let mut bound = Bound::new(); + let number = |v: &toml::Value| match v { + toml::Value::Integer(i) => Some(*i as f64), + toml::Value::Float(f) => Some(*f), + _ => None, + }; + for (metric, value) in table.unwrap_or_default() { + let range = match &value { + toml::Value::Array(pair) if pair.len() == 2 => { + match (number(&pair[0]), number(&pair[1])) { + (Some(min), Some(max)) if min <= max => Range { min, max }, + _ => return Err(format!("xfail {case}: bound {metric} must be [min, max]")), + } + } + v => match number(v) { + Some(max) => Range { + min: f64::NEG_INFINITY, + max, + }, + None => { + return Err(format!( + "xfail {case}: bound {metric} must be a number or [min, max]" + )); + } + }, + }; + bound.insert(metric, range); + } + Ok(bound) +} + +pub fn parse(text: &str) -> Result, String> { + let file: RawFile = toml::from_str(text).map_err(|e| e.to_string())?; + let require_bound = file.require_bound; + file.xfail + .into_iter() + .map(|raw| { + if raw.case.is_empty() || raw.class.is_empty() || raw.reason.is_empty() { + return Err(format!( + "xfail {:?}/{:?}: case, class and reason are required", + raw.case, raw.class + )); + } + if let Some(bad) = raw + .platforms + .iter() + .find(|p| !PLATFORMS.contains(&p.as_str())) + { + return Err(format!( + "xfail {}: unknown platform {bad:?} (use {PLATFORMS:?})", + raw.case + )); + } + let seeds = raw + .seeds + .as_deref() + .map(parse_seeds) + .transpose() + .map_err(|e| format!("xfail {}: {e}", raw.case))?; + let bound = parse_bound(&raw.case, raw.bound)?; + if require_bound && bound.is_empty() && seeds.is_none() { + return Err(format!( + "xfail {}/{}: a bound is required (the documented mechanism's signature: bound = {{ metric = max }} or [min, max])", + raw.case, raw.class + )); + } + if raw.starved && !raw.flaky { + return Err(format!( + "xfail {}/{}: starved entries must be flaky", + raw.case, raw.class + )); + } + Ok(Xfail { + case: raw.case, + class: raw.class, + platforms: raw.platforms, + seeds, + flaky: raw.flaky, + starved: raw.starved, + hosts: raw.hosts, + bound, + reason: raw.reason, + link: raw.link, + }) + }) + .collect() +} + +/// Loads an expectations file. A missing file is an empty list; a malformed +/// one panics with the path, since a silently ignored entry would hide a +/// regression. +pub fn load(path: &Path) -> Vec { + match std::fs::read_to_string(path) { + Ok(text) => parse(&text).unwrap_or_else(|e| panic!("{}: {e}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => panic!("{}: {e}", path.display()), + } +} + +fn quote(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for c in value.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\t' => out.push_str("\\t"), + c if c.is_control() => { + let _ = write!(out, "\\u{:04X}", c as u32); + } + c => out.push(c), + } + } + out.push('"'); + out +} + +/// Renders entries in the canonical layout (one key per line, fixed order). +pub fn render(entries: &[Xfail]) -> String { + let mut out = String::new(); + for x in entries { + out.push_str("[[xfail]]\n"); + let _ = writeln!(out, "case = {}", quote(&x.case)); + let _ = writeln!(out, "class = {}", quote(&x.class)); + if !x.platforms.is_empty() { + let platforms: Vec = x.platforms.iter().map(|p| quote(p)).collect(); + let _ = writeln!(out, "platforms = [{}]", platforms.join(", ")); + } + if let Some(seeds) = &x.seeds { + let _ = writeln!(out, "seeds = {}", quote(&format_seeds(seeds))); + } + if x.flaky { + out.push_str("flaky = true\n"); + } + if x.starved { + out.push_str("starved = true\n"); + } + if !x.hosts.is_empty() { + let hosts: Vec = x.hosts.iter().map(|h| quote(h)).collect(); + let _ = writeln!(out, "hosts = [{}]", hosts.join(", ")); + } + if !x.bound.is_empty() { + let parts: Vec = x + .bound + .iter() + .map(|(metric, range)| format!("{metric} = {}", range.render())) + .collect(); + let _ = writeln!(out, "bound = {{ {} }}", parts.join(", ")); + } + let _ = writeln!(out, "reason = {}", quote(&x.reason)); + let _ = writeln!(out, "link = {}", quote(&x.link)); + out.push('\n'); + } + out +} + +/// The comment block before the first entry, kept when a file is rewritten. +pub fn header(text: &str) -> String { + let mut out = String::new(); + for line in text.lines() { + if line.trim() == "[[xfail]]" { + break; + } + out.push_str(line); + out.push('\n'); + } + let trimmed = out.trim_end(); + if trimmed.is_empty() { + String::new() + } else { + format!("{trimmed}\n\n") + } +} + +/// Failures seen in a run: `(case, class) -> failing seeds`. Unseeded suites +/// use an empty seed set. +pub type Observed = BTreeMap<(String, String), BTreeSet>; + +/// Cases that ran: `case -> seeds that ran` (`None` for unseeded cases). +pub type Ran = BTreeMap>>; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Unexpected { + pub case: String, + pub class: String, + /// Seeds not covered by any xfail (empty for unseeded cases). + pub seeds: BTreeSet, + /// Why matching entries did not excuse it (outside their bound, or the + /// machine did not starve); empty when no entry names the class. + pub why: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Fixed { + pub xfail: Xfail, + pub case: String, + /// Listed seeds that passed; `None` when the class did not occur at all. + pub seeds: Option>, +} + +impl Fixed { + pub fn message(&self, file: &str) -> String { + let seeds = match &self.seeds { + Some(seeds) => format!(" for seeds {}", format_seeds(seeds)), + None => String::new(), + }; + format!( + "{} no longer fails with {}{seeds}: remove this xfail from {file} ({})", + self.case, self.xfail.class, self.xfail.reason + ) + } +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Verdict { + pub unexpected: Vec, + pub fixed: Vec, + /// Observed failures each excusing entry accepted: (case, class, entry). + pub excused: Vec<(String, String, Xfail)>, +} + +impl Verdict { + pub fn ok(&self) -> bool { + self.unexpected.is_empty() && self.fixed.is_empty() + } +} + +/// Measurements of a run's failing classes: `(case, class) -> metrics`. +pub type Measures = BTreeMap<(String, String), Metrics>; + +/// What a run knows beyond pass and fail: classes it could not measure, the +/// measured size of each failure, and the cases that ran while the machine +/// starved (the sentinel's evidence). +#[derive(Clone, Debug, Default)] +pub struct Evidence { + pub unobserved: Unobserved, + pub measures: Measures, + pub starved: BTreeSet, +} + +/// Whether `x` excuses `class` on `case`: the case must have starved for a +/// starved entry, and the measurements must lie inside the entry's bound. +fn excuses(x: &Xfail, case: &str, class: &str, evidence: &Evidence) -> Result<(), String> { + if x.starved && !evidence.starved.contains(case) { + return Err("applies only under a proven machine stall, and the sentinel saw none".into()); + } + within( + &x.bound, + evidence + .measures + .get(&(case.to_string(), class.to_string())), + ) + .map_err(|e| format!("outside the expected bound ({e})")) +} + +fn covered_by<'a>( + xfails: impl Iterator, + case: &str, + class: &str, + platform: &str, + source_host: Option<&str>, + seeds: &BTreeSet, + evidence: &Evidence, +) -> BTreeSet { + let mut covered = BTreeSet::new(); + for x in xfails.filter(|x| { + x.class == class + && x.applies_on_host(platform, source_host) + && glob_match(&x.case, case) + && excuses(x, case, class, evidence).is_ok() + }) { + match &x.seeds { + None => return seeds.clone(), + Some(listed) => covered.extend(listed.intersection(seeds).copied()), + } + } + covered +} + +/// Classes a case could not measure because an earlier check failed (for +/// example A/V sync when no audio marker could be identified): `(case, +/// class)`. An unobserved class is neither a failure nor a pass, so an xfail +/// for it is not reported as fixed and `bless` leaves it alone. The failure +/// that made it unobservable is judged on its own. +pub type Unobserved = BTreeSet<(String, String)>; + +/// Compares the failures of a run with the expectations, on `platform`. +pub fn judge(xfails: &[Xfail], platform: &str, ran: &Ran, observed: &Observed) -> Verdict { + judge_with(xfails, platform, ran, observed, &Evidence::default()) +} + +/// [`judge`] for runs where some classes could not be measured. +pub fn judge_observing( + xfails: &[Xfail], + platform: &str, + ran: &Ran, + observed: &Observed, + unobserved: &Unobserved, +) -> Verdict { + let evidence = Evidence { + unobserved: unobserved.clone(), + ..Default::default() + }; + judge_with(xfails, platform, ran, observed, &evidence) +} + +/// [`judge`] with measurements, unmeasurable classes and starvation +/// evidence: an entry excuses a failure only inside its bound. +pub fn judge_with( + xfails: &[Xfail], + platform: &str, + ran: &Ran, + observed: &Observed, + evidence: &Evidence, +) -> Verdict { + let mut verdict = Verdict::default(); + for ((case, class), seeds) in observed { + let named: Vec<&Xfail> = xfails + .iter() + .filter(|x| x.class == *class && x.matches(case, platform)) + .collect(); + if named.is_empty() { + verdict.unexpected.push(Unexpected { + case: case.clone(), + class: class.clone(), + seeds: seeds.clone(), + why: String::new(), + }); + continue; + } + let covered = covered_by( + named.iter().copied(), + case, + class, + platform, + host().as_deref(), + seeds, + evidence, + ); + let excused_fully = seeds.is_empty() + && named + .iter() + .any(|x| x.seeds.is_none() && excuses(x, case, class, evidence).is_ok()); + let uncovered: BTreeSet = seeds.difference(&covered).copied().collect(); + if !excused_fully && (seeds.is_empty() || !uncovered.is_empty()) { + let why: Vec = named + .iter() + .filter_map(|x| excuses(x, case, class, evidence).err()) + .collect(); + verdict.unexpected.push(Unexpected { + case: case.clone(), + class: class.clone(), + seeds: uncovered, + why: why.join("; "), + }); + continue; + } + if let Some(x) = named + .iter() + .find(|x| excuses(x, case, class, evidence).is_ok()) + { + verdict + .excused + .push((case.clone(), class.clone(), (*x).clone())); + } + } + for x in xfails.iter().filter(|x| !x.flaky && x.applies_on(platform)) { + for (case, ran_seeds) in ran.iter().filter(|(case, _)| glob_match(&x.case, case)) { + if evidence + .unobserved + .contains(&(case.clone(), x.class.clone())) + { + continue; + } + let seen = observed.get(&(case.clone(), x.class.clone())); + let seeds = match (&x.seeds, seen) { + (None, None) => None, + (None, Some(_)) => continue, + (Some(listed), seen) => { + let passed: BTreeSet = listed + .iter() + .filter(|seed| ran_seeds.as_ref().is_none_or(|ran| ran.contains(seed))) + .filter(|seed| seen.is_none_or(|seen| !seen.contains(seed))) + .copied() + .collect(); + if passed.is_empty() { + continue; + } + Some(passed) + } + }; + verdict.fixed.push(Fixed { + xfail: x.clone(), + case: case.clone(), + seeds, + }); + } + } + verdict +} + +/// Proposes changes for this platform and host without narrowing entries +/// for unobserved hosts, platforms or cases. Links of replaced entries +/// are kept; new proposals get the reason +/// `TODO: ` from `problems`. +pub fn bless( + xfails: &[Xfail], + platform: &str, + ran: &Ran, + observed: &Observed, + problems: &BTreeMap<(String, String), String>, +) -> Vec { + bless_observing( + xfails, + platform, + ran, + observed, + &Unobserved::new(), + problems, + ) +} + +/// [`bless`] that keeps entries whose class some matching case could not +/// measure in this run. +pub fn bless_observing( + xfails: &[Xfail], + platform: &str, + ran: &Ran, + observed: &Observed, + unobserved: &Unobserved, + problems: &BTreeMap<(String, String), String>, +) -> Vec { + let evidence = Evidence { + unobserved: unobserved.clone(), + ..Default::default() + }; + bless_with( + xfails, + platform, + host().as_deref(), + ran, + observed, + &evidence, + problems, + ) +} + +/// [`bless`] with measurements: a failure outside every entry's bound gets a +/// new entry (TODO reason) whose bound is the measured range, to be +/// reviewed against the mechanism. Unknown-host observations cannot remove +/// entries. All-host entries are retained because finite host observations +/// cannot establish their behavior on every possible host. +pub fn bless_with( + xfails: &[Xfail], + platform: &str, + source_host: Option<&str>, + ran: &Ran, + observed: &Observed, + evidence: &Evidence, + problems: &BTreeMap<(String, String), String>, +) -> Vec { + let unobserved = &evidence.unobserved; + let mut out: Vec = Vec::new(); + let mut kept_here: Vec = Vec::new(); + for x in xfails { + let matched: Vec<(&String, &Option>)> = ran + .iter() + .filter(|(case, _)| glob_match(&x.case, case)) + .collect(); + let unmeasured = matched + .iter() + .any(|(case, _)| unobserved.contains(&((*case).clone(), x.class.clone()))); + if source_host.is_none() + || x.hosts.is_empty() + || !x.applies_on_host(platform, source_host) + || matched.is_empty() + || x.flaky + || unmeasured + { + out.push(x.clone()); + if x.applies_on_host(platform, source_host) { + kept_here.push(x.clone()); + } + continue; + } + let still_exact = matched.iter().all(|(case, ran_seeds)| { + let seen = observed.get(&((*case).clone(), x.class.clone())); + match (&x.seeds, seen) { + (_, None) => false, + (None, Some(_)) => true, + (Some(listed), Some(seen)) => listed + .iter() + .filter(|s| ran_seeds.as_ref().is_none_or(|r| r.contains(s))) + .all(|s| seen.contains(s)), + } + }); + if still_exact { + out.push(x.clone()); + kept_here.push(x.clone()); + continue; + } + let mut others = x.effective_platforms(); + others.remove(platform); + if !others.is_empty() { + out.push(x.with_platforms(others)); + } + let other_hosts: Vec = x + .hosts + .iter() + .filter(|h| Some(h.as_str()) != source_host) + .cloned() + .collect(); + if !other_hosts.is_empty() { + out.push(Xfail { + platforms: vec![platform.to_string()], + hosts: other_hosts, + ..x.clone() + }); + } + if let Some(listed) = &x.seeds { + for (case, ran_seeds) in &matched { + let not_ran: BTreeSet = listed + .iter() + .filter(|s| ran_seeds.as_ref().is_some_and(|r| !r.contains(s))) + .copied() + .collect(); + if !not_ran.is_empty() { + let entry = Xfail { + case: (*case).clone(), + seeds: Some(not_ran), + platforms: vec![platform.to_string()], + hosts: source_host.into_iter().map(str::to_string).collect(), + ..x.clone() + }; + out.push(entry.clone()); + kept_here.push(entry); + } + } + } + } + for ((case, class), seeds) in observed { + let covered = covered_by( + kept_here.iter(), + case, + class, + platform, + source_host, + seeds, + evidence, + ); + let fully = kept_here.iter().any(|x| { + x.class == *class + && x.applies_on_host(platform, source_host) + && glob_match(&x.case, case) + && x.seeds.is_none() + && excuses(x, case, class, evidence).is_ok() + }); + if fully || (!seeds.is_empty() && covered == *seeds) { + continue; + } + let missing: BTreeSet = seeds.difference(&covered).copied().collect(); + let previous = xfails.iter().find(|x| { + x.class == *class + && x.applies_on_host(platform, source_host) + && glob_match(&x.case, case) + }); + let entry = Xfail { + case: case.clone(), + class: class.clone(), + platforms: vec![platform.to_string()], + seeds: (!seeds.is_empty()).then_some(missing), + flaky: false, + starved: false, + hosts: source_host.into_iter().map(str::to_string).collect(), + bound: if seeds.is_empty() { + evidence + .measures + .get(&(case.clone(), class.clone())) + .cloned() + .unwrap_or_default() + } else { + Bound::new() + }, + reason: format!( + "TODO: {}", + problems + .get(&(case.clone(), class.clone())) + .map_or("describe the failure", |p| p.as_str()) + ), + link: previous.map_or_else(String::new, |x| x.link.clone()), + }; + out.push(entry); + } + coalesce(out) +} + +/// Merges entries that differ only in platforms or in seeds of the same +/// case, then sorts them. +fn coalesce(entries: Vec) -> Vec { + let mut by_seeds: Vec = Vec::new(); + for x in entries { + if let Some(existing) = by_seeds.iter_mut().find(|e| { + e.case == x.case + && e.class == x.class + && e.flaky == x.flaky + && e.starved == x.starved + && e.hosts == x.hosts + && e.bound == x.bound + && e.reason == x.reason + && e.link == x.link + && e.platforms == x.platforms + && e.seeds.is_some() + && x.seeds.is_some() + }) { + if let (Some(a), Some(b)) = (&mut existing.seeds, &x.seeds) { + a.extend(b.iter().copied()); + } + } else { + by_seeds.push(x); + } + } + let mut merged: Vec = Vec::new(); + for x in by_seeds { + if let Some(existing) = merged.iter_mut().find(|e| { + e.case == x.case + && e.class == x.class + && e.seeds == x.seeds + && e.flaky == x.flaky + && e.starved == x.starved + && e.hosts == x.hosts + && e.bound == x.bound + && e.reason == x.reason + && e.link == x.link + }) { + let mut platforms = existing.effective_platforms(); + platforms.extend(x.effective_platforms()); + *existing = existing.with_platforms(platforms); + } else { + merged.push(x); + } + } + merged.sort_by(|a, b| { + (&a.case, &a.class, a.effective_platforms()).cmp(&( + &b.case, + &b.class, + b.effective_platforms(), + )) + }); + merged.dedup(); + merged +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Outcome { + Pass, + Xfail, + Fail, + Xpass, + Unsupported, + /// A wall-clock class failed while the machine starved the process + /// (cap-verify's sentinel); the test is retried once. Never blessed. + Inconclusive, +} + +/// One case's result, written as a JSON line to `CAP_VERIFY_REPORT_DIR`. +/// Suites may add fields (for example per-stream mismatch lists); readers +/// ignore what they do not know. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Record { + pub suite: String, + /// Expectations file, relative to the repository root. + pub expectations: String, + pub case: String, + pub platform: String, + #[serde(default)] + pub tier: String, + pub outcome: Outcome, + /// Failure class -> failing seeds (empty for unseeded cases). + #[serde(default)] + pub failures: BTreeMap>, + /// Seeds that ran, for seeded cases. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub seeds_ran: Option>, + /// Classes this case could not measure -> why (see [`Unobserved`]). + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub unobserved: BTreeMap, + /// First problem per class, used as the draft reason when blessing. + #[serde(default)] + pub problems: BTreeMap, + /// Measured size of each failing class (see [`within`]). + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub measures: BTreeMap, + /// Failing classes an expectation excused -> "strict", "flaky" or + /// "flaky, starved". + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub excused: BTreeMap, + /// The case ran while the starvation sentinel saw the machine stall. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub starved: bool, + #[serde(default)] + pub summary: String, + #[serde(default)] + pub repro: String, + #[serde(default)] + pub seconds: f64, +} + +pub fn report_dir() -> Option { + std::env::var_os("CAP_VERIFY_REPORT_DIR") + .filter(|dir| !dir.is_empty()) + .map(PathBuf::from) +} + +/// Appends records to `/-.jsonl` (no-op +/// when the variable is unset). Any extra JSON fields in `extra` are merged +/// into each line. +pub fn write_records(records: &[(Record, serde_json::Value)]) { + let Some(dir) = report_dir() else { + return; + }; + write_records_to(&dir, records); +} + +fn write_records_to(dir: &Path, records: &[(Record, serde_json::Value)]) { + static WRITER: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _guard = WRITER.lock().expect("report writer lock"); + std::fs::create_dir_all(dir).expect("create report directory"); + let Some((first, _)) = records.first() else { + return; + }; + let file = dir.join(format!("{}-{}.jsonl", first.suite, std::process::id())); + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file) + .expect("open report file"); + for (record, extra) in records { + let mut value = serde_json::to_value(record).expect("serialize report record"); + value["host"] = serde_json::to_value(host()).expect("serialize report host"); + if let (Some(map), Some(extra)) = (value.as_object_mut(), extra.as_object()) { + for (k, v) in extra { + map.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + let line = format!("{value}\n"); + file.write_all(line.as_bytes()) + .expect("write report record"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set(seeds: &[u64]) -> BTreeSet { + seeds.iter().copied().collect() + } + + fn entry(case: &str, class: &str, platforms: &[&str], seeds: Option<&[u64]>) -> Xfail { + Xfail { + case: case.into(), + class: class.into(), + platforms: platforms.iter().map(|p| p.to_string()).collect(), + seeds: seeds.map(set), + flaky: false, + starved: false, + hosts: host().into_iter().collect(), + bound: Bound::new(), + reason: "r".into(), + link: String::new(), + } + } + + #[test] + fn concurrent_reports_remain_complete_json_lines() { + let dir = std::env::temp_dir().join(format!( + "cap-verify-reports-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::thread::scope(|scope| { + for writer in 0..8 { + let dir = &dir; + scope.spawn(move || { + for index in 0..50 { + let record = Record { + suite: "concurrent".into(), + expectations: "expectations.toml".into(), + case: format!("{writer}/{index}"), + platform: "windows".into(), + tier: "fast".into(), + outcome: Outcome::Unsupported, + failures: BTreeMap::new(), + seeds_ran: None, + unobserved: BTreeMap::new(), + problems: BTreeMap::new(), + measures: BTreeMap::new(), + excused: BTreeMap::new(), + starved: false, + summary: "test record".repeat(100), + repro: String::new(), + seconds: 0.0, + }; + write_records_to( + dir, + &[( + record, + serde_json::json!({"writer": writer, "host": "incorrect"}), + )], + ); + } + }); + } + }); + let file = dir.join(format!("concurrent-{}.jsonl", std::process::id())); + let text = std::fs::read_to_string(file).unwrap(); + let cases: BTreeSet = text + .lines() + .map(|line| { + let value: serde_json::Value = serde_json::from_str(line).unwrap(); + assert_eq!(value["host"], serde_json::json!(host())); + let record: Record = serde_json::from_str(line).unwrap(); + assert_eq!(record.outcome, Outcome::Unsupported); + record.case + }) + .collect(); + assert_eq!(text.lines().count(), 400); + assert_eq!(cases.len(), 400); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn globs() { + assert!(glob_match("export/*", "export/basic/mp4")); + assert!(glob_match("export/*/avf", "export/jitter/avf")); + assert!(!glob_match("export/*/avf", "export/jitter/ffmpeg")); + assert!(glob_match("a*b*c", "abc")); + assert!(!glob_match("a*b*c", "acb")); + assert!(glob_match("a", "a")); + assert!(!glob_match("a", "ab")); + } + + #[test] + fn shards_partition_cases() { + let ids: Vec = (0..200).map(|i| format!("export/case-{i}/avf")).collect(); + let mut counts = [0usize; 3]; + for id in &ids { + let shard = shard_of(id, 3); + assert!((1..=3).contains(&shard)); + counts[(shard - 1) as usize] += 1; + } + assert!(counts.iter().all(|&n| n > 40), "{counts:?}"); + assert_eq!(parse_shard("2/3"), Some((2, 3))); + assert_eq!(parse_shard("4/3"), None); + assert_eq!(parse_shard("0/3"), None); + } + + #[test] + fn seed_lists_round_trip() { + let seeds = parse_seeds("0 2 3 5-9 12").unwrap(); + assert_eq!(seeds, set(&[0, 2, 3, 5, 6, 7, 8, 9, 12])); + assert_eq!(format_seeds(&seeds), "0 2 3 5-9 12"); + } + + #[test] + fn file_round_trips_and_rejects_typos() { + let entries = vec![ + entry("a/*", "stale", &["macos"], Some(&[1, 2, 3])), + Xfail { + flaky: true, + reason: "quote \" and \\ ok".into(), + link: "https://x".into(), + ..entry("b", "future", &[], None) + }, + ]; + assert_eq!(parse(&render(&entries)).unwrap(), entries); + assert!(parse("[[xfail]]\ncase = \"a\"\nclas = \"x\"\nreason = \"r\"\n").is_err()); + assert!( + parse( + "[[xfail]]\ncase = \"a\"\nclass = \"x\"\nreason = \"r\"\nplatforms = [\"osx\"]\n" + ) + .is_err() + ); + } + + #[test] + fn strict_in_both_directions_per_seed() { + let xfails = vec![entry("a", "stale", &[], Some(&[1, 2]))]; + let ran = Ran::from([("a".to_string(), Some(set(&[0, 1, 2, 3])))]); + let observed = Observed::from([(("a".into(), "stale".into()), set(&[2, 3]))]); + let verdict = judge(&xfails, "linux", &ran, &observed); + assert_eq!(verdict.unexpected.len(), 1); + assert_eq!(verdict.unexpected[0].seeds, set(&[3])); + assert_eq!(verdict.fixed.len(), 1); + assert_eq!(verdict.fixed[0].seeds, Some(set(&[1]))); + } + + #[test] + fn unobserved_classes_are_neither_fixed_nor_blessed_away() { + let xfails = vec![ + entry("a", "sync", &[], None), + entry("a", "audio", &[], None), + ]; + let ran = Ran::from([("a".to_string(), None)]); + let observed = Observed::from([(("a".into(), "audio".into()), set(&[]))]); + let unobserved = Unobserved::from([("a".to_string(), "sync".to_string())]); + assert!(judge_observing(&xfails, "linux", &ran, &observed, &unobserved).ok()); + let measured = judge(&xfails, "linux", &ran, &observed); + assert_eq!(measured.fixed.len(), 1); + assert_eq!(measured.fixed[0].xfail.class, "sync"); + let blessed = bless_observing( + &xfails, + "linux", + &ran, + &observed, + &unobserved, + &BTreeMap::new(), + ); + assert_eq!(blessed, coalesce(xfails.clone())); + let unexpected = Observed::from([(("a".into(), "audio".into()), set(&[]))]); + let verdict = judge_observing(&[], "linux", &ran, &unexpected, &unobserved); + assert_eq!( + verdict.unexpected.len(), + 1, + "the prerequisite failure still fails" + ); + } + + #[test] + fn seeds_that_did_not_run_are_not_fixed() { + let xfails = vec![entry("a", "stale", &[], Some(&[1, 50]))]; + let ran = Ran::from([("a".to_string(), Some(set(&[0, 1, 2])))]); + let observed = Observed::from([(("a".into(), "stale".into()), set(&[1]))]); + assert!(judge(&xfails, "linux", &ran, &observed).ok()); + } + + #[test] + fn unseeded_globs_and_platforms() { + let xfails = vec![ + entry("export/x/*", "future", &["macos"], None), + Xfail { + flaky: true, + ..entry("seek/y", "stale", &[], None) + }, + ]; + let ran = Ran::from([ + ("export/x/avf".to_string(), None), + ("export/x/ffmpeg".to_string(), None), + ("seek/y".to_string(), None), + ]); + let observed = Observed::from([(("export/x/avf".into(), "future".into()), set(&[]))]); + let verdict = judge(&xfails, "macos", &ran, &observed); + assert!(verdict.unexpected.is_empty()); + assert_eq!(verdict.fixed.len(), 1); + assert_eq!(verdict.fixed[0].case, "export/x/ffmpeg"); + let verdict = judge(&xfails, "linux", &ran, &observed); + assert_eq!(verdict.unexpected.len(), 1); + assert!(verdict.fixed.is_empty()); + } + + #[test] + fn bless_only_touches_this_platform() { + let xfails = vec![ + entry("a", "stale", &[], None), + entry("b", "future", &[], Some(&[1, 2, 40])), + entry("c", "stale", &["windows"], None), + ]; + let ran = Ran::from([ + ("a".to_string(), None), + ("b".to_string(), Some(set(&[0, 1, 2, 3]))), + ("c".to_string(), None), + ("d".to_string(), None), + ]); + let observed = Observed::from([ + (("b".into(), "future".into()), set(&[2, 3])), + (("d".into(), "missing".into()), set(&[])), + ]); + let problems = + BTreeMap::from([(("d".into(), "missing".into()), "3 frames missing".into())]); + let blessed = bless(&xfails, "macos", &ran, &observed, &problems); + assert!( + judge(&blessed, "macos", &ran, &observed).ok(), + "{blessed:#?}" + ); + let a: Vec<&Xfail> = blessed.iter().filter(|x| x.case == "a").collect(); + assert_eq!(a.len(), 1); + assert_eq!(a[0].platforms, vec!["linux", "windows"]); + assert!(blessed.contains(&entry("c", "stale", &["windows"], None))); + let b_mac: Vec<&Xfail> = blessed + .iter() + .filter(|x| x.case == "b" && x.applies_on("macos")) + .collect(); + assert_eq!(b_mac.len(), 2); + assert!( + b_mac + .iter() + .any(|x| x.seeds == Some(set(&[40])) && x.reason == "r") + ); + assert!(b_mac.iter().any(|x| { + x.seeds == Some(set(&[2, 3])) && x.reason == "TODO: describe the failure" + })); + let d = blessed.iter().find(|x| x.case == "d").unwrap(); + assert_eq!(d.reason, "TODO: 3 frames missing"); + assert_eq!(d.platforms, vec!["macos"]); + assert_eq!( + bless(&blessed, "macos", &ran, &observed, &problems), + blessed, + "blessing twice changes nothing" + ); + } + + #[test] + fn bless_requires_an_explanation_on_a_new_platform() { + let xfails = vec![entry("a", "stale", &["linux", "windows"], None)]; + let ran = Ran::from([("a".to_string(), None)]); + let observed = Observed::from([(("a".into(), "stale".into()), set(&[]))]); + let blessed = bless(&xfails, "macos", &ran, &observed, &BTreeMap::new()); + let mut proposed = entry("a", "stale", &["macos"], None); + proposed.reason = "TODO: describe the failure".into(); + assert_eq!(blessed, vec![xfails[0].clone(), proposed]); + } + + #[test] + fn bless_preserves_unobserved_host_and_platform_slices() { + let xfail = Xfail { + hosts: vec!["github-hosted".into(), "local".into()], + ..entry("a", "stale", &["macos", "windows"], None) + }; + let ran = Ran::from([("a".into(), None)]); + let blessed = bless_with( + &[xfail.clone()], + "macos", + Some("local"), + &ran, + &Observed::new(), + &Evidence::default(), + &BTreeMap::new(), + ); + assert_eq!(blessed.len(), 2); + assert!( + blessed + .iter() + .any(|x| x.platforms == ["windows"] && x.hosts == xfail.hosts) + ); + assert!( + blessed + .iter() + .any(|x| x.platforms == ["macos"] && x.hosts == ["github-hosted"]) + ); + assert!( + !blessed + .iter() + .any(|x| x.applies_on_host("macos", Some("local"))) + ); + + let untested = bless_with( + &[xfail.clone()], + "macos", + Some("blacksmith"), + &ran, + &Observed::new(), + &Evidence::default(), + &BTreeMap::new(), + ); + assert_eq!(untested, vec![xfail]); + } + + #[test] + fn bless_unknown_hosts_cannot_remove_or_narrow_entries() { + let xfails = vec![ + entry("a", "stale", &["macos"], Some(&[1, 2])), + Xfail { + hosts: Vec::new(), + ..entry("b", "stale", &["macos"], None) + }, + ]; + let ran = Ran::from([("a".into(), Some(set(&[1, 2]))), ("b".into(), None)]); + for observed in [ + Observed::new(), + Observed::from([(("a".into(), "stale".into()), set(&[1]))]), + ] { + let blessed = bless_with( + &xfails, + "macos", + None, + &ran, + &observed, + &Evidence::default(), + &BTreeMap::new(), + ); + for entry in &xfails { + assert!(blessed.contains(entry)); + } + } + let blessed = bless_with( + &xfails[1..], + "macos", + Some("blacksmith"), + &ran, + &Observed::new(), + &Evidence::default(), + &BTreeMap::new(), + ); + assert_eq!(blessed, xfails[1..]); + } + + fn measured(case: &str, class: &str, metrics: &[(&str, f64, f64)]) -> Evidence { + let mut evidence = Evidence::default(); + evidence.measures.insert( + (case.into(), class.into()), + metrics + .iter() + .map(|&(m, min, max)| (m.to_string(), Range { min, max })) + .collect(), + ); + evidence + } + + #[test] + fn bounds_limit_what_an_entry_excuses() { + let text = "require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"stale\"\nbound = { frames = 12, ms = [-210, -190] }\nreason = \"r\"\n"; + let xfails = parse(text).unwrap(); + assert_eq!(xfails[0].bound["frames"].max, 12.0); + assert_eq!(xfails[0].bound["ms"].min, -210.0); + assert_eq!(parse(&render(&xfails)).unwrap(), xfails); + let ran = Ran::from([("a".to_string(), None)]); + let observed = Observed::from([(("a".into(), "stale".into()), set(&[]))]); + let inside = measured( + "a", + "stale", + &[("frames", 3.0, 12.0), ("ms", -205.0, -195.0)], + ); + let verdict = judge_with(&xfails, "linux", &ran, &observed, &inside); + assert!(verdict.ok(), "{verdict:?}"); + assert_eq!(verdict.excused.len(), 1); + let more = measured( + "a", + "stale", + &[("frames", 13.0, 13.0), ("ms", -200.0, -200.0)], + ); + let verdict = judge_with(&xfails, "linux", &ran, &observed, &more); + assert_eq!(verdict.unexpected.len(), 1); + assert!( + verdict.unexpected[0].why.contains("frames 13"), + "{verdict:?}" + ); + let early = measured( + "a", + "stale", + &[("frames", 1.0, 1.0), ("ms", -300.0, -200.0)], + ); + assert_eq!( + judge_with(&xfails, "linux", &ran, &observed, &early) + .unexpected + .len(), + 1 + ); + let extra = measured( + "a", + "stale", + &[ + ("frames", 1.0, 1.0), + ("ms", -200.0, -200.0), + ("camera-frames", 2.0, 2.0), + ], + ); + let verdict = judge_with(&xfails, "linux", &ran, &observed, &extra); + assert!( + verdict.unexpected[0].why.contains("camera-frames 2"), + "{verdict:?}" + ); + let zero = measured( + "a", + "stale", + &[ + ("frames", 1.0, 1.0), + ("ms", -200.0, -200.0), + ("camera-frames", 0.0, 0.0), + ], + ); + assert!(judge_with(&xfails, "linux", &ran, &observed, &zero).ok()); + let unmeasured = measured("a", "stale", &[("frames", 1.0, 1.0)]); + let verdict = judge_with(&xfails, "linux", &ran, &observed, &unmeasured); + assert!(verdict.unexpected[0].why.contains("ms not measured")); + assert_eq!( + judge(&xfails, "linux", &ran, &observed).unexpected.len(), + 1, + "a bounded entry never excuses an unmeasured failure" + ); + assert!( + parse("require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"x\"\nreason = \"r\"\n") + .is_err() + ); + assert!( + parse("require_bound = true\n[[xfail]]\ncase = \"a\"\nclass = \"x\"\nseeds = \"1\"\nreason = \"r\"\n").is_ok(), + "seeded entries are bounded by their seeds" + ); + } + + #[test] + fn starved_entries_need_the_sentinel() { + let text = "[[xfail]]\ncase = \"a\"\nclass = \"error\"\nflaky = true\nstarved = true\nbound = { count = 1 }\nreason = \"r\"\n"; + let xfails = parse(text).unwrap(); + assert!(parse(&text.replace("flaky = true\n", "")).is_err()); + let ran = Ran::from([("a".to_string(), None)]); + let observed = Observed::from([(("a".into(), "error".into()), set(&[]))]); + let mut evidence = measured("a", "error", &[("count", 1.0, 1.0)]); + assert_eq!( + judge_with(&xfails, "linux", &ran, &observed, &evidence) + .unexpected + .len(), + 1 + ); + evidence.starved.insert("a".into()); + assert!(judge_with(&xfails, "linux", &ran, &observed, &evidence).ok()); + assert!(judge_with(&xfails, "linux", &ran, &Observed::new(), &evidence).ok()); + } + + #[test] + fn bless_bounds_new_entries_by_their_measurements() { + let ran = Ran::from([("a".to_string(), None)]); + let observed = Observed::from([(("a".into(), "sync".into()), set(&[]))]); + let evidence = measured("a", "sync", &[("ms", -201.0, -199.0)]); + let blessed = bless_with( + &[], + "macos", + host().as_deref(), + &ran, + &observed, + &evidence, + &BTreeMap::new(), + ); + assert_eq!(blessed.len(), 1); + assert_eq!( + blessed[0].bound["ms"], + Range { + min: -201.0, + max: -199.0 + } + ); + assert!(judge_with(&blessed, "macos", &ran, &observed, &evidence).ok()); + } +} diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rs new file mode 100644 index 00000000000..f1cdf634201 --- /dev/null +++ b/crates/verify-expect/src/main.rs @@ -0,0 +1,341 @@ +//! `cap-verify-expect check ...` validates expectations files. +//! `cap-verify-expect bless ...` rewrites every +//! expectations file named in the records so those runs pass exactly. Records +//! from several platforms (for example the CI report artifacts of all three +//! OSes) can be blessed in one call; each platform only changes its own +//! entries. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, + process::ExitCode, +}; + +use cap_verify_expect::{ + Evidence, Observed, Outcome, Ran, Record, bless_with, header, load, parse, render, +}; + +fn collect(path: &Path, out: &mut Vec) { + if path.is_dir() { + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + let mut entries: Vec = entries.flatten().map(|e| e.path()).collect(); + entries.sort(); + for entry in entries { + collect(&entry, out); + } + } else if path.extension().is_some_and(|ext| ext == "jsonl") { + out.push(path.to_path_buf()); + } +} + +fn repo_root() -> PathBuf { + std::env::var_os("CAP_VERIFY_ROOT") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) +} + +#[derive(Default)] +struct Run { + ran: Ran, + observed: Observed, + evidence: Evidence, + problems: BTreeMap<(String, String), String>, +} + +#[derive(serde::Deserialize)] +struct SourceRecord { + #[serde(flatten)] + record: Record, + #[serde(default)] + host: Option, +} + +type HostRuns = BTreeMap<(String, Option), Run>; +type Runs = BTreeMap; + +fn add_record(runs: &mut Runs, source: SourceRecord) { + let SourceRecord { record, host } = source; + if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) { + return; + } + let host = host + .map(|h| h.trim().to_string()) + .filter(|h| !h.is_empty() && h != "unknown"); + let run = runs + .entry(record.expectations.clone()) + .or_default() + .entry((record.platform.clone(), host)) + .or_default(); + let ran = run.ran.entry(record.case.clone()).or_insert(None); + if let Some(seeds) = &record.seeds_ran { + ran.get_or_insert_with(BTreeSet::new).extend(seeds); + } + for class in record.unobserved.keys() { + run.evidence + .unobserved + .insert((record.case.clone(), class.clone())); + } + if record.starved { + run.evidence.starved.insert(record.case.clone()); + } + for (class, metrics) in &record.measures { + let entry = run + .evidence + .measures + .entry((record.case.clone(), class.clone())) + .or_default(); + for (metric, range) in metrics { + entry + .entry(metric.clone()) + .and_modify(|r| { + r.include(range.min); + r.include(range.max); + }) + .or_insert(*range); + } + } + for (class, seeds) in &record.failures { + run.observed + .entry((record.case.clone(), class.clone())) + .or_default() + .extend(seeds); + if let Some(problem) = record.problems.get(class) { + run.problems + .entry((record.case.clone(), class.clone())) + .or_insert_with(|| problem.clone()); + } + } +} + +fn proposals( + entries: &[cap_verify_expect::Xfail], + runs: &HostRuns, +) -> Vec { + let mut entries = entries.to_vec(); + for ((platform, host), run) in runs { + entries = bless_with( + &entries, + platform, + host.as_deref(), + &run.ran, + &run.observed, + &run.evidence, + &run.problems, + ); + } + entries +} + +fn bless_command(inputs: &[String]) -> ExitCode { + let mut files = Vec::new(); + for input in inputs { + collect(Path::new(input), &mut files); + } + if files.is_empty() { + eprintln!("no .jsonl records under {inputs:?}"); + return ExitCode::FAILURE; + } + let mut runs = Runs::new(); + for file in &files { + let text = std::fs::read_to_string(file).unwrap_or_default(); + for line in text.lines().filter(|l| !l.trim().is_empty()) { + let record: SourceRecord = match serde_json::from_str(line) { + Ok(record) => record, + Err(e) => { + eprintln!("{}: skipping a record: {e}", file.display()); + continue; + } + }; + add_record(&mut runs, record); + } + } + let root = repo_root(); + for (expectations, platforms) in &runs { + let path = root.join(expectations); + let text = std::fs::read_to_string(&path).unwrap_or_default(); + let before = load(&path); + let entries = proposals(&before, platforms); + let added = entries.iter().filter(|e| !before.contains(e)).count(); + let removed = before.iter().filter(|e| !entries.contains(e)).count(); + let platform_list: Vec<_> = platforms.keys().collect(); + if added == 0 && removed == 0 { + println!("{expectations}: unchanged ({platform_list:?})"); + continue; + } + if let Err(e) = std::fs::write(&path, format!("{}{}", header(&text), render(&entries))) { + eprintln!("{}: {e}", path.display()); + return ExitCode::FAILURE; + } + println!( + "{expectations}: {removed} entries removed or changed, {added} written ({platform_list:?}); review the diff and fill in TODO reasons" + ); + } + ExitCode::SUCCESS +} + +fn check_command(paths: &[String]) -> ExitCode { + let mut ok = true; + for path in paths { + match std::fs::read_to_string(path) + .map_err(|e| e.to_string()) + .and_then(|t| parse(&t)) + { + Ok(entries) => { + let todo = entries + .iter() + .filter(|e| e.reason.starts_with("TODO")) + .count(); + if todo > 0 { + eprintln!("{path}: {todo} entries still have a TODO reason"); + ok = false; + } else { + println!("{path}: {} entries", entries.len()); + } + } + Err(e) => { + eprintln!("{path}: {e}"); + ok = false; + } + } + } + if ok { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + match args.split_first() { + Some((command, rest)) if command == "bless" && !rest.is_empty() => bless_command(rest), + Some((command, rest)) if command == "check" && !rest.is_empty() => check_command(rest), + _ => { + eprintln!( + "usage: cap-verify-expect bless ...\n cap-verify-expect check ..." + ); + ExitCode::from(2) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_verify_expect::{Metrics, Range}; + + fn source_record(source_host: Option<&str>, failure: Option) -> SourceRecord { + let mut value = serde_json::json!({ + "suite": "recording", + "expectations": "expectations.toml", + "case": "a", + "platform": "macos", + "outcome": if failure.is_some() { "xfail" } else { "pass" }, + }); + if let Some(host) = source_host { + value["host"] = host.into(); + } + if let Some(value_seen) = failure { + value["failures"] = serde_json::json!({ "sync": [] }); + value["measures"] = serde_json::json!({ "sync": { "ms": [value_seen, value_seen] } }); + } + serde_json::from_value(value).unwrap() + } + + #[test] + fn mixed_host_records_keep_separate_measurements_and_expectations() { + let entries = parse( + "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nplatforms = [\"macos\"]\nhosts = [\"local\", \"github-hosted\"]\nbound = { ms = [-5, 5] }\nreason = \"known offset\"\n", + ) + .unwrap(); + let mut runs = Runs::new(); + add_record(&mut runs, source_record(Some("local"), Some(3.0))); + add_record(&mut runs, source_record(Some("blacksmith"), Some(10.0))); + add_record(&mut runs, source_record(Some("github-hosted"), None)); + let host_runs = &runs["expectations.toml"]; + assert_eq!(host_runs.len(), 3); + let proposed = proposals(&entries, host_runs); + assert_eq!(proposed.len(), 2); + let local = proposed.iter().find(|x| x.hosts == ["local"]).unwrap(); + assert_eq!(local.bound, entries[0].bound); + assert_eq!(local.reason, entries[0].reason); + let blacksmith = proposed.iter().find(|x| x.hosts == ["blacksmith"]).unwrap(); + assert_eq!(blacksmith.bound["ms"], Range::at(10.0)); + assert!(blacksmith.reason.starts_with("TODO:")); + } + + #[test] + fn legacy_and_unknown_host_records_never_use_the_invoking_host() { + let entries = parse( + "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nhosts = [\"local\", \"github-hosted\"]\nreason = \"known offset\"\n", + ) + .unwrap(); + for host in [None, Some(""), Some("unknown")] { + let mut runs = Runs::new(); + add_record(&mut runs, source_record(host, None)); + add_record(&mut runs, source_record(Some("blacksmith"), None)); + let host_runs = &runs["expectations.toml"]; + assert!(host_runs.contains_key(&("macos".into(), None))); + assert_eq!(proposals(&entries, host_runs), entries); + } + } + + #[test] + fn expanded_signature_requires_a_new_explanation() { + let entries = parse( + "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nbound = { ms = [-5, 5] }\nreason = \"known offset\"\n", + ) + .unwrap(); + let ran = Ran::from([("a".into(), None)]); + let observed = Observed::from([(("a".into(), "sync".into()), BTreeSet::new())]); + let evidence = Evidence { + measures: BTreeMap::from([( + ("a".into(), "sync".into()), + Metrics::from([("ms".into(), Range::at(10.0))]), + )]), + ..Default::default() + }; + let proposals = bless_with( + &entries, + "macos", + Some("local"), + &ran, + &observed, + &evidence, + &BTreeMap::new(), + ); + assert_eq!(proposals.len(), 2); + assert!(proposals.contains(&entries[0])); + let path = std::env::temp_dir().join(format!( + "cap-verify-explanation-{}-{}.toml", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, render(&proposals)).unwrap(); + let status = check_command(&[path.to_string_lossy().into_owned()]); + std::fs::remove_file(path).unwrap(); + assert_eq!(status, ExitCode::FAILURE); + + let retained = bless_with( + &entries, + "macos", + Some("local"), + &ran, + &observed, + &Evidence { + measures: BTreeMap::from([( + ("a".into(), "sync".into()), + Metrics::from([("ms".into(), Range::at(3.0))]), + )]), + ..Default::default() + }, + &BTreeMap::new(), + ); + assert_eq!(retained, entries); + } +} diff --git a/crates/verify-stimulus/Cargo.toml b/crates/verify-stimulus/Cargo.toml new file mode 100644 index 00000000000..81905b4741d --- /dev/null +++ b/crates/verify-stimulus/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "cap-verify-stimulus" +version = "0.1.0" +edition = "2024" +publish = false +description = "Helper process of the real-capture recording oracle (crates/verify/tests/recording.rs): shows screen markers and plays system-audio markers" + +[lints] +workspace = true + +[[bin]] +name = "cap-verify-stimulus" +path = "src/main.rs" + +[dependencies] +cap-verify = { path = "../verify" } +anyhow = { workspace = true } +serde_json = "1" +workspace-hack = { version = "0.1", path = "../workspace-hack" } + +[target.'cfg(target_os = "macos")'.dependencies] +cidre = { workspace = true } +cpal = { workspace = true } +futures = { workspace = true } +objc2-foundation = "0.3.1" +scap-screencapturekit = { path = "../scap-screencapturekit" } +softbuffer = "0.4" +winit = "0.30" diff --git a/crates/verify-stimulus/src/main.rs b/crates/verify-stimulus/src/main.rs new file mode 100644 index 00000000000..528d9692fdf --- /dev/null +++ b/crates/verify-stimulus/src/main.rs @@ -0,0 +1,777 @@ +//! `cap-verify-stimulus `: see `cap_verify::stimulus`. + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("cap-verify-stimulus only runs on macOS"); + std::process::exit(2); +} + +#[cfg(target_os = "macos")] +fn main() { + if let Err(e) = app::run() { + println!("error {e}"); + std::process::exit(1); + } +} + +#[cfg(target_os = "macos")] +mod app { + use std::{ + fs::File, + io::{BufRead, Write}, + num::NonZeroU32, + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + mpsc, + }, + time::{Duration, Instant}, + }; + + use cap_verify::{ + audio_code::{self, AudioSource, MarkerPlacement}, + sched, + stimulus::{StimulusEvent, StimulusSpec, host_add_secs, host_now, host_secs}, + video_code::{self, FrameMarker, PixelOrder, StreamKind}, + }; + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + use objc2_foundation::{NSActivityOptions, NSProcessInfo, NSString}; + use winit::{ + application::ApplicationHandler, + dpi::{LogicalPosition, LogicalSize}, + event::{StartCause, WindowEvent}, + event_loop::{ActiveEventLoop, ControlFlow, EventLoop, EventLoopProxy}, + window::{Fullscreen, Window, WindowId, WindowLevel}, + }; + + #[derive(Debug)] + enum Command { + Go, + Quit, + } + + struct Log { + file: Mutex, + } + + impl Log { + fn write(&self, event: &StimulusEvent) { + if let Ok(line) = serde_json::to_string(event) + && let Ok(mut f) = self.file.lock() + { + let _ = writeln!(f, "{line}"); + let _ = f.flush(); + } + } + } + + fn say(word: &str) { + let mut out = std::io::stdout(); + let _ = writeln!(out, "{word}"); + let _ = out.flush(); + } + + pub fn run() -> anyhow::Result<()> { + let mut args = std::env::args().skip(1); + let spec: StimulusSpec = serde_json::from_str(&args.next().unwrap_or_default())?; + let log_path = PathBuf::from(args.next().unwrap_or_default()); + let log = Arc::new(Log { + file: Mutex::new(File::create(&log_path)?), + }); + let sentinel = Sentinel::start(log_path.with_file_name(SENTINEL_FILE)); + + use winit::platform::macos::{ActivationPolicy, EventLoopBuilderExtMacOS}; + let event_loop = EventLoop::::with_user_event() + .with_activation_policy(ActivationPolicy::Regular) + .with_activate_ignoring_other_apps(true) + .build()?; + let proxy = event_loop.create_proxy(); + spawn_stdin_reader(proxy); + + let epoch = Arc::new(AtomicU64::new(0)); + let audio = if spec.audio { + match start_audio(&spec, epoch.clone(), log.clone()) { + Ok(stream) => Some(stream), + Err(reason) => { + log.write(&StimulusEvent::AudioUnavailable { reason }); + None + } + } + } else { + None + }; + + let mut app = App { + spec, + log, + epoch, + go_at: None, + window: None, + surface: None, + size: (0, 0), + shown: None, + ready: false, + done: false, + error: None, + created: None, + idle_tick: 0, + last_idle: None, + thread: sched::current_thread(), + wake: (String::new(), None), + presented_at: None, + }; + let observer = start_observer(app.log.clone()); + let (qos, realtime) = raise_priority(app.spec.fps); + app.log.write(&StimulusEvent::Priority { qos, realtime }); + let process_info = NSProcessInfo::processInfo(); + let activity = unsafe { + process_info.beginActivityWithOptions_reason( + NSActivityOptions::UserInitiated | NSActivityOptions::LatencyCritical, + &NSString::from_str("Present recording verification markers on schedule"), + ) + }; + let result = event_loop.run_app(&mut app); + unsafe { process_info.endActivity(&activity) }; + result?; + drop(observer); + drop(audio); + sentinel.finish()?; + if let Some(error) = app.error { + anyhow::bail!(error); + } + Ok(()) + } + + use cap_verify::stimulus::{SENTINEL_FILE, SENTINEL_PERIOD_SECS}; + + /// A real-time thread that only wakes every `SENTINEL_PERIOD_SECS` and + /// records each wake (host ticks), for the whole life of the stimulus. + /// At priority 97 it preempts every timeshare thread of every process, + /// so a long gap between its wakes means the machine itself did not run + /// (the hypervisor descheduled the virtual CPUs, or the kernel did not + /// schedule), not that the recorder was busy. + struct Sentinel { + stop: Arc, + thread: std::thread::JoinHandle<(sched::Realtime, Option, Vec)>, + path: PathBuf, + } + + impl Sentinel { + fn start(path: PathBuf) -> Self { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let thread = { + let stop = stop.clone(); + std::thread::Builder::new() + .name("verify-sentinel".into()) + .spawn(move || { + let realtime = sched::request_realtime(SENTINEL_PERIOD_SECS, 0.0005); + let me = sched::current_thread(); + let mut wakes = Vec::with_capacity(64 * 1024); + let mut last = host_now(); + wakes.push(last); + while !stop.load(Ordering::Acquire) { + sched::wait_until(host_add_secs(last, SENTINEL_PERIOD_SECS)); + last = host_now(); + wakes.push(last); + } + (realtime, sched::thread_state(me), wakes) + }) + .expect("spawn sentinel") + }; + Self { stop, thread, path } + } + + /// Writes a JSON header line (scheduling, timebase, first wake), + /// then one line per wake: ticks since the previous wake. + fn finish(self) -> anyhow::Result<()> { + self.stop.store(true, Ordering::Release); + let (realtime, end, wakes) = self + .thread + .join() + .map_err(|_| anyhow::anyhow!("sentinel panicked"))?; + let info = cidre::mach::TimeBaseInfo::new(); + let mut out = serde_json::json!({ + "period_secs": SENTINEL_PERIOD_SECS, + "timebase": [info.numer, info.denom], + "first": wakes.first(), + "realtime": realtime, + "end": end, + }) + .to_string(); + out.push('\n'); + for w in wakes.windows(2) { + out.push_str(&(w[1] - w[0]).to_string()); + out.push('\n'); + } + std::fs::write(&self.path, out)?; + Ok(()) + } + } + + /// Stops the observer stream when dropped. + struct Observer { + stop: Arc, + thread: Option>, + } + + impl Drop for Observer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + /// A small ScreenCaptureKit stream of the primary display, owned by this + /// process: the recorder under test is a different client in a different + /// process, so a stretch where this stream also receives no new frames + /// while pictures are being presented is the window server or capture + /// service stalling (the input never reached the screen or the capture + /// service), not the recorder. + fn start_observer(log: Arc) -> Observer { + use cidre::{ns, sc}; + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let thread = { + let stop = stop.clone(); + std::thread::spawn(move || { + let fail = + |reason: String| log.write(&StimulusEvent::ObserverUnavailable { reason }); + let content = match futures::executor::block_on(sc::ShareableContent::current()) { + Ok(content) => content, + Err(e) => return fail(format!("shareable content: {e:?}")), + }; + let displays = content.displays(); + let Some(display) = displays.iter().next() else { + return fail("no display".into()); + }; + let filter = + sc::ContentFilter::with_display_excluding_windows(display, &ns::Array::new()); + let config = scap_screencapturekit::StreamCfgBuilder::default() + .with_width(160) + .with_height(120) + .with_fps(60.0) + .with_queue_depth(4) + .with_shows_cursor(false) + .build(); + let observed = log.clone(); + let capturer = scap_screencapturekit::Capturer::builder(filter, config) + .with_output_sample_buf_cb(move |frame| { + if let scap_screencapturekit::Frame::Screen(frame) = frame { + let host = cidre::cm::Clock::convert_host_time_to_sys_units( + frame.sample_buf().pts(), + ); + observed.write(&StimulusEvent::Observed { host }); + } + }) + .build(); + let capturer = match capturer { + Ok(capturer) => capturer, + Err(e) => return fail(format!("stream: {e:?}")), + }; + if let Err(e) = futures::executor::block_on(capturer.start()) { + return fail(format!("start: {e:?}")); + } + while !stop.load(Ordering::Acquire) { + std::thread::sleep(Duration::from_millis(20)); + } + let _ = futures::executor::block_on(capturer.stop()); + }) + }; + Observer { + stop, + thread: Some(thread), + } + } + + fn spawn_stdin_reader(proxy: EventLoopProxy) { + std::thread::spawn(move || { + let stdin = std::io::stdin(); + for line in stdin.lock().lines() { + let Ok(line) = line else { break }; + let command = match line.trim() { + "go" => Command::Go, + "quit" => Command::Quit, + _ => continue, + }; + if proxy.send_event(command).is_err() { + return; + } + } + let _ = proxy.send_event(Command::Quit); + }); + } + + struct App { + spec: StimulusSpec, + log: Arc, + epoch: Arc, + go_at: Option<(Instant, u64)>, + window: Option>, + surface: Option, Arc>>, + size: (u32, u32), + shown: Option, + ready: bool, + done: bool, + error: Option, + created: Option, + idle_tick: u32, + last_idle: Option, + thread: u32, + wake: (String, Option), + presented_at: Option<(u32, Instant)>, + } + + /// Hosted runners start jobs at utility QoS (scheduler base priority 20) + /// and children inherit it, so under recording load the stimulus loses + /// the CPU to the capture and encoder services for 50-190 ms at a time. + /// The pictures a user records come from foreground apps; the stimulus + /// asks for the same treatment (user-interactive QoS) and a time + /// constraint matching its frame period, so it keeps its schedule + /// (`cap_verify::sched`, shared with the synthetic camera and + /// microphone). Returns the kernel status of each request (0 = applied). + fn raise_priority(fps: f64) -> (i32, i32) { + let realtime = sched::request_realtime(1.0 / fps, 0.010); + (realtime.qos_status, realtime.policy_status) + } + + impl App { + /// The index that should be on screen now. + fn desired(&self, now: Instant) -> u32 { + let Some((go, _)) = self.go_at else { + return 0; + }; + let elapsed = now.saturating_duration_since(go).as_secs_f64(); + let last = self.spec.frame_count(); + let mut index = ((elapsed * self.spec.fps).floor() as u32 + 1).min(last); + while index > 1 && self.spec.held(self.spec.due_secs(index)) { + index -= 1; + } + index.max(self.shown.unwrap_or(0)) + } + + fn next_due(&self, now: Instant) -> Option { + let (go, _) = self.go_at?; + let shown = self.shown.unwrap_or(0); + let last = self.spec.frame_count(); + (shown + 1..=last) + .map(|i| go + Duration::from_secs_f64(self.spec.due_secs(i))) + .find(|&t| t > now) + .or(Some(now)) + } + + /// Before `go` the picture keeps frame 0 but the noise bands keep + /// moving at the frame rate, so the window server is compositing at + /// full rate by the time the recording starts (a cold paravirtual + /// display presents its first changes late). + fn idle_due(&self, now: Instant) -> Option { + if self.go_at.is_some() { + return None; + } + Some( + self.last_idle + .map_or(now, |t| t + Duration::from_secs_f64(1.0 / self.spec.fps)), + ) + } + + fn draw(&mut self, index: u32, salt: u64) { + let render_started = host_now(); + let (w, h) = self.size; + let Some(surface) = self.surface.as_mut() else { + return; + }; + if w == 0 || h == 0 { + return; + } + if surface + .resize(NonZeroU32::new(w).unwrap(), NonZeroU32::new(h).unwrap()) + .is_err() + { + return; + } + let Ok(mut buffer) = surface.buffer_mut() else { + return; + }; + let (w, h) = (w as usize, h as usize); + let band = if self.spec.busy { h / 8 } else { 0 }; + { + let bytes: &mut [u8] = unsafe { + std::slice::from_raw_parts_mut(buffer.as_mut_ptr().cast::(), w * h * 4) + }; + let stride = w * 4; + let marker = FrameMarker::new(StreamKind::Screen, 0, index); + video_code::draw_marker( + marker, + &mut bytes[band * stride..], + w, + h - 2 * band, + stride, + PixelOrder::Bgra, + ); + if band > 0 { + noise_band(&mut bytes[..band * stride], w, band, index, salt << 1); + noise_band( + &mut bytes[(h - band) * stride..], + w, + band, + index, + (salt << 1) | 1, + ); + } + } + if buffer.present().is_ok() { + let host = host_now(); + if self.shown != Some(index) { + self.log.write(&StimulusEvent::Frame { + index, + host, + render_started: Some(render_started), + }); + } + self.shown = Some(index); + if self.go_at.is_some() { + self.presented_at = Some((index, Instant::now())); + } + } + } + } + + /// Achromatic 16 px blocks whose levels change every frame. + fn noise_band(data: &mut [u8], w: usize, h: usize, index: u32, salt: u64) { + let stride = w * 4; + let mut seed = (u64::from(index) + 1).wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ salt; + let blocks_x = w.div_ceil(16); + let mut levels = vec![0u8; blocks_x * h.div_ceil(16)]; + for level in &mut levels { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + *level = 40 + (seed % 176) as u8; + } + for y in 0..h { + let row = &mut data[y * stride..y * stride + w * 4]; + for (x, px) in row.chunks_exact_mut(4).enumerate() { + let v = levels[(y / 16) * blocks_x + x / 16]; + px.copy_from_slice(&[v, v, v, 255]); + } + } + } + + impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window.is_some() { + return; + } + let mut attrs = Window::default_attributes() + .with_title("Cap Verify Stimulus") + .with_window_level(WindowLevel::AlwaysOnTop); + attrs = match self.spec.window { + None => attrs + .with_fullscreen(Some(Fullscreen::Borderless(event_loop.primary_monitor()))), + Some((x, y, w, h)) => attrs + .with_decorations(false) + .with_resizable(false) + .with_position(LogicalPosition::new(x, y)) + .with_inner_size(LogicalSize::new(w, h)), + }; + let window = match event_loop.create_window(attrs) { + Ok(w) => Arc::new(w), + Err(e) => { + self.error = Some(format!("window: {e}")); + event_loop.exit(); + return; + } + }; + let size = window.inner_size(); + self.size = (size.width, size.height); + let surface = softbuffer::Context::new(window.clone()) + .and_then(|context| softbuffer::Surface::new(&context, window.clone())); + match surface { + Ok(s) => self.surface = Some(s), + Err(e) => { + self.error = Some(format!("surface: {e}")); + event_loop.exit(); + return; + } + } + window.request_redraw(); + self.window = Some(window); + } + + fn user_event(&mut self, event_loop: &ActiveEventLoop, command: Command) { + match command { + Command::Go if self.go_at.is_none() => { + let host = host_now(); + self.go_at = Some((Instant::now(), host)); + self.epoch.store(host, Ordering::Release); + self.log.write(&StimulusEvent::Go { host }); + if let Some(window) = &self.window { + window.request_redraw(); + } + } + Command::Go => {} + Command::Quit => event_loop.exit(), + } + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + _id: WindowId, + event: WindowEvent, + ) { + match event { + WindowEvent::CloseRequested => event_loop.exit(), + WindowEvent::Resized(size) => { + self.size = (size.width, size.height); + if let Some(window) = &self.window { + window.request_redraw(); + } + } + WindowEvent::RedrawRequested => { + let now = Instant::now(); + let created = *self.created.get_or_insert(now); + let index = self.desired(now); + if self.go_at.is_none() { + if self.idle_due(now).is_some_and(|due| due <= now) { + self.idle_tick += 1; + self.last_idle = Some(now); + self.draw(0, u64::from(self.idle_tick)); + } + } else if self.shown != Some(index) { + self.draw(index, 0); + } + let warm = now.duration_since(created).as_secs_f64() >= self.spec.warmup; + if !self.ready && warm && self.shown == Some(0) { + self.ready = true; + let scale = self.window.as_ref().map_or(1.0, |w| w.scale_factor()); + self.log.write(&StimulusEvent::Ready { + host: host_now(), + width: self.size.0, + height: self.size.1, + scale, + }); + say("ready"); + } + if !self.done + && self.go_at.is_some() + && self.shown == Some(self.spec.frame_count()) + { + self.done = true; + self.log.write(&StimulusEvent::Done { host: host_now() }); + say("done"); + } + } + _ => {} + } + } + + fn new_events(&mut self, _event_loop: &ActiveEventLoop, cause: StartCause) { + let now = Instant::now(); + self.wake = match cause { + StartCause::ResumeTimeReached { + requested_resume, .. + } => ( + "timer".into(), + Some(now.saturating_duration_since(requested_resume).as_micros() as i64), + ), + StartCause::WaitCancelled { + requested_resume, .. + } => ( + "cancelled".into(), + requested_resume.map(|r| now.saturating_duration_since(r).as_micros() as i64), + ), + // A frame is drawn on the poll pass that follows the wake; + // keep the wake's own lateness. + StartCause::Poll => return, + StartCause::Init => ("init".into(), None), + }; + } + + fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { + let now = Instant::now(); + if let Some((index, at)) = self.presented_at.take() + && let Some(info) = sched::thread_state(self.thread) + { + self.log.write(&StimulusEvent::Sched { + index, + cause: self.wake.0.clone(), + wake_late_us: self.wake.1, + post_present_us: now.saturating_duration_since(at).as_micros() as i64, + cur_pri: info.cur_pri, + base_pri: info.base_pri, + cpu_us: info.cpu_us, + }); + } + if let Some(due) = self.idle_due(now) { + if due <= now + && let Some(window) = &self.window + { + window.request_redraw(); + } + event_loop.set_control_flow(ControlFlow::WaitUntil(due.max(now))); + return; + } + if self.done { + event_loop.set_control_flow(ControlFlow::Wait); + return; + } + if let Some(window) = &self.window + && self.desired(now) != self.shown.unwrap_or(0) + { + window.request_redraw(); + event_loop.set_control_flow(ControlFlow::Poll); + return; + } + match self.next_due(now) { + Some(next) => event_loop.set_control_flow(ControlFlow::WaitUntil(next)), + None => event_loop.set_control_flow(ControlFlow::Wait), + } + } + } + + struct AudioState { + sample_rate: u32, + channels: usize, + epoch: Arc, + schedule_origin: u64, + lead_secs: f64, + seconds: f64, + markers: Vec<(MarkerPlacement, Vec)>, + written: u64, + anchor: Option<(i64, u64)>, + events: mpsc::Sender, + } + + impl AudioState { + fn fill(&mut self, data: &mut [f32], callback_host: u64) { + data.fill(0.0); + let frames = data.len() / self.channels.max(1); + let start = self.written as i64; + self.written += frames as u64; + let go = self.epoch.load(Ordering::Acquire); + if go == 0 { + return; + } + let rate = f64::from(self.sample_rate); + let (anchor_sample, anchor_host) = *self.anchor.get_or_insert((start, callback_host)); + let end = start + frames as i64; + for (placement, wave) in &self.markers { + let host = host_add_secs(self.schedule_origin, placement.time_secs); + let after_go = host_secs(host, go); + if after_go < self.lead_secs || after_go > self.seconds { + continue; + } + let first = anchor_sample + (host_secs(host, anchor_host) * rate).round() as i64; + let last = first + wave.len() as i64; + if last <= start || first >= end { + continue; + } + if first >= start { + let offset = (first - start) as f64 / rate; + let _ = self.events.send(StimulusEvent::Audio { + index: placement.index, + host: host_add_secs(callback_host, offset), + }); + } + for i in 0..frames { + let at = start + i as i64; + if at < first || at >= last { + continue; + } + let v = wave[(at - first) as usize] * audio_code::DEFAULT_AMPLITUDE; + for c in 0..self.channels { + data[i * self.channels + c] += v; + } + } + } + } + } + + fn start_audio( + spec: &StimulusSpec, + epoch: Arc, + log: Arc, + ) -> Result { + use cpal::host::coreaudio::StreamInstantExt; + + let host = cpal::default_host(); + let device = match std::env::var("CAP_VERIFY_AUDIO_OUTPUT") { + Ok(name) if !name.is_empty() => host + .output_devices() + .map_err(|e| format!("output devices: {e}"))? + .find(|d| d.name().is_ok_and(|n| n == name)) + .ok_or_else(|| format!("no output device named {name}"))?, + _ => host + .default_output_device() + .ok_or("no default output device")?, + }; + let config = device + .default_output_config() + .map_err(|e| format!("output config: {e}"))?; + if config.sample_format() != cpal::SampleFormat::F32 { + return Err(format!("unsupported format {:?}", config.sample_format())); + } + let sample_rate = config.sample_rate().0; + let channels = config.channels(); + let schedule_origin = if spec.audio_origin == 0 { + host_now() + } else { + spec.audio_origin + }; + // The schedule shares its origin with the other marker sources of + // the case, so the per-source phases keep markers from overlapping. + let horizon = host_secs(host_now(), schedule_origin).max(0.0) + 120.0 + spec.seconds; + let markers = audio_code::standard_schedule( + AudioSource::System, + 0.0, + horizon.min(f64::from(audio_code::MAX_AUDIO_INDEX) * spec.audio_period), + spec.audio_period, + ) + .into_iter() + .map(|p| { + let wave = audio_code::marker_waveform(AudioSource::System, p.index, sample_rate); + (p, wave) + }) + .collect(); + let (tx, rx) = mpsc::channel::(); + { + let log = log.clone(); + std::thread::spawn(move || { + for event in rx { + log.write(&event); + } + }); + } + let mut state = AudioState { + sample_rate, + channels: usize::from(channels), + epoch, + schedule_origin, + lead_secs: 0.25, + seconds: spec.seconds, + markers, + written: 0, + anchor: None, + events: tx, + }; + let stream = device + .build_output_stream( + &config.config(), + move |data: &mut [f32], info: &cpal::OutputCallbackInfo| { + state.fill(data, info.timestamp().callback.as_host_time()); + }, + |e| eprintln!("stimulus audio error: {e}"), + None, + ) + .map_err(|e| format!("output stream: {e}"))?; + stream.play().map_err(|e| format!("play: {e}"))?; + log.write(&StimulusEvent::AudioStarted { + device: device.name().unwrap_or_default(), + sample_rate, + channels, + }); + Ok(stream) + } +} diff --git a/crates/verify/Cargo.toml b/crates/verify/Cargo.toml new file mode 100644 index 00000000000..4392a72d173 --- /dev/null +++ b/crates/verify/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "cap-verify" +version = "0.1.0" +edition = "2024" +publish = false + +[lints] +workspace = true + +[dependencies] +cap-media-info = { path = "../media-info" } +cap-project = { path = "../project" } +cap-recording = { path = "../recording" } +cap-timestamp = { path = "../timestamp" } + +anyhow = { workspace = true } +ffmpeg = { workspace = true } +flume = { workspace = true } +futures = { workspace = true } +relative-path = "1.9.3" +rustfft = "6.4.1" +serde = { workspace = true } +serde_json = "1" +tokio = { workspace = true } +cap-verify-expect = { path = "../verify-expect" } +workspace-hack = { version = "0.1", path = "../workspace-hack" } + +[target.'cfg(target_os = "macos")'.dependencies] +cap-enc-avfoundation = { path = "../enc-avfoundation" } +cidre = { workspace = true } + +[dev-dependencies] +cap-recording = { path = "../recording", features = ["test-utils"] } +cap-editor = { path = "../editor", features = ["test-hooks"] } +cap-audio = { path = "../audio" } +cap-export = { path = "../export" } +cap-rendering = { path = "../rendering" } +tempfile = "3" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[target.'cfg(target_os = "macos")'.dev-dependencies] +scap-targets = { path = "../scap-targets" } diff --git a/crates/verify/README.md b/crates/verify/README.md new file mode 100644 index 00000000000..fb01d84883c --- /dev/null +++ b/crates/verify/README.md @@ -0,0 +1,155 @@ +# cap-verify: engine verification suite + +Oracle tests for the playback, export, recording, editing and captions engines. +Fixtures are generated by code at test time. Every source video frame carries +its own identity as a block code, and every audio source carries +sample-accurate markers. An independent reference model computes which source +frame and sample each output frame and sample must show. A case passes only +when every checked frame and sample matches. + +## Suites + +The suites are listed in `scripts/verify/suites.json`: + +| Suite | What it checks | FAST tier | FULL tier | +| --- | --- | --- | --- | +| `verify-self` | marker codecs, reference model, report | linux | linux, macos | +| `export` | every exported frame (MP4, GIF, both decoders on macOS) | linux, macos, windows | all | +| `playback` | offline-driven editor playback frames | linux, macos | all | +| `seek` | random, keyframe and scrub seeks | linux, macos | all | +| `audio` | export and offline playback audio against sample-accurate markers | linux, macos, windows | all | +| `editing` | random edit sequences exported and checked frame by frame | linux | linux | +| `captions` | fake-transcriber captions: segments and rendered captions on the right frames | linux | linux | +| `decoder-model-ffmpeg` | FFmpeg decoder decisions on scripted media (`cap-rendering` `decoder::model`) | linux | linux | +| `decoder-model-avf` | AVAssetReader decoder decisions on scripted media | macos | macos | + +The upload fault suite (`apps/desktop-gpui/src/upload/verify_tests.rs` and +`upload/queue/verify_resume_tests.rs`: a mock Cap API with scripted 5xx, +resets, outages, hangs and quits, byte-identical objects and request and byte +budgets) lives in the GPUI app's own workspace. It runs inside +`cargo test --bin cap-gpui`, which the `CI / Build GPUI Desktop` job already +runs on all three OSes, so that job is its gate. The app-level caption tests +(`apps/desktop-gpui/src/transcription/verify_tests.rs`: a fake transcriber +behind the real extraction and caption pipeline) run in the same job. Run them +locally with +`cargo test --manifest-path apps/desktop-gpui/Cargo.toml --bin cap-gpui upload::` +(or `transcription::`). + +Platform-independent logic runs on Linux only in CI. Suites that test platform +code run on that platform: AVFoundation, VideoToolbox and Metal on macOS; +Media Foundation, Direct3D and WASAPI on Windows. When you run the suite +locally, the platform-independent suites run on every OS. + +`CAP_VERIFY_TIER=full` (the FULL tier) uses longer fixtures and more seeds. + +## Running locally + +```sh +scripts/verify/run.sh fast # what CI runs on this OS for a PR +scripts/verify/run.sh full # merge-queue and nightly tier +scripts/verify/run.sh fast --suites seek # one suite +scripts/verify/run.sh fast --case 'export/basic/*' +scripts/verify/run.sh fast --list # print the plan +``` + +Install cargo-nextest (`cargo install cargo-nextest --locked`) to run the +tests in parallel, one process per test. Without it, the script runs +`cargo test` per suite. The local run uses the `dev` profile and shares its +build with normal development. CI uses the `ci-verify` profile +(`CAP_VERIFY_CARGO_PROFILE=ci-verify`). Generated fixtures are cached in +`target/verify-fixtures`, or in `CAP_VERIFY_FIXTURE_DIR` when it is set. + +Every failure prints the exact frames or samples that are wrong, the seed, +and a one-line repro command, for example: + +```sh +CAP_VERIFY_CASE='export/backward-hold/backward-jumps/mp460/avf' CAP_VERIFY_TIER=fast cargo test -p cap-verify --test export -- --nocapture +CAP_DECODER_MODEL_CASE=decoder-model/avf/syn/edlist/playback CAP_DECODER_MODEL_SEED_INDEX=3 cargo test -p cap-rendering --lib decoder::model::avassetreader_answers_match_the_reference_model -- --nocapture +``` + +## Known failures: strict expectations + +Known bugs on `main` are strict expected failures. They are listed in +`crates/verify/expectations.toml` and in +`crates/rendering/src/decoder/model/expectations.toml`, using the format from +`crates/verify-expect`: + +```toml +[[xfail]] +case = "export/early-camera/full/*" # case id, * globs allowed +class = "future" # failure class that must still occur +platforms = ["macos"] # optional, absent = every platform +seeds = "0 3-7" # optional, seeded suites only +flaky = true # optional; state the measured rate +reason = "what is wrong" +link = "https://github.com/CapSoftware/Cap/pull/2595" # empty = no fix yet +``` + +- A failure that is not listed fails the run. +- A listed class or seed that no longer fails also fails the run, with + "remove this xfail". When a fix lands, its expected failures must be removed + in the same PR, so the fix is locked in. +- A class that cannot be measured because an earlier check failed (A/V sync + when no audio marker of a source could be identified) is recorded as "not + measured", never as a pass: its xfail is neither confirmed nor reported as + fixed, and bless leaves it alone. The failure that prevented the measurement + is judged on its own, so it still has to be listed. + +To update the files after a deliberate change, run: + +```sh +scripts/verify/bless.sh fast # run here and rewrite from the results +scripts/verify/bless.sh --run # rewrite from a CI run, all three OSes +``` + +Bless changes only the entries of the platforms that ran. It keeps the +reasons and links of existing entries, and gives each new failure a `TODO:` +reason. Replace every `TODO:` reason before you merge. Review the diff: a +blessed failure is a bug that the suite keeps checking. + +## CI + +`.github/workflows/engine-verify.yml`: + +- **Plan.** `scripts/verify/plan.py` maps the changed paths to suites through + the root workspace dependency graph (`cargo metadata`). The standalone + GPUI app is outside that graph, so any change under `apps/desktop-gpui/` + conservatively selects every engine suite. This selection does not replace + the app's own tests or prove coverage of its orchestration. A PR that changes + only docs or the web app runs no engine suite. +- **Tiers.** FAST runs on every PR push. FULL runs when the PR has the + `verify-full` label, in the merge queue, and nightly. +- **Jobs.** One job per platform builds the selected tests with the + `ci-verify` profile and runs them with nextest. Dependencies are optimised + and cached; workspace crates are not optimised. A platform with + `shards > 1` in `suites.json` uploads a nextest archive, and shard jobs run + partitions of it. +- **No retries, except for a proven machine stall.** Every test process + runs a starvation sentinel (`crates/verify/src/sentinel.rs`): a + high-priority thread that sleeps 2 ms and records how late it wakes, plus + the process CPU time over wall time. Each case record carries that series + (`machine`), and `sentinel-.log` keeps it for tests that are killed. + A wake gap of 100 ms or more proves the runner stopped scheduling the + process. Only wall-clock failures are excused by it: a real-time playback + case that delivered too few frames becomes `inconclusive`, and a test + killed at its timeout, by wgpu's 6.3 s queue drain or by a lost GPU device is + retryable when its sentinel log shows the gap. `scripts/verify/retry.py` reruns those tests + once with `CAP_VERIFY_RETRY=1`; a case that is inconclusive again fails as + `environment`, and the job passes only if every failure was retryable and + every retry passed. Content, frame identity, timestamps and sync never + become inconclusive. Any other failed test is rerun once only so that the + summary can label it flaky; the job still fails. +- **Summary.** Each job summary lists the failing cases, their exact frames + or samples, and a repro command. The report records are uploaded as the + `verify-report-*` artifacts, which `bless.sh --run` reads. +- **Gate.** Make the `Engine verify` job a required check. + +## Baseline comparison + +Run `python3 scripts/verify/diff.py BASE_DIR MAIN_DIR --json comparison.json` with downloaded report artifacts. Comma-separated run directories combine independent runs on each side. Keep both sides on the same OS image. + +A required track absent from a recording leaves its dependent classes unobserved with reason `track absent`. A newly observed class appears in the coverage-gained column and is still judged by main's own expectations. Coverage gain alone is not an equal or worse result. + +The `missing-after-hold` class measures an environment-sensitive ScreenCaptureKit idle-resume event. Compare at least three independent real-capture runs per side per OS image. A distribution is worse only when every main per-run count is above the baseline range. Insufficient repetitions and overlapping ranges are reported explicitly. Retries of one capture do not add independent samples. Other classes retain the strict zero-tolerance comparison. + +Finalize crash cases hold the test muxer at entry to finalization after source EOF. The parent kills it while the barrier is held and checks that the recorder never wrote `done`. Missing the barrier or finding `done` is a harness failure. The barrier exists only in the integration-test wrapper and is disabled for all ordinary cases. diff --git a/crates/verify/expectations.toml b/crates/verify/expectations.toml new file mode 100644 index 00000000000..09db7f24934 --- /dev/null +++ b/crates/verify/expectations.toml @@ -0,0 +1,673 @@ +# Strict expected failures for the engine verification suite +# (`cargo test -p cap-verify`), in the shared format of crates/verify-expect +# (also used by crates/rendering/src/decoder/model/expectations.toml): +# +# [[xfail]] +# case = "" +# class = "" +# bound = { = or [, ], ... } +# reason = "" +# link = "" +# platforms = ["macos"] (optional; absent = every platform) +# flaky = true (optional; timing-dependent, may pass. State the +# measured rate in the reason.) +# starved = true (optional, flaky only: excuses the failure only +# when the case's starvation sentinel saw the +# machine stall during the case) +# +# Every entry carries the bound of its documented mechanism. The suite +# measures each failing class, and an entry excuses a failure only when every +# measurement lies inside its bound; a measured metric the bound does not +# name must be zero. Metrics: +# - frame classes (stale, future, unreadable, missing, ...): -frames +# (frames affected per stream), frame (range of output frames or seek +# positions affected), delta (largest source-frame error); +# - classes from a single check: count (number of problems), plus the +# check's own measurement: offset-ms (A/V offset, audio minus screen, or +# screen minus audio for sync-screen-*), spread-ms, missing, unexpected, +# drift-ms, luma / cb / cr (code values), off-periods (frame periods off +# the timestamp grid), -start-late-ms, -end-lost. +# +# A listed class that no longer occurs fails the run with "remove this +# xfail", so fixes stay locked in. A class that occurs and is not listed, or +# that occurs outside its bound, fails the run. scripts/verify/bless.sh +# rewrites this file from a local run or from a CI run's report artifacts +# (`--run `, every platform), keeping reasons, links and bounds; new +# entries get a TODO reason and their measured range as the bound. Review the +# diff. +# +# "NEW:" marks bugs the suite found that no open PR fixed when they were +# recorded. #2638 fixes the AVAssetReader and pts-hole decoder entries +# (it supersedes #2595 and #2603), #2636 the decode-deadline entries. + +require_bound = true + +[[xfail]] +case = "export/jitter-camera-40s/full/mp460/avf" +class = "future" +platforms = ["macos"] +bound = { camera-frames = [500, 560], delta = 320, frame = [1580, 2399] } +reason = "Camera runaway: with a jittery ~12 fps camera at 30 fps nominal exported at 60 fps, the pts-hole map evicts the hole it just recorded once 64 wider ones are tracked, and the reader drifts ahead from output frame 1585 on (530 camera frames ahead, up to 298 source frames, every run; the screen is never affected)" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "export/jitter-camera-120s/full/mp460/avf" +class = "future" +platforms = ["macos"] +bound = { camera-frames = [5000, 5250], delta = 2232, frame = [1585, 7199] } +reason = "Same camera runaway on the 120 s fixture (full tier): from output frame 1585, 5146 camera frames ahead, up to 2232 source frames (74 s); 5081 to 5201 in the runs whose exports differ (see the nondeterministic entry). The screen is never affected" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "playback/jitter-camera-120s/full/seq60-from0/avf" +class = "future" +platforms = ["macos"] +bound = { camera-frames = [2300, 2800], delta = 1900, frame = [2480, 7199] } +reason = "Same camera runaway in offline playback of the 120 s fixture: from output frame 2490, 2485 to 2618 camera frames ahead, up to 1419 to 1726 source frames (the count varies with the decoder's request batching)" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "playback/jitter-camera-120s/full/seq60-from0/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { screen-frames = 1, camera-frames = 1, delta = 1, frame = [7199, 7199] } +reason = "The last output frame of the 120 s offline playback (7199, 119.983 s) shows the source frame before the right one in 1 of 3 full runs (38763192776). Mechanism: when the request stream pauses for 250 ms or more, the AVAssetReader decoder's idle timeout (IDLE_READER_TIMEOUT, crates/rendering/src/decoder/avassetreader.rs) clears its cache and reader state, and the final request is then answered with the newest frame before it. Reproduced on Apple silicon by pausing 300 ms before the last request (1 of 1); a 100 ms pause does not reproduce it. Bound: only the final frame, one source frame stale" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "export/jitter-camera-120s/full/mp460/avf" +class = "nondeterministic" +platforms = ["macos"] +flaky = true +bound = { count = 1, frames = [1, 5615], frame = [1585, 7199] } +reason = "Two exports of the 120 s jitter-camera project can differ, only inside the camera runaway of the future entry above (from output frame 1585 on; first difference measured at 1723 or later). Mechanism (scratch/jitter-probe3, runs 38847126270, 38849423910, 38849597307, instrumented decoder): export asks for camera frames one at a time (every batch held one request in 116 traced exports, so try_recv batching is not involved); when more than 250 ms pass between two camera requests the AVAssetReader decoder's idle timeout (IDLE_READER_TIMEOUT, crates/rendering/src/decoder/avassetreader.rs:48, teardown at :728-744) drops every reader, and inside the runaway the re-created reader answers that request correctly, so the runaway restarts from there and every later answer changes (a 390 ms stall before camera request 1105: answer 1105 in one export, 2345 in the other). A forced teardown at camera request 2000 gives 5094 wrong frames, identical in both runs; with the teardown disabled 0 of 12 samples differed although request gaps reached 439 ms. Under suite load the gap is wall-clock, hence flaky (3 of 8 runs beside the export suite; alone 16 of 16 identical). #2638 removes the runaway and with it this class (20 of 20 identical). Bound: same frame count, differences only at or after the runaway's first frame; the future entry bounds the wrong frames themselves (5000 to 5250 camera frames from frame 1585, at most 2232 ahead), and a camera that falls back to FFmpeg after a slow re-created reader fails as stale" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "*/jitter-camera-120s/full/*/ffmpeg" +class = "stale" +bound = { camera-frames = 1551, delta = 90, frame = [2493, 6767] } +reason = "Same pts-hole eviction on the FFmpeg decoder (full tier, 120 s): 1551 of 7199 camera frames stale, up to 90 source frames, output frames 2493 to 6767, identical in the MP4 export and in offline playback on every platform that runs it" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "export/backward-hold/backward-jumps/mp460/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { camera-frames = 1, delta = 1, frame = [2579, 2579] } +reason = "AVAssetReader serves one camera frame 1 source frame stale after the timeline jumps backward (output frame 2579: source 1048 for 1049): the reader chosen after the jump had already vended past the request and its previous vend was no longer cached, so the nearest older cached frame answered. Every run in probe 38762089521 (42 of 42) and on hosted macOS, but it passed on blacksmith-6vcpu-macos-15 in push run 38846099909: whether the previous vend is still cached depends on the decoder's wall-clock idle timeout, so it is flaky there" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "export/camera-pts-late/full/mp430/avf" +class = "future" +platforms = ["macos"] +bound = { camera-frames = 127, delta = 53, frame = [8, 134] } +reason = "A camera file whose pts start 2 s in runs ahead instead of holding its first frame: before the track's first sample every request steps the reader one sample further (127 camera frames, output frames 8 to 134, up to 53 source frames ahead)" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "export/camera-pts-late/full/mp430/ffmpeg" +class = "future" +bound = { camera-frames = 178, delta = 60, frame = [1, 178] } +reason = "NEW: FFmpeg decoder, same as the AVAssetReader case: a camera file whose pts start 2 s in runs ahead from the first output frame (178 of 179 frames, up to 60 source frames ahead, every platform)" +link = "" + +[[xfail]] +case = "seek/seek-gop/keyframes60/avf" +class = "stale" +platforms = ["macos"] +bound = { screen-frames = 3, camera-frames = 3, delta = 1, frame = [351, 701] } +reason = "Wrong-GOP seek: the reader starts on the keyframe time rounded to 10 ms, decodes the previous GOP and shows its last frame under the keyframe's number (requests between the keyframe pts and the next slot show keyframe - 1): 3 seeks, one source frame stale" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-native/keyframes60/avf" +class = "stale" +platforms = ["macos"] +bound = { screen-frames = 8, camera-frames = 8, delta = 44, frame = [45, 705] } +reason = "Wrong-GOP seek on the native fixture (see seek/seek-gop/keyframes60/avf): 8 seeks show a frame of the previous GOP, up to 44 source frames stale" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-gop/storm30/avf" +class = "stale" +platforms = ["macos"] +bound = { screen-frames = 30, camera-frames = 30, delta = 450, frame = [1, 900] } +reason = "A reused AVAssetReader answers from a frame cached by an earlier, unrelated seek (seek 349 shows 237): 20 of 3000 full-tier seeks (1 in the fast tier), up to 352 source frames stale. Which earlier seek's frame answers depends on timing, so the bound is the mechanism's: any frame of the fixture (450), at most 1% of the seeks" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-native/storm30/avf" +class = "stale" +platforms = ["macos"] +bound = { screen-frames = 40, camera-frames = 40, delta = 450, frame = [1, 900] } +reason = "Same reader reuse as seek/seek-gop/storm30/avf on the native fixture: 7 to 28 of 3000 full-tier seeks (2 in the fast tier), up to 243 to 287 source frames stale between runs; bounded like seek/seek-gop/storm30/avf" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-*/storm30/avf" +class = "future" +platforms = ["macos"] +flaky = true +bound = { screen-frames = 12, camera-frames = 12, delta = 2, frame = [1, 900] } +reason = "Full-tier seek storm: a few seeks are answered one or two source frames ahead (seek-gop frame 4 shows 5 for 3; 3 seeks on seek-gop, 11 on seek-native). Pool position capping: after a read the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame is reused for it without a reset and answers with its next sample (same mechanism as seek/seek-native/keyframes60/avf, minimal repro there). Whether a storm reaches that state depends on request timing: the decoder's 250 ms idle timeout (IDLE_READER_TIMEOUT) clears its cache and readers when requests pause, and with a 300 ms pause every 7 requests the seek-native storm has no ahead answer (measured on Apple silicon; 7 ahead without pauses). seek-gop: 3 of 3 full runs and locally; seek-native: 1 of 3 full runs (38763178760) and locally" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/vfr-holds/storm60/avf" +class = "future" +platforms = ["macos"] +bound = { screen-frames = 72, camera-frames = 14, delta = 253, frame = [1, 1800] } +reason = "Seek storm over long VFR holds: AVAssetReader shows a frame from the far side of a hold (expected 60, got 217): 72 screen and 14 camera seeks of the full storm (8 and 2 in the fast tier), up to 253 source frames ahead" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/vfr-holds/storm60/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { screen-frames = 60, camera-frames = 40, delta = 536, frame = [1, 1800] } +reason = "Same storm, stale answers from reused readers: 52 to 53 screen and 35 to 36 camera seeks in full runs, 0 screen and 2 camera in the fast tier, up to 536 source frames stale (timing-dependent reader reuse; 3 of 4 runs on main)" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/vfr-holds/into-holds60/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { camera-frames = 1, delta = 1, frame = [149, 149] } +reason = "NEW: AVAssetReader camera answers one frame stale when a region is revisited after other seeks (see seek/vfr-holds/revisit60/avf for the deterministic sequence): output frame 149, one source frame. Fails 12 of 13 runs on main; it passed once under parallel load, probably because the decoder's wall-clock paths (250 ms idle reader drop, 100 ms scrub cooldown) change which branch answers" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/vfr-holds/revisit60/avf" +class = "stale" +platforms = ["macos"] +bound = { camera-frames = 1, delta = 1, frame = [149, 149] } +reason = "NEW: seeks to 2.0 s, 3.5 s, 1.2 s, 2.3 s (each + 29 playback frames, 60 fps): the camera request at 2.483 s (slot 74, CFR camera) is answered with slot 73. Deterministic on runners: every run on main and with #2603; FFmpeg is correct. Repro: CAP_VERIFY_SEEK_SCRIPT='vfr-holds@60:2.0+29,3.5+29,1.2+29,2.3+29' cargo test -p cap-verify --test seek script -- --nocapture" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-gop/storm30/ffmpeg" +class = "stale" +bound = { screen-frames = 100, camera-frames = 100, delta = 100, frame = [1, 900] } +reason = "NEW: FFmpeg decoder seeks answer stale frames: the backward-seek lookback tolerance serves a cached frame 1 to 4 source frames old, and seeks near the start are answered by the first-frame shortcut with frame 0 (71 of 3000 full-tier seeks, 1 in the fast tier, up to 81 source frames stale). The count depends on the decoder's wall-clock paths (it varies between runs on macOS: 65 to 78 on the native fixture), so the bound is the mechanism's: at most 100 seeks (3.3%), never more stale than the 100 frames the shortcut and the lookback can reach" +link = "" + +[[xfail]] +case = "seek/seek-native/storm30/ffmpeg" +class = "stale" +bound = { screen-frames = 100, camera-frames = 100, delta = 100, frame = [1, 900] } +reason = "NEW: same FFmpeg lookback tolerance and first-frame shortcut as seek/seek-gop/storm30/ffmpeg on the native fixture: 62 to 78 of 3000 full-tier seeks (9 in the fast tier), up to 83 source frames stale; bounded like seek/seek-gop/storm30/ffmpeg" +link = "" + +[[xfail]] +case = "seek/vfr-holds/storm60/ffmpeg" +class = "stale" +bound = { screen-frames = 38, camera-frames = 67, delta = 76, frame = [1, 1800] } +reason = "NEW: same FFmpeg lookback tolerance and first-frame shortcut over long VFR holds: 38 screen and 67 camera seeks of the full storm (7 and 11 in the fast tier), up to 76 source frames stale" +link = "" + +# Decode deadline under machine starvation. These apply only to a case the +# starvation sentinel saw stall (starved = true) and only with the signature +# of the mechanism; the same symptom on a healthy machine is a failure. + +[[xfail]] +case = "playback/backward-hold/backward-jumps/seq30-from0/ffmpeg" +class = "error" +platforms = ["windows"] +flaky = true +starved = true +bound = { count = 1 } +reason = "NEW: under load offline playback gets no frame at all: a decode that misses its 2,000 ms deadline (crates/rendering/src/decoder/mod.rs, request_frame returns None) leaves the request unanswered (\"decoders returned no frame for clip 0 at 5.000s\"). Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 3.1 s, 38 gaps over 100 ms); same deadline as the export frame reorder entry. Bound: one error, only under a sentinel-proven stall" +link = "https://github.com/CapSoftware/Cap/pull/2636" + +[[xfail]] +case = "export/early-camera/full/mp430/ffmpeg" +class = "frame-count" +platforms = ["windows"] +flaky = true +starved = true +bound = { count = 1, missing = 1 } +reason = "NEW: under load an offline export loses its first frame: when the first decodes miss their 2,000 ms deadline the render loop advances before any usable frame exists (crates/rendering/src/lib.rs, initial decode failure path; sol-starved-export.md), so the export has 179 of 180 frames. Seen once on a starved windows-2022 runner (Engine verify 38841241061, windows shard 1, sentinel longest wake gap 4.2 s, 133 gaps over 100 ms). Bound: exactly one frame, only under a sentinel-proven stall" +link = "https://github.com/CapSoftware/Cap/pull/2636" + +[[xfail]] +case = "export/early-camera/full/mp430/ffmpeg" +class = "future" +platforms = ["windows"] +flaky = true +starved = true +bound = { screen-frames = 179, camera-frames = 179, delta = 2, frame = [0, 179] } +reason = "NEW: same initial decode-deadline loss as the frame-count entry: with the first frame gone every later frame shows the source one or two frames ahead (146 of 178 screen frames). Seen once on a starved windows-2022 runner (Engine verify 38841241061). Bound: at most two source frames ahead, only under a sentinel-proven stall" +link = "https://github.com/CapSoftware/Cap/pull/2636" + +# Instant mode (fixtures recorded through the real Instant muxers and +# finalize_instant_output). Each audio source is its own class, so a known +# loss or offset of one source cannot hide the other's. + +[[xfail]] +case = "export/instant-ungated-mic-late/output/mp4/native" +class = "sync-mic" +platforms = ["macos", "windows"] +bound = { count = 1, offset-ms = [-210, -190] } +reason = "NEW: the Instant audio track is anchored at its first sample (AudioAnchor::FirstFrame) and nothing aligns it to the screen afterwards: a mic that becomes ready 0.2 s after the screen plays 200 ms early in output.mp4 (-198.7 to -201.1 ms measured). Bound: early by the mic's start delay" +link = "" + +[[xfail]] +case = "export/instant-ungated-system-loopback/output/mp4/native" +class = "sync-system" +platforms = ["macos", "windows"] +bound = { count = 1, offset-ms = [-270, -250] } +reason = "NEW: same anchor, loopback-style system audio (WASAPI delivers packets only while sound plays): the first sound becomes audio zero, so all audio is early by the time to first sound (260 ms here; -253.7 to -260.2 ms measured)" +link = "" + +[[xfail]] +case = "export/instant-gated-system-only/output/mp4/native" +class = "sync-system" +platforms = ["macos", "windows"] +bound = { count = 1, offset-ms = [-240, -215] } +reason = "NEW: same first-sample anchor behind the start gate, loopback-only system audio: the first sound after the gate becomes audio zero, so the system audio is early by the time from the gate to the first sound (-226.7 ms in every run on macOS and Windows; Linux, which anchors Instant audio at the recording epoch, is only up to one frame late, see the gated sync-system entry)" +link = "" + +[[xfail]] +case = "export/instant-gated-*mic*/output/mp4/native" +class = "sync-mic" +bound = { count = 1, offset-ms = [5, 34] } +reason = "NEW: after the start gate opens, the screen and audio pipelines each take their own first admitted frame as zero, so audio is late by up to one video frame (33.3 ms at 30 fps; 12.6 to 17.2 ms measured with the gate opening between frames)" +link = "" + +[[xfail]] +case = "export/instant-gated-system-*/output/mp4/native" +class = "sync-system" +bound = { count = 1, offset-ms = [5, 34] } +reason = "NEW: the gate's per-pipeline zero, as the gated sync-mic entry: system audio late by up to one video frame (13.3 to 15.1 ms measured; on macOS and Windows the loopback-only variant is the first-sample anchor entry instead)" +link = "" + +[[xfail]] +case = "export/instant-gated-system-only/output/mp4/native" +class = "audio-system" +flaky = true +bound = { count = 1, missing = 2, lost = 2 } +reason = "NEW: gated loopback-only system audio sometimes loses markers (2 of 11 lost in 1 of 2 fresh recordings, with the track 958 ms early). Bound: at most 2 markers lost, none unexpected" +link = "" + +[[xfail]] +case = "export/instant/output/mp4/native" +class = "sync-system" +flaky = true +bound = { count = 1, offset-ms = [-2.7, 28.3], mic-offset-ms = [5, 34], system-minus-mic-ms = [-7.7, -5.7] } +reason = "Instant mixer input timing and the gate phase produce a -6.7 ms system-minus-mic mode, within 1 ms. Only this inter-source signature with the microphone inside its unchanged 5 to 34 ms bound is excused. Coordinator decision 09:20, 2026-10-16." +link = "https://github.com/CapSoftware/Cap/tree/fix/instant-mixer-input-alignment" + +[[xfail]] +case = "export/instant/output/mp4/native" +class = "sync-mic" +bound = { count = 1, offset-ms = [5, 34] } +reason = "NEW: see export/instant-gated-*: mic audio up to one video frame late after the start gate (14.2 to 15.3 ms measured)" +link = "" + +[[xfail]] +case = "*/basic/transition/*60*/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { screen-frames = 1, delta = 1, frame = [149, 149] } +reason = "During a cross-fade both clips read the same AVAssetReader decoder at two positions; a reset for one position drops the other position's last vended frame from the cache, so at 60 fps the incoming side of output frame 149 (2.483 s) shows source frame 115 instead of 116. Not the IOSurface source lifetime bug: it stays with that fix (sol-avf proof run 38762654200). Same frame in export and in offline playback. Fires in almost every run but depends on the decoder's cache state, which its 250 ms idle timeout (IDLE_READER_TIMEOUT) clears: a 300 ms pause in the request stream at 2.0, 2.42 or 2.48 s makes offline playback correct (measured on Apple silicon), and the export passed once on macos-15 (push run 38763177849). Bound: that one screen frame, one source frame stale" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "audio/audio-mix-*-a2/long-crossfade/mp4-*" +class = "stale" +platforms = ["linux", "windows"] +bound = { screen-frames = 18, delta = 49, frame = [61, 115] } +reason = "FFmpeg decoder, same-clip cross-fade (cuts 0.3-4.1 s and 3.9-7.8 s of one recording, 1.8 s transition): both sides read one decoder at two positions, so blend frames take a side up to 49 source frames stale (frame 61: 80 for 129; 18 frames, output frames 61 to 110, 6 of them where the stale side is the weaker one, inside the 1.8 s transition at output frames 61 to 115). Same mechanism as editing/export/same-clip-transition/forward/ffmpeg; identical on Linux and Windows and for the 30 s full-tier fixture (same cuts)" +link = "" + +[[xfail]] +case = "seek/seek-native/keyframes60/avf" +class = "future" +platforms = ["macos"] +bound = { screen-frames = 1, camera-frames = 1, delta = 1, frame = [1, 1] } +reason = "Pool position capping: after a read, the reader's pool position is capped to the newest request (crates/rendering/src/decoder/avassetreader.rs, update_decoder_position(decoder_idx, capped)), so a reader that already decoded past a frame still looks usable for it (find_best_decoder_for_time_with_reuse_threshold: position <= request), is reused without a reset, and answers with the next sample it decodes. Here output frame 1 (0.017 s) shows source 1 for source 0. Minimal: CAP_VERIFY_SEEK_SCRIPT='seek-native@60:0,8.8,0.016667' cargo test -p cap-verify --test seek script -- --nocapture. Deterministic on every macOS runner" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +[[xfail]] +case = "seek/seek-native/keyframes60/ffmpeg" +class = "stale" +platforms = ["linux", "windows"] +bound = { screen-frames = 2, camera-frames = 2, delta = 61, frame = [124, 490] } +reason = "NEW: FFmpeg decoder. Off macOS this fixture is libx264 (the AVFoundation encoder is unavailable), so its keyframe set and shuffled request order differ from macOS. Request 124 shows source frame 0 for 61, the first-frame shortcut bug of seek/seek-gop/home-then-play30/ffmpeg; request 490 shows 243 for 244: the 2-frame backward lookback tolerance. Identical on Linux and Windows" +link = "" + +[[xfail]] +case = "editing/mapping/duration-covered" +class = "coverage" +bound = { count = [1, 4], failed-per-mille = [95, 98] } +reason = "NEW: a fullscreen text segment (a hold) left past the end of the clips (place a text card near the end, then trim or delete clips; clip edits do not ripple text segments) still adds its length to TimelineConfiguration::duration(), but get_frame_mapping returns None from the content end on, so the hold and the time before it have no source frame. Deterministic seeds: 97 of 1000 fast-tier and 1925 of 20000 full-tier sequences fail, the ones that leave a hold past the end (failed-per-mille 97.0 and 96.3). Minimal repro: Hold at 0.92 of a 7 s clip, then trim the clip end" +link = "" + +[[xfail]] +case = "editing/export/hold-past-end" +class = "frame-count" +bound = { count = 1, missing = 30 } +reason = "NEW: same as editing/mapping/duration-covered, end to end: a 1 s fullscreen text at 5.4 s and the clip trimmed to 3.2 s give a 4.2 s timeline, but the export stops at the first unmapped frame: 96 frames instead of 126, and the text card is never rendered" +link = "" + +[[xfail]] +case = "captions/app/segmentation/unicode" +class = "caption-segments" +bound = { count = [1, 3] } +reason = "NEW (minor): caption chunking decides that a boundary word is a weak one-letter word with String::len() (bytes), so a one-letter word outside ASCII such as the French preposition 'à' is not weak and a caption can end on it (apps/desktop-gpui/src/transcription.rs caption_boundary_word_is_weak). The case reports the first three failing token streams; an unexpected error from the product would add a problem beyond them. Repro: cargo test --manifest-path apps/desktop-gpui/Cargo.toml --bin cap-gpui segmentation_matches_reference_unicode -- --nocapture" +link = "" + +[[xfail]] +case = "captions/legacy-migration/ends-at-timeline-end" +class = "caption-migration" +bound = { count = 1 } +reason = "NEW (minor): loading a legacy (output-timed) project drops a caption that ends exactly at the timeline end: synchronize_captions maps each end through source_start + (end - start) / timescale, which rounds just below the timeline end after a clip transition (clips 41/15 s and 82/15 s, 0.2 s transition: 7.9999999 < 8.0), so the end does not map and the whole caption is discarded" +link = "" + +[[xfail]] +case = "editing/export/multi-segment/seed-7" +class = "stale" +bound = { screen-frames = 30, camera-frames = 14, delta = 29, frame = [129, 157] } +reason = "FFmpeg decoder (full tier): after the timeline played clip 0 from 1.68 s to its end and then transitions back into clip 0 at 0 s, the frames after the transition show source frame 0 instead of 20-29 (deterministic: 22 screen and 10 camera frames, output frames 135 to 157, with the dominant side of each blend checked). Same mechanism as the seek/*/storm*/ffmpeg entries: backward seeks near the start answer frame 0" +link = "" + +[[xfail]] +case = "editing/export/multi-segment/seed-7" +class = "unreadable" +bound = { screen-frames = 1, frame = [141, 141] } +reason = "FFmpeg decoder (full tier): one blend frame of the transition back into clip 0 mixes two stale frames and its marker is unreadable (same bug as the stale entry above)" +link = "" + +[[xfail]] +case = "editing/export/basic/seed-10" +class = "stale" +bound = { screen-frames = 3, camera-frames = 3, delta = 2, frame = [106, 109] } +reason = "NEW: FFmpeg decoder (full tier, Linux): seed 10 splits the clip at 0.34 s, plays the later part first and cross-fades back into the clip start, so the requests for source frames 1 and 2 right after the transition are answered with frame 0 (frames 108-109 on Linux, 107-109 with the macOS-encoded fixture). The first-frame shortcut in crates/rendering/src/decoder/ffmpeg.rs (requested_frame <= MAX_FRAME_LOOKBACK_TOLERANCE answered with first_ever_frame, no reset) serves frame 0 for any request up to frame 2 that is not cached; same mechanism as seek/seek-gop/home-then-play30/ffmpeg. Deterministic" +link = "" + +[[xfail]] +case = "editing/export/basic/seed-37" +class = "stale" +bound = { screen-frames = 1, camera-frames = 1, delta = 1, frame = [12, 12] } +reason = "NEW: FFmpeg decoder (full tier, Linux), the first-frame shortcut of editing/export/basic/seed-10: in a cross-fade back into the clip start the weaker side's request for source frame 1 is answered with frame 0 (output frame 12; found by the two-sided blend check, run 38851482675)" +link = "" + +[[xfail]] +case = "editing/export/same-clip-transition/forward/ffmpeg" +class = "stale" +bound = { screen-frames = 8, camera-frames = 8, delta = 55, frame = [37, 58] } +reason = "NEW: FFmpeg decoder, cross-fade between two parts of the same recording clip (0-2 s, then 3-5.5 s, 0.8 s transition): the export reads one decoder at two positions interleaved and every third blend frame takes the far side from the near position (frame 37: incoming should be source 90, shows 35; up to 55 source frames stale). 6 frames where the wrong side dominates; the two-sided blend check finds 2 more where it is the weaker side (weights 0.08 and 0.21, measured on Apple silicon). Deterministic; AVAssetReader passes this case (its 60 fps variant is the */basic/transition/*60*/avf xfail)" +link = "" + +[[xfail]] +case = "editing/export/basic/seed-5" +class = "stale" +bound = { screen-frames = 2, camera-frames = 2, delta = 2, frame = [52, 54] } +reason = "FFmpeg decoder, same-clip cross-fade (see editing/export/same-clip-transition/forward/ffmpeg): blend frame 53 one source frame stale" +link = "" + +[[xfail]] +case = "editing/export/basic/seed-23" +class = "stale" +bound = { screen-frames = 6, camera-frames = 6, delta = 70, frame = [7, 12] } +reason = "FFmpeg decoder (full tier), same-clip cross-fade (see editing/export/same-clip-transition/forward/ffmpeg): blend frames 7 to 11 up to 70 source frames stale" +link = "" + +[[xfail]] +case = "seek/seek-gop/home-then-play30/ffmpeg" +class = "stale" +bound = { screen-frames = 31, camera-frames = 31, delta = 89, frame = [60, 90] } +reason = "NEW: FFmpeg decoder freezes on the first frame after a jump to the start. Seek 6.1 s, seek 0, seek 2.0 s and play: 31 of 122 frames show source frame 0 (up to 89 frames stale, about 1 s frozen). A request at grid 0..2 is answered by the first-frame shortcut (ffmpeg.rs, requested_frame <= MAX_FRAME_LOOKBACK_TOLERANCE) without resetting the reader, which stays near 6 s, and last_sent_frame becomes frame 0; the next request is then not 'far forward' of frame 0 (< FRAME_CACHE_SIZE), so no reset happens, the reader vends frames past the request and the answer falls back to last_sent_frame (frame 0) until playback outruns FRAME_CACHE_SIZE. Main decoder on Linux and on Windows without a D3D11 video device; macOS AVAssetReader passes. Minimal: CAP_VERIFY_SEEK_SCRIPT='seek-gop@60:6.1,0.0167,2.0667' (third request shows frame 0 for 61)" +link = "" + +[[xfail]] +case = "playback/basic/cuts-speeds-reorder/realtime30-from25/avf" +class = "stale" +platforms = ["macos"] +flaky = true +bound = { screen-frames = 2, camera-frames = 2, delta = 1, frame = [126, 127] } +reason = "NEW: the real-time playback loop (cap_editor::Playback prefetch, AVAssetReader) shows source frame 58 instead of 59 for the last two output frames of the 0.5x segment that follows the 2x segment (4.200 s and 4.233 s, screen and camera); offline frame stepping and export of the same timeline show 59. 3 of 3 runs on Apple silicon, 2 of 4 full runs on runners, where the loop presents fewer frames, so it is timing dependent there" +link = "https://github.com/CapSoftware/Cap/pull/2638" + +# Real capture (crates/verify/tests/recording.rs, verify-recording workflow). +# Sync classes are per pair (sync-screen-mic is screen minus mic, negative = +# the screen plays early). + +[[xfail]] +case = "recording/instant/screen-mic-system" +class = "sync-screen-mic" +platforms = ["macos"] +flaky = true +bound = { count = 1, offset-ms = [-200, -10] } +reason = "NEW: the Instant screen track starts at its first encoded frame (the VideoToolbox encoder starts after the audio pipeline), so the screen plays early against mic and system audio by the encoder start delay: runners -131 to -24 ms (median -53 to -74 over 20 recordings, fix-video-timestamps probes), developer Mac 340 to 570 ms. The delay varies: in Verify recording 38818894595 (macos-26) screen-mic was -8.5 ms, inside the allowed -10..+75 ms, so the check passed; flaky until the screen track is anchored to the recording start. Bound: screen early by at most 200 ms on runners" +link = "" + +[[xfail]] +case = "recording/instant/long-holds" +class = "sync-screen-mic" +platforms = ["macos"] +flaky = true +bound = { count = 1, offset-ms = [-200, -10] } +reason = "NEW: same encoder start delay as recording/instant/screen-mic-system on a 30 s recording with screen holds: -61 to -68 ms on runners (developer Mac: screen 375 ms early); flaky for the same reason" +link = "" + +[[xfail]] +case = "recording/instant/late-mic" +class = "sync-screen-mic" +platforms = ["macos"] +bound = { count = 1, offset-ms = [900, 1100] } +reason = "NEW: a microphone whose first sample arrives 1.0 s after the recording starts becomes audio zero, so all audio plays early by its start-up delay minus the screen's (screen late against the mic by 962 to 1040 ms on runners, every run). Bound: the mic's 1.0 s delay within 100 ms" +link = "" + +[[xfail]] +case = "recording/instant/system-only" +class = "sync-screen-system" +platforms = ["macos"] +bound = { count = 1, offset-ms = [100, 250] } +reason = "NEW: Instant with system audio and no microphone: the first ScreenCaptureKit audio buffer becomes audio zero, so system audio plays 140 to 184 ms early (developer Mac, 8 of 8). Runs only where system audio is played and required (CAP_VERIFY_REQUIRE_SYSTEM_AUDIO); elsewhere it reports unsupported" +link = "" + +[[xfail]] +case = "recording/instant/system-only" +class = "unreadable" +platforms = ["macos"] +flaky = true +bound = { count = 1, system-markers = 1 } +reason = "NEW: Instant system-only: one system-audio marker inside the track does not decode (a discontinuity inside the marker) in 4 of 5 developer-Mac runs" +link = "" + +[[xfail]] +case = "recording/instant/gated" +class = "sync-screen-mic" +platforms = ["macos"] +flaky = true +bound = { count = 1, offset-ms = [-150, 150] } +reason = "NEW: the app's Instant start (start gate armed by the start cue): the screen-to-audio offset varies run to run: runners -55 to +16 ms (6 of 20 runs below the -10 ms floor), developer Mac +2 to +385 ms (3 of 13 above +75 ms); logs show 'Start gate admitted first audio frame at the arm point' followed by overlap trims of up to 124 ms. Bound: the trim size, 150 ms either way" +link = "" + +[[xfail]] +case = "recording/instant/gated" +class = "missing" +platforms = ["macos"] +flaky = true +bound = { count = 1, mic-markers = 1, mic-after-start-s = [0, 1] } +reason = "NEW: gated Instant with microphone and system audio loses an audio marker near the start in 2 of 5 developer-Mac runs (and 3 of 3 in an earlier batch); not seen on runners. Bound: one mic marker within the first second after the gate" +link = "" + +[[xfail]] +case = "recording/instant/gated" +class = "unreadable" +platforms = ["macos"] +flaky = true +bound = { count = 1, mic-markers = 1 } +reason = "NEW: see the gated Instant missing-marker entry: the damaged marker straddles the admission point (one marker)" +link = "" + +[[xfail]] +case = "recording/instant/gated" +class = "timestamps" +platforms = ["macos"] +flaky = true +bound = { count = 1, mic-spread-ms = [4, 130] } +reason = "NEW: gated Instant microphone timeline not continuous after the gate opens: marker offsets vary by 5 to 124 ms within one recording on runners (overlap trims after admission; 852 ms once on a developer Mac). Bound: the trim size" +link = "" + +# Crash recovery on the software (libx264) encoder path. Blacksmith macOS VMs +# have no VideoToolbox hardware H.264 encoder (VTCompressionSessionCreate with +# RequireHardwareAcceleratedVideoEncoder returns -12903, probe run +# 38847791287), so h264_videotoolbox fails to open and every recording falls +# back to libx264 (`get_codec_and_options`, crates/enc-ffmpeg/src/video/h264.rs), +# which real Macs also use when the hardware encoder is busy or for targets +# above its estimated capacity (for example 4K at 60 fps). Its GOP is a frame +# count: g = 2 s x nominal fps (keyint 120, keyint_min 61 at 60 fps), scenecut=0, +# and ScreenCaptureKit delivers a variable rate (about 33 fps for this +# stimulus), so the DASH muxer, which cuts only at keyframes, writes 3.68 s +# segments instead of 2 s, and the segment in progress is a 0-byte .tmp at the +# crash (run 38845236912: segment_001 120 frames 3.68 s, segment_002.m4s.tmp +# empty). Recovery keeps only the complete segments. With the hardware encoder +# (hosted runners, real Apple silicon Macs) segments stay near 2 s. + +[[xfail]] +case = "recording/studio-recovery/*" +class = "missing" +platforms = ["macos"] +flaky = true +hosts = ["blacksmith"] +bound = { count = 1, screen-lost-s = [3, 5.5] } +reason = "NEW: software (libx264) encoder path: keyframes every 120 frames regardless of the variable capture rate make 3.7 s segments, and the in-progress segment is empty at a crash, so recovery loses 4.5 to 4.8 s against the 3 s budget in 9 of 10 Blacksmith runs (flaky: the loss depends on where the crash falls inside the segment). Bound: at most one 3.7 s segment plus the 1 s since the last presented picture" +link = "https://github.com/CapSoftware/Cap/actions/runs/38845236912" + +# First picture after a static screen. During a scripted hold ScreenCaptureKit +# delivers only Idle status buffers; its first Complete frame after the hold +# arrives 70 to 82 ms after the new picture was presented (steady state 23 to +# 31 ms after each present), and it already shows the next picture. Frame-level +# logs (run 38745219281, macos-15: every SCK screen buffer with its +# SCStreamFrameInfoStatus, every Cap send, every mux call) show 0 Complete +# frames between the post-hold picture and the next one, and 1797 Complete +# frames = 1797 Cap sends = 1797 mux calls: the picture never reaches Cap's +# capture callback and Cap drops nothing. The stimulus presents through the +# standard Core Animation path, so a user's first change after a static screen +# is affected the same way: it is recorded ~50 to 80 ms late and anything +# shown for less than that is lost. Whether a given post-hold picture is lost +# depends on whether the 70 to 82 ms resume outlasts that picture, so the +# entries are bounded by the number of holds. Blacksmith VMs resume in time +# (these cases pass there), so the entries apply to hosted runners and +# developer Macs only (hosts). + +[[xfail]] +case = "recording/studio-screen/mp4-holds" +class = "missing-after-hold" +platforms = ["macos"] +hosts = ["github-hosted", "local"] +bound = { count = 3, screen-pictures = 3 } +reason = "NEW (ScreenCaptureKit idle resume, see above): the first 20 fps picture after each of the 3 holds is never delivered by ScreenCaptureKit (first Complete frame 80 to 82 ms after it was presented); runners 18 of 18 holds in 6 runs. Bound: one picture per hold" +link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281" + +[[xfail]] +case = "recording/instant/long-holds" +class = "missing-after-hold" +platforms = ["macos"] +flaky = true +hosts = ["github-hosted", "local"] +bound = { count = 2, screen-pictures = 2 } +reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after a hold is never delivered when the resume takes longer than its 67 ms (first Complete frame 70 to 74 ms after it was presented); runners: hold at 6 s 6 of 6 runs, hold at 18 s 4 of 6. Flaky: when ScreenCaptureKit happens to resume within the picture's 67 ms both pictures are caught (fix-video-timestamps track). Bound: one picture per hold (2 holds)" +link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281" + +[[xfail]] +case = "recording/studio-camera/long" +class = "missing-after-hold" +platforms = ["macos"] +flaky = true +hosts = ["github-hosted", "local"] +bound = { count = 1, screen-pictures = 1 } +reason = "NEW (ScreenCaptureKit idle resume, see above): the first 15 fps picture after the 3 s hold is missing when the resume takes longer than its 67 ms; runners 5 of 6 runs (flaky because the 70 to 82 ms resume straddles the 67 ms picture). Bound: the one hold's picture" +link = "https://github.com/CapSoftware/Cap/actions/runs/38745219281" +# Crash safety (crates/verify/tests/crash.rs). Measured on Blacksmith runners +# (run 38850661948: macos-15 libx264, ubuntu-2404, windows-2025) with the suite +# merged into main 3ec0c9497; each fix has its own proof branch +# ci/fixproof-crash- (runs 38850662667, 38850661576, 38850663436, +# 38850662102), which removes exactly its entries. Fixes: #2640 (bug A), +# #2641 (bug B1), #2643 (bug C), #2644 (bug E). + +[[xfail]] +case = "crash/recorder-kill/inproc/static-screen" +class = "loss-time" +bound = { count = 1, secs = [2.5, 20.5] } +reason = "NEW (bug B1): fragments are cut only at keyframes and every recording encoder's keyframe interval is a frame count (g = 2 s x nominal fps; libx264 keyint 60 here, VideoToolbox 12), so when the screen delivers 3 frames per second a fragment spans 20 s of capture and a crash after 11 s loses all of it (3 of 3 platforms; macOS runners use libx264). Bound: at most one 20 s fragment of capture" +link = "https://github.com/CapSoftware/Cap/pull/2641" + +[[xfail]] +case = "crash/recorder-kill/inproc/static-screen" +class = "missing" +bound = { count = 1 } +reason = "NEW (bug B1, see loss-time): no fragment of the static recording is complete when the recorder dies, so recovery has nothing (no frames)" +link = "https://github.com/CapSoftware/Cap/pull/2641" + +[[xfail]] +case = "crash/recorder-kill/oop/r*" +class = "loss-time" +platforms = ["macos"] +bound = { count = 1, secs = [2.5, 10] } +reason = "NEW (bug C): the out-of-process muxer writes packets into a 1 MiB BufWriter around cap-muxer's stdin and flushes only at Start and Finish (oop_muxer.rs MuxerSubprocess::write_packet), so encoded video stays in the recorder's memory: killed after 4 to 8 s, nothing reached cap-muxer (init.mp4 0 bytes, no segment). Bound: everything handed before the kill, at most 10 s" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "crash/recorder-kill/oop/r*" +class = "missing" +platforms = ["macos"] +bound = { count = 1 } +reason = "NEW (bug C, see loss-time): recovery finds no segment at all (no frames)" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "crash/recorder-kill/oop/r*" +class = "loss" +platforms = ["macos"] +bound = { count = 1, frames = [64, 300] } +reason = "NEW (bug C, see loss-time): every handed frame is lost, more than one fragment plus the frames in flight (at most the 10 s handed at 30 fps)" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "crash/muxer-kill/oop" +class = "error" +platforms = ["macos"] +bound = { count = 1 } +reason = "NEW (bug C): after cap-muxer is killed the recorder keeps buffering packets for the dead child (no flush, so no write error and no respawn), and stop() fails the display track: \"subprocess was reaped with unusable output\"" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "crash/recorder-kill/oop/static-screen" +class = "loss-time" +platforms = ["macos"] +bound = { count = 1, secs = [2.5, 20.5] } +reason = "NEW (bugs B1 and C): the static recording's fragments span 20 s of capture (frame-count keyframe interval, see the in-process entry) and the out-of-process muxer had not received any packet (1 MiB unflushed buffer)" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "crash/recorder-kill/oop/static-screen" +class = "missing" +platforms = ["macos"] +bound = { count = 1 } +reason = "NEW (bugs B1 and C, see loss-time): nothing recoverable (no frames)" +link = "https://github.com/CapSoftware/Cap/pull/2643" + +[[xfail]] +case = "export/instant/output/mp4/native" +class = "sync-system" +platforms = ["macos"] +flaky = true +bound = { count = 1, offset-ms = [-56, -25], mic-offset-ms = [5, 34], system-minus-mic-ms = [-61, -59] } +reason = "Instant mixer input timing produces a -60 ms system-minus-mic mode, within 1 ms. Only this inter-source signature with the microphone inside its unchanged 5 to 34 ms bound is excused. Coordinator decision 09:20, 2026-10-16." +link = "https://github.com/CapSoftware/Cap/tree/fix/instant-mixer-input-alignment" diff --git a/crates/verify/investigations/instant-system-audio.md b/crates/verify/investigations/instant-system-audio.md new file mode 100644 index 00000000000..199adc36128 --- /dev/null +++ b/crates/verify/investigations/instant-system-audio.md @@ -0,0 +1,18 @@ +# Instant system audio investigation + +Opened 2026-10-16 by the suite integration executor under coordinator decision C. +Status: product cause confirmed by the mixer arrival-schedule probe. The fix is tracked on [fix/instant-mixer-input-alignment](https://github.com/CapSoftware/Cap/tree/fix/instant-mixer-input-alignment). This track is separate from AVFoundation decoding. + +| Evidence | System offset | Mic offset | +| --- | --- | --- | +| Control 38858394655 | -46.1 ms | +13.9 ms | +| Candidate 38861420654 | -45.9 ms | +14.1 ms | +| Candidate repeat 3, 38861426173 | +7.8 ms | +14.5 ms | + +All use fresh fixtures. The coordinator also records #2635 evidence of the mixer placing system audio about 60 ms ahead of the microphone in about half of Instant recordings. The native case reads the pipeline's output.mp4 directly, independently of the product decoder. + +The normal offset bound remains 5 to 34 ms. Coordinator decisions at 09:20 and 10:05 replace the default Instant system-sync expectation with two bounded flaky signatures: system minus microphone at -60.0 or -6.7 ms, within 1 ms, with a measured microphone offset inside 5 to 34 ms. The -60 ms signature remains macOS-only; the -6.7 ms signature applies on all platforms. Missing measurements, other offsets and other classes remain failures. + +AudioMixer feeds each input to amix without per-input timing, making each first sample its own zero. Scheduler-dependent 53.3 ms startup silence partly masks the resulting 60 ms source displacement. The separate product track places each input relative to the mixer start and makes padding timestamp-based; its acceptance is outside this suite integration. + +Initial code trace: `crates/verify/src/pipeline.rs` builds the Instant audio pipeline with the same epoch anchoring as the product. `crates/recording/src/output_pipeline/core.rs` attaches sources to `AudioMixer` and then applies `AudioGapTracker` overlap handling to its output. Both the mixer source alignment and later overlap trimming remain candidates. No local capture or benchmark was started while PID 72210 was live. diff --git a/crates/verify/src/audio_code.rs b/crates/verify/src/audio_code.rs new file mode 100644 index 00000000000..7e498c0761e --- /dev/null +++ b/crates/verify/src/audio_code.rs @@ -0,0 +1,491 @@ +//! Audio markers. +//! +//! A marker is a 20 ms linear chirp (the sync symbol, located to a fraction +//! of a sample by cross-correlation) followed by 20 FSK bits of 5 ms each: a +//! 12-bit marker index and a CRC-8. Each source (microphone, system audio, +//! media track) has its own chirp direction/band and its own pair of bit +//! tones, so markers from different sources can be located independently in +//! a mix. The chirp start is the marker's reference position. + +use rustfft::{FftPlanner, num_complex::Complex32}; +use serde::{Deserialize, Serialize}; + +pub const CHIRP_SECS: f64 = 0.020; +pub const BIT_SECS: f64 = 0.005; +pub const INDEX_BITS: u32 = 12; +const CRC_BITS: u32 = 8; +const PAYLOAD_BITS: u32 = INDEX_BITS + CRC_BITS; +pub const MARKER_SECS: f64 = CHIRP_SECS + PAYLOAD_BITS as f64 * BIT_SECS; +pub const MAX_AUDIO_INDEX: u32 = (1 << INDEX_BITS) - 1; +pub const DEFAULT_AMPLITUDE: f32 = 0.3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AudioSource { + Mic, + System, + Media, +} + +struct SourceParams { + chirp_from: f64, + chirp_to: f64, + bit_zero: f64, + bit_one: f64, +} + +impl AudioSource { + pub const ALL: [AudioSource; 3] = [AudioSource::Mic, AudioSource::System, AudioSource::Media]; + + fn params(self) -> SourceParams { + match self { + AudioSource::Mic => SourceParams { + chirp_from: 600.0, + chirp_to: 3600.0, + bit_zero: 1000.0, + bit_one: 1600.0, + }, + AudioSource::System => SourceParams { + chirp_from: 3600.0, + chirp_to: 600.0, + bit_zero: 2200.0, + bit_one: 2800.0, + }, + AudioSource::Media => SourceParams { + chirp_from: 1200.0, + chirp_to: 4800.0, + bit_zero: 3400.0, + bit_one: 4000.0, + }, + } + } + + pub fn name(self) -> &'static str { + match self { + AudioSource::Mic => "mic", + AudioSource::System => "system", + AudioSource::Media => "media", + } + } + + /// Phase of this source's markers as a fraction of the schedule period, + /// so sources recorded together never overlap. + fn schedule_phase_secs(self) -> f64 { + match self { + AudioSource::Mic => 0.0, + AudioSource::System => 1.0 / 3.0, + AudioSource::Media => 2.0 / 3.0, + } + } +} + +fn crc8(value: u32) -> u8 { + let mut crc: u8 = 0; + for byte in value.to_le_bytes().iter().take(2) { + crc ^= byte; + for _ in 0..8 { + crc = if crc & 0x80 != 0 { + (crc << 1) ^ 0x07 + } else { + crc << 1 + }; + } + } + crc +} + +fn payload_bits(index: u32) -> u32 { + index | (u32::from(crc8(index)) << INDEX_BITS) +} + +fn hann(i: usize, n: usize) -> f64 { + if n <= 1 { + return 1.0; + } + 0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / (n - 1) as f64).cos() +} + +/// Short raised-cosine fade at both ends of a symbol to avoid clicks. +fn edge_taper(i: usize, n: usize, ramp: usize) -> f64 { + let ramp = ramp.min(n / 2).max(1); + if i < ramp { + 0.5 - 0.5 * (std::f64::consts::PI * i as f64 / ramp as f64).cos() + } else if i >= n - ramp { + let k = n - 1 - i; + 0.5 - 0.5 * (std::f64::consts::PI * k as f64 / ramp as f64).cos() + } else { + 1.0 + } +} + +fn chirp(source: AudioSource, sample_rate: u32) -> Vec { + let p = source.params(); + let n = (CHIRP_SECS * f64::from(sample_rate)).round() as usize; + let rate = f64::from(sample_rate); + let k = (p.chirp_to - p.chirp_from) / CHIRP_SECS; + let ramp = (0.002 * rate) as usize; + (0..n) + .map(|i| { + let t = i as f64 / rate; + let phase = 2.0 * std::f64::consts::PI * (p.chirp_from * t + 0.5 * k * t * t); + (phase.sin() * edge_taper(i, n, ramp)) as f32 + }) + .collect() +} + +/// Marker waveform at unit amplitude, starting with the chirp. +pub fn marker_waveform(source: AudioSource, index: u32, sample_rate: u32) -> Vec { + assert!( + index <= MAX_AUDIO_INDEX, + "audio marker index {index} too large" + ); + let p = source.params(); + let rate = f64::from(sample_rate); + let mut out = chirp(source, sample_rate); + let bits = payload_bits(index); + let chirp_len = out.len(); + let ramp = (0.0008 * rate) as usize; + for bit in 0..PAYLOAD_BITS { + let start = chirp_len + (f64::from(bit) * BIT_SECS * rate).round() as usize; + let end = chirp_len + (f64::from(bit + 1) * BIT_SECS * rate).round() as usize; + let freq = if bits >> bit & 1 == 1 { + p.bit_one + } else { + p.bit_zero + }; + out.resize(end, 0.0); + let n = end - start; + for (i, sample) in out[start..end].iter_mut().enumerate() { + let t = (start + i) as f64 / rate; + *sample = + ((2.0 * std::f64::consts::PI * freq * t).sin() * edge_taper(i, n, ramp)) as f32; + } + } + out +} + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct MarkerPlacement { + pub index: u32, + /// Chirp start, in seconds from the start of the source track. + pub time_secs: f64, +} + +/// Deterministic marker schedule: one marker per `period_secs`, offset by a +/// per-source phase and a small per-index jitter so a whole-period shift can +/// never alias to a zero error. +pub fn standard_schedule( + source: AudioSource, + start_secs: f64, + duration_secs: f64, + period_secs: f64, +) -> Vec { + let mut out = Vec::new(); + let mut index = 0u32; + loop { + let jitter = (u64::from(index).wrapping_mul(2_654_435_761) % 97) as f64 / 97.0 + * (period_secs / 3.0 - MARKER_SECS).max(0.0); + let t = start_secs + + source.schedule_phase_secs() * period_secs + + f64::from(index) * period_secs + + jitter; + if t + MARKER_SECS > start_secs + duration_secs || index > MAX_AUDIO_INDEX { + break; + } + out.push(MarkerPlacement { + index, + time_secs: t, + }); + index += 1; + } + out +} + +/// Mixes markers into `out` (mono) at `amplitude`. Placement times are in +/// seconds from sample 0 of `out`. +pub fn render_markers( + out: &mut [f32], + sample_rate: u32, + source: AudioSource, + placements: &[MarkerPlacement], + amplitude: f32, +) { + for placement in placements { + let start = (placement.time_secs * f64::from(sample_rate)).round() as isize; + let wave = marker_waveform(source, placement.index, sample_rate); + for (i, v) in wave.iter().enumerate() { + let at = start + i as isize; + if at >= 0 + && let Some(sample) = out.get_mut(at as usize) + { + *sample += v * amplitude; + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct LocatedMarker { + pub source: AudioSource, + /// `None` when the chirp was found but the payload did not decode. + pub index: Option, + /// Chirp start in seconds from the first analysed sample. + pub time_secs: f64, + /// Estimated amplitude relative to a unit-amplitude marker. + pub gain: f64, + /// Normalised correlation of the chirp (1.0 = exact shape). + pub correlation: f64, +} + +const MIN_CORRELATION: f64 = 0.55; + +/// Finds every marker of `source` in a mono signal. +pub fn locate_markers( + samples: &[f32], + sample_rate: u32, + source: AudioSource, +) -> Vec { + let template = chirp(source, sample_rate); + let m = template.len(); + if samples.len() < m || m == 0 { + return Vec::new(); + } + let corr = cross_correlate(samples, &template); + let template_energy: f64 = template.iter().map(|v| f64::from(*v).powi(2)).sum(); + let mut prefix = Vec::with_capacity(samples.len() + 1); + prefix.push(0.0f64); + for v in samples { + let last = *prefix.last().unwrap_or(&0.0); + prefix.push(last + f64::from(*v).powi(2)); + } + let lags = samples.len() - m + 1; + let norm = |lag: usize| -> f64 { + let energy = prefix[lag + m] - prefix[lag]; + if energy <= 1e-12 { + 0.0 + } else { + corr[lag] / (energy * template_energy).sqrt() + } + }; + let suppress = (MARKER_SECS * 0.8 * f64::from(sample_rate)) as usize; + let mut out = Vec::new(); + let mut lag = 0; + while lag < lags { + let c = norm(lag); + if c < MIN_CORRELATION { + lag += 1; + continue; + } + // Take the best peak in the neighbourhood. + let end = (lag + suppress).min(lags); + let (best, best_c) = (lag..end) + .map(|l| (l, norm(l))) + .fold((lag, c), |acc, x| if x.1 > acc.1 { x } else { acc }); + let refined = refine_peak(&corr, best); + let gain = corr[best] / template_energy; + let time_secs = refined / f64::from(sample_rate); + let index = decode_payload(samples, sample_rate, source, refined); + out.push(LocatedMarker { + source, + index, + time_secs, + gain, + correlation: best_c, + }); + lag = best + suppress; + } + out +} + +/// Identifies markers whose chirp was found but whose payload did not +/// decode, by their timing pattern: the schedule's per-index jitter makes +/// the gaps between markers unique, so one constant shift maps the chirps +/// onto the expected times. Returns `(index, chirp time)` pairs, or `None` +/// unless a single shift places at least three chirps within `tolerance` +/// of an expected marker and no other shift places as many. +pub fn identify_by_schedule( + expected: &[(u32, f64)], + chirps: &[f64], + tolerance: f64, +) -> Option> { + let pairs = |shift: f64| -> Vec<(u32, f64)> { + chirps + .iter() + .filter_map(|&c| { + expected + .iter() + .find(|&&(_, t)| (c - shift - t).abs() <= tolerance) + .map(|&(index, _)| (index, c)) + }) + .collect() + }; + let mut scored: Vec<(usize, f64)> = Vec::new(); + for &c in chirps { + for &(_, t) in expected { + let shift = c - t; + if scored.iter().any(|&(_, s)| (s - shift).abs() <= tolerance) { + continue; + } + scored.push((pairs(shift).len(), shift)); + } + } + scored.sort_by(|a, b| b.0.cmp(&a.0)); + let &(best, shift) = scored.first()?; + let runner_up = scored.get(1).map_or(0, |s| s.0); + (best >= 3 && runner_up < best).then(|| pairs(shift)) +} + +fn refine_peak(corr: &[f64], at: usize) -> f64 { + if at == 0 || at + 1 >= corr.len() { + return at as f64; + } + let (a, b, c) = (corr[at - 1], corr[at], corr[at + 1]); + let denom = a - 2.0 * b + c; + if denom.abs() < 1e-12 { + return at as f64; + } + at as f64 + (0.5 * (a - c) / denom).clamp(-0.5, 0.5) +} + +fn goertzel_power(samples: &[f32], start: usize, len: usize, rate: f64, freq: f64) -> f64 { + let w = 2.0 * std::f64::consts::PI * freq / rate; + let coeff = 2.0 * w.cos(); + let (mut s1, mut s2) = (0.0f64, 0.0f64); + for i in 0..len { + let x = f64::from(samples[start + i]) * hann(i, len); + let s = x + coeff * s1 - s2; + s2 = s1; + s1 = s; + } + s1 * s1 + s2 * s2 - coeff * s1 * s2 +} + +fn decode_payload( + samples: &[f32], + sample_rate: u32, + source: AudioSource, + chirp_start: f64, +) -> Option { + let p = source.params(); + let rate = f64::from(sample_rate); + let chirp_len = (CHIRP_SECS * rate).round(); + let mut bits = 0u32; + for bit in 0..PAYLOAD_BITS { + let start = chirp_start + chirp_len + (f64::from(bit) * BIT_SECS * rate); + let len = (BIT_SECS * rate).round() as usize; + // Analyse the middle 80% of the bit to skip the transitions. + let guard = len / 10; + let start = start.round() as isize + guard as isize; + let len = len - 2 * guard; + if start < 0 || start as usize + len > samples.len() { + return None; + } + let zero = goertzel_power(samples, start as usize, len, rate, p.bit_zero); + let one = goertzel_power(samples, start as usize, len, rate, p.bit_one); + let (hi, lo) = if one > zero { (one, zero) } else { (zero, one) }; + if hi < lo * 4.0 { + return None; + } + if one > zero { + bits |= 1 << bit; + } + } + let index = bits & MAX_AUDIO_INDEX; + (payload_bits(index) == bits).then_some(index) +} + +fn cross_correlate(signal: &[f32], template: &[f32]) -> Vec { + let n = signal.len() + template.len(); + let size = n.next_power_of_two(); + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(size); + let ifft = planner.plan_fft_inverse(size); + let mut a: Vec = signal + .iter() + .map(|&v| Complex32::new(v, 0.0)) + .chain(std::iter::repeat(Complex32::new(0.0, 0.0))) + .take(size) + .collect(); + let mut b: Vec = template + .iter() + .map(|&v| Complex32::new(v, 0.0)) + .chain(std::iter::repeat(Complex32::new(0.0, 0.0))) + .take(size) + .collect(); + fft.process(&mut a); + fft.process(&mut b); + for (x, y) in a.iter_mut().zip(&b) { + *x *= y.conj(); + } + ifft.process(&mut a); + let scale = 1.0 / size as f64; + a.iter() + .take(signal.len()) + .map(|c| f64::from(c.re) * scale) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn locates_markers_of_each_source_in_a_mix() { + let rate = 48_000; + let secs = 6.0; + let mut mix = vec![0.0f32; (secs * f64::from(rate)) as usize]; + let mut expected = Vec::new(); + for source in AudioSource::ALL { + let schedule = standard_schedule(source, 0.1, secs - 0.2, 0.5); + render_markers(&mut mix, rate, source, &schedule, 0.2); + expected.push((source, schedule)); + } + for (source, schedule) in expected { + let found = locate_markers(&mix, rate, source); + assert_eq!(found.len(), schedule.len(), "{source:?}"); + for (f, e) in found.iter().zip(&schedule) { + assert_eq!(f.index, Some(e.index)); + assert!((f.time_secs - e.time_secs).abs() < 1.0 / f64::from(rate)); + assert!((f.gain - 0.2).abs() < 0.03, "gain {}", f.gain); + } + } + } + + #[test] + fn schedule_pattern_identifies_undecodable_markers_without_aliasing() { + let schedule = standard_schedule(AudioSource::Mic, 0.1, 5.9, 0.5); + let expected: Vec<(u32, f64)> = schedule + .iter() + .skip(1) + .map(|p| (p.index, p.time_secs)) + .collect(); + for shift in [-0.519, 0.0, 0.5, 1.0, 0.0145] { + let chirps: Vec = schedule.iter().map(|p| p.time_secs + shift).collect(); + let found = identify_by_schedule(&expected, &chirps, 0.002).expect("identified"); + assert!( + found.len() >= expected.len() - 2, + "shift {shift}: {found:?}" + ); + for (index, time) in found { + let truth = schedule.iter().find(|p| p.index == index).unwrap(); + assert!( + (time - shift - truth.time_secs).abs() < 1e-9, + "shift {shift}" + ); + } + } + assert!(identify_by_schedule(&expected, &[1.0, 2.0], 0.002).is_none()); + } + + #[test] + fn crc_rejects_flipped_payloads() { + for index in [0, 1, 77, MAX_AUDIO_INDEX] { + let bits = payload_bits(index); + for b in 0..PAYLOAD_BITS { + let flipped = bits ^ (1 << b); + let i = flipped & MAX_AUDIO_INDEX; + assert_ne!(payload_bits(i), flipped); + } + } + } +} diff --git a/crates/verify/src/audio_fixture.rs b/crates/verify/src/audio_fixture.rs new file mode 100644 index 00000000000..99ebc1eaff2 --- /dev/null +++ b/crates/verify/src/audio_fixture.rs @@ -0,0 +1,123 @@ +//! Fixtures for the audio, captions and editing tracks: projects whose mic, +//! system audio and timeline music carry distinct audio markers, written by +//! [`crate::fixture::write_fixture`] plus a music file. + +use std::path::Path; + +use cap_project::{AudioTrackSegment, ProjectConfiguration}; + +use crate::{ + audio_code::{self, AudioSource}, + catalog::{SCREEN_H, SCREEN_W}, + fixture::{self, AudioTrackSpec, FixtureSpec, SegmentSpec, Timing, VideoTrackSpec}, + media::{self, AudioCodec}, + report::Tier, +}; + +/// Bump when the music writer changes so cached fixtures are rebuilt. +const AUDIO_GENERATOR_VERSION: u32 = 2; +/// Music file written next to `recording-meta.json`. +pub const MUSIC_FILE: &str = "music.ogg"; +pub const MUSIC_SECS: f64 = 12.0; + +fn audio(source: AudioSource, duration: f64, start_time: f64) -> AudioTrackSpec { + AudioTrackSpec { + source, + sample_rate: 48_000, + channels: if source == AudioSource::System { 2 } else { 1 }, + duration, + start_time, + } +} + +/// One clip: screen at 30 fps (starting 0.10 s into the recording), mic +/// starting 0.5 s in (the latest track, so screen and system audio get +/// alignment offsets), stereo system audio from 0, plus a stereo music file. +/// The mic start is one marker period so the sources' markers stay apart +/// once aligned. +pub fn mix(tier: Tier) -> FixtureSpec { + let duration = match tier { + Tier::Fast => 8.0, + Tier::Full => 30.0, + }; + let mut display = VideoTrackSpec::new(SCREEN_W, SCREEN_H, 30, Timing::Constant { duration }); + display.start_time = 0.10; + FixtureSpec { + name: format!("audio-mix-{duration:.0}s-a{AUDIO_GENERATOR_VERSION}"), + segments: vec![SegmentSpec { + display, + camera: None, + mic: Some(audio(AudioSource::Mic, duration - 0.5, 0.5)), + system_audio: Some(audio(AudioSource::System, duration, 0.0)), + }], + } +} + +/// Two clips (pause/resume): the second has a mic only. +pub fn multi(_tier: Tier) -> FixtureSpec { + let screen = + |duration| VideoTrackSpec::new(SCREEN_W, SCREEN_H, 30, Timing::Constant { duration }); + FixtureSpec { + name: format!("audio-multi-a{AUDIO_GENERATOR_VERSION}"), + segments: vec![ + SegmentSpec { + display: screen(5.0), + camera: None, + mic: Some(audio(AudioSource::Mic, 5.0, 0.0)), + system_audio: Some(audio(AudioSource::System, 5.0, 0.0)), + }, + SegmentSpec { + display: screen(4.0), + camera: None, + mic: Some(audio(AudioSource::Mic, 4.0, 0.0)), + system_audio: None, + }, + ], + } +} + +/// Writes the fixture tracks and the music file. +pub fn write(spec: &FixtureSpec, dir: &Path) -> anyhow::Result<()> { + fixture::write_fixture(spec, dir)?; + let rate = 48_000; + let mut mono = vec![0.0f32; (MUSIC_SECS * f64::from(rate)) as usize]; + let schedule = audio_code::standard_schedule(AudioSource::Media, 0.1, MUSIC_SECS - 0.1, 0.5); + audio_code::render_markers( + &mut mono, + rate, + AudioSource::Media, + &schedule, + audio_code::DEFAULT_AMPLITUDE, + ); + media::write_audio( + &dir.join(MUSIC_FILE), + &[mono.clone(), mono], + rate, + AudioCodec::Opus, + 128_000, + ) +} + +/// Adds a timeline music segment playing [`MUSIC_FILE`]. +pub fn add_music( + config: &mut ProjectConfiguration, + start: f64, + end: f64, + trim_start: f64, + volume_db: f32, +) { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.audio_segments.push(AudioTrackSegment { + start, + end, + track: 0, + path: MUSIC_FILE.into(), + name: None, + enabled: true, + trim_start, + volume_db, + fade_in: 0.0, + fade_out: 0.0, + duration: Some(MUSIC_SECS), + }); +} diff --git a/crates/verify/src/audio_model.rs b/crates/verify/src/audio_model.rs new file mode 100644 index 00000000000..5ac4f95a50f --- /dev/null +++ b/crates/verify/src/audio_model.rs @@ -0,0 +1,923 @@ +//! Reference model and oracle for audio: where every source marker must be +//! heard in an output (export or offline playback), at what level, and +//! nowhere else. +//! +//! Written from the project's documented rules, not by calling the +//! production mapping: +//! - Timeline segments play back to back; segment `k` covers +//! `(end - start) / timescale` output seconds and maps output time `u` in it +//! to recording time `start + u * timescale`. A transition into segment `k` +//! overlaps the two segments by its clamped duration. A cross-fade mixes +//! the outgoing clip at `cos(p * pi / 2)` and the incoming one at +//! `sin(p * pi / 2)` (equal power, `p` = progress through the overlap); a +//! fade through black at `max(1 - 2p, 0)` and `max(2p - 1, 0)`. Markers +//! wholly inside the overlap whose blend gain is at least +//! `BLEND_REQUIRED_GAIN` must be heard at that gain; the rest may be heard +//! at any level. +//! - A recording audio track sample at track time `t` plays at recording time +//! `t - offset`, where `offset` is the clip's stored offset for that track +//! (mic or system audio; auto-calculated offsets align every track to the +//! latest track start). +//! - Stale-startup repair: a track whose recording-meta gap summary reports +//! at least 3 startup overlap drops, at least 3 dropped frames and 100 to +//! 2000 ms of startup overlap trimmed plays that many ms earlier (its offset +//! is reduced by the trimmed time). +//! - Track gain is `10^(dB / 20)`; at or below -30 dB, or with the project +//! muted, the track is silent. Clip volume multiplies the recording mix. A +//! mono track is centre-panned at -3 dB (0.707 per channel). +//! - Retimed segments (timescale != 1) are silent in the default `mute` speed +//! audio mode; `matchSpeed` plays the source resampled (the marker is +//! compressed in time by the timescale and its pitch scaled with it); +//! `maintainPitch` time-stretches and is not marker-checked. +//! - Timeline music (audio segments) plays in output time from `trim_start`, +//! independent of recording edits, at `10^(volume_db / 20)` (silent at or +//! below -60 dB), with linear fades that are not level-checked. +//! - Studio Sound processes the microphone only: mic markers must still be +//! heard exactly once at the right time, but their level is not checked. + +use std::path::Path; + +use anyhow::{Context, bail}; +use cap_project::{ + ClipSpeedAudioMode, ClipTransitionType, ProjectConfiguration, RecordingMeta, + StudioRecordingMeta, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + audio_code::{self, AudioSource, LocatedMarker, MARKER_SECS}, + media, +}; + +/// Sample rate every analysis runs at (the editor's mixing rate). +pub const ANALYSIS_RATE: u32 = 48_000; +/// Largest allowed difference between a marker's heard and expected time. +pub const TIME_TOLERANCE_SECS: f64 = 0.005; +/// Largest allowed relative level error where the level is checked. +pub const GAIN_TOLERANCE: f64 = 0.25; +const MATCH_WINDOW_SECS: f64 = 0.040; +/// Detections quieter than this (-60 dB below a unit marker) are numerical +/// noise: the chirp correlation is normalised, so near-silent rounding +/// residue can correlate. Real markers are heard at 0.05 or more. +const MIN_HEARD_GAIN: f64 = 1e-3; +/// Chirps of one source correlate weakly (up to ~0.7) with another source's +/// chirp and bit tones. Undecodable detections below this are that +/// cross-talk; unprocessed markers correlate above 0.95 even after AAC. +/// Studio Sound bends the mic chirp (down to ~0.74) and can smear its bits, +/// so processed mic markers are matched by time instead. +const CROSSTALK_CORRELATION: f64 = 0.8; +const MIN_TRANSITION: f64 = 0.05; +const MONO_PAN: f64 = 0.707; +/// Blend gain from which a marker inside a transition must be heard (and its +/// level checked); quieter blended markers may be missed by the detector. +const BLEND_REQUIRED_GAIN: f64 = 0.75; +/// Offset of the point a marker's level is measured at (the chirp centre). +const CHIRP_CENTRE_SECS: f64 = 0.010; + +/// Gain a transition applies to the outgoing (`incoming == false`) or +/// incoming clip at progress `p`. +fn blend_gain(kind: ClipTransitionType, p: f64, incoming: bool) -> f64 { + let p = p.clamp(0.0, 1.0); + match (kind, incoming) { + (ClipTransitionType::CrossFade, false) => (p * std::f64::consts::FRAC_PI_2).cos(), + (ClipTransitionType::CrossFade, true) => (p * std::f64::consts::FRAC_PI_2).sin(), + (ClipTransitionType::FadeThroughBlack, false) => (1.0 - 2.0 * p).max(0.0), + (ClipTransitionType::FadeThroughBlack, true) => (2.0 * p - 1.0).max(0.0), + } +} + +/// One marker found in a source file. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)] +pub struct SourceMarker { + pub index: u32, + /// Chirp start in seconds of file time. + pub time_secs: f64, + /// Level relative to a unit marker, averaged over the file's channels. + pub gain: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TrackAudioTruth { + pub source: AudioSource, + pub channels: usize, + pub markers: Vec, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct SegmentAudioTruth { + pub mic: Option, + pub system: Option, +} + +/// Markers of every recording audio track (and timeline music files), read +/// back from the files. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AudioTruth { + pub segments: Vec, + /// Music file path (as written in the project) -> markers. + pub music: Vec<(String, TrackAudioTruth)>, +} + +/// Locates every marker of `source` in a file (all channels averaged). +pub fn read_track_truth(path: &Path, source: AudioSource) -> anyhow::Result { + let decoded = media::decode_audio(path, Some(ANALYSIS_RATE)) + .with_context(|| format!("decode {path:?}"))?; + let found = audio_code::locate_markers(&decoded.mono(), ANALYSIS_RATE, source); + let mut markers = Vec::new(); + for marker in found { + let Some(index) = marker.index else { + bail!( + "{path:?}: undecodable {} marker at {:.3}s", + source.name(), + marker.time_secs + ); + }; + markers.push(SourceMarker { + index, + time_secs: marker.time_secs + decoded.start_secs, + gain: marker.gain, + }); + } + if markers.is_empty() { + bail!("{path:?}: no {} markers", source.name()); + } + Ok(TrackAudioTruth { + source, + channels: decoded.channels.len(), + markers, + }) +} + +impl AudioTruth { + /// Reads every audio track of a multi-segment studio project plus the + /// music files referenced by `config`. Cached in the project directory. + pub fn read(project: &Path, config: &ProjectConfiguration) -> anyhow::Result { + let cache = project.join("verify-audio-truth.json"); + let mut truth: AudioTruth = std::fs::read_to_string(&cache) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) + .unwrap_or_default(); + let meta = RecordingMeta::load_for_project(project) + .map_err(|e| anyhow::anyhow!("load meta: {e:?}"))?; + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + bail!("not a multi-segment studio project"); + }; + let mut changed = false; + if truth.segments.len() != inner.segments.len() { + truth.segments = inner + .segments + .iter() + .map(|segment| -> anyhow::Result { + Ok(SegmentAudioTruth { + mic: segment + .mic + .as_ref() + .map(|m| { + read_track_truth(&project.join(m.path.as_str()), AudioSource::Mic) + }) + .transpose()?, + system: segment + .system_audio + .as_ref() + .map(|s| { + read_track_truth( + &project.join(s.path.as_str()), + AudioSource::System, + ) + }) + .transpose()?, + }) + }) + .collect::>()?; + changed = true; + } + if let Some(timeline) = &config.timeline { + for segment in &timeline.audio_segments { + if truth.music.iter().any(|(p, _)| *p == segment.path) { + continue; + } + let path = Path::new(&segment.path); + let path = if path.is_absolute() { + path.to_path_buf() + } else { + project.join(path) + }; + truth.music.push(( + segment.path.clone(), + read_track_truth(&path, AudioSource::Media)?, + )); + changed = true; + } + } + if changed { + let _ = std::fs::write(&cache, serde_json::to_string(&truth)?); + } + Ok(truth) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Presence { + /// Must be heard exactly once. + Required, + /// May be heard (cut by an edit boundary, inside a transition or fade); + /// if heard it must be at the right time. + Optional, +} + +/// Where one source marker must be heard in the output. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ExpectedMarker { + pub source: AudioSource, + pub index: u32, + /// Output time of the chirp start. + pub time_secs: f64, + /// Expected level of the output's channel average, when checked. + pub gain: Option, + pub presence: Presence, + /// Time compression of the marker (the segment timescale for + /// `matchSpeed`, else 1). + pub rate: f64, + /// Recording clip and recording time the marker belongs to (`None` for + /// timeline music), for A/V checks and reports. + pub recording: Option<(usize, f64)>, + /// Whether the marker's payload survives processing. Studio Sound can + /// smear the FSK bits; such a marker is matched by its chirp time. + pub payload: bool, + /// Whether the marker sounds inside a clip transition, where the video + /// is a blend of two frames and has no single frame to time it against. + #[serde(default)] + pub blended: bool, +} + +/// An output window in which detections of `source` are not judged. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct UncheckedWindow { + pub source: AudioSource, + pub start: f64, + pub end: f64, + pub reason: String, +} + +/// A window of output time with time-compressed markers (`matchSpeed`). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RateWindow { + pub start: f64, + pub end: f64, + pub rate: f64, +} + +#[derive(Clone, Debug)] +struct ModelSegment { + clip: usize, + start: f64, + end: f64, + timescale: f64, + mode: ClipSpeedAudioMode, + volume: f64, + transition_in: f64, + transition_kind: ClipTransitionType, +} + +impl ModelSegment { + fn duration(&self) -> f64 { + (self.end - self.start) / self.timescale + } +} + +/// Clip track offsets as stored in the project. +#[derive(Clone, Copy, Debug)] +struct ClipAudioOffsets { + mic: f64, + system: f64, +} + +pub struct AudioModel { + pub expected: Vec, + pub unchecked: Vec, + pub rate_windows: Vec, + pub duration: f64, +} + +fn db_gain(db: f32) -> Option { + (db > -30.0).then(|| 10f64.powf(f64::from(db) / 20.0)) +} + +impl AudioModel { + pub fn new( + meta: &RecordingMeta, + project: &ProjectConfiguration, + truth: &AudioTruth, + ) -> anyhow::Result { + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + bail!("the audio model covers multi-segment studio projects"); + }; + let timeline = project + .timeline + .as_ref() + .ok_or_else(|| anyhow::anyhow!("project has no timeline"))?; + let repair = |audio: Option<&cap_project::AudioMeta>| { + audio + .and_then(|a| a.gap_summary.as_ref()) + .filter(|s| { + s.startup_overlap_drops >= 3 + && s.overlap_dropped_frames >= 3 + && (100..=2000).contains(&s.startup_overlap_trimmed_ms) + }) + .map_or(0.0, |s| -f64::from(s.startup_overlap_trimmed_ms) / 1000.0) + }; + let offsets: Vec = (0..inner.segments.len()) + .map(|i| { + let clip = project.clips.iter().find(|c| c.index as usize == i); + let segment = &inner.segments[i]; + ClipAudioOffsets { + mic: clip.map_or(0.0, |c| f64::from(c.offsets.mic)) + + repair(segment.mic.as_ref()), + system: clip.map_or(0.0, |c| f64::from(c.offsets.system_audio)) + + repair(segment.system_audio.as_ref()), + } + }) + .collect(); + let mut segments: Vec = timeline + .segments + .iter() + .map(|s| ModelSegment { + clip: s.recording_clip as usize, + start: s.start, + end: s.end, + timescale: s.timescale, + mode: s.speed_audio_mode.unwrap_or_default(), + volume: s + .volume + .filter(|v| v.is_finite()) + .unwrap_or(1.0) + .clamp(0.0, 2.0), + transition_in: 0.0, + transition_kind: ClipTransitionType::CrossFade, + }) + .collect(); + for transition in &timeline.transitions { + let k = transition.segment_index as usize; + if k == 0 || k >= segments.len() { + continue; + } + let max = segments[k - 1].duration().min(segments[k].duration()) / 2.0; + if transition.duration.is_finite() && transition.duration > 0.0 && max >= MIN_TRANSITION + { + segments[k].transition_in = transition.duration.clamp(MIN_TRANSITION, max); + segments[k].transition_kind = transition.kind; + } + } + let mut output_starts = Vec::with_capacity(segments.len()); + let mut at = 0.0; + for (k, segment) in segments.iter().enumerate() { + output_starts.push(at); + let next_in = segments.get(k + 1).map_or(0.0, |n| n.transition_in); + at += segment.duration() - next_in; + } + let duration = at; + let audio = &project.audio; + let mut expected = Vec::new(); + let mut unchecked = Vec::new(); + let mut rate_windows = Vec::new(); + // Whether each recording marker in `expected` sounds inside a transition. + let mut in_blend: Vec = Vec::new(); + + for (k, segment) in segments.iter().enumerate() { + let out_start = output_starts[k]; + let out_end = out_start + segment.duration(); + // Output window shared with a neighbour's transition. + let blend_until = out_start + segment.transition_in; + let next_in = segments.get(k + 1).map_or(0.0, |n| n.transition_in); + let next_kind = segments + .get(k + 1) + .map_or(ClipTransitionType::CrossFade, |n| n.transition_kind); + let blend_from = out_end - next_in; + let Some(clip_truth) = truth.segments.get(segment.clip) else { + continue; + }; + let retimed = (segment.timescale - 1.0).abs() > f64::EPSILON; + if retimed && segment.mode == ClipSpeedAudioMode::MaintainPitch { + for source in [AudioSource::Mic, AudioSource::System] { + unchecked.push(UncheckedWindow { + source, + start: out_start, + end: out_end, + reason: "maintainPitch time stretch".into(), + }); + } + continue; + } + let silent = audio.mute || (retimed && segment.mode == ClipSpeedAudioMode::Mute); + if retimed && !silent { + rate_windows.push(RateWindow { + start: out_start, + end: out_end, + rate: segment.timescale, + }); + } + let tracks = [ + ( + clip_truth.mic.as_ref(), + offsets[segment.clip].mic, + db_gain(audio.mic_volume_db), + audio.improve, + ), + ( + clip_truth.system.as_ref(), + offsets[segment.clip].system, + db_gain(audio.system_volume_db), + false, + ), + ]; + for (track, offset, gain, processed) in tracks { + let Some(track) = track else { continue }; + let Some(gain) = gain.filter(|_| !silent) else { + continue; + }; + let pan = if track.channels == 1 { MONO_PAN } else { 1.0 }; + let marker_len = MARKER_SECS * segment.timescale; + for marker in &track.markers { + let recording_time = marker.time_secs - offset; + if recording_time + marker_len <= segment.start || recording_time >= segment.end + { + continue; + } + let time = out_start + (recording_time - segment.start) / segment.timescale; + let whole = recording_time >= segment.start + && recording_time + marker_len <= segment.end; + let marker_end = time + MARKER_SECS / segment.timescale; + let blended = time < blend_until || marker_end > blend_from; + // Blend gain when the whole marker lies in one overlap. + let blend = if !blended { + Some(1.0) + } else if marker_end <= blend_until && time >= out_start { + let p = (time + CHIRP_CENTRE_SECS - out_start) / segment.transition_in; + Some(blend_gain(segment.transition_kind, p, true)) + } else if time >= blend_from && marker_end <= out_end && next_in > 0.0 { + let p = (time + CHIRP_CENTRE_SECS - blend_from) / next_in; + Some(blend_gain(next_kind, p, false)) + } else { + None + }; + let level_known = !processed && segment.volume > 0.0; + let audible = blend.filter(|&b| !blended || b >= BLEND_REQUIRED_GAIN); + let presence = if whole && audible.is_some() && segment.volume > 0.0 { + Presence::Required + } else { + Presence::Optional + }; + if segment.volume == 0.0 { + continue; + } + in_blend.push(blended); + expected.push(ExpectedMarker { + source: track.source, + index: marker.index, + time_secs: time, + gain: audible + .filter(|_| level_known) + .map(|b| marker.gain * pan * gain * segment.volume * b), + presence, + rate: if retimed { segment.timescale } else { 1.0 }, + recording: Some((segment.clip, recording_time)), + payload: !processed, + blended, + }); + } + } + } + + // Two markers of one source sounding at once (the two sides of a + // transition) disturb each other's detection: neither is required. + let overlapping: Vec = expected + .iter() + .zip(&in_blend) + .map(|(e, &blended)| { + blended + && expected.iter().any(|o: &ExpectedMarker| { + !std::ptr::eq(e, o) + && o.source == e.source + && (o.time_secs - e.time_secs).abs() < MARKER_SECS + 0.005 + }) + }) + .collect(); + for (e, overlaps) in expected.iter_mut().zip(overlapping) { + if overlaps { + e.presence = Presence::Optional; + e.gain = None; + } + } + + for music in &timeline.audio_segments { + let gain = if music.volume_db <= -60.0 { + continue; + } else { + 10f64.powf(f64::from(music.volume_db) / 20.0) + }; + if !music.enabled || music.end <= music.start { + continue; + } + let Some((_, track)) = truth.music.iter().find(|(p, _)| *p == music.path) else { + bail!("no truth for music file {}", music.path); + }; + let pan = if track.channels == 1 { MONO_PAN } else { 1.0 }; + let trim = music.trim_start.max(0.0); + let seg_end = music.end.min(duration); + for marker in &track.markers { + let time = music.start + marker.time_secs - trim; + if time + MARKER_SECS <= music.start || time >= seg_end { + continue; + } + let whole = time >= music.start && time + MARKER_SECS <= seg_end; + let faded = time < music.start + music.fade_in.max(0.0) + || time + MARKER_SECS > music.end - music.fade_out.max(0.0); + expected.push(ExpectedMarker { + source: AudioSource::Media, + index: marker.index, + time_secs: time, + gain: (!faded).then_some(marker.gain * pan * gain), + presence: if whole { + Presence::Required + } else { + Presence::Optional + }, + rate: 1.0, + recording: None, + payload: true, + blended: false, + }); + } + } + expected.sort_by(|a, b| a.time_secs.total_cmp(&b.time_secs)); + Ok(Self { + expected, + unchecked, + rate_windows, + duration, + }) + } + + /// Expectations for an output that starts at timeline time `from` and + /// covers `length` seconds; times become relative to the output start. + pub fn window(&self, from: f64, length: f64) -> Self { + let to = from + length; + let expected = self + .expected + .iter() + .filter(|e| e.time_secs + MARKER_SECS / e.rate > from && e.time_secs < to) + .map(|e| { + let whole = e.time_secs >= from && e.time_secs + MARKER_SECS / e.rate <= to; + ExpectedMarker { + time_secs: e.time_secs - from, + presence: if whole { + e.presence + } else { + Presence::Optional + }, + ..e.clone() + } + }) + .collect(); + Self { + expected, + unchecked: self + .unchecked + .iter() + .map(|w| UncheckedWindow { + start: w.start - from, + end: w.end - from, + ..w.clone() + }) + .collect(), + rate_windows: self + .rate_windows + .iter() + .map(|w| RateWindow { + start: w.start - from, + end: w.end - from, + rate: w.rate, + }) + .collect(), + duration: length, + } + } + + /// Appends `other` (a later output window) shifted by `at` seconds. + pub fn append(&mut self, other: Self, at: f64) { + self.expected + .extend(other.expected.into_iter().map(|e| ExpectedMarker { + time_secs: e.time_secs + at, + ..e + })); + self.unchecked + .extend(other.unchecked.into_iter().map(|w| UncheckedWindow { + start: w.start + at, + end: w.end + at, + ..w + })); + self.rate_windows + .extend(other.rate_windows.into_iter().map(|w| RateWindow { + start: w.start + at, + end: w.end + at, + rate: w.rate, + })); + self.duration = self.duration.max(at + other.duration); + } + + pub fn required(&self) -> usize { + self.expected + .iter() + .filter(|e| e.presence == Presence::Required) + .count() + } +} + +/// A located marker with its time relative to the analysed output. +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct HeardMarker { + pub source: AudioSource, + pub index: Option, + pub time_secs: f64, + pub gain: f64, + pub rate: f64, + pub correlation: f64, +} + +/// Finds every marker of every source in a mono output (at +/// [`ANALYSIS_RATE`]), including time-compressed markers inside +/// `rate_windows`. `start_secs` is the output time of sample 0. +pub fn hear(mono: &[f32], start_secs: f64, rate_windows: &[RateWindow]) -> Vec { + let mut heard = Vec::new(); + for source in AudioSource::ALL { + for m in audio_code::locate_markers(mono, ANALYSIS_RATE, source) { + if m.gain.abs() < MIN_HEARD_GAIN { + continue; + } + let time = m.time_secs + start_secs; + if rate_windows + .iter() + .any(|w| time >= w.start - 0.001 && time < w.end) + { + continue; + } + heard.push(heard_marker(m, time, 1.0)); + } + for window in rate_windows { + let scaled_rate = (f64::from(ANALYSIS_RATE) / window.rate).round() as u32; + let from = (((window.start - start_secs) * f64::from(ANALYSIS_RATE)) + .floor() + .max(0.0)) as usize; + let to = ((((window.end - start_secs) * f64::from(ANALYSIS_RATE)).ceil()) as usize + + (MARKER_SECS * f64::from(ANALYSIS_RATE)) as usize) + .min(mono.len()); + if from >= to { + continue; + } + for m in audio_code::locate_markers(&mono[from..to], scaled_rate, source) { + if m.gain.abs() < MIN_HEARD_GAIN { + continue; + } + let time = start_secs + + from as f64 / f64::from(ANALYSIS_RATE) + + m.time_secs * f64::from(scaled_rate) / f64::from(ANALYSIS_RATE); + if time >= window.start - 0.001 && time < window.end { + heard.push(heard_marker(m, time, window.rate)); + } + } + } + } + heard.sort_by(|a, b| a.time_secs.total_cmp(&b.time_secs)); + heard +} + +fn heard_marker(m: LocatedMarker, time_secs: f64, rate: f64) -> HeardMarker { + HeardMarker { + source: m.source, + index: m.index, + time_secs, + gain: m.gain, + rate, + correlation: m.correlation, + } +} + +/// The verdict of one output against the model. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AudioReport { + pub required: usize, + pub matched: usize, + pub missing: Vec, + pub doubled: Vec, + pub unexpected: Vec, + pub mistimed: Vec, + pub level: Vec, + pub damaged: Vec, + pub max_time_error_ms: f64, + pub max_gain_error: f64, + pub min_correlation: f64, + /// (expected, heard) pairs, for A/V checks. + #[serde(skip)] + pub pairs: Vec<(ExpectedMarker, HeardMarker)>, +} + +impl AudioReport { + pub fn passed(&self) -> bool { + self.missing.is_empty() + && self.doubled.is_empty() + && self.unexpected.is_empty() + && self.mistimed.is_empty() + && self.level.is_empty() + && self.damaged.is_empty() + } + + /// Failure classes for `expectations.toml` (empty when passed). + pub fn classes(&self) -> Vec<&'static str> { + [ + ("audio-missing", &self.missing), + ("audio-doubled", &self.doubled), + ("audio-unexpected", &self.unexpected), + ("audio-timing", &self.mistimed), + ("audio-level", &self.level), + ("audio-undecodable", &self.damaged), + ] + .into_iter() + .filter(|(_, list)| !list.is_empty()) + .map(|(class, _)| class) + .collect() + } + + pub fn summary(&self) -> String { + let mut parts = vec![format!( + "audio {}/{} required markers matched, max time error {:.3} ms, max level error {:.0}%, min correlation {:.2}", + self.matched, + self.required, + self.max_time_error_ms, + self.max_gain_error * 100.0, + self.min_correlation + )]; + for (label, list) in [ + ("missing", &self.missing), + ("doubled", &self.doubled), + ("unexpected", &self.unexpected), + ("mistimed", &self.mistimed), + ("wrong level", &self.level), + ("undecodable", &self.damaged), + ] { + if !list.is_empty() { + parts.push(format!( + "{} {label} (first: {})", + list.len(), + list.iter().take(4).cloned().collect::>().join("; ") + )); + } + } + parts.join(", ") + } +} + +fn describe(e: &ExpectedMarker) -> String { + match e.recording { + Some((clip, t)) => format!( + "{} #{} at {:.4}s (clip {clip} {:.4}s)", + e.source.name(), + e.index, + e.time_secs, + t + ), + None => format!("{} #{} at {:.4}s", e.source.name(), e.index, e.time_secs), + } +} + +/// Matches what was heard against the model. +pub fn check(model: &AudioModel, heard: &[HeardMarker]) -> AudioReport { + let mut report = AudioReport { + required: model.required(), + min_correlation: 1.0, + ..Default::default() + }; + let mut hits = vec![0usize; model.expected.len()]; + for h in heard { + let unchecked = model.unchecked.iter().any(|w| { + w.source == h.source + && h.time_secs >= w.start - MATCH_WINDOW_SECS + && h.time_secs < w.end + }); + if unchecked || h.time_secs < -MATCH_WINDOW_SECS || h.time_secs > model.duration + 0.5 { + continue; + } + let candidate = model + .expected + .iter() + .enumerate() + .filter(|(_, e)| { + e.source == h.source && (e.time_secs - h.time_secs).abs() <= MATCH_WINDOW_SECS + }) + .min_by(|a, b| { + let key = |e: &ExpectedMarker| { + (h.index != Some(e.index), (e.time_secs - h.time_secs).abs()) + }; + let (ka, kb) = (key(a.1), key(b.1)); + ka.0.cmp(&kb.0).then(ka.1.total_cmp(&kb.1)) + }); + let time_only = h.index.is_none() && candidate.is_some_and(|(_, e)| !e.payload); + if h.index.is_none() && !time_only && h.correlation < CROSSTALK_CORRELATION { + continue; + } + let Some((i, e)) = candidate else { + report.unexpected.push(format!( + "{} #{} at {:.4}s (correlation {:.2})", + h.source.name(), + h.index.map_or("?".into(), |i| i.to_string()), + h.time_secs, + h.correlation + )); + continue; + }; + match h.index { + None if e.presence == Presence::Optional => continue, + None if !e.payload => {} + None => { + report + .damaged + .push(format!("{} heard at {:.4}s", describe(e), h.time_secs)); + hits[i] += 1; + continue; + } + Some(index) if index != e.index => { + report.unexpected.push(format!( + "{} #{index} at {:.4}s (near {})", + h.source.name(), + h.time_secs, + describe(e) + )); + continue; + } + Some(_) => {} + } + hits[i] += 1; + report.min_correlation = report.min_correlation.min(h.correlation); + let error = h.time_secs - e.time_secs; + report.max_time_error_ms = report.max_time_error_ms.max(error.abs() * 1000.0); + if error.abs() > TIME_TOLERANCE_SECS { + report.mistimed.push(format!( + "{} heard at {:.4}s ({:+.2} ms)", + describe(e), + h.time_secs, + error * 1000.0 + )); + } + if let Some(gain) = e.gain { + let rel = h.gain / gain - 1.0; + report.max_gain_error = report.max_gain_error.max(rel.abs()); + if rel.abs() > GAIN_TOLERANCE { + report.level.push(format!( + "{} level {:.3}, expected {:.3}", + describe(e), + h.gain, + gain + )); + } + } + report.pairs.push((e.clone(), *h)); + } + for (e, hits) in model.expected.iter().zip(hits) { + match (e.presence, hits) { + (Presence::Required, 0) => report.missing.push(describe(e)), + (Presence::Required, 1) => report.matched += 1, + (_, 0 | 1) => {} + (_, n) => report + .doubled + .push(format!("{} heard {n} times", describe(e))), + } + } + report +} + +/// Splits interleaved stereo into its channel average. +pub fn interleaved_mono(samples: &[f32], channels: usize) -> Vec { + samples + .chunks_exact(channels.max(1)) + .map(|frame| frame.iter().sum::() / frame.len() as f32) + .collect() +} + +/// Peak absolute difference between two PCM buffers in dBFS (`None` when +/// identical), and the first differing sample. +pub fn pcm_difference(a: &[f32], b: &[f32]) -> Option<(f64, usize)> { + let mut peak = 0.0f32; + let mut first = None; + for (i, (x, y)) in a.iter().zip(b).enumerate() { + let d = (x - y).abs(); + if d > 0.0 && first.is_none() { + first = Some(i); + } + peak = peak.max(d); + } + if a.len() != b.len() && first.is_none() { + first = Some(a.len().min(b.len())); + peak = peak.max(1.0); + } + first.map(|i| (20.0 * f64::from(peak.max(1e-12)).log10(), i)) +} diff --git a/crates/verify/src/catalog.rs b/crates/verify/src/catalog.rs new file mode 100644 index 00000000000..99a0e597d25 --- /dev/null +++ b/crates/verify/src/catalog.rs @@ -0,0 +1,336 @@ +//! The fixture catalogue. Sizes and lengths depend on the tier: the fast +//! tier keeps every fixture a few seconds long at 320x180 or below; the full +//! tier lengthens the cases whose bugs need time to build up. + +use crate::{ + audio_code::AudioSource, + fixture::{AudioTrackSpec, FixtureSpec, SegmentSpec, Timing, VideoTrackSpec}, + report::Tier, +}; + +pub const SCREEN_W: u32 = 320; +pub const SCREEN_H: u32 = 180; +pub const CAMERA_W: u32 = 160; +pub const CAMERA_H: u32 = 120; + +fn screen(fps: u32, timing: Timing) -> VideoTrackSpec { + VideoTrackSpec::new(SCREEN_W, SCREEN_H, fps, timing) +} + +fn camera(timing: Timing) -> VideoTrackSpec { + VideoTrackSpec::new(CAMERA_W, CAMERA_H, 30, timing) +} + +fn audio(source: AudioSource, duration: f64) -> AudioTrackSpec { + AudioTrackSpec { + source, + sample_rate: 48_000, + channels: if source == AudioSource::System { 2 } else { 1 }, + duration, + start_time: 0.0, + } +} + +/// Screen + camera + mic + system audio, constant 30 fps. +pub fn basic(_tier: Tier) -> FixtureSpec { + let duration = 6.0; + FixtureSpec { + name: "basic".into(), + segments: vec![SegmentSpec { + display: screen(30, Timing::Constant { duration }), + camera: Some(camera(Timing::Constant { duration })), + mic: Some(audio(AudioSource::Mic, duration)), + system_audio: Some(audio(AudioSource::System, duration)), + }], + } +} + +/// VFR screen with static holds longer than the decoders' frame cache +/// (2.6 s and 4.2 s at 30 fps > 90 frames) plus a narrow 0.4 s hold, and a +/// 60 fps nominal rate so holds are wide on the request grid. +pub fn vfr_holds(_tier: Tier) -> FixtureSpec { + let duration = 14.0; + FixtureSpec { + name: "vfr-holds".into(), + segments: vec![SegmentSpec { + display: screen( + 60, + Timing::Holds { + duration, + holds: vec![(1.0, 2.6), (5.0, 0.4), (7.0, 4.2)], + }, + ), + camera: Some(camera(Timing::Constant { duration })), + mic: None, + system_audio: None, + }], + } +} + +/// Camera nominally 30 fps delivering ~12 fps with jitter, as real webcams +/// do in low light. Long enough for the decoder's gap tracking to fill up. +pub fn jitter_camera(tier: Tier) -> FixtureSpec { + let duration = match tier { + Tier::Fast => 40.0, + Tier::Full => 120.0, + }; + FixtureSpec { + name: format!("jitter-camera-{duration:.0}s"), + segments: vec![SegmentSpec { + display: screen(30, Timing::Constant { duration }), + camera: Some(camera(Timing::Jittery { + duration, + seed: 0x5EED_0001, + holds: Vec::new(), + })), + mic: None, + system_audio: None, + }], + } +} + +/// Screen at 30 fps with keyframes every 25 frames (0.833 s: never on a +/// 10 ms grid), for seek-to-keyframe exactness. +pub fn seek_gop(_tier: Tier) -> FixtureSpec { + let duration = 12.0; + let mut display = screen(30, Timing::Constant { duration }); + display.gop = 25; + let mut cam = camera(Timing::Constant { duration }); + cam.gop = 25; + FixtureSpec { + name: "seek-gop".into(), + segments: vec![SegmentSpec { + display, + camera: Some(cam), + mic: None, + system_audio: None, + }], + } +} + +/// The seek fixture written by the product's macOS encoder: AVAssetWriter +/// with VideoToolbox, its own GOP structure and a 1/600 time base. +/// Keyframes at k/30 s are off the 10 ms grid for most k. Identical to +/// `seek_gop` elsewhere. +pub fn seek_gop_native(_tier: Tier) -> FixtureSpec { + let duration = 12.0; + let native = |mut track: VideoTrackSpec| { + track.encoder = crate::fixture::TrackEncoder::AvFoundation; + track + }; + FixtureSpec { + name: "seek-native".into(), + segments: vec![SegmentSpec { + display: native(screen(30, Timing::Constant { duration })), + camera: Some(native(camera(Timing::Constant { duration }))), + mic: None, + system_audio: None, + }], + } +} + +/// Camera that starts 1.5 s after the screen (it is the latest track). +pub fn late_camera(_tier: Tier) -> FixtureSpec { + let duration = 6.0; + let mut cam = camera(Timing::Constant { duration: 4.5 }); + cam.start_time = 1.5; + FixtureSpec { + name: "late-camera".into(), + segments: vec![SegmentSpec { + display: screen(30, Timing::Constant { duration }), + camera: Some(cam), + mic: None, + system_audio: None, + }], + } +} + +/// Camera that starts before a slow microphone: the mic is the latest track, +/// so the camera needs a positive alignment offset. +pub fn early_camera(_tier: Tier) -> FixtureSpec { + let duration = 6.0; + let mut display = screen(30, Timing::Constant { duration }); + display.start_time = 0.2; + let mut cam = camera(Timing::Constant { duration }); + cam.start_time = 0.0; + let mut mic = audio(AudioSource::Mic, duration - 1.0); + mic.start_time = 1.0; + FixtureSpec { + name: "early-camera".into(), + segments: vec![SegmentSpec { + display, + camera: Some(cam), + mic: Some(mic), + system_audio: None, + }], + } +} + +/// Camera file whose pts start 2 s in (frames before that do not exist). +pub fn camera_pts_late(_tier: Tier) -> FixtureSpec { + let duration = 6.0; + let mut cam = camera(Timing::Constant { duration: 4.0 }); + cam.pts_offset = 2.0; + FixtureSpec { + name: "camera-pts-late".into(), + segments: vec![SegmentSpec { + display: screen(30, Timing::Constant { duration }), + camera: Some(cam), + mic: None, + system_audio: None, + }], + } +} + +/// Two recording segments (pause/resume), the second without a camera. +pub fn multi_segment(_tier: Tier) -> FixtureSpec { + FixtureSpec { + name: "multi-segment".into(), + segments: vec![ + SegmentSpec { + display: screen(30, Timing::Constant { duration: 4.0 }), + camera: Some(camera(Timing::Constant { duration: 4.0 })), + mic: None, + system_audio: None, + }, + SegmentSpec { + display: screen(30, Timing::Constant { duration: 3.0 }), + camera: None, + mic: None, + system_audio: None, + }, + ], + } +} + +/// A jittery 60 fps-nominal screen (~25 fps delivered, so nearly every +/// frame interval is a hole on the request grid) with a 10 s static hold at +/// 30-40 s. Timelines that play past the hold, then elsewhere, then jump +/// back into it exercise the decoders' bounded hole tracking. +pub fn backward_hold(_tier: Tier) -> FixtureSpec { + let duration = 52.0; + let mut display = screen( + 60, + Timing::Jittery { + duration, + seed: 0x5EED_0002, + holds: vec![(30.0, 10.0)], + }, + ); + display.gop = 30; + FixtureSpec { + name: "backward-hold".into(), + segments: vec![SegmentSpec { + display, + camera: Some(camera(Timing::Constant { duration })), + mic: None, + system_audio: None, + }], + } +} + +/// Recorded through the real pipeline: two recording segments (pause and +/// resume), the first with a jittery camera that is the last device to +/// start, a static-screen hold, a mic that starts late and loopback-style +/// system audio; the second screen and mic only. (A camera that starts +/// before the mic is covered by `early_camera`.) +pub fn recorded_studio(_tier: Tier) -> crate::pipeline::RecordedSpec { + use crate::pipeline::{Delivery, RecordedSegment, RecordedSpec, RecordedVideo}; + let screen = |delivery| RecordedVideo { + width: SCREEN_W, + height: SCREEN_H, + fps: 30, + delivery, + start_delay: 0.0, + }; + RecordedSpec { + name: "studio".into(), + segments: vec![ + RecordedSegment { + duration: 6.0, + screen: screen(Delivery::Holds(vec![(2.0, 1.5)])), + camera: Some(RecordedVideo { + width: CAMERA_W, + height: CAMERA_H, + fps: 30, + delivery: Delivery::Jittery { seed: 0x5EED_0003 }, + start_delay: 0.3, + }), + mic_delay: Some(0.15), + system_audio: true, + }, + RecordedSegment { + duration: 4.0, + screen: screen(Delivery::Steady), + camera: None, + mic_delay: Some(0.1), + system_audio: false, + }, + ], + } +} + +/// Instant recording through the real Instant muxers, started like the app +/// does (sources warm up, then the start gate opens at 0.517 s, between two frames): steady 30 fps +/// screen with a static hold, a mic that becomes ready at 0.2 s, +/// loopback-style system audio. +pub fn recorded_instant(_tier: Tier) -> crate::pipeline::InstantSpec { + use crate::pipeline::{Delivery, InstantSpec, RecordedVideo}; + InstantSpec { + name: "instant".into(), + duration: 6.0, + screen: RecordedVideo { + width: SCREEN_W, + height: SCREEN_H, + fps: 30, + delivery: Delivery::Holds(vec![(2.5, 1.2)]), + start_delay: 0.0, + }, + mic_delay: Some(0.2), + system_audio: true, + system_continuous: false, + arm_at: Some(0.517), + } +} + +/// Instant variants isolating each audio source's start behaviour: no start +/// gate, so a source that delivers its first sample after the screen does. +pub fn instant_variants(tier: Tier) -> Vec { + let base = recorded_instant(tier); + vec![ + crate::pipeline::InstantSpec { + name: "instant-ungated-mic-late".into(), + system_audio: false, + arm_at: None, + ..base.clone() + }, + crate::pipeline::InstantSpec { + name: "instant-ungated-system-loopback".into(), + mic_delay: None, + arm_at: None, + ..base.clone() + }, + crate::pipeline::InstantSpec { + name: "instant-ungated-system-continuous-mic-late".into(), + system_continuous: true, + arm_at: None, + ..base.clone() + }, + crate::pipeline::InstantSpec { + name: "instant-gated-system-only".into(), + mic_delay: None, + ..base.clone() + }, + crate::pipeline::InstantSpec { + name: "instant-gated-system-continuous-mic-late".into(), + system_continuous: true, + ..base.clone() + }, + crate::pipeline::InstantSpec { + name: "instant-gated-mic-only".into(), + system_audio: false, + ..base.clone() + }, + base, + ] +} diff --git a/crates/verify/src/fixture.rs b/crates/verify/src/fixture.rs new file mode 100644 index 00000000000..de6ac91972e --- /dev/null +++ b/crates/verify/src/fixture.rs @@ -0,0 +1,591 @@ +//! Fixture projects. Every fixture is generated by code from a spec and a +//! seed; nothing binary is checked in. +//! +//! Two generators share the project layout: +//! - [`write_fixture`] writes tracks directly with exact presentation times. +//! It is fast and deterministic and is how exotic timing is produced: VFR +//! holds of several seconds, jittery cameras, late starts. +//! - [`crate::pipeline`] records through the real recording pipeline in real +//! time (channel sources -> mux loop -> encoders -> containers). +//! +//! Source truth is never taken from the spec: [`SourceTruth::read`] decodes +//! every written track and reads each frame's marker and presentation time. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use cap_project::{ + AudioMeta, BackgroundSource, ClipConfiguration, MultipleSegment, MultipleSegments, Platform, + ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, + StudioRecordingStatus, TimelineConfiguration, TimelineSegment, VideoMeta, +}; +use relative_path::RelativePathBuf; +use serde::{Deserialize, Serialize}; + +use crate::{ + audio_code::{self, AudioSource}, + media::{self, AudioCodec, VideoWriter, VideoWriterConfig}, + video_code::{self, FrameMarker, StreamKind}, +}; + +/// Bump when generated output changes so cached fixtures are rebuilt. +pub const GENERATOR_VERSION: u32 = 2; + +/// splitmix64: tiny deterministic PRNG; every fixture prints its seed. +#[derive(Clone)] +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Self(seed) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + pub fn f64(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + + pub fn range(&mut self, lo: u64, hi: u64) -> u64 { + lo + self.next_u64() % (hi - lo + 1) + } +} + +/// Source frame slots on the nominal grid: frame `k` is presented at +/// `k / fps`. Slot numbers double as marker indices, so a report saying +/// "source frame 1797" also says "59.9 s into the track at 30 fps". +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub enum Timing { + /// Every slot from 0 to `duration * fps`. + Constant { duration: f64 }, + /// Constant rate with no frames inside the given (start, length) holds, + /// like a static screen under VFR capture. + Holds { + duration: f64, + holds: Vec<(f64, f64)>, + }, + /// A source delivering below its nominal rate with jitter: consecutive + /// frames are 1..=4 slots apart (mean ~2.4), with a rare 6..=12 slot + /// stall, and no frames inside `holds`. + Jittery { + duration: f64, + seed: u64, + holds: Vec<(f64, f64)>, + }, + /// Explicit slots. + Slots(Vec), +} + +impl Timing { + pub fn slots(&self, fps: u32) -> Vec { + match self { + Timing::Constant { duration } => { + (0..(duration * f64::from(fps)).round() as u32).collect() + } + Timing::Holds { duration, holds } => (0..(duration * f64::from(fps)).round() as u32) + .filter(|&k| { + let t = f64::from(k) / f64::from(fps); + !holds.iter().any(|&(s, l)| t > s && t < s + l) + }) + .collect(), + Timing::Jittery { + duration, + seed, + holds, + } => { + let mut rng = Rng::new(*seed); + let end = (duration * f64::from(fps)).round() as u32; + let mut out = Vec::new(); + let mut k = 0; + while k < end { + let t = f64::from(k) / f64::from(fps); + if !holds.iter().any(|&(s, l)| t > s && t < s + l) { + out.push(k); + } + let r = rng.f64(); + k += if r < 0.02 { + rng.range(6, 12) as u32 + } else if r < 0.17 { + 1 + } else if r < 0.62 { + 2 + } else if r < 0.87 { + 3 + } else { + 4 + }; + } + out + } + Timing::Slots(slots) => slots.clone(), + } + } + + pub fn duration(&self, fps: u32) -> f64 { + match self { + Timing::Constant { duration } + | Timing::Holds { duration, .. } + | Timing::Jittery { duration, .. } => *duration, + Timing::Slots(slots) => { + f64::from(slots.last().copied().unwrap_or(0) + 1) / f64::from(fps) + } + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct VideoTrackSpec { + pub width: u32, + pub height: u32, + pub fps: u32, + pub timing: Timing, + /// Frames between keyframes. Pick values whose keyframe times are not on + /// a 10 ms grid to exercise seek rounding. + pub gop: u32, + pub b_frames: u32, + /// Presentation time of the first slot inside the file (0 for every real + /// recording; non-zero models a file whose pts start late). + pub pts_offset: f64, + /// `start_time` written to recording-meta.json. + pub start_time: f64, + /// Encoder used for the track. + #[serde(default)] + pub encoder: TrackEncoder, +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub enum TrackEncoder { + /// libx264 with exact GOP control (all platforms). + #[default] + X264, + /// The product's macOS AVAssetWriter/VideoToolbox encoder. Falls back to + /// libx264 on other platforms. + AvFoundation, +} + +impl VideoTrackSpec { + pub fn new(width: u32, height: u32, fps: u32, timing: Timing) -> Self { + Self { + width, + height, + fps, + timing, + gop: fps * 2 + 1, + b_frames: 0, + pts_offset: 0.0, + start_time: 0.0, + encoder: TrackEncoder::X264, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct AudioTrackSpec { + pub source: AudioSource, + pub sample_rate: u32, + pub channels: u16, + pub duration: f64, + pub start_time: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct SegmentSpec { + pub display: VideoTrackSpec, + pub camera: Option, + pub mic: Option, + pub system_audio: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct FixtureSpec { + pub name: String, + pub segments: Vec, +} + +/// Stable cache key over a spec and the generator version. +pub fn cache_key(name: &str, spec: &impl Serialize) -> String { + let json = serde_json::to_string(spec).unwrap_or_default(); + let mut hash: u64 = 0xcbf2_9ce4_8422_2325 ^ u64::from(GENERATOR_VERSION); + for byte in json.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + format!("{name}-{hash:016x}") +} + +impl FixtureSpec { + pub fn cache_key(&self) -> String { + cache_key(&self.name, self) + } +} + +pub(crate) fn segment_dir(i: usize) -> String { + format!("content/segments/segment-{i}") +} + +fn write_video_track( + path: &Path, + kind: StreamKind, + segment: u8, + spec: &VideoTrackSpec, +) -> anyhow::Result<()> { + #[cfg(target_os = "macos")] + if spec.encoder == TrackEncoder::AvFoundation { + let times: Vec<(u32, f64)> = spec + .timing + .slots(spec.fps) + .into_iter() + .map(|slot| { + ( + slot, + spec.pts_offset + f64::from(slot) / f64::from(spec.fps), + ) + }) + .collect(); + return crate::fixture_avf::write_video_track( + path, + |slot| FrameMarker::new(kind, segment, slot), + spec.width, + spec.height, + spec.fps, + ×, + ); + } + let mut config = VideoWriterConfig::new(spec.width, spec.height, spec.fps); + config.gop = spec.gop; + config.b_frames = spec.b_frames; + let mut writer = VideoWriter::create(path, &config)?; + for slot in spec.timing.slots(spec.fps) { + let marker = FrameMarker::new(kind, segment, slot); + let rgba = video_code::marker_rgba(marker, spec.width as usize, spec.height as usize); + writer.write_rgba( + &rgba, + spec.pts_offset + f64::from(slot) / f64::from(spec.fps), + )?; + } + writer.finish()?; + Ok(()) +} + +fn write_audio_track(path: &Path, spec: &AudioTrackSpec) -> anyhow::Result<()> { + let samples = (spec.duration * f64::from(spec.sample_rate)).round() as usize; + let mut mono = vec![0.0f32; samples]; + let schedule = audio_code::standard_schedule(spec.source, 0.1, spec.duration - 0.1, 0.5); + audio_code::render_markers( + &mut mono, + spec.sample_rate, + spec.source, + &schedule, + audio_code::DEFAULT_AMPLITUDE, + ); + let channels = vec![mono; usize::from(spec.channels.max(1))]; + media::write_audio(path, &channels, spec.sample_rate, AudioCodec::Opus, 96_000) +} + +/// Writes recording-meta.json and a default project-config.json for the +/// given segments and returns the meta. +pub fn write_project_files( + dir: &Path, + name: &str, + segments: Vec, +) -> anyhow::Result { + let mut timeline = Vec::new(); + let mut clips = Vec::new(); + for (i, segment) in segments.iter().enumerate() { + let display = dir.join(segment.display.path.as_str()); + let duration = media::video_packet_times(&display)? + .last() + .copied() + .unwrap_or(0.0) + + 1.0 / f64::from(segment.display.fps); + timeline.push(TimelineSegment { + recording_clip: i as u32, + start: 0.0, + end: duration, + timescale: 1.0, + name: None, + speed_audio_mode: None, + hide_cursor: None, + volume: None, + }); + clips.push(ClipConfiguration { + index: i as u32, + offsets: segment.calculate_audio_offsets(), + offsets_auto_calculated: true, + }); + } + let meta = RecordingMeta { + platform: Some(Platform::default()), + project_path: dir.to_path_buf(), + pretty_name: name.to_string(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { + inner: MultipleSegments { + segments, + cursors: Default::default(), + status: Some(StudioRecordingStatus::Complete), + }, + })), + upload: None, + }; + meta.save_for_project() + .map_err(|e| anyhow::anyhow!("save recording meta: {e:?}"))?; + let mut config = verification_project_config(); + config.timeline = Some(TimelineConfiguration { + segments: timeline, + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + style_segments: Vec::new(), + image_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }); + config.clips = clips; + config.write(dir)?; + Ok(meta) +} + +/// Product defaults with a neutral background (never mistaken for a key +/// colour) and no screen motion blur (which blends neighbouring frames). +pub fn verification_project_config() -> ProjectConfiguration { + let mut config = ProjectConfiguration::default(); + config.background.source = BackgroundSource::Color { + value: [60, 60, 60], + alpha: 255, + }; + config.screen_motion_blur = 0.0; + config +} + +/// Writes a fixture project with exact timing into `dir` (created). +pub fn write_fixture(spec: &FixtureSpec, dir: &Path) -> anyhow::Result<()> { + if dir.exists() { + std::fs::remove_dir_all(dir)?; + } + let mut segments = Vec::new(); + for (i, segment) in spec.segments.iter().enumerate() { + let seg_dir = segment_dir(i); + std::fs::create_dir_all(dir.join(&seg_dir))?; + let rel = |file: &str| RelativePathBuf::from(format!("{seg_dir}/{file}")); + write_video_track( + &dir.join(rel("display.mp4").as_str()), + StreamKind::Screen, + i as u8, + &segment.display, + ) + .context("display track")?; + let camera = match &segment.camera { + Some(camera) => { + write_video_track( + &dir.join(rel("camera.mp4").as_str()), + StreamKind::Camera, + i as u8, + camera, + ) + .context("camera track")?; + Some(VideoMeta { + path: rel("camera.mp4"), + fps: camera.fps, + start_time: Some(camera.start_time), + device_id: None, + }) + } + None => None, + }; + let audio = |spec: &Option, file: &str| -> anyhow::Result<_> { + Ok(match spec { + Some(a) => { + write_audio_track(&dir.join(rel(file).as_str()), a)?; + Some(AudioMeta { + path: rel(file), + start_time: Some(a.start_time), + device_id: None, + gap_summary: None, + }) + } + None => None, + }) + }; + let mic = audio(&segment.mic, "audio-input.ogg")?; + let system_audio = audio(&segment.system_audio, "system_audio.ogg")?; + segments.push(MultipleSegment { + display: VideoMeta { + path: rel("display.mp4"), + fps: segment.display.fps, + start_time: Some(segment.display.start_time), + device_id: None, + }, + camera, + mic, + system_audio, + cursor: None, + keyboard: None, + display_notch: None, + }); + } + write_project_files(dir, &spec.name, segments)?; + Ok(()) +} + +/// Root for generated fixtures: `CAP_VERIFY_FIXTURE_DIR`, else +/// `/verify-fixtures`. CI caches this directory keyed by the +/// generator sources. +pub fn fixture_root() -> PathBuf { + if let Some(dir) = std::env::var_os("CAP_VERIFY_FIXTURE_DIR") { + return PathBuf::from(dir); + } + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| manifest.join("../../target")); + target.join("verify-fixtures") +} + +/// Returns a ready fixture, generating it unless a complete cached copy with +/// the same key exists. Safe to call from parallel test processes. +pub fn ensure_fixture( + spec: &FixtureSpec, + generate: impl FnOnce(&FixtureSpec, &Path) -> anyhow::Result<()>, +) -> anyhow::Result { + ensure_fixture_keyed(spec.cache_key(), |dir| generate(spec, dir)) +} + +/// [`ensure_fixture`] for any generator, keyed by `key`. +pub fn ensure_fixture_keyed( + key: String, + generate: impl FnOnce(&Path) -> anyhow::Result<()>, +) -> anyhow::Result { + let root = fixture_root(); + std::fs::create_dir_all(&root)?; + let dir = root.join(format!("{key}.cap")); + let done = root.join(format!("{key}.complete")); + if done.exists() && dir.exists() { + return Ok(dir); + } + static GENERATING: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _serial = GENERATING.lock().unwrap_or_else(|e| e.into_inner()); + if done.exists() && dir.exists() { + return Ok(dir); + } + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let staging = root.join(format!( + "{key}.{}-{}.tmp.cap", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + generate(&staging)?; + // Paths inside recording-meta.json are relative, so the staged project + // stays valid after the rename. + if dir.exists() { + let _ = std::fs::remove_dir_all(&staging); + } else if let Err(error) = std::fs::rename(&staging, &dir) { + if !dir.exists() { + bail!("publish fixture {dir:?}: {error}"); + } + let _ = std::fs::remove_dir_all(&staging); + } + std::fs::write(&done, key)?; + Ok(dir) +} + +/// One decoded source frame: its presentation time in the file and marker. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct SourceFrame { + pub pts_secs: f64, + pub index: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TrackTruth { + pub kind: StreamKind, + pub segment: u8, + /// Frames in presentation order. + pub frames: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SegmentTruth { + pub screen: TrackTruth, + pub camera: Option, +} + +/// Ground truth for a project, read back from its files. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SourceTruth { + pub segments: Vec, +} + +impl SourceTruth { + pub fn read(project: &Path) -> anyhow::Result { + let cache = project.join("verify-truth.json"); + if let Ok(text) = std::fs::read_to_string(&cache) + && let Ok(truth) = serde_json::from_str(&text) + { + return Ok(truth); + } + let meta = RecordingMeta::load_for_project(project) + .map_err(|e| anyhow::anyhow!("load meta: {e:?}"))?; + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + bail!("not a multi-segment studio project"); + }; + let mut segments = Vec::new(); + for (i, segment) in inner.segments.iter().enumerate() { + let screen = read_track( + &project.join(segment.display.path.as_str()), + StreamKind::Screen, + i as u8, + )?; + let camera = match &segment.camera { + Some(camera) => Some(read_track( + &project.join(camera.path.as_str()), + StreamKind::Camera, + i as u8, + )?), + None => None, + }; + segments.push(SegmentTruth { screen, camera }); + } + let truth = SourceTruth { segments }; + let _ = std::fs::write(&cache, serde_json::to_string(&truth)?); + Ok(truth) + } +} + +fn read_track(path: &Path, kind: StreamKind, segment: u8) -> anyhow::Result { + let mut frames = Vec::new(); + let mut bad = Vec::new(); + media::decode_video(path, |picture| { + match video_code::read_stream(&picture.view(), kind).marker() { + Some(marker) if marker.segment == segment => frames.push(SourceFrame { + pts_secs: picture.pts_secs, + index: marker.index, + }), + other => bad.push((picture.pts_secs, other)), + } + })?; + if !bad.is_empty() { + bail!( + "{path:?}: {} source frames without a readable {} marker (first at {:.3}s)", + bad.len(), + kind.name(), + bad[0].0 + ); + } + frames.sort_by(|a, b| a.pts_secs.total_cmp(&b.pts_secs)); + Ok(TrackTruth { + kind, + segment, + frames, + }) +} diff --git a/crates/verify/src/fixture_avf.rs b/crates/verify/src/fixture_avf.rs new file mode 100644 index 00000000000..d81b63e01c4 --- /dev/null +++ b/crates/verify/src/fixture_avf.rs @@ -0,0 +1,111 @@ +//! macOS: writes a track through the product's AVFoundation encoder +//! (`cap-enc-avfoundation`, AVAssetWriter + VideoToolbox), the encoder Studio +//! recordings use on macOS, fed NV12 IOSurface buffers with exact times. + +use std::{path::Path, time::Duration}; + +use anyhow::{anyhow, bail}; +use cap_enc_avfoundation::{MP4Encoder, QueueFrameError}; +use cap_media_info::{RawVideoFormat, VideoInfo}; +use cidre::{arc, cf, cm, cv}; + +use crate::video_code::{self, FrameMarker}; + +pub fn pixel_buffer( + width: usize, + height: usize, + rgba: &[u8], +) -> anyhow::Result> { + let io_surface_properties = cf::Dictionary::new(); + let keys: [&cf::Type; 1] = [cv::pixel_buffer::keys::io_surf_props().as_ref()]; + let values: [&cf::Type; 1] = [io_surface_properties.as_ref()]; + let attrs = cf::Dictionary::with_keys_values(&keys, &values) + .ok_or_else(|| anyhow!("pixel buffer attributes"))?; + let mut buf = cv::PixelBuf::new(width, height, cv::PixelFormat::_420V, Some(&attrs)) + .map_err(|e| anyhow!("pixel buffer: {e:?}"))?; + unsafe { + buf.lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT) + .result() + .map_err(|e| anyhow!("lock: {e:?}"))?; + let y_stride = buf.plane_bytes_per_row(0); + let uv_stride = buf.plane_bytes_per_row(1); + let y_base = buf.plane_base_address(0).cast_mut(); + let uv_base = buf.plane_base_address(1).cast_mut(); + let ycc = |x: usize, y: usize| { + let i = (y * width + x) * 4; + video_code::rgb_to_ycbcr(rgba[i], rgba[i + 1], rgba[i + 2]) + }; + for y in 0..height { + for x in 0..width { + let (l, _, _) = ycc(x, y); + *y_base.add(y * y_stride + x) = (16.0 + l * 219.0 / 255.0).round() as u8; + } + } + for y in 0..height / 2 { + for x in 0..width / 2 { + let (_, cb, cr) = ycc(x * 2, y * 2); + *uv_base.add(y * uv_stride + x * 2) = (128.0 + cb * 224.0 / 255.0).round() as u8; + *uv_base.add(y * uv_stride + x * 2 + 1) = + (128.0 + cr * 224.0 / 255.0).round() as u8; + } + } + buf.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT); + } + Ok(buf) +} + +pub fn write_video_track( + path: &Path, + marker_for: impl Fn(u32) -> FrameMarker, + width: u32, + height: u32, + fps: u32, + times: &[(u32, f64)], +) -> anyhow::Result<()> { + let info = VideoInfo::from_raw(RawVideoFormat::Nv12, width, height, fps); + let mut encoder = + MP4Encoder::init(path.to_path_buf(), info, None, None).map_err(|e| anyhow!("{e}"))?; + let mut last = Duration::ZERO; + for &(slot, pts) in times { + let rgba = video_code::marker_rgba(marker_for(slot), width as usize, height as usize); + let buf = pixel_buffer(width as usize, height as usize, &rgba)?; + let format_desc = cm::VideoFormatDesc::with_image_buf(&buf) + .map_err(|e| anyhow!("format description: {e:?}"))?; + let timing = cm::SampleTimingInfo { + duration: cm::Time::new(1_000_000 / i64::from(fps), 1_000_000), + pts: cm::Time::new((pts * 1e6).round() as i64, 1_000_000), + dts: cm::Time::invalid(), + }; + let sample = cm::SampleBuf::with_image_buf( + &buf, + true, + None, + std::ptr::null(), + &format_desc, + &timing, + ) + .map_err(|e| anyhow!("sample buffer: {e:?}"))?; + let timestamp = Duration::from_secs_f64(pts); + let mut queued = false; + for _ in 0..20_000 { + match encoder.queue_video_frame(sample.clone(), timestamp) { + Ok(()) => { + queued = true; + break; + } + Err(QueueFrameError::NotReadyForMore) => { + std::thread::sleep(Duration::from_micros(250)); + } + Err(e) => bail!("queue frame at {pts}: {e}"), + } + } + if !queued { + bail!("encoder never became ready at {pts}"); + } + last = timestamp; + } + encoder + .finish(Some(last + Duration::from_secs_f64(1.0 / f64::from(fps)))) + .map_err(|e| anyhow!("finish: {e}"))?; + Ok(()) +} diff --git a/crates/verify/src/lib.rs b/crates/verify/src/lib.rs new file mode 100644 index 00000000000..2460603f81e --- /dev/null +++ b/crates/verify/src/lib.rs @@ -0,0 +1,20 @@ +//! Engine verification suite: marker codecs, fixture generation, a reference +//! timeline model and oracle checkers. + +pub mod audio_code; +pub mod audio_fixture; +pub mod audio_model; +pub mod catalog; +pub mod fixture; +#[cfg(target_os = "macos")] +pub mod fixture_avf; +pub mod media; +pub mod model; +pub mod oracle; +pub mod pipeline; +pub mod report; +#[cfg(target_os = "macos")] +pub mod sched; +pub mod sentinel; +pub mod stimulus; +pub mod video_code; diff --git a/crates/verify/src/media.rs b/crates/verify/src/media.rs new file mode 100644 index 00000000000..0a397881dea --- /dev/null +++ b/crates/verify/src/media.rs @@ -0,0 +1,606 @@ +//! Thin FFmpeg helpers used to write fixtures and read outputs. These are +//! test tools: simple, sequential and independent of the product's decoders. + +use std::path::Path; + +use anyhow::{Context, anyhow, bail}; +use ffmpeg::{ + ChannelLayout, Dictionary, Rational, + codec::{self, Flags as CodecFlags}, + format::{self, Pixel, Sample, sample::Type as SampleType}, + frame, + media::Type, + software::{resampling, scaling}, +}; + +use crate::video_code::{PackedFrame, PixelOrder, PixelSource, Yuv420Frame}; + +pub fn init() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + ffmpeg::init().expect("ffmpeg init"); + ffmpeg::util::log::set_level(ffmpeg::util::log::Level::Error); + }); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoCodec { + H264, + Hevc, +} + +#[derive(Clone, Debug)] +pub struct VideoWriterConfig { + pub width: u32, + pub height: u32, + /// Nominal frame rate written to the stream (the track's `r_frame_rate`). + pub fps: u32, + pub codec: VideoCodec, + /// Distance between keyframes, in frames. + pub gop: u32, + pub b_frames: u32, + /// Target bitrate in bits per second; `None` uses constant quality. + pub bitrate: Option, + /// Stream time base denominator (ticks per second). + pub time_base: i32, +} + +impl VideoWriterConfig { + pub fn new(width: u32, height: u32, fps: u32) -> Self { + Self { + width, + height, + fps, + codec: VideoCodec::H264, + gop: fps, + b_frames: 0, + bitrate: None, + time_base: 90_000, + } + } +} + +fn find_video_encoder(codec: VideoCodec) -> anyhow::Result { + let names: &[&str] = match codec { + VideoCodec::H264 => &["libx264", "h264_videotoolbox", "h264_mf"], + // The bundled FFmpeg's libx265 crashes inside avcodec after a few + // frames, so HEVC uses the platform hardware encoders the product + // itself uses (VideoToolbox, Media Foundation). + VideoCodec::Hevc => &["hevc_videotoolbox", "hevc_mf"], + }; + if let Ok(name) = std::env::var("CAP_VERIFY_FORCE_ENCODER") { + return ffmpeg::encoder::find_by_name(&name).ok_or_else(|| anyhow!("no {name}")); + } + names + .iter() + .find_map(|name| ffmpeg::encoder::find_by_name(name)) + .ok_or_else(|| anyhow!("no encoder available for {codec:?}")) +} + +/// Writes an MP4 (or MOV) with exactly the presentation timestamps given. +pub struct VideoWriter { + output: format::context::Output, + encoder: ffmpeg::encoder::Video, + scaler: scaling::Context, + yuv: frame::Video, + rgba: frame::Video, + encoder_time_base: Rational, + stream_time_base: Rational, + last_pts: Option, + frames: u64, +} + +impl VideoWriter { + pub fn create(path: &Path, config: &VideoWriterConfig) -> anyhow::Result { + init(); + let codec = find_video_encoder(config.codec)?; + let mut output = format::output(&path).with_context(|| format!("open {path:?}"))?; + let global_header = output + .format() + .flags() + .contains(format::Flags::GLOBAL_HEADER); + let time_base = Rational::new(1, config.time_base); + let mut ctx = codec::context::Context::new_with_codec(codec); + if codec.name() == "libx264" { + ctx.set_threading(codec::threading::Config::count(1)); + } + let mut enc = ctx.encoder().video()?; + enc.set_width(config.width); + enc.set_height(config.height); + enc.set_format(Pixel::YUV420P); + enc.set_time_base(time_base); + enc.set_frame_rate(Some(Rational::new(config.fps as i32, 1))); + enc.set_gop(config.gop); + enc.set_max_b_frames(config.b_frames as usize); + if let Some(bitrate) = config.bitrate { + enc.set_bit_rate(bitrate as usize); + } + if global_header { + enc.set_flags(CodecFlags::GLOBAL_HEADER); + } + let mut options = Dictionary::new(); + if codec.name() == "libx264" { + options.set("preset", "ultrafast"); + options.set("x264-params", "scenecut=0:open-gop=0"); + if config.bitrate.is_none() { + options.set("crf", "23"); + } + } + let encoder = enc.open_as_with(codec, options)?; + let stream_time_base = { + let mut stream = output.add_stream(codec)?; + stream.set_parameters(&encoder); + stream.set_time_base(time_base); + stream.set_rate(Rational::new(config.fps as i32, 1)); + stream.time_base() + }; + output.write_header()?; + let stream_time_base = output + .stream(0) + .map(|s| s.time_base()) + .unwrap_or(stream_time_base); + let scaler = scaling::Context::get( + Pixel::RGBA, + config.width, + config.height, + Pixel::YUV420P, + config.width, + config.height, + scaling::Flags::BILINEAR, + )?; + Ok(Self { + output, + encoder, + scaler, + yuv: frame::Video::new(Pixel::YUV420P, config.width, config.height), + rgba: frame::Video::new(Pixel::RGBA, config.width, config.height), + encoder_time_base: time_base, + stream_time_base, + last_pts: None, + frames: 0, + }) + } + + /// Queues one RGBA picture (tightly packed) presented at `pts_secs`. + pub fn write_rgba(&mut self, rgba: &[u8], pts_secs: f64) -> anyhow::Result<()> { + let width = self.rgba.width() as usize; + let height = self.rgba.height() as usize; + let stride = self.rgba.stride(0); + let dst = self.rgba.data_mut(0); + for y in 0..height { + dst[y * stride..y * stride + width * 4] + .copy_from_slice(&rgba[y * width * 4..(y + 1) * width * 4]); + } + let mut yuv = frame::Video::new(Pixel::YUV420P, self.yuv.width(), self.yuv.height()); + self.scaler.run(&self.rgba, &mut yuv)?; + let pts = (pts_secs * f64::from(self.encoder_time_base.denominator()) + / f64::from(self.encoder_time_base.numerator())) + .round() as i64; + if self.last_pts.is_some_and(|last| pts <= last) { + bail!("non-increasing pts {pts_secs}"); + } + self.last_pts = Some(pts); + yuv.set_pts(Some(pts)); + self.encoder.send_frame(&yuv)?; + self.frames += 1; + self.drain() + } + + fn drain(&mut self) -> anyhow::Result<()> { + let mut packet = ffmpeg::Packet::empty(); + while self.encoder.receive_packet(&mut packet).is_ok() { + packet.set_stream(0); + packet.rescale_ts(self.encoder_time_base, self.stream_time_base); + packet.write_interleaved(&mut self.output)?; + } + Ok(()) + } + + pub fn finish(mut self) -> anyhow::Result { + self.encoder.send_eof()?; + self.drain()?; + self.output.write_trailer()?; + Ok(self.frames) + } +} + +/// A decoded picture in a layout the marker reader understands. +pub struct Picture { + pub frame: frame::Video, + pub pts_secs: f64, +} + +impl Picture { + pub fn view(&self) -> PictureView<'_> { + PictureView::new(&self.frame) + } +} + +pub enum PictureView<'a> { + Yuv(Yuv420Frame<'a>), + Packed(PackedFrame<'a>), +} + +impl<'a> PictureView<'a> { + pub fn new(frame: &'a frame::Video) -> Self { + let width = frame.width() as usize; + let height = frame.height() as usize; + match frame.format() { + Pixel::NV12 => PictureView::Yuv(Yuv420Frame { + width, + height, + y: frame.data(0), + y_stride: frame.stride(0), + u: frame.data(1), + u_stride: frame.stride(1), + v: None, + }), + Pixel::RGBA | Pixel::BGRA => PictureView::Packed(PackedFrame { + data: frame.data(0), + width, + height, + stride: frame.stride(0), + order: if frame.format() == Pixel::RGBA { + PixelOrder::Rgba + } else { + PixelOrder::Bgra + }, + }), + _ => PictureView::Yuv(Yuv420Frame { + width, + height, + y: frame.data(0), + y_stride: frame.stride(0), + u: frame.data(1), + u_stride: frame.stride(1), + v: Some((frame.data(2), frame.stride(2))), + }), + } + } +} + +impl PixelSource for PictureView<'_> { + fn width(&self) -> usize { + match self { + PictureView::Yuv(f) => f.width(), + PictureView::Packed(f) => f.width(), + } + } + + fn height(&self) -> usize { + match self { + PictureView::Yuv(f) => f.height(), + PictureView::Packed(f) => f.height(), + } + } + + fn luma(&self, x: usize, y: usize) -> f64 { + match self { + PictureView::Yuv(f) => f.luma(x, y), + PictureView::Packed(f) => f.luma(x, y), + } + } + + fn chroma(&self, x: usize, y: usize) -> (f64, f64) { + match self { + PictureView::Yuv(f) => f.chroma(x, y), + PictureView::Packed(f) => f.chroma(x, y), + } + } +} + +fn readable_format(format: Pixel) -> bool { + matches!( + format, + Pixel::YUV420P | Pixel::YUVJ420P | Pixel::NV12 | Pixel::RGBA | Pixel::BGRA + ) +} + +/// Decodes every picture of the best video stream in presentation order. +pub fn decode_video(path: &Path, mut on_picture: impl FnMut(Picture)) -> anyhow::Result<()> { + init(); + let mut input = format::input(&path).with_context(|| format!("open {path:?}"))?; + let stream = input + .streams() + .best(Type::Video) + .ok_or_else(|| anyhow!("{path:?} has no video stream"))?; + let index = stream.index(); + let time_base = stream.time_base(); + let mut decoder = codec::context::Context::from_parameters(stream.parameters())? + .decoder() + .video()?; + let mut scaler: Option = None; + let mut emit = |decoded: &frame::Video, scaler: &mut Option| { + let pts = decoded.timestamp().or(decoded.pts()).unwrap_or(0); + let pts_secs = + pts as f64 * f64::from(time_base.numerator()) / f64::from(time_base.denominator()); + let frame = if readable_format(decoded.format()) { + decoded.clone() + } else { + let s = scaler.get_or_insert_with(|| { + scaling::Context::get( + decoded.format(), + decoded.width(), + decoded.height(), + Pixel::YUV420P, + decoded.width(), + decoded.height(), + scaling::Flags::POINT, + ) + .expect("scaler") + }); + let mut out = frame::Video::empty(); + s.run(decoded, &mut out).expect("convert picture"); + out + }; + on_picture(Picture { frame, pts_secs }); + }; + let mut decoded = frame::Video::empty(); + for (stream, packet) in input.packets() { + if stream.index() != index { + continue; + } + decoder.send_packet(&packet)?; + while decoder.receive_frame(&mut decoded).is_ok() { + emit(&decoded, &mut scaler); + } + } + decoder.send_eof()?; + while decoder.receive_frame(&mut decoded).is_ok() { + emit(&decoded, &mut scaler); + } + Ok(()) +} + +/// Packet presentation times of the best video stream, sorted. +pub fn video_packet_times(path: &Path) -> anyhow::Result> { + init(); + let mut input = format::input(&path).with_context(|| format!("open {path:?}"))?; + let stream = input + .streams() + .best(Type::Video) + .ok_or_else(|| anyhow!("{path:?} has no video stream"))?; + let index = stream.index(); + let tb = stream.time_base(); + let mut times: Vec = input + .packets() + .filter(|(s, _)| s.index() == index) + .filter_map(|(_, p)| p.pts()) + .map(|pts| pts as f64 * f64::from(tb.numerator()) / f64::from(tb.denominator())) + .collect(); + times.sort_by(f64::total_cmp); + Ok(times) +} + +/// Presentation times of the keyframes of the best video stream, sorted. +pub fn keyframe_times(path: &Path) -> anyhow::Result> { + init(); + let mut input = format::input(&path).with_context(|| format!("open {path:?}"))?; + let stream = input + .streams() + .best(Type::Video) + .ok_or_else(|| anyhow!("{path:?} has no video stream"))?; + let index = stream.index(); + let tb = stream.time_base(); + let mut times: Vec = input + .packets() + .filter(|(s, p)| s.index() == index && p.is_key()) + .filter_map(|(_, p)| p.pts()) + .map(|pts| pts as f64 * f64::from(tb.numerator()) / f64::from(tb.denominator())) + .collect(); + times.sort_by(f64::total_cmp); + Ok(times) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioCodec { + Aac, + Opus, + Flac, +} + +/// Writes mono or multichannel f32 audio (channels given separately). +pub fn write_audio( + path: &Path, + channels: &[Vec], + sample_rate: u32, + codec: AudioCodec, + bitrate: usize, +) -> anyhow::Result<()> { + init(); + let codec_name = match codec { + AudioCodec::Aac => "aac", + AudioCodec::Opus => "libopus", + AudioCodec::Flac => "flac", + }; + let encoder_codec = + ffmpeg::encoder::find_by_name(codec_name).ok_or_else(|| anyhow!("no {codec_name}"))?; + let layout = ChannelLayout::default(channels.len() as i32); + let mut output = format::output(&path)?; + let global_header = output + .format() + .flags() + .contains(format::Flags::GLOBAL_HEADER); + let mut enc = codec::context::Context::new_with_codec(encoder_codec) + .encoder() + .audio()?; + let sample_format = encoder_codec + .audio() + .ok() + .and_then(|a| a.formats().and_then(|mut f| f.next())) + .unwrap_or(Sample::F32(SampleType::Planar)); + enc.set_rate(sample_rate as i32); + enc.set_channel_layout(layout); + enc.set_format(sample_format); + enc.set_bit_rate(bitrate); + enc.set_time_base(Rational::new(1, sample_rate as i32)); + if global_header { + enc.set_flags(CodecFlags::GLOBAL_HEADER); + } + let mut encoder = enc.open_as(encoder_codec)?; + { + let mut stream = output.add_stream(encoder_codec)?; + stream.set_parameters(&encoder); + stream.set_time_base(Rational::new(1, sample_rate as i32)); + } + output.write_header()?; + let stream_tb = output.stream(0).map(|s| s.time_base()).unwrap(); + let frame_size = match encoder.frame_size() { + 0 => 1024, + n => n as usize, + }; + let mut resampler = resampling::Context::get( + Sample::F32(SampleType::Planar), + layout, + sample_rate, + sample_format, + layout, + sample_rate, + )?; + let total = channels.first().map_or(0, Vec::len); + let mut pos = 0; + let mut packet = ffmpeg::Packet::empty(); + let enc_tb = Rational::new(1, sample_rate as i32); + let mut drain = |encoder: &mut ffmpeg::encoder::Audio, + output: &mut format::context::Output| + -> anyhow::Result<()> { + while encoder.receive_packet(&mut packet).is_ok() { + packet.set_stream(0); + packet.rescale_ts(enc_tb, stream_tb); + packet.write_interleaved(output)?; + } + Ok(()) + }; + while pos < total { + let n = frame_size.min(total - pos); + let mut planar = frame::Audio::new(Sample::F32(SampleType::Planar), n, layout); + planar.set_rate(sample_rate); + for (ch, data) in channels.iter().enumerate() { + let plane: &mut [f32] = planar.plane_mut(ch); + plane[..n].copy_from_slice(&data[pos..pos + n]); + } + let mut converted = frame::Audio::empty(); + resampler.run(&planar, &mut converted)?; + converted.set_pts(Some(pos as i64)); + converted.set_rate(sample_rate); + encoder.send_frame(&converted)?; + drain(&mut encoder, &mut output)?; + pos += n; + } + encoder.send_eof()?; + drain(&mut encoder, &mut output)?; + output.write_trailer()?; + Ok(()) +} + +pub struct DecodedAudio { + pub sample_rate: u32, + /// One vector per channel. + pub channels: Vec>, + /// Presentation time of the first decoded sample, in seconds. + pub start_secs: f64, +} + +impl DecodedAudio { + pub fn mono(&self) -> Vec { + let n = self.channels.iter().map(Vec::len).min().unwrap_or(0); + let count = self.channels.len().max(1) as f32; + (0..n) + .map(|i| self.channels.iter().map(|c| c[i]).sum::() / count) + .collect() + } +} + +/// Decodes the best audio stream (optionally resampled) to planar f32. +pub fn decode_audio(path: &Path, out_rate: Option) -> anyhow::Result { + init(); + let mut input = format::input(&path).with_context(|| format!("open {path:?}"))?; + let stream = input + .streams() + .best(Type::Audio) + .ok_or_else(|| anyhow!("{path:?} has no audio stream"))?; + let index = stream.index(); + let tb = stream.time_base(); + // The container start time already accounts for encoder priming that the + // decoder strips (Opus pre-skip, AAC edit lists); the first frame's pts + // does not always. + let container_start = (stream.start_time() != ffmpeg::ffi::AV_NOPTS_VALUE).then(|| { + stream.start_time() as f64 * f64::from(tb.numerator()) / f64::from(tb.denominator()) + }); + let mut decoder = codec::context::Context::from_parameters(stream.parameters())? + .decoder() + .audio()?; + let mut resampler: Option = None; + let mut channels: Vec> = Vec::new(); + let mut start_secs = None; + let mut rate = 0; + let mut handle = |decoded: &frame::Audio, + resampler: &mut Option| + -> anyhow::Result<()> { + if start_secs.is_none() { + let pts = decoded.pts().or(decoded.timestamp()).unwrap_or(0); + start_secs = Some(pts as f64 * f64::from(tb.numerator()) / f64::from(tb.denominator())); + } + let target_rate = out_rate.unwrap_or(decoded.rate()); + rate = target_rate; + let layout = if decoded.channel_layout().is_empty() { + ChannelLayout::default(decoded.channels() as i32) + } else { + decoded.channel_layout() + }; + let r = match resampler { + Some(r) => r, + None => resampler.insert(resampling::Context::get( + decoded.format(), + layout, + decoded.rate(), + Sample::F32(SampleType::Planar), + layout, + target_rate, + )?), + }; + let mut out = frame::Audio::empty(); + let mut input = decoded.clone(); + input.set_channel_layout(layout); + r.run(&input, &mut out)?; + append_planar(&mut channels, &out); + Ok(()) + }; + let mut decoded = frame::Audio::empty(); + for (s, packet) in input.packets() { + if s.index() != index { + continue; + } + decoder.send_packet(&packet)?; + while decoder.receive_frame(&mut decoded).is_ok() { + handle(&decoded, &mut resampler)?; + } + } + decoder.send_eof()?; + while decoder.receive_frame(&mut decoded).is_ok() { + handle(&decoded, &mut resampler)?; + } + if let Some(r) = resampler.as_mut() { + let mut out = frame::Audio::empty(); + while r.flush(&mut out).is_ok() && out.samples() > 0 { + append_planar(&mut channels, &out); + } + } + Ok(DecodedAudio { + sample_rate: rate, + channels, + start_secs: container_start.or(start_secs).unwrap_or(0.0), + }) +} + +fn append_planar(channels: &mut Vec>, out: &frame::Audio) { + let n = out.samples(); + let count = out.channels() as usize; + if channels.len() < count { + channels.resize(count, Vec::new()); + } + for (ch, data) in channels.iter_mut().enumerate().take(count) { + let plane: &[f32] = out.plane(ch); + data.extend_from_slice(&plane[..n]); + } +} diff --git a/crates/verify/src/model.rs b/crates/verify/src/model.rs new file mode 100644 index 00000000000..dd8da79dd72 --- /dev/null +++ b/crates/verify/src/model.rs @@ -0,0 +1,481 @@ +//! Reference timeline model: what every output frame must show. +//! +//! Written for the tests from the project's documented rules, not by calling +//! the production mapping: +//! - Output frame `i` at `fps` is output time `i / fps`; an export has +//! `ceil(fps * duration)` frames. +//! - Timeline segments play back to back; segment `k` covers +//! `(end - start) / timescale` output seconds and maps output time `u` +//! inside it to recording time `start + u * timescale`. A clip transition +//! into segment `k` overlaps the two segments by its (clamped) duration: +//! a cross-fade frame must be a blend of the outgoing segment's frame (at +//! its last `duration` seconds) and the incoming one's; fades through black +//! are not checked. +//! - Every track of a recording segment is aligned to the latest track start +//! (`latest_start_time - track.start_time` is added to the request). The +//! camera additionally takes the clip's camera offset when the user set it; +//! an auto-calculated camera offset describes the same start alignment and +//! must not be applied a second time. +//! - A request at track time `s` asks for frame slot `floor(s * fps_meta)`, +//! where `fps_meta` is the track's nominal rate from recording-meta.json, +//! and is answered by the newest source frame whose slot +//! `round(pts * fps_meta)` is at or before it (the first frame before the +//! track starts). Requests within float rounding of a slot boundary accept +//! either neighbour. +//! - The camera is absent when hidden, when the segment has no camera, and +//! inside a "hide camera" scene. A scene fades in over `transition_in` +//! before its start and out over `transition_out` before its end; fades +//! are not checked. + +use cap_project::{ProjectConfiguration, RecordingMeta, SceneMode, StudioRecordingMeta}; +use serde::{Deserialize, Serialize}; + +use crate::{ + fixture::{SourceTruth, TrackTruth}, + oracle::StreamExpectation, + video_code::StreamKind, +}; + +const MIN_TRANSITION: f64 = 0.05; +const BOUNDARY_EPSILON: f64 = 1e-9; +const SLOT_EPSILON: f64 = 2e-3; + +#[derive(Clone, Debug)] +struct ModelSegment { + clip: usize, + start: f64, + end: f64, + timescale: f64, + transition_in: f64, + /// The incoming transition is a cross-fade (fades through black are not + /// checked). + crossfade: bool, +} + +impl ModelSegment { + fn duration(&self) -> f64 { + (self.end - self.start) / self.timescale + } +} + +#[derive(Clone, Debug)] +struct TrackTiming { + fps: u32, + offset: f64, +} + +#[derive(Clone, Debug)] +struct ClipTiming { + screen: TrackTiming, + camera: Option, + user_camera_offset: f64, +} + +#[derive(Clone, Copy, Debug)] +enum SceneRule { + HideCamera, + Unchecked, +} + +#[derive(Clone, Debug)] +struct Scene { + start: f64, + end: f64, + fade_in: f64, + fade_out: f64, + rule: SceneRule, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FrameExpectation { + pub screen: StreamExpectation, + pub camera: StreamExpectation, + /// (recording clip, recording time) the frame maps to, for reports. + pub source: Option<(usize, f64)>, +} + +impl FrameExpectation { + pub fn get(&self, kind: StreamKind) -> &StreamExpectation { + match kind { + StreamKind::Screen => &self.screen, + StreamKind::Camera => &self.camera, + } + } +} + +enum Mapping { + Single { + segment: usize, + source_time: f64, + }, + Transition { + outgoing: (usize, f64), + incoming: (usize, f64), + crossfade: bool, + progress: f64, + }, +} + +pub struct ReferenceModel { + segments: Vec, + output_starts: Vec, + clips: Vec, + truth: SourceTruth, + camera_hidden: bool, + scenes: Vec, + unsupported: Option, +} + +impl ReferenceModel { + pub fn new( + meta: &RecordingMeta, + project: &ProjectConfiguration, + truth: SourceTruth, + ) -> anyhow::Result { + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + anyhow::bail!("the model covers multi-segment studio projects"); + }; + let clips = inner + .segments + .iter() + .enumerate() + .map(|(i, segment)| { + let starts = [ + segment.display.start_time, + segment.camera.as_ref().and_then(|c| c.start_time), + segment.mic.as_ref().and_then(|m| m.start_time), + segment.system_audio.as_ref().and_then(|s| s.start_time), + ]; + let present = [ + true, + segment.camera.is_some(), + segment.mic.is_some(), + segment.system_audio.is_some(), + ]; + let all_known = starts + .iter() + .zip(present) + .all(|(start, present)| !present || start.is_some()); + let latest = all_known.then(|| { + starts + .iter() + .flatten() + .copied() + .fold(f64::NEG_INFINITY, f64::max) + }); + let align = |start: Option| match (latest, start) { + (Some(latest), Some(start)) => latest - start, + _ => 0.0, + }; + let clip = project.clips.iter().find(|c| c.index as usize == i); + let user_camera_offset = clip + .filter(|c| !c.offsets_auto_calculated) + .map_or(0.0, |c| f64::from(c.offsets.camera)); + ClipTiming { + screen: TrackTiming { + fps: segment.display.fps, + offset: align(segment.display.start_time), + }, + camera: segment.camera.as_ref().map(|c| TrackTiming { + fps: c.fps, + offset: align(c.start_time), + }), + user_camera_offset, + } + }) + .collect(); + let timeline = project + .timeline + .as_ref() + .ok_or_else(|| anyhow::anyhow!("project has no timeline"))?; + let mut segments: Vec = timeline + .segments + .iter() + .map(|s| ModelSegment { + clip: s.recording_clip as usize, + start: s.start, + end: s.end, + timescale: s.timescale, + transition_in: 0.0, + crossfade: false, + }) + .collect(); + for transition in &timeline.transitions { + let k = transition.segment_index as usize; + if k == 0 || k >= segments.len() { + continue; + } + let max = segments[k - 1].duration().min(segments[k].duration()) / 2.0; + if transition.duration.is_finite() && transition.duration > 0.0 && max >= MIN_TRANSITION + { + segments[k].transition_in = transition.duration.clamp(MIN_TRANSITION, max); + segments[k].crossfade = + matches!(transition.kind, cap_project::ClipTransitionType::CrossFade); + } + } + let mut output_starts = Vec::with_capacity(segments.len()); + let mut at = 0.0; + for (k, segment) in segments.iter().enumerate() { + output_starts.push(at); + let next_in = segments.get(k + 1).map_or(0.0, |n| n.transition_in); + at += segment.duration() - next_in; + } + let scenes = timeline + .scene_segments + .iter() + .filter(|s| !matches!(s.mode, SceneMode::Default)) + .map(|s| Scene { + start: s.start, + end: s.end, + fade_in: s.transition_in.max(0.0), + fade_out: s.transition_out.max(0.0), + rule: if matches!(s.mode, SceneMode::HideCamera) { + SceneRule::HideCamera + } else { + SceneRule::Unchecked + }, + }) + .collect(); + let unsupported = if timeline.text_segments.iter().any(|t| t.enabled) { + Some("text segments".to_string()) + } else if !timeline.zoom_segments.is_empty() { + Some("zoom segments".to_string()) + } else if !timeline.camera3d_segments.is_empty() { + Some("3D camera segments".to_string()) + } else { + None + }; + Ok(Self { + segments, + output_starts, + clips, + truth, + camera_hidden: project.camera.hide, + scenes, + unsupported, + }) + } + + pub fn duration(&self) -> f64 { + let total: f64 = self.segments.iter().map(ModelSegment::duration).sum(); + total - self.segments.iter().map(|s| s.transition_in).sum::() + } + + pub fn output_frames(&self, fps: u32) -> u32 { + (f64::from(fps) * self.duration()).ceil() as u32 + } + + fn map(&self, t: f64) -> Option { + for (k, segment) in self.segments.iter().enumerate() { + let start = self.output_starts[k]; + let next_in = self.segments.get(k + 1).map_or(0.0, |n| n.transition_in); + if t >= start && t < start + segment.transition_in { + let elapsed = t - start; + let prev = &self.segments[k - 1]; + let out_tick = prev.duration() - segment.transition_in + elapsed; + return Some(Mapping::Transition { + outgoing: (k - 1, prev.start + out_tick * prev.timescale), + incoming: (k, segment.start + elapsed * segment.timescale), + crossfade: segment.crossfade, + progress: (elapsed / segment.transition_in).clamp(0.0, 1.0), + }); + } + if t >= start + segment.transition_in && t < start + segment.duration() - next_in { + let elapsed = t - start; + if elapsed > segment.duration() { + return None; + } + return Some(Mapping::Single { + segment: k, + source_time: segment.start + elapsed * segment.timescale, + }); + } + } + None + } + + fn near_boundary(&self, t: f64) -> bool { + self.segments.iter().enumerate().any(|(k, segment)| { + let start = self.output_starts[k]; + let edges = [ + start, + start + segment.transition_in, + start + segment.duration(), + ]; + edges.iter().any(|e| (t - e).abs() < BOUNDARY_EPSILON) + }) + } + + fn track(&self, clip: usize, kind: StreamKind) -> Option<(&TrackTruth, &TrackTiming)> { + let truth = self.truth.segments.get(clip)?; + let timing = self.clips.get(clip)?; + match kind { + StreamKind::Screen => Some((&truth.screen, &timing.screen)), + StreamKind::Camera => Some((truth.camera.as_ref()?, timing.camera.as_ref()?)), + } + } + + /// Frames a decoder must answer for a request at recording time `time` + /// (before the track's own alignment offset). + pub fn answer(&self, clip: usize, kind: StreamKind, time: f64) -> StreamExpectation { + let Some((track, timing)) = self.track(clip, kind) else { + return StreamExpectation::Absent; + }; + let mut request = time + timing.offset; + if kind == StreamKind::Camera { + request += self.clips[clip].user_camera_offset; + } + let x = request.max(0.0) * f64::from(timing.fps); + let nearest = x.round(); + let slots: Vec = if (x - nearest).abs() < SLOT_EPSILON { + vec![nearest as i64 - 1, nearest as i64] + } else { + vec![x.floor() as i64] + }; + let numbers: Vec = track + .frames + .iter() + .map(|f| (f.pts_secs * f64::from(timing.fps)).round() as i64) + .collect(); + let mut allowed = Vec::new(); + for slot in slots { + let upto = numbers.partition_point(|&n| n <= slot); + let target = if upto == 0 { + numbers[0] + } else { + numbers[upto - 1] + }; + for (frame, &n) in track.frames.iter().zip(&numbers) { + if n == target && !allowed.contains(&frame.index) { + allowed.push(frame.index); + } + } + } + allowed.sort_unstable(); + StreamExpectation::Frame { + segment: track.segment, + allowed, + } + } + + fn scene_rule(&self, t: f64) -> Option<(SceneRule, bool)> { + self.scenes.iter().find_map(|scene| { + // A scene fades in before its start and out before its end. + if t < scene.start - scene.fade_in || t >= scene.end { + return None; + } + let fading = t < scene.start || t >= scene.end - scene.fade_out; + Some((scene.rule, fading)) + }) + } + + pub fn expect_at(&self, t: f64) -> FrameExpectation { + let unchecked = |reason: &str| FrameExpectation { + screen: StreamExpectation::Unchecked { + reason: reason.into(), + }, + camera: StreamExpectation::Unchecked { + reason: reason.into(), + }, + source: None, + }; + if let Some(reason) = &self.unsupported { + return unchecked(reason); + } + if self.near_boundary(t) { + return unchecked("on a timeline segment boundary"); + } + let (clip, source_time, mut screen, mut camera) = match self.map(t) { + Some(Mapping::Single { + segment, + source_time, + }) => { + let clip = self.segments[segment].clip; + let camera = if self.camera_hidden { + StreamExpectation::Absent + } else { + self.answer(clip, StreamKind::Camera, source_time) + }; + ( + clip, + source_time, + self.answer(clip, StreamKind::Screen, source_time), + camera, + ) + } + Some(Mapping::Transition { + outgoing, + incoming, + crossfade, + progress, + }) => { + if !crossfade { + return unchecked("fade-through-black transition"); + } + let (out_clip, in_clip) = ( + self.segments[outgoing.0].clip, + self.segments[incoming.0].clip, + ); + let blend = |kind: StreamKind| { + let a = self.answer(out_clip, kind, outgoing.1); + let b = self.answer(in_clip, kind, incoming.1); + match (a, b) { + ( + StreamExpectation::Frame { + segment: sa, + allowed: aa, + }, + StreamExpectation::Frame { + segment: sb, + allowed: ab, + }, + ) => StreamExpectation::Blend { + outgoing: (sa, aa), + incoming: (sb, ab), + progress, + }, + (StreamExpectation::Absent, StreamExpectation::Absent) => { + StreamExpectation::Absent + } + _ => StreamExpectation::Unchecked { + reason: "transition between camera and no camera".into(), + }, + } + }; + let camera = if self.camera_hidden { + StreamExpectation::Absent + } else { + blend(StreamKind::Camera) + }; + (in_clip, incoming.1, blend(StreamKind::Screen), camera) + } + None => return unchecked("past the end of the timeline"), + }; + match self.scene_rule(t) { + Some((_, true)) => { + camera = StreamExpectation::Unchecked { + reason: "scene fade".into(), + }; + } + Some((SceneRule::HideCamera, false)) => camera = StreamExpectation::Absent, + Some((SceneRule::Unchecked, false)) => { + screen = StreamExpectation::Unchecked { + reason: "scene layout".into(), + }; + camera = StreamExpectation::Unchecked { + reason: "scene layout".into(), + }; + } + None => {} + } + FrameExpectation { + screen, + camera, + source: Some((clip, source_time)), + } + } + + pub fn expect(&self, output_frame: u32, fps: u32) -> FrameExpectation { + self.expect_at(f64::from(output_frame) / f64::from(fps)) + } +} diff --git a/crates/verify/src/oracle.rs b/crates/verify/src/oracle.rs new file mode 100644 index 00000000000..80f1b11c485 --- /dev/null +++ b/crates/verify/src/oracle.rs @@ -0,0 +1,617 @@ +//! Compares decoded markers against the reference model's expectations. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::video_code::{Readout, StreamKind}; + +/// What one output frame must show for one stream. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "expect", rename_all = "lowercase")] +pub enum StreamExpectation { + /// The stream must show one of `allowed` (the first is canonical; more + /// than one only when the product's documented rule is ambiguous, e.g. a + /// request within float rounding of a frame boundary). + Frame { segment: u8, allowed: Vec }, + /// The stream must not be visible. + Absent, + /// A cross-fade between two source frames (a clip transition). Each side + /// lists its allowed indices like `Frame`. + Blend { + outgoing: (u8, Vec), + incoming: (u8, Vec), + /// Transition progress (0 = all outgoing). The measured blend weight + /// must be within `BLEND_PROGRESS_TOLERANCE` of it (the tolerance + /// covers blending in linear light, where the code's luma weight is + /// a gamma curve of the progress). + progress: f64, + }, + /// Not checked, with the reason. + Unchecked { reason: String }, +} + +impl StreamExpectation { + pub fn frame(segment: u8, index: u32) -> Self { + StreamExpectation::Frame { + segment, + allowed: vec![index], + } + } + + pub fn canonical(&self) -> Option<(u8, u32)> { + match self { + StreamExpectation::Frame { segment, allowed } => { + allowed.first().map(|&i| (*segment, i)) + } + StreamExpectation::Blend { outgoing, .. } => { + outgoing.1.first().map(|&i| (outgoing.0, i)) + } + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MismatchKind { + /// An older source frame than the model's. + Stale, + /// A source frame from the future. + Ahead, + /// A frame from a different recording segment. + WrongSegment, + /// The stream is visible but its code is damaged. + Unreadable, + /// The stream should be visible and is not. + Missing, + /// The stream should be hidden and is visible. + Unexpected, + /// A transition frame blends the right two frames with the wrong weight. + BlendWeight, +} + +/// Largest allowed difference between a transition frame's measured blend +/// weight and the transition progress. +pub const BLEND_PROGRESS_TOLERANCE: f64 = 0.1; + +/// Source frames searched on each side of a cross-fade for a pair that +/// explains the picture better than the expected one. +pub const BLEND_SEARCH: i64 = 12; + +/// Source frames searched on one side of a cross-fade with the other side +/// held at its expected frame (a reader left far behind or ahead). +pub const BLEND_SEARCH_ONE_SIDE: i64 = 240; + +/// How much better (in worst per-cell residual) another pair must fit than +/// the expected pair before the blend is read as that pair. +pub const BLEND_SIDE_MARGIN: f64 = 0.05; + +/// How many times smaller the other pair's residual must be. +pub const BLEND_SIDE_RATIO: f64 = 3.0; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Mismatch { + /// Position in the checked sequence (request number for seek runs). + pub position: usize, + pub output_frame: u32, + pub time_secs: f64, + pub kind: MismatchKind, + pub expected: Option<(u8, u32)>, + pub observed: Option<(u8, u32)>, + /// observed - expected, in source frames, for stale/ahead. + pub delta: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct StreamReport { + pub stream: Option, + pub checked: u32, + pub unchecked: u32, + pub stale: u32, + pub ahead: u32, + pub wrong_segment: u32, + pub unreadable: u32, + pub missing: u32, + pub unexpected: u32, + #[serde(default)] + pub blend_weight: u32, + pub max_ahead: i64, + pub max_stale: i64, + /// Source frames the model shows that never appeared. + pub skipped_source_frames: Vec, + /// Source frames shown on more output frames than the model says. + pub repeated_source_frames: Vec, + pub mismatches: Vec, +} + +impl StreamReport { + pub fn failures(&self) -> u32 { + self.stale + + self.ahead + + self.wrong_segment + + self.unreadable + + self.missing + + self.unexpected + + self.blend_weight + } + + pub fn passed(&self) -> bool { + self.failures() == 0 + } + + pub fn summary(&self) -> String { + let name = self.stream.map_or("stream", StreamKind::name); + if self.passed() { + return format!("{name}: {} frames correct", self.checked); + } + let mut parts = Vec::new(); + for (label, n) in [ + ("stale", self.stale), + ("ahead", self.ahead), + ("wrong segment", self.wrong_segment), + ("unreadable", self.unreadable), + ("missing", self.missing), + ("unexpected", self.unexpected), + ("blended with the wrong weight", self.blend_weight), + ] { + if n > 0 { + parts.push(format!("{n} {label}")); + } + } + let mut text = format!( + "{name}: {} of {} frames wrong ({})", + self.failures(), + self.checked, + parts.join(", ") + ); + if self.max_ahead > 0 { + text.push_str(&format!(", up to {} source frames ahead", self.max_ahead)); + } + if self.max_stale > 0 { + text.push_str(&format!(", up to {} stale", self.max_stale)); + } + if let Some(first) = self.mismatches.first() { + let shown: Vec = self + .mismatches + .iter() + .take(4) + .map(|m| match (&m.kind, &m.detail) { + (MismatchKind::BlendWeight, Some(detail)) => format!( + "#{} frame {} ({:.3}s) {detail}", + m.position, m.output_frame, m.time_secs + ), + _ => format!( + "#{} frame {} ({:.3}s) expected {} got {}", + m.position, + m.output_frame, + m.time_secs, + m.expected.map_or("-".into(), |(s, i)| format!("{s}:{i}")), + m.observed.map_or("-".into(), |(s, i)| format!("{s}:{i}")), + ), + }) + .collect(); + text.push_str(&format!("; {}", shown.join("; "))); + let _ = first; + } + text + } +} + +/// Checks one stream of a sequence of output frames. `observed[i]` is the +/// readout of output frame `frame_numbers[i]` (defaults to i). +pub fn check_stream( + stream: StreamKind, + expectations: &[StreamExpectation], + observed: &[Readout], + fps: u32, +) -> StreamReport { + let frames: Vec = (0..observed.len() as u32).collect(); + check_stream_at(stream, expectations, observed, &frames, fps) +} + +pub fn check_stream_at( + stream: StreamKind, + expectations: &[StreamExpectation], + observed: &[Readout], + frame_numbers: &[u32], + fps: u32, +) -> StreamReport { + let mut report = StreamReport { + stream: Some(stream), + ..Default::default() + }; + let mut expected_counts: BTreeMap<(u8, u32), u32> = BTreeMap::new(); + let mut observed_counts: BTreeMap<(u8, u32), u32> = BTreeMap::new(); + for (position, ((expectation, readout), &frame)) in expectations + .iter() + .zip(observed) + .zip(frame_numbers) + .enumerate() + { + let time_secs = f64::from(frame) / f64::from(fps); + let push = |report: &mut StreamReport, + kind: MismatchKind, + observed: Option<(u8, u32)>, + delta: Option, + detail: Option| { + report.mismatches.push(Mismatch { + position, + output_frame: frame, + time_secs, + kind, + expected: expectation.canonical(), + observed, + delta, + detail, + }); + }; + match expectation { + StreamExpectation::Unchecked { .. } => { + report.unchecked += 1; + continue; + } + StreamExpectation::Absent => { + report.checked += 1; + match readout { + Readout::Absent => {} + Readout::Marker { marker, .. } => { + report.unexpected += 1; + push( + &mut report, + MismatchKind::Unexpected, + Some((marker.segment, marker.index)), + None, + None, + ); + } + Readout::Unreadable { reason } => { + report.unexpected += 1; + push( + &mut report, + MismatchKind::Unexpected, + None, + None, + Some(reason.clone()), + ); + } + Readout::Blend { incoming, .. } => { + report.unexpected += 1; + push( + &mut report, + MismatchKind::Unexpected, + Some((incoming.segment, incoming.index)), + None, + Some("blend".into()), + ); + } + } + } + StreamExpectation::Blend { + outgoing, + incoming, + progress: expected_progress, + } => { + report.checked += 1; + if let Readout::Blend { + progress: observed, .. + } = readout + && (observed - expected_progress).abs() > BLEND_PROGRESS_TOLERANCE + { + report.blend_weight += 1; + report.mismatches.push(Mismatch { + position, + output_frame: frame, + time_secs, + kind: MismatchKind::BlendWeight, + expected: None, + observed: None, + delta: None, + detail: Some(format!( + "transition blend weight {observed:.2}, expected progress {expected_progress:.2}" + )), + }); + } + let sides: Vec<(u8, u32)> = match readout { + Readout::Blend { + outgoing: o, + incoming: i, + .. + } => vec![(o.segment, o.index), (i.segment, i.index)], + Readout::Marker { marker, .. } => vec![(marker.segment, marker.index)], + Readout::Absent => { + report.missing += 1; + push(&mut report, MismatchKind::Missing, None, None, None); + continue; + } + Readout::Unreadable { reason } => { + report.unreadable += 1; + push( + &mut report, + MismatchKind::Unreadable, + None, + None, + Some(reason.clone()), + ); + continue; + } + }; + let fits = |got: (u8, u32), side: &(u8, Vec)| { + got.0 == side.0 && side.1.contains(&got.1) + }; + let ok = match sides.as_slice() { + [o, i] => fits(*o, outgoing) && fits(*i, incoming), + [one] => fits(*one, outgoing) || fits(*one, incoming), + _ => false, + }; + if !ok { + let (got, side) = match sides.as_slice() { + [o, _] if !fits(*o, outgoing) => (*o, outgoing), + [_, i] => (*i, incoming), + [one] => (*one, incoming), + _ => continue, + }; + let reference = i64::from(*side.1.first().unwrap_or(&0)); + let delta = i64::from(got.1) - reference; + let kind = if got.0 != side.0 { + report.wrong_segment += 1; + MismatchKind::WrongSegment + } else if delta < 0 { + report.stale += 1; + report.max_stale = report.max_stale.max(-delta); + MismatchKind::Stale + } else { + report.ahead += 1; + report.max_ahead = report.max_ahead.max(delta); + MismatchKind::Ahead + }; + report.mismatches.push(Mismatch { + position, + output_frame: frame, + time_secs, + kind, + expected: Some((side.0, reference as u32)), + observed: Some(got), + delta: Some(delta), + detail: Some( + if std::ptr::eq(side, incoming) { + "transition, incoming side" + } else { + "transition, outgoing side" + } + .into(), + ), + }); + } + } + StreamExpectation::Frame { segment, allowed } => { + report.checked += 1; + let canonical = (*segment, allowed[0]); + *expected_counts.entry(canonical).or_default() += 1; + match readout { + Readout::Absent => { + report.missing += 1; + push(&mut report, MismatchKind::Missing, None, None, None); + } + Readout::Blend { .. } => { + report.unreadable += 1; + push( + &mut report, + MismatchKind::Unreadable, + None, + None, + Some("blend where a single frame was expected".into()), + ); + } + Readout::Unreadable { reason } => { + report.unreadable += 1; + push( + &mut report, + MismatchKind::Unreadable, + None, + None, + Some(reason.clone()), + ); + } + Readout::Marker { marker, .. } => { + let got = (marker.segment, marker.index); + if marker.segment != *segment { + *observed_counts.entry(got).or_default() += 1; + report.wrong_segment += 1; + push( + &mut report, + MismatchKind::WrongSegment, + Some(got), + None, + None, + ); + } else if allowed.contains(&marker.index) { + *observed_counts.entry(canonical).or_default() += 1; + } else { + *observed_counts.entry(got).or_default() += 1; + let lo = *allowed.iter().min().unwrap_or(&0); + let hi = *allowed.iter().max().unwrap_or(&0); + if marker.index < lo { + let delta = i64::from(marker.index) - i64::from(lo); + report.stale += 1; + report.max_stale = report.max_stale.max(-delta); + push( + &mut report, + MismatchKind::Stale, + Some(got), + Some(delta), + None, + ); + } else { + let delta = i64::from(marker.index) - i64::from(hi); + report.ahead += 1; + report.max_ahead = report.max_ahead.max(delta); + push( + &mut report, + MismatchKind::Ahead, + Some(got), + Some(delta), + None, + ); + } + } + } + } + } + } + } + report.skipped_source_frames = expected_counts + .iter() + .filter(|(key, _)| !observed_counts.contains_key(key)) + .map(|((_, index), _)| *index) + .collect(); + report.repeated_source_frames = observed_counts + .iter() + .filter(|(key, n)| **n > expected_counts.get(key).copied().unwrap_or(0)) + .map(|((_, index), _)| *index) + .collect(); + report +} + +/// Reads one stream of a picture knowing what the model expects there: a +/// plain decode, or for a cross-fade the best-fitting blend of the expected +/// outgoing and incoming codes (searching nearby frames when none fits, so +/// a wrong answer is reported as such rather than as unreadable). +pub fn read_expecting( + img: &impl crate::video_code::PixelSource, + kind: StreamKind, + expectation: &StreamExpectation, +) -> Readout { + use crate::video_code::{self, BLEND_TOLERANCE, FrameMarker}; + let StreamExpectation::Blend { + outgoing, + incoming, + progress: expected_progress, + } = expectation + else { + return video_code::read_stream(img, kind); + }; + let cells = match video_code::read_cells(img, kind) { + Ok(cells) => cells, + Err(readout) => return readout, + }; + let best_of = |pairs: &mut dyn Iterator| { + pairs + .map(|(o, i)| { + let (p, residual) = video_code::blend_fit(&cells, o, i); + (o, i, p, residual) + }) + .min_by(|a, b| a.3.total_cmp(&b.3)) + }; + let marker = |seg: u8, index: i64| { + (0..=i64::from(video_code::MAX_INDEX)) + .contains(&index) + .then(|| FrameMarker::new(kind, seg, index as u32)) + }; + let expected = best_of(&mut outgoing.1.iter().flat_map(|&o| { + incoming.1.iter().filter_map(move |&i| { + Some(( + marker(outgoing.0, i64::from(o))?, + marker(incoming.0, i64::from(i))?, + )) + }) + })); + if let Some((o, i, p, residual)) = expected + && residual <= BLEND_TOLERANCE + { + // Both sides are verified: a wrong frame on the weaker side leaves + // residuals of only about its weight, which the tolerance alone would + // accept. Any other pair of nearby frames that explains the picture + // clearly better than the expected pair is what the stream shows. + let allowed = |side: &(u8, Vec), m: FrameMarker| { + m.segment == side.0 && side.1.contains(&m.index) + }; + let (o0, i0) = (i64::from(o.index), i64::from(i.index)); + let near = (-BLEND_SEARCH..=BLEND_SEARCH).flat_map(|d_o| { + (-BLEND_SEARCH..=BLEND_SEARCH).filter_map(move |d_i| { + Some((marker(o.segment, o0 + d_o)?, marker(i.segment, i0 + d_i)?)) + }) + }); + // One side wrong by far (a stale reader position) with the other + // side right. + let far = (-BLEND_SEARCH_ONE_SIDE..=BLEND_SEARCH_ONE_SIDE).flat_map(|d| { + [ + marker(o.segment, o0 + d).map(|ao| (ao, i)), + marker(i.segment, i0 + d).map(|ai| (o, ai)), + ] + .into_iter() + .flatten() + }); + // The same two frames with the sides swapped (weight 1 - p) are the + // same picture, not an alternative: a cross-fade inside one clip has + // both sides in the same segment. + let swapped = |ao: FrameMarker, ai: FrameMarker| ao == i && ai == o; + let alternative = best_of(&mut near.chain(far).filter(|(ao, ai)| { + !(swapped(*ao, *ai) || allowed(outgoing, *ao) && allowed(incoming, *ai)) + })); + if std::env::var("CAP_VERIFY_TRACE_BLEND").as_deref() == Ok("2") { + eprintln!( + "[cap-verify-blend-sides] {}", + serde_json::json!({ + "stream": kind, + "expected": [o.index, i.index], + "p": p, + "progress": expected_progress, + "residual": residual, + "alternative": alternative.map(|(ao, ai, ap, ares)| (ao.index, ai.index, ap, ares)), + }) + ); + } + // Clearly better, absolutely and relative to the expected pair's + // residual: on a noisy stream (the Linux camera code, residuals 0.13 + // to 0.17 on correct blends) a neighbouring pair can fit a little + // better by chance; a wrong side leaves the expected pair with a + // residual several times the alternative's. + if let Some((ao, ai, ap, ares)) = alternative + && ares + BLEND_SIDE_MARGIN < residual + && ares * BLEND_SIDE_RATIO < residual + { + return Readout::Blend { + outgoing: ao, + incoming: ai, + progress: ap, + residual: ares, + }; + } + return Readout::Blend { + outgoing: o, + incoming: i, + progress: p, + residual, + }; + } + let single = video_code::read_stream(img, kind); + if single.marker().is_some() { + return single; + } + let (o0, i0) = ( + i64::from(*outgoing.1.first().unwrap_or(&0)), + i64::from(*incoming.1.first().unwrap_or(&0)), + ); + let nearby = best_of(&mut (-12i64..=12).flat_map(|d_o| { + (-12i64..=12).filter_map(move |d_i| { + Some((marker(outgoing.0, o0 + d_o)?, marker(incoming.0, i0 + d_i)?)) + }) + })); + match nearby { + Some((o, i, p, residual)) if residual <= BLEND_TOLERANCE => Readout::Blend { + outgoing: o, + incoming: i, + progress: p, + residual, + }, + _ => Readout::Unreadable { + reason: format!( + "transition blend matches no expected pair (best residual {:.2})", + expected.map_or(f64::NAN, |e| e.3) + ), + }, + } +} diff --git a/crates/verify/src/pipeline.rs b/crates/verify/src/pipeline.rs new file mode 100644 index 00000000000..b1fa13003cf --- /dev/null +++ b/crates/verify/src/pipeline.rs @@ -0,0 +1,894 @@ +//! Fixtures recorded through the real recording pipeline. +//! +//! Synthetic sources (marker frames, marker audio) are fed through the same +//! channel sources the sync matrix uses into `OutputPipeline` with the +//! product's muxers and encoders (`Mp4Muxer` for screen and camera, +//! `OggMuxer` for microphone and system audio). Frames are emitted in real +//! time because the pipeline pins video timestamps to the wall clock, so a +//! fixture costs its duration; keep them short. The timing that lands in +//! the files is whatever the pipeline produced: tests read the truth back +//! from the files ([`crate::fixture::SourceTruth`]), never from the spec. + +use std::{path::Path, time::Duration}; + +use anyhow::{Context, anyhow}; +use cap_media_info::{AudioInfo, RawVideoFormat, Sample, Type, VideoInfo}; +use cap_project::{AudioMeta, MultipleSegment, VideoMeta}; +use cap_recording::{ + AudioFrame, ChannelAudioSource, ChannelAudioSourceConfig, ChannelVideoSource, + ChannelVideoSourceConfig, OutputPipeline, + ffmpeg::{FFmpegVideoFrame, Mp4Muxer, OggMuxer}, +}; +use cap_timestamp::{Timestamp, Timestamps}; +use relative_path::RelativePathBuf; +use serde::{Deserialize, Serialize}; + +use crate::{ + audio_code::{self, AudioSource}, + fixture::{Rng, segment_dir, write_project_files}, + video_code::{self, FrameMarker, PixelOrder, StreamKind}, +}; + +const AUDIO_RATE: u32 = 48_000; +const AUDIO_CHUNK_SECS: f64 = 0.02; + +/// How a synthetic video device delivers frames. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub enum Delivery { + /// Every slot of the nominal rate. + Steady, + /// No frames inside the (start, length) windows: a static screen. + Holds(Vec<(f64, f64)>), + /// Slots skipped at random (1..=3 slots apart), like a camera in low + /// light. + Jittery { seed: u64 }, +} + +impl Delivery { + /// The slots a device with this delivery emits over `duration` seconds: + /// the generator's truth for what a recording must contain. + pub fn slots(&self, fps: u32, duration: f64) -> Vec { + let end = (duration * f64::from(fps)) as u32; + match self { + Delivery::Steady => (0..end).collect(), + Delivery::Holds(holds) => (0..end) + .filter(|&k| { + let t = f64::from(k) / f64::from(fps); + !holds.iter().any(|&(s, l)| t > s && t < s + l) + }) + .collect(), + Delivery::Jittery { seed } => { + let mut rng = Rng::new(*seed); + let mut out = Vec::new(); + let mut k = 0; + while k < end { + out.push(k); + k += rng.range(1, 3) as u32; + } + out + } + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct RecordedVideo { + pub width: u32, + pub height: u32, + pub fps: u32, + pub delivery: Delivery, + /// Seconds after the segment starts before the device delivers. + pub start_delay: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct RecordedSegment { + pub duration: f64, + pub screen: RecordedVideo, + pub camera: Option, + /// Microphone starting this many seconds in (continuous delivery). + pub mic_delay: Option, + /// System audio delivered loopback-style (packets only while a marker + /// plays; the recorder synthesises the silence between). + pub system_audio: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct RecordedSpec { + pub name: String, + pub segments: Vec, +} + +impl RecordedSpec { + /// Key for the fixture cache: the spec plus the recording code that made + /// the fixture ([`product_source_hash`]). + pub fn cache_key(&self) -> String { + crate::fixture::cache_key(&self.name, &(self, product_source_hash())) + } +} + +/// Workspace crates whose code shapes a recorded fixture (capture pipeline, +/// muxers, encoders, timestamps, audio conversion). +const RECORDING_SOURCE_CRATES: &[&str] = &[ + "recording", + "cap-muxer", + "cap-muxer-protocol", + "enc-ffmpeg", + "enc-avfoundation", + "enc-mediafoundation", + "media-info", + "media", + "timestamp", + "audio", + "frame-converter", + "project", +]; + +/// Hash of the product sources that recorded fixtures are made by. Recorded +/// fixtures are product output: keying them by the spec alone would let a +/// cached fixture recorded by older code (the CI fixture cache is keyed by +/// the generator sources only) hide a regression in the recording pipeline. +/// With this in the key, any change to those crates, to Cargo.lock or to +/// this recorder harness re-records the fixtures. +pub fn product_source_hash() -> u64 { + static HASH: std::sync::OnceLock = std::sync::OnceLock::new(); + *HASH.get_or_init(|| { + fn collect(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if path.file_name().is_some_and(|n| n != "target") { + collect(&path, out); + } + } else if path + .extension() + .is_some_and(|e| e == "rs" || e == "toml" || e == "wgsl" || e == "metal") + { + out.push(path); + } + } + } + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + let root = manifest + .join("../..") + .canonicalize() + .unwrap_or_else(|_| manifest.join("../..")); + let mut files = Vec::new(); + for name in RECORDING_SOURCE_CRATES { + collect(&root.join("crates").join(name), &mut files); + } + files.push(root.join("Cargo.lock")); + files.push(root.join("crates/verify/src/pipeline.rs")); + files.sort(); + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + let mut feed = |bytes: &[u8]| { + for &byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + }; + for file in &files { + let Ok(contents) = std::fs::read(file) else { + continue; + }; + feed( + file.strip_prefix(&root) + .unwrap_or(file) + .to_string_lossy() + .as_bytes(), + ); + feed(&contents); + } + hash + }) +} + +fn bgra_frame(marker: FrameMarker, width: u32, height: u32) -> ffmpeg::frame::Video { + let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, width, height); + let stride = frame.stride(0); + video_code::draw_marker( + marker, + frame.data_mut(0), + width as usize, + height as usize, + stride, + PixelOrder::Bgra, + ); + frame +} + +/// Marker track for a whole segment, mono, sample 0 = segment start. +fn marker_track(source: AudioSource, duration: f64) -> Vec { + let mut mono = vec![0.0f32; (duration * f64::from(AUDIO_RATE)) as usize]; + let schedule = audio_code::standard_schedule(source, 0.1, duration - 0.1, 0.5); + audio_code::render_markers( + &mut mono, + AUDIO_RATE, + source, + &schedule, + audio_code::DEFAULT_AMPLITUDE, + ); + mono +} + +fn audio_chunk(track: &[f32], chunk_t: f64, info: &AudioInfo) -> ffmpeg::frame::Audio { + let frames = (f64::from(AUDIO_RATE) * AUDIO_CHUNK_SECS) as usize; + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + frames, + info.channel_layout(), + ); + frame.set_rate(AUDIO_RATE); + let start = (chunk_t * f64::from(AUDIO_RATE)).round() as usize; + let channels = usize::from(info.channels as u16); + let data = frame.data_mut(0); + for (i, sample) in data.chunks_exact_mut(4 * channels).enumerate() { + let v = track.get(start + i).copied().unwrap_or(0.0); + for ch in sample.chunks_exact_mut(4) { + ch.copy_from_slice(&v.to_le_bytes()); + } + } + frame +} + +/// Emitters wait for this before their first frame, so every device starts +/// delivering only once all pipelines are built. Emitting from the pipeline +/// epoch instead queued the frames of a slow setup and delivered them as a +/// burst, which the recorder (rightly) treats as stale overlap and drops. +type StartGate = tokio::sync::watch::Receiver>; + +async fn started(mut gate: StartGate) -> std::time::Instant { + loop { + if let Some(base) = *gate.borrow() { + return base; + } + if gate.changed().await.is_err() { + return std::time::Instant::now(); + } + } +} + +struct VideoLeg { + pipeline: OutputPipeline, + emit: tokio::task::JoinHandle<()>, + rel: RelativePathBuf, + fps: u32, +} + +#[allow(clippy::too_many_arguments)] +async fn video_leg( + dir: &Path, + rel: RelativePathBuf, + kind: StreamKind, + segment: u8, + video: &RecordedVideo, + duration: f64, + timestamps: Timestamps, + start: StartGate, +) -> anyhow::Result { + let info = VideoInfo::from_raw(RawVideoFormat::Bgra, video.width, video.height, video.fps); + let (tx, rx) = flume::bounded::(32); + let slots = video + .delivery + .slots(video.fps, duration - video.start_delay); + let (fps, width, height, delay) = (video.fps, video.width, video.height, video.start_delay); + let emit = tokio::spawn(async move { + let base = started(start).await; + for slot in slots { + let t = delay + f64::from(slot) / f64::from(fps); + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = FFmpegVideoFrame { + inner: bgra_frame(FrameMarker::new(kind, segment, slot), width, height), + timestamp: Timestamp::Instant(at), + }; + if tx.send_async(frame).await.is_err() { + break; + } + } + }); + let pipeline = OutputPipeline::builder(dir.join(rel.as_str())) + .with_video::>(ChannelVideoSourceConfig::new(info, rx)) + .with_timestamps(timestamps) + .build::(()) + .await + .map_err(|e| anyhow!("{kind:?} pipeline: {e}"))?; + Ok(VideoLeg { + pipeline, + emit, + rel, + fps: video.fps, + }) +} + +type AudioEmit = tokio::task::JoinHandle>; + +async fn record_segment( + dir: &Path, + index: usize, + spec: &RecordedSegment, +) -> anyhow::Result { + let seg = segment_dir(index); + std::fs::create_dir_all(dir.join(&seg))?; + let rel = |file: &str| RelativePathBuf::from(format!("{seg}/{file}")); + let timestamps = Timestamps::now(); + let (start_tx, start) = tokio::sync::watch::channel(None); + + let screen = video_leg( + dir, + rel("display.mp4"), + StreamKind::Screen, + index as u8, + &spec.screen, + spec.duration, + timestamps, + start.clone(), + ) + .await?; + let camera = match &spec.camera { + Some(camera) => Some( + video_leg( + dir, + rel("camera.mp4"), + StreamKind::Camera, + index as u8, + camera, + spec.duration, + timestamps, + start.clone(), + ) + .await?, + ), + None => None, + }; + + let mono_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 1) + .map_err(|e| anyhow!("audio info: {e:?}"))?; + let stereo_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 2) + .map_err(|e| anyhow!("audio info: {e:?}"))?; + + let mut mic: Option<(OutputPipeline, AudioEmit, RelativePathBuf)> = None; + if let Some(delay) = spec.mic_delay { + let track = marker_track(AudioSource::Mic, spec.duration); + let (mut tx, rx) = futures::channel::mpsc::channel::(32); + let duration = spec.duration; + let gate = start.clone(); + let emit = tokio::spawn(async move { + use futures::SinkExt; + let base = started(gate).await; + let first = (delay / AUDIO_CHUNK_SECS).ceil() as usize; + let last = (duration / AUDIO_CHUNK_SECS) as usize; + for k in first..last { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = + AudioFrame::new(audio_chunk(&track, t, &mono_info), Timestamp::Instant(at)); + if tx.send(frame).await.is_err() { + break; + } + } + tx + }); + let pipeline = OutputPipeline::builder(dir.join(rel("audio-input.ogg").as_str())) + .with_audio_source::(ChannelAudioSourceConfig::new(mono_info, rx)) + .with_timestamps(timestamps) + .build::(()) + .await + .map_err(|e| anyhow!("mic pipeline: {e}"))?; + mic = Some((pipeline, emit, rel("audio-input.ogg"))); + } + + let mut system: Option<(OutputPipeline, AudioEmit, RelativePathBuf)> = None; + if spec.system_audio { + let track = marker_track(AudioSource::System, spec.duration); + let schedule = + audio_code::standard_schedule(AudioSource::System, 0.1, spec.duration - 0.1, 0.5); + let (mut tx, rx) = futures::channel::mpsc::channel::(32); + let gate = start.clone(); + let emit = tokio::spawn(async move { + use futures::SinkExt; + let base = started(gate).await; + for placement in schedule { + let first = (placement.time_secs / AUDIO_CHUNK_SECS).floor() as usize; + let last = ((placement.time_secs + audio_code::MARKER_SECS) / AUDIO_CHUNK_SECS) + .ceil() as usize; + for k in first..last { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = AudioFrame::new( + audio_chunk(&track, t, &stereo_info), + Timestamp::Instant(at), + ); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx + }); + let pipeline = OutputPipeline::builder(dir.join(rel("system_audio.ogg").as_str())) + .with_audio_source::(ChannelAudioSourceConfig::new(stereo_info, rx)) + .with_timestamps(timestamps) + .with_audio_anchor(cap_recording::AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| anyhow!("system audio pipeline: {e}"))?; + system = Some((pipeline, emit, rel("system_audio.ogg"))); + } + + let _ = start_tx.send(Some(std::time::Instant::now() + Duration::from_millis(50))); + + // Wait for every device to finish delivering, keeping the audio senders + // open (real devices deliver until stop), then stop like the recorder. + screen.emit.await.context("screen emit")?; + let camera = match camera { + Some(leg) => { + let VideoLeg { + pipeline, + emit, + rel, + fps, + } = leg; + emit.await.context("camera emit")?; + Some((pipeline, rel, fps)) + } + None => None, + }; + let mut held = Vec::new(); + let mut audio_pipelines = Vec::new(); + for (name, leg) in [("mic", mic), ("system", system)] { + if let Some((pipeline, emit, rel)) = leg { + held.push(emit.await.context("audio emit")?); + audio_pipelines.push((name, pipeline, rel)); + } + } + tokio::time::sleep(Duration::from_millis(300)).await; + + let finished_screen = screen + .pipeline + .stop() + .await + .map_err(|e| anyhow!("screen stop: {e}"))?; + let start = |first: Timestamp| first.signed_duration_since_secs(timestamps); + let display = VideoMeta { + path: screen.rel, + fps: screen.fps, + start_time: Some(start(finished_screen.first_timestamp)), + device_id: None, + }; + let camera = match camera { + Some((pipeline, rel, fps)) => { + let finished = pipeline + .stop() + .await + .map_err(|e| anyhow!("camera stop: {e}"))?; + Some(VideoMeta { + path: rel, + fps, + start_time: Some(start(finished.first_timestamp)), + device_id: None, + }) + } + None => None, + }; + let mut mic_meta = None; + let mut system_meta = None; + for (name, pipeline, rel) in audio_pipelines { + let finished = pipeline + .stop() + .await + .map_err(|e| anyhow!("{name} stop: {e}"))?; + let meta = AudioMeta { + path: rel, + start_time: Some(start(finished.first_timestamp)), + device_id: None, + gap_summary: finished + .audio_gap_summary + .map(|s| cap_project::AudioGapSummary { + total_overlap_trimmed_ms: s.total_overlap_trimmed_ms, + startup_overlap_trimmed_ms: s.startup_overlap_trimmed_ms, + overlap_dropped_frames: s.overlap_dropped_frames, + startup_overlap_drops: s.startup_overlap_drops, + }), + }; + if name == "mic" { + mic_meta = Some(meta); + } else { + system_meta = Some(meta); + } + } + drop(held); + Ok(MultipleSegment { + display, + camera, + mic: mic_meta, + system_audio: system_meta, + cursor: None, + keyboard: None, + display_notch: None, + }) +} + +/// The emitters sleep until each frame's time. Windows wakes timers on its +/// default 15.6 ms tick, which by itself makes frames up to a tick late +/// (more under load); the product raises the timer resolution the same way +/// for playback and export. +fn high_resolution_timer() { + #[cfg(windows)] + { + #[link(name = "winmm")] + unsafe extern "system" { + fn timeBeginPeriod(period: u32) -> u32; + } + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + assert_eq!( + unsafe { timeBeginPeriod(1) }, + 0, + "set Windows timer resolution" + ); + }); + } +} + +pub async fn record_studio(spec: &RecordedSpec, dir: &Path) -> anyhow::Result<()> { + high_resolution_timer(); + crate::media::init(); + if dir.exists() { + std::fs::remove_dir_all(dir)?; + } + let mut segments = Vec::new(); + for (i, segment) in spec.segments.iter().enumerate() { + segments.push(record_segment(dir, i, segment).await?); + } + write_project_files(dir, &spec.name, segments)?; + Ok(()) +} + +/// An Instant recording: segmented screen video and one mixed audio track +/// (microphone and system audio) through the Instant muxers, finalised into +/// `content/output.mp4` by the product's `finalize_instant_output`. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct InstantSpec { + pub name: String, + pub duration: f64, + pub screen: RecordedVideo, + pub mic_delay: Option, + pub system_audio: bool, + /// System audio delivered continuously from the start (macOS capture) + /// instead of loopback-style packets only while a marker plays. + #[serde(default)] + pub system_continuous: bool, + /// Arm a recording start gate this many seconds after the sources start + /// (the app's start cue); `None` records from the first frame. + #[serde(default)] + pub arm_at: Option, +} + +impl InstantSpec { + /// Key for the fixture cache, including [`product_source_hash`]. + pub fn cache_key(&self) -> String { + crate::fixture::cache_key( + &format!("instant-{}", self.name), + &(self, product_source_hash()), + ) + } +} + +/// Where each marker was emitted, on the recording clock (seconds after the +/// pipeline epoch). Written next to the project as `verify-emitted.json`. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Emitted { + /// (slot, emit time) of every screen frame. + pub screen: Vec<(u32, f64)>, + pub fps: u32, + /// (source, marker index, chirp time) of every audio marker. + pub audio: Vec<(AudioSource, u32, f64)>, + /// (slot, time the frame was actually handed to the pipeline), to tell a + /// late emitter (a starved test runtime) from late processing. + #[serde(default)] + pub screen_sent: Vec<(u32, f64)>, +} + +pub async fn record_instant(spec: &InstantSpec, dir: &Path) -> anyhow::Result<()> { + high_resolution_timer(); + use cap_recording::{DashSegmentedAudioMuxer, DashSegmentedAudioMuxerConfig}; + crate::media::init(); + if dir.exists() { + std::fs::remove_dir_all(dir)?; + } + let content = dir.join("content"); + std::fs::create_dir_all(&content)?; + let timestamps = Timestamps::now(); + let base = timestamps.instant(); + let gate = spec + .arm_at + .map(|_| cap_recording::RecordingStartGate::new()); + let mut emitted = Emitted { + fps: spec.screen.fps, + ..Default::default() + }; + + let video = &spec.screen; + let info = VideoInfo::from_raw(RawVideoFormat::Bgra, video.width, video.height, video.fps); + let (tx, rx) = flume::bounded::(32); + let slots = video + .delivery + .slots(video.fps, spec.duration - video.start_delay); + for &slot in &slots { + emitted.screen.push(( + slot, + video.start_delay + f64::from(slot) / f64::from(video.fps), + )); + } + let (fps, width, height, delay) = (video.fps, video.width, video.height, video.start_delay); + let video_emit = tokio::spawn(async move { + let mut sent = Vec::new(); + for slot in slots { + let t = delay + f64::from(slot) / f64::from(fps); + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = FFmpegVideoFrame { + inner: bgra_frame(FrameMarker::new(StreamKind::Screen, 0, slot), width, height), + timestamp: Timestamp::Instant(at), + }; + if tx.send_async(frame).await.is_err() { + break; + } + sent.push((slot, base.elapsed().as_secs_f64())); + } + sent + }); + let mut video_builder = OutputPipeline::builder(content.join("display")) + .with_video::>(ChannelVideoSourceConfig::new(info, rx)) + .with_timestamps(timestamps); + if let Some(gate) = &gate { + video_builder = video_builder.with_start_gate(Some(gate.clone())); + } + let video_pipeline = video_builder + .build::( + cap_recording::ffmpeg::SegmentedVideoMuxerConfig { + segment_duration: Duration::from_secs(2), + ..Default::default() + }, + ) + .await + .map_err(|e| anyhow!("instant video pipeline: {e}"))?; + + let mono_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 1) + .map_err(|e| anyhow!("audio info: {e:?}"))?; + let stereo_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 2) + .map_err(|e| anyhow!("audio info: {e:?}"))?; + let mut audio_emits: Vec = Vec::new(); + let mut builder = OutputPipeline::builder(content.join("audio")).with_timestamps(timestamps); + if let Some(gate) = &gate { + builder = builder.with_start_gate(Some(gate.clone())); + } + // Mirrors the product's Instant audio pipeline, which anchors at the + // recording epoch on Linux only. + if cfg!(target_os = "linux") { + builder = builder.with_audio_anchor(cap_recording::AudioAnchor::PipelineEpoch); + } + let has_audio = spec.mic_delay.is_some() || spec.system_audio; + if spec.system_audio { + let track = marker_track(AudioSource::System, spec.duration); + let schedule = + audio_code::standard_schedule(AudioSource::System, 0.1, spec.duration - 0.1, 0.5); + emitted.audio.extend( + schedule + .iter() + .map(|p| (AudioSource::System, p.index, p.time_secs)), + ); + let (mut tx, rx) = futures::channel::mpsc::channel::(32); + let continuous = spec.system_continuous; + let duration = spec.duration; + audio_emits.push(tokio::spawn(async move { + use futures::SinkExt; + if continuous { + for k in 0..(duration / AUDIO_CHUNK_SECS) as usize { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = AudioFrame::new( + audio_chunk(&track, t, &stereo_info), + Timestamp::Instant(at), + ); + if tx.send(frame).await.is_err() { + break; + } + } + return tx; + } + for placement in schedule { + let first = (placement.time_secs / AUDIO_CHUNK_SECS).floor() as usize; + let last = ((placement.time_secs + audio_code::MARKER_SECS) / AUDIO_CHUNK_SECS) + .ceil() as usize; + for k in first..last { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = AudioFrame::new( + audio_chunk(&track, t, &stereo_info), + Timestamp::Instant(at), + ); + if tx.send(frame).await.is_err() { + return tx; + } + } + } + tx + })); + builder = builder.with_audio_source::(ChannelAudioSourceConfig::new( + stereo_info, + rx, + )); + } + if let Some(delay) = spec.mic_delay { + let track = marker_track(AudioSource::Mic, spec.duration); + let schedule = + audio_code::standard_schedule(AudioSource::Mic, 0.1, spec.duration - 0.1, 0.5); + emitted.audio.extend( + schedule + .iter() + .filter(|p| p.time_secs >= delay) + .map(|p| (AudioSource::Mic, p.index, p.time_secs)), + ); + let (mut tx, rx) = futures::channel::mpsc::channel::(32); + let duration = spec.duration; + audio_emits.push(tokio::spawn(async move { + use futures::SinkExt; + let first = (delay / AUDIO_CHUNK_SECS).ceil() as usize; + let last = (duration / AUDIO_CHUNK_SECS) as usize; + for k in first..last { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = base + Duration::from_secs_f64(t); + tokio::time::sleep_until(at.into()).await; + let frame = + AudioFrame::new(audio_chunk(&track, t, &mono_info), Timestamp::Instant(at)); + if tx.send(frame).await.is_err() { + break; + } + } + tx + })); + builder = builder + .with_audio_source::(ChannelAudioSourceConfig::new(mono_info, rx)); + } + let audio_pipeline = if has_audio { + Some( + builder + .build::(DashSegmentedAudioMuxerConfig::default()) + .await + .map_err(|e| anyhow!("instant audio pipeline: {e}"))?, + ) + } else { + None + }; + + if let (Some(gate), Some(arm_at)) = (&gate, spec.arm_at) { + tokio::time::sleep_until((base + Duration::from_secs_f64(arm_at)).into()).await; + gate.arm(); + } + emitted.screen_sent = video_emit.await.context("video emit")?; + let mut held = Vec::new(); + for emit in audio_emits { + held.push(emit.await.context("audio emit")?); + } + tokio::time::sleep(Duration::from_millis(300)).await; + video_pipeline + .stop() + .await + .map_err(|e| anyhow!("instant video stop: {e}"))?; + if let Some(audio) = audio_pipeline { + audio + .stop() + .await + .map_err(|e| anyhow!("instant audio stop: {e}"))?; + } + drop(held); + let output = content.join("output.mp4"); + let display_dir = content.join("display"); + let audio_dir = content.join("audio"); + if !audio_dir.exists() { + std::fs::create_dir_all(&audio_dir)?; + } + tokio::task::spawn_blocking(move || { + cap_recording::recovery::RecoveryManager::finalize_instant_output( + &display_dir, + &audio_dir, + &output, + ) + }) + .await? + .map_err(|e| anyhow!("finalize instant output: {e}"))?; + let meta = cap_project::RecordingMeta { + platform: Some(cap_project::Platform::default()), + project_path: dir.to_path_buf(), + pretty_name: spec.name.clone(), + sharing: None, + inner: cap_project::RecordingMetaInner::Instant( + cap_project::InstantRecordingMeta::Complete { + fps: spec.screen.fps, + sample_rate: has_audio.then_some(AUDIO_RATE), + }, + ), + upload: None, + }; + meta.save_for_project() + .map_err(|e| anyhow!("save instant meta: {e:?}"))?; + if let Some(arm_at) = spec.arm_at { + // Frames and markers within a frame of the arm instant may land on + // either side of it. + let margin = 0.05; + emitted.screen.retain(|e| e.1 >= arm_at + margin); + emitted.audio.retain(|e| e.2 >= arm_at + margin); + } + std::fs::write( + dir.join("verify-emitted.json"), + serde_json::to_string(&emitted)?, + )?; + Ok(()) +} + +/// Records one epoch-anchored audio track the way the product records +/// loopback system audio (`AudioAnchor::PipelineEpoch`, no start gate): +/// the first captured frame is stamped `head` seconds after the pipeline +/// epoch and a continuous 1 kHz tone follows for `tone` seconds, in 20 ms +/// chunks delivered in real time. Returns the file and the track start the +/// pipeline reports, in seconds after the epoch. +pub async fn record_epoch_audio( + dir: &Path, + head: f64, + tone: f64, +) -> anyhow::Result<(std::path::PathBuf, f64)> { + let stereo_info = AudioInfo::new(Sample::F32(Type::Packed), AUDIO_RATE, 2) + .map_err(|e| anyhow!("audio info: {e:?}"))?; + let samples = (tone * f64::from(AUDIO_RATE)) as usize; + let track: Vec = (0..samples) + .map(|n| { + let t = n as f64 / f64::from(AUDIO_RATE); + (0.5 * (2.0 * std::f64::consts::PI * 1000.0 * t).sin()) as f32 + }) + .collect(); + let path = dir.join("system_audio.ogg"); + let timestamps = Timestamps::now(); + let (mut tx, rx) = futures::channel::mpsc::channel::(32); + let pipeline = OutputPipeline::builder(path.clone()) + .with_audio_source::(ChannelAudioSourceConfig::new(stereo_info, rx)) + .with_timestamps(timestamps) + .with_audio_anchor(cap_recording::AudioAnchor::PipelineEpoch) + .build::(()) + .await + .map_err(|e| anyhow!("epoch audio pipeline: {e}"))?; + { + use futures::SinkExt; + let chunks = (tone / AUDIO_CHUNK_SECS).round() as usize; + for k in 0..chunks { + let t = k as f64 * AUDIO_CHUNK_SECS; + let at = timestamps.instant() + Duration::from_secs_f64(head + t); + tokio::time::sleep_until(at.into()).await; + let frame = + AudioFrame::new(audio_chunk(&track, t, &stereo_info), Timestamp::Instant(at)); + tx.send(frame) + .await + .map_err(|e| anyhow!("send audio: {e}"))?; + } + } + tokio::time::sleep(Duration::from_millis(300)).await; + let finished = pipeline + .stop() + .await + .map_err(|e| anyhow!("epoch audio stop: {e}"))?; + drop(tx); + Ok(( + path, + finished + .first_timestamp + .signed_duration_since_secs(timestamps), + )) +} diff --git a/crates/verify/src/report.rs b/crates/verify/src/report.rs new file mode 100644 index 00000000000..9ecabe828dc --- /dev/null +++ b/crates/verify/src/report.rs @@ -0,0 +1,702 @@ +//! Case results, the strict expected-failure list and the JSON report. +//! +//! Every case ends in [`settle`]: its failure classes are judged against +//! `crates/verify/expectations.toml` with the shared strict rules of +//! `cap-verify-expect` (a listed class passes as XFAIL, a listed class that +//! no longer occurs fails with "remove this xfail", any other failure fails +//! with the exact mismatches and a one-line local repro command), and the +//! result is written as a JSON record to `CAP_VERIFY_REPORT_DIR`. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::PathBuf, + time::Instant, +}; + +use serde::{Deserialize, Serialize}; + +use crate::oracle::StreamReport; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Tier { + Fast, + Full, +} + +/// `CAP_VERIFY_TIER=full` selects the merge-gate/nightly tier; anything else +/// (or unset) is the fast PR tier. +pub fn tier() -> Tier { + match std::env::var("CAP_VERIFY_TIER").as_deref() { + Ok("full") => Tier::Full, + _ => Tier::Fast, + } +} + +pub fn platform() -> &'static str { + if cfg!(target_os = "macos") { + "macos" + } else if cfg!(target_os = "windows") { + "windows" + } else { + "linux" + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CaseResult { + pub id: String, + pub platform: String, + pub tier: Tier, + pub passed: bool, + pub summary: String, + pub streams: Vec, + pub notes: Vec, + pub seconds: f64, + pub repro: String, + pub outcome: String, +} + +/// Classes that measure wall-clock behaviour of the runner as much as of Cap +/// (real-time playback delivering fewer frames than it should). Only these +/// may become inconclusive, and only under proven starvation. +const WALL_CLOCK_CLASSES: &[&str] = &["dropped"]; + +/// Set by scripts/verify/retry.py when it reruns a test once. +fn retrying() -> bool { + std::env::var("CAP_VERIFY_RETRY").as_deref() == Ok("1") +} + +pub struct Case { + pub id: String, + started: Instant, + machine: crate::sentinel::Mark, + pub streams: Vec, + pub problems: Vec, + /// The class of each entry of `problems` (a stream report's classes, + /// comma-separated). + pub problem_classes: Vec, + /// Failure classes seen (see `expectations.toml`). + pub classes: BTreeSet, + /// First problem per class (the draft reason when blessing). + class_problems: BTreeMap, + pub notes: Vec, + /// Classes this case could not measure because an earlier check failed + /// -> why (see `cap_verify_expect::Unobserved`). + pub unobserved: BTreeMap, + /// Set when this runner cannot run the case at all (for example no + /// D3D11 video device for Media Foundation on a GPU-less Windows VM). + pub unsupported: Option, + /// Measured size of each failing class: class -> metric -> range. An + /// expected failure excuses a class only inside its entry's bound. + measures: BTreeMap, + /// Additive metrics (counts): (class, metric) -> total. + counts: BTreeMap<(String, String), f64>, +} + +/// A case's analysis state, for discarding one analysis (see +/// [`Case::rollback`]). +pub struct CaseCheckpoint { + problems: usize, + problem_classes: usize, + streams: usize, + notes: usize, + classes: BTreeSet, + class_problems: BTreeMap, + unobserved: BTreeMap, + measures: BTreeMap, + counts: BTreeMap<(String, String), f64>, +} + +impl CaseCheckpoint { + /// Problems recorded before the checkpoint. + pub fn problems(&self) -> usize { + self.problems + } +} + +impl Case { + pub fn checkpoint(&self) -> CaseCheckpoint { + CaseCheckpoint { + problems: self.problems.len(), + problem_classes: self.problem_classes.len(), + streams: self.streams.len(), + notes: self.notes.len(), + classes: self.classes.clone(), + class_problems: self.class_problems.clone(), + unobserved: self.unobserved.clone(), + measures: self.measures.clone(), + counts: self.counts.clone(), + } + } + + /// Restores everything an analysis since `checkpoint` recorded. + pub fn rollback(&mut self, checkpoint: CaseCheckpoint) { + self.problems.truncate(checkpoint.problems); + self.problem_classes.truncate(checkpoint.problem_classes); + self.streams.truncate(checkpoint.streams); + self.notes.truncate(checkpoint.notes); + self.classes = checkpoint.classes; + self.class_problems = checkpoint.class_problems; + self.unobserved = checkpoint.unobserved; + self.measures = checkpoint.measures; + self.counts = checkpoint.counts; + } + + pub fn new(id: impl Into) -> Self { + let id = id.into(); + eprintln!("[cap-verify] case {id} ({})", platform()); + Self { + id, + started: Instant::now(), + machine: crate::sentinel::mark(), + streams: Vec::new(), + problems: Vec::new(), + problem_classes: Vec::new(), + classes: Default::default(), + class_problems: Default::default(), + notes: Vec::new(), + unobserved: Default::default(), + unsupported: None, + measures: Default::default(), + counts: Default::default(), + } + } + + /// Records one measurement of a failing class (an offset, a spread, a + /// count of affected items); the record keeps its range per metric. + pub fn measure(&mut self, class: &str, metric: &str, value: f64) { + self.measures + .entry(class.to_string()) + .or_default() + .entry(metric.to_string()) + .and_modify(|r| r.include(value)) + .or_insert(cap_verify_expect::Range::at(value)); + } + + /// Adds to a counted metric of a class (totals across checks). + pub fn count(&mut self, class: &str, metric: &str, n: f64) { + *self + .counts + .entry((class.to_string(), metric.to_string())) + .or_insert(0.0) += n; + } + + /// [`Case::problem`] with one measurement of its size. + pub fn problem_measured( + &mut self, + class: &str, + metric: &str, + value: f64, + text: impl Into, + ) { + self.measure(class, metric, value); + self.problem(class, text); + } + + /// Marks the case as impossible on this machine. With + /// `CAP_VERIFY_REQUIRE_SUPPORTED=1` (CI) it is a failure of class + /// `unsupported` unless the case matches one of the comma-separated globs + /// in `CAP_VERIFY_UNSUPPORTED_ALLOW` (the per-platform allowlist in + /// scripts/verify/suites.json, each with its reason). + pub fn unsupported(&mut self, reason: impl Into) { + let reason = reason.into(); + let allowed = std::env::var("CAP_VERIFY_UNSUPPORTED_ALLOW") + .unwrap_or_default() + .split(',') + .map(str::trim) + .any(|glob| !glob.is_empty() && glob_match(glob, &self.id)); + if std::env::var("CAP_VERIFY_REQUIRE_SUPPORTED").as_deref() == Ok("1") && !allowed { + self.problem( + "unsupported", + format!("unsupported on this machine: {reason}"), + ); + } else { + self.unsupported = Some(reason); + } + } + + /// Records a failure of the given class (`error`, `frame-count`, + /// `nondeterministic`, ...). + pub fn problem(&mut self, class: &str, text: impl Into) { + let text = text.into(); + self.count(class, "count", 1.0); + self.classes.insert(class.to_string()); + self.class_problems + .entry(class.to_string()) + .or_insert_with(|| text.clone()); + self.problem_classes.push(class.to_string()); + self.problems.push(text); + } + + /// Records that `class` could not be measured because the check it + /// depends on failed. The case must also report that failure: an + /// unobserved class never makes a case pass, it only stops an expected + /// failure of `class` from counting as fixed. + pub fn unobserved(&mut self, class: &str, why: impl Into) { + self.unobserved + .entry(class.to_string()) + .or_insert_with(|| why.into()); + } + + pub fn required_track_absent(&mut self, source: &str) { + let classes: &[&str] = match source { + "mic" => &["sync-screen-mic", "sync-system-mic", "sync-camera-mic"], + "system" => &["sync-screen-system", "sync-system-mic"], + "camera" => &["sync-camera-mic", "camera-clock"], + _ => &[], + }; + for class in classes { + self.unobserved(class, "track absent"); + } + } + + pub fn note(&mut self, text: impl Into) { + self.notes.push(text.into()); + } + + pub fn stream(&mut self, report: StreamReport) { + if !report.passed() { + use crate::oracle::MismatchKind; + let stream = report.stream.map_or("stream", |s| s.name()); + let mut classes = Vec::new(); + for (class, n, kind) in [ + ("stale", report.stale, MismatchKind::Stale), + ("future", report.ahead, MismatchKind::Ahead), + ( + "wrong-segment", + report.wrong_segment, + MismatchKind::WrongSegment, + ), + ("unreadable", report.unreadable, MismatchKind::Unreadable), + ("missing", report.missing, MismatchKind::Missing), + ("unexpected", report.unexpected, MismatchKind::Unexpected), + ("blend", report.blend_weight, MismatchKind::BlendWeight), + ] { + if n > 0 { + // Signature metrics: frames affected (in total and per + // stream), the output frames they fall on, and the + // largest source-frame error. + self.count(class, &format!("{stream}-frames"), f64::from(n)); + for m in report.mismatches.iter().filter(|m| m.kind == kind) { + self.measure(class, "frame", f64::from(m.output_frame)); + if let Some(delta) = m.delta { + self.measure(class, "delta", delta.unsigned_abs() as f64); + } + } + self.classes.insert(class.to_string()); + self.class_problems + .entry(class.to_string()) + .or_insert_with(|| report.summary()); + classes.push(class); + } + } + self.problem_classes.push(classes.join(",")); + self.problems.push(report.summary()); + } + self.streams.push(report); + } + + pub fn failed(&self) -> bool { + !self.problems.is_empty() + } +} + +pub use cap_verify_expect::{Xfail, glob_match}; + +/// Expectations file, relative to the repository root. +const EXPECTATIONS: &str = "crates/verify/expectations.toml"; + +pub fn expectations_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("expectations.toml") +} + +pub fn load_expectations() -> Vec { + cap_verify_expect::load(&expectations_path()) +} + +/// Entries that apply to `id` on this platform. +pub fn xfails_for(id: &str) -> Vec { + let platform = platform(); + load_expectations() + .into_iter() + .filter(|x| x.matches(id, platform)) + .collect() +} + +fn repro_command(id: &str) -> String { + let test = id.split('/').next().unwrap_or("export"); + format!( + "CAP_VERIFY_CASE='{id}' CAP_VERIFY_TIER={} cargo test -p cap-verify --test {test} -- --nocapture", + match tier() { + Tier::Fast => "fast", + Tier::Full => "full", + } + ) +} + +/// Whether a case runs: `CAP_VERIFY_CASE`, `CAP_VERIFY_SKIP` and +/// `CAP_VERIFY_SHARD` (see `cap_verify_expect::case_selected`). +pub fn selected(id: &str) -> bool { + cap_verify_expect::case_selected(id) +} + +fn write_result( + result: &CaseResult, + classes: &BTreeMap, + unobserved: &BTreeMap, + machine: &crate::sentinel::Window, + measures: &BTreeMap, + excused: &BTreeMap, +) { + use cap_verify_expect::{Outcome, Record}; + let record = Record { + suite: "cap-verify".into(), + expectations: EXPECTATIONS.into(), + case: result.id.clone(), + platform: result.platform.clone(), + tier: match result.tier { + Tier::Fast => "fast".into(), + Tier::Full => "full".into(), + }, + outcome: match result.outcome.as_str() { + "pass" => Outcome::Pass, + "unsupported" => Outcome::Unsupported, + "xfail" => Outcome::Xfail, + "XPASS" => Outcome::Xpass, + "inconclusive" => Outcome::Inconclusive, + _ => Outcome::Fail, + }, + failures: classes + .keys() + .map(|class| (class.clone(), Default::default())) + .collect(), + seeds_ran: None, + unobserved: unobserved.clone(), + problems: classes.clone(), + measures: measures.clone(), + excused: excused.clone(), + starved: machine.starved(), + summary: result.summary.clone(), + repro: result.repro.clone(), + seconds: result.seconds, + }; + let extra = serde_json::json!({ + "streams": result.streams, + "notes": result.notes, + "machine": machine, + }); + cap_verify_expect::write_records(&[(record, extra)]); +} + +/// Settles a case against the expectations file. Returns an error message +/// when the test must fail. +pub fn settle(mut case: Case) -> Result { + let measured: Vec = case + .unobserved + .keys() + .filter(|class| case.classes.contains(*class)) + .cloned() + .collect(); + for class in measured { + case.unobserved.remove(&class); + } + if !case.unobserved.is_empty() && !case.failed() { + let classes: Vec<&String> = case.unobserved.keys().collect(); + case.problem( + "harness", + format!("{classes:?} reported unmeasurable without the failure that prevented it"), + ); + } + let xfails = xfails_for(&case.id); + let failed = case.failed(); + let repro = repro_command(&case.id); + let platform = platform(); + // Per-stream frame counts are zero for every stream the case checked + // but that a frame class did not hit, so a bound can pin them. + let stream_names: BTreeSet<&'static str> = case + .streams + .iter() + .filter_map(|s| s.stream.map(|k| k.name())) + .collect(); + let frame_classes: BTreeSet = case + .counts + .keys() + .filter(|(_, metric)| { + stream_names + .iter() + .any(|name| *metric == format!("{name}-frames")) + }) + .map(|(class, _)| class.clone()) + .collect(); + for class in frame_classes { + for name in &stream_names { + case.count(&class, &format!("{name}-frames"), 0.0); + } + } + for ((class, metric), total) in std::mem::take(&mut case.counts) { + case.measures + .entry(class) + .or_default() + .insert(metric, cap_verify_expect::Range::at(total)); + } + case.measures + .retain(|class, _| case.classes.contains(class)); + let ran = cap_verify_expect::Ran::from([(case.id.clone(), None)]); + let observed: cap_verify_expect::Observed = case + .classes + .iter() + .map(|class| ((case.id.clone(), class.clone()), Default::default())) + .collect(); + let unobserved: cap_verify_expect::Unobserved = case + .unobserved + .keys() + .map(|class| (case.id.clone(), class.clone())) + .collect(); + // Machine starvation excuses only wall-clock classes, and only until the + // retry: content, frame identity, timestamps and sync never (except + // through a `starved` expectation with its own bound). + let machine = crate::sentinel::window(case.machine); + case.notes.push(machine.describe()); + let evidence = cap_verify_expect::Evidence { + unobserved, + measures: case + .measures + .iter() + .map(|(class, metrics)| ((case.id.clone(), class.clone()), metrics.clone())) + .collect(), + starved: if machine.starved() { + BTreeSet::from([case.id.clone()]) + } else { + BTreeSet::new() + }, + }; + let verdict = cap_verify_expect::judge_with(&xfails, platform, &ran, &observed, &evidence); + for u in verdict.unexpected.iter().filter(|u| !u.why.is_empty()) { + case.notes.push(format!( + "{} is listed in {EXPECTATIONS} but not excused: {}", + u.class, u.why + )); + } + let excused: BTreeMap = verdict + .excused + .iter() + .map(|(_, class, x)| { + let kind = match (x.flaky, x.starved) { + (_, true) => "flaky, starved", + (true, false) => "flaky", + (false, false) => "strict", + }; + (class.clone(), kind.to_string()) + }) + .collect(); + let mut unexpected: Vec<&String> = verdict.unexpected.iter().map(|u| &u.class).collect(); + let fixed: Vec<&Xfail> = verdict.fixed.iter().map(|f| &f.xfail).collect(); + let wall_clock_only = !unexpected.is_empty() + && unexpected + .iter() + .all(|class| WALL_CLOCK_CLASSES.contains(&class.as_str())); + let environment = String::from("environment"); + let inconclusive = wall_clock_only && machine.starved() && !retrying(); + if wall_clock_only && machine.starved() && retrying() { + case.problem( + "environment", + format!( + "inconclusive again on retry ({}): the machine starved this case twice", + machine.describe() + ), + ); + unexpected = vec![&environment]; + } + let outcome = if case.unsupported.is_some() && !failed { + "unsupported" + } else if inconclusive { + "inconclusive" + } else if !unexpected.is_empty() { + "FAIL" + } else if !fixed.is_empty() { + "XPASS" + } else if failed { + "xfail" + } else { + "pass" + }; + let mut summary = if let Some(reason) = case.unsupported.as_ref().filter(|_| !failed) { + format!("not run: {reason}") + } else if failed { + let mut text = case.problems.join("\n "); + for u in verdict.unexpected.iter().filter(|u| !u.why.is_empty()) { + text.push_str(&format!( + "\n ({} has an expectation, but this failure is {})", + u.class, u.why + )); + } + text + } else { + let streams: Vec = case + .streams + .iter() + .map(|s| format!("{} ({} unchecked)", s.summary(), s.unchecked)) + .collect(); + format!("all checks passed: {}", streams.join(", ")) + }; + for (class, why) in &case.unobserved { + summary.push_str(&format!("\n ({class} not measured: {why})")); + } + for x in &xfails { + summary.push_str(&format!( + "\n (expected {}{}{}: {} {})", + x.class, + if x.flaky { ", flaky" } else { "" }, + if x.starved { + ", only under a machine stall" + } else { + "" + }, + x.reason, + x.link + )); + } + let result = CaseResult { + id: case.id.clone(), + platform: platform.to_string(), + tier: tier(), + passed: matches!(outcome, "pass" | "xfail" | "unsupported"), + summary: summary.clone(), + streams: case.streams, + notes: case.notes, + seconds: case.started.elapsed().as_secs_f64(), + repro: repro.clone(), + outcome: outcome.to_string(), + }; + write_result( + &result, + &case.class_problems, + &case.unobserved, + &machine, + &case.measures, + &excused, + ); + eprintln!( + "[cap-verify] {outcome} {} in {:.1}s\n {summary}", + result.id, result.seconds + ); + match outcome { + "inconclusive" => Err(format!( + "case {} inconclusive: {} failed while the machine starved the process ({}); retried once by scripts/verify/retry.py\n repro: {repro}", + result.id, + unexpected + .iter() + .map(|c| c.as_str()) + .collect::>() + .join(", "), + machine.describe() + )), + "FAIL" => Err(format!( + "case {} failed ({}):\n {summary}\n repro: {repro}", + result.id, + unexpected + .iter() + .map(|c| c.as_str()) + .collect::>() + .join(", ") + )), + "XPASS" => Err(format!( + "case {} no longer fails with {}: remove that xfail from {}\n repro: {repro}", + result.id, + fixed + .iter() + .map(|x| x.class.as_str()) + .collect::>() + .join(", "), + expectations_path().display() + )), + _ => Ok(result), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn absent_required_tracks_are_unobserved_not_passes() { + let mut case = Case::new("recording/studio-recovery/crash"); + case.problem("missing", "no mic track recorded"); + case.required_track_absent("mic"); + for class in ["sync-screen-mic", "sync-system-mic", "sync-camera-mic"] { + assert_eq!( + case.unobserved.get(class).map(String::as_str), + Some("track absent") + ); + } + assert!(!case.unobserved.contains_key("sync-screen-system")); + assert!(case.failed()); + case.required_track_absent("camera"); + assert_eq!( + case.unobserved.get("camera-clock").map(String::as_str), + Some("track absent") + ); + case.required_track_absent("system"); + assert_eq!( + case.unobserved + .get("sync-screen-system") + .map(String::as_str), + Some("track absent") + ); + } + #[test] + fn instant_system_offset_expectation_requires_its_exact_signature() { + let entries = cap_verify_expect::parse(include_str!("../expectations.toml")).unwrap(); + let entries: Vec<_> = entries + .iter() + .filter(|entry| entry.class == "sync-system" && entry.flaky) + .collect(); + assert_eq!(entries.len(), 2); + let metrics = |offset, mic| { + BTreeMap::from([ + ("count".into(), cap_verify_expect::Range::at(1.0)), + ("offset-ms".into(), cap_verify_expect::Range::at(offset)), + ("mic-offset-ms".into(), cap_verify_expect::Range::at(mic)), + ( + "system-minus-mic-ms".into(), + cap_verify_expect::Range::at(((offset - mic) * 10.0_f64).round() / 10.0), + ), + ]) + }; + let accepts = |metrics: &BTreeMap| { + entries + .iter() + .any(|entry| cap_verify_expect::within(&entry.bound, Some(metrics)).is_ok()) + }; + for (offset, mic) in [(-46.1, 13.9), (7.8, 14.5), (-47.1, 13.9), (8.8, 14.5)] { + assert!(accepts(&metrics(offset, mic))); + } + for (offset, mic) in [ + (-47.2, 13.9), + (-45.0, 13.9), + (6.7, 14.5), + (8.9, 14.5), + (-25.0, 35.0), + (-56.0, 4.0), + (0.0, 14.5), + ] { + assert!(!accepts(&metrics(offset, mic))); + } + for required in ["mic-offset-ms", "system-minus-mic-ms"] { + let mut absent = metrics(-46.1, 13.9); + absent.remove(required); + assert!(!accepts(&absent)); + } + for entry in &entries { + assert!(entry.matches("export/instant/output/mp4/native", "macos")); + } + assert_eq!( + entries + .iter() + .filter(|entry| entry.matches("export/instant/output/mp4/native", "linux")) + .count(), + 1 + ); + } +} diff --git a/crates/verify/src/sched.rs b/crates/verify/src/sched.rs new file mode 100644 index 00000000000..df8960fd95f --- /dev/null +++ b/crates/verify/src/sched.rs @@ -0,0 +1,250 @@ +//! Real-time scheduling for the verification suite's input sources. +//! +//! Hosted runners start jobs at utility QoS (scheduler base priority 20) and +//! every thread inherits it. A blocking sleep at that tier may be coalesced +//! by up to `min(8 * requested, 150 ms)` (XNU timer coalescing with idle +//! amplification), and a successful user-interactive QoS request does not +//! lift the task's clamp: synthetic sources at QoS 33 still ran at priority +//! 20 and woke about 151 ms late (run 38766191045). The inputs a user +//! records come from a foreground app and device callbacks the system runs +//! in real time, so every input thread asks for a time constraint matching +//! its period, and the result is read back from the kernel instead of +//! trusting the request's status. + +use serde::{Deserialize, Serialize}; + +/// A time-constraint policy in milliseconds. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct Constraint { + pub period_ms: f64, + pub computation_ms: f64, + pub constraint_ms: f64, + pub preemptible: bool, +} + +/// A thread's effective scheduling state (`THREAD_EXTENDED_INFO`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ThreadState { + pub policy: i32, + pub base_pri: i32, + pub cur_pri: i32, + pub max_pri: i32, + pub cpu_us: u64, +} + +/// Lowest scheduler priority of the real-time band (`BASEPRI_RTQUEUES`). +pub const REALTIME_PRIORITY: i32 = 97; + +/// The outcome of [`request_realtime`]: every kernel status, the policy the +/// kernel reports back and the thread's effective state right after. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Realtime { + pub requested: Constraint, + pub qos_status: i32, + pub policy_status: i32, + pub readback_status: i32, + /// The kernel answered the readback with the default policy: the thread + /// is not time-constrained. + pub readback_default: bool, + pub granted: Option, + pub state: Option, +} + +impl Realtime { + /// `Ok` when the kernel applied the time constraint (the computation + /// budget may be raised by the kernel, never lowered) and the thread + /// runs in the real-time band. + pub fn obtained(&self) -> Result<(), String> { + if self.policy_status != 0 { + return Err(format!( + "thread_policy_set(TIME_CONSTRAINT) returned {}", + self.policy_status + )); + } + if self.readback_status != 0 || self.readback_default { + return Err(format!( + "thread_policy_get(TIME_CONSTRAINT) returned {} (default policy: {})", + self.readback_status, self.readback_default + )); + } + let Some(granted) = self.granted else { + return Err("no time constraint read back".into()); + }; + let tick = 0.001; + if (granted.period_ms - self.requested.period_ms).abs() > tick + || (granted.constraint_ms - self.requested.constraint_ms).abs() > tick + || granted.computation_ms + tick < self.requested.computation_ms + || granted.computation_ms > granted.constraint_ms + tick + || granted.preemptible != self.requested.preemptible + { + return Err(format!( + "kernel granted {granted:?} for requested {:?}", + self.requested + )); + } + state_is_realtime(self.state) + } +} + +/// `Ok` when the thread state is in the real-time band. +pub fn state_is_realtime(state: Option) -> Result<(), String> { + match state { + Some(state) if state.base_pri >= REALTIME_PRIORITY => Ok(()), + Some(state) => Err(format!( + "thread base priority {} is below the real-time band ({REALTIME_PRIORITY})", + state.base_pri + )), + None => Err("thread_info(THREAD_EXTENDED_INFO) failed".into()), + } +} + +#[repr(C)] +struct ThreadExtendedInfo { + user_time: u64, + system_time: u64, + cpu_usage: i32, + policy: i32, + run_state: i32, + flags: i32, + sleep_time: i32, + cur_pri: i32, + priority: i32, + max_priority: i32, + name: [u8; 64], +} + +#[repr(C)] +#[derive(Default)] +struct TimeConstraintPolicy { + period: u32, + computation: u32, + constraint: u32, + preemptible: i32, +} + +const QOS_CLASS_USER_INTERACTIVE: u32 = 0x21; +const THREAD_TIME_CONSTRAINT_POLICY: u32 = 2; +const THREAD_EXTENDED_INFO: i32 = 5; + +unsafe extern "C" { + fn pthread_self() -> usize; + fn pthread_mach_thread_np(thread: usize) -> u32; + fn thread_info(thread: u32, flavor: i32, info: *mut i32, count: *mut u32) -> i32; + fn thread_policy_set(thread: u32, flavor: u32, policy: *const i32, count: u32) -> i32; + fn thread_policy_get( + thread: u32, + flavor: u32, + policy: *mut i32, + count: *mut u32, + get_default: *mut i32, + ) -> i32; + fn pthread_set_qos_class_self_np(qos: u32, relative_priority: i32) -> i32; + fn mach_wait_until(deadline: u64) -> i32; +} + +/// The calling thread's Mach port, owned by pthread for the thread's +/// lifetime (unlike `mach_thread_self`, it adds no port reference). +pub fn current_thread() -> u32 { + unsafe { pthread_mach_thread_np(pthread_self()) } +} + +fn ticks_per_ms() -> f64 { + let info = cidre::mach::TimeBaseInfo::new(); + 1e6 * f64::from(info.denom) / f64::from(info.numer) +} + +/// Asks for user-interactive QoS and a time constraint of `period` +/// (constraint = period, preemptible) with a bounded `computation` budget, +/// then reads both back. +pub fn request_realtime(period_secs: f64, computation_secs: f64) -> Realtime { + let per_ms = ticks_per_ms(); + let ticks = |secs: f64| (secs * 1e3 * per_ms).round() as u32; + let requested = Constraint { + period_ms: period_secs * 1e3, + computation_ms: computation_secs * 1e3, + constraint_ms: period_secs * 1e3, + preemptible: true, + }; + let qos_status = unsafe { pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0) }; + let thread = current_thread(); + let policy = TimeConstraintPolicy { + period: ticks(period_secs), + computation: ticks(computation_secs), + constraint: ticks(period_secs), + preemptible: 1, + }; + let count = (std::mem::size_of::() / 4) as u32; + let policy_status = unsafe { + thread_policy_set( + thread, + THREAD_TIME_CONSTRAINT_POLICY, + (&raw const policy).cast(), + count, + ) + }; + let mut readback = TimeConstraintPolicy::default(); + let mut readback_count = count; + let mut get_default = 0i32; + let readback_status = unsafe { + thread_policy_get( + thread, + THREAD_TIME_CONSTRAINT_POLICY, + (&raw mut readback).cast(), + &mut readback_count, + &mut get_default, + ) + }; + let ms = |ticks: u32| f64::from(ticks) / per_ms; + Realtime { + requested, + qos_status, + policy_status, + readback_status, + readback_default: get_default != 0, + granted: (readback_status == 0).then(|| Constraint { + period_ms: ms(readback.period), + computation_ms: ms(readback.computation), + constraint_ms: ms(readback.constraint), + preemptible: readback.preemptible != 0, + }), + state: thread_state(thread), + } +} + +/// The effective scheduling state of `thread`. +pub fn thread_state(thread: u32) -> Option { + let mut info = ThreadExtendedInfo { + user_time: 0, + system_time: 0, + cpu_usage: 0, + policy: 0, + run_state: 0, + flags: 0, + sleep_time: 0, + cur_pri: 0, + priority: 0, + max_priority: 0, + name: [0; 64], + }; + let mut count = (std::mem::size_of::() / 4) as u32; + let status = unsafe { + thread_info( + thread, + THREAD_EXTENDED_INFO, + (&raw mut info).cast(), + &mut count, + ) + }; + (status == 0).then_some(ThreadState { + policy: info.policy, + base_pri: info.priority, + cur_pri: info.cur_pri, + max_pri: info.max_priority, + cpu_us: (info.user_time + info.system_time) / 1000, + }) +} + +/// Blocks until the host clock reaches `deadline` (Mach absolute ticks). +pub fn wait_until(deadline: u64) { + unsafe { mach_wait_until(deadline) }; +} diff --git a/crates/verify/src/sentinel.rs b/crates/verify/src/sentinel.rs new file mode 100644 index 00000000000..0a19ffe959b --- /dev/null +++ b/crates/verify/src/sentinel.rs @@ -0,0 +1,251 @@ +//! Machine-starvation sentinel for the engine suites. +//! +//! A test that measures wall-clock behaviour (real-time playback) can fail +//! because the runner stopped scheduling the process, not because Cap is +//! wrong. To tell the two apart from evidence rather than by guessing, every +//! test process runs one high-priority thread that sleeps a fixed short +//! interval and records how late it wakes. A wake gap of +//! [`STARVATION_GAP_MS`] or more means the machine did not run the process +//! for that long. The series is kept in memory for [`crate::report`] (per +//! case) and appended to `/sentinel-.log`, so a +//! test killed by its timeout still leaves its evidence. +//! +//! Only classes that legitimately depend on wall-clock timing may be excused +//! by starvation, and only after a retry (see `scripts/verify/retry.py`). +//! Content, frame identity, export timestamps and sync never are. + +use std::{ + io::Write, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +/// The sentinel's sleep. +const PERIOD: Duration = Duration::from_millis(2); +/// Gaps at or above this are kept (Windows timers tick at about 15.6 ms, so a +/// healthy sentinel there wakes up to about 32 ms apart). +const RECORD_GAP_MS: f64 = 50.0; +/// A wake gap this long during a case proves the machine starved it (the +/// real-capture freeze sentinel uses the same limit). +pub const STARVATION_GAP_MS: f64 = 100.0; + +struct State { + started: Instant, + gaps: Mutex>, +} + +fn state() -> &'static State { + static STATE: OnceLock = OnceLock::new(); + STATE.get_or_init(|| { + let state = State { + started: Instant::now(), + gaps: Mutex::new(Vec::new()), + }; + let log = std::env::var_os("CAP_VERIFY_REPORT_DIR").and_then(|dir| { + let dir = std::path::PathBuf::from(dir); + std::fs::create_dir_all(&dir).ok()?; + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(dir.join(format!("sentinel-{}.log", std::process::id()))) + .ok()?; + let args: Vec = std::env::args().skip(1).collect(); + let exe = std::env::current_exe() + .ok() + .and_then(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned())) + .unwrap_or_default(); + let _ = writeln!( + file, + "{}", + serde_json::json!({ "pid": std::process::id(), "exe": exe, "args": args }) + ); + Some(file) + }); + let _ = std::thread::Builder::new() + .name("cap-verify-sentinel".into()) + .spawn(move || run(log)); + state + }) +} + +/// How often the log records that the process is still alive, so a test +/// killed by its timeout leaves the time of its end (scripts/verify/retry.py +/// only excuses a failure whose stall overlaps the time before it). +const HEARTBEAT: Duration = Duration::from_secs(2); + +fn run(mut log: Option) { + raise_priority(); + let state = state(); + let mut last = Instant::now(); + let mut beat = Instant::now(); + loop { + std::thread::sleep(PERIOD); + let now = Instant::now(); + let gap_ms = now.duration_since(last).as_secs_f64() * 1000.0; + last = now; + if now.duration_since(beat) >= HEARTBEAT { + beat = now; + if let Some(file) = log.as_mut() { + let _ = writeln!( + file, + "{}", + serde_json::json!({ "alive": now.duration_since(state.started).as_secs_f64() }) + ); + } + } + if gap_ms >= RECORD_GAP_MS { + let at = now.duration_since(state.started).as_secs_f64() - gap_ms / 1000.0; + if let Ok(mut gaps) = state.gaps.lock() { + gaps.push((at, gap_ms)); + } + if let Some(file) = log.as_mut() { + let _ = writeln!( + file, + "{}", + serde_json::json!({ "at": at, "gap_ms": gap_ms }) + ); + } + } + } +} + +#[cfg(target_os = "macos")] +fn raise_priority() { + let _ = crate::sched::request_realtime(PERIOD.as_secs_f64(), 0.000_5); +} + +#[cfg(windows)] +fn raise_priority() { + unsafe extern "system" { + fn GetCurrentThread() -> isize; + fn SetThreadPriority(thread: isize, priority: i32) -> i32; + } + const THREAD_PRIORITY_TIME_CRITICAL: i32 = 15; + // SAFETY: GetCurrentThread returns a pseudo handle that needs no close. + unsafe { + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL); + } +} + +#[cfg(not(any(target_os = "macos", windows)))] +fn raise_priority() {} + +/// Start of a measured window (a case). +#[derive(Clone, Copy)] +pub struct Mark { + at: f64, + cpu: Option, + wall: Instant, +} + +/// What the sentinel saw during a window. +#[derive(Clone, Debug, serde::Serialize)] +pub struct Window { + pub seconds: f64, + /// Process CPU time over wall time (all threads; above 1 on several cores). + pub cpu_per_wall: Option, + pub max_gap_ms: f64, + /// Wake gaps of at least 50 ms: (seconds since the window started, ms). + pub gaps: Vec<(f64, f64)>, +} + +impl Window { + pub fn starved(&self) -> bool { + self.max_gap_ms >= STARVATION_GAP_MS + } + + pub fn describe(&self) -> String { + let over = self + .gaps + .iter() + .filter(|(_, gap)| *gap >= STARVATION_GAP_MS) + .count(); + format!( + "machine: sentinel longest wake gap {:.0} ms, {over} at or over {STARVATION_GAP_MS:.0} ms, CPU/wall {}", + self.max_gap_ms, + self.cpu_per_wall + .map_or("unknown".to_string(), |r| format!("{r:.2}")) + ) + } +} + +pub fn mark() -> Mark { + let state = state(); + Mark { + at: state.started.elapsed().as_secs_f64(), + cpu: process_cpu_secs(), + wall: Instant::now(), + } +} + +pub fn window(since: Mark) -> Window { + let state = state(); + let now = state.started.elapsed().as_secs_f64(); + let gaps: Vec<(f64, f64)> = state + .gaps + .lock() + .map(|gaps| { + gaps.iter() + .filter(|(at, gap)| at + gap / 1000.0 >= since.at && *at <= now) + .map(|(at, gap)| ((at - since.at).max(0.0), *gap)) + .collect() + }) + .unwrap_or_default(); + let seconds = since.wall.elapsed().as_secs_f64(); + let cpu_per_wall = since + .cpu + .zip(process_cpu_secs()) + .map(|(start, end)| (end - start) / seconds.max(1e-6)); + Window { + seconds, + cpu_per_wall, + max_gap_ms: gaps.iter().map(|(_, gap)| *gap).fold(0.0, f64::max), + gaps, + } +} + +/// User plus system CPU time of this process. +#[cfg(unix)] +fn process_cpu_secs() -> Option { + unsafe extern "C" { + fn getrusage(who: i32, usage: *mut i64) -> i32; + } + // struct rusage starts with two struct timeval (user, system), 16 bytes + // each on 64-bit macOS and Linux; the rest is longs. A larger buffer + // keeps the call in bounds on either. + let mut usage = [0i64; 32]; + // SAFETY: RUSAGE_SELF (0) and a buffer larger than struct rusage. + if unsafe { getrusage(0, usage.as_mut_ptr()) } != 0 { + return None; + } + // tv_usec is 32 bits on macOS (then padding), 64 on Linux. + let usec = |v: i64| (v & 0xffff_ffff) as f64 / 1e6; + Some(usage[0] as f64 + usec(usage[1]) + usage[2] as f64 + usec(usage[3])) +} + +#[cfg(windows)] +fn process_cpu_secs() -> Option { + unsafe extern "system" { + fn GetCurrentProcess() -> isize; + fn GetProcessTimes( + process: isize, + creation: *mut u64, + exit: *mut u64, + kernel: *mut u64, + user: *mut u64, + ) -> i32; + } + let (mut creation, mut exit, mut kernel, mut user) = (0u64, 0u64, 0u64, 0u64); + // SAFETY: FILETIME is two u32 in little-endian order, the same layout as + // a u64; GetCurrentProcess returns a pseudo handle that needs no close. + let ok = unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }; + (ok != 0).then(|| (kernel + user) as f64 / 1e7) +} diff --git a/crates/verify/src/stimulus.rs b/crates/verify/src/stimulus.rs new file mode 100644 index 00000000000..0820d75b234 --- /dev/null +++ b/crates/verify/src/stimulus.rs @@ -0,0 +1,302 @@ +//! Real-capture stimulus: a helper process (`cap-verify-stimulus`) that shows +//! a window rendering the screen marker of a monotonically increasing frame +//! index at a fixed rate and plays system-audio markers through the default +//! output, logging the host time of every presented frame and every marker. +//! +//! Protocol: the helper presents frame 0 and prints `ready`; the test starts +//! its recording and writes `go`; the helper then advances one index per +//! `1 / fps` (holding the picture inside `holds`), prints `done` after the +//! last frame and keeps the last frame on screen until `quit` (or EOF). +//! Every event is a JSON line in the log file; host times are mach absolute +//! ticks, the clock Cap stamps captured media with on macOS. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StimulusSpec { + pub fps: f64, + pub seconds: f64, + /// (start, length) seconds after `go` during which the picture is held. + pub holds: Vec<(f64, f64)>, + /// Pseudo-random achromatic noise bands above and below the marker, new + /// every frame, so the encoder sees dense changing content. + pub busy: bool, + /// Seconds of moving noise before `ready` (frame 0 keeps showing). + pub warmup: f64, + pub audio: bool, + pub audio_period: f64, + /// Host time the system-audio marker schedule counts from (0 = helper + /// start). Share it with the other marker sources of a case. + pub audio_origin: u64, + /// `None` = borderless fullscreen on the primary display; otherwise a + /// window at (x, y, width, height) in points. + pub window: Option<(i32, i32, u32, u32)>, +} + +impl StimulusSpec { + pub fn frame_count(&self) -> u32 { + (self.seconds * self.fps).round() as u32 + } + + /// Seconds after `go` at which frame `index` (>= 1) is due. + pub fn due_secs(&self, index: u32) -> f64 { + f64::from(index.saturating_sub(1)) / self.fps + } + + pub fn held(&self, secs: f64) -> bool { + self.holds.iter().any(|&(s, l)| secs > s && secs < s + l) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum StimulusEvent { + Ready { + host: u64, + width: u32, + height: u32, + scale: f64, + }, + Go { + host: u64, + }, + Frame { + index: u32, + /// Host time right after the frame was handed to the window server. + host: u64, + #[serde(default)] + render_started: Option, + }, + AudioStarted { + device: String, + sample_rate: u32, + channels: u16, + }, + AudioUnavailable { + reason: String, + }, + /// Chirp start of a system-audio marker at the output, from the render + /// callback's host time. + Audio { + index: u32, + host: u64, + }, + Done { + host: u64, + }, + /// A frame delivered to the stimulus's own ScreenCaptureKit stream (a + /// separate client in a separate process from the recorder), with the + /// host time it was captured: independent proof of what the window + /// server composited and the capture service delivered. + Observed { + host: u64, + }, + ObserverUnavailable { + reason: String, + }, + /// Kernel status of the stimulus's priority requests (0 = applied). + Priority { + qos: i32, + realtime: i32, + }, + /// Scheduling diagnostics for the frame just presented: how late the + /// run loop woke for it, how long the loop took to get back to waiting + /// after presenting, and the main thread's scheduler state. + Sched { + index: u32, + cause: String, + wake_late_us: Option, + post_present_us: i64, + cur_pri: i32, + base_pri: i32, + cpu_us: u64, + }, +} + +pub fn read_log(path: &Path) -> anyhow::Result> { + let text = std::fs::read_to_string(path)?; + text.lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).map_err(Into::into)) + .collect() +} + +/// The stimulus's real-time sentinel log, next to its event log. +pub const SENTINEL_FILE: &str = "sentinel.txt"; +/// The sentinel's wake period. +pub const SENTINEL_PERIOD_SECS: f64 = 0.002; + +/// Every wake of the stimulus's real-time sentinel thread (host ticks) and +/// the scheduling it ran under. +#[cfg(target_os = "macos")] +#[derive(Clone, Debug)] +pub struct SentinelLog { + pub realtime: crate::sched::Realtime, + pub end: Option, + pub wakes: Vec, +} + +#[cfg(target_os = "macos")] +impl SentinelLog { + pub fn read(path: &std::path::Path) -> anyhow::Result { + #[derive(serde::Deserialize)] + struct Header { + first: Option, + realtime: crate::sched::Realtime, + end: Option, + } + let text = std::fs::read_to_string(path)?; + let mut lines = text.lines(); + let header: Header = serde_json::from_str(lines.next().unwrap_or_default())?; + let mut wakes = Vec::new(); + if let Some(mut at) = header.first { + wakes.push(at); + for line in lines { + at += line.trim().parse::()?; + wakes.push(at); + } + } + Ok(Self { + realtime: header.realtime, + end: header.end, + wakes, + }) + } + + /// Gaps between consecutive wakes: (start host, seconds). + pub fn gaps(&self) -> impl Iterator + '_ { + self.wakes.windows(2).map(|w| (w[0], host_secs(w[1], w[0]))) + } +} + +#[cfg(target_os = "macos")] +pub fn host_now() -> u64 { + cidre::mach::abs_time() +} + +/// Seconds between two mach host times (`a - b`). +#[cfg(target_os = "macos")] +pub fn host_secs(a: u64, b: u64) -> f64 { + let info = cidre::mach::TimeBaseInfo::new(); + let ticks = a as f64 - b as f64; + ticks * f64::from(info.numer) / f64::from(info.denom) / 1e9 +} + +#[cfg(target_os = "macos")] +pub fn host_add_secs(a: u64, secs: f64) -> u64 { + let info = cidre::mach::TimeBaseInfo::new(); + (a as f64 + secs * 1e9 * f64::from(info.denom) / f64::from(info.numer)).round() as u64 +} + +#[cfg(target_os = "macos")] +pub use process::StimulusProcess; + +#[cfg(target_os = "macos")] +mod process { + use std::{ + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Child, ChildStdin, Command, Stdio}, + sync::mpsc, + time::Duration, + }; + + use anyhow::{Context, bail}; + + use super::{StimulusEvent, StimulusSpec, read_log}; + + pub struct StimulusProcess { + child: Child, + stdin: Option, + lines: mpsc::Receiver, + log: PathBuf, + } + + impl StimulusProcess { + pub fn spawn(binary: &Path, spec: &StimulusSpec, log: &Path) -> anyhow::Result { + let mut child = Command::new(binary) + .arg(serde_json::to_string(spec)?) + .arg(log) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .with_context(|| format!("spawn {}", binary.display()))?; + let stdout = child.stdout.take().context("stimulus stdout")?; + let (tx, lines) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if tx.send(line).is_err() { + break; + } + } + }); + let stdin = child.stdin.take(); + Ok(Self { + child, + stdin, + lines, + log: log.to_path_buf(), + }) + } + + pub fn pid(&self) -> u32 { + self.child.id() + } + + pub fn wait_for(&mut self, word: &str, timeout: Duration) -> anyhow::Result<()> { + let deadline = std::time::Instant::now() + timeout; + loop { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + match self.lines.recv_timeout(left) { + Ok(line) if line.trim() == word => return Ok(()), + Ok(line) if line.starts_with("error") => bail!("stimulus: {line}"), + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Timeout) => { + bail!("stimulus did not report `{word}` within {timeout:?}") + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + bail!("stimulus exited before `{word}`") + } + } + } + } + + pub fn send(&mut self, word: &str) -> anyhow::Result<()> { + let stdin = self.stdin.as_mut().context("stimulus stdin closed")?; + writeln!(stdin, "{word}")?; + stdin.flush()?; + Ok(()) + } + + /// Closes the window and returns the event log. + pub fn finish(mut self) -> anyhow::Result> { + let _ = self.send("quit"); + self.stdin = None; + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + if self.child.try_wait()?.is_some() { + break; + } + if std::time::Instant::now() > deadline { + let _ = self.child.kill(); + let _ = self.child.wait(); + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + read_log(&self.log) + } + } + + impl Drop for StimulusProcess { + fn drop(&mut self) { + if matches!(self.child.try_wait(), Ok(None)) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } + } +} diff --git a/crates/verify/src/video_code.rs b/crates/verify/src/video_code.rs new file mode 100644 index 00000000000..399b8ad8dc0 --- /dev/null +++ b/crates/verify/src/video_code.rs @@ -0,0 +1,837 @@ +//! Video frame markers. +//! +//! Every synthetic source frame is filled with its stream's key colour and +//! carries an 8x8 block code in its centre: a ring of 28 alternating pilot +//! cells (which give the local black and white levels and lock the geometry) +//! around 36 data cells holding a 16-bit frame index, a 4-bit tag (stream kind +//! and recording segment) and a CRC-16 of both. +//! +//! The reader does not need to know where the renderer put the stream: it +//! finds the stream's key-coloured region, derives the code square from the +//! region's centre and height (centre crops, rounded corners, circular camera +//! bubbles and padding all preserve both), refines the geometry against the +//! pilot ring and only then reads the data cells. A cell whose level is not +//! clearly black or white, a pilot that disagrees, or a CRC mismatch makes the +//! frame unreadable rather than wrong: a decoded index is either right or the +//! frame is reported as damaged. + +use serde::{Deserialize, Serialize}; + +pub const GRID: usize = 8; +#[cfg(test)] +const DATA_CELLS: usize = (GRID - 2) * (GRID - 2); +const INDEX_BITS: u32 = 16; +const TAG_BITS: u32 = 4; +const PAYLOAD_BITS: u32 = INDEX_BITS + TAG_BITS; +const WHITENING: u64 = 0x9_A5C3_5E71; +pub const MAX_INDEX: u32 = (1 << INDEX_BITS) - 1; +pub const MAX_SEGMENT: u8 = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum StreamKind { + Screen, + Camera, +} + +impl StreamKind { + pub const ALL: [StreamKind; 2] = [StreamKind::Screen, StreamKind::Camera]; + + pub fn key_rgb(self) -> [u8; 3] { + match self { + StreamKind::Screen => [32, 96, 224], + StreamKind::Camera => [224, 64, 128], + } + } + + /// Side of the code square as a fraction of the frame height. The camera + /// code is larger because the bubble is small, and stays inside the + /// inscribed circle of the centre square (half diagonal 0.42 < 0.5). + pub fn code_side_fraction(self) -> f64 { + match self { + StreamKind::Screen => 0.5, + StreamKind::Camera => 0.6, + } + } + + fn tag(self) -> u8 { + match self { + StreamKind::Screen => 0, + StreamKind::Camera => 1, + } + } + + fn from_tag(tag: u8) -> Option { + match tag { + 0 => Some(StreamKind::Screen), + 1 => Some(StreamKind::Camera), + _ => None, + } + } + + fn key_chroma(self) -> (f64, f64) { + let [r, g, b] = self.key_rgb(); + let (_, cb, cr) = rgb_to_ycbcr(r, g, b); + (cb, cr) + } + + pub fn name(self) -> &'static str { + match self { + StreamKind::Screen => "screen", + StreamKind::Camera => "camera", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct FrameMarker { + pub kind: StreamKind, + pub segment: u8, + pub index: u32, +} + +impl FrameMarker { + pub fn new(kind: StreamKind, segment: u8, index: u32) -> Self { + assert!( + index <= MAX_INDEX, + "frame index {index} exceeds {MAX_INDEX}" + ); + assert!( + segment <= MAX_SEGMENT, + "segment {segment} exceeds {MAX_SEGMENT}" + ); + Self { + kind, + segment, + index, + } + } + + fn payload(self) -> u32 { + let tag = u32::from(self.kind.tag() | (self.segment << 2)); + self.index | (tag << INDEX_BITS) + } + + fn bits(self) -> u64 { + let payload = self.payload(); + let crc = crc16(&payload.to_le_bytes()[..3]); + (u64::from(payload) | (u64::from(crc) << PAYLOAD_BITS)) ^ WHITENING + } + + fn from_bits(bits: u64) -> Option { + let bits = bits ^ WHITENING; + let payload = (bits & ((1 << PAYLOAD_BITS) - 1)) as u32; + let crc = (bits >> PAYLOAD_BITS) as u16; + if crc16(&payload.to_le_bytes()[..3]) != crc { + return None; + } + let tag = (payload >> INDEX_BITS) as u8; + Some(Self { + kind: StreamKind::from_tag(tag & 0b11)?, + segment: tag >> 2, + index: payload & MAX_INDEX, + }) + } +} + +pub(crate) fn crc16(bytes: &[u8]) -> u16 { + let mut crc: u16 = 0xFFFF; + for &byte in bytes { + crc ^= u16::from(byte) << 8; + for _ in 0..8 { + crc = if crc & 0x8000 != 0 { + (crc << 1) ^ 0x1021 + } else { + crc << 1 + }; + } + } + crc +} + +fn pilot_is_white(row: usize, col: usize) -> bool { + (row + col) % 2 == 1 +} + +fn is_pilot(row: usize, col: usize) -> bool { + row == 0 || col == 0 || row == GRID - 1 || col == GRID - 1 +} + +fn data_bit_index(row: usize, col: usize) -> usize { + (row - 1) * (GRID - 2) + (col - 1) +} + +fn cell_is_white(bits: u64, row: usize, col: usize) -> bool { + if is_pilot(row, col) { + pilot_is_white(row, col) + } else { + bits >> data_bit_index(row, col) & 1 == 1 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PixelOrder { + Rgba, + Bgra, +} + +/// Square code geometry in pixel space. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CodeGeometry { + pub x0: f64, + pub y0: f64, + pub side: f64, +} + +impl CodeGeometry { + pub fn for_frame(kind: StreamKind, width: usize, height: usize) -> Self { + let side = (kind.code_side_fraction() * height as f64) + .min(width as f64) + .floor(); + Self { + x0: ((width as f64 - side) / 2.0).floor(), + y0: ((height as f64 - side) / 2.0).floor(), + side, + } + } + + fn cell_center(&self, row: usize, col: usize) -> (f64, f64) { + let cell = self.side / GRID as f64; + ( + self.x0 + (col as f64 + 0.5) * cell, + self.y0 + (row as f64 + 0.5) * cell, + ) + } +} + +/// Draws a full marker frame (key colour background plus the code) into a +/// packed 4-byte-per-pixel buffer. +pub fn draw_marker( + marker: FrameMarker, + data: &mut [u8], + width: usize, + height: usize, + stride: usize, + order: PixelOrder, +) { + let [kr, kg, kb] = marker.kind.key_rgb(); + let key = match order { + PixelOrder::Rgba => [kr, kg, kb, 255], + PixelOrder::Bgra => [kb, kg, kr, 255], + }; + let geometry = CodeGeometry::for_frame(marker.kind, width, height); + let bits = marker.bits(); + let cell = geometry.side / GRID as f64; + for y in 0..height { + let row = &mut data[y * stride..y * stride + width * 4]; + let fy = y as f64 + 0.5 - geometry.y0; + let code_row = (fy >= 0.0 && fy < geometry.side).then(|| (fy / cell) as usize); + for (x, px) in row.chunks_exact_mut(4).enumerate() { + let fx = x as f64 + 0.5 - geometry.x0; + let value = match code_row { + Some(r) if fx >= 0.0 && fx < geometry.side => { + let c = ((fx / cell) as usize).min(GRID - 1); + let r = r.min(GRID - 1); + Some(if cell_is_white(bits, r, c) { 255 } else { 0 }) + } + _ => None, + }; + match value { + Some(v) => px.copy_from_slice(&[v, v, v, 255]), + None => px.copy_from_slice(&key), + } + } + } +} + +/// Allocates and draws an RGBA marker frame. +pub fn marker_rgba(marker: FrameMarker, width: usize, height: usize) -> Vec { + let mut data = vec![0u8; width * height * 4]; + draw_marker( + marker, + &mut data, + width, + height, + width * 4, + PixelOrder::Rgba, + ); + data +} + +pub fn rgb_to_ycbcr(r: u8, g: u8, b: u8) -> (f64, f64, f64) { + let (r, g, b) = (f64::from(r), f64::from(g), f64::from(b)); + ( + 0.299 * r + 0.587 * g + 0.114 * b, + -0.168_736 * r - 0.331_264 * g + 0.5 * b, + 0.5 * r - 0.418_688 * g - 0.081_312 * b, + ) +} + +/// Read access to a decoded picture in luma + centred chroma. +pub trait PixelSource { + fn width(&self) -> usize; + fn height(&self) -> usize; + fn luma(&self, x: usize, y: usize) -> f64; + /// (Cb - 128, Cr - 128) at the pixel. + fn chroma(&self, x: usize, y: usize) -> (f64, f64); +} + +pub struct PackedFrame<'a> { + pub data: &'a [u8], + pub width: usize, + pub height: usize, + pub stride: usize, + pub order: PixelOrder, +} + +impl PackedFrame<'_> { + fn rgb(&self, x: usize, y: usize) -> (u8, u8, u8) { + let i = y * self.stride + x * 4; + let px = &self.data[i..i + 3]; + match self.order { + PixelOrder::Rgba => (px[0], px[1], px[2]), + PixelOrder::Bgra => (px[2], px[1], px[0]), + } + } +} + +impl PixelSource for PackedFrame<'_> { + fn width(&self) -> usize { + self.width + } + + fn height(&self) -> usize { + self.height + } + + fn luma(&self, x: usize, y: usize) -> f64 { + let (r, g, b) = self.rgb(x, y); + rgb_to_ycbcr(r, g, b).0 + } + + fn chroma(&self, x: usize, y: usize) -> (f64, f64) { + let (r, g, b) = self.rgb(x, y); + let (_, cb, cr) = rgb_to_ycbcr(r, g, b); + (cb, cr) + } +} + +/// Planar 4:2:0 picture (YUV420P), or semi-planar NV12 when `v` is `None` +/// and `u` holds interleaved CbCr. +pub struct Yuv420Frame<'a> { + pub width: usize, + pub height: usize, + pub y: &'a [u8], + pub y_stride: usize, + pub u: &'a [u8], + pub u_stride: usize, + pub v: Option<(&'a [u8], usize)>, +} + +impl PixelSource for Yuv420Frame<'_> { + fn width(&self) -> usize { + self.width + } + + fn height(&self) -> usize { + self.height + } + + fn luma(&self, x: usize, y: usize) -> f64 { + f64::from(self.y[y * self.y_stride + x]) + } + + fn chroma(&self, x: usize, y: usize) -> (f64, f64) { + let (cx, cy) = (x / 2, y / 2); + let (cb, cr) = match self.v { + Some((v, v_stride)) => (self.u[cy * self.u_stride + cx], v[cy * v_stride + cx]), + None => { + let i = cy * self.u_stride + cx * 2; + (self.u[i], self.u[i + 1]) + } + }; + (f64::from(cb) - 128.0, f64::from(cr) - 128.0) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "lowercase")] +pub enum Readout { + /// No region of this stream's key colour is visible. + Absent, + /// The stream is visible but its code could not be read reliably. + Unreadable { reason: String }, + Marker { + marker: FrameMarker, + /// Worst data-cell distance from the black/white threshold, as a + /// fraction of the pilot contrast (0.5 = perfect). + margin: f64, + mirrored: bool, + }, + /// A cross-fade of two codes (`progress` 0 = all outgoing). + Blend { + outgoing: FrameMarker, + incoming: FrameMarker, + progress: f64, + residual: f64, + }, +} + +impl Readout { + pub fn marker(&self) -> Option { + match self { + Readout::Marker { marker, .. } => Some(*marker), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Region { + pub x0: f64, + pub y0: f64, + pub x1: f64, + pub y1: f64, +} + +const KEY_CHROMA_TOLERANCE: f64 = 38.0; +const KEY_MIN_CHROMA: f64 = 36.0; + +fn is_key(chroma: (f64, f64), key: (f64, f64)) -> bool { + let magnitude = chroma.0.hypot(chroma.1); + if magnitude < KEY_MIN_CHROMA { + return false; + } + // Limited-range video scales chroma by 224/255; compare against both. + let full = (chroma.0 - key.0).hypot(chroma.1 - key.1); + let limited = (chroma.0 - key.0 * 0.878).hypot(chroma.1 - key.1 * 0.878); + full.min(limited) < KEY_CHROMA_TOLERANCE +} + +/// Bounding box of the stream's key-coloured region. +pub fn locate_region(img: &impl PixelSource, kind: StreamKind) -> Option { + let (w, h) = (img.width(), img.height()); + if w < 8 || h < 8 { + return None; + } + let step = (w.min(h) / 240).max(1); + let key = kind.key_chroma(); + let cols = w.div_ceil(step); + let rows = h.div_ceil(step); + let mut row_counts = vec![0usize; rows]; + let mut col_counts = vec![0usize; cols]; + for (ry, row_count) in row_counts.iter_mut().enumerate() { + let y = ry * step; + for (cx, col_count) in col_counts.iter_mut().enumerate() { + if is_key(img.chroma(cx * step, y), key) { + *row_count += 1; + *col_count += 1; + } + } + } + let total: usize = row_counts.iter().sum(); + if total < 12 { + return None; + } + let extent = |counts: &[usize]| -> Option<(usize, usize)> { + let max = *counts.iter().max()?; + let threshold = (max / 10).max(2); + let first = counts.iter().position(|&c| c >= threshold)?; + let last = counts.iter().rposition(|&c| c >= threshold)?; + Some((first, last)) + }; + let (r0, r1) = extent(&row_counts)?; + let (c0, c1) = extent(&col_counts)?; + let refine = |coarse: usize, dir: isize, horizontal: bool| -> f64 { + // Walk outward from the coarse edge at full resolution along the + // centre line of the region to find the exact key boundary. + let (lo, hi) = if horizontal { (0, w) } else { (0, h) }; + let mid_other = if horizontal { + ((r0 + r1) * step / 2).min(h - 1) + } else { + ((c0 + c1) * step / 2).min(w - 1) + }; + let probe = |p: usize| { + let (x, y) = if horizontal { + (p, mid_other) + } else { + (mid_other, p) + }; + is_key(img.chroma(x, y), key) + }; + let mut p = (coarse * step).clamp(lo, hi - 1) as isize; + let limit = (2 * step) as isize + 2; + let mut moved = 0; + while moved < limit { + let next = p + dir; + if next < lo as isize || next >= hi as isize || !probe(next as usize) { + break; + } + p = next; + moved += 1; + } + if dir < 0 { p as f64 } else { p as f64 + 1.0 } + }; + // Rows/columns through the centre may cross the code (no key colour), + // so refine horizontal edges on the region's top band instead when the + // centre line misses; the coarse value is kept when refinement fails. + let x0 = refine(c0, -1, true).min((c0 * step) as f64); + let x1 = refine(c1, 1, true).max((c1 * step + 1) as f64); + let y0 = refine(r0, -1, false).min((r0 * step) as f64); + let y1 = refine(r1, 1, false).max((r1 * step + 1) as f64); + Some(Region { x0, y0, x1, y1 }) +} + +struct CellSample { + levels: [[f64; GRID]; GRID], +} + +fn sample_cells(img: &impl PixelSource, geometry: &CodeGeometry) -> Option { + let (w, h) = (img.width() as f64, img.height() as f64); + let cell = geometry.side / GRID as f64; + let mut levels = [[0.0; GRID]; GRID]; + let offsets = [-0.22, 0.0, 0.22]; + for (row, row_levels) in levels.iter_mut().enumerate() { + for (col, level) in row_levels.iter_mut().enumerate() { + let (cx, cy) = geometry.cell_center(row, col); + let mut sum = 0.0; + let mut n = 0.0; + for oy in offsets { + for ox in offsets { + let x = cx + ox * cell; + let y = cy + oy * cell; + if x < 0.0 || y < 0.0 || x >= w || y >= h { + return None; + } + sum += img.luma(x as usize, y as usize); + n += 1.0; + } + } + *level = sum / n; + } + } + Some(CellSample { levels }) +} + +struct PilotFit { + black: f64, + white: f64, + margin: f64, +} + +fn pilot_fit(sample: &CellSample) -> PilotFit { + let mut black_sum = 0.0; + let mut white_sum = 0.0; + let mut black_max = f64::MIN; + let mut white_min = f64::MAX; + let mut nb = 0.0; + let mut nw = 0.0; + for (row, row_levels) in sample.levels.iter().enumerate() { + for (col, &v) in row_levels.iter().enumerate() { + if !is_pilot(row, col) { + continue; + } + if pilot_is_white(row, col) { + white_sum += v; + nw += 1.0; + white_min = white_min.min(v); + } else { + black_sum += v; + nb += 1.0; + black_max = black_max.max(v); + } + } + } + PilotFit { + black: black_sum / nb, + white: white_sum / nw, + margin: white_min - black_max, + } +} + +const MIN_CONTRAST: f64 = 48.0; +const MIN_CELL_MARGIN: f64 = 0.18; + +/// Reads the code of one stream from a picture, locating it by key colour. +pub fn read_stream(img: &impl PixelSource, kind: StreamKind) -> Readout { + match stream_square(img, kind) { + Ok((cx, cy, side)) => read_code_near(img, kind, cx, cy, side), + Err(readout) => readout, + } +} + +/// Cell levels of a located code, normalised so the pilot black level is 0 +/// and white is 1. +#[derive(Clone, Debug)] +pub struct CodeCells { + pub levels: [[f64; GRID]; GRID], + pub contrast: f64, +} + +/// Finds the code square near (cx, cy) with the given side, refining +/// position and scale against the pilot ring. +fn locate_cells( + img: &impl PixelSource, + cx: f64, + cy: f64, + side: f64, +) -> Result<(PilotFit, CellSample), Readout> { + let cell = side / GRID as f64; + let shift_step = (cell / 6.0).max(0.5); + let mut best: Option<(PilotFit, CellSample)> = None; + for scale in [1.0, 0.97, 1.03, 0.94, 1.06] { + for dy in -2..=2 { + for dx in -2..=2 { + let s = side * scale; + let geometry = CodeGeometry { + x0: cx - s / 2.0 + f64::from(dx) * shift_step, + y0: cy - s / 2.0 + f64::from(dy) * shift_step, + side: s, + }; + let Some(sample) = sample_cells(img, &geometry) else { + continue; + }; + let fit = pilot_fit(&sample); + if best.as_ref().is_none_or(|(b, _)| fit.margin > b.margin) { + best = Some((fit, sample)); + } + } + } + } + let Some((fit, sample)) = best else { + return Err(Readout::Unreadable { + reason: "code square falls outside the picture".into(), + }); + }; + let contrast = fit.white - fit.black; + if contrast < MIN_CONTRAST { + return Err(Readout::Unreadable { + reason: format!("pilot contrast {contrast:.0} too low"), + }); + } + if fit.margin < contrast * 0.3 { + return Err(Readout::Unreadable { + reason: format!( + "pilot ring not resolved (margin {:.2})", + fit.margin / contrast + ), + }); + } + Ok((fit, sample)) +} + +/// Locates the stream's code square from its key-coloured region. +fn stream_square(img: &impl PixelSource, kind: StreamKind) -> Result<(f64, f64, f64), Readout> { + let Some(region) = locate_region(img, kind) else { + return Err(Readout::Absent); + }; + let height = region.y1 - region.y0; + let width = region.x1 - region.x0; + let side = (kind.code_side_fraction() * height).min(width); + if side < 12.0 { + return Err(Readout::Unreadable { + reason: format!("{} region too small ({width:.0}x{height:.0})", kind.name()), + }); + } + Ok(( + (region.x0 + region.x1) / 2.0, + (region.y0 + region.y1) / 2.0, + side, + )) +} + +/// Normalised cell levels of the stream's code (for blend checks). +pub fn read_cells(img: &impl PixelSource, kind: StreamKind) -> Result { + let (cx, cy, side) = stream_square(img, kind)?; + let (fit, sample) = locate_cells(img, cx, cy, side)?; + let contrast = fit.white - fit.black; + let mut levels = [[0.0; GRID]; GRID]; + for (row, row_levels) in levels.iter_mut().enumerate() { + for (col, level) in row_levels.iter_mut().enumerate() { + *level = (sample.levels[row][col] - fit.black) / contrast; + } + } + Ok(CodeCells { levels, contrast }) +} + +/// How well a cross-fade of `outgoing` into `incoming` explains the cells: +/// fits the mix `progress` over the cells whose bits differ and returns +/// (progress, worst residual over all data cells). +pub fn blend_fit(cells: &CodeCells, outgoing: FrameMarker, incoming: FrameMarker) -> (f64, f64) { + let (a, b) = (outgoing.bits(), incoming.bits()); + let bit = + |bits: u64, row: usize, col: usize| f64::from(u8::from(cell_is_white(bits, row, col))); + let mut num = 0.0; + let mut den = 0.0; + for (row, row_levels) in cells.levels.iter().enumerate() { + for (col, &v) in row_levels.iter().enumerate() { + let (x, y) = (bit(a, row, col), bit(b, row, col)); + if x != y { + // v = x + p (y - x) + num += (v - x) * (y - x); + den += (y - x) * (y - x); + } + } + } + let p = if den > 0.0 { + (num / den).clamp(0.0, 1.0) + } else { + 0.5 + }; + let mut worst: f64 = 0.0; + for (row, row_levels) in cells.levels.iter().enumerate() { + for (col, &v) in row_levels.iter().enumerate() { + if is_pilot(row, col) { + continue; + } + let (x, y) = (bit(a, row, col), bit(b, row, col)); + worst = worst.max((v - (x + p * (y - x))).abs()); + } + } + (p, worst) +} + +/// Largest per-cell residual a blend may leave and still count as a match. +pub const BLEND_TOLERANCE: f64 = 0.3; + +/// Reads a code whose square is approximately centred at (cx, cy) with the +/// given side, refining position and scale against the pilot ring. +pub fn read_code_near( + img: &impl PixelSource, + kind: StreamKind, + cx: f64, + cy: f64, + side: f64, +) -> Readout { + let (fit, sample) = match locate_cells(img, cx, cy, side) { + Ok(found) => found, + Err(readout) => return readout, + }; + let contrast = fit.white - fit.black; + let threshold = (fit.white + fit.black) / 2.0; + let mut worst = f64::MAX; + let mut bits = 0u64; + let mut mirrored_bits = 0u64; + for (row, row_levels) in sample.levels.iter().enumerate() { + for (col, &v) in row_levels.iter().enumerate() { + if is_pilot(row, col) { + continue; + } + worst = worst.min((v - threshold).abs() / contrast); + if v > threshold { + bits |= 1 << data_bit_index(row, col); + mirrored_bits |= 1 << data_bit_index(row, GRID - 1 - col); + } + } + } + if worst < MIN_CELL_MARGIN { + return Readout::Unreadable { + reason: format!("ambiguous data cell (margin {worst:.2})"), + }; + } + for (candidate, mirrored) in [(bits, false), (mirrored_bits, true)] { + if let Some(marker) = FrameMarker::from_bits(candidate) { + if marker.kind != kind { + return Readout::Unreadable { + reason: format!( + "{} region carries a {} code", + kind.name(), + marker.kind.name() + ), + }; + } + return Readout::Marker { + marker, + margin: worst, + mirrored, + }; + } + } + Readout::Unreadable { + reason: "checksum mismatch".into(), + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FrameReadout { + pub screen: Readout, + pub camera: Readout, +} + +impl FrameReadout { + pub fn get(&self, kind: StreamKind) -> &Readout { + match kind { + StreamKind::Screen => &self.screen, + StreamKind::Camera => &self.camera, + } + } +} + +pub fn read_frame(img: &impl PixelSource) -> FrameReadout { + FrameReadout { + screen: read_stream(img, StreamKind::Screen), + camera: read_stream(img, StreamKind::Camera), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn packed(data: &[u8], w: usize, h: usize) -> PackedFrame<'_> { + PackedFrame { + data, + width: w, + height: h, + stride: w * 4, + order: PixelOrder::Rgba, + } + } + + #[test] + fn bits_round_trip_for_every_tag() { + for kind in StreamKind::ALL { + for segment in 0..=MAX_SEGMENT { + for index in [0, 1, 2, 255, 256, 4095, 30_000, MAX_INDEX] { + let marker = FrameMarker::new(kind, segment, index); + assert_eq!(FrameMarker::from_bits(marker.bits()), Some(marker)); + } + } + } + } + + #[test] + fn single_and_double_bit_errors_never_decode_as_another_marker() { + let marker = FrameMarker::new(StreamKind::Screen, 0, 1234); + let bits = marker.bits(); + for i in 0..DATA_CELLS { + assert_eq!(FrameMarker::from_bits(bits ^ (1 << i)), None); + for j in i + 1..DATA_CELLS { + assert_eq!(FrameMarker::from_bits(bits ^ (1 << i) ^ (1 << j)), None); + } + } + } + + #[test] + fn clean_frames_decode_at_many_sizes() { + for (w, h) in [(320, 180), (640, 360), (1280, 720), (100, 100), (64, 48)] { + for kind in StreamKind::ALL { + for index in [0, 7, 1000, MAX_INDEX] { + let marker = FrameMarker::new(kind, 2, index); + let data = marker_rgba(marker, w, h); + let readout = read_stream(&packed(&data, w, h), kind); + assert_eq!(readout.marker(), Some(marker), "{w}x{h} {readout:?}"); + } + } + } + } + + #[test] + fn other_stream_reads_absent() { + let data = marker_rgba(FrameMarker::new(StreamKind::Screen, 0, 5), 320, 180); + assert_eq!( + read_stream(&packed(&data, 320, 180), StreamKind::Camera), + Readout::Absent + ); + } +} diff --git a/crates/verify/tests/audio.rs b/crates/verify/tests/audio.rs new file mode 100644 index 00000000000..714ee5fbbf8 --- /dev/null +++ b/crates/verify/tests/audio.rs @@ -0,0 +1,1043 @@ +//! Audio oracle: every mic, system audio and music marker must be heard +//! exactly once, at the time and level the reference audio model computes, +//! in exports and in offline editor playback, and in sync with the video. + +mod audio_support; +mod common; + +use audio_support::{ExportPath, OfflinePlayer}; +use cap_project::{ClipSpeedAudioMode, ProjectConfiguration}; +use cap_verify::{ + audio_fixture, + audio_model::{self, AudioModel}, + fixture::FixtureSpec, + report::{self, Case, Tier}, +}; + +struct Variant { + edit: &'static str, + apply: fn(&mut ProjectConfiguration), +} + +fn full(_: &mut ProjectConfiguration) {} + +/// Cuts, a backward jump (1.0-2.0 plays again after 3.4-5.0) and a skip. +fn cuts(config: &mut ProjectConfiguration) { + common::set_cuts( + config, + &[ + (0.5, 2.6, 1.0), + (3.4, 5.0, 1.0), + (1.0, 2.0, 1.0), + (5.5, 7.8, 1.0), + ], + ); +} + +fn speeds(config: &mut ProjectConfiguration, mode: Option) { + common::set_cuts( + config, + &[ + (0.3, 2.0, 1.0), + (2.0, 4.0, 2.0), + (4.0, 5.0, 0.5), + (5.0, 7.0, 1.5), + (7.0, 7.8, 1.0), + ], + ); + for segment in &mut config.timeline.as_mut().expect("timeline").segments { + segment.speed_audio_mode = mode; + } +} + +/// Retimed segments in the default speed audio mode are silent. +fn speeds_muted(config: &mut ProjectConfiguration) { + speeds(config, None); +} + +/// Retimed segments resampled like tape (markers compressed in time). +fn speeds_match(config: &mut ProjectConfiguration) { + speeds(config, Some(ClipSpeedAudioMode::MatchSpeed)); +} + +/// Mic at -6 dB, system audio at the -30 dB mute floor, clip volume 0.5 on +/// the second segment and 0 on the third. +fn levels(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(0.2, 3.0, 1.0), (3.0, 5.5, 1.0), (5.5, 7.8, 1.0)]); + config.audio.mic_volume_db = -6.0; + config.audio.system_volume_db = -30.0; + let segments = &mut config.timeline.as_mut().expect("timeline").segments; + segments[1].volume = Some(0.5); + segments[2].volume = Some(0.0); +} + +/// Project mute: recording audio silent, timeline music still plays. +fn muted_with_music(config: &mut ProjectConfiguration) { + config.audio.mute = true; + audio_fixture::add_music(config, 1.0, 7.0, 0.5, -6.0); +} + +/// Music on top of cuts: music follows output time, not the edits. +fn music_over_cuts(config: &mut ProjectConfiguration) { + cuts(config); + audio_fixture::add_music(config, 0.75, 6.5, 2.0, -3.0); +} + +fn studio_sound(config: &mut ProjectConfiguration) { + cuts(config); + config.audio.improve = true; +} + +fn transition(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(0.3, 3.5, 1.0), (4.5, 7.8, 1.0)]); + common::add_transition(config, 1, 0.4); +} + +/// A long cross-fade: markers early in the overlap must keep the outgoing +/// clip's level and the incoming clip must ramp up (equal-power gains). +fn long_crossfade(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(0.3, 4.1, 1.0), (3.9, 7.8, 1.0)]); + common::add_transition(config, 1, 1.8); +} + +/// A long fade through black: each side fades linearly to silence over half +/// the overlap. +fn long_fade_black(config: &mut ProjectConfiguration) { + long_crossfade(config); + config.timeline.as_mut().expect("timeline").transitions[0].kind = + cap_project::ClipTransitionType::FadeThroughBlack; +} + +fn variants() -> Vec { + vec![ + Variant { + edit: "full", + apply: full, + }, + Variant { + edit: "cuts", + apply: cuts, + }, + Variant { + edit: "speeds-muted", + apply: speeds_muted, + }, + Variant { + edit: "speeds-match", + apply: speeds_match, + }, + Variant { + edit: "levels", + apply: levels, + }, + Variant { + edit: "muted-music", + apply: muted_with_music, + }, + Variant { + edit: "music-cuts", + apply: music_over_cuts, + }, + Variant { + edit: "studio-sound", + apply: studio_sound, + }, + Variant { + edit: "transition", + apply: transition, + }, + Variant { + edit: "long-crossfade", + apply: long_crossfade, + }, + Variant { + edit: "long-fade-black", + apply: long_fade_black, + }, + ] +} + +const FPS: u32 = 30; +const RES: (u32, u32) = (320, 180); + +fn export_case(spec: &FixtureSpec, variant: &Variant, path: ExportPath) -> Option { + export_case_at(spec, variant, path, FPS) +} + +fn export_case_at( + spec: &FixtureSpec, + variant: &Variant, + path: ExportPath, + fps: u32, +) -> Option { + let id = format!("audio/{}/{}/{}{fps}", spec.name, variant.edit, path.label()); + if !report::selected(&id) { + return None; + } + let project = audio_support::project(spec); + let rt = common::runtime(); + let mut config = project.config(); + (variant.apply)(&mut config); + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + match rt.block_on(audio_support::export( + &project, + config.clone(), + path, + fps, + RES, + dir.path(), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => match path { + ExportPath::Gif => { + if audio_support::has_audio_stream(&out) { + case.problem("audio-unexpected", "GIF export has an audio stream"); + } + } + ExportPath::Mov if !audio_support::has_audio_stream(&out) => { + case.note( + "MOV export carries no audio (the app uses MOV for cursor-only overlays)", + ); + } + _ => { + let model = audio_support::audio_model(&project, &config); + match audio_support::exported_mono(&out) { + Err(e) => case.problem("error", e), + Ok(mono) => { + let report = audio_support::check_pcm(&mut case, "export", &model, &mono); + let video = project.model(&config); + let frames = + common::read_output(&out, Some(&|i| video.expect(i as u32, fps))); + common::check_sequence(&mut case, &video, &frames, fps, true); + audio_support::check_av(&mut case, &project.meta, &report, &frames, fps); + // The audio track must cover the whole video, and end + // within one AAC frame (encoder padding) after it. + let audio_secs = mono.len() as f64 / f64::from(audio_model::ANALYSIS_RATE); + let video_secs = frames.len() as f64 / f64::from(fps); + let slack = 1024.0 / f64::from(audio_model::ANALYSIS_RATE) + 0.001; + if audio_secs < video_secs - 0.002 || audio_secs > video_secs + slack { + case.problem( + "audio-duration", + format!( + "audio track lasts {audio_secs:.4} s, video {video_secs:.4} s (slack {slack:.4} s)" + ), + ); + } + } + } + } + }, + } + report::settle(case).err() +} + +fn run_exports(spec: FixtureSpec, variants: &[Variant], paths: &[ExportPath]) { + let mut failures = Vec::new(); + for variant in variants { + for &path in paths { + failures.extend(export_case(&spec, variant, path)); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn export_mp4() { + run_exports( + audio_fixture::mix(report::tier()), + &variants(), + &[ExportPath::App, ExportPath::Plain], + ); +} + +/// Studio Sound at 10 fps (`cap export --fps 10`). The cuts' backward jump +/// sends the export through the editor's `AudioRenderer` instead of the +/// streaming mixer, and at 10 fps every video frame asks it for 4800 +/// samples: more than one 4096-sample Studio Sound block, so the blocks +/// after the first in a request must line up too. +#[test] +fn export_studio_sound_low_fps() { + let failures: Vec = export_case_at( + &audio_fixture::mix(report::tier()), + &Variant { + edit: "studio-sound", + apply: studio_sound, + }, + ExportPath::App, + 10, + ) + .into_iter() + .collect(); + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn studio_sound_music() { + let spec = audio_fixture::mix(report::tier()); + let project = audio_support::project(&spec); + let rt = common::runtime(); + let mut config = project.config(); + studio_sound(&mut config); + audio_fixture::add_music(&mut config, 1.0, 7.0, 0.5, -6.0); + let mut recording_config = config.clone(); + recording_config + .timeline + .as_mut() + .expect("timeline") + .audio_segments + .clear(); + let mut failures = Vec::new(); + for fps in [30, 10] { + for path in [ExportPath::App, ExportPath::Plain] { + let id = format!( + "audio/{}/studio-sound-music/{}{fps}", + spec.name, + path.label() + ); + if !report::selected(&id) { + continue; + } + let mut case = Case::new(&id); + let combined_dir = common::scratch_dir(&id); + let recording_dir = common::scratch_dir(&format!("{id}-recording")); + let outputs: Result>, String> = [ + (config.clone(), combined_dir.path()), + (recording_config.clone(), recording_dir.path()), + ] + .into_iter() + .map(|(config, dir)| { + rt.block_on(audio_support::export(&project, config, path, fps, RES, dir)) + .and_then(|out| audio_support::exported_mono(&out)) + }) + .collect(); + match outputs { + Ok(outputs) => audio_support::check_music_contribution( + &mut case, + &project, + &config, + &outputs[1], + &outputs[0], + ), + Err(error) => case.problem("error", error), + } + failures.extend(report::settle(case).err()); + } + } + let id = format!("audio/{}/studio-sound-music/playback", spec.name); + if report::selected(&id) { + let mut case = Case::new(&id); + let duration = audio_support::audio_model(&project, &config).duration; + let mut outputs = Vec::new(); + for config in [config.clone(), recording_config] { + let mut player = OfflinePlayer::open(&rt, &project, config); + player.play(0.0); + outputs.push(audio_support::mono(&player.pull(duration + 0.3))); + if player.starved > 0 { + case.problem("audio-starved", format!("{} pulls starved", player.starved)); + } + } + audio_support::check_music_contribution( + &mut case, + &project, + &config, + &outputs[1], + &outputs[0], + ); + failures.extend(report::settle(case).err()); + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn studio_sound_music_is_additive() { + let spec = audio_fixture::mix(report::tier()); + let id = format!("audio/{}/studio-sound-music/additive", spec.name); + if !report::selected(&id) { + return; + } + let project = audio_support::project(&spec); + let rt = common::runtime(); + let mut config = project.config(); + cuts(&mut config); + audio_fixture::add_music(&mut config, 1.0, 7.0, 0.5, -6.0); + let media = rt + .block_on(cap_editor::create_segments( + &project.meta, + project.meta.studio_meta().expect("studio"), + false, + )) + .expect("segments"); + let segments = rt.block_on(cap_editor::get_audio_segments(&media)); + let music = cap_editor::load_music_tracks_uncached(&config, &project.path); + let source = cap_audio::AudioData::from_file(project.path.join(audio_fixture::MUSIC_FILE)) + .expect("music source"); + assert_eq!(source.channels(), 2); + let frames = + (config.timeline.as_ref().expect("timeline").duration() * 48_000.0).round() as usize; + let render = |config: &ProjectConfiguration, block: usize| { + let mut renderer = + cap_editor::AudioRenderer::new(segments.clone()).with_music(music.clone()); + let mut pcm = Vec::new(); + while pcm.len() / 2 < frames { + let count = block.min(frames - pcm.len() / 2); + let (_, chunk) = renderer.render_frame_raw(count, config).expect("audio"); + assert!(!chunk.is_empty()); + pcm.extend(chunk); + } + pcm + }; + let mut case = Case::new(&id); + for improve in [false, true] { + config.audio.improve = improve; + let mut recording_config = config.clone(); + recording_config + .timeline + .as_mut() + .expect("timeline") + .audio_segments + .clear(); + for block in [1024, 1600, 4800] { + let combined = render(&config, block); + let recording = render(&recording_config, block); + assert_eq!(combined.len(), recording.len()); + let mut worst = 0.0f32; + for (sample, (combined, recording)) in combined.iter().zip(&recording).enumerate() { + assert!(combined.is_finite() && combined.abs() < 0.99); + let frame = sample / 2; + let expected = if (48_000..336_000).contains(&frame) { + source.samples()[(frame - 24_000) * 2 + sample % 2] * 10.0_f32.powf(-6.0 / 20.0) + } else { + 0.0 + }; + let difference = (combined - recording - expected).abs(); + worst = worst.max(difference); + assert!( + difference <= 1e-6, + "Studio Sound {improve}, block {block}, sample {sample}: music differs by {difference}" + ); + } + case.note(format!( + "Studio Sound {improve}, block {block}: music max sample error {worst:.9}" + )); + } + } + common::finish(case); +} + +/// Full decode of audio recorded at another rate than 48 kHz (44.1 kHz USB +/// microphones are common): the resampler's buffered tail must be flushed, +/// so the decoded track lasts exactly as long as the source, its last +/// milliseconds still carry the signal, and every sample sits at its time. +#[test] +fn resampled_decode_tail() { + const TONE_HZ: f64 = 1000.0; + const AMPLITUDE: f64 = 0.5; + let mut failures = Vec::new(); + for rate in [44_100u32, 32_000] { + let id = format!("audio/decode/resample-{rate}/tail"); + if !report::selected(&id) { + continue; + } + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + let path = dir.path().join("tone.wav"); + let frames = rate as usize; + let tone: Vec = (0..frames) + .map(|n| { + let t = n as f64 / f64::from(rate); + (AMPLITUDE * (2.0 * std::f64::consts::PI * TONE_HZ * t).sin() * 32767.0).round() + as i16 + }) + .collect(); + write_wav_mono(&path, rate, &tone); + match cap_audio::AudioData::from_file(&path) { + Err(e) => case.problem("error", format!("decode: {e}")), + Ok(data) => { + let channels = usize::from(data.channels()); + let mono: Vec = data + .samples() + .chunks(channels) + .map(|c| f64::from(c[0])) + .collect(); + let expected_len = frames * 48_000 / rate as usize; + if mono.len().abs_diff(expected_len) > 2 { + case.problem( + "duration", + format!( + "decoded {} samples at 48 kHz, the {frames}-sample {rate} Hz source lasts {expected_len} ({:+.2} ms)", + mono.len(), + (mono.len() as f64 - expected_len as f64) / 48.0 + ), + ); + } + let ideal = |m: usize| { + AMPLITUDE * (2.0 * std::f64::consts::PI * TONE_HZ * m as f64 / 48_000.0).sin() + }; + let worst = (4_800..mono.len().min(expected_len).saturating_sub(4_800)) + .map(|m| ((mono[m] - ideal(m)).abs(), m)) + .max_by(|a, b| a.0.total_cmp(&b.0)); + if let Some((error, m)) = worst + && error > 0.02 + { + case.problem( + "timing", + format!( + "sample {m} ({:.3} s) is {error:.3} away from the source tone at that time", + m as f64 / 48_000.0 + ), + ); + } + let tail = 240.min(mono.len()); + let rms = (mono[mono.len() - tail..].iter().map(|v| v * v).sum::() + / tail.max(1) as f64) + .sqrt(); + let tone_rms = AMPLITUDE / std::f64::consts::SQRT_2; + if rms < 0.6 * tone_rms { + case.problem( + "duration", + format!( + "the last 5 ms are {:.1} dB below the tone (rms {rms:.3}, tone {tone_rms:.3})", + 20.0 * (rms / tone_rms).log10() + ), + ); + } + case.note(format!( + "{} samples (expected {expected_len}), worst interior error {:.4}, tail rms {rms:.3}", + mono.len(), + worst.map_or(0.0, |w| w.0) + )); + } + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// A 16-bit PCM mono WAV file. +fn write_wav_mono(path: &std::path::Path, rate: u32, samples: &[i16]) { + let data_len = (samples.len() * 2) as u32; + let mut bytes = Vec::with_capacity(44 + data_len as usize); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_len).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&rate.to_le_bytes()); + bytes.extend_from_slice(&(rate * 2).to_le_bytes()); + bytes.extend_from_slice(&2u16.to_le_bytes()); + bytes.extend_from_slice(&16u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_len.to_le_bytes()); + for s in samples { + bytes.extend_from_slice(&s.to_le_bytes()); + } + std::fs::write(path, bytes).expect("write wav"); +} + +#[test] +fn export_other_formats() { + run_exports( + audio_fixture::mix(report::tier()), + &[Variant { + edit: "cuts", + apply: cuts, + }], + &[ExportPath::Mov, ExportPath::Gif], + ); +} + +#[test] +fn export_multi_clip() { + run_exports( + audio_fixture::multi(report::tier()), + &[ + Variant { + edit: "full", + apply: full, + }, + Variant { + edit: "reorder", + apply: |config| { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.segments.reverse(); + }, + }, + ], + &[ExportPath::App, ExportPath::Plain], + ); +} + +/// Plays the whole timeline from 0 through the offline output. +fn playback_case(spec: &FixtureSpec, variant: &Variant) -> Option { + let id = format!("audio/{}/{}/playback", spec.name, variant.edit); + if !report::selected(&id) { + return None; + } + let project = audio_support::project(spec); + let rt = common::runtime(); + let mut config = project.config(); + (variant.apply)(&mut config); + let model = audio_support::audio_model(&project, &config); + let mut case = Case::new(&id); + let mut player = OfflinePlayer::open(&rt, &project, config); + player.play(0.0); + let pcm = player.pull(model.duration + 0.3); + audio_support::check_pcm( + &mut case, + "play from 0", + &model.window(0.0, audio_support::frames_secs(&pcm)), + &audio_support::mono(&pcm), + ); + if player.starved > 0 { + case.problem( + "audio-starved", + format!("{} pulls starved (samples never rendered)", player.starved), + ); + } + report::settle(case).err() +} + +#[test] +fn playback() { + let spec = audio_fixture::mix(report::tier()); + let failures: Vec = variants() + .iter() + .filter_map(|variant| playback_case(&spec, variant)) + .collect(); + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn playback_multi_clip() { + let spec = audio_fixture::multi(report::tier()); + let failures: Vec = [Variant { + edit: "full", + apply: full, + }] + .iter() + .filter_map(|variant| playback_case(&spec, variant)) + .collect(); + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// One play/seek script step. +#[derive(Clone, Copy, Debug)] +enum Step { + /// Press play at a timeline position (a pause before it is implied). + Play(f64), + /// Live seek during playback. + Seek(f64), + /// Pull this many seconds of output. + Pull(f64), +} + +/// Runs a script; returns the output of every play/seek span as +/// (timeline start, interleaved PCM). +fn run_script(player: &mut OfflinePlayer, script: &[Step]) -> Vec<(f64, Vec)> { + let mut spans: Vec<(f64, Vec)> = Vec::new(); + for step in script { + match *step { + Step::Play(at) => { + player.play(at); + spans.push((at, Vec::new())); + } + Step::Seek(to) => { + player.seek(to); + spans.push((to, Vec::new())); + } + Step::Pull(secs) => { + let pcm = player.pull(secs); + spans.last_mut().expect("pull after play").1.extend(pcm); + } + } + } + spans +} + +fn span_model(model: &AudioModel, spans: &[(f64, Vec)]) -> Vec { + spans + .iter() + .map(|(start, pcm)| model.window(*start, audio_support::frames_secs(pcm))) + .collect() +} + +const SPAN: f64 = 1.2; + +/// Pause/play and live seeks: every span is checked against the model, and +/// every span's PCM must equal what main's player produces for it: a play +/// equals a fresh output's first play at that position; a live seek keeps +/// reading the render the play started. +fn transport_case( + spec: &FixtureSpec, + edit: &str, + apply: fn(&mut ProjectConfiguration), +) -> Option { + let id = format!("audio/{}/{edit}/transport", spec.name); + if !report::selected(&id) { + return None; + } + let project = audio_support::project(spec); + let rt = common::runtime(); + let mut config = project.config(); + apply(&mut config); + let model = audio_support::audio_model(&project, &config); + let mut case = Case::new(&id); + let script = [ + Step::Play(2.0), + Step::Pull(SPAN), + Step::Play(4.5), + Step::Pull(SPAN), + Step::Play(0.6), + Step::Pull(SPAN), + Step::Play(2.0), + Step::Pull(SPAN), + Step::Seek(5.2), + Step::Pull(SPAN), + Step::Seek(1.1), + Step::Pull(SPAN), + ]; + let mut player = OfflinePlayer::open(&rt, &project, config.clone()); + let spans = run_script(&mut player, &script); + for ((start, pcm), expected) in spans.iter().zip(span_model(&model, &spans)) { + audio_support::check_pcm( + &mut case, + &format!("span at {start:.2}s"), + &expected, + &audio_support::mono(pcm), + ); + } + // Reference PCM for every span from fresh outputs: a play is a fresh + // play there; a seek target is read from the play it happened in. + let mut play_start = 0.0; + let mut offset_in_play = 0.0; + let mut kinds = Vec::new(); + for step in &script { + match *step { + Step::Play(at) => { + play_start = at; + offset_in_play = 0.0; + kinds.push((at, None)); + } + Step::Seek(to) => kinds.push((to, Some((play_start, offset_in_play)))), + Step::Pull(secs) => offset_in_play += secs, + } + } + for ((start, pcm), (_, seek)) in spans.iter().zip(&kinds) { + let mut fresh = OfflinePlayer::open(&rt, &project, config.clone()); + let reference = match seek { + None => { + fresh.play(*start); + fresh.pull(audio_support::frames_secs(pcm)) + } + Some((play_at, _)) => { + fresh.play(*play_at); + fresh.seek(*start); + fresh.pull(audio_support::frames_secs(pcm)) + } + }; + if seek.is_some() { + let mut cold = OfflinePlayer::open(&rt, &project, config.clone()); + cold.play(*start); + let cold = cold.pull(audio_support::frames_secs(pcm)); + case.note(match audio_model::pcm_difference(pcm, &cold) { + None => format!("seek to {start:.2}s: also identical to a fresh play there"), + Some((db, _)) => format!( + "seek to {start:.2}s: differs from a fresh play there by {db:.1} dBFS (main keeps the play's render)" + ), + }); + } + let label = match seek { + None => format!("play at {start:.2}s"), + Some((play_at, _)) => format!("seek to {start:.2}s (play from {play_at:.2}s)"), + }; + match audio_model::pcm_difference(pcm, &reference) { + None => case.note(format!("{label}: PCM identical to a fresh output")), + Some((db, at)) => case.problem("pcm-mismatch", format!( + "{label}: PCM differs from a fresh output (peak difference {db:.1} dBFS, first at {:.4}s into the span)", + (at / audio_support::CHANNELS) as f64 / f64::from(audio_support::RATE) + )), + } + } + if player.starved > 0 { + case.problem("audio-starved", format!("{} pulls starved", player.starved)); + } + report::settle(case).err() +} + +#[test] +fn transport() { + let spec = audio_fixture::mix(report::tier()); + let mut failures = Vec::new(); + failures.extend(transport_case(&spec, "full", full)); + failures.extend(transport_case(&spec, "studio-sound-full", |config| { + config.audio.improve = true; + })); + if report::tier() == Tier::Full { + failures.extend(transport_case(&spec, "cuts", cuts)); + failures.extend(transport_case(&spec, "studio-sound-cuts", studio_sound)); + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// The oracle itself: real playback PCM passes, and the same PCM with one +/// device block (10.7 ms) of inserted silence, one dropped block, a +/// repeated half second, a 6 ms shift (tolerance 5 ms) or a 6 dB drop on one source fails +/// with the right class. +#[test] +fn oracle_self_check() { + let spec = audio_fixture::mix(report::tier()); + let id = format!("audio/{}/oracle-self-check", spec.name); + if !report::selected(&id) { + return; + } + let project = audio_support::project(&spec); + let rt = common::runtime(); + let mut config = project.config(); + music_over_cuts(&mut config); + let model = audio_support::audio_model(&project, &config); + let mut player = OfflinePlayer::open(&rt, &project, config); + player.play(0.0); + let pcm = audio_support::mono(&player.pull(model.duration + 0.3)); + let model = model.window(0.0, pcm.len() as f64 / f64::from(audio_support::RATE)); + let at = |secs: f64| (secs * f64::from(audio_support::RATE)) as usize; + let block = audio_support::BLOCK; + let gap = { + let mut v = pcm.clone(); + v.splice(at(3.0)..at(3.0), std::iter::repeat_n(0.0, block)); + v + }; + let dropped = { + let mut v = pcm.clone(); + v.drain(at(3.0)..at(3.0) + block); + v + }; + let repeated = { + let mut v = pcm.clone(); + let copy = v[at(2.0)..at(2.5)].to_vec(); + v.splice(at(2.5)..at(2.5), copy); + v + }; + let shifted = { + let mut v = vec![0.0; at(0.006)]; + v.extend_from_slice(&pcm); + v + }; + let mut case = Case::new(&id); + let classes = |mono: &[f32]| { + audio_model::check(&model, &audio_model::hear(mono, 0.0, &model.rate_windows)).classes() + }; + let original = classes(&pcm); + if !original.is_empty() { + case.problem("error", format!("unmodified playback fails: {original:?}")); + } + for (label, mono, must) in [ + ("10.7 ms gap", &gap, "audio-timing"), + ("10.7 ms drop", &dropped, "audio-timing"), + ("0.5 s repeat", &repeated, "audio-unexpected"), + ("6 ms shift", &shifted, "audio-timing"), + ] { + let found = classes(mono); + if found.contains(&must) { + case.note(format!("{label}: caught as {found:?}")); + } else { + case.problem( + "error", + format!("{label} not caught as {must} (classes {found:?})"), + ); + } + } + let quiet = { + let mut config = project.config(); + music_over_cuts(&mut config); + config.audio.system_volume_db = -6.0; + audio_support::audio_model(&project, &config) + }; + let found = audio_model::check( + &quiet.window(0.0, pcm.len() as f64 / f64::from(audio_support::RATE)), + &audio_model::hear(&pcm, 0.0, &quiet.rate_windows), + ) + .classes(); + if found.contains(&"audio-level") { + case.note(format!("6 dB system level error: caught as {found:?}")); + } else { + case.problem( + "error", + format!("6 dB level error not caught (classes {found:?})"), + ); + } + common::finish(case); +} + +/// Stereo system audio recorded on the left channel only must come out on +/// the left channel only. Every other case mixes identical channels and +/// checks the channel average, so it cannot see swapped or summed channels. +#[test] +fn stereo_channels() { + use cap_verify::{ + audio_code::{self, AudioSource}, + fixture, + media::{self, AudioCodec}, + }; + let id = format!("audio/stereo-left/full/{}{FPS}", ExportPath::App.label()); + if !report::selected(&id) { + return; + } + common::init(); + let mut spec = audio_fixture::mix(report::tier()); + spec.name = format!("{}-left", spec.name); + spec.segments[0].mic = None; + let path = fixture::ensure_fixture(&spec, |spec, dir| { + audio_fixture::write(spec, dir)?; + let file = dir.join("content/segments/segment-0/system_audio.ogg"); + let decoded = media::decode_audio(&file, None)?; + let left = decoded.channels[0].clone(); + let right = vec![0.0; left.len()]; + media::write_audio( + &file, + &[left, right], + decoded.sample_rate, + AudioCodec::Opus, + 96_000, + ) + }) + .unwrap_or_else(|e| panic!("fixture {}: {e:?}", spec.name)); + let project = common::load(&spec.name, path); + let config = project.config(); + let model = audio_support::audio_model(&project, &config); + let required = model + .expected + .iter() + .filter(|e| { + e.source == AudioSource::System && e.presence == audio_model::Presence::Required + }) + .count(); + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + match common::runtime().block_on(audio_support::export( + &project, + config, + ExportPath::App, + FPS, + RES, + dir.path(), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => match media::decode_audio(&out, Some(audio_model::ANALYSIS_RATE)) { + Err(e) => case.problem("error", format!("decode audio: {e:?}")), + Ok(decoded) if decoded.channels.len() != 2 => case.problem( + "audio-channels", + format!( + "export has {} audio channels, expected 2", + decoded.channels.len() + ), + ), + Ok(decoded) => { + let heard = |channel: &[f32]| { + audio_code::locate_markers( + channel, + audio_model::ANALYSIS_RATE, + AudioSource::System, + ) + .into_iter() + .filter(|m| m.index.is_some() && m.gain > 0.02) + .count() + }; + let (left, right) = (heard(&decoded.channels[0]), heard(&decoded.channels[1])); + if left < required || right > 0 { + case.problem( + "audio-channels", + format!( + "left-only system audio: {left} markers heard on the left (expected {required}), {right} on the right (expected 0)" + ), + ); + } + } + }, + } + common::finish(case); +} + +/// A recording whose mic gap summary reports a stale startup burst: the +/// editor plays the mic earlier by the trimmed time (stale-startup repair), +/// in exports and in playback. +/// +/// The trim is one whole marker period (500 ms), so the repaired mic markers +/// keep their own schedule phase and stay clear of the system-audio markers. +/// A 300 ms trim moved every mic marker to 33 ms after the start of the +/// system marker with the same index; the two overlapped for 87 ms and the +/// overlapped system marker sometimes failed to decode. +#[test] +fn timing_repair() { + use cap_verify::fixture; + const TRIMMED_MS: u32 = 500; + common::init(); + let mut spec = audio_fixture::mix(report::tier()); + spec.name = format!("{}-repair{TRIMMED_MS}", spec.name); + let path = fixture::ensure_fixture(&spec, |spec, dir| { + audio_fixture::write(spec, dir)?; + let meta_path = dir.join("recording-meta.json"); + let mut meta: serde_json::Value = serde_json::from_slice(&std::fs::read(&meta_path)?)?; + meta["segments"][0]["mic"]["gap_summary"] = serde_json::json!({ + "total_overlap_trimmed_ms": TRIMMED_MS, + "startup_overlap_trimmed_ms": TRIMMED_MS, + "overlap_dropped_frames": 4, + "startup_overlap_drops": 4 + }); + std::fs::write(&meta_path, serde_json::to_vec_pretty(&meta)?)?; + Ok(()) + }) + .unwrap_or_else(|e| panic!("fixture {}: {e:?}", spec.name)); + let project = common::load(&spec.name, path); + let config = project.config(); + let model = audio_support::audio_model(&project, &config); + let mut failures = Vec::new(); + for path in [ExportPath::App, ExportPath::Plain] { + let id = format!("audio/{}/full/{}{FPS}", spec.name, path.label()); + if !report::selected(&id) { + continue; + } + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + match common::runtime().block_on(audio_support::export( + &project, + config.clone(), + path, + FPS, + RES, + dir.path(), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => match audio_support::exported_mono(&out) { + Err(e) => case.problem("error", e), + Ok(mono) => { + audio_support::check_pcm(&mut case, "export", &model, &mono); + } + }, + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + let id = format!("audio/{}/full/playback", spec.name); + if report::selected(&id) { + let rt = common::runtime(); + let mut case = Case::new(&id); + let mut player = OfflinePlayer::open(&rt, &project, config); + player.play(0.0); + let pcm = player.pull(model.duration + 0.3); + audio_support::check_pcm( + &mut case, + "play from 0", + &model.window(0.0, audio_support::frames_secs(&pcm)), + &audio_support::mono(&pcm), + ); + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + common::assert_passed(failures); +} diff --git a/crates/verify/tests/audio_support/mod.rs b/crates/verify/tests/audio_support/mod.rs new file mode 100644 index 00000000000..07e8500e3e3 --- /dev/null +++ b/crates/verify/tests/audio_support/mod.rs @@ -0,0 +1,483 @@ +#![allow(dead_code)] + +//! Shared pieces of the audio, captions and editing tests: marker fixtures, +//! the offline playback driver and export helpers. + +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +use cap_editor::{AudioOutput, AudioSegment, MusicTracks, OfflineAudioSink, PlaySpec}; +use cap_project::{ProjectConfiguration, RecordingMeta, StudioRecordingMeta, XY}; +use cap_verify::{ + audio_fixture, + audio_model::{self, AudioModel, AudioReport, AudioTruth}, + fixture::{self, FixtureSpec, SourceTruth}, + media, + report::Case, + video_code::StreamKind, +}; +use tokio::sync::watch; + +use crate::common::{self, OutputFrame, Project}; + +pub const RATE: u32 = cap_editor::HEADLESS_SAMPLE_RATE; +pub const CHANNELS: usize = cap_editor::HEADLESS_CHANNELS as usize; +pub const BLOCK: usize = cap_editor::HEADLESS_BLOCK_FRAMES; +/// A pull waits at most this long for its samples to render. +const RENDER_TIMEOUT: Duration = Duration::from_secs(60); + +pub fn project(spec: &FixtureSpec) -> Project { + common::init(); + let path = fixture::ensure_fixture(spec, audio_fixture::write) + .unwrap_or_else(|e| panic!("fixture {}: {e:?}", spec.name)); + let meta = RecordingMeta::load_for_project(&path).expect("fixture meta"); + let truth = SourceTruth::read(&path).expect("fixture truth"); + Project { + name: spec.name.clone(), + path, + meta, + truth, + } +} + +pub fn audio_model(project: &Project, config: &ProjectConfiguration) -> AudioModel { + let truth = AudioTruth::read(&project.path, config).expect("audio truth"); + AudioModel::new(&project.meta, config, &truth).expect("audio model") +} + +/// Records an audio verdict on a case. +pub fn judge(case: &mut Case, label: &str, report: &AudioReport) { + if report.passed() { + case.note(format!("{label}: {}", report.summary())); + } else { + let classes = report.classes(); + for class in &classes[1..] { + case.classes.insert((*class).to_string()); + } + case.problem(classes[0], format!("{label}: {}", report.summary())); + } +} + +/// Checks a mono output starting at output time `start_secs`. +pub fn check_pcm(case: &mut Case, label: &str, model: &AudioModel, mono: &[f32]) -> AudioReport { + let heard = audio_model::hear(mono, 0.0, &model.rate_windows); + let report = audio_model::check(model, &heard); + if std::env::var_os("CAP_VERIFY_DUMP_AUDIO").is_some() { + for h in &heard { + eprintln!(" heard {label}: {h:?}"); + } + for e in &model.expected { + eprintln!(" expected {label}: {e:?}"); + } + } + judge(case, label, &report); + report +} + +pub fn check_music_contribution( + case: &mut Case, + project: &Project, + config: &ProjectConfiguration, + recording: &[f32], + combined: &[f32], +) { + let mut recording_config = config.clone(); + recording_config + .timeline + .as_mut() + .expect("timeline") + .audio_segments + .clear(); + check_pcm( + case, + "recording", + &audio_model(project, &recording_config), + recording, + ); + if recording.len() != combined.len() { + case.problem( + "audio-duration", + format!( + "adding music changed the sample count: {} to {}", + recording.len(), + combined.len() + ), + ); + return; + } + // Enhanced mic markers overlap the music's chirp band. Subtraction removes + // their unknown waveform while retaining music timing, payload and gain checks. + let music: Vec = combined + .iter() + .zip(recording) + .map(|(combined, recording)| combined - recording) + .collect(); + let mut model = audio_model(project, config); + model + .expected + .retain(|marker| marker.source == cap_verify::audio_code::AudioSource::Media); + check_pcm(case, "music contribution", &model, &music); +} + +/// Plays a project's audio through the editor's real output source +/// pipeline (`AudioOutput::play` -> `install_source` -> pre-render buffer +/// -> playhead sync -> fill), pulled offline block by block. +pub struct OfflinePlayer { + output: AudioOutput, + sink: OfflineAudioSink, + segments: Vec, + music: MusicTracks, + pub config: ProjectConfiguration, + pub duration: f64, + playhead: Option>, + generation: Option, + /// Pulls whose samples did not render in time (silence was played). + pub starved: usize, +} + +impl OfflinePlayer { + pub fn open( + rt: &tokio::runtime::Runtime, + project: &Project, + config: ProjectConfiguration, + ) -> Self { + let meta = project.meta.studio_meta().expect("studio").clone(); + let segments = rt.block_on(async { + let media = cap_editor::create_segments(&project.meta, &meta, false) + .await + .expect("segments"); + cap_editor::get_audio_segments(&media).await + }); + let music = cap_editor::load_music_tracks_uncached(&config, &project.path); + let (output, sink) = AudioOutput::new_offline(); + let duration = config.timeline.as_ref().map_or(0.0, |t| t.duration()); + Self { + output, + sink, + segments, + music, + config, + duration, + playhead: None, + generation: None, + starved: 0, + } + } + + /// Presses play at `start` (stopping any current playback first). + pub fn play(&mut self, start: f64) { + self.stop(); + let (tx, rx) = watch::channel(start); + let generation = self + .output + .play(PlaySpec { + segments: self.segments.clone(), + music: self.music.clone(), + project: self.config.clone(), + duration_secs: self.duration, + start_playhead_secs: start, + playhead_rx: rx, + }) + .expect("offline audio output accepted the play request"); + self.playhead = Some(tx); + self.generation = Some(generation); + } + + /// Live seek while playing, as the playback loop sends it. + pub fn seek(&mut self, to: f64) { + if let Some(tx) = &self.playhead { + let _ = tx.send(to); + } + } + + pub fn stop(&mut self) { + if let Some(generation) = self.generation.take() { + self.output.stop_playback(generation); + } + self.playhead = None; + } + + /// Pulls at least `secs` of interleaved output in device-sized blocks. + pub fn pull(&mut self, secs: f64) -> Vec { + let blocks = (secs * f64::from(RATE) / BLOCK as f64).ceil() as usize; + let mut out = Vec::with_capacity(blocks * BLOCK * CHANNELS); + let mut block = vec![0.0f32; BLOCK * CHANNELS]; + for _ in 0..blocks { + if !self.sink.pull(&mut block, RENDER_TIMEOUT) { + self.starved += 1; + } + out.extend_from_slice(&block); + } + out + } +} + +impl Drop for OfflinePlayer { + fn drop(&mut self) { + self.stop(); + } +} + +pub fn frames_secs(interleaved: &[f32]) -> f64 { + (interleaved.len() / CHANNELS) as f64 / f64::from(RATE) +} + +pub fn mono(interleaved: &[f32]) -> Vec { + audio_model::interleaved_mono(interleaved, CHANNELS) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExportPath { + /// What the app does: `build_for_mp4` (streaming audio when eligible) + /// then `export_prepared`. + App, + /// `build` then `export` (CLI and other callers). + Plain, + Mov, + Gif, +} + +impl ExportPath { + pub fn label(self) -> &'static str { + match self { + ExportPath::App => "mp4-app", + ExportPath::Plain => "mp4-plain", + ExportPath::Mov => "mov", + ExportPath::Gif => "gif", + } + } + + fn extension(self) -> &'static str { + match self { + ExportPath::App | ExportPath::Plain => "mp4", + ExportPath::Mov => "mov", + ExportPath::Gif => "gif", + } + } +} + +pub async fn export( + project: &Project, + config: ProjectConfiguration, + path: ExportPath, + fps: u32, + resolution: (u32, u32), + dir: &Path, +) -> Result { + let out = dir.join(format!("out.{}", path.extension())); + let builder = cap_export::ExporterBase::builder(project.path.clone()) + .with_config(config) + .with_output_path(out); + let resolution_base = XY::new(resolution.0, resolution.1); + let mp4 = cap_export::mp4::Mp4ExportSettings { + fps, + resolution_base, + compression: cap_export::mp4::ExportCompression::Social, + custom_bpp: None, + force_ffmpeg_decoder: false, + optimize_filesize: false, + }; + match path { + ExportPath::App => { + let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let base = builder + .build_for_mp4(cancel) + .await + .map_err(|e| format!("exporter build: {e}"))?; + mp4.export_prepared(base, |_| true).await + } + ExportPath::Plain => { + let base = builder + .build() + .await + .map_err(|e| format!("exporter build: {e}"))?; + mp4.export(base, |_| true).await + } + ExportPath::Mov => { + let base = builder + .build() + .await + .map_err(|e| format!("exporter build: {e}"))?; + cap_export::mov::MovExportSettings { + fps, + resolution_base, + cursor_only: false, + } + .export(base, |_| true) + .await + } + ExportPath::Gif => { + let base = builder + .build() + .await + .map_err(|e| format!("exporter build: {e}"))?; + cap_export::gif::GifExportSettings { + fps, + resolution_base, + quality: None, + } + .export(base, |_| true) + .await + } + } +} + +/// Whether a media file has an audio stream. +pub fn has_audio_stream(path: &Path) -> bool { + media::init(); + ffmpeg::format::input(&path) + .map(|input| input.streams().best(ffmpeg::media::Type::Audio).is_some()) + .unwrap_or(false) +} + +/// Decodes an exported file's audio as mono at the analysis rate, with +/// sample 0 at output time 0. +pub fn exported_mono(path: &Path) -> Result, String> { + let decoded = media::decode_audio(path, Some(audio_model::ANALYSIS_RATE)) + .map_err(|e| format!("decode audio: {e:?}"))?; + let mut mono = decoded.mono(); + let lead = (decoded.start_secs * f64::from(audio_model::ANALYSIS_RATE)).round() as i64; + if lead > 0 { + let mut padded = vec![0.0; lead as usize]; + padded.extend_from_slice(&mono); + mono = padded; + } else if lead < 0 { + mono.drain(..(-lead as usize).min(mono.len())); + } + Ok(mono) +} + +/// Per clip: (display alignment offset, display nominal fps). +pub fn display_timing(meta: &RecordingMeta) -> Vec<(f64, u32)> { + let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else { + return Vec::new(); + }; + inner + .segments + .iter() + .map(|segment| { + let starts = [ + segment.display.start_time, + segment.camera.as_ref().and_then(|c| c.start_time), + segment.mic.as_ref().and_then(|m| m.start_time), + segment.system_audio.as_ref().and_then(|s| s.start_time), + ]; + let latest = starts + .iter() + .flatten() + .copied() + .fold(f64::NEG_INFINITY, f64::max); + let offset = match segment.display.start_time { + Some(start) if latest.is_finite() => latest - start, + _ => 0.0, + }; + (offset, segment.display.fps) + }) + .collect() +} + +/// A/V offset per heard recording marker: the output time the marker is +/// heard minus the output time the screen first shows the source frame +/// that contains the marker's recording time. Must be within one output +/// frame (or one source frame at the segment's speed) plus 5 ms. +pub fn check_av( + case: &mut Case, + meta: &RecordingMeta, + report: &AudioReport, + frames: &[OutputFrame], + fps: u32, +) { + let timing = display_timing(meta); + // Screen frames the video oracle already found wrong show the wrong + // source frame, so they cannot time a marker: a marker whose first + // showing is one of them is not measured (the video failure is judged + // on its own). + let wrong: std::collections::BTreeSet = case + .streams + .iter() + .filter(|s| s.stream == Some(StreamKind::Screen)) + .flat_map(|s| s.mismatches.iter().map(|m| m.output_frame as usize)) + .collect(); + let mut worst = 0.0f64; + let mut bad = Vec::new(); + let mut checked = 0; + let mut unmeasured = 0; + for (expected, heard) in &report.pairs { + // Partial markers and markers inside a transition (a video blend) + // have no single source frame to compare with. + if expected.presence != audio_model::Presence::Required || expected.blended { + continue; + } + let Some((clip, recording_time)) = expected.recording else { + continue; + }; + let Some(&(offset, src_fps)) = timing.get(clip) else { + continue; + }; + let slot = ((recording_time + offset) * f64::from(src_fps)).floor() as i64; + // Search near the heard time only: a timeline can show the same + // source frame again elsewhere (backward jumps, reorders). + let first = ((heard.time_secs - 0.2) * f64::from(fps)).floor().max(0.0) as usize; + let shown = frames + .iter() + .enumerate() + .skip(first) + .take((0.4 * f64::from(fps)).ceil() as usize) + .find(|(_, f)| { + f.readout.get(StreamKind::Screen).marker().is_some_and(|m| { + usize::from(m.segment) == clip + && (slot..=slot + i64::from(src_fps) / 4).contains(&i64::from(m.index)) + }) + }); + let Some((i, _)) = shown else { + continue; + }; + if wrong.contains(&i) { + unmeasured += 1; + continue; + } + checked += 1; + let av = heard.time_secs - i as f64 / f64::from(fps); + let limit = (1.0 / f64::from(fps)).max(1.0 / (f64::from(src_fps) * heard.rate)) + 0.005; + worst = worst.max(av.abs()); + if av.abs() > limit { + bad.push(format!( + "{} #{} heard at {:.4}s, its source frame first shown at {:.4}s ({:+.1} ms)", + expected.source.name(), + expected.index, + heard.time_secs, + i as f64 / f64::from(fps), + av * 1000.0 + )); + } + } + let summary = format!( + "A/V: {checked} markers, worst offset {:.1} ms (limit one frame + 5 ms)", + worst * 1000.0 + ); + if unmeasured > 0 { + case.note(format!( + "A/V: {unmeasured} markers not measured: their source frame first shows on a frame the video oracle found wrong" + )); + if checked == 0 { + case.unobserved( + "av-sync", + format!("all {unmeasured} timeable markers first show on wrong video frames"), + ); + } + } + if bad.is_empty() { + case.note(summary); + } else { + case.problem( + "av-sync", + format!( + "{summary}, {} out of sync (first: {})", + bad.len(), + bad.iter().take(3).cloned().collect::>().join("; ") + ), + ); + } +} diff --git a/crates/verify/tests/captions.rs b/crates/verify/tests/captions.rs new file mode 100644 index 00000000000..8d34a8e0bf7 --- /dev/null +++ b/crates/verify/tests/captions.rs @@ -0,0 +1,1913 @@ +//! Captions oracle, with no transcription model anywhere: +//! +//! - `projection_*`: random source-timed captions (gaps, punctuation, words +//! stretched across silences, text-only captions, style overrides) through +//! random edit sequences (`edits.rs`) must project onto the output timeline +//! exactly as the reference model below says (`derive_caption_track_segments` +//! via `synchronize_captions`, the call the editor, export and render farm +//! make). +//! - `legacy_migration`: output-timed captions from older projects are moved +//! to source time on load; checked against the model and for round trips. +//! - `round_trip_on_disk`: captions, settings and overrides survive +//! project-config.json and the legacy captions.json sidecar. +//! - `transcription_audio`: a fake transcriber hears the fixture's audio +//! markers in the transcription input assembled the way the app does +//! (per-take mic + system audio aligned by the clip offsets, takes +//! concatenated by display duration) and returns one word per marker; every +//! word must sit at its source time. +//! - `render_*`: exports with captions; every output frame must show exactly +//! the caption the model puts there (identified against per-text glyph +//! templates), or none. +//! - `editor_open`: opening a project in the editor engine migrates and +//! re-projects stored captions as the model says. +//! +//! Reference model (documented rules of the caption projection, written for +//! the test): +//! - Source time is the recordings concatenated: clip `c` starts at the sum +//! of the earlier clips' display durations. +//! - A word's end is clamped to `start + 2.5 s`; a caption with words ends at +//! its last clamped word. +//! - Each caption is cut by every timeline segment, in timeline order: the +//! part of each word inside the segment's source range (non-empty) is +//! placed at `segment output start + (t - range start) / timescale` and +//! moved past fullscreen-text holds (a range ending exactly at a hold ends +//! before it). A piece spans its first to last word; its text joins the +//! words with spaces, except before punctuation that attaches to the +//! previous word. A caption without words maps its own range. +//! - A caption cut into several pieces gets ids `::edl`; pieces are +//! sorted by output start (stable); style overrides follow the source id. +//! - Rendering: at output time `t` the first piece with +//! `start <= t < min(end, last word start + 2.5)` is shown. + +mod common; +mod edits; + +use std::path::Path; + +use cap_project::{ + CaptionSegment, CaptionSettings, CaptionTrackSegment, CaptionWord, CaptionsData, + ProjectConfiguration, RecordingMeta, StudioRecordingMeta, +}; +use cap_verify::{ + audio_code::{self, AudioSource, MARKER_SECS}, + catalog, + fixture::{AudioTrackSpec, FixtureSpec, Rng, SegmentSpec, Timing, VideoTrackSpec}, + media, + report::{self, Case, Tier}, + video_code::PixelSource, +}; +use edits::{Op, OpMix, TimeModel}; + +const MAX_WORD_SECS: f64 = 2.5; +/// Projected times are stored as f32. +const TIME_TOLERANCE: f64 = 1e-4; +const WORD_TOLERANCE_SECS: f64 = 0.001; + +/// Settles a case, printing its notes (the core report prints them only in +/// the JSON report). +fn settle(case: Case) -> Option { + for note in &case.notes { + eprintln!("[cap-verify] {}: {note}", case.id); + } + report::settle(case).err() +} + +fn finish(case: Case) { + if let Some(message) = settle(case) { + panic!("{message}"); + } +} + +fn seeds(fast: u64, full: u64) -> u64 { + match report::tier() { + Tier::Fast => fast, + Tier::Full => full, + } +} + +fn attaches_to_previous(text: &str) -> bool { + const ATTACHING: &str = ",.!?;:%)]}'’、。！？；：，"; + text.trim() + .chars() + .next() + .is_some_and(|c| ATTACHING.contains(c)) +} + +fn join_words<'a>(words: impl IntoIterator) -> String { + let mut text = String::new(); + for word in words.into_iter().map(str::trim).filter(|w| !w.is_empty()) { + if !text.is_empty() && !attaches_to_previous(word) { + text.push(' '); + } + text.push_str(word); + } + text +} + +#[derive(Clone, Debug, Default)] +struct Overrides { + fade: Option, + position: Option, + color: Option, + font_size: Option, +} + +#[derive(Clone, Debug)] +struct Piece { + id: String, + start: f64, + end: f64, + text: String, + words: Vec<(String, f64, f64)>, + overrides: Overrides, +} + +impl Piece { + /// Where the renderer stops showing the piece. + fn visible_end(&self) -> f64 { + match self.words.last() { + Some(w) => self.end.min(w.1 + MAX_WORD_SECS), + None => self.end, + } + } +} + +fn clip_starts(durations: &[f64]) -> Vec { + let mut starts = vec![0.0]; + for d in durations { + starts.push(starts.last().unwrap() + d); + } + starts +} + +/// Projects source-timed captions through `config`'s edit list. +fn model_project( + captions: &[CaptionSegment], + config: &ProjectConfiguration, + durations: &[f64], +) -> Vec { + let time = TimeModel::new(config); + let starts = clip_starts(durations); + let mut overrides: Vec<(String, Overrides)> = Vec::new(); + for t in &edits::timeline(config).caption_segments { + let source = t.id.split("::edl").next().unwrap_or(&t.id).to_string(); + if !overrides.iter().any(|(id, _)| *id == source) { + overrides.push(( + source, + Overrides { + fade: t.fade_duration_override, + position: t.position_override.clone(), + color: t.color_override.clone(), + font_size: t.font_size_override, + }, + )); + } + } + let mut out = Vec::new(); + for caption in captions { + let words: Vec<(String, f64, f64)> = caption + .words + .iter() + .map(|w| { + let start = f64::from(w.start); + let end = f64::from(w.end).min(start + MAX_WORD_SECS); + (w.text.clone(), start, end) + }) + .collect(); + let mut pieces = Vec::new(); + for s in &time.segments { + let lo = starts[s.clip as usize] + s.start; + let hi = starts[s.clip as usize] + s.end; + let place = |a: f64, b: f64| -> Option<(f64, f64)> { + let (a, b) = (a.max(lo), b.min(hi)); + (a < b).then(|| { + ( + time.to_output(s.out_start + (a - lo) / s.timescale), + time.to_output_end(s.out_start + (b - lo) / s.timescale), + ) + }) + }; + if words.is_empty() { + let end = f64::from(caption.end); + if let Some((a, b)) = place(f64::from(caption.start), end) { + pieces.push((a, b, caption.text.clone(), Vec::new())); + } + continue; + } + let placed: Vec<(String, f64, f64)> = words + .iter() + .filter_map(|(text, a, b)| place(*a, *b).map(|(a, b)| (text.clone(), a, b))) + .collect(); + if let (Some(first), Some(last)) = (placed.first(), placed.last()) { + let text = join_words(placed.iter().map(|w| w.0.as_str())); + pieces.push((first.1, last.2, text, placed.clone())); + } + } + let count = pieces.len(); + let source_overrides = overrides + .iter() + .find(|(id, _)| *id == caption.id) + .map(|(_, o)| o.clone()) + .unwrap_or_default(); + for (i, (start, end, text, words)) in pieces.into_iter().enumerate() { + out.push(Piece { + id: if count == 1 { + caption.id.clone() + } else { + format!("{}::edl{i}", caption.id) + }, + start, + end, + text, + words, + overrides: source_overrides.clone(), + }); + } + } + out.sort_by(|a, b| a.start.total_cmp(&b.start)); + out +} + +/// Moves output-timed (legacy) captions to source time: a caption keeps +/// only if both ends map; unmappable words drop. +fn model_migrate( + legacy: &[CaptionSegment], + config: &ProjectConfiguration, + durations: &[f64], +) -> Vec { + let time = TimeModel::new(config); + let starts = clip_starts(durations); + let to_source = |o: f32| time.output_to_source(f64::from(o), &starts); + let mut out: Vec = legacy + .iter() + .filter_map(|c| { + Some(CaptionSegment { + id: c.id.clone(), + start: to_source(c.start)? as f32, + end: to_source(c.end)? as f32, + text: c.text.clone(), + words: c + .words + .iter() + .filter_map(|w| { + Some(CaptionWord { + text: w.text.clone(), + start: to_source(w.start)? as f32, + end: to_source(w.end)? as f32, + }) + }) + .collect(), + }) + }) + .collect(); + out.sort_by(|a, b| a.start.total_cmp(&b.start)); + out +} + +fn close(a: f64, b: f64) -> bool { + (a - b).abs() <= TIME_TOLERANCE +} + +fn describe(p: &Piece) -> String { + format!("{} {:.4}-{:.4} \"{}\"", p.id, p.start, p.end, p.text) +} + +fn describe_track(t: &CaptionTrackSegment) -> String { + format!("{} {:.4}-{:.4} \"{}\"", t.id, t.start, t.end, t.text) +} + +/// Indices of `starts` in output order, pieces starting together (within +/// the time tolerance) ordered by id: which of two pieces that start +/// together is listed first is not specified. +fn tie_order(items: &[(f64, &str)]) -> Vec { + let mut order: Vec = (0..items.len()).collect(); + order.sort_by(|&a, &b| items[a].0.total_cmp(&items[b].0)); + let mut i = 0; + while i < order.len() { + let mut j = i + 1; + while j < order.len() && items[order[j]].0 - items[order[j - 1]].0 <= TIME_TOLERANCE { + j += 1; + } + order[i..j].sort_by(|&a, &b| items[a].1.cmp(items[b].1)); + i = j; + } + order +} + +/// Compares the engine's caption track with the model's pieces. +fn compare_track(product: &[CaptionTrackSegment], model: &[Piece]) -> Vec { + let product_order = tie_order( + &product + .iter() + .map(|p| (p.start, p.id.as_str())) + .collect::>(), + ); + let model_order = tie_order( + &model + .iter() + .map(|p| (p.start, p.id.as_str())) + .collect::>(), + ); + let product: Vec<&CaptionTrackSegment> = product_order.iter().map(|&i| &product[i]).collect(); + let model: Vec<&Piece> = model_order.iter().map(|&i| &model[i]).collect(); + let mut problems = Vec::new(); + if product.len() != model.len() { + problems.push(format!( + "{} caption track segments, the model expects {}", + product.len(), + model.len() + )); + } + for (p, m) in product.iter().zip(&model) { + let words_match = p.words.len() == m.words.len() + && p.words.iter().zip(&m.words).all(|(a, b)| { + a.text == b.0 && close(f64::from(a.start), b.1) && close(f64::from(a.end), b.2) + }); + let overrides_match = p.fade_duration_override == m.overrides.fade + && p.position_override == m.overrides.position + && p.color_override == m.overrides.color + && p.font_size_override == m.overrides.font_size; + if p.id != m.id + || !close(p.start, m.start) + || !close(p.end, m.end) + || p.text != m.text + || !words_match + || !overrides_match + { + problems.push(format!( + "engine {}{}, model {}", + describe_track(p), + if overrides_match { + "" + } else { + " (overrides differ)" + }, + describe(m) + )); + } + } + problems +} + +const VOCABULARY: [&str; 24] = [ + "hello", "world", "is", "a", "river", "cloud", "it", "to", "über", "café", "日本", "so", + "screen", "record", "of", "we", "on", "capture", "edit", "timeline", "in", "up", "my", "an", +]; +const PUNCTUATION: [&str; 6] = [",", ".", "?", "!", "'s", "%"]; + +fn random_captions(rng: &mut Rng, total: f64) -> Vec { + let mut captions = Vec::new(); + let mut t = rng.f64() * 0.5; + while t < total + 1.0 { + let id = format!("c{}", captions.len()); + if rng.range(0, 9) == 0 { + let end = t + 0.2 + rng.f64() * 2.0; + captions.push(CaptionSegment { + id, + start: t as f32, + end: end as f32, + text: format!("plain text {}", captions.len()), + words: Vec::new(), + }); + t = end + rng.f64() * 0.5; + continue; + } + let mut words = Vec::new(); + for i in 0..rng.range(1, 8) { + if i > 0 { + t += match rng.range(0, 3) { + 0 => 0.0, + 1 => rng.f64() * 0.2, + 2 => rng.f64() * 0.6, + _ => rng.f64() * 1.5, + }; + } + let length = if rng.range(0, 19) == 0 { + 3.0 + rng.f64() * 4.0 + } else { + 0.05 + rng.f64() * 0.6 + }; + let text = if i > 0 && rng.range(0, 5) == 0 { + PUNCTUATION[rng.range(0, PUNCTUATION.len() as u64 - 1) as usize] + } else { + VOCABULARY[rng.range(0, VOCABULARY.len() as u64 - 1) as usize] + }; + words.push(CaptionWord { + text: text.to_string(), + start: t as f32, + end: (t + length) as f32, + }); + t += length; + } + let start = words[0].start; + let mut end = words.last().unwrap().end; + if rng.range(0, 3) == 0 { + end += (rng.f64() * 1.0) as f32; + } + captions.push(CaptionSegment { + id, + start, + end, + text: join_words(words.iter().map(|w| w.text.as_str())), + words, + }); + t += rng.f64() * 1.2; + } + captions +} + +fn caption_settings() -> CaptionSettings { + CaptionSettings { + enabled: true, + ..Default::default() + } +} + +fn set_captions( + config: &mut ProjectConfiguration, + captions: Vec, + source_timed: bool, +) { + config.captions = Some(CaptionsData { + segments: captions, + settings: caption_settings(), + source_timed, + }); +} + +/// Style overrides on some current track pieces. `per_source` styles every +/// piece of a caption alike; otherwise pieces of one caption may differ and +/// the first piece's style is the one that survives re-projection. +fn random_overrides(rng: &mut Rng, config: &mut ProjectConfiguration, per_source: bool) { + let timeline = config.timeline.as_mut().expect("timeline"); + let mut styled: Vec<(String, CaptionTrackSegment)> = Vec::new(); + for piece in &mut timeline.caption_segments { + let source = cap_project::source_caption_id(&piece.id).to_string(); + if per_source && let Some((_, style)) = styled.iter().find(|(id, _)| *id == source) { + piece.fade_duration_override = style.fade_duration_override; + piece.position_override = style.position_override.clone(); + piece.color_override = style.color_override.clone(); + piece.font_size_override = style.font_size_override; + continue; + } + if rng.range(0, 3) == 0 { + piece.fade_duration_override = Some(rng.range(0, 5) as f32 / 10.0); + piece.position_override = Some("top-center".into()); + piece.color_override = Some(format!("#{:06X}", rng.range(0, 0xFF_FFFF))); + piece.font_size_override = Some(rng.range(12, 96) as u32); + } + styled.push((source, piece.clone())); + } +} + +fn projection_problems( + base: &ProjectConfiguration, + ops: &[Op], + captions: &[CaptionSegment], + durations: &[f64], + override_seed: u64, +) -> Vec { + let mut config = base.clone(); + set_captions(&mut config, captions.to_vec(), true); + cap_project::synchronize_captions(&mut config, durations); + random_overrides(&mut Rng::new(override_seed), &mut config, false); + for op in ops { + edits::apply(&mut config, op); + } + let model = model_project(captions, &config, durations); + cap_project::synchronize_captions(&mut config, durations); + let mut problems = compare_track(&edits::timeline(&config).caption_segments, &model); + let again = edits::timeline(&config).caption_segments.clone(); + cap_project::synchronize_captions(&mut config, durations); + let after = &edits::timeline(&config).caption_segments; + if serde_json::to_value(&again).ok() != serde_json::to_value(after).ok() { + problems.push("re-synchronizing an unchanged project changed the caption track".into()); + } + problems +} + +fn projection_case(name: &str, mix: OpMix, seeds: u64) { + let id = format!("captions/projection/{name}"); + if !report::selected(&id) { + return; + } + let mut case = Case::new(&id); + let (mut failed, mut pieces) = (0, 0usize); + for seed in 0..seeds { + let mut rng = Rng::new(0xCA97_0000 ^ (seed << 8) ^ name.len() as u64); + let durations = edits::random_durations(&mut rng); + let base = edits::base_config(&durations); + let captions = random_captions(&mut rng, durations.iter().sum()); + let count = rng.range(0, 8) as usize; + let ops = edits::random_sequence(&mut rng, &base, count, mix); + let problems = projection_problems(&base, &ops, &captions, &durations, seed); + pieces += model_project(&captions, &edits::replay(&base, &ops), &durations).len(); + if problems.is_empty() { + continue; + } + failed += 1; + if failed > 3 { + continue; + } + let minimal = edits::shrink(&ops, |ops| { + !projection_problems(&base, ops, &captions, &durations, seed).is_empty() + }); + let minimal_problems = projection_problems(&base, &minimal, &captions, &durations, seed); + case.problem("caption-track", format!( + "seed {seed}: clips {durations:?}, {} captions, shrunk from {} to {} ops {minimal:?}: {} problems (first: {})", + captions.len(), + ops.len(), + minimal.len(), + minimal_problems.len(), + minimal_problems.iter().take(3).cloned().collect::>().join("; ") + )); + } + case.note(format!( + "{seeds} random caption sets and edit sequences, {pieces} projected pieces, {failed} failed" + )); + if failed > 3 { + case.problem( + "caption-track", + format!("{failed} sequences failed in total"), + ); + } + finish(case); +} + +#[test] +fn projection_cuts_speeds() { + projection_case( + "cuts-speeds", + OpMix { + transitions: false, + holds: false, + }, + seeds(1000, 20000), + ); +} + +#[test] +fn projection_transitions() { + projection_case( + "transitions", + OpMix { + transitions: true, + holds: false, + }, + seeds(1000, 20000), + ); +} + +#[test] +fn projection_holds() { + projection_case( + "holds", + OpMix { + transitions: false, + holds: true, + }, + seeds(1000, 20000), + ); +} + +#[test] +fn projection_all() { + projection_case( + "all", + OpMix { + transitions: true, + holds: true, + }, + seeds(1000, 20000), + ); +} + +/// Fullscreen text holds placed exactly where captions end or start (as when +/// a text card is snapped to a caption): a caption ending exactly at a hold +/// ends before it, one starting there starts after it, also when an earlier +/// hold has already moved the boundary. +#[test] +fn projection_hold_boundaries() { + let id = "captions/projection/hold-boundaries"; + if !report::selected(id) { + return; + } + let mut case = Case::new(id); + let durations = [6.0]; + let mut config = edits::base_config(&durations); + { + let timeline = config.timeline.as_mut().expect("timeline"); + // Output time; the second hold starts at gapless 4.0 s. + timeline + .text_segments + .push(edits::fullscreen_text(2.0, 3.0)); + timeline + .text_segments + .push(edits::fullscreen_text(5.0, 5.5)); + } + let word = |text: &str, start: f32, end: f32| CaptionWord { + text: text.into(), + start, + end, + }; + let captions = vec![ + CaptionSegment { + id: "ends-at-hold".into(), + start: 1.0, + end: 2.0, + text: "apple river".into(), + words: vec![word("apple", 1.0, 1.5), word("river", 1.5, 2.0)], + }, + CaptionSegment { + id: "starts-at-hold".into(), + start: 2.0, + end: 3.0, + text: "cloud stone".into(), + words: vec![word("cloud", 2.0, 2.5), word("stone", 2.5, 3.0)], + }, + CaptionSegment { + id: "text-only-ends-at-second-hold".into(), + start: 3.5, + end: 4.0, + text: "maple tiger".into(), + words: Vec::new(), + }, + ]; + set_captions(&mut config, captions.clone(), true); + cap_project::synchronize_captions(&mut config, &durations); + let model = model_project(&captions, &config, &durations); + let product = &edits::timeline(&config).caption_segments; + let mut problems = compare_track(product, &model); + // The documented outcome, written out: the model must agree with it. + for (id, start, end) in [ + ("ends-at-hold", 1.0, 2.0), + ("starts-at-hold", 3.0, 4.0), + ("text-only-ends-at-second-hold", 4.5, 5.0), + ] { + if !model + .iter() + .any(|p| p.id == id && close(p.start, start) && close(p.end, end)) + { + problems.push(format!("model has no piece {id} {start:.1}-{end:.1}")); + } + } + if problems.is_empty() { + case.note(format!("{} pieces at hold boundaries", model.len())); + } else { + case.problem( + "caption-track", + format!("{} problems: {}", problems.len(), problems.join("; ")), + ); + } + finish(case); +} + +fn track_as_captions(track: &[CaptionTrackSegment]) -> Vec { + track + .iter() + .map(|t| CaptionSegment { + id: t.id.clone(), + start: t.start as f32, + end: t.end as f32, + text: t.text.clone(), + words: t.words.clone(), + }) + .collect() +} + +fn migration_problems( + base: &ProjectConfiguration, + ops: &[Op], + captions: &[CaptionSegment], + durations: &[f64], +) -> Vec { + let mut config = edits::replay(base, ops); + set_captions(&mut config, captions.to_vec(), true); + cap_project::synchronize_captions(&mut config, durations); + let mut shown = edits::timeline(&config).caption_segments.clone(); + // Captions ending exactly at the timeline end are covered by + // `legacy_migration_caption_at_timeline_end`. + let end = TimeModel::new(&config).duration(); + shown.retain(|t| (t.end - end).abs() > TIME_TOLERANCE); + let legacy = track_as_captions(&shown); + config + .timeline + .as_mut() + .expect("timeline") + .caption_segments + .clear(); + set_captions(&mut config, legacy.clone(), false); + let expected = model_migrate(&legacy, &config, durations); + cap_project::synchronize_captions(&mut config, durations); + let captions_data = config.captions.as_ref().expect("captions"); + let mut problems = Vec::new(); + if !captions_data.source_timed { + problems.push("captions still marked output-timed after loading".into()); + } + let migrated = &captions_data.segments; + if migrated.len() != expected.len() { + problems.push(format!( + "{} captions after migration, the model expects {}", + migrated.len(), + expected.len() + )); + } + for (a, b) in migrated.iter().zip(&expected) { + let same = a.id == b.id + && close(f64::from(a.start), f64::from(b.start)) + && close(f64::from(a.end), f64::from(b.end)) + && a.words.len() == b.words.len() + && a.words.iter().zip(&b.words).all(|(x, y)| { + x.text == y.text + && close(f64::from(x.start), f64::from(y.start)) + && close(f64::from(x.end), f64::from(y.end)) + }); + if !same { + problems.push(format!( + "migrated {} {:.4}-{:.4}, model {} {:.4}-{:.4}", + a.id, a.start, a.end, b.id, b.start, b.end + )); + } + } + problems.extend(compare_track( + &edits::timeline(&config).caption_segments, + &model_project(&expected, &config, durations), + )); + // A legacy caption inside one segment, away from transitions and holds, + // must come back where it was. + let time = TimeModel::new(&config); + let track = &edits::timeline(&config).caption_segments; + for piece in &shown { + let inside = time.segments.iter().enumerate().any(|(k, s)| { + let next_in = time.segments.get(k + 1).map_or(0.0, |n| n.transition_in); + let lo = time.to_output(s.out_start + s.transition_in); + let hi = time.to_output_end(s.out_start + s.duration - next_in); + piece.start > lo + TIME_TOLERANCE + && piece.end < hi - TIME_TOLERANCE + && time.held_before(piece.end) == time.held_before(piece.start) + }); + let clean = piece + .words + .iter() + .all(|w| f64::from(w.end - w.start) < MAX_WORD_SECS); + if inside + && clean + && !track.iter().any(|t| { + t.text == piece.text && close(t.start, piece.start) && close(t.end, piece.end) + }) + { + problems.push(format!( + "legacy caption {} did not come back in place after loading", + describe_track(piece) + )); + } + } + problems +} + +/// A legacy caption that ends exactly where the timeline ends (its last +/// word ran to the end of the recording) must survive loading. +#[test] +fn legacy_migration_caption_at_timeline_end() { + let id = "captions/legacy-migration/ends-at-timeline-end"; + if !report::selected(id) { + return; + } + let durations = [41.0 / 15.0, 82.0 / 15.0]; + let mut config = edits::base_config(&durations); + common::add_transition(&mut config, 1, 0.2); + let end = edits::timeline(&config).duration() as f32; + let legacy = vec![CaptionSegment { + id: "last".into(), + start: end - 2.2, + end, + text: "until the end".into(), + words: vec![ + CaptionWord { + text: "until".into(), + start: end - 2.2, + end: end - 1.5, + }, + CaptionWord { + text: "the".into(), + start: end - 1.5, + end: end - 1.0, + }, + CaptionWord { + text: "end".into(), + start: end - 1.0, + end, + }, + ], + }]; + let expected = model_migrate(&legacy, &config, &durations); + set_captions(&mut config, legacy, false); + cap_project::synchronize_captions(&mut config, &durations); + let mut case = Case::new(id); + let migrated = &config.captions.as_ref().expect("captions").segments; + if migrated.len() == expected.len() && expected.len() == 1 { + case.note(format!( + "caption ending at {end:.4}s kept at source {:.4}-{:.4}", + migrated[0].start, migrated[0].end + )); + } else { + case.problem( + "caption-migration", + format!( + "a legacy caption ending at the {end:.4}s timeline end: {} captions after loading, the model keeps {}", + migrated.len(), + expected.len() + ), + ); + } + finish(case); +} + +#[test] +fn legacy_migration() { + let id = "captions/legacy-migration"; + if !report::selected(id) { + return; + } + let mut case = Case::new(id); + let seeds = seeds(1000, 20000); + let mut failed = 0; + for seed in 0..seeds { + let mut rng = Rng::new(0xCA97_1000 ^ (seed << 8)); + let durations = edits::random_durations(&mut rng); + let base = edits::base_config(&durations); + let captions = random_captions(&mut rng, durations.iter().sum()); + let count = rng.range(0, 8) as usize; + let ops = edits::random_sequence( + &mut rng, + &base, + count, + OpMix { + transitions: true, + holds: true, + }, + ); + if migration_problems(&base, &ops, &captions, &durations).is_empty() { + continue; + } + failed += 1; + if failed > 3 { + continue; + } + let minimal = edits::shrink(&ops, |ops| { + !migration_problems(&base, ops, &captions, &durations).is_empty() + }); + let problems = migration_problems(&base, &minimal, &captions, &durations); + case.problem("caption-migration", format!( + "seed {seed}: clips {durations:?}, shrunk to ops {minimal:?}: {} problems (first: {})", + problems.len(), + problems + .iter() + .take(3) + .cloned() + .collect::>() + .join("; ") + )); + } + case.note(format!("{seeds} legacy projects migrated, {failed} failed")); + if failed > 3 { + case.problem( + "caption-migration", + format!("{failed} sequences failed in total"), + ); + } + finish(case); +} + +fn styled_settings(rng: &mut Rng) -> CaptionSettings { + CaptionSettings { + enabled: true, + font: ["System Sans-Serif", "System Serif", "System Monospace"][rng.range(0, 2) as usize] + .into(), + size: rng.range(12, 96) as u32, + color: "#12AB34".into(), + background_color: "#00C800".into(), + background_opacity: rng.range(0, 100) as u32, + position: "top-left".into(), + italic: true, + font_weight: 500, + outline: true, + outline_color: "#FF0000".into(), + export_with_subtitles: true, + highlight_color: "#0000FF".into(), + fade_duration: 0.3, + linger_duration: 0.7, + word_transition_duration: 0.1, + active_word_highlight: true, + manual_position: Some(cap_project::XY::new(0.25, 0.75)), + preset: "custom".into(), + animation: "pop".into(), + highlight_style: "pill".into(), + uppercase: true, + } +} + +#[test] +fn round_trip_on_disk() { + let id = "captions/round-trip"; + if !report::selected(id) { + return; + } + let mut case = Case::new(id); + let dir = common::scratch_dir(id); + let seeds = seeds(20, 200); + for seed in 0..seeds { + let mut rng = Rng::new(0xCA97_2000 ^ seed); + let durations = edits::random_durations(&mut rng); + let base = edits::base_config(&durations); + let ops = edits::random_sequence( + &mut rng, + &base, + 4, + OpMix { + transitions: true, + holds: true, + }, + ); + let mut config = edits::replay(&base, &ops); + config.captions = Some(CaptionsData { + segments: random_captions(&mut rng, durations.iter().sum()), + settings: styled_settings(&mut rng), + source_timed: true, + }); + cap_project::synchronize_captions(&mut config, &durations); + random_overrides(&mut rng, &mut config, true); + let project = dir.path().join(format!("seed-{seed}")); + std::fs::create_dir_all(&project).expect("project dir"); + if let Err(e) = config.write(&project) { + case.problem("error", format!("seed {seed}: write failed: {e}")); + continue; + } + let loaded = match ProjectConfiguration::load(&project) { + Ok(loaded) => loaded, + Err(e) => { + case.problem("error", format!("seed {seed}: load failed: {e}")); + continue; + } + }; + let json = |c: &ProjectConfiguration| { + ( + serde_json::to_value(&c.captions).expect("captions json"), + serde_json::to_value(&edits::timeline(c).caption_segments).expect("track json"), + ) + }; + if json(&config) != json(&loaded) { + case.problem( + "caption-roundtrip", + format!("seed {seed}: captions changed through project-config.json"), + ); + } + let mut resynced = loaded.clone(); + cap_project::synchronize_captions(&mut resynced, &durations); + if json(&resynced) != json(&loaded) { + case.problem( + "caption-roundtrip", + format!("seed {seed}: re-projecting a loaded project changed its captions"), + ); + } + } + // The legacy captions.json sidecar of older projects: output-timed + // captions with no track, picked up when project-config.json has none. + let project = common::project(&catalog::basic(report::tier())); + let copy = dir.path().join("sidecar.cap"); + copy_project(&project.path, ©); + let mut rng = Rng::new(0xCA97_2FFF); + let legacy = CaptionsData { + segments: random_captions(&mut rng, 5.0), + settings: styled_settings(&mut rng), + source_timed: false, + }; + std::fs::write( + copy.join("captions.json"), + serde_json::to_string(&legacy).expect("sidecar json"), + ) + .expect("write sidecar"); + let meta = RecordingMeta::load_for_project(©).expect("meta"); + let loaded = meta.project_config(); + match &loaded.captions { + None => case.problem("caption-roundtrip", "captions.json sidecar was not loaded"), + Some(c) => { + if serde_json::to_value(c).ok() != serde_json::to_value(&legacy).ok() { + case.problem( + "caption-roundtrip", + "captions.json sidecar loaded with different captions", + ); + } + } + } + case.note(format!( + "{seeds} projects through project-config.json, plus the captions.json sidecar" + )); + finish(case); +} + +fn copy_project(from: &Path, to: &Path) { + for entry in walk(from) { + let rel = entry.strip_prefix(from).expect("inside project"); + let target = to.join(rel); + std::fs::create_dir_all(target.parent().expect("parent")).expect("mkdir"); + std::fs::copy(&entry, &target).expect("copy"); + } +} + +fn walk(dir: &Path) -> Vec { + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir).expect("read dir").flatten() { + let path = entry.path(); + if path.is_dir() { + out.extend(walk(&path)); + } else { + out.push(path); + } + } + out +} + +/// Two takes; every track starts at a different time, so the clip offsets +/// matter (mic 0.25 s late, system audio 0.1 s early, display 0.05 s late). +fn transcription_fixture() -> FixtureSpec { + let screen = |duration: f64, start_time: f64| { + let mut v = VideoTrackSpec::new( + catalog::SCREEN_W, + catalog::SCREEN_H, + 30, + Timing::Constant { duration }, + ); + v.start_time = start_time; + v + }; + let audio = |source: AudioSource, duration: f64, start_time: f64| AudioTrackSpec { + source, + sample_rate: 48_000, + channels: if source == AudioSource::System { 2 } else { 1 }, + duration, + start_time, + }; + FixtureSpec { + name: "captions-takes".into(), + segments: vec![ + SegmentSpec { + display: screen(5.0, 0.05), + camera: None, + mic: Some(audio(AudioSource::Mic, 5.0, 0.25)), + system_audio: Some(audio(AudioSource::System, 5.2, -0.1)), + }, + SegmentSpec { + display: screen(4.0, 0.0), + camera: None, + mic: Some(audio(AudioSource::Mic, 4.0, 0.0)), + system_audio: None, + }, + ], + } +} + +const RATE: u32 = 48_000; + +/// Display durations as the recordings' containers report them: the length +/// of each take on the concatenated source timeline. Checked against the +/// spec (whole frames) to within one frame. +fn display_durations(case: &mut Case, project: &common::Project, spec: &FixtureSpec) -> Vec { + let durations: Vec = cap_rendering::ProjectRecordingsMeta::new( + &project.path, + project.meta.studio_meta().expect("studio"), + ) + .expect("recordings meta") + .segments + .iter() + .map(|s| s.display.duration) + .collect(); + for (d, s) in durations.iter().zip(&spec.segments) { + let truth = s.display.timing.duration(s.display.fps); + if (d - truth).abs() > 1.0 / f64::from(s.display.fps) + 1e-6 { + case.problem( + "duration", + format!("display duration {d:.4}s, the track has {truth:.4}s of frames"), + ); + } + } + durations +} + +fn interleaved(decoded: &media::DecodedAudio) -> Vec { + let frames = decoded.channels.iter().map(Vec::len).min().unwrap_or(0); + let mut samples = Vec::with_capacity(frames * decoded.channels.len()); + for i in 0..frames { + for channel in &decoded.channels { + samples.push(channel[i]); + } + } + samples +} + +/// The transcription input, built the way the app's transcription does: +/// per take, system audio then mic, aligned by the clip offsets, over the +/// take's display duration; takes concatenated. +fn transcription_audio(project: &common::Project) -> Vec { + let Some(StudioRecordingMeta::MultipleSegments { inner }) = project.meta.studio_meta() else { + panic!("multi-segment fixture"); + }; + let recordings = cap_rendering::ProjectRecordingsMeta::new( + &project.path, + project.meta.studio_meta().expect("studio"), + ) + .expect("recordings meta"); + let takes: Vec = inner + .segments + .iter() + .zip(&recordings.segments) + .map(|(segment, recording)| { + let offsets = segment.calculate_audio_offsets(); + let mut sources = Vec::new(); + for (audio, offset) in [ + (&segment.system_audio, offsets.system_audio), + (&segment.mic, offsets.mic), + ] { + let Some(audio) = audio else { continue }; + let decoded = + media::decode_audio(&project.path.join(audio.path.as_str()), Some(RATE)) + .expect("decode source audio"); + sources.push(cap_editor::TranscriptionAudioSource { + samples: interleaved(&decoded), + channels: decoded.channels.len(), + offset_secs: f64::from(offset), + }); + } + cap_editor::TranscriptionAudioTake { + display_duration_secs: recording.display.duration, + sources, + } + }) + .collect(); + cap_editor::assemble_transcription_audio(&takes, RATE).expect("assemble") +} + +fn word_text(source: AudioSource, index: u32) -> String { + format!("{}{index}", source.name()) +} + +/// The fake transcriber: one word per audio marker heard, timed from the +/// marker's chirp to its end. +fn fake_transcribe(audio: &[f32]) -> Vec { + let mut words: Vec = [AudioSource::Mic, AudioSource::System] + .into_iter() + .flat_map(|source| { + audio_code::locate_markers(audio, RATE, source) + .into_iter() + .filter_map(move |m| { + Some(CaptionWord { + text: word_text(source, m.index?), + start: m.time_secs as f32, + end: (m.time_secs + MARKER_SECS) as f32, + }) + }) + }) + .collect(); + words.sort_by(|a, b| a.start.total_cmp(&b.start)); + words +} + +/// Where every marker must be heard, from each source file's own markers +/// and the spec's track start times: a track sample at file time `m` +/// happened at `track start + m`; a take's transcription time 0 is its +/// latest track start, and takes follow each other by display length. +fn expected_words( + project: &common::Project, + spec: &FixtureSpec, + durations: &[f64], +) -> Vec<(String, f64)> { + let Some(StudioRecordingMeta::MultipleSegments { inner }) = project.meta.studio_meta() else { + panic!("multi-segment fixture"); + }; + let mut take_start = 0.0; + let mut out = Vec::new(); + for ((segment, seg_spec), &display) in inner.segments.iter().zip(&spec.segments).zip(durations) + { + let latest = [ + Some(seg_spec.display.start_time), + seg_spec.mic.as_ref().map(|a| a.start_time), + seg_spec.system_audio.as_ref().map(|a| a.start_time), + ] + .into_iter() + .flatten() + .fold(f64::NEG_INFINITY, f64::max); + for (meta, track) in [ + (&segment.mic, &seg_spec.mic), + (&segment.system_audio, &seg_spec.system_audio), + ] { + let (Some(meta), Some(track)) = (meta, track) else { + continue; + }; + let decoded = media::decode_audio(&project.path.join(meta.path.as_str()), Some(RATE)) + .expect("decode source audio"); + for marker in audio_code::locate_markers(&decoded.mono(), RATE, track.source) { + let Some(index) = marker.index else { continue }; + let t = track.start_time + decoded.start_secs + marker.time_secs - latest; + if t >= 0.0 && t + MARKER_SECS <= display { + out.push((word_text(track.source, index), take_start + t)); + } + } + } + take_start += display; + } + out.sort_by(|a, b| a.1.total_cmp(&b.1)); + out +} + +#[test] +fn transcription_audio_words() { + let id = "captions/transcription-audio/takes"; + if !report::selected(id) { + return; + } + let spec = transcription_fixture(); + let project = common::project(&spec); + let mut case = Case::new(id); + let durations = display_durations(&mut case, &project, &spec); + let words = fake_transcribe(&transcription_audio(&project)); + let expected = expected_words(&project, &spec, &durations); + let mut problems = Vec::new(); + let mut worst = 0.0f64; + for (text, time) in &expected { + let hits: Vec<&CaptionWord> = words + .iter() + .filter(|w| &w.text == text && (f64::from(w.start) - time).abs() < 0.05) + .collect(); + match hits.as_slice() { + [w] => { + let error = f64::from(w.start) - time; + worst = worst.max(error.abs()); + if error.abs() > WORD_TOLERANCE_SECS { + problems.push(format!("{text} at {:.4}s, expected {time:.4}s", w.start)); + } + } + [] => problems.push(format!("{text} missing (expected at {time:.4}s)")), + _ => problems.push(format!("{text} heard {} times", hits.len())), + } + } + if words.len() != expected.len() { + problems.push(format!( + "{} words heard, {} expected", + words.len(), + expected.len() + )); + } + if expected.len() < 10 { + problems.push(format!("only {} reference words", expected.len())); + } + if problems.is_empty() { + case.note(format!( + "{} words at their source times (worst {:.3} ms)", + words.len(), + worst * 1000.0 + )); + } else { + case.problem( + "caption-words", + format!( + "{} problems (first: {})", + problems.len(), + problems + .iter() + .take(4) + .cloned() + .collect::>() + .join("; ") + ), + ); + } + // The transcribed captions through a reorder of the two takes, projected + // with the spec's display lengths as the source of truth. + let captions: Vec = words + .chunks(3) + .enumerate() + .map(|(i, chunk)| CaptionSegment { + id: format!("segment-{i}"), + start: chunk[0].start, + end: chunk[chunk.len() - 1].end, + text: join_words(chunk.iter().map(|w| w.text.as_str())), + words: chunk.to_vec(), + }) + .collect(); + let mut config = project.config(); + config + .timeline + .as_mut() + .expect("timeline") + .segments + .reverse(); + set_captions(&mut config, captions.clone(), true); + cap_project::synchronize_captions(&mut config, &durations); + let track_problems = compare_track( + &edits::timeline(&config).caption_segments, + &model_project(&captions, &config, &durations), + ); + if !track_problems.is_empty() { + case.problem( + "caption-track", + format!( + "reordered takes: {} caption problems (first: {})", + track_problems.len(), + track_problems[0] + ), + ); + } + finish(case); +} + +/// Caption style that is easy to find: an opaque green box with white text, +/// no fades, animation or word highlight. +fn render_settings() -> CaptionSettings { + CaptionSettings { + enabled: true, + size: 72, + color: "#FFFFFF".into(), + background_color: "#00C800".into(), + background_opacity: 100, + position: "bottom-center".into(), + outline: false, + fade_duration: 0.0, + animation: "none".into(), + active_word_highlight: false, + ..Default::default() + } +} + +const WORDS: [&str; 30] = [ + "apple", "river", "cloud", "stone", "maple", "tiger", "piano", "lemon", "orbit", "glass", + "honey", "zebra", "candle", "forest", "rocket", "violet", "marble", "tunnel", "pepper", + "falcon", "garden", "silver", "anchor", "button", "copper", "dragon", "eclipse", "fabric", + "harbor", "island", +]; + +/// Scripted words: three per caption, one caption every ~1.35 s, with a +/// word stretched across a silence in the third caption. +fn scripted_captions(total: f64) -> Vec { + let mut captions = Vec::new(); + let mut t = 0.2; + let mut word = 0; + while t + 1.2 < total && word + 3 <= WORDS.len() { + let mut words = Vec::new(); + for i in 0..3 { + let start = t + 0.4 * i as f64; + let mut end = start + 0.35; + if captions.len() == 2 && i == 2 { + end = start + 6.0; + } + words.push(CaptionWord { + text: WORDS[word].into(), + start: start as f32, + end: end as f32, + }); + word += 1; + } + captions.push(CaptionSegment { + id: format!("segment-{}", captions.len()), + start: words[0].start, + end: words[2].end, + text: join_words(words.iter().map(|w| w.text.as_str())), + words, + }); + t += 1.35; + } + captions +} + +#[derive(Clone)] +struct CaptionPixels { + box_pixels: usize, + /// Bright (text) pixels inside the box's rows, full width. + glyphs: Vec, + bbox: Option<(usize, usize, usize, usize)>, +} + +fn caption_pixels(view: &impl PixelSource) -> CaptionPixels { + let (w, h) = (view.width(), view.height()); + let mut box_pixels = 0; + let mut bbox: Option<(usize, usize, usize, usize)> = None; + for y in h / 2..h { + for x in 0..w { + let (cb, cr) = view.chroma(x, y); + if cb < -35.0 && cr < -40.0 { + box_pixels += 1; + bbox = Some(match bbox { + None => (x, y, x, y), + Some((x0, y0, x1, y1)) => (x0.min(x), y0.min(y), x1.max(x), y1.max(y)), + }); + } + } + } + let glyphs = (h / 2..h) + .flat_map(|y| (0..w).map(move |x| (x, y))) + .map(|(x, y)| { + bbox.is_some_and(|(x0, y0, x1, y1)| x >= x0 && x <= x1 && y >= y0 && y <= y1) + && view.luma(x, y) > 170.0 + }) + .collect(); + CaptionPixels { + box_pixels, + glyphs, + bbox, + } +} + +fn iou(a: &[bool], b: &[bool]) -> f64 { + let (mut both, mut any) = (0usize, 0usize); + for (x, y) in a.iter().zip(b) { + both += usize::from(*x && *y); + any += usize::from(*x || *y); + } + if any == 0 { + 1.0 + } else { + both as f64 / any as f64 + } +} + +/// A frame shows a caption box when this many box pixels are present. +const BOX_PRESENT: usize = 200; +const SAME_TEXT_IOU: f64 = 0.75; + +fn decode_captions(path: &Path) -> Vec { + let mut frames = Vec::new(); + media::decode_video(path, |picture| frames.push(caption_pixels(&picture.view()))) + .expect("decode export"); + frames +} + +const RENDER_FPS: u32 = 30; +const RENDER_RES: (u32, u32) = (480, 270); + +fn render_export( + rt: &tokio::runtime::Runtime, + project: &common::Project, + config: ProjectConfiguration, + out: &Path, +) -> Result, String> { + let run = common::ExportRun { + fps: RENDER_FPS, + resolution: RENDER_RES, + format: common::Format::Mp4, + decoder: common::Decoder::Default, + }; + let path = rt.block_on(common::export(project, config, &run, out))?; + Ok(decode_captions(&path)) +} + +/// Glyph templates: every piece text rendered alone in its own window of an +/// unedited timeline. +fn templates( + rt: &tokio::runtime::Runtime, + project: &common::Project, + texts: &[String], + durations: &[f64], + out: &Path, +) -> Result, String> { + let slot = 0.3; + let mut config = project.config(); + config.camera.hide = true; + let captions: Vec = texts + .iter() + .enumerate() + .map(|(i, text)| CaptionSegment { + id: format!("t{i}"), + start: (i as f64 * slot) as f32, + end: (i as f64 * slot + slot - 0.05) as f32, + text: text.clone(), + words: Vec::new(), + }) + .collect(); + config.captions = Some(CaptionsData { + segments: captions, + settings: render_settings(), + source_timed: true, + }); + // Projected here so the templates do not depend on the export's own + // projection (which `render_*` checks). + cap_project::synchronize_captions(&mut config, durations); + let duration = edits::timeline(&config).duration(); + if texts.len() as f64 * slot > duration { + return Err(format!( + "{} templates do not fit in {duration:.1}s", + texts.len() + )); + } + let frames = render_export(rt, project, config, out)?; + texts + .iter() + .enumerate() + .map(|(i, text)| { + let frame = ((i as f64 * slot + 0.12) * f64::from(RENDER_FPS)).round() as usize; + let pixels = frames + .get(frame) + .filter(|p| p.box_pixels >= BOX_PRESENT) + .ok_or_else(|| format!("template \"{text}\" not rendered at frame {frame}"))?; + Ok((text.clone(), pixels.clone())) + }) + .collect() +} + +struct RenderEdit { + name: &'static str, + apply: fn(&mut ProjectConfiguration), + /// Apply the edit after the caption track was projected, leaving the + /// stored track stale (the export must re-project it). + stale_track: bool, + /// Source captions for the recording's total length. + captions: fn(f64) -> Vec, + /// Check frames that fall exactly on a caption start or end too. Only + /// for captions whose boundaries are exact in f32 and on the frame grid. + strict_edges: bool, +} + +fn render_case( + spec: FixtureSpec, + edit: &RenderEdit, + rt: &tokio::runtime::Runtime, +) -> Option { + let id = format!("captions/render/{}/{}", spec.name, edit.name); + if !report::selected(&id) { + return None; + } + let project = common::project(&spec); + let mut case = Case::new(&id); + let durations = display_durations(&mut case, &project, &spec); + let captions = (edit.captions)(durations.iter().sum()); + let mut config = project.config(); + config.camera.hide = true; + if !edit.stale_track { + (edit.apply)(&mut config); + } + config.captions = Some(CaptionsData { + segments: captions.clone(), + settings: render_settings(), + source_timed: true, + }); + cap_project::synchronize_captions(&mut config, &durations); + if edit.stale_track { + (edit.apply)(&mut config); + } + let pieces = model_project(&captions, &config, &durations); + let dir = common::scratch_dir(&id); + let mut texts: Vec = pieces.iter().map(|p| p.text.clone()).collect(); + texts.sort(); + texts.dedup(); + let result = templates( + rt, + &project, + &texts, + &durations, + &dir.path().join("templates.mp4"), + ) + .and_then(|templates| { + render_export(rt, &project, config.clone(), &dir.path().join("out.mp4")) + .map(|frames| (templates, frames)) + }); + match result { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok((templates, frames)) => { + let time = TimeModel::new(&config); + let expected_frames = (time.duration() * f64::from(RENDER_FPS)).ceil() as usize; + if frames.len() != expected_frames { + case.problem( + "frame-count", + format!( + "{} frames, the timeline needs {expected_frames}", + frames.len() + ), + ); + } + let mut wrong = Vec::new(); + let (mut checked, mut shown) = (0, 0); + for (i, pixels) in frames.iter().enumerate() { + let t = i as f64 / f64::from(RENDER_FPS); + let edge = pieces + .iter() + .any(|p| (t - p.start).abs() < 1e-3 || (t - p.visible_end()).abs() < 1e-3); + if edge && !edit.strict_edges { + continue; + } + checked += 1; + let expected = pieces + .iter() + .find(|p| t >= p.start && t < p.visible_end()) + .map(|p| p.text.as_str()); + let present = pixels.box_pixels >= BOX_PRESENT; + let seen = present.then(|| { + templates + .iter() + .map(|(text, template)| { + (text.as_str(), iou(&template.glyphs, &pixels.glyphs)) + }) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .filter(|(_, score)| *score >= SAME_TEXT_IOU) + .map(|(text, _)| text) + }); + let ok = match (expected, seen) { + (None, None) => true, + (Some(e), Some(Some(s))) => e == s, + _ => false, + }; + shown += usize::from(expected.is_some()); + if !ok { + wrong.push(format!( + "frame {i} ({t:.3}s): expected {expected:?}, shows {}", + match seen { + None => "no caption".to_string(), + Some(None) => format!("an unknown caption (box {:?})", pixels.bbox), + Some(Some(s)) => format!("\"{s}\""), + } + )); + } + } + let summary = format!( + "{} pieces, {checked} frames checked, {shown} with a caption", + pieces.len() + ); + if wrong.is_empty() { + case.note(summary); + } else { + case.problem( + "caption-frame", + format!( + "{summary}; {} frames wrong (first: {})", + wrong.len(), + wrong.iter().take(4).cloned().collect::>().join("; ") + ), + ); + } + } + } + settle(case) +} + +fn render_edits() -> Vec { + vec![ + RenderEdit { + name: "full", + apply: |_| {}, + stale_track: false, + captions: scripted_captions, + strict_edges: false, + }, + RenderEdit { + name: "cuts-speeds-reorder", + apply: |c| { + common::set_cuts( + c, + &[ + (0.5, 2.0, 1.0), + (2.5, 4.0, 2.0), + (1.0, 2.0, 0.5), + (4.0, 5.8, 1.5), + ], + ) + }, + stale_track: false, + captions: scripted_captions, + strict_edges: false, + }, + RenderEdit { + name: "transition", + apply: |c| { + common::set_cuts(c, &[(0.0, 2.5, 1.0), (3.5, 6.0, 1.0)]); + common::add_transition(c, 1, 0.4); + }, + stale_track: false, + captions: scripted_captions, + strict_edges: false, + }, + RenderEdit { + name: "stale-track", + apply: |c| common::set_cuts(c, &[(2.7, 6.0, 1.0), (0.0, 2.0, 1.0)]), + stale_track: true, + captions: scripted_captions, + strict_edges: false, + }, + RenderEdit { + // Half speed: the third caption's stretched last word lasts + // 4.6 s on the output, so the renderer's 2.5 s word limit + // decides when it disappears. + name: "slow", + apply: |c| common::set_cuts(c, &[(0.0, 6.0, 0.5)]), + stale_track: false, + captions: scripted_captions, + strict_edges: false, + }, + RenderEdit { + name: "back-to-back", + apply: |_| {}, + stale_track: false, + captions: back_to_back_captions, + strict_edges: true, + }, + ] +} + +/// Captions that follow each other without a gap, every boundary on a half +/// second (exact in f32 and on the 30 fps frame grid): on the frame where +/// one caption ends the next one shows, and after the last one nothing. +fn back_to_back_captions(total: f64) -> Vec { + let mut captions = Vec::new(); + let mut start = 0.5; + while start + 1.0 <= total - 0.5 && captions.len() < 5 { + let k = captions.len() * 2; + let words: Vec = (0..2) + .map(|i| CaptionWord { + text: WORDS[k + i].into(), + start: (start + 0.5 * i as f64) as f32, + end: (start + 0.5 * (i + 1) as f64) as f32, + }) + .collect(); + captions.push(CaptionSegment { + id: format!("segment-{}", captions.len()), + start: words[0].start, + end: words[1].end, + text: join_words(words.iter().map(|w| w.text.as_str())), + words, + }); + start += 1.0; + } + captions +} + +#[test] +fn render_single_clip() { + let rt = common::runtime(); + let failures: Vec = render_edits() + .iter() + .filter_map(|edit| render_case(catalog::basic(report::tier()), edit, &rt)) + .collect(); + common::assert_passed(failures); +} + +#[test] +fn render_multi_clip() { + let rt = common::runtime(); + let edit = RenderEdit { + name: "reorder", + apply: |c| { + let timeline = c.timeline.as_mut().expect("timeline"); + timeline.segments.reverse(); + }, + stale_track: true, + captions: scripted_captions, + strict_edges: false, + }; + let failures: Vec = render_case(catalog::multi_segment(report::tier()), &edit, &rt) + .into_iter() + .collect(); + common::assert_passed(failures); +} + +/// Opens a copy of `project` whose stored config is `config` in the editor +/// engine and returns the config it settles on. +fn open_in_editor( + rt: &tokio::runtime::Runtime, + project: &common::Project, + config: &ProjectConfiguration, + dir: &Path, + sidecar: Option<&CaptionsData>, +) -> Result { + let copy = dir.join("project.cap"); + copy_project(&project.path, ©); + config + .write(©) + .map_err(|e| format!("write config: {e}"))?; + if let Some(sidecar) = sidecar { + std::fs::write( + copy.join("captions.json"), + serde_json::to_string(sidecar).expect("sidecar json"), + ) + .map_err(|e| format!("write sidecar: {e}"))?; + } + rt.block_on(async { + let editor = cap_editor::EditorInstance::new( + copy.clone(), + |_| {}, + Box::new(|_: cap_editor::EditorFrameOutput, _: cap_editor::FrameLayout| {}), + None, + ) + .await?; + let loaded = editor.project_config.1.borrow().clone(); + editor.dispose().await; + Ok(loaded) + }) +} + +#[test] +fn editor_open() { + let rt = common::runtime(); + let spec = catalog::multi_segment(report::tier()); + let project = common::project(&spec); + let mut durations_check = Case::new("captions/editor-open/durations"); + let durations = display_durations(&mut durations_check, &project, &spec); + let captions = (0..) + .map(|seed| random_captions(&mut Rng::new(0xCA97_3000 + seed), durations.iter().sum())) + .find(|captions| captions.len() >= 5) + .expect("captions"); + let edit = |config: &mut ProjectConfiguration| { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.segments.reverse(); + timeline.segments[0].start += 0.5; + timeline.segments[1].timescale = 1.5; + }; + let mut failures = Vec::new(); + for variant in [ + "source-timed-stale-track", + "legacy-output-timed", + "legacy-sidecar", + ] { + let id = format!("captions/editor-open/{variant}"); + if !report::selected(&id) { + continue; + } + let mut case = Case::new(&id); + for problem in &durations_check.problems { + case.problem("duration", problem.clone()); + } + let dir = common::scratch_dir(&id); + let mut config = project.config(); + let (expected_captions, sidecar) = match variant { + "source-timed-stale-track" => { + set_captions(&mut config, captions.clone(), true); + cap_project::synchronize_captions(&mut config, &durations); + edit(&mut config); + (captions.clone(), None) + } + _ => { + edit(&mut config); + let mut shown = config.clone(); + set_captions(&mut shown, captions.clone(), true); + cap_project::synchronize_captions(&mut shown, &durations); + let legacy = track_as_captions(&edits::timeline(&shown).caption_segments); + let migrated = model_migrate(&legacy, &config, &durations); + if variant == "legacy-output-timed" { + set_captions(&mut config, legacy, false); + (migrated, None) + } else { + let sidecar = CaptionsData { + segments: legacy, + settings: caption_settings(), + source_timed: false, + }; + (migrated, Some(sidecar)) + } + } + }; + match open_in_editor(&rt, &project, &config, dir.path(), sidecar.as_ref()) { + Err(e) => case.problem("error", format!("editor failed to open: {e}")), + Ok(loaded) => match &loaded.captions { + None => case.problem("caption-migration", "no captions after opening"), + Some(data) => { + if !data.source_timed { + case.problem( + "caption-migration", + "captions still output-timed after opening", + ); + } + let same = data.segments.len() == expected_captions.len() + && data.segments.iter().zip(&expected_captions).all(|(a, b)| { + a.id == b.id + && close(f64::from(a.start), f64::from(b.start)) + && close(f64::from(a.end), f64::from(b.end)) + && a.words.len() == b.words.len() + }); + if !same { + case.problem( + "caption-migration", + format!( + "stored captions after opening differ from the model ({} vs {})", + data.segments.len(), + expected_captions.len() + ), + ); + } + let problems = compare_track( + &edits::timeline(&loaded).caption_segments, + &model_project(&expected_captions, &loaded, &durations), + ); + if problems.is_empty() { + case.note(format!( + "{} captions, {} track pieces as the model says", + data.segments.len(), + edits::timeline(&loaded).caption_segments.len() + )); + } else { + case.problem( + "caption-track", + format!("{} track problems (first: {})", problems.len(), problems[0]), + ); + } + } + }, + } + failures.extend(settle(case)); + } + common::assert_passed(failures); +} diff --git a/crates/verify/tests/codec.rs b/crates/verify/tests/codec.rs new file mode 100644 index 00000000000..4fa8ae31075 --- /dev/null +++ b/crates/verify/tests/codec.rs @@ -0,0 +1,356 @@ +//! Marker codec robustness: the video code must survive lossy encoding, +//! scaling, compositing and blur, and the audio markers must be located to +//! well under a millisecond after lossy encoding and resampling. + +use cap_verify::{ + audio_code::{self, AudioSource}, + media::{self, AudioCodec, VideoCodec, VideoWriter, VideoWriterConfig}, + oracle::{self, StreamExpectation}, + video_code::{self, FrameMarker, PackedFrame, PixelOrder, Readout, StreamKind}, +}; + +struct Rgba { + data: Vec, + w: usize, + h: usize, +} + +impl Rgba { + fn new(w: usize, h: usize, rgb: [u8; 3]) -> Self { + let mut data = vec![255u8; w * h * 4]; + for px in data.chunks_exact_mut(4) { + px[..3].copy_from_slice(&rgb); + } + Self { data, w, h } + } + + fn view(&self) -> PackedFrame<'_> { + PackedFrame { + data: &self.data, + width: self.w, + height: self.h, + stride: self.w * 4, + order: PixelOrder::Rgba, + } + } + + fn get(&self, x: usize, y: usize) -> [u8; 4] { + let i = (y * self.w + x) * 4; + [ + self.data[i], + self.data[i + 1], + self.data[i + 2], + self.data[i + 3], + ] + } + + /// Bilinear resample of a source rectangle into a new image. + fn crop_scale(&self, sx: f64, sy: f64, sw: f64, sh: f64, w: usize, h: usize) -> Rgba { + let mut out = Rgba::new(w, h, [0, 0, 0]); + for y in 0..h { + for x in 0..w { + let fx = sx + (x as f64 + 0.5) * sw / w as f64 - 0.5; + let fy = sy + (y as f64 + 0.5) * sh / h as f64 - 0.5; + let x0 = fx.floor().clamp(0.0, (self.w - 1) as f64) as usize; + let y0 = fy.floor().clamp(0.0, (self.h - 1) as f64) as usize; + let x1 = (x0 + 1).min(self.w - 1); + let y1 = (y0 + 1).min(self.h - 1); + let ax = (fx - x0 as f64).clamp(0.0, 1.0); + let ay = (fy - y0 as f64).clamp(0.0, 1.0); + let i = (y * w + x) * 4; + for c in 0..3 { + let v = |px: [u8; 4]| f64::from(px[c]); + let top = v(self.get(x0, y0)) * (1.0 - ax) + v(self.get(x1, y0)) * ax; + let bottom = v(self.get(x0, y1)) * (1.0 - ax) + v(self.get(x1, y1)) * ax; + out.data[i + c] = (top * (1.0 - ay) + bottom * ay).round() as u8; + } + } + } + out + } + + /// Pastes `src` at (x, y) through a rounded-rectangle mask of radius `r` + /// (r = half the side gives a circle), anti-aliased over one pixel. + fn paste_rounded(&mut self, src: &Rgba, ox: usize, oy: usize, r: f64) { + for y in 0..src.h { + for x in 0..src.w { + let (px, py) = (x as f64 + 0.5, y as f64 + 0.5); + let cx = px.clamp(r, src.w as f64 - r); + let cy = py.clamp(r, src.h as f64 - r); + let d = (px - cx).hypot(py - cy) - r; + let alpha = (0.5 - d).clamp(0.0, 1.0); + let (tx, ty) = (ox + x, oy + y); + if tx >= self.w || ty >= self.h || alpha == 0.0 { + continue; + } + let i = (ty * self.w + tx) * 4; + let j = (y * src.w + x) * 4; + for c in 0..3 { + let blended = f64::from(self.data[i + c]) * (1.0 - alpha) + + f64::from(src.data[j + c]) * alpha; + self.data[i + c] = blended.round() as u8; + } + } + } + } + + fn box_blur(&self) -> Rgba { + let mut out = Rgba::new(self.w, self.h, [0, 0, 0]); + for y in 0..self.h { + for x in 0..self.w { + for c in 0..3 { + let mut sum = 0u32; + let mut n = 0u32; + for dy in -1i32..=1 { + for dx in -1i32..=1 { + let xx = x as i32 + dx; + let yy = y as i32 + dy; + if xx >= 0 + && yy >= 0 + && (xx as usize) < self.w + && (yy as usize) < self.h + { + sum += u32::from(self.get(xx as usize, yy as usize)[c]); + n += 1; + } + } + } + out.data[(y * self.w + x) * 4 + c] = (sum / n) as u8; + } + } + } + out + } +} + +fn marker_frame(marker: FrameMarker, w: usize, h: usize) -> Rgba { + Rgba { + data: video_code::marker_rgba(marker, w, h), + w, + h, + } +} + +/// A composited editor-like output: background, padded screen with rounded +/// corners, circular camera bubble bottom right. +fn composite(screen: u32, camera: u32, out_w: usize, out_h: usize, blur: bool) -> Rgba { + let screen_src = marker_frame(FrameMarker::new(StreamKind::Screen, 0, screen), 1280, 720); + let camera_src = marker_frame(FrameMarker::new(StreamKind::Camera, 0, camera), 640, 360); + let mut out = Rgba::new(out_w, out_h, [60, 60, 60]); + let pad = out_w as f64 * 0.06; + let sw = out_w as f64 - 2.0 * pad; + let sh = sw * 9.0 / 16.0; + let screen_img = screen_src.crop_scale(0.0, 0.0, 1280.0, 720.0, sw as usize, sh as usize); + let sy = ((out_h as f64 - sh) / 2.0).max(0.0); + out.paste_rounded(&screen_img, pad as usize, sy as usize, sw * 0.03); + let bubble = (out_h as f64 * 0.3).round() as usize; + let camera_img = camera_src.crop_scale(140.0, 0.0, 360.0, 360.0, bubble, bubble); + let bx = out_w - bubble - (out_w as f64 * 0.03) as usize; + let by = out_h - bubble - (out_h as f64 * 0.04) as usize; + out.paste_rounded(&camera_img, bx, by, bubble as f64 / 2.0); + if blur { out.box_blur() } else { out } +} + +fn encode_decode( + frames: &[Rgba], + codec: VideoCodec, + bitrate: u64, + dir: &std::path::Path, +) -> Option> { + let (w, h) = (frames[0].w as u32, frames[0].h as u32); + let mut config = VideoWriterConfig::new(w, h, 30); + config.codec = codec; + config.bitrate = Some(bitrate); + config.gop = 30; + let path = dir.join(format!("{codec:?}-{w}x{h}-{bitrate}.mp4")); + let mut writer = match VideoWriter::create(&path, &config) { + Ok(writer) => writer, + Err(error) if codec == VideoCodec::Hevc => { + eprintln!("skipping HEVC: {error}"); + return None; + } + Err(error) => panic!("{error:?}"), + }; + for (i, frame) in frames.iter().enumerate() { + writer.write_rgba(&frame.data, i as f64 / 30.0).unwrap(); + } + writer.finish().unwrap(); + let mut pictures = Vec::new(); + media::decode_video(&path, |p| pictures.push(p)).unwrap(); + Some(pictures) +} + +#[test] +fn video_code_survives_low_bitrate_h264_and_hevc_on_full_frames() { + let dir = tempfile::tempdir().unwrap(); + for (kind, w, h) in [ + (StreamKind::Screen, 640, 360), + (StreamKind::Camera, 320, 240), + ] { + let indices: Vec = (0..45).map(|i| (i * 977 + 13) % 65_536).collect(); + let frames: Vec = indices + .iter() + .map(|&i| marker_frame(FrameMarker::new(kind, 1, i), w, h)) + .collect(); + for codec in [VideoCodec::H264, VideoCodec::Hevc] { + let Some(pictures) = encode_decode(&frames, codec, 120_000, dir.path()) else { + continue; + }; + assert_eq!(pictures.len(), frames.len()); + for (picture, &index) in pictures.iter().zip(&indices) { + let readout = video_code::read_stream(&picture.view(), kind); + assert_eq!( + readout.marker(), + Some(FrameMarker::new(kind, 1, index)), + "{codec:?} {kind:?} at {:.3}s: {readout:?}", + picture.pts_secs + ); + } + } + } +} + +#[test] +fn video_code_survives_compositing_scaling_rounding_bubble_and_blur() { + let dir = tempfile::tempdir().unwrap(); + // 640x360 is a typical verification output; 320x180 puts the screen at + // a quarter of its source size and the camera code cells at ~6 px. + for (out_w, out_h, bitrate) in [(640, 360, 150_000), (320, 180, 80_000)] { + for blur in [false, true] { + let pairs: Vec<(u32, u32)> = (0..40).map(|i| (i * 3 + 100, i / 2 + 7)).collect(); + let frames: Vec = pairs + .iter() + .map(|&(s, c)| composite(s, c, out_w, out_h, blur)) + .collect(); + for (raw, &(s, c)) in frames.iter().zip(&pairs) { + let readout = video_code::read_frame(&raw.view()); + assert_eq!(readout.screen.marker().map(|m| m.index), Some(s)); + assert_eq!(readout.camera.marker().map(|m| m.index), Some(c)); + } + for codec in [VideoCodec::H264, VideoCodec::Hevc] { + let Some(pictures) = encode_decode(&frames, codec, bitrate, dir.path()) else { + continue; + }; + for (picture, &(s, c)) in pictures.iter().zip(&pairs) { + let readout = video_code::read_frame(&picture.view()); + assert_eq!( + ( + readout.screen.marker().map(|m| m.index), + readout.camera.marker().map(|m| m.index) + ), + (Some(s), Some(c)), + "{codec:?} {out_w}x{out_h} blur={blur}: {readout:?}" + ); + } + } + } + } +} + +#[test] +fn damaged_codes_read_as_unreadable_never_as_another_frame() { + let w = 320; + let h = 180; + let marker = FrameMarker::new(StreamKind::Screen, 0, 4242); + let mut hits = 0; + // Smear a grey bar across successive rows of the code: every damaged + // picture must read as the true marker or as unreadable. + for bar in 0..h { + let mut frame = marker_frame(marker, w, h); + for y in bar..(bar + 6).min(h) { + for x in 0..w { + let i = (y * w + x) * 4; + frame.data[i..i + 3].copy_from_slice(&[128, 128, 128]); + } + } + match video_code::read_stream(&frame.view(), StreamKind::Screen) { + Readout::Marker { marker: m, .. } => { + assert_eq!(m, marker); + hits += 1; + } + Readout::Unreadable { .. } | Readout::Absent | Readout::Blend { .. } => {} + } + } + assert!(hits > 0); +} + +#[test] +fn sequence_oracle_flags_wrong_duplicated_and_missing_frames_after_encode() { + let dir = tempfile::tempdir().unwrap(); + let expected: Vec = (0..60).collect(); + let mut shown = expected.clone(); + shown[10] = 9; // duplicate of the previous frame (stale) + shown[20] = 23; // a frame from the future + shown.remove(30); // one frame skipped... + shown.insert(31, 31); // ...and one repeated to keep the length + let frames: Vec = shown + .iter() + .map(|&i| marker_frame(FrameMarker::new(StreamKind::Screen, 0, i), 320, 180)) + .collect(); + let pictures = encode_decode(&frames, VideoCodec::H264, 100_000, dir.path()).unwrap(); + let observed: Vec = pictures + .iter() + .map(|p| video_code::read_stream(&p.view(), StreamKind::Screen)) + .collect(); + let expectations: Vec = expected + .iter() + .map(|&i| StreamExpectation::frame(0, i)) + .collect(); + let report = oracle::check_stream(StreamKind::Screen, &expectations, &observed, 30); + let wrong: Vec = report.mismatches.iter().map(|m| m.output_frame).collect(); + assert_eq!(wrong, vec![10, 20, 30]); + assert_eq!(report.stale, 1); + assert_eq!(report.ahead, 2); + assert_eq!(report.skipped_source_frames, vec![10, 20, 30]); + assert_eq!(report.repeated_source_frames, vec![9, 23, 31]); +} + +#[test] +fn audio_markers_locate_within_a_millisecond_after_lossy_codecs_and_resampling() { + let dir = tempfile::tempdir().unwrap(); + let rate = 48_000; + let secs = 8.0; + let mut mix = vec![0.0f32; (secs * f64::from(rate)) as usize]; + let mut schedules = Vec::new(); + for source in [AudioSource::Mic, AudioSource::System] { + let schedule = audio_code::standard_schedule(source, 0.25, secs - 0.5, 0.5); + audio_code::render_markers(&mut mix, rate, source, &schedule, 0.25); + schedules.push((source, schedule)); + } + for (codec, ext, bitrate) in [ + (AudioCodec::Aac, "m4a", 64_000), + (AudioCodec::Opus, "ogg", 32_000), + ] { + let path = dir.path().join(format!("markers.{ext}")); + media::write_audio(&path, &[mix.clone(), mix.clone()], rate, codec, bitrate).unwrap(); + for out_rate in [None, Some(44_100), Some(16_000)] { + let decoded = media::decode_audio(&path, out_rate).unwrap(); + let mono = decoded.mono(); + for (source, schedule) in &schedules { + let found = audio_code::locate_markers(&mono, decoded.sample_rate, *source); + assert_eq!( + found.len(), + schedule.len(), + "{codec:?} {out_rate:?} {source:?}" + ); + let mut worst = 0.0f64; + for (f, e) in found.iter().zip(schedule) { + assert_eq!(f.index, Some(e.index), "{codec:?} {out_rate:?} {source:?}"); + let error = (f.time_secs + decoded.start_secs - e.time_secs).abs(); + worst = worst.max(error); + assert!((f.gain / 0.25 - 1.0).abs() < 0.4, "gain {}", f.gain); + } + assert!( + worst < 0.001, + "{codec:?} {out_rate:?} {source:?}: worst error {:.3} ms", + worst * 1e3 + ); + eprintln!( + "{codec:?} {out_rate:?} {source:?}: {} markers, worst {:.3} ms", + found.len(), + worst * 1e3 + ); + } + } + } +} diff --git a/crates/verify/tests/common/mod.rs b/crates/verify/tests/common/mod.rs new file mode 100644 index 00000000000..e6ca89c7e87 --- /dev/null +++ b/crates/verify/tests/common/mod.rs @@ -0,0 +1,1225 @@ +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; + +use cap_project::{ + ClipTransition, ClipTransitionType, ProjectConfiguration, RecordingMeta, TimelineSegment, XY, +}; +use cap_verify::{ + fixture::{self, FixtureSpec, SourceTruth}, + media, + model::ReferenceModel, + oracle::{self, StreamExpectation}, + report::{self, Case}, + video_code::{self, FrameReadout, StreamKind}, +}; + +pub fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .thread_stack_size(16 * 1024 * 1024) + .enable_all() + .build() + .expect("runtime") +} + +/// `CAP_VERIFY_LOG=` (EnvFilter syntax, e.g. `wgpu_hal=warn,cap_rendering=info`) +/// prints product and wgpu logs to stderr, for triage on remote runners. +pub fn init() { + media::init(); + static LOG: std::sync::Once = std::sync::Once::new(); + LOG.call_once(|| { + if let Ok(filter) = std::env::var("CAP_VERIFY_LOG") { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::new(filter)) + .with_writer(std::io::stderr) + .try_init(); + } + }); +} + +/// A ready fixture project plus its decoded source truth. +pub struct Project { + pub name: String, + pub path: PathBuf, + pub meta: RecordingMeta, + pub truth: SourceTruth, +} + +pub fn project(spec: &FixtureSpec) -> Project { + init(); + let path = fixture::ensure_fixture(spec, fixture::write_fixture) + .unwrap_or_else(|e| panic!("fixture {}: {e:?}", spec.name)); + load(&spec.name, path) +} + +pub fn load(name: &str, path: PathBuf) -> Project { + let meta = RecordingMeta::load_for_project(&path).expect("fixture meta"); + let truth = SourceTruth::read(&path).expect("fixture truth"); + Project { + name: name.to_string(), + path, + meta, + truth, + } +} + +/// A fixture recorded through the real recording pipeline (real time on +/// first use, cached afterwards). +pub fn recorded(spec: &cap_verify::pipeline::RecordedSpec) -> Project { + init(); + let path = fixture::ensure_fixture_keyed(spec.cache_key(), |dir| { + runtime().block_on(cap_verify::pipeline::record_studio(spec, dir)) + }) + .unwrap_or_else(|e| panic!("recorded fixture {}: {e:?}", spec.name)); + load(&format!("rec-{}", spec.name), path) +} + +impl Project { + pub fn config(&self) -> ProjectConfiguration { + self.meta.project_config() + } + + pub fn model(&self, config: &ProjectConfiguration) -> ReferenceModel { + ReferenceModel::new(&self.meta, config, self.truth.clone()).expect("model") + } +} + +/// Replaces the timeline with `(start, end, timescale)` cuts of clip 0. +pub fn set_cuts(config: &mut ProjectConfiguration, cuts: &[(f64, f64, f64)]) { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.segments = cuts + .iter() + .map(|&(start, end, timescale)| TimelineSegment { + recording_clip: 0, + start, + end, + timescale, + name: None, + speed_audio_mode: None, + hide_cursor: None, + volume: None, + }) + .collect(); +} + +/// Replaces the timeline with `(clip, start, end, timescale)` segments. +pub fn set_clip_cuts(config: &mut ProjectConfiguration, cuts: &[(u32, f64, f64, f64)]) { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.segments = cuts + .iter() + .map(|&(clip, start, end, timescale)| TimelineSegment { + recording_clip: clip, + start, + end, + timescale, + name: None, + speed_audio_mode: None, + hide_cursor: None, + volume: None, + }) + .collect(); +} + +pub fn add_transition(config: &mut ProjectConfiguration, segment_index: u32, duration: f64) { + let timeline = config.timeline.as_mut().expect("timeline"); + timeline.transitions.push(ClipTransition { + segment_index, + kind: ClipTransitionType::CrossFade, + duration, + }); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Format { + Mp4, + Gif, + Mov, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Decoder { + /// The platform default: AVAssetReader on macOS, Media Foundation on + /// Windows, FFmpeg on Linux. + Default, + /// `--force-ffmpeg-decoder`. + Ffmpeg, +} + +impl Decoder { + pub fn label(self) -> &'static str { + match self { + Decoder::Default if cfg!(target_os = "macos") => "avf", + Decoder::Default if cfg!(target_os = "windows") => "mf", + Decoder::Default => "ffmpeg", + Decoder::Ffmpeg => "ffmpeg", + } + } + + /// Decoders that are actually distinct on this platform. + pub fn all() -> Vec { + if cfg!(any(target_os = "macos", target_os = "windows")) { + vec![Decoder::Default, Decoder::Ffmpeg] + } else { + vec![Decoder::Default] + } + } + + /// The `DecoderType` display prefix this choice must produce. A silent + /// fallback (Media Foundation or AVAssetReader failing over to FFmpeg) + /// would otherwise test the wrong decoder under this label. + pub fn expected_type(self) -> &'static str { + match self { + Decoder::Default if cfg!(target_os = "macos") => "AVAssetReader", + Decoder::Default if cfg!(target_os = "windows") => "MediaFoundation", + _ => "FFmpeg", + } + } +} + +/// Records which decoder each stream actually used and flags any stream +/// that is not on the requested decoder. +pub fn check_decoders(case: &mut Case, decoder: Decoder, segments: &[cap_editor::SegmentMedia]) { + let mut used = Vec::new(); + for (i, segment) in segments.iter().enumerate() { + let streams = + std::iter::once(("screen", Some(segment.decoders.screen_decoder_status()))).chain( + std::iter::once(("camera", segment.decoders.camera_decoder_status())), + ); + for (stream, status) in streams { + let Some(status) = status else { continue }; + let kind = status.decoder_type.to_string(); + used.push(format!("{stream}{i}={kind}")); + if !kind.starts_with(decoder.expected_type()) { + case.problem( + "decoder-fallback", + format!( + "segment {i} {stream} decoded by {kind}, expected {} (fallback reason: {})", + decoder.expected_type(), + status.fallback_reason.as_deref().unwrap_or("none") + ), + ); + } + } + } + case.note(format!("decoders: {}", used.join(" "))); +} + +/// Why `decoder` cannot run on this machine at all, probed once per +/// process. Only Windows has such a case: the Media Foundation decoder +/// needs a D3D11 device with video support, which GPU-less VMs (hosted CI +/// runners, where D3D11 falls back to WARP) cannot create, so the product +/// falls back to FFmpeg there. Any other Media Foundation failure stays a +/// `decoder-fallback` test failure. +pub fn decoder_unsupported(project: &Project, decoder: Decoder) -> Option { + if decoder != Decoder::Default || !cfg!(target_os = "windows") { + return None; + } + static PROBE: std::sync::OnceLock> = std::sync::OnceLock::new(); + PROBE + .get_or_init(|| { + let meta = project.meta.studio_meta()?; + let segments = runtime() + .block_on(cap_editor::create_segments(&project.meta, meta, false)) + .ok()?; + let reason = segments + .first()? + .decoders + .screen_decoder_status() + .fallback_reason + .clone()?; + reason.contains("D3D11CreateDevice").then(|| { + format!( + "Media Foundation needs a D3D11 device with video support and this machine has none (WARP has no video decode): {reason}" + ) + }) + }) + .clone() +} + +/// Starts a case, or settles it as unsupported and returns `None` when this +/// machine cannot run `decoder` (see [`decoder_unsupported`]). +pub fn begin(id: &str, project: &Project, decoder: Decoder) -> Option { + let mut case = Case::new(id); + if let Some(reason) = decoder_unsupported(project, decoder) { + case.unsupported(reason); + if case.unsupported.is_some() { + let _ = report::settle(case); + return None; + } + } + Some(case) +} + +/// Opens the project's decoders the way the exporter does and checks them. +/// The exporter's own decoders are private, so this probes the same +/// `create_segments` decision on the same files. +pub async fn probe_decoders(case: &mut Case, project: &Project, decoder: Decoder) { + let Some(meta) = project.meta.studio_meta() else { + return; + }; + match cap_editor::create_segments(&project.meta, meta, decoder == Decoder::Ffmpeg).await { + Ok(segments) => check_decoders(case, decoder, &segments), + Err(e) => case.problem("error", format!("decoder probe failed: {e}")), + } +} + +pub struct ExportRun { + pub fps: u32, + pub resolution: (u32, u32), + pub format: Format, + pub decoder: Decoder, +} + +pub async fn export( + project: &Project, + config: ProjectConfiguration, + run: &ExportRun, + out: &Path, +) -> Result { + let force = run.decoder == Decoder::Ffmpeg; + let base = cap_export::ExporterBase::builder(project.path.clone()) + .with_config(config) + .with_force_ffmpeg_decoder(force) + .with_output_path(out.to_path_buf()) + .build() + .await + .map_err(|e| format!("exporter build: {e}"))?; + let resolution_base = XY::new(run.resolution.0, run.resolution.1); + match run.format { + Format::Mp4 => { + cap_export::mp4::Mp4ExportSettings { + fps: run.fps, + resolution_base, + compression: cap_export::mp4::ExportCompression::Social, + custom_bpp: None, + force_ffmpeg_decoder: force, + optimize_filesize: false, + } + .export(base, |_| true) + .await + } + Format::Mov => { + cap_export::mov::MovExportSettings { + fps: run.fps, + resolution_base, + cursor_only: false, + } + .export(base, |_| true) + .await + } + Format::Gif => { + cap_export::gif::GifExportSettings { + fps: run.fps, + resolution_base, + quality: None, + } + .export(base, |_| true) + .await + } + } +} + +pub struct OutputFrame { + pub pts_secs: f64, + pub readout: FrameReadout, + /// Hash of the decoded picture, for determinism checks. + pub hash: u64, + /// Picture size. + pub size: (usize, usize), + /// Camera key region of every `LAYOUT_SAMPLE_EVERY`th frame, as fractions + /// of the picture size. + pub camera_region: Option, + /// Mean Y, Cb, Cr code values of a flat band of the screen's key colour + /// and the stream's colour tag, sampled like `camera_region` (YUV + /// outputs only). + pub screen_colour: Option, +} + +#[derive(Clone, Copy, Debug)] +pub struct ScreenColour { + pub ycbcr: [f64; 3], + pub space: ffmpeg::util::color::Space, + pub range: ffmpeg::util::color::Range, +} + +/// Measured minus expected (Y, Cb, Cr) code values for `rgb` encoded the way +/// the stream's colour tag says (BT.709 unless tagged BT.601; limited range +/// unless tagged full). +pub fn colour_error(colour: &ScreenColour, rgb: [u8; 3]) -> [f64; 3] { + use ffmpeg::util::color::{Range, Space}; + let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0); + let (kr, kb) = if matches!(colour.space, Space::BT470BG | Space::SMPTE170M) { + (0.299, 0.114) + } else { + (0.2126, 0.0722) + }; + let y = kr * r + (1.0 - kr - kb) * g + kb * b; + let (cb, cr) = ((b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr))); + let expected = if colour.range == Range::JPEG { + [255.0 * y, 128.0 + 255.0 * cb, 128.0 + 255.0 * cr] + } else { + [16.0 + 219.0 * y, 128.0 + 224.0 * cb, 128.0 + 224.0 * cr] + }; + [0, 1, 2].map(|k| colour.ycbcr[k] - expected[k]) +} + +/// Mean code values and tag of a rectangle (fractions of the picture). +pub fn rect_colour( + frame: &ffmpeg::frame::Video, + view: &impl video_code::PixelSource, + rect: [f64; 4], +) -> Option { + if !matches!( + frame.format(), + ffmpeg::format::Pixel::YUV420P | ffmpeg::format::Pixel::NV12 + ) { + return None; + } + let (w, h) = (view.width() as f64, view.height() as f64); + let (x0, x1) = ( + (rect[0] * w) as usize, + ((rect[2] * w) as usize).max((rect[0] * w) as usize + 1), + ); + let (y0, y1) = ( + (rect[1] * h) as usize, + ((rect[3] * h) as usize).max((rect[1] * h) as usize + 1), + ); + let mut sum = [0.0; 3]; + let mut n = 0.0; + for y in y0..y1 { + for x in x0..x1 { + let (cb, cr) = view.chroma(x, y); + sum[0] += view.luma(x, y); + sum[1] += cb + 128.0; + sum[2] += cr + 128.0; + n += 1.0; + } + } + Some(ScreenColour { + ycbcr: sum.map(|v| v / n), + space: frame.color_space(), + range: frame.color_range(), + }) +} + +/// Mean code values of the band 8-16% down the screen's key region, 20-80% +/// across (above the code square and the default camera position). +fn screen_colour( + frame: &ffmpeg::frame::Video, + view: &impl video_code::PixelSource, +) -> Option { + if !matches!( + frame.format(), + ffmpeg::format::Pixel::YUV420P | ffmpeg::format::Pixel::NV12 + ) { + return None; + } + let region = video_code::locate_region(view, StreamKind::Screen)?; + let (w, h) = (region.x1 - region.x0, region.y1 - region.y0); + let (x0, x1) = ( + (region.x0 + 0.2 * w) as usize, + (region.x0 + 0.8 * w) as usize, + ); + let (y0, y1) = ( + (region.y0 + 0.08 * h) as usize, + (region.y0 + 0.16 * h) as usize, + ); + let mut sum = [0.0; 3]; + let mut n = 0.0; + for y in y0..y1.max(y0 + 1) { + for x in x0..x1.max(x0 + 1) { + let (cb, cr) = view.chroma(x, y); + sum[0] += view.luma(x, y); + sum[1] += cb + 128.0; + sum[2] += cr + 128.0; + n += 1.0; + } + } + Some(ScreenColour { + ycbcr: sum.map(|v| v / n), + space: frame.color_space(), + range: frame.color_range(), + }) +} + +/// Frames between camera placement samples (placement does not change +/// between frames of the verification projects). +pub const LAYOUT_SAMPLE_EVERY: usize = 10; + +fn picture_hash(frame: &ffmpeg::frame::Video) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + let planes = if frame.format() == ffmpeg::format::Pixel::NV12 { + 2 + } else { + 3 + }; + for plane in 0..planes.min(frame.planes()) { + let width = frame.plane_width(plane) as usize + * if frame.format() == ffmpeg::format::Pixel::NV12 && plane == 1 { + 2 + } else { + 1 + }; + let stride = frame.stride(plane); + let data = frame.data(plane); + for row in 0..frame.plane_height(plane) as usize { + for &byte in &data[row * stride..row * stride + width] { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + } + } + hash +} + +/// Decodes an output and reads every frame, using the model's expectation +/// for frame `i` (when given) to verify transition blends. +pub fn read_output( + path: &Path, + expect: Option<&dyn Fn(usize) -> cap_verify::model::FrameExpectation>, +) -> Vec { + let mut frames = Vec::new(); + media::decode_video(path, |picture| { + let view = picture.view(); + let readout = match expect { + Some(expect) => read_expecting(&view, &expect(frames.len())), + None => video_code::read_frame(&view), + }; + let size = ( + picture.frame.width() as usize, + picture.frame.height() as usize, + ); + let camera_region = (frames.len() % LAYOUT_SAMPLE_EVERY == 0 + && matches!(readout.camera, video_code::Readout::Marker { .. })) + .then(|| video_code::locate_region(&view, StreamKind::Camera)) + .flatten() + .map(|r| video_code::Region { + x0: r.x0 / size.0 as f64, + x1: r.x1 / size.0 as f64, + y0: r.y0 / size.1 as f64, + y1: r.y1 / size.1 as f64, + }); + let screen_colour = (frames.len() % LAYOUT_SAMPLE_EVERY == 0) + .then(|| screen_colour(&picture.frame, &view)) + .flatten(); + frames.push(OutputFrame { + pts_secs: picture.pts_secs, + readout, + hash: picture_hash(&picture.frame), + size, + camera_region, + screen_colour, + }) + }) + .unwrap_or_else(|e| panic!("decode {path:?}: {e:?}")); + frames +} + +pub fn read_expecting( + img: &impl video_code::PixelSource, + expectation: &cap_verify::model::FrameExpectation, +) -> FrameReadout { + let readout = FrameReadout { + screen: oracle::read_expecting(img, StreamKind::Screen, &expectation.screen), + camera: oracle::read_expecting(img, StreamKind::Camera, &expectation.camera), + }; + if std::env::var("CAP_VERIFY_TRACE_BLEND").as_deref() == Ok("1") { + for (kind, expected, observed) in [ + (StreamKind::Screen, &expectation.screen, &readout.screen), + (StreamKind::Camera, &expectation.camera, &readout.camera), + ] { + if matches!(expected, StreamExpectation::Blend { .. }) + && let Ok(cells) = video_code::read_cells(img, kind) + { + eprintln!( + "[cap-verify-blend] {}", + serde_json::json!({ + "stream": kind, + "expected": expected, + "observed": observed, + "contrast": cells.contrast, + "levels": cells.levels, + }) + ); + } + } + } + readout +} + +/// Checks a sequential output (export or sequential playback) frame by frame +/// against the model. +pub fn check_sequence( + case: &mut Case, + model: &ReferenceModel, + frames: &[OutputFrame], + fps: u32, + check_timestamps: bool, +) { + let expected_count = model.output_frames(fps); + if frames.len() as u32 != expected_count { + case.measure( + "frame-count", + "missing", + f64::from(expected_count) - frames.len() as f64, + ); + case.problem( + "frame-count", + format!( + "output has {} frames, the model expects {expected_count} ({:.3}s at {fps} fps)", + frames.len(), + model.duration() + ), + ); + } + if check_timestamps { + let off_grid: Vec<(usize, f64)> = frames + .iter() + .enumerate() + .filter(|(i, f)| (f.pts_secs - *i as f64 / f64::from(fps)).abs() > 0.5 / f64::from(fps)) + .map(|(i, f)| (i, f.pts_secs)) + .collect(); + if !off_grid.is_empty() { + // Signature: how many frames, from which frame, and how far off + // the grid in frame periods (a one-slot shift is exactly 1). + for &(i, pts) in &off_grid { + case.measure("timestamps", "frame", i as f64); + case.measure( + "timestamps", + "off-periods", + ((pts * f64::from(fps) - i as f64).abs() * 100.0).round() / 100.0, + ); + } + case.count("timestamps", "frames", off_grid.len() as f64); + case.problem( + "timestamps", + format!( + "output timestamps off the {fps} fps grid in {} frames: {:?}", + off_grid.len(), + &off_grid[..off_grid.len().min(5)] + ), + ); + } + } + let n = frames.len().min(expected_count as usize); + let expectations: Vec<_> = (0..n as u32).map(|i| model.expect(i, fps)).collect(); + for kind in StreamKind::ALL { + let exp: Vec = + expectations.iter().map(|e| e.get(kind).clone()).collect(); + let obs: Vec<_> = frames[..n] + .iter() + .map(|f| f.readout.get(kind).clone()) + .collect(); + case.stream(oracle::check_stream(kind, &exp, &obs, fps)); + } +} + +/// Product contract for exported MP4s: a keyframe at least every +/// `DEFAULT_KEYFRAME_INTERVAL_SECS` (2 s, crates/enc-ffmpeg h264.rs), so +/// shared videos seek quickly. +pub const MAX_KEYFRAME_INTERVAL_SECS: f64 = 2.0; +/// Colour checks on a flat key-colour band (code values). +pub const COLOUR_LEVEL_TOLERANCE: f64 = 4.0; +pub const COLOUR_MATRIX_TOLERANCE: f64 = 2.0; + +/// Presentation checks every export gets on top of frame identity: the +/// keyframe interval (MP4), camera mirroring (the code reads mirrored exactly +/// when the project mirrors the camera; the screen never is), and camera +/// placement (the camera's key region lies in the corner/edge the project +/// asks for). +pub fn check_presentation( + case: &mut Case, + config: &ProjectConfiguration, + frames: &[OutputFrame], + path: &Path, + fps: u32, + keyframes: bool, +) { + if keyframes { + match media::keyframe_times(path) { + Err(e) => case.problem("error", format!("keyframes: {e}")), + Ok(times) => { + let end = frames.len() as f64 / f64::from(fps); + let slack = 1.0 / f64::from(fps) + 1e-3; + let mut previous = 0.0; + let mut worst: Option<(f64, f64)> = None; + if times.first().is_none_or(|&t| t > slack) { + case.problem( + "keyframes", + format!("the first frame is not a keyframe (keyframes {times:?})"), + ); + } + for &t in times.iter().chain(std::iter::once(&end)) { + let gap = t - previous; + if gap > MAX_KEYFRAME_INTERVAL_SECS + slack + && worst.is_none_or(|(_, g)| gap > g) + { + worst = Some((previous, gap)); + } + previous = t; + } + if let Some((from, gap)) = worst { + case.problem( + "keyframes", + format!( + "{gap:.3} s without a keyframe from {from:.3} s (contract: at most {MAX_KEYFRAME_INTERVAL_SECS} s; {} keyframes in {end:.3} s)", + times.len() + ), + ); + } + } + } + } + + let mut wrong_mirror = Vec::new(); + for (i, frame) in frames.iter().enumerate() { + for kind in StreamKind::ALL { + if let video_code::Readout::Marker { mirrored, .. } = frame.readout.get(kind) { + let expected = kind == StreamKind::Camera && config.camera.mirror; + if *mirrored != expected { + wrong_mirror.push((i, kind.name(), *mirrored)); + } + } + } + } + if !wrong_mirror.is_empty() { + case.problem( + "mirror", + format!( + "{} frames show a stream mirrored wrongly (camera mirror setting {}): first {:?}", + wrong_mirror.len(), + config.camera.mirror, + &wrong_mirror[..wrong_mirror.len().min(5)] + ), + ); + } + + // Colour level: the screen's key colour must decode, under the stream's + // own colour tag (what players use), to the source luma. (Its chroma + // barely separates BT.601 from BT.709; `check_colour` covers the matrix + // with a saturated colour.) + let level: Vec<(usize, f64)> = frames + .iter() + .enumerate() + .filter_map(|(i, frame)| { + let colour = frame.screen_colour?; + let [dy, _, _] = colour_error(&colour, StreamKind::Screen.key_rgb()); + (dy.abs() > COLOUR_LEVEL_TOLERANCE).then_some((i, dy)) + }) + .collect(); + for (_, dy) in &level { + case.measure("colour-level", "luma", *dy); + } + if let Some((i, dy)) = level.first() { + case.problem( + "colour-level", + format!( + "screen key colour luma off by {dy:+.1} code values in {} sampled frames (first frame {i}; tolerance {COLOUR_LEVEL_TOLERANCE})", + level.len() + ), + ); + } + + use cap_project::{CameraXPosition, CameraYPosition}; + let mut misplaced = Vec::new(); + for (i, frame) in frames.iter().enumerate() { + let Some(r) = frame.camera_region else { + continue; + }; + let (cx, cy) = ((r.x0 + r.x1) / 2.0, (r.y0 + r.y1) / 2.0); + let x_ok = match config.camera.position.x { + CameraXPosition::Left => cx < 0.5, + CameraXPosition::Center => (cx - 0.5).abs() < 0.05, + CameraXPosition::Right => cx > 0.5, + }; + let y_ok = match config.camera.position.y { + CameraYPosition::Top => cy < 0.5, + CameraYPosition::Bottom => cy > 0.5, + }; + if !(x_ok && y_ok) { + misplaced.push(( + i, + (cx * 1000.0).round() / 1000.0, + (cy * 1000.0).round() / 1000.0, + )); + } + } + if !misplaced.is_empty() { + case.problem( + "layout", + format!( + "camera drawn away from its {:?}/{:?} position in {} sampled frames (frame, centre x, centre y as fractions): {:?}", + config.camera.position.x, + config.camera.position.y, + misplaced.len(), + &misplaced[..misplaced.len().min(5)] + ), + ); + } +} + +pub fn finish(case: Case) { + if let Err(message) = report::settle(case) { + panic!("{message}"); + } +} + +pub fn scratch_dir(name: &str) -> tempfile::TempDir { + let root = fixture::fixture_root().join("scratch"); + std::fs::create_dir_all(&root).expect("scratch root"); + tempfile::Builder::new() + .prefix(&format!("{}-", name.replace('/', "_"))) + .tempdir_in(root) + .expect("scratch dir") +} + +/// Drives the editor's decode + render path offline, one requested frame +/// at a time (no wall clock): `create_segments` decoders, `FrameRenderer`, +/// `RendererLayers`. Seeks use `get_frames_initial` like the preview +/// renderer; frames that follow a seek use `get_frames` like playback. +pub struct Player { + constants: std::sync::Arc, + segments: Vec, + renderer: Option>, + layers: cap_rendering::RendererLayers, + zooms: Vec, + outgoing_zooms: std::collections::HashMap, + config: ProjectConfiguration, + duration: f64, + resolution: XY, +} + +impl Player { + pub async fn open( + project: &Project, + config: ProjectConfiguration, + decoder: Decoder, + resolution: (u32, u32), + ) -> Result { + let meta = project.meta.studio_meta().ok_or("not studio")?.clone(); + let recordings = cap_rendering::ProjectRecordingsMeta::new(&project.path, &meta)?; + let constants = std::sync::Arc::new( + cap_rendering::RenderVideoConstants::new( + &recordings.segments, + project.meta.clone(), + meta.clone(), + ) + .await + .map_err(|e| format!("render constants: {e}"))?, + ); + let segments = + cap_editor::create_segments(&project.meta, &meta, decoder == Decoder::Ffmpeg).await?; + let mut layers = cap_rendering::RendererLayers::new_with_options( + &constants.device, + &constants.queue, + constants.is_software_adapter, + ); + let (sw, sh) = segments[0].decoders.screen_video_dimensions(); + let camera = segments[0].decoders.camera_video_dimensions(); + layers.prepare_for_video_dimensions( + &constants.device, + sw, + sh, + camera.map(|c| c.0), + camera.map(|c| c.1), + ); + let duration = config.timeline.as_ref().map_or(0.0, |t| t.duration()); + let zooms = segments + .iter() + .map(|s| { + cap_rendering::ZoomTransformTimeline::from_project( + &config, + &s.cursor, + duration, + constants.options.screen_size, + ) + }) + .collect(); + // The renderer borrows the constants; both live as long as the + // player, and the Arc keeps the constants' address stable. + let constants_ref: &'static cap_rendering::RenderVideoConstants = + unsafe { &*std::sync::Arc::as_ptr(&constants) }; + Ok(Self { + renderer: Some(cap_rendering::FrameRenderer::new(constants_ref)), + constants, + segments, + layers, + zooms, + outgoing_zooms: Default::default(), + config, + duration, + resolution: XY::new(resolution.0, resolution.1), + }) + } + + /// Flags streams that are not on `decoder` (checked after the run, so a + /// mid-run fallback is caught too). + pub fn check_decoders(&self, case: &mut Case, decoder: Decoder) { + check_decoders(case, decoder, &self.segments); + case.note(format!( + "render adapter: {}{}", + self.constants.adapter_name(), + if self.constants.is_software_adapter { + " (software)" + } else { + "" + } + )); + } + + pub fn prefer_sequential(&self) { + for segment in &self.segments { + segment.decoders.prefer_sequential_access(); + } + } + + /// Renders output frame `frame` and reads its markers. `None` when the + /// timeline has no segment there or the decoders returned nothing. + pub async fn frame( + &mut self, + frame: u32, + fps: u32, + seek: bool, + expectation: Option<&cap_verify::model::FrameExpectation>, + ) -> Result { + let t = f64::from(frame) / f64::from(fps); + let mapping = self + .config + .timeline + .as_ref() + .and_then(|timeline| timeline.get_frame_mapping(t)) + .ok_or_else(|| format!("no timeline segment at {t:.3}s"))?; + let (incoming, outgoing) = match mapping { + cap_project::TimelineFrameMapping::Single { source, .. } + | cap_project::TimelineFrameMapping::Hold { source, .. } => { + ((source.segment.recording_clip, source.source_time), None) + } + cap_project::TimelineFrameMapping::Transition { + outgoing, + incoming, + kind, + progress, + .. + } => ( + (incoming.segment.recording_clip, incoming.source_time), + Some(( + outgoing.segment.recording_clip, + outgoing.source_time, + kind, + progress, + )), + ), + }; + let incoming_frames = self.fetch(incoming.0, incoming.1, seek).await?; + let rendered = if let Some((clip, time, kind, progress)) = outgoing { + let outgoing_frames = self.fetch(clip, time, seek).await?; + let screen_size = self.constants.options.screen_size; + let outgoing_zoom = self.outgoing_zooms.entry(clip).or_insert_with(|| { + cap_rendering::ZoomTransformTimeline::from_project_for_outgoing_clip( + &self.config, + &self.segments[clip as usize].cursor, + self.duration, + screen_size, + clip, + ) + }); + outgoing_zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32); + let outgoing_uniforms = cap_rendering::ProjectUniforms::new( + &self.constants, + &self.config, + frame, + fps, + self.resolution, + &self.segments[clip as usize].cursor, + &outgoing_frames, + self.duration, + outgoing_zoom, + ); + let zoom = &mut self.zooms[incoming.0 as usize]; + zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32); + let incoming_uniforms = cap_rendering::ProjectUniforms::new( + &self.constants, + &self.config, + frame, + fps, + self.resolution, + &self.segments[incoming.0 as usize].cursor, + &incoming_frames, + self.duration, + zoom, + ); + self.renderer + .as_mut() + .expect("renderer") + .render_transition_immediate( + cap_rendering::TransitionRenderInput { + segment_frames: outgoing_frames, + uniforms: outgoing_uniforms, + cursor: &self.segments[clip as usize].cursor, + render_display: true, + }, + cap_rendering::TransitionRenderInput { + segment_frames: incoming_frames, + uniforms: incoming_uniforms, + cursor: &self.segments[incoming.0 as usize].cursor, + render_display: true, + }, + kind, + progress as f32, + &mut self.layers, + ) + .await + .map_err(|e| format!("render transition: {e}"))? + } else { + let media = &self.segments[incoming.0 as usize]; + let zoom = &mut self.zooms[incoming.0 as usize]; + zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32); + let uniforms = cap_rendering::ProjectUniforms::new( + &self.constants, + &self.config, + frame, + fps, + self.resolution, + &media.cursor, + &incoming_frames, + self.duration, + zoom, + ); + self.renderer + .as_mut() + .expect("renderer") + .render_immediate( + incoming_frames, + uniforms, + &media.cursor, + true, + &mut self.layers, + ) + .await + .map_err(|e| format!("render: {e}"))? + }; + let view = video_code::PackedFrame { + data: &rendered.data, + width: rendered.width as usize, + height: rendered.height as usize, + stride: rendered.padded_bytes_per_row as usize, + order: video_code::PixelOrder::Rgba, + }; + Ok(match expectation { + Some(expectation) => read_expecting(&view, expectation), + None => video_code::read_frame(&view), + }) + } +} + +impl Player { + /// Decodes the frames for recording `clip` at recording `time`, as a + /// seek (`get_frames_initial`) or as playback (`get_frames`). + async fn fetch( + &self, + clip: u32, + time: f64, + seek: bool, + ) -> Result { + let media = &self.segments[clip as usize]; + let offsets = self + .config + .clips + .iter() + .find(|c| c.index == clip) + .map(|c| c.offsets) + .unwrap_or_default(); + let needs_camera = !self.config.camera.hide; + if seek { + media + .decoders + .get_frames_initial(time as f32, needs_camera, true, offsets) + .await + } else { + media + .decoders + .get_frames(time as f32, needs_camera, true, offsets) + .await + } + .ok_or_else(|| format!("decoders returned no frame for clip {clip} at {time:.3}s")) + } +} + +impl Drop for Player { + fn drop(&mut self) { + // Drop the renderer before the constants it borrows. + self.renderer.take(); + } +} + +/// Checks a list of (output frame, readout) pairs against the model. +pub fn check_requests( + case: &mut Case, + model: &ReferenceModel, + frames: &[u32], + readouts: &[FrameReadout], + fps: u32, +) { + let expectations: Vec<_> = frames.iter().map(|&f| model.expect(f, fps)).collect(); + for kind in StreamKind::ALL { + let exp: Vec = + expectations.iter().map(|e| e.get(kind).clone()).collect(); + let obs: Vec<_> = readouts.iter().map(|r| r.get(kind).clone()).collect(); + case.stream(oracle::check_stream_at(kind, &exp, &obs, frames, fps)); + } +} + +pub fn assert_passed(failures: Vec) { + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// What the playback loop reported about one real-time run. +#[derive(Debug, Default)] +pub struct RealtimeTelemetry { + /// From `Playback::start` to the clock starting (warm-up done), if it did. + pub ready_after: Option, + pub warmup_buffered: Option<(usize, usize)>, + /// Frames the loop skipped on purpose, by reason. + pub skipped: std::collections::BTreeMap, + pub renderer_dropped: u64, +} + +impl RealtimeTelemetry { + pub fn describe(&self) -> String { + format!( + "playback ready after {}, warm-up buffered {}, skipped {:?}, renderer dropped {}", + self.ready_after.map_or("never".to_string(), |d| format!( + "{:.0} ms", + d.as_secs_f64() * 1000.0 + )), + self.warmup_buffered + .map_or("-".to_string(), |(b, t)| format!("{b}/{t}")), + self.skipped, + self.renderer_dropped + ) + } +} + +/// Runs the editor's real-time playback loop (`cap_editor::Playback`: warm-up, +/// prefetch, frame cache, wall clock) from `start_frame` and returns every +/// frame it delivered, read with `expect(frame number)` (so transition frames +/// are read as blends), plus the loop's telemetry. The `seconds` window starts +/// when the playback clock starts (warm-up finished), not when `start` +/// returns, so a slow warm-up does not eat the measured window. +/// Video only (`create_segments_without_audio`): the audio clock has its own +/// oracle. Dropped frames are normal under load; delivered ones must be right. +#[allow(clippy::too_many_arguments)] +pub async fn realtime_playback( + case: &mut Case, + project: &Project, + config: ProjectConfiguration, + decoder: Decoder, + resolution: (u32, u32), + fps: u32, + start_frame: u32, + seconds: f64, + expect: &dyn Fn(u32) -> cap_verify::model::FrameExpectation, +) -> Result<(Vec<(u32, FrameReadout)>, RealtimeTelemetry), String> { + use std::sync::{Arc, Mutex}; + const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + let meta = project.meta.studio_meta().ok_or("not studio")?.clone(); + let recordings = cap_rendering::ProjectRecordingsMeta::new(&project.path, &meta)?; + let constants = Arc::new( + cap_rendering::RenderVideoConstants::new( + &recordings.segments, + project.meta.clone(), + meta.clone(), + ) + .await + .map_err(|e| format!("render constants: {e}"))?, + ); + let segments = Arc::new( + cap_editor::create_segments_without_audio(&project.meta, &meta, decoder == Decoder::Ffmpeg) + .await?, + ); + // The case is labelled by decoder: it must be the decoder that runs. + check_decoders(case, decoder, &segments); + let layers_rx = cap_editor::start_renderer_layers_creation(&constants, &config); + let layers_rx = cap_editor::finish_renderer_layers_creation(layers_rx).await; + let delivered: Arc>> = Default::default(); + let sink = delivered.clone(); + let frame_cb: cap_editor::EditorFrameCallback = Box::new(move |output, _| { + if let cap_editor::EditorFrameOutput::Rgba(frame) = output { + sink.lock().unwrap().push(frame); + } + }); + let renderer = Arc::new(cap_editor::Renderer::spawn( + constants.clone(), + frame_cb, + layers_rx, + )?); + let (_project_tx, project_rx) = tokio::sync::watch::channel(config); + let (audio_output, _audio_sink) = cap_editor::AudioOutput::new_offline(); + let (telemetry, mut telemetry_rx) = cap_editor::PlaybackTelemetry::channel(); + let playback = cap_editor::Playback { + renderer, + render_constants: constants, + start_frame_number: start_frame, + project: project_rx, + segment_medias: segments, + music: cap_editor::MusicTracks::new(), + audio_output: Arc::new(audio_output), + telemetry: Some(telemetry), + }; + let started = std::time::Instant::now(); + let handle = playback + .start(fps, XY::new(resolution.0, resolution.1)) + .await + .map_err(|e| format!("playback start: {e:?}"))?; + let mut report = RealtimeTelemetry::default(); + let absorb = |event: cap_editor::PlaybackTelemetryEvent, report: &mut RealtimeTelemetry| { + use cap_editor::PlaybackTelemetryEvent as E; + match event { + E::WarmupComplete { + buffered_frames, + target_frames, + .. + } => report.warmup_buffered = Some((buffered_frames, target_frames)), + E::ClockStarted { .. } => report.ready_after = Some(started.elapsed()), + E::FrameSkipped { + skipped, reason, .. + } => *report.skipped.entry(format!("{reason:?}")).or_default() += u64::from(skipped), + E::RendererDropped { .. } => report.renderer_dropped += 1, + _ => {} + } + }; + let ready_deadline = tokio::time::Instant::now() + READY_TIMEOUT; + while report.ready_after.is_none() { + match tokio::time::timeout_at(ready_deadline, telemetry_rx.recv()).await { + Ok(Some(event)) => absorb(event, &mut report), + _ => break, + } + } + if report.ready_after.is_some() { + tokio::time::sleep(std::time::Duration::from_secs_f64(seconds)).await; + } + handle.stop(); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + while let Ok(event) = telemetry_rx.try_recv() { + absorb(event, &mut report); + } + let frames = std::mem::take(&mut *delivered.lock().unwrap()); + let frames = frames + .into_iter() + .map(|frame| { + let view = video_code::PackedFrame { + data: &frame.data, + width: frame.width as usize, + height: frame.height as usize, + stride: frame.padded_bytes_per_row as usize, + order: video_code::PixelOrder::Rgba, + }; + ( + frame.frame_number, + read_expecting(&view, &expect(frame.frame_number)), + ) + }) + .collect(); + Ok((frames, report)) +} diff --git a/crates/verify/tests/crash.rs b/crates/verify/tests/crash.rs new file mode 100644 index 00000000000..33c3b8c2061 --- /dev/null +++ b/crates/verify/tests/crash.rs @@ -0,0 +1,1732 @@ +//! Crash safety of fragmented Studio recording: the out-of-process muxer +//! against the in-process one, and recovery after the recorder, the muxer or +//! the disk fails. +//! +//! Synthetic marker frames go through the product's Studio display muxers. +//! macOS: the in-process `MacOSFragmentedM4SMuxer` and the out-of-process +//! `OutOfProcessFragmentedM4SMuxer` (a `cap-muxer` child), fed NV12 IOSurface +//! sample buffers like ScreenCaptureKit delivers. Linux and Windows: the +//! shared `SegmentedVideoEncoder` writer through `ffmpeg::SegmentedVideoMuxer` +//! (the Linux Studio display muxer; on Windows the writer behind +//! `WindowsFragmentedM4SMuxer`, whose Direct3D capture frames cannot be +//! synthesized, so Windows has no out-of-process case here). +//! +//! Every frame handed to the pipeline is logged (index, capture time) line by +//! line, so the truth survives a SIGKILL of the recorder. Recovery is judged +//! against what is physically on disk after the failure: every fragment whose +//! box structure is complete must come back frame for frame at its capture +//! time. Frames lost to a failure must be the newest ones and fit in the +//! fragment being written: one GOP (`GOP_SECS` x fps frames, the encoder's +//! keyframe interval, so the longest a segment can be when frames arrive at +//! the nominal rate) plus `IN_FLIGHT` frames still inside the pipeline. + +mod common; + +use std::{ + collections::BTreeMap, + io::Write, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +use anyhow::{Context, anyhow, bail}; +use cap_media_info::{RawVideoFormat, VideoInfo}; +use cap_recording::{ + ChannelVideoSource, ChannelVideoSourceConfig, OutputPipeline, recovery::RecoveryManager, +}; +use cap_timestamp::{Timestamp, Timestamps}; +use cap_verify::{ + fixture::Rng, + media, + report::{self, Case, Tier}, + video_code::{self, FrameMarker, StreamKind}, +}; + +const FPS: u32 = 30; +const WIDTH: u32 = 640; +const HEIGHT: u32 = 360; +/// `DEFAULT_KEYFRAME_INTERVAL_SECS` in crates/enc-ffmpeg/src/video/h264.rs: +/// the encoder emits a keyframe every `GOP_SECS * fps` frames and both DASH +/// writers cut a segment only at a keyframe. +const GOP_SECS: f64 = 2.0; +const GOP: u32 = (GOP_SECS as u32) * FPS; +/// Frames that can still be inside the pipeline when the process dies: the +/// one the encoder holds back to stamp real durations (segmented_stream.rs +/// `queue_frame`), the one being encoded, and the one in the muxer channel. +/// The emitter runs at the nominal rate on an idle encoder, so the channel +/// holds at most one frame at a time. +const IN_FLIGHT: u32 = 3; +/// Recorded length of every case. +const SECONDS: f64 = 8.0; +/// Truth timestamps vs presentation times: the encoder time base and the +/// container round to at most 1/90000 s; half a millisecond is far below the +/// 33 ms frame period and above any rounding. +const PTS_TOLERANCE: f64 = 0.0005; +const CHILD_ENV: &str = "CAP_VERIFY_CRASH_CHILD"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + /// The default in-process writer. + InProcess, + /// `out_of_process_muxer` on (macOS only here). + OutOfProcess, +} + +impl Arm { + fn label(self) -> &'static str { + match self { + Arm::InProcess => "inproc", + Arm::OutOfProcess => "oop", + } + } + + fn parse(text: &str) -> Option { + match text { + "inproc" => Some(Arm::InProcess), + "oop" => Some(Arm::OutOfProcess), + _ => None, + } + } +} + +fn arms() -> Vec { + if cfg!(target_os = "macos") { + vec![Arm::InProcess, Arm::OutOfProcess] + } else { + vec![Arm::InProcess] + } +} + +/// One frame the emitter handed to the pipeline. +#[derive(Clone, Copy, Debug)] +struct Handed { + index: u32, + /// Seconds after the first handed frame. + secs: f64, +} + +/// What a recording run left behind: the truth and the display directory. +struct Run { + handed: Vec, + display: PathBuf, +} + +#[cfg(target_os = "macos")] +mod source { + use super::*; + use cap_recording::{ + MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig, OutOfProcessFragmentedM4SMuxer, + OutOfProcessFragmentedM4SMuxerConfig, screen_capture, + }; + use cidre::cm; + + pub type Frame = screen_capture::VideoFrame; + + pub fn info() -> VideoInfo { + VideoInfo::from_raw(RawVideoFormat::Nv12, WIDTH, HEIGHT, FPS) + } + + pub type Picture = cidre::arc::R; + + /// Renders ahead of time so emission keeps the nominal rate. + pub fn picture(index: u32) -> anyhow::Result { + picture_sized(index, WIDTH, HEIGHT, FPS) + } + + pub fn picture_sized(index: u32, width: u32, height: u32, fps: u32) -> anyhow::Result { + let rgba = video_code::marker_rgba( + FrameMarker::new(StreamKind::Screen, 0, index), + width as usize, + height as usize, + ); + let buf = cap_verify::fixture_avf::pixel_buffer(width as usize, height as usize, &rgba)?; + let format_desc = cm::VideoFormatDesc::with_image_buf(&buf) + .map_err(|e| anyhow!("format description: {e:?}"))?; + let timing = cm::SampleTimingInfo { + duration: cm::Time::new(1, fps as i32), + pts: cm::Time::new(i64::from(index), fps as i32), + dts: cm::Time::invalid(), + }; + cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &format_desc, &timing) + .map_err(|e| anyhow!("sample buffer: {e:?}")) + } + + pub fn frame(picture: Picture, at: Instant) -> Frame { + screen_capture::VideoFrame { + sample_buf: picture, + timestamp: Timestamp::Instant(at), + } + } + + pub async fn pipeline( + display: &Path, + arm: Arm, + rx: flume::Receiver, + timestamps: Timestamps, + ) -> anyhow::Result { + pipeline_with(display, arm, rx, timestamps, info()).await + } + + pub async fn pipeline_with( + display: &Path, + arm: Arm, + rx: flume::Receiver, + timestamps: Timestamps, + info: VideoInfo, + ) -> anyhow::Result { + let builder = OutputPipeline::builder(display.to_path_buf()) + .with_video::>(ChannelVideoSourceConfig::new(info, rx)) + .with_timestamps(timestamps); + match arm { + Arm::InProcess => { + builder + .build::>( + MacOSFragmentedM4SMuxerConfig::default(), + ) + .await + } + Arm::OutOfProcess => { + builder + .build::>( + OutOfProcessFragmentedM4SMuxerConfig::default(), + ) + .await + } + } + } +} + +#[cfg(not(target_os = "macos"))] +mod source { + use super::*; + use cap_recording::ffmpeg::{FFmpegVideoFrame, SegmentedVideoMuxer, SegmentedVideoMuxerConfig}; + + pub type Frame = FFmpegVideoFrame; + + pub fn info() -> VideoInfo { + VideoInfo::from_raw(RawVideoFormat::Bgra, WIDTH, HEIGHT, FPS) + } + + pub type Picture = ffmpeg::frame::Video; + + /// Renders ahead of time so emission keeps the nominal rate. + pub fn picture(index: u32) -> anyhow::Result { + let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, WIDTH, HEIGHT); + let stride = frame.stride(0); + video_code::draw_marker( + FrameMarker::new(StreamKind::Screen, 0, index), + frame.data_mut(0), + WIDTH as usize, + HEIGHT as usize, + stride, + video_code::PixelOrder::Bgra, + ); + Ok(frame) + } + + pub fn frame(picture: Picture, at: Instant) -> Frame { + FFmpegVideoFrame { + inner: picture, + timestamp: Timestamp::Instant(at), + } + } + + pub async fn pipeline( + display: &Path, + arm: Arm, + rx: flume::Receiver, + timestamps: Timestamps, + ) -> anyhow::Result { + anyhow::ensure!(arm == Arm::InProcess, "no out-of-process muxer here"); + OutputPipeline::builder(display.to_path_buf()) + .with_video::>(ChannelVideoSourceConfig::new(info(), rx)) + .with_timestamps(timestamps) + // The Studio display config (capture_pipeline.rs, X11Capture): + // 2 s segments. + .build::>(SegmentedVideoMuxerConfig { + segment_duration: Duration::from_secs(2), + ..Default::default() + }) + .await + } +} + +struct ObservedMuxer { + inner: M, + output_path: PathBuf, +} + +impl cap_recording::Muxer for ObservedMuxer +where + M::Config: Send, +{ + type Config = M::Config; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + audio_config: Option, + pause_flag: std::sync::Arc, + tasks: &mut cap_recording::TaskPool, + ) -> anyhow::Result { + let inner = M::setup( + config, + output_path.clone(), + video_config, + audio_config, + pause_flag, + tasks, + ) + .await?; + Ok(Self { inner, output_path }) + } + + fn stop(&mut self) { + self.inner.stop(); + std::fs::write(self.output_path.join("mux-video-stopped"), b"stopped") + .expect("write EOF acknowledgement"); + } + + fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { + finalization_barrier(&self.output_path)?; + self.inner.finish(timestamp) + } + + fn set_health_sender(&mut self, tx: cap_recording::HealthSender) { + self.inner.set_health_sender(tx); + } +} + +impl> cap_recording::VideoMuxer + for ObservedMuxer +where + M::Config: Send, +{ + type VideoFrame = source::Frame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + self.inner.send_video_frame(frame, timestamp) + } +} + +impl cap_recording::AudioMuxer for ObservedMuxer +where + M::Config: Send, +{ + fn send_audio_frame( + &mut self, + frame: cap_recording::AudioFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + self.inner.send_audio_frame(frame, timestamp) + } +} + +fn finalization_barrier(display: &Path) -> anyhow::Result<()> { + let hold = display.join("hold-finalization"); + if !hold.exists() { + return Ok(()); + } + std::fs::write(display.join("finalization-entered"), b"entered")?; + let deadline = Instant::now() + Duration::from_secs(45); + while hold.exists() { + anyhow::ensure!(Instant::now() < deadline, "finalization barrier timed out"); + std::thread::sleep(Duration::from_millis(2)); + } + Ok(()) +} + +/// Line-per-event truth log that a SIGKILL cannot lose (no buffering). +struct TruthLog(Option); + +impl TruthLog { + fn line(&mut self, text: &str) { + if let Some(file) = &mut self.0 { + let _ = file.write_all(format!("{text}\n").as_bytes()); + } + } +} + +/// Hooks a case runs while the recorder is live. +#[derive(Default)] +struct Live { + /// Called with every handed index (after it was handed). + on_frame: Option>, +} + +/// Records `seconds` of marker frames into `display` and stops like the +/// recorder (`OutputPipeline::stop`). +/// Every `step`-th slot of the nominal rate carries a frame (1 = steady; more +/// is a mostly static screen, where ScreenCaptureKit delivers only changes). +async fn record( + display: &Path, + arm: Arm, + seconds: f64, + step: u32, + truth: Option<&Path>, + mut live: Live, +) -> anyhow::Result { + std::fs::create_dir_all(display)?; + let mut log = TruthLog(match truth { + Some(path) => Some(std::fs::File::create(path)?), + None => None, + }); + let count = (seconds * f64::from(FPS)) as u32; + let slots: Vec = (0..count).step_by(step.max(1) as usize).collect(); + let mut pictures = slots + .iter() + .copied() + .map(source::picture) + .collect::>>()? + .into_iter(); + let timestamps = Timestamps::now(); + let (tx, rx) = flume::bounded::(8); + let mut pipeline = source::pipeline(display, arm, rx, timestamps).await?; + let mut health = pipeline.take_health_rx(); + let base = Instant::now() + Duration::from_millis(50); + let mut handed = Vec::new(); + for index in slots { + let at = base + Duration::from_secs_f64(f64::from(index) / f64::from(FPS)); + tokio::time::sleep_until(at.into()).await; + let picture = pictures.next().expect("rendered"); + let frame = source::frame(picture, at); + tx.send_async(frame) + .await + .map_err(|_| anyhow!("pipeline stopped taking frames at {index}"))?; + let secs = f64::from(index) / f64::from(FPS); + handed.push(Handed { index, secs }); + log.line(&format!("f {index} {secs:.6}")); + if let Some(hook) = live.on_frame.as_mut() { + hook(index); + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + drop(tx); + tokio::time::timeout(Duration::from_secs(5), async { + while !display.join("mux-video-stopped").exists() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .context("waiting for mux-video EOF")?; + log.line("stop"); + let finished = pipeline.stop().await; + if let Some(health) = health.as_mut() { + while let Ok(event) = health.try_recv() { + log.line(&format!("health {event:?}")); + } + } + finished.map_err(|e| anyhow!("stop: {e:#}"))?; + log.line("done"); + Ok(Run { + handed, + display: display.to_path_buf(), + }) +} + +fn read_truth(path: &Path) -> anyhow::Result<(Vec, bool, Vec)> { + let text = std::fs::read_to_string(path).with_context(|| format!("read {path:?}"))?; + let mut handed = Vec::new(); + let mut stopped = false; + let mut health = Vec::new(); + for line in text.lines() { + let mut parts = line.split(' '); + match parts.next() { + Some("f") => { + let (Some(index), Some(secs)) = (parts.next(), parts.next()) else { + continue; + }; + // A kill can cut the last line short. + if let (Ok(index), Ok(secs)) = (index.parse(), secs.parse()) { + if line.len() > 2 && secs_complete(line) { + handed.push(Handed { index, secs }); + } + } + } + Some("stop") => stopped = true, + Some("health") => health.push(line[7..].to_string()), + _ => {} + } + } + Ok((handed, stopped, health)) +} + +/// The writer prints 6 decimals; a shorter field was cut by the kill. +fn secs_complete(line: &str) -> bool { + line.rsplit(' ') + .next() + .and_then(|s| s.split('.').nth(1)) + .is_some_and(|d| d.len() == 6) +} + +/// Runs only as a child of `crash_safety` (`CAP_VERIFY_CRASH_CHILD` set): +/// records into the given directory until it is killed or finishes. +#[test] +fn crash_child_recorder() { + let Ok(spec) = std::env::var(CHILD_ENV) else { + return; + }; + common::init(); + let fields: BTreeMap<&str, &str> = spec + .split(';') + .filter_map(|kv| kv.split_once('=')) + .collect(); + let arm = Arm::parse(fields["arm"]).expect("arm"); + let display = PathBuf::from(fields["display"]); + let truth = PathBuf::from(fields["truth"]); + let seconds: f64 = fields["seconds"].parse().expect("seconds"); + let step: u32 = fields["step"].parse().expect("step"); + if arm == Arm::OutOfProcess { + cap_recording::oop_muxer::resolve_muxer_binary().expect("child recorder: cap-muxer binary"); + } + let rt = common::runtime(); + rt.block_on(record( + &display, + arm, + seconds, + step, + Some(&truth), + Live::default(), + )) + .expect("child recording"); +} + +struct ChildRecorder { + child: Child, + truth: PathBuf, +} + +impl ChildRecorder { + fn spawn( + display: &Path, + truth: &Path, + arm: Arm, + seconds: f64, + step: u32, + ) -> anyhow::Result { + let exe = std::env::current_exe()?; + let mut command = Command::new(exe); + if let Ok(muxer) = cap_recording::oop_muxer::resolve_muxer_binary() { + command.env(cap_recording::oop_muxer::ENV_BIN_PATH, muxer); + } + let child = command + .args([ + "crash_child_recorder", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .env( + CHILD_ENV, + format!( + "arm={};display={};truth={};seconds={seconds};step={step}", + arm.label(), + display.display(), + truth.display() + ), + ) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + Ok(Self { + child, + truth: truth.to_path_buf(), + }) + } + + /// Polls the truth log until `ready` holds (or the child exits). + fn wait_until( + &mut self, + timeout: Duration, + mut ready: impl FnMut(&[Handed], bool) -> bool, + ) -> anyhow::Result<()> { + let deadline = Instant::now() + timeout; + loop { + if self.truth.exists() { + let (handed, stopped, _) = read_truth(&self.truth)?; + if ready(&handed, stopped) { + return Ok(()); + } + } + if let Some(status) = self.child.try_wait()? { + bail!("child recorder exited early: {status}"); + } + if Instant::now() > deadline { + bail!("child recorder did not reach the kill point in {timeout:?}"); + } + std::thread::sleep(Duration::from_millis(2)); + } + } + + /// SIGKILL (TerminateProcess on Windows); returns the cap-muxer child + /// pids it left behind. + fn kill(mut self) -> anyhow::Result> { + let orphans = muxer_children(self.child.id()); + self.child.kill()?; + self.child.wait()?; + Ok(orphans) + } +} + +impl Drop for ChildRecorder { + fn drop(&mut self) { + if !matches!(self.child.try_wait(), Ok(Some(_))) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } + } +} + +/// `cap-muxer` processes whose parent is `pid`. +fn muxer_children(pid: u32) -> Vec { + if cfg!(windows) { + return Vec::new(); + } + let Ok(out) = Command::new("pgrep") + .args(["-P", &pid.to_string(), "-x", "cap-muxer"]) + .output() + else { + return Vec::new(); + }; + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| l.trim().parse().ok()) + .collect() +} + +fn alive(pid: u32) -> bool { + Command::new("kill") + .args(["-0", &pid.to_string()]) + .stderr(Stdio::null()) + .status() + .is_ok_and(|s| s.success()) +} + +fn signal(pid: u32, sig: &str) -> anyhow::Result<()> { + let status = Command::new("kill") + .args([&format!("-{sig}"), &pid.to_string()]) + .status()?; + anyhow::ensure!(status.success(), "kill -{sig} {pid} failed"); + Ok(()) +} + +/// After a recorder crash the orphaned cap-muxer sees EOF and finishes its +/// open segment; recovery happens on the next launch, after it exited. +fn wait_for_exit(pids: &[u32], case: &mut Case) { + let started = Instant::now(); + for &pid in pids { + while alive(pid) && started.elapsed() < Duration::from_secs(20) { + std::thread::sleep(Duration::from_millis(20)); + } + if alive(pid) { + case.problem( + "orphan", + format!("cap-muxer {pid} still running 20 s after the recorder died"), + ); + } + } + if !pids.is_empty() { + case.note(format!( + "orphaned cap-muxer exited {:.2}s after the kill", + started.elapsed().as_secs_f64() + )); + } +} + +/// Top-level ISO BMFF boxes span the file exactly and include moof + mdat: +/// the writer finished this fragment (independent of recovery.rs). +fn fragment_complete(path: &Path) -> bool { + let Ok(data) = std::fs::read(path) else { + return false; + }; + let mut at = 0usize; + let (mut moof, mut mdat) = (false, false); + while at + 8 <= data.len() { + let size32 = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as usize; + let kind = &data[at + 4..at + 8]; + let size = match size32 { + 0 => return false, + 1 => { + if at + 16 > data.len() { + return false; + } + u64::from_be_bytes(data[at + 8..at + 16].try_into().unwrap()) as usize + } + n => n, + }; + if size < 8 { + return false; + } + moof |= kind == b"moof"; + mdat |= kind == b"mdat"; + at += size; + } + at == data.len() && moof && mdat +} + +fn segment_index(name: &str) -> Option { + name.strip_prefix("segment_")? + .trim_end_matches(".tmp") + .strip_suffix(".m4s")? + .parse() + .ok() +} + +/// Physically complete fragments of one writer directory (finished +/// `segment_NNN.m4s`, or a finished `.m4s.tmp` the writer had not renamed +/// yet), in index order. +fn complete_fragments(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut found: BTreeMap = BTreeMap::new(); + for entry in entries.flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + let Some(index) = segment_index(name) else { + continue; + }; + if !fragment_complete(&path) { + continue; + } + // Prefer the renamed file when both exist. + if name.ends_with(".tmp") && found.contains_key(&index) { + continue; + } + found.insert(index, path); + } + found.into_values().collect() +} + +/// Writer directories: the display directory and every `respawn-N` beside +/// it (out-of-process muxer restarts), in order. +fn writer_dirs(display: &Path) -> Vec { + let mut dirs = vec![display.to_path_buf()]; + let mut respawns: Vec<(u32, PathBuf)> = std::fs::read_dir(display) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + let n = name.strip_prefix("respawn-")?.parse().ok()?; + Some((n, e.path())) + }) + .collect(); + respawns.sort(); + dirs.extend(respawns.into_iter().map(|(_, p)| p)); + dirs +} + +/// Decoded (marker index, presentation seconds) of a set of fragments, +/// joined with their init segment into a temporary file. +fn decode_fragments( + dir: &Path, + fragments: &[PathBuf], + scratch: &Path, +) -> anyhow::Result> { + if fragments.is_empty() { + return Ok(Vec::new()); + } + let mut joined = std::fs::read(dir.join("init.mp4")).context("init.mp4")?; + for fragment in fragments { + joined.extend(std::fs::read(fragment)?); + } + let path = scratch.join(format!( + "joined-{}.mp4", + dir.file_name().unwrap_or_default().to_string_lossy() + )); + std::fs::write(&path, joined)?; + let decoded = decode_markers(&path); + let _ = std::fs::remove_file(&path); + decoded +} + +fn decode_markers(path: &Path) -> anyhow::Result> { + let mut out = Vec::new(); + let mut unreadable = 0usize; + media::decode_video(path, |picture| { + let readout = video_code::read_frame(&picture.view()); + match readout.get(StreamKind::Screen).marker() { + Some(marker) => out.push((marker.index, picture.pts_secs)), + None => unreadable += 1, + } + })?; + anyhow::ensure!( + unreadable == 0, + "{unreadable} pictures without a readable marker" + ); + Ok(out) +} + +/// The frames of every physically complete fragment, per fragment, writer +/// directories in order. +fn fragments_on_disk(display: &Path, scratch: &Path) -> anyhow::Result>> { + let mut out = Vec::new(); + for dir in writer_dirs(display) { + for fragment in complete_fragments(&dir) { + out.push(decode_fragments( + &dir, + std::slice::from_ref(&fragment), + scratch, + )?); + } + } + Ok(out) +} + +fn pending_fragment_frames( + display: &Path, + scratch: &Path, +) -> anyhow::Result> { + let mut frames = std::collections::BTreeSet::new(); + for dir in writer_dirs(display) { + let Ok(bytes) = std::fs::read(dir.join("manifest.json")) else { + continue; + }; + let manifest: serde_json::Value = serde_json::from_slice(&bytes)?; + let entries = manifest + .get("segments") + .or_else(|| manifest.get("fragments")); + let Some(entries) = entries.and_then(serde_json::Value::as_array) else { + continue; + }; + for fragment in complete_fragments(&dir) { + let Some(index) = fragment + .file_name() + .and_then(|n| n.to_str()) + .and_then(segment_index) + else { + continue; + }; + let pending = entries.iter().any(|entry| { + entry + .get("is_complete") + .and_then(serde_json::Value::as_bool) + == Some(false) + && entry + .get("path") + .and_then(serde_json::Value::as_str) + .and_then(|path| Path::new(path).file_name()) + .and_then(|name| name.to_str()) + .and_then(segment_index) + == Some(index) + }); + if pending { + frames.extend( + decode_fragments(&dir, &[fragment], scratch)? + .into_iter() + .map(|frame| frame.0), + ); + } + } + } + Ok(frames) +} + +/// Every frame inside the physically complete fragments, in order. +fn on_disk(display: &Path, scratch: &Path) -> anyhow::Result> { + Ok(fragments_on_disk(display, scratch)?.concat()) +} + +/// A fragment covers the capture time from its first frame to the first +/// frame of the next one, and the encoders must start a new one once +/// `GOP_SECS` have passed: the span may exceed `GOP_SECS` only by the +/// interval before the frame that closed it. +fn check_fragment_spans(case: &mut Case, fragments: &[Vec<(u32, f64)>], handed: &[Handed]) { + let time: BTreeMap = handed.iter().map(|h| (h.index, h.secs)).collect(); + let next_after = |index: u32| handed.iter().find(|h| h.index > index).map(|h| h.secs); + let mut worst = 0.0f64; + for fragment in fragments { + let (Some(first), Some(last)) = (fragment.first(), fragment.last()) else { + continue; + }; + let (Some(&start), Some(&end)) = (time.get(&first.0), time.get(&last.0)) else { + continue; + }; + let Some(next) = next_after(last.0) else { + continue; + }; + let span = next - start; + worst = worst.max(span); + if span > GOP_SECS + (next - end) + 1e-6 { + case.problem( + "fragment-span", + format!( + "fragment of frames {}..={} covers {span:.2}s of capture, more than {GOP_SECS}s plus the {:.2}s before the frame that closed it", + first.0, + last.0, + next - end + ), + ); + } + } + case.note(format!( + "longest complete fragment covers {worst:.2}s of capture" + )); +} + +/// Out-of-process only: after the recorder died, the orphaned cap-muxer +/// sees EOF and must finish the fragment it had open (nothing it received +/// may be lost). +fn check_open_fragment(case: &mut Case, display: &Path) { + for dir in writer_dirs(display) { + for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with("segment_") + && name.ends_with(".m4s.tmp") + && !fragment_complete(&path) + { + let size = entry.metadata().map(|m| m.len()).unwrap_or(0); + case.problem( + "open-fragment-lost", + format!( + "the orphaned cap-muxer left {name} incomplete ({size} bytes): the packets it had received are lost" + ), + ); + } + } + } +} + +/// The recovered display: the app's display finalization +/// (`RecoveryManager::finalize_to_progressive_mp4`, the same call for a clean +/// stop and for crash recovery). +fn finalize(display: &Path, out: &Path) -> anyhow::Result> { + RecoveryManager::finalize_to_progressive_mp4(display, out) + .map_err(|e| anyhow!("finalize: {e}"))?; + decode_markers(out) +} + +/// Checks `frames` (decoded index, pts) against the truth: no duplicates, +/// increasing indices, each frame at its capture time relative to the first +/// recovered frame. Returns the recovered index set. +fn check_timeline(case: &mut Case, label: &str, frames: &[(u32, f64)], handed: &[Handed]) { + let truth: BTreeMap = handed.iter().map(|h| (h.index, h.secs)).collect(); + let Some(&(first_index, first_pts)) = frames.first() else { + case.problem("missing", format!("{label}: no frames")); + return; + }; + let Some(&first_truth) = truth.get(&first_index) else { + case.problem( + "unexpected", + format!("{label}: frame {first_index} was never handed"), + ); + return; + }; + let mut previous: Option = None; + let mut worst = (0.0f64, 0u32); + let mut off = Vec::new(); + for &(index, pts) in frames { + if let Some(p) = previous + && index <= p + { + case.problem( + "order", + format!("{label}: frame {index} after {p} (repeat or reorder)"), + ); + } + previous = Some(index); + let Some(&t) = truth.get(&index) else { + case.problem( + "unexpected", + format!("{label}: frame {index} was never handed"), + ); + continue; + }; + let error = (pts - first_pts) - (t - first_truth); + if error.abs() > worst.0.abs() { + worst = (error, index); + } + if error.abs() > PTS_TOLERANCE { + off.push(index); + } + } + if !off.is_empty() { + case.problem( + "timestamps", + format!( + "{label}: {} frames off their capture time by more than {:.1} ms, worst {:+.1} ms at frame {} (first {:?})", + off.len(), + PTS_TOLERANCE * 1e3, + worst.0 * 1e3, + worst.1, + &off[..off.len().min(5)] + ), + ); + } + case.note(format!( + "{label}: {} frames {}..={}, worst pts error {:+.2} ms", + frames.len(), + frames.first().map(|f| f.0).unwrap_or(0), + frames.last().map(|f| f.0).unwrap_or(0), + worst.0 * 1e3 + )); +} + +/// Recovery keeps everything physically complete and loses at most the +/// fragment being written plus the frames in flight, all at the end. +fn check_recovery( + case: &mut Case, + recovered: &[(u32, f64)], + physical: &[(u32, f64)], + handed: &[Handed], + pending: &std::collections::BTreeSet, +) { + check_timeline(case, "recovered", recovered, handed); + let recovered_set: std::collections::BTreeSet = recovered.iter().map(|f| f.0).collect(); + let physical_set: std::collections::BTreeSet = physical.iter().map(|f| f.0).collect(); + let dropped: Vec = physical_set.difference(&recovered_set).copied().collect(); + if !dropped.is_empty() { + case.count("dropped-complete", "frames", dropped.len() as f64); + case.count( + "dropped-complete", + "outside-pending", + dropped + .iter() + .filter(|index| !pending.contains(index)) + .count() as f64, + ); + case.problem( + "dropped-complete", + format!( + "recovery dropped {} frames that were inside complete fragments on disk: {}..={}", + dropped.len(), + dropped[0], + dropped[dropped.len() - 1] + ), + ); + } + let extra: Vec = recovered_set.difference(&physical_set).copied().collect(); + if !extra.is_empty() { + case.problem( + "unexpected", + format!( + "recovery produced {} frames that no complete fragment holds: {:?}", + extra.len(), + &extra[..extra.len().min(5)] + ), + ); + } + let last_physical = physical.iter().map(|f| f.0).max(); + let lost: Vec<&Handed> = handed + .iter() + .filter(|h| last_physical.is_none_or(|p| h.index > p)) + .collect(); + let lost_tail = lost.len() as u32; + // Capture time the crash lost: from the first lost frame to the last + // handed one, plus that frame's own period. + let lost_secs = match (lost.first(), lost.last()) { + (Some(a), Some(b)) => b.secs - a.secs + 1.0 / f64::from(FPS), + _ => 0.0, + }; + let interior: Vec = handed + .iter() + .map(|h| h.index) + .filter(|i| last_physical.is_some_and(|p| *i < p) && !physical_set.contains(i)) + .collect(); + if !interior.is_empty() { + case.count("missing", "frames", interior.len() as f64); + case.problem( + "missing", + format!( + "{} handed frames missing before the last complete fragment's end: {:?}", + interior.len(), + &interior[..interior.len().min(5)] + ), + ); + } + let bound = GOP + IN_FLIGHT; + // In time: one fragment of capture time, plus the in-flight frames and + // the frame that closes the fragment at the delivered interval (a static + // screen delivers far fewer frames than the nominal rate). + let interval = handed + .windows(2) + .map(|w| w[1].secs - w[0].secs) + .fold(1.0 / f64::from(FPS), f64::max); + let bound_secs = GOP_SECS + f64::from(IN_FLIGHT + 1) * interval; + case.note(format!( + "handed {} frames, complete on disk up to {:?}, lost tail {lost_tail} frames / {lost_secs:.2}s (bound {bound} frames / {bound_secs:.2}s)", + handed.len(), + last_physical + )); + if lost_tail > bound { + case.measure("loss", "frames", f64::from(lost_tail)); + case.problem( + "loss", + format!( + "the crash lost the last {lost_tail} handed frames, more than one fragment ({GOP}) plus {IN_FLIGHT} in flight" + ), + ); + } + if lost_secs > bound_secs + 1e-6 { + case.measure("loss-time", "secs", (lost_secs * 100.0).round() / 100.0); + case.problem( + "loss-time", + format!( + "the crash lost {lost_secs:.2}s of capture, more than one {GOP_SECS}s fragment plus {} frames in flight ({bound_secs:.2}s at the delivered interval)", IN_FLIGHT + 1 + ), + ); + } +} + +fn scratch_root() -> PathBuf { + std::env::var_os("CAP_VERIFY_SCRATCH") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join("cap-verify-crash") +} + +fn fresh(dir: &Path) -> anyhow::Result<()> { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir)?; + Ok(()) +} + +/// Clean recordings through both writers must give the same frames at the +/// same times, all of them, at their capture times. +fn equality(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> { + let scratch = scratch_root().join("equality"); + fresh(&scratch)?; + let mut results = Vec::new(); + for arm in arms() { + let display = scratch.join(arm.label()).join("display"); + let run = rt.block_on(record(&display, arm, SECONDS, 1, None, Live::default()))?; + let out = scratch.join(format!("{}.mp4", arm.label())); + let frames = finalize(&run.display, &out)?; + check_timeline(case, arm.label(), &frames, &run.handed); + let handed: std::collections::BTreeSet = run.handed.iter().map(|h| h.index).collect(); + let got: std::collections::BTreeSet = frames.iter().map(|f| f.0).collect(); + let missing: Vec = handed.difference(&got).copied().collect(); + if !missing.is_empty() { + case.problem( + "missing", + format!( + "{}: {} of {} handed frames missing after a clean stop: {:?}", + arm.label(), + missing.len(), + handed.len(), + &missing[..missing.len().min(8)] + ), + ); + } + results.push((arm, frames)); + } + if let [(_, a), (_, b)] = results.as_slice() { + let differ = a.len() != b.len() + || a.iter() + .zip(b) + .any(|(x, y)| x.0 != y.0 || (x.1 - y.1).abs() > PTS_TOLERANCE); + if differ { + case.problem( + "unequal", + format!( + "in-process and out-of-process outputs differ: {} vs {} frames", + a.len(), + b.len() + ), + ); + } else { + case.note(format!("in-process == out-of-process: {} frames", a.len())); + } + } + let _ = std::fs::remove_dir_all(&scratch); + Ok(()) +} + +/// Static-screen recording: a new picture every `STATIC_STEP` slots (3 per +/// second at 30 fps), killed after `STATIC_KILL_SECS`. +const STATIC_STEP: u32 = 10; +const STATIC_SECONDS: f64 = 14.0; +const STATIC_KILL_SECS: f64 = 11.0; + +/// Kills at 2*GOP + k for k = 0..=12 and reports the first k at which the +/// second fragment was complete on disk. +fn writer_latency(arm: Arm, case: &mut Case) -> anyhow::Result<()> { + let mut first_complete = None; + for k in 0..=12u32 { + let name = format!("latency{k}"); + let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label())); + fresh(&scratch)?; + let display = scratch.join("display"); + let truth = scratch.join("truth.txt"); + let mut child = ChildRecorder::spawn(&display, &truth, arm, 5.0, 1)?; + let n = 2 * GOP + k; + child.wait_until(Duration::from_secs(40), |handed, _| { + handed.last().is_some_and(|h| h.index >= n) + })?; + let orphans = child.kill()?; + wait_for_exit(&orphans, case); + let physical = on_disk(&display, &scratch)?; + let done = physical.iter().any(|f| f.0 == 2 * GOP - 1); + case.note(format!( + "kill after frame {n}: fragment 2 complete = {done}" + )); + let _ = std::fs::remove_dir_all(&scratch); + if done && first_complete.is_none() { + first_complete = Some(k); + } + } + case.note(format!( + "{}: fragment complete on disk {first_complete:?} frames after its closing keyframe was handed", + arm.label() + )); + Ok(()) +} + +/// When the recorder process dies. Indices are handed-frame indices. +#[derive(Clone, Copy, Debug)] +enum KillPoint { + /// After frame N was handed. + Frame(u32), + /// Some milliseconds after the recorder started stopping. + Finalize(u64), +} + +fn kill_points(seed: u64) -> Vec<(String, KillPoint)> { + let mut rng = Rng::new(seed); + let total = (SECONDS * f64::from(FPS)) as u32; + let mut points = Vec::new(); + // Mid-fragment, at seeded random frames past the first fragment. + for k in 0..2 { + let lo = GOP + 15; + let index = lo + (rng.next_u64() % u64::from(total - lo - 10)) as u32; + points.push((format!("random{k}"), KillPoint::Frame(index))); + } + // Around the second rollover: the keyframe that closes fragment 2 is + // frame 2*GOP; the writer renames the segment when the packet after it + // arrives (the encoder holds one back), and the manifest learns about + // it on its next 10-frame scan. + for offset in [0u32, 2, 5] { + points.push(( + format!("rollover+{offset}"), + KillPoint::Frame(2 * GOP + offset), + )); + } + for delay in [0u64, 40] { + points.push((format!("finalize+{delay}ms"), KillPoint::Finalize(delay))); + } + points +} + +/// Kills a child recorder at `point` and recovers what it left. +/// `seconds`/`step` shape the recording (see `record`). +fn recorder_kill( + arm: Arm, + name: &str, + point: KillPoint, + seconds: f64, + step: u32, + case: &mut Case, +) -> anyhow::Result<()> { + let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label())); + fresh(&scratch)?; + let display = scratch.join("display"); + let truth = scratch.join("truth.txt"); + if matches!(point, KillPoint::Finalize(_)) { + std::fs::create_dir_all(&display)?; + std::fs::write(display.join("hold-finalization"), b"hold")?; + } + let mut child = ChildRecorder::spawn(&display, &truth, arm, seconds, step)?; + let timeout = Duration::from_secs_f64(seconds + 30.0); + match point { + KillPoint::Frame(n) => child.wait_until(timeout, |handed, _| { + handed.last().is_some_and(|h| h.index >= n) + })?, + KillPoint::Finalize(ms) => { + child.wait_until(timeout, |_, stopped| { + stopped && display.join("finalization-entered").exists() + })?; + std::thread::sleep(Duration::from_millis(ms)); + } + } + let orphans = child.kill()?; + wait_for_exit(&orphans, case); + let (handed, stopped, _) = read_truth(&truth)?; + if matches!(point, KillPoint::Finalize(_)) { + check_finalize_coverage(case, &display, &truth)?; + } + case.note(format!( + "killed after frame {:?}{}", + handed.last().map(|h| h.index), + if stopped { " while stopping" } else { "" } + )); + if arm == Arm::OutOfProcess { + check_open_fragment(case, &display); + } + let fragments = fragments_on_disk(&display, &scratch)?; + check_fragment_spans(case, &fragments, &handed); + let physical = fragments.concat(); + let pending = pending_fragment_frames(&display, &scratch)?; + let recovered = match finalize(&display, &scratch.join("recovered.mp4")) { + Ok(frames) => frames, + Err(e) if physical.is_empty() => { + case.note(format!("nothing complete on disk; recovery: {e:#}")); + Vec::new() + } + Err(e) => return Err(e), + }; + check_recovery(case, &recovered, &physical, &handed, &pending); + if std::env::var_os("CAP_VERIFY_KEEP").is_none() { + let _ = std::fs::remove_dir_all(&scratch); + } + Ok(()) +} + +/// Out-of-process only: kill cap-muxer mid-recording. The recording must +/// continue in a respawned muxer, and recovery must keep every complete +/// fragment, at its capture time (a muxer crash must not shift later video +/// against audio). +#[cfg(unix)] +fn muxer_kill(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> { + let scratch = scratch_root().join("muxer-kill"); + fresh(&scratch)?; + let display = scratch.join("display"); + let kill_at = GOP + GOP / 2; + let me = std::process::id(); + let killed = std::sync::Arc::new(std::sync::Mutex::new(None::)); + let killed_hook = killed.clone(); + let live = Live { + on_frame: Some(Box::new(move |index| { + if index == kill_at + && let Some(&pid) = muxer_children(me).first() + { + let _ = signal(pid, "KILL"); + *killed_hook.lock().unwrap() = Some(pid); + } + })), + }; + let run = rt.block_on(record(&display, Arm::OutOfProcess, SECONDS, 1, None, live))?; + let Some(pid) = *killed.lock().unwrap() else { + bail!("no cap-muxer child to kill at frame {kill_at}"); + }; + case.note(format!("killed cap-muxer {pid} after frame {kill_at}")); + let respawns = writer_dirs(&display).len() - 1; + if respawns == 0 { + case.problem("respawn", "no respawned muxer after the kill"); + } + let physical = on_disk(&display, &scratch)?; + let recovered = finalize(&display, &scratch.join("recovered.mp4"))?; + check_timeline(case, "recovered", &recovered, &run.handed); + let recovered_set: std::collections::BTreeSet = recovered.iter().map(|f| f.0).collect(); + let dropped: Vec = physical + .iter() + .map(|f| f.0) + .filter(|i| !recovered_set.contains(i)) + .collect(); + if !dropped.is_empty() { + case.count("dropped-complete", "frames", dropped.len() as f64); + + case.problem( + "dropped-complete", + format!( + "finalize dropped {} frames inside complete fragments", + dropped.len() + ), + ); + } + // Loss: from the last frame the dead muxer finished to the first frame + // the respawned one kept (the next keyframe): at most the killed + // fragment plus one GOP until the encoder's next keyframe. + let indices: Vec = recovered.iter().map(|f| f.0).collect(); + let gap = indices + .windows(2) + .map(|w| w[1] - w[0] - 1) + .max() + .unwrap_or(0); + let last = run.handed.last().map(|h| h.index).unwrap_or(0); + let tail = last.saturating_sub(*indices.last().unwrap_or(&0)); + case.note(format!( + "{respawns} respawn(s); recovered {} of {} frames, largest gap {gap} frames, tail {tail}", + recovered.len(), + run.handed.len() + )); + if gap > 2 * GOP + IN_FLIGHT { + case.problem( + "loss", + format!( + "a muxer crash lost {gap} consecutive frames ({:.2}s), more than two fragments plus {IN_FLIGHT} in flight", + f64::from(gap) / f64::from(FPS) + ), + ); + } + if tail > IN_FLIGHT { + case.problem( + "loss", + format!("the respawned muxer lost the last {tail} frames of a clean stop"), + ); + } + let _ = std::fs::remove_dir_all(&scratch); + Ok(()) +} + +/// CPU seconds and resident memory of a process (`ps`). +#[cfg(target_os = "macos")] +fn process_usage(pid: u32) -> Option<(f64, u64)> { + let out = Command::new("ps") + .args(["-o", "time=,rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let mut fields = text.split_whitespace(); + let time = fields.next()?; + let rss: u64 = fields.next()?.parse().ok()?; + let mut secs = 0.0; + for part in time.split(':') { + secs = secs * 60.0 + part.parse::().ok()?; + } + Some((secs, rss * 1024)) +} + +/// Total CPU seconds of every VideoToolbox encoder service process (the +/// hardware encoder's host process; shared by every client on the machine). +#[cfg(target_os = "macos")] +fn encoder_service_cpu() -> f64 { + let Ok(out) = Command::new("pgrep") + .args(["-x", "VTEncoderXPCService"]) + .output() + else { + return 0.0; + }; + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| l.trim().parse().ok()) + .filter_map(process_usage) + .map(|u| u.0) + .sum() +} + +/// Overhead of the out-of-process muxer at a real Studio size: 1920x1080 +/// NV12 at 60 fps for `PERF_SECONDS`, interleaved A/B rounds. Reports CPU +/// seconds of the recorder and of cap-muxer per recorded second and their +/// peak resident memory (measurement only; the default decision reads it). +#[cfg(target_os = "macos")] +const PERF_SECONDS: f64 = 10.0; + +#[cfg(target_os = "macos")] +fn overhead(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> { + let (width, height, fps) = (1920u32, 1080u32, 60u32); + let ring: Vec = (0..fps) + .map(|i| source::picture_sized(i, width, height, fps)) + .collect::>()?; + let me = std::process::id(); + let mut results: BTreeMap<&str, Vec<(f64, f64, u64, u64)>> = BTreeMap::new(); + for round in 0..3 { + let order = if round % 2 == 0 { + [Arm::InProcess, Arm::OutOfProcess] + } else { + [Arm::OutOfProcess, Arm::InProcess] + }; + for arm in order { + let scratch = scratch_root().join(format!("perf-{}-{round}", arm.label())); + fresh(&scratch)?; + let display = scratch.join("display"); + let ring = ring.clone(); + let sample = rt.block_on(async { + let info = VideoInfo::from_raw(RawVideoFormat::Nv12, width, height, fps); + let (tx, rx) = flume::bounded::(8); + let pipeline = + source::pipeline_with(&display, arm, rx, Timestamps::now(), info).await?; + let before = process_usage(me).map(|u| u.0).unwrap_or(0.0); + let service_before = encoder_service_cpu(); + let base = Instant::now() + Duration::from_millis(50); + let count = (PERF_SECONDS * f64::from(fps)) as u32; + let (mut peak_self, mut peak_muxer, mut muxer_cpu) = (0u64, 0u64, 0.0f64); + for index in 0..count { + let at = base + Duration::from_secs_f64(f64::from(index) / f64::from(fps)); + tokio::time::sleep_until(at.into()).await; + let picture = ring[(index % fps) as usize].clone(); + tx.send_async(source::frame(picture, at)) + .await + .map_err(|_| anyhow!("pipeline stopped"))?; + if index % (fps / 2) == 0 { + if let Some((_, rss)) = process_usage(me) { + peak_self = peak_self.max(rss); + } + for pid in muxer_children(me) { + if let Some((cpu, rss)) = process_usage(pid) { + peak_muxer = peak_muxer.max(rss); + muxer_cpu = muxer_cpu.max(cpu); + } + } + } + } + for pid in muxer_children(me) { + if let Some((cpu, _)) = process_usage(pid) { + muxer_cpu = muxer_cpu.max(cpu); + } + } + drop(tx); + pipeline.stop().await.map_err(|e| anyhow!("stop: {e:#}"))?; + let after = process_usage(me).map(|u| u.0).unwrap_or(0.0); + let service = (encoder_service_cpu() - service_before).max(0.0); + anyhow::Ok(( + (after - before) / PERF_SECONDS, + muxer_cpu / PERF_SECONDS + service / PERF_SECONDS, + peak_self, + peak_muxer, + )) + })?; + case.note(format!( + "round {round} {}: recorder {:.3} CPU s/s, cap-muxer + VideoToolbox service {:.3} CPU s/s, recorder peak {} MB, cap-muxer peak {} MB", + arm.label(), + sample.0, + sample.1, + sample.2 / 1_000_000, + sample.3 / 1_000_000 + )); + results.entry(arm.label()).or_default().push(sample); + let _ = std::fs::remove_dir_all(&scratch); + } + } + for (arm, samples) in &results { + let mut total: Vec = samples.iter().map(|s| s.0 + s.1).collect(); + total.sort_by(f64::total_cmp); + case.note(format!( + "{arm}: median total CPU {:.3} s per recorded second (min {:.3}, max {:.3})", + total[total.len() / 2], + total[0], + total[total.len() - 1] + )); + } + Ok(()) +} + +#[test] +fn crash_safety() { + if std::env::var(CHILD_ENV).is_ok() { + return; + } + common::init(); + if cfg!(target_os = "macos") && std::env::var_os("CAP_MUXER_BIN").is_none() { + // Built next to the test binary by `cargo build -p cap-muxer`. + if let Some(bin) = std::env::current_exe() + .ok() + .and_then(|exe| Some(exe.parent()?.parent()?.join("cap-muxer"))) + .filter(|p| p.exists()) + { + let _ = cap_recording::oop_muxer::set_muxer_binary_override(bin); + } + } + let rt = common::runtime(); + let tier = report::tier(); + let mut failures = Vec::new(); + let mut settle = |case: Case| { + for note in &case.notes { + eprintln!(" note: {note}"); + } + if let Err(e) = report::settle(case) { + failures.push(e); + } + }; + + let oop_ready = cap_recording::oop_muxer::resolve_muxer_binary(); + + if cfg!(target_os = "macos") && report::selected("crash/equality") { + let mut case = Case::new("crash/equality"); + if let Err(e) = &oop_ready { + case.problem("harness", format!("cap-muxer not found: {e:#}")); + } else if let Err(e) = equality(&rt, &mut case) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + + #[cfg(target_os = "macos")] + if tier == Tier::Full && report::selected("crash/oop-overhead") { + let mut case = Case::new("crash/oop-overhead"); + if let Err(e) = overhead(&rt, &mut case) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + + #[cfg(unix)] + if cfg!(target_os = "macos") && report::selected("crash/muxer-kill/oop") { + let mut case = Case::new("crash/muxer-kill/oop"); + if let Err(e) = muxer_kill(&rt, &mut case) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + + let seed: u64 = std::env::var("CAP_VERIFY_CRASH_SEED") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(20261016); + for arm in arms() { + for (name, point) in kill_points(seed) { + let id = format!("crash/recorder-kill/{}/{name}", arm.label()); + if !report::selected(&id) || (tier == Tier::Fast && name.starts_with("random1")) { + continue; + } + let mut case = Case::new(&id); + case.note(format!("seed {seed}, point {point:?}")); + if let Err(e) = recorder_kill(arm, &name, point, SECONDS, 1, &mut case) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + // A mostly static screen: one frame every STATIC_STEP slots. A + // fragment must still close every GOP_SECS of capture time. + let id = format!("crash/recorder-kill/{}/static-screen", arm.label()); + if report::selected(&id) { + let mut case = Case::new(&id); + let kill = (STATIC_KILL_SECS * f64::from(FPS)) as u32; + if let Err(e) = recorder_kill( + arm, + "static", + KillPoint::Frame(kill), + STATIC_SECONDS, + STATIC_STEP, + &mut case, + ) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + // Writer latency: how many frames after the keyframe that closes a + // fragment the fragment is complete on disk (measurement only). + let id = format!("crash/writer-latency/{}", arm.label()); + if tier == Tier::Full && report::selected(&id) { + let mut case = Case::new(&id); + if let Err(e) = writer_latency(arm, &mut case) { + case.problem("error", format!("{e:#}")); + } + settle(case); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +fn check_finalize_coverage(case: &mut Case, display: &Path, truth: &Path) -> anyhow::Result<()> { + let done = std::fs::read_to_string(truth)? + .lines() + .any(|line| line == "done"); + let entered = display.join("finalization-entered").is_file(); + let held = display.join("hold-finalization").is_file(); + case.note(format!( + "finalization coverage: entered={entered}, held={held}, done={done}" + )); + if done || !entered || !held { + case.problem("harness", "kill missed the held finalization window"); + } + Ok(()) +} + +#[test] +fn finalize_barrier_is_default_off_and_blocks_until_released() { + let dir = tempfile::tempdir().unwrap(); + finalization_barrier(dir.path()).unwrap(); + assert!(!dir.path().join("finalization-entered").exists()); + std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap(); + let display = dir.path().to_path_buf(); + let (tx, rx) = std::sync::mpsc::channel(); + let handle = std::thread::spawn(move || { + finalization_barrier(&display).unwrap(); + tx.send(()).unwrap(); + }); + let deadline = Instant::now() + Duration::from_secs(5); + while !dir.path().join("finalization-entered").exists() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(1)); + } + assert!(rx.try_recv().is_err()); + std::fs::remove_file(dir.path().join("hold-finalization")).unwrap(); + rx.recv_timeout(Duration::from_secs(5)).unwrap(); + handle.join().unwrap(); +} + +#[test] +fn finalize_done_is_a_coverage_miss() { + let dir = tempfile::tempdir().unwrap(); + let truth = dir.path().join("truth.txt"); + std::fs::write(&truth, b"stop\ndone\n").unwrap(); + std::fs::write(dir.path().join("finalization-entered"), b"entered").unwrap(); + std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap(); + let mut case = Case::new("finalize-coverage-test"); + check_finalize_coverage(&mut case, dir.path(), &truth).unwrap(); + assert!(case.classes.contains("harness")); +} + +#[test] +fn duplicate_and_extra_frames_still_fail_after_eof() { + let handed = vec![ + Handed { + index: 0, + secs: 0.0, + }, + Handed { + index: 1, + secs: 1.0 / f64::from(FPS), + }, + ]; + for frames in [ + vec![(0, 0.0), (1, 1.0 / 30.0), (1, 2.0 / 30.0)], + vec![(0, 0.0), (1, 1.0 / 30.0), (2, 2.0 / 30.0)], + ] { + let mut case = Case::new("eof-mutation-test"); + check_timeline(&mut case, "mutated", &frames, &handed); + assert!(case.classes.contains("order") || case.classes.contains("unexpected")); + } +} + +#[test] +fn pending_fragment_bound_rejects_other_complete_frame_loss() { + let handed = vec![ + Handed { + index: 0, + secs: 0.0, + }, + Handed { + index: 1, + secs: 1.0 / f64::from(FPS), + }, + ]; + let physical = vec![(0, 0.0), (1, 1.0 / f64::from(FPS))]; + // The bound the pending-fragment (#2640) entry used: frames inside + // complete pending fragments only. + let bound = cap_verify_expect::Bound::from([ + ("count".into(), cap_verify_expect::Range::at(1.0)), + ( + "frames".into(), + cap_verify_expect::Range { + min: 1.0, + max: 60.0, + }, + ), + ("outside-pending".into(), cap_verify_expect::Range::at(0.0)), + ]); + for (outside, accepted) in [(0.0, true), (1.0, false)] { + let metrics = BTreeMap::from([ + ("count".into(), cap_verify_expect::Range::at(1.0)), + ("frames".into(), cap_verify_expect::Range::at(1.0)), + ( + "outside-pending".into(), + cap_verify_expect::Range::at(outside), + ), + ]); + assert_eq!( + cap_verify_expect::within(&bound, Some(&metrics)).is_ok(), + accepted + ); + } + let mut case = Case::new("pending-fragment-test"); + check_recovery( + &mut case, + &physical[..1], + &physical, + &handed, + &std::collections::BTreeSet::from([1]), + ); + assert!(case.classes.contains("dropped-complete")); + assert!(!case.classes.contains("loss")); +} diff --git a/crates/verify/tests/editing.rs b/crates/verify/tests/editing.rs new file mode 100644 index 00000000000..63090c7f0cd --- /dev/null +++ b/crates/verify/tests/editing.rs @@ -0,0 +1,519 @@ +//! Editing oracle: seeded random sequences of timeline edits (split, delete, +//! trim start/end, speed, reorder, clip volume, clip transitions, fullscreen +//! text holds) on project configs. +//! +//! - `mapping_*`: for every sequence, the engine's timeline mapping +//! (`TimelineConfiguration::duration`, `get_frame_mapping`, +//! `get_segment_time`) must agree with the independent model in +//! `edits::TimeModel` on a 1/60 s grid, at random times and just inside +//! every edge; without holds, also with the core reference model +//! (`cap_verify::model`). Failing sequences are shrunk to a minimal list +//! of ops before reporting. +//! - `export_*`: a few random edit sequences per fixture exported and +//! checked frame for frame against the core reference model. +//! +//! Undo/redo lives in the desktop app (`ProjectHistory` snapshots whole +//! project configs), not in an engine crate, so it is out of scope here. + +mod common; +mod edits; + +use cap_project::ProjectConfiguration; +use cap_verify::{ + catalog, + fixture::Rng, + model::ReferenceModel, + report::{self, Case, Tier}, +}; +use common::{Decoder, ExportRun, Format}; +use edits::{Mapping, Op, OpMix, TimeModel}; + +const TOLERANCE: f64 = 1e-9; +const EDGE_EPSILON: f64 = 1e-6; + +/// Output times to probe: a 1/60 s grid, random times and both sides of +/// every edge. +fn probe_times(model: &TimeModel, rng: &mut Rng) -> Vec { + let duration = model.duration(); + let mut times: Vec = (0..(duration * 60.0).ceil() as usize) + .map(|i| i as f64 / 60.0) + .collect(); + times.extend((0..64).map(|_| rng.f64() * duration)); + let mut edges = Vec::new(); + for (k, s) in model.segments.iter().enumerate() { + let next_in = model.segments.get(k + 1).map_or(0.0, |n| n.transition_in); + for e in [ + s.out_start, + s.out_start + s.transition_in, + s.out_start + s.duration - next_in, + ] { + edges.push(model.to_output(e)); + } + } + for &(a, b) in &model.holds { + edges.extend([a, b]); + } + for edge in edges { + times.extend([edge - 1e-4, edge + 1e-4]); + } + times.retain(|&t| t >= 0.0 && t < duration); + times +} + +#[derive(Clone, Copy, PartialEq)] +enum Check { + /// Engine mapping against the models. + Mapping, + /// Every output time inside `duration()` has a source frame. + Coverage, +} + +impl Check { + fn class(self) -> &'static str { + match self { + Check::Mapping => "mapping", + Check::Coverage => "coverage", + } + } +} + +/// Checks one edited config; returns the problems. +fn check_config( + config: &ProjectConfiguration, + durations: &[f64], + seed: u64, + check: Check, +) -> Vec { + let timeline = edits::timeline(config); + let model = TimeModel::new(config); + let mut problems = Vec::new(); + if check == Check::Coverage { + let mut rng = Rng::new(seed); + for t in probe_times(&model, &mut rng) { + if !model.near_edge(t, EDGE_EPSILON) && timeline.get_frame_mapping(t).is_none() { + problems.push(format!( + "t={t:.6}: no source frame inside the {:.6}s timeline", + timeline.duration() + )); + } + } + return problems; + } + if (timeline.duration() - model.duration()).abs() > TOLERANCE { + problems.push(format!( + "duration {:.9}, the model expects {:.9}", + timeline.duration(), + model.duration() + )); + } + let core = if model.holds.is_empty() { + let (meta, truth) = edits::synthetic_meta(durations); + Some(ReferenceModel::new(&meta, config, truth).expect("core model")) + } else { + None + }; + let mut rng = Rng::new(seed ^ 0x7123_4567); + for t in probe_times(&model, &mut rng) { + if model.near_edge(t, EDGE_EPSILON) { + continue; + } + let expected = model.map(t); + let actual = edits::product_map(timeline, t); + match (&expected, &actual) { + (Some(e), Some(a)) if edits::same_mapping(e, a, TOLERANCE) => {} + (None, None) => {} + _ => problems.push(format!("t={t:.6}: engine {actual:?}, model {expected:?}")), + } + let shown = config.get_segment_time(t).map(|(time, segment)| { + let index = timeline + .segments + .iter() + .position(|s| std::ptr::eq(s, segment)) + .expect("segment from this timeline"); + (index, time) + }); + let expected_shown = expected.map(|m| (m.shown().segment, m.shown().time)); + let shown_ok = match (shown, expected_shown) { + (Some((a, x)), Some((b, y))) => a == b && (x - y).abs() <= TOLERANCE, + (None, None) => true, + _ => false, + }; + if !shown_ok { + problems.push(format!( + "t={t:.6}: get_segment_time {shown:?}, model {expected_shown:?}" + )); + } + if let (Some(core), Some(Mapping::Single { source, .. })) = (&core, &expected) { + let clip = timeline.segments[source.segment].recording_clip as usize; + match core.expect_at(t).source { + Some((c, time)) if c == clip && (time - source.time).abs() <= TOLERANCE => {} + other => problems.push(format!( + "t={t:.6}: core reference model {other:?}, edit model clip {clip} at {:.9}", + source.time + )), + } + } + } + problems +} + +fn mapping_case(name: &str, mix: OpMix, seeds: u64, check: Check) { + let id = format!("editing/mapping/{name}"); + if !report::selected(&id) { + return; + } + let mut case = Case::new(&id); + let mut failed = 0; + let mut probes = 0usize; + for seed in 0..seeds { + let mut rng = Rng::new(0xED17_0000 ^ (seed << 8) ^ name.len() as u64); + let durations = edits::random_durations(&mut rng); + let base = edits::base_config(&durations); + let count = rng.range(1, 10) as usize; + let ops = edits::random_sequence(&mut rng, &base, count, mix); + let config = edits::replay(&base, &ops); + probes += probe_times(&TimeModel::new(&config), &mut Rng::new(seed)).len(); + let problems = check_config(&config, &durations, seed, check); + if problems.is_empty() { + continue; + } + failed += 1; + if failed > 3 { + continue; + } + let minimal = edits::shrink(&ops, |ops| { + !check_config(&edits::replay(&base, ops), &durations, seed, check).is_empty() + }); + let minimal_problems = + check_config(&edits::replay(&base, &minimal), &durations, seed, check); + case.problem(check.class(), format!( + "seed {seed}: clips {durations:?}, shrunk from {} to {} ops {minimal:?}: {} mismatches (first: {})", + ops.len(), + minimal.len(), + minimal_problems.len(), + minimal_problems + .iter() + .take(3) + .cloned() + .collect::>() + .join("; ") + )); + } + case.note(format!( + "{seeds} random edit sequences, {probes} probe times, {failed} sequences failed" + )); + if failed > 3 { + case.problem(check.class(), format!("{failed} sequences failed in total")); + } + if failed > 0 { + // The share of the (seeded, deterministic) sequences that fail: the + // signature an expectation bounds. + case.measure( + check.class(), + "failed-per-mille", + (failed as f64 * 10_000.0 / seeds as f64).round() / 10.0, + ); + } + finish(case); +} + +/// Settles a case, printing its notes (the core report prints them only in +/// the JSON report). +fn settle(case: Case) -> Option { + for note in &case.notes { + eprintln!("[cap-verify] {}: {note}", case.id); + } + report::settle(case).err() +} + +fn finish(case: Case) { + if let Some(message) = settle(case) { + panic!("{message}"); + } +} + +fn seeds(fast: u64, full: u64) -> u64 { + match report::tier() { + Tier::Fast => fast, + Tier::Full => full, + } +} + +#[test] +fn mapping_cuts_speeds() { + mapping_case( + "cuts-speeds", + OpMix { + transitions: false, + holds: false, + }, + seeds(1000, 20000), + Check::Mapping, + ); +} + +#[test] +fn mapping_transitions() { + mapping_case( + "transitions", + OpMix { + transitions: true, + holds: false, + }, + seeds(1000, 20000), + Check::Mapping, + ); +} + +#[test] +fn mapping_holds() { + mapping_case( + "holds", + OpMix { + transitions: false, + holds: true, + }, + seeds(1000, 20000), + Check::Mapping, + ); +} + +#[test] +fn mapping_all() { + mapping_case( + "all", + OpMix { + transitions: true, + holds: true, + }, + seeds(1000, 20000), + Check::Mapping, + ); +} + +/// Fullscreen text holds left past the end of the clips (place a text card, +/// then trim or delete clips) still count towards `duration()`, but those +/// output times have no source frame, so renders stop early. +#[test] +fn mapping_duration_covered() { + mapping_case( + "duration-covered", + OpMix { + transitions: true, + holds: true, + }, + seeds(1000, 20000), + Check::Coverage, + ); +} + +/// Random edits of a real fixture exported and checked frame by frame. +fn export_cases(spec: cap_verify::fixture::FixtureSpec, seeds: u64) -> Vec { + let project = common::project(&spec); + let base = project.config(); + let rt = common::runtime(); + let fps = 30; + let decoders = Decoder::all(); + let mut failures = Vec::new(); + for seed in 0..seeds { + let id = format!("editing/export/{}/seed-{seed}", spec.name); + if !report::selected(&id) { + continue; + } + let mut rng = Rng::new(0xED17_1000 ^ (seed << 8) ^ spec.name.len() as u64); + let count = rng.range(3, 8) as usize; + let mut ops = edits::random_sequence( + &mut rng, + &base, + count, + OpMix { + transitions: true, + holds: false, + }, + ); + // Odd seeds end with a transition on every boundary, long ones and + // ones below the minimum included, so clamping is always exercised. + if seed % 2 == 1 { + let segments = edits::timeline(&edits::replay(&base, &ops)).segments.len(); + for segment in 1..segments { + ops.push(Op::Transition { + segment, + duration: [3.0, 0.02, 0.4][(segment + seed as usize) % 3], + }); + } + } + let config = edits::replay(&base, &ops); + let mut case = Case::new(&id); + case.note(format!("ops: {ops:?}")); + let model = project.model(&config); + let dir = common::scratch_dir(&id); + let run = ExportRun { + fps, + resolution: (320, 180), + format: Format::Mp4, + decoder: decoders[seed as usize % decoders.len()], + }; + case.note(format!("decoder {}", run.decoder.label())); + match rt.block_on(common::export( + &project, + config, + &run, + &dir.path().join("out.mp4"), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => { + let frames = common::read_output(&out, Some(&|i| model.expect(i as u32, fps))); + common::check_sequence(&mut case, &model, &frames, fps, true); + } + } + failures.extend(settle(case)); + } + failures +} + +#[test] +fn export_single_clip() { + common::assert_passed(export_cases(catalog::basic(report::tier()), seeds(8, 40))); +} + +#[test] +fn export_multi_clip() { + common::assert_passed(export_cases( + catalog::multi_segment(report::tier()), + seeds(6, 30), + )); +} + +/// A cross-fade between two parts of the same recording clip: the export +/// reads one decoder at two positions, interleaved, for the whole +/// transition. Forward (later part incoming) and backward. +#[test] +fn same_clip_transition_export() { + let project = common::project(&catalog::basic(report::tier())); + let rt = common::runtime(); + let fps = 30; + let mut failures = Vec::new(); + for (name, cuts) in [ + ("forward", [(0.0, 2.0, 1.0), (3.0, 5.5, 1.0)]), + ("backward", [(3.0, 5.5, 1.0), (0.5, 2.5, 1.0)]), + ] { + for decoder in Decoder::all() { + let id = format!( + "editing/export/same-clip-transition/{name}/{}", + decoder.label() + ); + if !report::selected(&id) { + continue; + } + let mut config = project.config(); + common::set_cuts(&mut config, &cuts); + common::add_transition(&mut config, 1, 0.8); + let model = project.model(&config); + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + let run = ExportRun { + fps, + resolution: (320, 180), + format: Format::Mp4, + decoder, + }; + match rt.block_on(common::export( + &project, + config, + &run, + &dir.path().join("out.mp4"), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => { + let frames = common::read_output(&out, Some(&|i| model.expect(i as u32, fps))); + common::check_sequence(&mut case, &model, &frames, fps, true); + } + } + failures.extend(settle(case)); + } + } + common::assert_passed(failures); +} + +/// Deterministic repro for `editing/mapping/duration-covered`: a 1 s +/// fullscreen text card at 5.4 s, then the clip trimmed to 3.2 s. The +/// timeline reports 4.2 s; the export must have as many frames. +#[test] +fn hold_past_end_export() { + let id = "editing/export/hold-past-end"; + if !report::selected(id) { + return; + } + let project = common::project(&catalog::basic(report::tier())); + let mut config = project.config(); + let timeline = config.timeline.as_mut().expect("timeline"); + timeline + .text_segments + .push(edits::fullscreen_text(5.4, 6.4)); + timeline.segments[0].end = 3.2; + let duration = timeline.duration(); + let fps = 30; + let mut case = Case::new(id); + let dir = common::scratch_dir(id); + let run = ExportRun { + fps, + resolution: (320, 180), + format: Format::Mp4, + decoder: Decoder::Default, + }; + match common::runtime().block_on(common::export( + &project, + config, + &run, + &dir.path().join("out.mp4"), + )) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(out) => { + let frames = common::read_output(&out, None).len(); + let expected = (duration * f64::from(fps)).ceil() as usize; + if frames == expected { + case.note(format!("{frames} frames for the {duration:.3}s timeline")); + } else { + case.measure("frame-count", "missing", expected as f64 - frames as f64); + case.problem( + "frame-count", + format!( + "{frames} frames exported, the {duration:.3}s timeline needs {expected}" + ), + ); + } + } + } + finish(case); +} + +#[test] +fn shrinker_finds_the_minimal_sequence() { + let base = edits::base_config(&[6.0]); + let ops = vec![ + Op::Volume { + segment: 0, + volume: 0.5, + }, + Op::Split { + segment: 0, + at: 0.5, + }, + Op::Speed { + segment: 1, + timescale: 2.0, + }, + Op::Move { from: 0, to: 1 }, + ]; + let fails = |ops: &[Op]| { + let config = edits::replay(&base, ops); + edits::timeline(&config) + .segments + .first() + .is_some_and(|s| s.timescale == 2.0) + }; + assert!(fails(&ops)); + let minimal = edits::shrink(&ops, fails); + assert_eq!(minimal.len(), 3, "{minimal:?}"); + assert!(!minimal.iter().any(|op| matches!(op, Op::Volume { .. }))); +} diff --git a/crates/verify/tests/edits/mod.rs b/crates/verify/tests/edits/mod.rs new file mode 100644 index 00000000000..53898427ab1 --- /dev/null +++ b/crates/verify/tests/edits/mod.rs @@ -0,0 +1,734 @@ +//! Random timeline edits shared by `editing.rs` and `captions.rs`. +//! +//! - [`Op`]: one user edit (split, delete, trim, speed, reorder, clip +//! volume, clip transition, fullscreen text hold). Parameters are +//! fractions, so any sub-sequence of a recorded sequence replays to a valid +//! project; an op that no longer applies is skipped. That makes failures +//! shrinkable: [`shrink`] drops ops while the failure persists. +//! - [`TimeModel`]: an independent model of the output -> recording mapping, +//! written from the documented rules rather than by calling +//! `cap_project`: +//! - segments play back to back; segment `k` lasts +//! `(end - start) / timescale` and maps `u` seconds into it to recording +//! time `start + u * timescale`; +//! - a clip transition into `k` overlaps `k - 1` and `k` by its duration, +//! clamped to `[0.05, min(dur(k - 1), dur(k)) / 2]` and dropped when that +//! range is empty; inside it both sides play, `k - 1` from +//! `dur(k - 1) - transition` on; +//! - enabled fullscreen text segments are holds: the recording clock +//! pauses for their output-time window (overlapping or touching windows +//! merge), showing the frame at the hold start; the timeline grows by the +//! held time. + +#![allow(dead_code)] + +use cap_project::{ + ClipConfiguration, ClipTransition, ClipTransitionType, MultipleSegment, MultipleSegments, + Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, + StudioRecordingStatus, TextSegment, TimelineConfiguration, TimelineFrameMapping, + TimelineSegment, VideoMeta, +}; +use cap_verify::fixture::{Rng, SegmentTruth, SourceFrame, SourceTruth, TrackTruth}; +use cap_verify::video_code::StreamKind; +use relative_path::RelativePathBuf; + +/// No edit leaves a timeline segment shorter than this (recording seconds). +pub const MIN_SOURCE: f64 = 0.4; +pub const MIN_TRANSITION: f64 = 0.05; +const SPEEDS: [f64; 6] = [0.5, 0.75, 1.0, 1.25, 1.5, 2.0]; + +#[derive(Clone, Debug, PartialEq)] +pub enum Op { + Split { segment: usize, at: f64 }, + Delete { segment: usize }, + TrimStart { segment: usize, by: f64 }, + TrimEnd { segment: usize, by: f64 }, + Speed { segment: usize, timescale: f64 }, + Move { from: usize, to: usize }, + Volume { segment: usize, volume: f64 }, + Transition { segment: usize, duration: f64 }, + Hold { at: f64, length: f64 }, +} + +#[derive(Clone, Copy, Debug)] +pub struct OpMix { + pub transitions: bool, + pub holds: bool, +} + +pub fn timeline(config: &ProjectConfiguration) -> &TimelineConfiguration { + config.timeline.as_ref().expect("timeline") +} + +fn timeline_mut(config: &mut ProjectConfiguration) -> &mut TimelineConfiguration { + config.timeline.as_mut().expect("timeline") +} + +fn source_len(segment: &TimelineSegment) -> f64 { + segment.end - segment.start +} + +/// Keeps transitions sorted, one per boundary, on boundaries that exist. +fn tidy_transitions(timeline: &mut TimelineConfiguration) { + let count = timeline.segments.len(); + timeline + .transitions + .retain(|t| t.segment_index >= 1 && (t.segment_index as usize) < count); + timeline.transitions.sort_by_key(|t| t.segment_index); + timeline + .transitions + .dedup_by(|later, earlier| later.segment_index == earlier.segment_index); +} + +/// Applies `op`; false when it does not apply to this project (it is then +/// skipped, which keeps every sub-sequence replayable). +pub fn apply(config: &mut ProjectConfiguration, op: &Op) -> bool { + let total = timeline(config).duration(); + let timeline = timeline_mut(config); + let count = timeline.segments.len(); + let valid = |index: usize| index < count; + match *op { + Op::Split { segment, at } => { + if !valid(segment) || source_len(&timeline.segments[segment]) < 2.0 * MIN_SOURCE { + return false; + } + let s = &timeline.segments[segment]; + let cut = s.start + MIN_SOURCE + at * (source_len(s) - 2.0 * MIN_SOURCE); + let mut tail = s.clone(); + tail.start = cut; + timeline.segments[segment].end = cut; + timeline.segments.insert(segment + 1, tail); + for t in &mut timeline.transitions { + if t.segment_index as usize > segment { + t.segment_index += 1; + } + } + } + Op::Delete { segment } => { + if !valid(segment) || count < 2 { + return false; + } + timeline.segments.remove(segment); + let k = segment as u32; + timeline + .transitions + .retain(|t| t.segment_index != k && t.segment_index != k + 1); + for t in &mut timeline.transitions { + if t.segment_index > k + 1 { + t.segment_index -= 1; + } + } + } + Op::TrimStart { segment, by } | Op::TrimEnd { segment, by } => { + if !valid(segment) { + return false; + } + let s = &mut timeline.segments[segment]; + let room = source_len(s) - MIN_SOURCE; + if room <= 0.0 { + return false; + } + if matches!(op, Op::TrimStart { .. }) { + s.start += by * room; + } else { + s.end -= by * room; + } + } + Op::Speed { segment, timescale } => { + if !valid(segment) { + return false; + } + timeline.segments[segment].timescale = timescale; + } + Op::Move { from, to } => { + if !valid(from) || !valid(to) || from == to { + return false; + } + let moved = timeline.segments.remove(from); + timeline.segments.insert(to, moved); + timeline.transitions.clear(); + } + Op::Volume { segment, volume } => { + if !valid(segment) { + return false; + } + timeline.segments[segment].volume = Some(volume); + } + Op::Transition { segment, duration } => { + if segment == 0 || !valid(segment) { + return false; + } + timeline + .transitions + .retain(|t| t.segment_index as usize != segment); + timeline.transitions.push(ClipTransition { + segment_index: segment as u32, + kind: ClipTransitionType::CrossFade, + duration, + }); + } + Op::Hold { at, length } => { + let start = at * total; + timeline + .text_segments + .push(fullscreen_text(start, start + length)); + } + } + tidy_transitions(timeline); + true +} + +pub fn fullscreen_text(start: f64, end: f64) -> TextSegment { + serde_json::from_value(serde_json::json!({ + "start": start, + "end": end, + "content": "hold", + "layout": "fullscreen", + })) + .expect("text segment") +} + +pub fn random_op(rng: &mut Rng, config: &ProjectConfiguration, mix: OpMix) -> Op { + let count = timeline(config).segments.len() as u64; + let segment = |rng: &mut Rng| rng.range(0, count - 1) as usize; + loop { + let op = match rng.range(0, 9) { + 0 | 1 => Op::Split { + segment: segment(rng), + at: rng.f64(), + }, + 2 => Op::Delete { + segment: segment(rng), + }, + 3 => Op::TrimStart { + segment: segment(rng), + by: rng.f64() * 0.6, + }, + 4 => Op::TrimEnd { + segment: segment(rng), + by: rng.f64() * 0.6, + }, + 5 => Op::Speed { + segment: segment(rng), + timescale: SPEEDS[rng.range(0, SPEEDS.len() as u64 - 1) as usize], + }, + 6 => Op::Move { + from: segment(rng), + to: segment(rng), + }, + 7 => Op::Volume { + segment: segment(rng), + volume: [0.0, 0.5, 1.5][rng.range(0, 2) as usize], + }, + 8 if mix.transitions => Op::Transition { + segment: segment(rng), + duration: [0.02, 0.2, 0.4, 0.8, 3.0][rng.range(0, 4) as usize], + }, + 9 if mix.holds => Op::Hold { + at: rng.f64(), + length: 0.2 + rng.f64() * 1.5, + }, + _ => continue, + }; + return op; + } +} + +/// `count` random ops that all apply, starting from `base`. +pub fn random_sequence( + rng: &mut Rng, + base: &ProjectConfiguration, + count: usize, + mix: OpMix, +) -> Vec { + let mut config = base.clone(); + let mut ops = Vec::new(); + let mut attempts = 0; + while ops.len() < count && attempts < count * 20 { + attempts += 1; + let op = random_op(rng, &config, mix); + if apply(&mut config, &op) { + ops.push(op); + } + } + ops +} + +pub fn replay(base: &ProjectConfiguration, ops: &[Op]) -> ProjectConfiguration { + let mut config = base.clone(); + for op in ops { + apply(&mut config, op); + } + config +} + +/// Greedy shrink: drops ops one at a time while `fails` still fails. +pub fn shrink(ops: &[Op], fails: impl Fn(&[Op]) -> bool) -> Vec { + let mut current = ops.to_vec(); + loop { + let mut reduced = false; + let mut i = 0; + while i < current.len() { + let mut candidate = current.clone(); + candidate.remove(i); + if fails(&candidate) { + current = candidate; + reduced = true; + } else { + i += 1; + } + } + if !reduced { + return current; + } + } +} + +/// A project whose recording clips last `durations` seconds, each on the +/// timeline once, in order. +pub fn base_config(durations: &[f64]) -> ProjectConfiguration { + let mut config = cap_verify::fixture::verification_project_config(); + config.timeline = Some(TimelineConfiguration { + segments: durations + .iter() + .enumerate() + .map(|(i, &d)| TimelineSegment { + recording_clip: i as u32, + timescale: 1.0, + start: 0.0, + end: d, + name: None, + speed_audio_mode: None, + volume: None, + hide_cursor: None, + }) + .collect(), + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + style_segments: Vec::new(), + image_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }); + config.clips = (0..durations.len()) + .map(|i| ClipConfiguration { + index: i as u32, + offsets: Default::default(), + offsets_auto_calculated: true, + }) + .collect(); + config +} + +/// 1 to 3 recording clips of 2 to 8 s (whole 30 fps frames). +pub fn random_durations(rng: &mut Rng) -> Vec { + (0..rng.range(1, 3)) + .map(|_| rng.range(60, 240) as f64 / 30.0) + .collect() +} + +/// In-memory recording meta + source truth for `durations` (30 fps screen, +/// no other tracks), so the core reference model can be built without media. +pub fn synthetic_meta(durations: &[f64]) -> (RecordingMeta, SourceTruth) { + let segments = durations + .iter() + .enumerate() + .map(|(i, _)| MultipleSegment { + display: VideoMeta { + path: RelativePathBuf::from(format!("content/segments/segment-{i}/display.mp4")), + fps: 30, + start_time: Some(0.0), + device_id: None, + }, + camera: None, + mic: None, + system_audio: None, + cursor: None, + keyboard: None, + display_notch: None, + }) + .collect(); + let meta = RecordingMeta { + platform: Some(Platform::default()), + project_path: std::env::temp_dir(), + pretty_name: "synthetic".into(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { + inner: MultipleSegments { + segments, + cursors: Default::default(), + status: Some(StudioRecordingStatus::Complete), + }, + })), + upload: None, + }; + let truth = SourceTruth { + segments: durations + .iter() + .enumerate() + .map(|(i, &d)| SegmentTruth { + screen: TrackTruth { + kind: StreamKind::Screen, + segment: i as u8, + frames: (0..(d * 30.0).round() as u32) + .map(|k| SourceFrame { + pts_secs: f64::from(k) / 30.0, + index: k, + }) + .collect(), + }, + camera: None, + }) + .collect(), + }; + (meta, truth) +} + +#[derive(Clone, Debug)] +pub struct Placed { + pub clip: u32, + pub start: f64, + pub end: f64, + pub timescale: f64, + pub duration: f64, + pub out_start: f64, + pub transition_in: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Source { + pub segment: usize, + pub time: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Mapping { + Single { + source: Source, + output_end: f64, + }, + Transition { + outgoing: Source, + incoming: Source, + progress: f64, + output_end: f64, + }, + Hold { + source: Source, + output_end: f64, + }, +} + +impl Mapping { + /// The source a single-frame consumer shows: the incoming side of a + /// transition. + pub fn shown(&self) -> Source { + match *self { + Mapping::Single { source, .. } | Mapping::Hold { source, .. } => source, + Mapping::Transition { incoming, .. } => incoming, + } + } +} + +pub struct TimeModel { + pub segments: Vec, + /// Merged hold windows in output time. + pub holds: Vec<(f64, f64)>, +} + +impl TimeModel { + pub fn new(config: &ProjectConfiguration) -> Self { + let timeline = timeline(config); + let mut segments: Vec = timeline + .segments + .iter() + .map(|s| Placed { + clip: s.recording_clip, + start: s.start, + end: s.end, + timescale: s.timescale, + duration: (s.end - s.start) / s.timescale, + out_start: 0.0, + transition_in: 0.0, + }) + .collect(); + for k in 1..segments.len() { + // The last transition listed for a boundary wins. + let Some(t) = timeline + .transitions + .iter() + .rev() + .find(|t| t.segment_index as usize == k) + else { + continue; + }; + let max = segments[k - 1].duration.min(segments[k].duration) / 2.0; + if t.duration.is_finite() && t.duration > 0.0 && max >= MIN_TRANSITION { + segments[k].transition_in = t.duration.max(MIN_TRANSITION).min(max); + } + } + let mut at = 0.0; + for s in &mut segments { + at -= s.transition_in; + s.out_start = at; + at += s.duration; + } + let mut windows: Vec<(f64, f64)> = timeline + .text_segments + .iter() + .filter(|t| { + t.enabled && t.layout == cap_project::TextLayout::Fullscreen && t.end > t.start + }) + .map(|t| (t.start, t.end)) + .collect(); + windows.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut holds: Vec<(f64, f64)> = Vec::new(); + for w in windows { + match holds.last_mut() { + Some(last) if w.0 <= last.1 => last.1 = last.1.max(w.1), + _ => holds.push(w), + } + } + Self { segments, holds } + } + + /// Output duration without holds. + pub fn base_duration(&self) -> f64 { + self.segments + .iter() + .map(|s| s.duration - s.transition_in) + .sum() + } + + pub fn duration(&self) -> f64 { + self.base_duration() + self.holds.iter().map(|h| h.1 - h.0).sum::() + } + + fn transition_out(&self, k: usize) -> f64 { + self.segments.get(k + 1).map_or(0.0, |n| n.transition_in) + } + + /// Hold-free mapping at gapless time `e`. + pub fn base_map(&self, e: f64) -> Option { + for (k, s) in self.segments.iter().enumerate() { + let u = e - s.out_start; + if s.transition_in > 0.0 && u >= 0.0 && u < s.transition_in { + let prev = &self.segments[k - 1]; + return Some(Mapping::Transition { + outgoing: Source { + segment: k - 1, + time: prev.start + (prev.duration - s.transition_in + u) * prev.timescale, + }, + incoming: Source { + segment: k, + time: s.start + u * s.timescale, + }, + progress: u / s.transition_in, + output_end: s.out_start + s.transition_in, + }); + } + let single_end = s.out_start + s.duration - self.transition_out(k); + if u >= s.transition_in && e < single_end { + return Some(Mapping::Single { + source: Source { + segment: k, + time: s.start + u * s.timescale, + }, + output_end: single_end, + }); + } + } + None + } + + /// Held output time before output time `o`. + pub fn held_before(&self, o: f64) -> f64 { + self.holds + .iter() + .map(|&(a, b)| (o.min(b) - a).max(0.0)) + .sum() + } + + /// Gapless -> output time, landing after every hold that starts at or + /// before the gapless point. + pub fn to_output(&self, e: f64) -> f64 { + let mut held = 0.0; + for &(a, b) in &self.holds { + if e >= a - held { + held += b - a; + } + } + e + held + } + + /// Gapless -> output time for the end of a range: a range ending exactly + /// where a hold starts ends before the hold. + pub fn to_output_end(&self, e: f64) -> f64 { + let mut held = 0.0; + for &(a, b) in &self.holds { + if e > a - held { + held += b - a; + } + } + e + held + } + + /// Output -> recording position on the concatenated source (clip `c` + /// starts at the sum of the earlier clips' durations), preferring the + /// last timeline segment that contains the gapless time, ends included. + pub fn output_to_source(&self, o: f64, clip_starts: &[f64]) -> Option { + let e = o - self.held_before(o); + self.segments.iter().rev().find_map(|s| { + let edited_end = s.out_start + s.duration; + (e >= s.out_start && e <= edited_end) + .then(|| clip_starts[s.clip as usize] + s.start + (e - s.out_start) * s.timescale) + }) + } + + pub fn map(&self, o: f64) -> Option { + if let Some(&(a, b)) = self.holds.iter().find(|&&(a, b)| o >= a && o < b) { + let source = self.base_map(a - self.held_before(a))?.shown(); + return Some(Mapping::Hold { + source, + output_end: b, + }); + } + let next_hold = self.holds.iter().map(|h| h.0).find(|&a| a > o); + let end = |e: f64| { + let out = self.to_output(e); + next_hold.map_or(out, |a| out.min(a)) + }; + Some(match self.base_map(o - self.held_before(o))? { + Mapping::Single { source, output_end } => Mapping::Single { + source, + output_end: end(output_end), + }, + Mapping::Transition { + outgoing, + incoming, + progress, + output_end, + } => Mapping::Transition { + outgoing, + incoming, + progress, + output_end: end(output_end), + }, + hold @ Mapping::Hold { .. } => hold, + }) + } + + /// Whether output time `o` is within `eps` of an edge where the mapping + /// changes (either side is then acceptable). + pub fn near_edge(&self, o: f64, eps: f64) -> bool { + if self + .holds + .iter() + .any(|&(a, b)| (o - a).abs() < eps || (o - b).abs() < eps) + { + return true; + } + let e = o - self.held_before(o); + self.segments.iter().enumerate().any(|(k, s)| { + [ + s.out_start, + s.out_start + s.transition_in, + s.out_start + s.duration - self.transition_out(k), + s.out_start + s.duration, + ] + .iter() + .any(|edge| (e - edge).abs() < eps) + }) + } +} + +fn product_source(source: &cap_project::TimelineSource<'_>) -> Source { + Source { + segment: source.segment_index, + time: source.source_time, + } +} + +/// The engine's mapping in model terms. +pub fn product_map(timeline: &TimelineConfiguration, o: f64) -> Option { + Some(match timeline.get_frame_mapping(o)? { + TimelineFrameMapping::Single { source, output_end } => Mapping::Single { + source: product_source(&source), + output_end, + }, + TimelineFrameMapping::Transition { + outgoing, + incoming, + progress, + output_end, + .. + } => Mapping::Transition { + outgoing: product_source(&outgoing), + incoming: product_source(&incoming), + progress, + output_end, + }, + TimelineFrameMapping::Hold { source, output_end } => Mapping::Hold { + source: product_source(&source), + output_end, + }, + }) +} + +fn close(a: f64, b: f64, tolerance: f64) -> bool { + (a - b).abs() <= tolerance +} + +fn same_source(a: Source, b: Source, tolerance: f64) -> bool { + a.segment == b.segment && close(a.time, b.time, tolerance) +} + +pub fn same_mapping(a: &Mapping, b: &Mapping, tolerance: f64) -> bool { + match (a, b) { + ( + Mapping::Single { + source: s1, + output_end: e1, + }, + Mapping::Single { + source: s2, + output_end: e2, + }, + ) + | ( + Mapping::Hold { + source: s1, + output_end: e1, + }, + Mapping::Hold { + source: s2, + output_end: e2, + }, + ) => same_source(*s1, *s2, tolerance) && close(*e1, *e2, tolerance), + ( + Mapping::Transition { + outgoing: o1, + incoming: i1, + progress: p1, + output_end: e1, + }, + Mapping::Transition { + outgoing: o2, + incoming: i2, + progress: p2, + output_end: e2, + }, + ) => { + same_source(*o1, *o2, tolerance) + && same_source(*i1, *i2, tolerance) + && close(*p1, *p2, tolerance) + && close(*e1, *e2, tolerance) + } + _ => false, + } +} diff --git a/crates/verify/tests/export.rs b/crates/verify/tests/export.rs new file mode 100644 index 00000000000..cd40593b451 --- /dev/null +++ b/crates/verify/tests/export.rs @@ -0,0 +1,1506 @@ +//! Export oracle: every exported frame must show exactly the source frames +//! the reference model computes, for both streams, on every decoder. + +mod common; + +use cap_project::{ProjectConfiguration, SceneMode, SceneSegment, XY}; +use cap_verify::{ + catalog, + fixture::FixtureSpec, + report::{self, Case, Tier}, +}; +use common::{Decoder, ExportRun, Format}; + +struct Variant { + edit: &'static str, + apply: fn(&mut ProjectConfiguration), +} + +fn no_edit(_: &mut ProjectConfiguration) {} + +fn case_id(fixture: &str, edit: &str, format: Format, fps: u32, decoder: Decoder) -> String { + let format = match format { + Format::Mp4 => "mp4", + Format::Gif => "gif", + Format::Mov => "mov", + }; + format!("export/{fixture}/{edit}/{format}{fps}/{}", decoder.label()) +} + +/// Exports `spec` with each edit, format, rate and decoder and checks every +/// frame. With `determinism`, the first combination is exported twice and +/// the decoded pictures must be identical. +fn run( + spec: FixtureSpec, + variants: &[Variant], + formats: &[(Format, u32)], + decoders: &[Decoder], + resolution: (u32, u32), + determinism: bool, +) -> Vec { + run_project( + common::project(&spec), + variants, + formats, + decoders, + resolution, + determinism, + ) +} + +fn run_project( + project: common::Project, + variants: &[Variant], + formats: &[(Format, u32)], + decoders: &[Decoder], + resolution: (u32, u32), + determinism: bool, +) -> Vec { + let rt = common::runtime(); + let mut failures = Vec::new(); + for variant in variants { + for &(format, fps) in formats { + for &decoder in decoders { + let id = case_id(&project.name, variant.edit, format, fps, decoder); + if !report::selected(&id) { + continue; + } + let mut config = project.config(); + (variant.apply)(&mut config); + let model = project.model(&config); + let Some(mut case) = common::begin(&id, &project, decoder) else { + continue; + }; + let dir = common::scratch_dir(&id); + let ext = match format { + Format::Gif => "gif", + Format::Mov => "mov", + Format::Mp4 => "mp4", + }; + let run = ExportRun { + fps, + resolution, + format, + decoder, + }; + rt.block_on(common::probe_decoders(&mut case, &project, decoder)); + let first = dir.path().join(format!("a.{ext}")); + match rt.block_on(common::export(&project, config.clone(), &run, &first)) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(path) => { + let frames = + common::read_output(&path, Some(&|i| model.expect(i as u32, fps))); + common::check_sequence( + &mut case, + &model, + &frames, + fps, + format != Format::Mov, + ); + common::check_presentation( + &mut case, + &config, + &frames, + &path, + fps, + format == Format::Mp4, + ); + if determinism { + let second = dir.path().join(format!("b.{ext}")); + match rt.block_on(common::export(&project, config, &run, &second)) { + Err(e) => { + case.problem("error", format!("second export failed: {e}")) + } + Ok(path) => { + let again = common::read_output(&path, None); + let differing: Vec = frames + .iter() + .zip(&again) + .enumerate() + .filter(|(_, (a, b))| a.hash != b.hash) + .map(|(i, _)| i) + .collect(); + if !differing.is_empty() || frames.len() != again.len() { + case.count( + "nondeterministic", + "frames", + differing.len() as f64, + ); + case.measure( + "nondeterministic", + "length-difference", + again.len() as f64 - frames.len() as f64, + ); + for &i in &differing { + case.measure("nondeterministic", "frame", i as f64); + } + case.problem( + "nondeterministic", + format!( + "nondeterministic: {} of {} frames differ between two exports (first {:?})", + differing.len(), + frames.len(), + &differing[..differing.len().min(5)] + )); + } + } + } + } + } + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + } + } + failures +} + +const RES: (u32, u32) = (480, 270); + +fn cuts_speeds_reorder(config: &mut ProjectConfiguration) { + common::set_cuts( + config, + &[ + (0.5, 2.0, 1.0), + (2.5, 4.0, 2.0), + (1.0, 2.0, 0.5), + (4.0, 5.8, 1.5), + ], + ); +} + +fn transition(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(0.0, 2.5, 1.0), (3.5, 6.0, 1.0)]); + common::add_transition(config, 1, 0.4); +} + +fn camera_hidden(config: &mut ProjectConfiguration) { + config.camera.hide = true; +} + +fn hide_camera_scene(config: &mut ProjectConfiguration) { + config + .timeline + .as_mut() + .expect("timeline") + .scene_segments + .push(SceneSegment { + start: 2.0, + end: 4.0, + mode: SceneMode::HideCamera, + split_layout: None, + transition_in: 0.3, + transition_out: 0.3, + }); +} + +/// A camera offset the user set on the clip (not auto-calculated): the camera +/// shows the frame `0.3 s` later than the screen's. +fn camera_user_offset(config: &mut ProjectConfiguration) { + match config.clips.iter_mut().find(|c| c.index == 0) { + Some(clip) => { + clip.offsets.camera = 0.3; + clip.offsets_auto_calculated = false; + } + None => config.clips.push(cap_project::ClipConfiguration { + index: 0, + offsets: cap_project::ClipOffsets { + camera: 0.3, + ..Default::default() + }, + offsets_auto_calculated: false, + }), + } +} + +#[test] +fn camera_offset() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::basic(report::tier()), + &[Variant { + edit: "camera-user-offset", + apply: camera_user_offset, + }], + &[(Format::Mp4, 30)], + &Decoder::all(), + RES, + false, + )); + common::assert_passed(failures); +} + +#[test] +fn basic() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::basic(report::tier()), + &[ + Variant { + edit: "full", + apply: no_edit, + }, + Variant { + edit: "cuts-speeds-reorder", + apply: cuts_speeds_reorder, + }, + Variant { + edit: "transition", + apply: transition, + }, + Variant { + edit: "camera-hidden", + apply: camera_hidden, + }, + Variant { + edit: "hide-camera-scene", + apply: hide_camera_scene, + }, + ], + &[(Format::Mp4, 30), (Format::Mp4, 60)], + &Decoder::all(), + RES, + false, + )); + common::assert_passed(failures); +} + +#[test] +fn gif() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::basic(report::tier()), + &[Variant { + edit: "cuts-speeds-reorder", + apply: cuts_speeds_reorder, + }], + &[(Format::Gif, 15)], + &[Decoder::Default], + (320, 180), + false, + )); + common::assert_passed(failures); +} + +#[test] +fn vfr_holds() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::vfr_holds(report::tier()), + &[Variant { + edit: "full", + apply: no_edit, + }], + &[(Format::Mp4, 30), (Format::Mp4, 60)], + &Decoder::all(), + RES, + false, + )); + common::assert_passed(failures); +} + +/// Long recordings with a jittery ~12 fps camera at 30 fps nominal exported +/// at 60 fps: the camera must never run ahead of the screen. +#[test] +fn jitter_camera() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::jitter_camera(report::tier()), + &[Variant { + edit: "full", + apply: no_edit, + }], + &[(Format::Mp4, 60)], + &Decoder::all(), + RES, + report::tier() == Tier::Full, + )); + common::assert_passed(failures); +} + +fn backward_jumps(config: &mut ProjectConfiguration) { + common::set_cuts( + config, + &[(5.0, 35.0, 1.0), (44.0, 50.0, 1.0), (28.0, 45.0, 1.0)], + ); +} + +/// Play past a long hold, elsewhere, then jump back into the hold. +#[test] +fn backward_hold() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::backward_hold(report::tier()), + &[Variant { + edit: "backward-jumps", + apply: backward_jumps, + }], + &[(Format::Mp4, 30), (Format::Mp4, 60)], + &Decoder::all(), + RES, + false, + )); + common::assert_passed(failures); +} + +#[test] +fn start_offsets() { + let mut failures: Vec = Vec::new(); + for spec in [ + catalog::late_camera(report::tier()), + catalog::early_camera(report::tier()), + catalog::camera_pts_late(report::tier()), + ] { + failures.extend(run( + spec, + &[Variant { + edit: "full", + apply: no_edit, + }], + &[(Format::Mp4, 30)], + &Decoder::all(), + RES, + false, + )); + } + common::assert_passed(failures); +} + +#[test] +fn multi_segment() { + let mut failures: Vec = Vec::new(); + failures.extend(run( + catalog::multi_segment(report::tier()), + &[Variant { + edit: "full", + apply: no_edit, + }], + &[(Format::Mp4, 30)], + &Decoder::all(), + RES, + true, + )); + common::assert_passed(failures); +} + +fn across_segments(config: &mut ProjectConfiguration) { + common::set_clip_cuts( + config, + &[(0, 0.5, 3.0, 1.0), (1, 0.5, 2.5, 1.0), (0, 3.5, 5.5, 1.5)], + ); +} + +#[test] +#[ignore] +fn prepare_recorded_fixtures() { + common::recorded(&catalog::recorded_studio(report::tier())); + for spec in catalog::instant_variants(report::tier()) { + cap_verify::fixture::ensure_fixture_keyed(spec.cache_key(), |dir| { + common::runtime().block_on(cap_verify::pipeline::record_instant(&spec, dir)) + }) + .expect("instant fixture"); + } +} + +/// A Studio project recorded through the real recording pipeline. +#[test] +fn recorded_studio() { + let failures = run_project( + common::recorded(&catalog::recorded_studio(report::tier())), + &[ + Variant { + edit: "full", + apply: no_edit, + }, + Variant { + edit: "across-segments", + apply: across_segments, + }, + ], + &[(Format::Mp4, 30), (Format::Mp4, 60)], + &Decoder::all(), + RES, + false, + ); + common::assert_passed(failures); +} + +/// The recorded Studio fixture against what its sources emitted (the +/// generator's truth): every emitted screen and camera frame must be in the +/// files. The export cases above take their truth from those files, so a +/// recording that lost its start or its final frames would otherwise only +/// shorten their model. +#[test] +fn recorded_studio_coverage() { + let id = "export/rec-studio/recording/emitted-frames"; + if !report::selected(id) { + return; + } + let spec = catalog::recorded_studio(report::tier()); + let project = common::recorded(&spec); + let mut case = Case::new(id); + for (i, segment) in spec.segments.iter().enumerate() { + let truth = project.truth.segments.get(i); + for (name, video, track) in [ + ("screen", Some(&segment.screen), truth.map(|t| &t.screen)), + ( + "camera", + segment.camera.as_ref(), + truth.and_then(|t| t.camera.as_ref()), + ), + ] { + let Some(video) = video else { + continue; + }; + let emitted = video + .delivery + .slots(video.fps, segment.duration - video.start_delay); + let recorded: std::collections::BTreeSet = track + .map(|t| t.frames.iter().map(|f| f.index).collect()) + .unwrap_or_default(); + let missing: Vec = emitted + .iter() + .filter(|slot| !recorded.contains(slot)) + .copied() + .collect(); + if missing.is_empty() { + case.note(format!( + "segment {i} {name}: all {} emitted frames recorded", + emitted.len() + )); + } else { + case.count( + "frames-missing", + &format!("{name}-frames"), + missing.len() as f64, + ); + case.problem( + "frames-missing", + format!( + "segment {i} {name}: {} of {} emitted frames are not in the recording (first {:?}, last {:?})", + missing.len(), + emitted.len(), + &missing[..missing.len().min(6)], + missing.last() + ), + ); + } + } + } + common::finish(case); +} + +/// Instant mode: the finalised `output.mp4` is the deliverable. Every +/// emitted screen frame must appear once, in order, on its own timing, and +/// every audio marker must be present, within 5 ms of the screen. +#[test] +fn instant_output() { + let mut failures = Vec::new(); + for spec in catalog::instant_variants(report::tier()) { + if let Some(failure) = instant_case(spec) { + failures.push(failure); + } + } + common::assert_passed(failures); +} + +fn instant_case(spec: cap_verify::pipeline::InstantSpec) -> Option { + use cap_verify::{ + audio_code::{self, AudioSource}, + fixture, media, + pipeline::{self, Emitted}, + video_code::{self, StreamKind}, + }; + let id = format!("export/{}/output/mp4/native", spec.name); + if !report::selected(&id) { + return None; + } + common::init(); + let dir = fixture::ensure_fixture_keyed(spec.cache_key(), |dir| { + common::runtime().block_on(pipeline::record_instant(&spec, dir)) + }) + .expect("instant fixture"); + let emitted: Emitted = serde_json::from_str( + &std::fs::read_to_string(dir.join("verify-emitted.json")).expect("emitted"), + ) + .expect("emitted json"); + let output = dir.join("content/output.mp4"); + let mut case = Case::new(&id); + let mut frames = Vec::new(); + media::decode_video(&output, |p| { + frames.push(( + p.pts_secs, + video_code::read_stream(&p.view(), StreamKind::Screen), + )) + }) + .expect("decode instant output"); + let shown: Vec<(f64, u32)> = frames + .iter() + .filter_map(|(pts, r)| r.marker().map(|m| (*pts, m.index))) + .collect(); + if shown.len() != frames.len() { + case.count("unreadable", "frames", (frames.len() - shown.len()) as f64); + case.problem( + "unreadable", + format!( + "{} of {} frames unreadable", + frames.len() - shown.len(), + frames.len() + ), + ); + } + let emitted_slots: Vec = emitted.screen.iter().map(|e| e.0).collect(); + let shown_slots: Vec = shown.iter().map(|s| s.1).collect(); + let missing: Vec = emitted_slots + .iter() + .filter(|s| !shown_slots.contains(s)) + .copied() + .collect(); + let repeated: Vec = shown_slots + .windows(2) + .filter(|w| w[1] <= w[0]) + .map(|w| w[1]) + .collect(); + // Frames missing from the file: content, never excused by the machine + // (`dropped` is the real-time playback class). + if !missing.is_empty() { + case.count("frames-missing", "frames", missing.len() as f64); + case.problem( + "frames-missing", + format!( + "{} emitted frames missing from the output (first {:?})", + missing.len(), + &missing[..missing.len().min(8)] + ), + ); + } + if !repeated.is_empty() { + case.count("stale", "frames", repeated.len() as f64); + case.problem( + "stale", + format!( + "{} frames repeated or out of order (first {:?})", + repeated.len(), + &repeated[..repeated.len().min(8)] + ), + ); + } + // Frames within the start gate's margin have no recorded emit time and + // are not timed. + let video_offsets: Vec = shown + .iter() + .filter_map(|&(pts, slot)| { + emitted + .screen + .iter() + .find(|e| e.0 == slot) + .map(|e| pts - e.1) + }) + .collect(); + let spread = |v: &[f64]| { + let lo = v.iter().copied().fold(f64::INFINITY, f64::min); + let hi = v.iter().copied().fold(f64::NEG_INFINITY, f64::max); + hi - lo + }; + let median = |v: &[f64]| { + let mut v = v.to_vec(); + v.sort_by(f64::total_cmp); + v.get(v.len() / 2).copied().unwrap_or(f64::NAN) + }; + let frame = 1.0 / f64::from(emitted.fps); + if video_offsets.is_empty() { + case.count("frames-missing", "frames", emitted_slots.len() as f64); + case.problem( + "frames-missing", + "no emitted screen frame identified in the output", + ); + case.unobserved("timestamps", "no emitted screen frame identified"); + case.unobserved("sync-mic", "no emitted screen frame identified"); + case.unobserved("sync-system", "no emitted screen frame identified"); + } + let video_spread = spread(&video_offsets); + case.note(format!("video timing spread {:.2} ms", video_spread * 1e3)); + let late = emitted + .screen_sent + .iter() + .filter_map(|&(slot, sent)| { + let planned = emitted.screen.iter().find(|e| e.0 == slot)?.1; + Some((slot, sent - planned)) + }) + .max_by(|a, b| a.1.total_cmp(&b.1)); + if let Some((slot, late)) = late { + case.note(format!( + "test emitter at most {:.2} ms late (slot {slot})", + late * 1e3 + )); + } + if !video_offsets.is_empty() && video_spread > frame / 2.0 { + case.measure( + "timestamps", + "spread-ms", + (video_spread * 1e4).round() / 10.0, + ); + case.problem( + "timestamps", + format!( + "video frame times drift by {:.1} ms against emission (allowed {:.1})", + video_spread * 1e3, + frame / 2.0 * 1e3 + ), + ); + } + let audio = media::decode_audio(&output, None).expect("decode instant audio"); + let mono = audio.mono(); + let mut mic_offset_ms = None; + for source in [AudioSource::Mic, AudioSource::System] { + let expected: Vec<(u32, f64)> = emitted + .audio + .iter() + .filter(|e| e.0 == source) + .map(|e| (e.1, e.2)) + .collect(); + if expected.is_empty() { + continue; + } + // Each source is its own class, so a silenced mic cannot hide behind + // a known system-audio loss (or the other way round). + let audio_class = format!("audio-{}", source.name()); + let sync_class = format!("sync-{}", source.name()); + // Markers are matched by their decoded identity anywhere in the + // track, so an arbitrarily large A/V offset is still measured. + let found = audio_code::locate_markers(&mono, audio.sample_rate, source); + let mut offsets = Vec::new(); + let mut absent = Vec::new(); + for &(index, t) in &expected { + match found.iter().find(|f| f.index == Some(index)) { + Some(f) => offsets.push(f.time_secs + audio.start_secs - t), + None => absent.push(index), + } + } + let extra = found.len().saturating_sub(expected.len() - absent.len()); + if !absent.is_empty() || extra > 0 { + let undecoded = found.iter().filter(|f| f.index.is_none()).count(); + case.count(&audio_class, "missing", absent.len() as f64); + case.count(&audio_class, "unexpected", extra as f64); + case.problem( + &audio_class, + format!( + "{}: {} markers missing (first {:?}), {} unexpected ({} chirps without a decodable identity)", + source.name(), + absent.len(), + &absent[..absent.len().min(5)], + extra, + undecoded + ), + ); + let listed: Vec = found + .iter() + .take(16) + .map(|f| { + format!( + "{:.4}s {} corr {:.2} gain {:.3}", + f.time_secs + audio.start_secs, + f.index.map_or("undecoded".to_string(), |i| format!("#{i}")), + f.correlation, + f.gain + ) + }) + .collect(); + case.note(format!( + "{} chirps found: [{}]; expected {:?}", + source.name(), + listed.join(", "), + expected + .iter() + .map(|&(i, t)| format!("#{i} {t:.4}s")) + .collect::>() + )); + } + // Chirps whose payload was damaged can still be identified by the + // schedule's timing pattern, so a large offset stays measurable. + let chirps: Vec = found + .iter() + .filter(|f| f.index.is_none()) + .map(|f| f.time_secs + audio.start_secs) + .collect(); + if let Some(pattern) = audio_code::identify_by_schedule(&expected, &chirps, 0.002) { + let mut added = 0; + for (index, time) in pattern { + if absent.contains(&index) + && let Some(&(_, t)) = expected.iter().find(|e| e.0 == index) + { + offsets.push(time - t); + added += 1; + } + } + case.note(format!( + "{}: {added} markers without a decodable identity identified by their timing pattern", + source.name() + )); + if !absent.is_empty() { + // Markers whose chirp is there but whose payload is damaged + // (the audio itself survived) versus markers that are gone. + case.count( + &audio_class, + "lost", + absent.len().saturating_sub(added) as f64, + ); + } + } else if !absent.is_empty() { + case.count(&audio_class, "lost", absent.len() as f64); + } + if offsets.is_empty() { + case.unobserved( + &sync_class, + format!( + "{}: none of the {} expected markers was identified in the output", + source.name(), + expected.len() + ), + ); + continue; + } + if video_offsets.is_empty() { + continue; + } + let av = median(&offsets) - median(&video_offsets); + case.note(format!( + "{}: {} markers, spread {:.2} ms, A/V offset {:+.2} ms", + source.name(), + offsets.len(), + spread(&offsets) * 1e3, + av * 1e3 + )); + eprintln!( + "[cap-verify] instant {}: spread {:.2} ms, A/V {:+.2} ms", + source.name(), + spread(&offsets) * 1e3, + av * 1e3 + ); + if spread(&offsets) > 0.005 { + case.measure( + &audio_class, + "drift-ms", + (spread(&offsets) * 1e4).round() / 10.0, + ); + case.problem( + &audio_class, + format!( + "{} marker timing drifts by {:.2} ms", + source.name(), + spread(&offsets) * 1e3 + ), + ); + } + if source == AudioSource::Mic { + mic_offset_ms = Some((av * 1e4).round() / 10.0); + } + if av.abs() > 0.005 { + if source == AudioSource::System + && case.id == "export/instant/output/mp4/native" + && let Some(mic_offset_ms) = mic_offset_ms + { + case.measure(&sync_class, "mic-offset-ms", mic_offset_ms); + case.measure( + &sync_class, + "system-minus-mic-ms", + ((av * 1e3 - mic_offset_ms) * 10.0).round() / 10.0, + ); + } + case.measure(&sync_class, "offset-ms", (av * 1e4).round() / 10.0); + case.problem( + &sync_class, + format!("{} is {:+.1} ms off the screen", source.name(), av * 1e3), + ); + } + } + report::settle(case).err() +} + +/// ProRes MOV export. +#[test] +fn mov() { + let failures = run( + catalog::basic(report::tier()), + &[Variant { + edit: "cuts-speeds-reorder", + apply: cuts_speeds_reorder, + }], + &[(Format::Mov, 30)], + &[Decoder::Default], + RES, + false, + ); + common::assert_passed(failures); +} + +fn short_cut(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(1.0, 2.5, 1.0), (4.0, 5.0, 2.0)]); +} + +/// 4K output (sources upscaled): the full-size encode and render path. +#[test] +fn uhd() { + let (formats, decoders): (&[(Format, u32)], Vec) = match report::tier() { + Tier::Fast => (&[(Format::Mp4, 30)], vec![Decoder::Default]), + Tier::Full => (&[(Format::Mp4, 30), (Format::Mp4, 60)], Decoder::all()), + }; + let failures = run( + catalog::basic(report::tier()), + &[Variant { + edit: "uhd-short-cut", + apply: short_cut, + }], + formats, + &decoders, + (3840, 2160), + false, + ); + common::assert_passed(failures); +} + +/// Fixed aspect ratios (Wide, Vertical, Square, Classic, Tall): the frame is +/// the crop's longer side (relative to the aspect) with the other side +/// derived from the aspect, rounded up to even numbers. It must hold the +/// exact aspect-fit size (never smaller, or the picture is squeezed into a +/// frame that is too small) and be at most 2 px larger on each side. +#[test] +fn aspect_frame_size() { + use cap_project::AspectRatio; + use cap_rendering::{ProjectUniforms, RenderOptions}; + let id = "export/frame-size/aspect-ratios"; + if !report::selected(id) { + return; + } + let mut case = Case::new(id); + let mut checked = 0; + let mut wrong = Vec::new(); + for (w, h) in [ + (1000u32, 700u32), + (1512, 982), + (1920, 1080), + (1081, 1919), + (333, 777), + (2559, 1599), + (1470, 956), + (3024, 1964), + ] { + for (aspect, ratio) in [ + (AspectRatio::Wide, 16.0 / 9.0), + (AspectRatio::Vertical, 9.0 / 16.0), + (AspectRatio::Square, 1.0), + (AspectRatio::Classic, 4.0 / 3.0), + (AspectRatio::Tall, 3.0 / 4.0), + ] { + let mut config = cap_verify::fixture::verification_project_config(); + config.aspect_ratio = Some(aspect.clone()); + config.background.padding = 0.0; + config.background.crop = None; + let options = RenderOptions { + camera_size: None, + screen_size: cap_project::XY::new(w, h), + preserve_screen_alpha: false, + }; + let (bw, bh) = ProjectUniforms::get_base_size(&options, &config); + let (fw, fh) = (f64::from(w), f64::from(h)); + let (ew, eh) = if fw / fh > ratio { + (fw, fw / ratio) + } else { + (fh * ratio, fh) + }; + checked += 1; + let fits = |got: u32, exact: f64| { + got % 2 == 0 && f64::from(got) >= exact - 1e-6 && f64::from(got) < exact + 2.0 + }; + if !fits(bw, ew) || !fits(bh, eh) { + wrong.push(format!( + "{w}x{h} at {aspect:?}: frame {bw}x{bh}, exact fit {ew:.2}x{eh:.2}" + )); + } + } + } + if wrong.is_empty() { + case.note(format!("{checked} crop sizes and aspect ratios")); + } else { + case.problem( + "frame-size", + format!( + "{} of {checked} frames are not the smallest even size holding the aspect fit: {}", + wrong.len(), + wrong[..wrong.len().min(4)].join("; ") + ), + ); + } + common::finish(case); +} + +#[derive(Clone, Copy, Debug)] +struct MeasuredColour { + ycbcr: [f64; 3], + space: ffmpeg::util::color::Space, + range: ffmpeg::util::color::Range, +} + +/// Measured minus expected (Y, Cb, Cr) code values for `rgb` encoded the way +/// the stream's colour tag says (BT.709 unless tagged BT.601; limited range +/// unless tagged full), the way players decode it. +fn colour_error(colour: &MeasuredColour, rgb: [u8; 3]) -> [f64; 3] { + use ffmpeg::util::color::{Range, Space}; + let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0); + let (kr, kb) = if matches!(colour.space, Space::BT470BG | Space::SMPTE170M) { + (0.299, 0.114) + } else { + (0.2126, 0.0722) + }; + let y = kr * r + (1.0 - kr - kb) * g + kb * b; + let (cb, cr) = ((b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr))); + let expected = if colour.range == Range::JPEG { + [255.0 * y, 128.0 + 255.0 * cb, 128.0 + 255.0 * cr] + } else { + [16.0 + 219.0 * y, 128.0 + 224.0 * cb, 128.0 + 224.0 * cr] + }; + [0, 1, 2].map(|k| colour.ycbcr[k] - expected[k]) +} + +/// Mean code values and tag of a rectangle (fractions of the picture). +fn rect_colour( + frame: &ffmpeg::frame::Video, + view: &impl cap_verify::video_code::PixelSource, + rect: [f64; 4], +) -> Option { + if !matches!( + frame.format(), + ffmpeg::format::Pixel::YUV420P | ffmpeg::format::Pixel::NV12 + ) { + return None; + } + let (w, h) = (view.width() as f64, view.height() as f64); + let (x0, x1) = ((rect[0] * w) as usize, (rect[2] * w) as usize); + let (y0, y1) = ((rect[1] * h) as usize, (rect[3] * h) as usize); + let mut sum = [0.0; 3]; + let mut n = 0.0; + for y in y0..y1.max(y0 + 1) { + for x in x0..x1.max(x0 + 1) { + let (cb, cr) = view.chroma(x, y); + sum[0] += view.luma(x, y); + sum[1] += cb + 128.0; + sum[2] += cr + 128.0; + n += 1.0; + } + } + Some(MeasuredColour { + ycbcr: sum.map(|v| v / n), + space: frame.color_space(), + range: frame.color_range(), + }) +} + +/// Allowed error of a flat background (code values): luma level, chroma. +const COLOUR_LEVEL_TOLERANCE: f64 = 4.0; +const COLOUR_MATRIX_TOLERANCE: f64 = 2.0; + +/// Colour fidelity of pixels the renderer generates in RGB (the background): +/// a flat background must decode, under the stream's own colour tag (what +/// players use), to the configured colour. Red separates BT.601 from BT.709 +/// (about 13 luma and 9 chroma code values apart); black and white sit at +/// the ends of the limited range (Y 16 and 235, neutral chroma) under every +/// matrix, so they pin the range independently of the matrix question. +#[test] +fn colour() { + use cap_project::BackgroundSource; + use cap_verify::media; + let spec = catalog::basic(report::tier()); + let mut failures = Vec::new(); + for (name, background) in [ + ("colour-red", [220u8, 40, 40]), + ("colour-black", [0, 0, 0]), + ("colour-white", [255, 255, 255]), + ] { + let id = case_id(&spec.name, name, Format::Mp4, 30, Decoder::Default); + if !report::selected(&id) { + continue; + } + let project = common::project(&spec); + let mut config = project.config(); + config.background.source = BackgroundSource::Color { + value: background.map(u16::from), + alpha: 255, + }; + config.background.padding = 40.0; + let Some(mut case) = common::begin(&id, &project, Decoder::Default) else { + continue; + }; + common::runtime().block_on(common::probe_decoders( + &mut case, + &project, + Decoder::Default, + )); + let dir = common::scratch_dir(&id); + let run = ExportRun { + fps: 30, + resolution: RES, + format: Format::Mp4, + decoder: Decoder::Default, + }; + let out = dir.path().join("a.mp4"); + match common::runtime().block_on(common::export(&project, config, &run, &out)) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(path) => { + let mut errors = Vec::new(); + let mut index = 0usize; + let decoded = media::decode_video(&path, |picture| { + let i = index; + index += 1; + if i % 10 != 0 { + return; + } + let view = picture.view(); + if let Some(colour) = + rect_colour(&picture.frame, &view, [0.01, 0.01, 0.06, 0.06]) + { + errors.push((i, colour_error(&colour, background), colour)); + } + }); + if let Err(e) = decoded { + case.problem("error", format!("decode: {e:?}")); + } + if errors.is_empty() { + case.problem("error", "no YUV frames to measure"); + } + for (_, e, _) in errors + .iter() + .filter(|(_, e, _)| e[0].abs() > COLOUR_LEVEL_TOLERANCE) + { + case.measure("colour-level", "luma", e[0]); + } + for (_, e, _) in errors + .iter() + .filter(|(_, e, _)| e[1].abs().max(e[2].abs()) > COLOUR_MATRIX_TOLERANCE) + { + case.measure("colour-matrix", "cb", e[1]); + case.measure("colour-matrix", "cr", e[2]); + } + if let Some((i, [dy, _, _], colour)) = errors + .iter() + .find(|(_, e, _)| e[0].abs() > COLOUR_LEVEL_TOLERANCE) + { + case.problem( + "colour-level", + format!( + "background luma off by {dy:+.1} code values (Y {:.1}) for the stream's {:?} {:?} tag (frame {i})", + colour.ycbcr[0], colour.space, colour.range + ), + ); + } + if let Some((i, [_, dcb, dcr], colour)) = errors + .iter() + .find(|(_, e, _)| e[1].abs().max(e[2].abs()) > COLOUR_MATRIX_TOLERANCE) + { + case.problem( + "colour-matrix", + format!( + "background chroma off by Cb {dcb:+.1} Cr {dcr:+.1} for the stream's {:?} tag (frame {i})", + colour.space + ), + ); + } + if let Some((_, e, colour)) = errors.first() { + case.note(format!( + "frame 0: Y {:.1} Cb {:.1} Cr {:.1} ({:?} {:?}), error {:+.1} {:+.1} {:+.1}", + colour.ycbcr[0], + colour.ycbcr[1], + colour.ycbcr[2], + colour.space, + colour.range, + e[0], + e[1], + e[2] + )); + } + } + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + common::assert_passed(failures); +} + +/// Background crop: the export must show the cropped part of the screen. +/// Frame identity alone cannot see a crop (the code is still readable when +/// the crop is ignored), so the screen code's square is located in every +/// tenth frame (the low-chroma block inside the screen's key region) and +/// must sit where the crop puts it. +#[test] +fn crop() { + use cap_project::Crop; + use cap_verify::{ + media, + video_code::{self, PixelSource, StreamKind}, + }; + let spec = catalog::basic(report::tier()); + let id = case_id(&spec.name, "crop", Format::Mp4, 30, Decoder::Default); + if !report::selected(&id) { + return; + } + let project = common::project(&spec); + let (src_w, src_h) = (f64::from(catalog::SCREEN_W), f64::from(catalog::SCREEN_H)); + let crop = Crop { + position: XY::new(20, 10), + size: XY::new(256, 144), + }; + let mut config = project.config(); + config.camera.hide = true; + config.background.crop = Some(crop.clone()); + let model = project.model(&config); + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + let run = ExportRun { + fps: 30, + resolution: RES, + format: Format::Mp4, + decoder: Decoder::Default, + }; + let out = dir.path().join("a.mp4"); + match common::runtime().block_on(common::export(&project, config.clone(), &run, &out)) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(path) => { + let mut frames = Vec::new(); + let (cx0, cy0) = (f64::from(crop.position.x), f64::from(crop.position.y)); + let (cw, ch) = (f64::from(crop.size.x), f64::from(crop.size.y)); + let side = StreamKind::Screen.code_side_fraction() * src_h; + let mut misplaced = Vec::new(); + let mut index = 0usize; + let decoded = media::decode_video(&path, |picture| { + let i = index; + index += 1; + let view = picture.view(); + let region = video_code::locate_region(&view, StreamKind::Screen); + let expected = region.map(|region| { + let (rw, rh) = (region.x1 - region.x0, region.y1 - region.y0); + ( + region.x0 + (src_w / 2.0 - cx0) / cw * rw, + region.y0 + (src_h / 2.0 - cy0) / ch * rh, + side / ch * rh, + ) + }); + // The code is off the screen region's centre, so it is read + // where the crop puts it. + let screen = match expected { + Some((x, y, s)) => { + video_code::read_code_near(&view, StreamKind::Screen, x, y, s) + } + None => video_code::Readout::Absent, + }; + frames.push(common::OutputFrame { + pts_secs: picture.pts_secs, + readout: video_code::FrameReadout { + screen, + camera: video_code::read_stream(&view, StreamKind::Camera), + }, + hash: 0, + size: (view.width(), view.height()), + camera_region: None, + screen_colour: None, + }); + if i % common::LAYOUT_SAMPLE_EVERY != 0 { + return; + } + let (Some(region), Some((expect_cx, expect_cy, expect_side))) = (region, expected) + else { + misplaced.push(format!("frame {i}: no screen region")); + return; + }; + let rh = region.y1 - region.y0; + let (mut x0, mut y0, mut x1, mut y1) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN); + for y in (region.y0.ceil() as usize)..(region.y1 as usize).min(view.height()) { + for x in (region.x0.ceil() as usize)..(region.x1 as usize).min(view.width()) { + let (cb, cr) = view.chroma(x, y); + if cb.hypot(cr) < 16.0 { + x0 = x0.min(x as f64); + y0 = y0.min(y as f64); + x1 = x1.max(x as f64 + 1.0); + y1 = y1.max(y as f64 + 1.0); + } + } + } + let tolerance = 0.03 * rh; + let (cx, cy, s) = ((x0 + x1) / 2.0, (y0 + y1) / 2.0, y1 - y0); + if x1 < x0 + || (cx - expect_cx).abs() > tolerance + || (cy - expect_cy).abs() > tolerance + || (s - expect_side).abs() > tolerance + { + misplaced.push(format!( + "frame {i}: code square centre ({cx:.1}, {cy:.1}) side {s:.1}, crop puts it at ({expect_cx:.1}, {expect_cy:.1}) side {expect_side:.1}" + )); + } + }); + if let Err(e) = decoded { + case.problem("error", format!("decode: {e:?}")); + } + common::check_sequence(&mut case, &model, &frames, 30, true); + if !misplaced.is_empty() { + case.problem( + "layout", + format!( + "cropped screen misplaced in {} sampled frames: {}", + misplaced.len(), + misplaced[..misplaced.len().min(3)].join("; ") + ), + ); + } + } + } + common::finish(case); +} + +/// Audio of a Studio project recorded through the real recording pipeline: +/// every marker the synthetic mic and loopback system audio emitted (at a +/// known time on the same clock as the screen frames) must be heard once in +/// the export, in sync with the screen frame emitted at the same time. This +/// checks the recording pipeline's audio timestamps (anchoring, gap filling, +/// start times in recording-meta) end to end; the audio oracle's own fixtures +/// are written with exact timing and never pass through the recorder. +#[test] +fn recorded_studio_audio() { + use cap_verify::{ + audio_code::{self, AudioSource, MARKER_SECS}, + media, + video_code::StreamKind, + }; + let spec = catalog::recorded_studio(report::tier()); + let id = case_id( + &format!("rec-{}", spec.name), + "full-audio", + Format::Mp4, + 30, + Decoder::Default, + ); + if !report::selected(&id) { + return; + } + let project = common::recorded(&spec); + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + let run = ExportRun { + fps: 30, + resolution: RES, + format: Format::Mp4, + decoder: Decoder::Default, + }; + let out = dir.path().join("a.mp4"); + match common::runtime().block_on(common::export(&project, project.config(), &run, &out)) { + Err(e) => case.problem("error", format!("export failed: {e}")), + Ok(path) => { + let frames = common::read_output(&path, None); + match media::decode_audio(&path, Some(48_000)) { + Err(e) => case.problem("error", format!("decode audio: {e:?}")), + Ok(decoded) => { + let mut mono = decoded.mono(); + let lead = (decoded.start_secs * 48_000.0).round() as i64; + if lead > 0 { + let mut padded = vec![0.0; lead as usize]; + padded.extend_from_slice(&mono); + mono = padded; + } else if lead < 0 { + mono.drain(..(-lead as usize).min(mono.len())); + } + let frame = 1.0 / 30.0; + let mut missing = Vec::new(); + let mut doubled = Vec::new(); + let mut off = Vec::new(); + let mut checked = 0; + for (source, emitted_from) in [ + ( + AudioSource::Mic, + spec.segments + .iter() + .map(|s| s.mic_delay) + .collect::>(), + ), + ( + AudioSource::System, + spec.segments + .iter() + .map(|s| s.system_audio.then_some(0.0)) + .collect(), + ), + ] { + let heard = audio_code::locate_markers(&mono, 48_000, source); + if std::env::var_os("CAP_VERIFY_DUMP_AUDIO").is_some() { + for h in &heard { + eprintln!( + " heard {}: {:?} at {:.4}s gain {:.3}", + source.name(), + h.index, + h.time_secs, + h.gain + ); + } + let file = project.path.join("content/segments/segment-0").join( + if source == AudioSource::Mic { + "audio-input.ogg" + } else { + "system_audio.ogg" + }, + ); + if let Ok(src) = media::decode_audio(&file, Some(48_000)) { + for h in audio_code::locate_markers(&src.mono(), 48_000, source) { + eprintln!( + " file {}: {:?} at {:.4}s (+{:.4}) gain {:.3}", + source.name(), + h.index, + h.time_secs, + src.start_secs, + h.gain + ); + } + } + } + for (clip, segment) in spec.segments.iter().enumerate() { + let Some(from) = emitted_from[clip] else { + continue; + }; + let schedule = audio_code::standard_schedule( + source, + 0.1, + segment.duration - 0.1, + 0.5, + ); + for placement in schedule { + let t = placement.time_secs; + if t < from || t + MARKER_SECS > segment.duration - 0.05 { + continue; + } + let slot = (t * 30.0).floor() as u32; + let Some(shown) = frames.iter().position(|f| { + f.readout.get(StreamKind::Screen).marker().is_some_and(|m| { + usize::from(m.segment) == clip && m.index == slot + }) + }) else { + continue; + }; + let shown_at = shown as f64 * frame; + let hits: Vec<_> = heard + .iter() + .filter(|h| { + h.index == Some(placement.index) + && (h.time_secs - shown_at).abs() < 1.0 + }) + .collect(); + let what = format!( + "{} #{} (clip {clip}, {t:.3} s)", + source.name(), + placement.index + ); + match hits.as_slice() { + [] => missing.push(what), + [h] => { + checked += 1; + let av = h.time_secs - shown_at; + if av.abs() > frame + 0.005 { + off.push(format!("{what} {:+.1} ms", av * 1000.0)); + } + } + _ => doubled.push(what), + } + } + } + } + if !missing.is_empty() || !doubled.is_empty() { + case.problem( + "audio", + format!( + "recorded markers: {} missing (first {:?}), {} heard twice (first {:?})", + missing.len(), + missing.first(), + doubled.len(), + doubled.first() + ), + ); + } + if !off.is_empty() { + case.problem( + "sync", + format!( + "{} of {checked} recorded markers out of sync with the screen frame emitted with them (limit one frame + 5 ms): {}", + off.len(), + off[..off.len().min(3)].join("; ") + ), + ); + } else { + case.note(format!("{checked} recorded markers in sync")); + } + } + } + } + } + common::finish(case); +} + +/// Epoch-anchored audio (the product's loopback system audio): the track's +/// time 0 is the pipeline epoch, so silence must stand in for the time +/// between the epoch and the first captured frame, or every sample plays +/// early by that much. A head under the recorder's 70 ms gap threshold is +/// covered only by that head silence; a longer one is also caught by gap +/// filling. Recorded fresh every run (no fixture cache). +#[test] +fn recorded_epoch_audio_head() { + use cap_verify::media; + const TOLERANCE_SECS: f64 = 0.005; + let rt = common::runtime(); + let mut failures = Vec::new(); + for head in [0.04, 0.3] { + let id = format!("export/recorded-epoch-audio/head-{:.0}ms", head * 1000.0); + if !report::selected(&id) { + continue; + } + let mut case = Case::new(&id); + let dir = common::scratch_dir(&id); + match rt.block_on(cap_verify::pipeline::record_epoch_audio( + dir.path(), + head, + 0.6, + )) { + Err(e) => case.problem("error", format!("record: {e}")), + Ok((path, track_start)) => match media::decode_audio(&path, Some(48_000)) { + Err(e) => case.problem("error", format!("decode: {e:?}")), + Ok(decoded) => { + let mono = decoded.mono(); + match mono.iter().position(|v| v.abs() > 0.05) { + None => case.problem("audio-missing", "the recorded tone is silent"), + Some(index) => { + let heard = track_start + decoded.start_secs + index as f64 / 48_000.0; + let error = heard - head; + if error.abs() > TOLERANCE_SECS { + case.problem( + "sync", + format!( + "audio captured {:.1} ms after the epoch is heard at {:.1} ms on the track ({:+.1} ms; track start {:.1} ms, file start {:.1} ms)", + head * 1000.0, + heard * 1000.0, + error * 1000.0, + track_start * 1000.0, + decoded.start_secs * 1000.0 + ), + ); + } + case.note(format!( + "tone heard at {:.2} ms (captured at {:.1} ms)", + heard * 1000.0, + head * 1000.0 + )); + } + } + } + }, + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + common::assert_passed(failures); +} diff --git a/crates/verify/tests/playback.rs b/crates/verify/tests/playback.rs new file mode 100644 index 00000000000..bcf429dd457 --- /dev/null +++ b/crates/verify/tests/playback.rs @@ -0,0 +1,362 @@ +//! Playback oracle: the editor's decode + render path, driven offline frame +//! by frame, must show exactly the frames the model (and therefore the +//! export) shows: from the start, from the middle, through holds, across +//! cuts and backward timeline jumps. + +mod common; + +use cap_project::ProjectConfiguration; +use cap_verify::{ + catalog, + fixture::{FixtureSpec, Rng}, + report::{self, Tier}, +}; +use common::Decoder; + +const RES: (u32, u32) = (480, 270); + +fn no_edit(_: &mut ProjectConfiguration) {} + +fn cuts(config: &mut ProjectConfiguration) { + common::set_cuts( + config, + &[ + (0.5, 2.0, 1.0), + (2.5, 4.0, 2.0), + (1.0, 2.0, 0.5), + (4.0, 5.8, 1.5), + ], + ); +} + +fn backward_jumps(config: &mut ProjectConfiguration) { + common::set_cuts( + config, + &[(5.0, 35.0, 1.0), (44.0, 50.0, 1.0), (28.0, 45.0, 1.0)], + ); +} + +/// Plays `[start, end)` output frames sequentially (the first as a seek). +fn sequential( + spec: FixtureSpec, + edit: (&str, fn(&mut ProjectConfiguration)), + fps: u32, + start_fraction: f64, +) -> Vec { + sequential_project(common::project(&spec), edit, fps, start_fraction) +} + +fn sequential_project( + project: common::Project, + edit: (&str, fn(&mut ProjectConfiguration)), + fps: u32, + start_fraction: f64, +) -> Vec { + let rt = common::runtime(); + let mut failures = Vec::new(); + for decoder in Decoder::all() { + let id = format!( + "playback/{}/{}/seq{fps}-from{:.0}/{}", + project.name, + edit.0, + start_fraction * 100.0, + decoder.label() + ); + if !report::selected(&id) { + continue; + } + let mut config = project.config(); + (edit.1)(&mut config); + let model = project.model(&config); + let total = model.output_frames(fps); + let first = (f64::from(total) * start_fraction) as u32; + let Some(mut case) = common::begin(&id, &project, decoder) else { + continue; + }; + let frames: Vec = (first..total).collect(); + let result = rt.block_on(async { + let mut player = common::Player::open(&project, config, decoder, RES).await?; + let mut readouts = Vec::new(); + for &f in &frames { + readouts.push( + player + .frame(f, fps, f == first, Some(&model.expect(f, fps))) + .await?, + ); + } + player.check_decoders(&mut case, decoder); + Ok::<_, String>(readouts) + }); + match result { + Ok(readouts) => common::check_requests(&mut case, &model, &frames, &readouts, fps), + Err(e) => case.problem("error", e), + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + failures +} + +#[test] +fn basic_sequential() { + let mut failures: Vec = Vec::new(); + failures.extend(sequential( + catalog::basic(report::tier()), + ("full", no_edit), + 30, + 0.0, + )); + failures.extend(sequential( + catalog::basic(report::tier()), + ("cuts-speeds-reorder", cuts), + 60, + 0.0, + )); + common::assert_passed(failures); +} + +#[test] +fn play_from_middle() { + let mut failures: Vec = Vec::new(); + let seed = 0x9A1D_0001; + let mut rng = Rng::new(seed); + eprintln!("seed {seed:#x}"); + for _ in 0..2 { + let start = 0.2 + rng.f64() * 0.6; + failures.extend(sequential( + catalog::basic(report::tier()), + ("full", no_edit), + 30, + start, + )); + } + failures.extend(sequential( + catalog::vfr_holds(report::tier()), + ("full", no_edit), + 60, + 0.3, + )); + common::assert_passed(failures); +} + +#[test] +fn vfr_holds_sequential() { + let mut failures: Vec = Vec::new(); + failures.extend(sequential( + catalog::vfr_holds(report::tier()), + ("full", no_edit), + 30, + 0.0, + )); + common::assert_passed(failures); +} + +#[test] +fn backward_jumps_sequential() { + let mut failures: Vec = Vec::new(); + failures.extend(sequential( + catalog::backward_hold(report::tier()), + ("backward-jumps", backward_jumps), + 30, + 0.0, + )); + common::assert_passed(failures); +} + +#[test] +fn jitter_camera_sequential() { + let mut failures: Vec = Vec::new(); + let fps = if report::tier() == Tier::Full { 60 } else { 30 }; + failures.extend(sequential( + catalog::jitter_camera(report::tier()), + ("full", no_edit), + fps, + 0.0, + )); + common::assert_passed(failures); +} + +#[test] +fn multi_segment_sequential() { + let mut failures: Vec = Vec::new(); + failures.extend(sequential( + catalog::multi_segment(report::tier()), + ("full", no_edit), + 30, + 0.0, + )); + common::assert_passed(failures); +} + +#[test] +fn recorded_studio_sequential() { + let failures = sequential_project( + common::recorded(&catalog::recorded_studio(report::tier())), + ("full", no_edit), + 60, + 0.0, + ); + common::assert_passed(failures); +} + +fn transition(config: &mut ProjectConfiguration) { + common::set_cuts(config, &[(0.0, 2.5, 1.0), (3.5, 6.0, 1.0)]); + common::add_transition(config, 1, 0.4); +} + +fn hide_camera_scene(config: &mut ProjectConfiguration) { + config + .timeline + .as_mut() + .expect("timeline") + .scene_segments + .push(cap_project::SceneSegment { + start: 2.0, + end: 4.0, + mode: cap_project::SceneMode::HideCamera, + split_layout: None, + transition_in: 0.3, + transition_out: 0.3, + }); +} + +fn camera_hidden(config: &mut ProjectConfiguration) { + config.camera.hide = true; +} + +#[test] +fn edits_sequential() { + let mut failures = Vec::new(); + for edit in [ + ("transition", transition as fn(&mut ProjectConfiguration)), + ("hide-camera-scene", hide_camera_scene), + ("camera-hidden", camera_hidden), + ] { + failures.extend(sequential(catalog::basic(report::tier()), edit, 30, 0.0)); + failures.extend(sequential(catalog::basic(report::tier()), edit, 60, 0.0)); + } + common::assert_passed(failures); +} + +/// Longest acceptable real-time playback warm-up (measured at most 0.53 s, +/// WARP-rendered Windows included). +const REALTIME_WARMUP_LIMIT: std::time::Duration = std::time::Duration::from_secs(2); + +/// The real-time playback loop (`cap_editor::Playback`), not just the decode +/// and render path: every frame it delivers must show what the model puts at +/// that frame number. Frames may be dropped under load (macOS CI presents at +/// a few fps); at least `MIN_DELIVERED` must arrive once the clock has +/// started. Under a machine stall the starvation sentinel makes a `dropped` +/// failure inconclusive and the test is retried once. +#[test] +fn realtime_loop() { + const MIN_DELIVERED: usize = 8; + let rt = common::runtime(); + let mut failures = Vec::new(); + let project = common::project(&catalog::basic(report::tier())); + for (edit, apply, start_fraction) in [ + ("full", no_edit as fn(&mut ProjectConfiguration), 0.0), + ("cuts-speeds-reorder", cuts, 0.25), + ("transition", transition, 0.2), + ] { + for decoder in Decoder::all() { + let id = format!( + "playback/{}/{edit}/realtime30-from{:.0}/{}", + project.name, + start_fraction * 100.0, + decoder.label() + ); + if !report::selected(&id) { + continue; + } + let mut config = project.config(); + apply(&mut config); + let model = project.model(&config); + let total = model.output_frames(30); + let first = (f64::from(total) * start_fraction) as u32; + let seconds = f64::from(total - first) / 30.0 + 0.5; + let Some(mut case) = common::begin(&id, &project, decoder) else { + continue; + }; + match rt.block_on(common::realtime_playback( + &mut case, + &project, + config, + decoder, + RES, + 30, + first, + seconds, + &|f| model.expect(f, 30), + )) { + Err(e) => case.problem("error", e), + Ok((delivered, telemetry)) => { + case.note(telemetry.describe()); + let delivered: Vec<_> = + delivered.into_iter().filter(|(f, _)| *f < total).collect(); + // Performance floor, relative to what the runner class + // delivers (measured on runs 38841247123, 38844291490 and + // 38846099909): macOS and Linux present 0.88 to 0.99 of + // the frames; warm-up took at most 0.53 s. Windows CI + // renders on the WARP software rasterizer, whose speed + // (0.13 to 0.96 of the frames on Blacksmith, run + // 38851482675, depending on the fixture) measures WARP, + // not Cap, so only the identity checks and the minimum + // below apply there. A wall-clock class: a proven + // machine stall makes it inconclusive, never excused. + let floor = if cfg!(windows) { 0.0 } else { 0.75 }; + let due = (total - first) as usize; + let share = delivered.len() as f64 / due.max(1) as f64; + if let Some(ready) = telemetry.ready_after + && ready > REALTIME_WARMUP_LIMIT + { + case.measure("dropped", "warm-up-s", ready.as_secs_f64()); + case.problem( + "dropped", + format!( + "playback took {:.2} s to start (limit {:.1} s)", + ready.as_secs_f64(), + REALTIME_WARMUP_LIMIT.as_secs_f64() + ), + ); + } + if telemetry.ready_after.is_some() && share < floor { + case.measure("dropped", "delivered-share", share); + case.problem( + "dropped", + format!( + "real-time playback presented {} of {due} frames ({:.0}%, floor {:.0}% on this runner class)", + delivered.len(), + share * 100.0, + floor * 100.0 + ), + ); + } + if telemetry.ready_after.is_none() { + case.problem( + "dropped", + "the playback clock never started (warm-up did not finish within 20 s)", + ); + } else if delivered.len() < MIN_DELIVERED { + case.problem( + "dropped", + format!( + "real-time playback delivered {} frames after it was ready, expected at least {MIN_DELIVERED} ({})", + delivered.len(), + telemetry.describe() + ), + ); + } + let frames: Vec = delivered.iter().map(|(f, _)| *f).collect(); + let readouts: Vec<_> = delivered.into_iter().map(|(_, r)| r).collect(); + common::check_requests(&mut case, &model, &frames, &readouts, 30); + } + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + } + common::assert_passed(failures); +} diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs new file mode 100644 index 00000000000..33c4a3a163b --- /dev/null +++ b/crates/verify/tests/recording.rs @@ -0,0 +1,3704 @@ +//! Real-capture recording oracle (macOS). +//! +//! A helper process shows the screen marker of a monotonically increasing +//! frame index at a fixed rate and plays system-audio markers; synthetic +//! camera and microphone feeds deliver their own markers with exact capture +//! times. Cap's real Studio and Instant pipelines record it all through +//! ScreenCaptureKit and VideoToolbox, stop and finalize exactly like the app +//! and the CLI, and every output frame and audio marker is then checked +//! against the host-clock truth logged by the sources. +//! +//! Capture mode (`CAP_VERIFY_CAPTURE`): `display` records the primary display +//! with a fullscreen stimulus (CI); `window` records only the stimulus window +//! (default on a developer Mac, so other screen activity cannot leak in); +//! `off` skips. Without Screen Recording permission the cases skip unless +//! `CAP_VERIFY_REQUIRE_CAPTURE=1`. + +#![cfg(target_os = "macos")] + +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::Duration, +}; + +use cap_media_info::{RawVideoFormat, VideoInfo}; +use cap_project::{InstantRecordingMeta, RecordingMeta, RecordingMetaInner, StudioRecordingMeta}; +use cap_recording::{ + NativeCameraFrame, RecordingDefaults, SendableShareableContent, StudioQuality, + feeds::{ + Admission, camera::synthetic::SyntheticCamera, microphone::synthetic::SyntheticMicrophone, + }, + instant_recording, + recovery::RecoveryManager, + screen_capture::ScreenCaptureTarget, + studio_recording, +}; +use cap_timestamp::{MachAbsoluteTimestamp, Timestamp}; +use cap_verify::{ + audio_code::{self, AudioSource, MarkerPlacement}, + fixture_avf, media, + report::{self, Case, Tier}, + sched, + stimulus::{ + SENTINEL_FILE, SentinelLog, StimulusEvent, StimulusProcess, StimulusSpec, host_add_secs, + host_now, host_secs, + }, + video_code::{self, FrameMarker, Readout, StreamKind}, +}; + +static ATTEMPT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +const MIC_RATE: u32 = 48_000; +const MIC_CHUNK: usize = 480; +const CAMERA_FPS: u32 = 30; +const CAMERA_SIZE: (usize, usize) = (640, 360); +const AUDIO_PERIOD: f64 = 0.5; +const WINDOW_TITLE: &str = "Cap Verify Stimulus"; +const SYSTEM_AUDIO_STARTUP: f64 = 1.0; +const START_CUE_MS: u64 = 400; +/// Frames a track may lose while its encoder starts (the capture pool is full +/// until VideoToolbox is ready; #2573 measured 0-6 per run). +const STARTUP_SECS: f64 = 1.0; +const STARTUP_DROPS: usize = 6; +const SCREEN_MISS_FRACTION: f64 = 0.03; +/// Video frames are stamped from their source capture time until the drift +/// anchor, two seconds into the track (`VideoDriftTracker` in +/// crates/recording/src/output_pipeline/core.rs); after it, from the +/// processing wall clock. +const DRIFT_ANCHOR_SECS: f64 = 2.0; +/// Stimulus rate for cases captured at 30 fps (Instant, Studio with a +/// camera). A picture is only guaranteed a sample when it stays on screen +/// for one capture interval plus one refresh (50 ms at 30 fps); 20 fps +/// pictures last exactly that long, so about half of them were excused from +/// the drop checks. At 15 fps every picture lasts 67 ms and must be captured. +const STIMULUS_FPS_30: f64 = 15.0; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + Studio { + fragmented: bool, + camera: bool, + }, + Instant, + /// Fragmented Studio whose on-disk state is snapshotted mid-recording + /// (an APFS clone of the project, exactly what a crash leaves behind: + /// in-progress meta, finished fragments, the fragment being written), + /// then recovered the way the app recovers it. + StudioCrashed, +} + +#[derive(Clone, Debug)] +struct Spec { + id: &'static str, + tier: Tier, + mode: Mode, + seconds: f64, + stimulus_fps: f64, + holds: Vec<(f64, f64)>, + /// Pairs of consecutive camera frames dropped this many seconds after + /// the recording started (the start-up drops #2573 fixed). + camera_drops_at: Vec, + /// The app's start sequence: the recording is built behind a + /// `RecordingStartGate` that the start cue arms a moment later. + gated: bool, + /// The microphone delivers its first sample this long after the + /// recording started (a device that wakes late). + mic_delay: f64, + mic: bool, + system_audio: bool, +} + +fn specs() -> Vec { + vec![ + Spec { + id: "recording/studio-screen/fragmented", + tier: Tier::Fast, + mode: Mode::Studio { + fragmented: true, + camera: false, + }, + seconds: 8.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/studio-camera/startup-drops", + tier: Tier::Fast, + mode: Mode::Studio { + fragmented: true, + camera: true, + }, + seconds: 8.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![], + camera_drops_at: vec![0.3, 1.1], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/instant/screen-mic-system", + tier: Tier::Fast, + mode: Mode::Instant, + seconds: 8.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/studio-screen/gated", + tier: Tier::Fast, + mode: Mode::Studio { + fragmented: true, + camera: false, + }, + seconds: 8.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: true, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/instant/gated", + tier: Tier::Fast, + mode: Mode::Instant, + seconds: 8.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![], + camera_drops_at: vec![], + gated: true, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/instant/late-mic", + tier: Tier::Fast, + mode: Mode::Instant, + seconds: 8.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 1.0, + mic: true, + system_audio: false, + }, + Spec { + id: "recording/instant/system-only", + tier: Tier::Fast, + mode: Mode::Instant, + seconds: 8.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: false, + system_audio: true, + }, + Spec { + id: "recording/studio-screen/late-mic", + tier: Tier::Full, + mode: Mode::Studio { + fragmented: true, + camera: false, + }, + seconds: 8.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.3, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/studio-recovery/crash", + tier: Tier::Fast, + mode: Mode::StudioCrashed, + seconds: 8.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/studio-screen/mp4-holds", + tier: Tier::Full, + mode: Mode::Studio { + fragmented: false, + camera: false, + }, + seconds: 30.0, + stimulus_fps: 20.0, + holds: vec![(5.0, 2.0), (12.0, 0.5), (20.0, 4.0)], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/studio-camera/long", + tier: Tier::Full, + mode: Mode::Studio { + fragmented: true, + camera: true, + }, + seconds: 30.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![(10.0, 3.0)], + camera_drops_at: vec![0.5, 1.5, 12.0], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + Spec { + id: "recording/instant/long-holds", + tier: Tier::Full, + mode: Mode::Instant, + seconds: 30.0, + stimulus_fps: STIMULUS_FPS_30, + holds: vec![(6.0, 2.0), (18.0, 5.0)], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }, + ] +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CaptureMode { + Display, + Window, +} + +fn capture_mode() -> Option { + match std::env::var("CAP_VERIFY_CAPTURE").as_deref() { + Ok("display") => Some(CaptureMode::Display), + Ok("window") => Some(CaptureMode::Window), + Ok("off") => None, + _ if std::env::var_os("CI").is_some() => Some(CaptureMode::Display), + _ => Some(CaptureMode::Window), + } +} + +/// System audio is required on a developer Mac and, with +/// `CAP_VERIFY_REQUIRE_SYSTEM_AUDIO=1`, anywhere. +fn require_system_audio() -> bool { + match std::env::var("CAP_VERIFY_REQUIRE_SYSTEM_AUDIO").as_deref() { + Ok("1") | Ok("true") => true, + Ok(_) => false, + Err(_) => std::env::var_os("CI").is_none(), + } +} + +/// `CAP_VERIFY_STIMULUS_AUDIO=0` keeps the stimulus silent, so the system +/// audio track carries no markers and is not checked (unless system audio +/// is required). On a host whose output device does reach +/// ScreenCaptureKit but whose system-audio timing is not yet characterised +/// (Blacksmith with BlackHole), this keeps the run equivalent to hosted +/// runners, where the output devices give ScreenCaptureKit only silence. +fn stimulus_plays_system_audio() -> bool { + !matches!( + std::env::var("CAP_VERIFY_STIMULUS_AUDIO").as_deref(), + Ok("0") | Ok("false") + ) +} + +fn env_flag(name: &str) -> bool { + matches!(std::env::var(name).as_deref(), Ok("1") | Ok("true")) +} + +/// Hosted runners start jobs at utility QoS (scheduler priority 20) and +/// every thread of the test inherits it, below what an app the user is +/// running gets. The runtime also drives the synthetic camera and +/// microphone, which stand in for capture callbacks the system delivers at +/// high priority; at utility priority a loaded runner delays them and their +/// lateness shows up as recording timing errors. Run the runtime at the +/// priority the product itself asks for its recording threads. +#[cfg(target_os = "macos")] +fn raise_thread_qos() { + const QOS_CLASS_USER_INITIATED: u32 = 0x19; + unsafe extern "C" { + fn pthread_set_qos_class_self_np(qos: u32, relative_priority: i32) -> i32; + } + unsafe { pthread_set_qos_class_self_np(QOS_CLASS_USER_INITIATED, 0) }; +} + +fn runtime() -> tokio::runtime::Runtime { + raise_thread_qos(); + tokio::runtime::Builder::new_multi_thread() + .on_thread_start(raise_thread_qos) + .worker_threads(4) + .thread_stack_size(16 * 1024 * 1024) + .enable_all() + .build() + .expect("runtime") +} + +fn init() { + media::init(); + if std::env::var_os("RUST_LOG").is_some() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .try_init(); + } +} + +#[test] +fn real_capture() { + init(); + let Some(capture) = capture_mode() else { + eprintln!("[cap-verify] recording: CAP_VERIFY_CAPTURE=off, skipping"); + return; + }; + let rt = runtime(); + let shareable = match rt.block_on(cidre::sc::ShareableContent::current()) { + Ok(content) => SendableShareableContent::from(content), + Err(e) => { + let message = format!("ScreenCaptureKit unavailable ({e:?}); grant Screen Recording"); + if env_flag("CAP_VERIFY_REQUIRE_CAPTURE") { + panic!("{message}"); + } + eprintln!("[cap-verify] recording: {message}, skipping"); + return; + } + }; + let tier = report::tier(); + let repeat: usize = std::env::var("CAP_VERIFY_REPEAT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1); + // A cold runner (WindowServer, ScreenCaptureKit, VideoToolbox) delivers + // its first recording late and irregularly; one discarded recording + // brings every case onto the steady state a user's machine is in. + if specs() + .iter() + .any(|s| report::selected(s.id) && (s.tier == Tier::Fast || tier == Tier::Full)) + { + let warmup = Spec { + id: "recording/warmup", + tier: Tier::Fast, + mode: Mode::Studio { + fragmented: true, + camera: false, + }, + seconds: 2.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: true, + }; + let mut case = Case::new(warmup.id); + let started = std::time::Instant::now(); + let result = rt.block_on(async { + let scratch = scratch_root().join("warmup"); + let _ = std::fs::remove_dir_all(&scratch); + std::fs::create_dir_all(&scratch)?; + let recorded = record( + &warmup, + capture, + &shareable, + &scratch, + host_now(), + &mut case, + ) + .await; + let _ = std::fs::remove_dir_all(&scratch); + recorded.map(|_| ()) + }); + eprintln!( + "[cap-verify] warm-up recording {} in {:.1}s", + if result.is_ok() { "done" } else { "failed" }, + started.elapsed().as_secs_f64() + ); + } + let mut failures = Vec::new(); + for spec in specs() { + if !report::selected(spec.id) || (spec.tier == Tier::Full && tier == Tier::Fast) { + continue; + } + if !spec.mic && !require_system_audio() { + // Recorded, so the summary shows it (and CI's + // CAP_VERIFY_REQUIRE_SUPPORTED fails it unless allowlisted). + let mut case = Case::new(spec.id); + case.unsupported( + "system audio is this case's only audio and this host does not play and require system audio (CAP_VERIFY_REQUIRE_SYSTEM_AUDIO unset)", + ); + if let Err(e) = report::settle(case) { + failures.push(e); + } + continue; + } + for attempt in 0..repeat { + ATTEMPT.store(attempt + 1, Ordering::Relaxed); + let mut case = Case::new(spec.id); + if repeat > 1 { + case.note(format!("repeat {}/{repeat}", attempt + 1)); + } + if let Err(e) = rt.block_on(run_case(&spec, capture, &shareable, &mut case)) { + case.problem("error", format!("{e:#}")); + } + for note in &case.notes { + eprintln!(" note: {note}"); + } + if let Err(e) = report::settle(case) { + failures.push(e); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +/// Host-clock truth for one source: marker index -> host time (seconds from +/// the case reference). +type Truth = BTreeMap; + +struct Feeds { + mic: Arc, + camera: Option>, + stop: Arc, + started: Arc, + mic_truth: Truth, + mic_task: std::thread::JoinHandle, + camera_task: Option, SourceTrace)>>, +} + +/// Computation budget per period of the synthetic sources' time +/// constraint: a camera frame (marker, NV12 pixel buffer, sample) took +/// 1.6 ms median, 4.7 ms p95 on hosted runners (run 38766191045); a mic +/// chunk is pre-rendered and only copied. +const CAMERA_COMPUTATION_SECS: f64 = 0.010; +const MIC_COMPUTATION_SECS: f64 = 0.001; + +/// One hand-off of a synthetic source (host ticks). +#[derive(Clone, Copy, Debug)] +struct Handoff { + /// Camera slot or microphone chunk number. + index: u32, + /// When the frame was captured, or the chunk's last sample was. + due: u64, + /// When the source's wait for `due` returned. + wake: u64, + /// When the frame or chunk was ready to hand over. + built: u64, + /// When the hand-off returned. + pushed: u64, + admission: Admission, + /// The source thread's scheduler priority when it woke. + pri: i32, +} + +/// Every hand-off of one synthetic source plus its scheduling: the +/// independent evidence for a source verdict, persisted per attempt. +#[derive(Clone, Debug)] +struct SourceTrace { + name: &'static str, + realtime: sched::Realtime, + /// Scheduling state after the last hand-off. + end: Option, + handoffs: Vec, +} + +impl SourceTrace { + fn start(name: &'static str, period: f64, computation: f64, capacity: usize) -> (Self, u32) { + let realtime = sched::request_realtime(period, computation); + ( + Self { + name, + realtime, + end: None, + handoffs: Vec::with_capacity(capacity), + }, + sched::current_thread(), + ) + } + + /// Hands over through `push`, timing each step. + fn handoff( + &mut self, + index: u32, + due: u64, + wake: u64, + pri: i32, + push: impl FnOnce() -> Admission, + ) -> Admission { + let built = host_now(); + let admission = push(); + self.handoffs.push(Handoff { + index, + due, + wake, + built, + pushed: host_now(), + admission, + pri, + }); + admission + } +} + +/// Waits for `due` on the source thread and samples its priority. +fn wait_for(thread: u32, due: u64) -> (u64, i32) { + sched::wait_until(due); + let wake = host_now(); + (wake, sched::thread_state(thread).map_or(-1, |s| s.cur_pri)) +} + +/// Synthetic sources stand in for capture devices, which the system feeds +/// from its own real-time threads and which never wait for the recorder: +/// each runs on a dedicated thread under a time constraint of its own +/// period (`cap_verify::sched`, shared with the stimulus) and hands over +/// with `try_send`, exactly like the device callbacks. A feed that cannot +/// keep up therefore shows as rejected hand-offs (a product failure), and a +/// late hand-off can only come from the source thread itself. +fn start_feeds( + reference: u64, + max_secs: f64, + camera: bool, + drops_at: Vec, + mic_delay: f64, +) -> Feeds { + let stop = Arc::new(AtomicBool::new(false)); + let started = Arc::new(AtomicU64::new(0)); + let mic = Arc::new(SyntheticMicrophone::spawn( + "Cap Verify Synthetic Microphone", + MIC_RATE, + 1, + )); + let origin = host_now(); + // Marker times count from the case reference, like the system audio + // stimulus, so the two sources' phases never overlap. + let base = host_secs(origin, reference); + let schedule: Vec = + audio_code::standard_schedule(AudioSource::Mic, 0.0, base + max_secs, AUDIO_PERIOD) + .into_iter() + .filter(|p| p.time_secs >= base + 0.3) + .map(|p| MarkerPlacement { + index: p.index, + time_secs: p.time_secs - base, + }) + .collect(); + let rate = f64::from(MIC_RATE); + let mic_truth: Truth = schedule + .iter() + .map(|p| (p.index, base + (p.time_secs * rate).round() / rate)) + .collect(); + let mic_task = { + let mic = mic.clone(); + let stop = stop.clone(); + let started = started.clone(); + std::thread::Builder::new() + .name("verify-synthetic-mic".into()) + .spawn(move || { + // Render the whole signal before going real-time: the + // constrained loop only copies chunks. + let mut signal = vec![0.0f32; (max_secs * f64::from(MIC_RATE)) as usize]; + audio_code::render_markers( + &mut signal, + MIC_RATE, + AudioSource::Mic, + &schedule, + audio_code::DEFAULT_AMPLITUDE, + ); + let (mut trace, thread) = SourceTrace::start( + "microphone", + MIC_CHUNK as f64 / f64::from(MIC_RATE), + MIC_COMPUTATION_SECS, + signal.len() / MIC_CHUNK + 1, + ); + for (n, chunk) in signal.chunks(MIC_CHUNK).enumerate() { + if stop.load(Ordering::Acquire) { + break; + } + let first = n * MIC_CHUNK; + let capture = host_add_secs(origin, first as f64 / f64::from(MIC_RATE)); + let delivered = + host_add_secs(origin, (first + chunk.len()) as f64 / f64::from(MIC_RATE)); + let (wake, pri) = wait_for(thread, delivered); + if mic_delay > 0.0 { + let start = started.load(Ordering::Acquire); + if start == 0 || host_secs(capture, start) < mic_delay { + continue; + } + } + trace.handoff(n as u32, delivered, wake, pri, || { + mic.try_push( + chunk, + Timestamp::MachAbsoluteTime(MachAbsoluteTimestamp::new(capture)), + ) + }); + } + trace.end = sched::thread_state(thread); + trace + }) + .expect("spawn synthetic microphone") + }; + let (camera, camera_task) = if camera { + let info = VideoInfo::from_raw( + RawVideoFormat::Nv12, + CAMERA_SIZE.0 as u32, + CAMERA_SIZE.1 as u32, + CAMERA_FPS, + ); + let feed = Arc::new(SyntheticCamera::spawn("Cap Verify Synthetic Camera", info)); + let task = { + let feed = feed.clone(); + let stop = stop.clone(); + let started = started.clone(); + std::thread::Builder::new() + .name("verify-synthetic-camera".into()) + .spawn(move || { + let slots = (max_secs * f64::from(CAMERA_FPS)) as u32; + let (mut trace, thread) = SourceTrace::start( + "camera", + 1.0 / f64::from(CAMERA_FPS), + CAMERA_COMPUTATION_SECS, + slots as usize, + ); + let origin = host_now(); + let base = host_secs(origin, reference); + let mut truth = Truth::new(); + let mut dropped = Vec::new(); + let mut drop_slots: Option> = None; + for slot in 0..slots { + if stop.load(Ordering::Acquire) { + break; + } + let secs = f64::from(slot) / f64::from(CAMERA_FPS); + let capture = host_add_secs(origin, secs); + let (wake, pri) = wait_for(thread, capture); + let start = started.load(Ordering::Acquire); + if drop_slots.is_none() && start != 0 { + let since = host_secs(start, origin); + drop_slots = Some( + drops_at + .iter() + .flat_map(|at| { + let s = + ((since + at) * f64::from(CAMERA_FPS)).ceil() as u32; + [s, s + 1] + }) + .collect(), + ); + } + if drop_slots.as_ref().is_some_and(|d| d.contains(&slot)) { + dropped.push(slot); + continue; + } + let Some(frame) = camera_frame(slot, capture) else { + continue; + }; + trace.handoff(slot, capture, wake, pri, || feed.try_push_native(frame)); + // Intended frames stay in the truth even when the + // feed rejected them: the recording must not + // silently lack them. + truth.insert(slot, base + secs); + } + trace.end = sched::thread_state(thread); + (truth, dropped, trace) + }) + .expect("spawn synthetic camera") + }; + (Some(feed), Some(task)) + } else { + (None, None) + }; + Feeds { + mic, + mic_truth, + camera, + stop, + started, + mic_task, + camera_task, + } +} + +struct StopOnDrop(Arc); + +impl Drop for StopOnDrop { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +/// The product classes a synthetic source's hand-offs can fail with. +fn ingress_class(source: &str) -> &'static str { + match source { + "camera" => "camera-ingress", + _ => "mic-ingress", + } +} + +/// What one source's hand-offs inside the recording window show. +#[derive(Debug, Default)] +struct SourceVerdict { + /// Product failures: (class, message). + product: Vec<(&'static str, String)>, + /// The source itself handed over more than `SOURCE_STALL_SECS` late + /// (the host stalled its thread): (lateness, inconclusive reason). + stalled: Option<(f64, String)>, + /// Worst lateness inside the window, seconds. + worst: f64, + /// Hand-offs inside the window, and those the feed rejected outside it + /// (before the recording started or after it stopped). + inside: usize, + rejected_outside: usize, +} + +/// Judges one source over the recording window (seconds from `reference`, +/// by due time). A hand-off is late by `built - due`, as before. +/// +/// A rejected hand-off (feed mailbox full or closed) is a product failure: +/// the real capture callbacks `try_send` the same way and lose the frame. +/// A late hand-off is attributed by its debt chain: while a hand-off was +/// due before the previous one returned, the time spent inside the feed's +/// hand-offs is the feed's. When the lateness without that time stays within +/// `SOURCE_STALL_SECS`, Cap delayed the source: a product failure, never +/// inconclusive. Otherwise the source thread itself woke or ran late. +fn judge_source(trace: &SourceTrace, reference: u64, window: (f64, f64)) -> SourceVerdict { + let class = ingress_class(trace.name); + let mut verdict = SourceVerdict::default(); + let mut rejected: Vec<(u32, f64, Admission)> = Vec::new(); + let mut cap_debt = 0.0f64; + let mut held = 0usize; + let mut held_worst: Option<(f64, String)> = None; + let mut previous: Option<&Handoff> = None; + for h in &trace.handoffs { + let busy = previous.is_some_and(|p| p.pushed > h.due); + cap_debt = match previous { + Some(p) if busy => cap_debt + host_secs(p.pushed, p.built), + _ => 0.0, + }; + let due = host_secs(h.due, reference); + let inside = due >= window.0 && due <= window.1; + if h.admission != Admission::Accepted { + if inside { + rejected.push((h.index, due, h.admission)); + } else { + verdict.rejected_outside += 1; + } + } + if inside { + verdict.inside += 1; + let late = host_secs(h.built, h.due); + verdict.worst = verdict.worst.max(late); + if late > SOURCE_STALL_SECS { + let cap = cap_debt.min(late); + let asleep_from = previous.map_or(h.due, |p| p.pushed.max(h.due)); + let breakdown = format!( + "woke {:.1} ms past its deadline at scheduler priority {}, built in {:.1} ms, \ + {:.1} ms inside Cap's hand-offs of earlier frames", + host_secs(h.wake, asleep_from).max(0.0) * 1000.0, + h.pri, + host_secs(h.built, h.wake) * 1000.0, + cap * 1000.0 + ); + if late - cap <= SOURCE_STALL_SECS { + held += 1; + if held_worst.as_ref().is_none_or(|(worst, _)| late > *worst) { + held_worst = Some(( + late, + format!( + "hand-off {} due at {due:.2}s, {:.0} ms late: {breakdown}", + h.index, + late * 1000.0 + ), + )); + } + } else if verdict + .stalled + .as_ref() + .is_none_or(|(worst, _)| late > *worst) + { + verdict.stalled = Some(( + late, + format!( + "{} source stalled: hand-off {} due at {due:.2}s was {:.0} ms late ({breakdown})", + trace.name, + h.index, + late * 1000.0 + ), + )); + } + } + } + previous = Some(h); + } + if let Some((_, worst)) = held_worst { + verdict.product.push(( + class, + format!( + "Cap's {} feed held the source: {held} hand-offs more than {:.0} ms late because of time spent inside the feed; worst {worst}", + trace.name, + SOURCE_STALL_SECS * 1000.0 + ), + )); + } + if let Some((index, at, admission)) = rejected.first() { + let full = rejected + .iter() + .filter(|(_, _, a)| *a == Admission::Full) + .count(); + verdict.product.push(( + class, + format!( + "Cap's {} feed rejected {} of {} hand-offs inside the recording ({full} mailbox full, {} closed); first {index} at {at:.2}s ({admission:?})", + trace.name, + rejected.len(), + verdict.inside, + rejected.len() - full + ), + )); + } + verdict +} + +/// `Err` when a source did not run under its real-time constraint for the +/// whole recording: its timing would not stand in for a capture device. +fn source_scheduling( + trace: &SourceTrace, + reference: u64, + window: (f64, f64), +) -> Result<(), String> { + trace.realtime.obtained().map_err(|e| { + format!( + "{} source: real-time scheduling not obtained: {e}", + trace.name + ) + })?; + sched::state_is_realtime(trace.end).map_err(|e| { + format!( + "{} source: real-time scheduling lost by the end: {e}", + trace.name + ) + })?; + if let Some(h) = trace.handoffs.iter().find(|h| { + let due = host_secs(h.due, reference); + due >= window.0 && due <= window.1 && h.pri < sched::REALTIME_PRIORITY + }) { + return Err(format!( + "{} source: hand-off {} ran at scheduler priority {}, below the real-time band", + trace.name, h.index, h.pri + )); + } + Ok(()) +} + +/// Writes every source's scheduling and hand-offs for one attempt (all +/// attempts, including discarded retries) and returns the file. +fn persist_sources( + stem: &str, + reference: u64, + rec: &Recorded, + verdicts: &[SourceVerdict], + fallback: &Path, +) -> PathBuf { + let dir = std::env::var_os("CAP_VERIFY_DUMP") + .map(|d| PathBuf::from(d).join("sources")) + .unwrap_or_else(|| fallback.to_path_buf()); + let path = dir.join(format!("{stem}.json")); + let info = cidre::mach::TimeBaseInfo::new(); + let sources: Vec = rec + .sources + .iter() + .zip(verdicts) + .map(|(trace, verdict)| { + serde_json::json!({ + "name": trace.name, + "realtime": trace.realtime, + "end": trace.end, + "worst_late_ms": verdict.worst * 1000.0, + "stalled": verdict.stalled.as_ref().map(|(_, reason)| reason), + "product": verdict.product.iter().map(|(class, text)| format!("{class}: {text}")).collect::>(), + "handoffs": trace.handoffs.iter().map(|h| ( + h.index, + h.due, + h.wake, + h.built, + h.pushed, + match h.admission { + Admission::Accepted => "ok", + Admission::Full => "full", + Admission::Closed => "closed", + }, + h.pri, + )).collect::>(), + }) + }) + .collect(); + let value = serde_json::json!({ + "attempt": stem, + "timebase": [info.numer, info.denom], + "reference": reference, + "window_secs": [rec.started, rec.stopped], + "columns": ["index", "due", "wake", "built", "pushed", "admission", "pri"], + "sources": sources, + }); + let written = + std::fs::create_dir_all(&dir).and_then(|()| std::fs::write(&path, value.to_string())); + if let Err(e) = written { + eprintln!("[cap-verify] could not write {}: {e}", path.display()); + } + path +} + +fn camera_frame(slot: u32, capture: u64) -> Option { + let (w, h) = CAMERA_SIZE; + let rgba = video_code::marker_rgba(FrameMarker::new(StreamKind::Camera, 0, slot), w, h); + let sample_buf = camera_sample(&rgba, w, h, capture).ok()?; + Some(NativeCameraFrame { + sample_buf, + timestamp: Timestamp::MachAbsoluteTime(MachAbsoluteTimestamp::new(capture)), + }) +} + +fn camera_sample( + rgba: &[u8], + w: usize, + h: usize, + host: u64, +) -> anyhow::Result> { + use cidre::cm; + let buf = fixture_avf::pixel_buffer(w, h, rgba)?; + let format_desc = cm::VideoFormatDesc::with_image_buf(&buf) + .map_err(|e| anyhow::anyhow!("format description: {e:?}"))?; + let nanos = (host_secs(host, 0) * 1e9).round() as i64; + let timing = cm::SampleTimingInfo { + duration: cm::Time::new(1, CAMERA_FPS as i32), + pts: cm::Time::new(nanos, 1_000_000_000), + dts: cm::Time::invalid(), + }; + cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &format_desc, &timing) + .map_err(|e| anyhow::anyhow!("sample buffer: {e:?}")) +} + +enum Handle { + Studio(studio_recording::ActorHandle), + Instant(instant_recording::ActorHandle), +} + +/// Capture target plus every window except the stimulus, which is excluded +/// from the capture filter so system prompts and the developer's apps never +/// reach the recording. +async fn capture_setup( + capture: CaptureMode, +) -> anyhow::Result<( + ScreenCaptureTarget, + Vec, + SendableShareableContent, +)> { + let stimulus_window = find_window(WINDOW_TITLE); + let excluded = scap_targets::Window::list() + .into_iter() + .map(|w| w.id()) + .filter(|id| Some(id) != stimulus_window.as_ref()) + .collect(); + let shareable = cidre::sc::ShareableContent::current() + .await + .map_err(|e| anyhow::anyhow!("shareable content: {e:?}"))? + .into(); + let target = match capture { + CaptureMode::Display => ScreenCaptureTarget::Display { + id: scap_targets::Display::primary().id(), + }, + CaptureMode::Window => ScreenCaptureTarget::Window { + id: stimulus_window.ok_or_else(|| anyhow::anyhow!("stimulus window not found"))?, + }, + }; + Ok((target, excluded, shareable)) +} + +/// The app's Studio builder (`RecordingDefaults::apply_to_studio_builder`) +/// with the capture-independent settings pinned: no keyboard capture (it +/// would prompt for Input Monitoring) and Balanced quality on every host. +fn studio_builder( + project: PathBuf, + target: ScreenCaptureTarget, + excluded: Vec, + fragmented: bool, + camera: bool, + system_audio: bool, +) -> (studio_recording::ActorBuilder, u32) { + let defaults = RecordingDefaults::default(); + let builder = defaults + .apply_to_studio_builder( + studio_recording::Actor::builder(project, target).with_system_audio(system_audio), + camera, + None, + ) + .with_excluded_windows(excluded) + .with_keyboard_capture(false) + .with_fragmented(fragmented) + .with_quality(StudioQuality::Balanced); + (builder, defaults.studio_max_fps(camera, None)) +} + +fn stimulus_binary() -> PathBuf { + std::env::var_os("CAP_VERIFY_STIMULUS_BIN") + .map(PathBuf::from) + .unwrap_or_else(|| { + // Built by `cargo build -p cap-verify-stimulus` into the same + // target directory as this test (target//deps/..). + let exe = std::env::current_exe().expect("test executable path"); + let path = exe + .parent() + .and_then(|deps| deps.parent()) + .expect("target profile directory") + .join("cap-verify-stimulus"); + assert!( + path.exists(), + "{} not found: build it first with `cargo build -p cap-verify-stimulus` (same profile), or set CAP_VERIFY_STIMULUS_BIN", + path.display() + ); + path + }) +} + +fn find_window(title: &str) -> Option { + scap_targets::Window::list() + .into_iter() + .find(|w| w.name().as_deref() == Some(title)) + .map(|w| w.id()) +} + +struct Recorded { + project: PathBuf, + started: f64, + stopped: f64, + stimulus: Vec, + mic_truth: Truth, + camera: Option<(Truth, Vec)>, + max_fps: u32, + mic_delay: f64, + /// The stimulus's scripted holds and rate, to locate the first picture + /// after each hold. + holds: Vec<(f64, f64)>, + stimulus_fps: f64, + /// Every hand-off of the synthetic microphone and camera. + sources: Vec, + /// The stimulus process's real-time sentinel. + sentinel: Result, + /// Recovery after a crash may lose at most this many seconds before the + /// crash (the in-flight fragment). + loss_budget: Option, +} + +async fn run_case( + spec: &Spec, + capture: CaptureMode, + shareable: &SendableShareableContent, + case: &mut Case, +) -> anyhow::Result<()> { + let scratch = scratch_root().join(format!( + "{}-r{}", + spec.id.replace('/', "-"), + ATTEMPT.load(Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&scratch); + let mut tries = 0; + let (recorded, reference) = loop { + tries += 1; + let attempt_scratch = scratch.join(format!("attempt-{tries}")); + std::fs::create_dir_all(&attempt_scratch)?; + let reference = host_now(); + let recorded = record(spec, capture, shareable, &attempt_scratch, reference, case).await?; + let window = (recorded.started, recorded.stopped); + let verdicts: Vec = recorded + .sources + .iter() + .map(|trace| judge_source(trace, reference, window)) + .collect(); + let stem = format!( + "{}-r{}-a{tries}", + spec.id.replace('/', "-"), + ATTEMPT.load(Ordering::Relaxed) + ); + let evidence = persist_sources(&stem, reference, &recorded, &verdicts, &attempt_scratch); + case.note(format!( + "attempt {tries} sources ({}): {}", + evidence.display(), + recorded + .sources + .iter() + .zip(&verdicts) + .map(|(trace, verdict)| format!( + "{} {} hand-offs, worst {:.1} ms late, priority {}", + trace.name, + verdict.inside, + verdict.worst * 1000.0, + trace.realtime.state.map_or(-1, |s| s.base_pri) + )) + .collect::>() + .join("; ") + )); + for (trace, verdict) in recorded.sources.iter().zip(&verdicts) { + if verdict.rejected_outside > 0 { + case.note(format!( + "{} feed rejected {} hand-offs outside the recording", + trace.name, verdict.rejected_outside + )); + } + } + if let Some(reason) = recorded + .sources + .iter() + .find_map(|trace| source_scheduling(trace, reference, window).err()) + { + case.problem( + "harness", + format!("{reason}; evidence {}", evidence.display()), + ); + case.note(format!("artifacts kept at {}", scratch.display())); + return Ok(()); + } + let sentinel = match &recorded.sentinel { + Ok(log) => log, + Err(e) => { + case.problem("harness", format!("stimulus sentinel log unavailable: {e}")); + case.note(format!("artifacts kept at {}", scratch.display())); + return Ok(()); + } + }; + let sentinel_evidence = keep_sentinel_log(&attempt_scratch, &stem); + if let Err(e) = sentinel + .realtime + .obtained() + .and_then(|()| sched::state_is_realtime(sentinel.end)) + { + case.problem( + "harness", + format!("stimulus sentinel: real-time scheduling not obtained or lost: {e}"), + ); + case.note(format!("artifacts kept at {}", scratch.display())); + return Ok(()); + } + let freezes = machine_freezes(sentinel, reference, window); + case.note(format!( + "attempt {tries} sentinel ({}): {}", + sentinel_evidence.display(), + describe_sentinel(sentinel, reference, window) + )); + let product: Vec<&(&str, String)> = verdicts.iter().flat_map(|v| &v.product).collect(); + let product_failed = !product.is_empty(); + // The recorder's own feed failed its source: a product failure that + // a retry must not hide. The output checks still run. + for (class, text) in product { + case.problem(class, format!("{text}; evidence {}", evidence.display())); + } + if !product_failed && let Some(reason) = inconclusive(spec, &recorded, &verdicts) { + let reason = format!("{reason}; evidence {}", evidence.display()); + keep_stimulus_log(&attempt_scratch, &stem); + if tries < 2 { + case.note(format!("inconclusive, retrying: {reason}")); + continue; + } + case.problem("environment", format!("inconclusive twice: {reason}")); + case.note(format!("artifacts kept at {}", scratch.display())); + return Ok(()); + } + let before = case.checkpoint(); + IMPLICATED.with(|m| m.borrow_mut().clear()); + analyze(&recorded, reference, case)?; + let implicated = IMPLICATED.with(|m| std::mem::take(&mut *m.borrow_mut())); + let excuse = (!product_failed) + .then(|| frozen_failures(case, before.problems(), &implicated, &freezes)) + .flatten(); + let Some(reason) = excuse else { + break (recorded, reference); + }; + // Every unexpected failure lies inside a machine freeze: the + // recording cannot tell Cap from the host. Discard the analysis. + case.rollback(before); + let reason = format!("{reason}; evidence {}", sentinel_evidence.display()); + keep_stimulus_log(&attempt_scratch, &stem); + if tries < 2 { + case.note(format!("inconclusive, retrying: {reason}")); + continue; + } + case.problem("environment", format!("inconclusive twice: {reason}")); + case.note(format!("artifacts kept at {}", scratch.display())); + return Ok(()); + }; + if self_check_enabled() { + self_check(&recorded, reference, case)?; + } + if !case.failed() && !env_flag("CAP_VERIFY_KEEP") { + let _ = std::fs::remove_dir_all(&scratch); + } else { + case.note(format!("artifacts kept at {}", scratch.display())); + } + Ok(()) +} + +/// A synthetic source that hands a frame or chunk over this late did not +/// produce its input on time: no capture device delivers this late, so the +/// host stalled the source (a VM-wide stall measured 286 ms; with the +/// sources on their own threads, steady hand-offs are within a few ms). +const SOURCE_STALL_SECS: f64 = 0.1; + +/// A run whose input was faulty, proven by a log independent of the +/// recorder, cannot tell a capture problem from a harness problem: it is +/// retried once, and a case that stays inconclusive fails as `environment`. +/// - the stimulus was starved (it skipped or stalled its own pictures); +/// - the stimulus's own ScreenCaptureKit stream (another client, another +/// process) received no new frame for more than four stimulus periods +/// while pictures were being presented: the window server or capture +/// service stalled, so the pictures never reached any recorder; +/// - a synthetic source's own thread handed over input more than +/// `SOURCE_STALL_SECS` late inside the recording (`judge_source`; time +/// spent inside Cap's feed is a product failure instead). +fn inconclusive(spec: &Spec, rec: &Recorded, verdicts: &[SourceVerdict]) -> Option { + if let Some(reason) = starved(spec, &rec.stimulus) { + return Some(reason); + } + if let Some(reason) = compositor_stall(spec, &rec.stimulus) { + return Some(reason); + } + verdicts + .iter() + .find_map(|v| v.stalled.as_ref().map(|(_, reason)| reason.clone())) +} + +/// A gap this long between wakes of the stimulus's real-time sentinel +/// means the whole machine stopped running (see `Sentinel` in the stimulus). +/// Nothing the recorder does can delay a priority-97 thread of another +/// process that long: it preempts every timeshare thread, the recorder's +/// real-time threads (audio and capture callbacks, the synthetic sources) +/// each run a few ms per period and are demoted by the kernel if they +/// overrun their computation budget, so only a host that runs no virtual +/// CPU, or a kernel that schedules nothing, leaves the sentinel asleep. +const SENTINEL_FREEZE_SECS: f64 = 0.1; +/// Failures attributed to a freeze: frames or markers captured from shortly +/// before it (in flight when it began) until shortly after it (the burst of +/// frames the recorder catches up with). +const FREEZE_MARGIN_BEFORE: f64 = 0.15; +const FREEZE_MARGIN_AFTER: f64 = 0.3; + +/// Sentinel gaps over `SENTINEL_FREEZE_SECS` that overlap the recording +/// window: (start, end) in seconds from the reference. +fn machine_freezes(log: &SentinelLog, reference: u64, window: (f64, f64)) -> Vec<(f64, f64)> { + log.gaps() + .filter(|(_, gap)| *gap > SENTINEL_FREEZE_SECS) + .map(|(start, gap)| { + let start = host_secs(start, reference); + (start, start + gap) + }) + .filter(|(start, end)| *end >= window.0 && *start <= window.1) + .collect() +} + +fn describe_sentinel(log: &SentinelLog, reference: u64, window: (f64, f64)) -> String { + let gaps: Vec = log + .gaps() + .filter(|(start, _)| { + let at = host_secs(*start, reference); + at >= window.0 && at <= window.1 + }) + .map(|(_, gap)| gap) + .collect(); + let max = gaps.iter().copied().fold(0.0f64, f64::max); + format!( + "{} wakes in the recording, priority {}, longest gap {:.1} ms, {} over 10 ms, {} over {:.0} ms", + gaps.len(), + log.realtime.state.map_or(-1, |s| s.base_pri), + max * 1000.0, + gaps.iter().filter(|g| **g > 0.010).count(), + gaps.iter().filter(|g| **g > SENTINEL_FREEZE_SECS).count(), + SENTINEL_FREEZE_SECS * 1000.0 + ) +} + +/// `Some(reason)` when the analysis added unexpected failures (classes the +/// case has no xfail for) and every one of them is localized to frames or +/// markers captured within a machine freeze. Failures that are not +/// localized (sync medians, spreads, unreadable or unknown frames) are never +/// attributed to a freeze, and a case without a freeze keeps every failure. +fn frozen_failures( + case: &Case, + from: usize, + implicated: &BTreeMap>, + freezes: &[(f64, f64)], +) -> Option { + if freezes.is_empty() { + return None; + } + let expected: Vec = report::xfails_for(&case.id) + .into_iter() + .map(|x| x.class) + .collect(); + let unexpected: Vec = (from..case.problems.len()) + .filter(|&i| { + case.problem_classes[i] + .split(',') + .any(|class| !expected.iter().any(|e| e == class)) + }) + .collect(); + if unexpected.is_empty() { + return None; + } + let inside = |t: f64| { + freezes.iter().any(|&(start, end)| { + t >= start - FREEZE_MARGIN_BEFORE && t <= end + FREEZE_MARGIN_AFTER + }) + }; + for &i in &unexpected { + let times = implicated.get(&i)?; + if times.is_empty() || !times.iter().all(|&t| inside(t)) { + return None; + } + } + Some(format!( + "machine froze (stimulus sentinel asleep {}) and every unexpected failure lies within it: {}", + freezes + .iter() + .map(|(start, end)| format!("{:.0} ms at {start:.2}s", (end - start) * 1000.0)) + .collect::>() + .join(", "), + unexpected + .iter() + .map(|&i| format!("{}: {}", case.problem_classes[i], case.problems[i])) + .collect::>() + .join("; ") + )) +} + +/// Keeps every attempt's sentinel log with the source evidence. +fn keep_sentinel_log(attempt_scratch: &Path, stem: &str) -> PathBuf { + let Some(dir) = std::env::var_os("CAP_VERIFY_DUMP") else { + return attempt_scratch.join(SENTINEL_FILE); + }; + let dir = PathBuf::from(dir).join("sources"); + let path = dir.join(format!("{stem}-sentinel.txt")); + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::copy(attempt_scratch.join(SENTINEL_FILE), &path); + path +} + +thread_local! { + /// Capture times (seconds from the reference) of the frames or markers + /// behind each localized problem, by its index in `Case::problems`. + static IMPLICATED: std::cell::RefCell>> = + const { std::cell::RefCell::new(BTreeMap::new()) }; +} + +/// Localizes the problem just added to `case`. +fn implicate(case: &Case, times: impl IntoIterator) { + if let Some(index) = case.problems.len().checked_sub(1) { + IMPLICATED.with(|m| m.borrow_mut().insert(index, times.into_iter().collect())); + } +} + +/// Keeps a discarded attempt's stimulus log (its independent capture +/// observer) next to the source evidence: the case's scratch directory is +/// removed when a retry passes. +fn keep_stimulus_log(attempt_scratch: &Path, stem: &str) { + if let Some(dir) = std::env::var_os("CAP_VERIFY_DUMP") { + let dir = PathBuf::from(dir).join("sources"); + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::copy( + attempt_scratch.join("stimulus.jsonl"), + dir.join(format!("{stem}-stimulus.jsonl")), + ); + } +} + +/// Longest stretch without a new frame on the stimulus's own capture stream +/// while the stimulus was changing its picture (outside scripted holds and +/// the period after them, where a capture stream legitimately resumes late). +fn compositor_stall(spec: &Spec, events: &[StimulusEvent]) -> Option { + let go = events.iter().find_map(|e| match e { + StimulusEvent::Go { host } => Some(*host), + _ => None, + })?; + let done = events.iter().find_map(|e| match e { + StimulusEvent::Done { host } => Some(host_secs(*host, go)), + _ => None, + })?; + let observed: Vec = events + .iter() + .filter_map(|e| match e { + StimulusEvent::Observed { host } => Some(host_secs(*host, go)), + _ => None, + }) + .collect(); + if observed.is_empty() { + return None; + } + let period = 1.0 / spec.stimulus_fps; + let quiet = |t: f64| { + spec.holds + .iter() + .any(|&(s, l)| t > s - period && t < s + l + 4.0 * period) + }; + let mut points: Vec = observed + .into_iter() + .filter(|t| *t >= 0.0 && *t <= done) + .collect(); + points.insert(0, 0.0); + points.push(done); + let (gap, at) = points + .windows(2) + .filter(|w| !quiet(w[0]) && !quiet(w[1])) + .map(|w| (w[1] - w[0], w[0])) + .fold((0.0f64, 0.0f64), |m, x| if x.0 > m.0 { x } else { m }); + (gap > 4.0 * period).then(|| { + format!( + "screen capture stalled for every client: the stimulus's own capture stream received no frame for {:.0} ms at {:.2}s after go", + gap * 1000.0, + at + ) + }) +} + +/// The stimulus is the truth source: when the host starves it (it skipped +/// frames or stalled outside the scripted holds) a run cannot tell capture +/// problems from harness problems, so it is retried once. +fn starved(spec: &Spec, events: &[StimulusEvent]) -> Option { + let go = events.iter().find_map(|e| match e { + StimulusEvent::Go { host } => Some(*host), + _ => None, + })?; + let frames: Vec<(u32, f64)> = events + .iter() + .filter_map(|e| match e { + StimulusEvent::Frame { index, host, .. } if *index > 0 => { + Some((*index, host_secs(*host, go))) + } + _ => None, + }) + .collect(); + let expected = (1..=(spec.seconds * spec.stimulus_fps).round() as u32) + .filter(|index| { + let time = f64::from(index - 1) / spec.stimulus_fps; + !spec + .holds + .iter() + .any(|&(start, length)| time > start && time < start + length) + }) + .count(); + let longest_draw = events + .iter() + .filter_map(|event| match event { + StimulusEvent::Frame { + host, + render_started: Some(start), + .. + } => Some(host_secs(*host, *start)), + _ => None, + }) + .fold(0.0f64, f64::max); + let draw_note = format!("longest draw {:.0} ms", longest_draw * 1000.0); + if frames.len() * 10 < expected * 9 { + return Some(format!( + "presented {} of {expected} frames; {draw_note}", + frames.len() + )); + } + let period = 1.0 / spec.stimulus_fps; + let held = |t: f64| { + spec.holds + .iter() + .any(|&(s, l)| t > s - period && t < s + l + period) + }; + let stall = frames + .windows(2) + .filter(|w| !held(w[0].1) && !held(w[1].1)) + .map(|w| w[1].1 - w[0].1) + .fold(0.0f64, f64::max); + (stall > 4.0 * period).then(|| { + format!( + "stalled {:.0} ms between frames; {draw_note}", + stall * 1000.0 + ) + }) +} + +#[test] +fn scripted_holds_do_not_count_as_stimulus_starvation() { + let mut spec = specs().remove(0); + spec.seconds = 2.0; + spec.holds = vec![(0.5, 0.5)]; + let reference = host_now(); + let mut events = vec![StimulusEvent::Go { host: reference }]; + for index in (1..=11).chain(21..=40) { + events.push(StimulusEvent::Frame { + index, + host: host_add_secs(reference, f64::from(index - 1) / spec.stimulus_fps), + render_started: None, + }); + } + assert_eq!(starved(&spec, &events), None); +} + +#[test] +fn capture_service_stalls_are_inconclusive_but_hold_resumes_are_not() { + let spec = &specs()[0]; + let go = host_now(); + let period = 1.0 / spec.stimulus_fps; + let mut events = vec![StimulusEvent::Go { host: go }]; + let observe = |events: &mut Vec, from: f64, to: f64| { + let mut t = from; + while t < to { + events.push(StimulusEvent::Observed { + host: host_add_secs(go, t), + }); + t += period; + } + }; + observe(&mut events, 0.0, 2.0); + observe(&mut events, 3.05, spec.seconds); + events.push(StimulusEvent::Done { + host: host_add_secs(go, spec.seconds), + }); + assert!(compositor_stall(spec, &events).is_some_and(|r| r.contains("1100 ms at 1.95s"))); + let mut held = spec.clone(); + held.holds = vec![(2.0, 1.0)]; + assert_eq!(compositor_stall(&held, &events), None); +} + +#[test] +fn observer_silence_excuses_only_a_covered_window() { + let truth: Truth = (0..10u32).map(|i| (i, f64::from(i) * 0.05)).collect(); + let check = |observed: Vec| VideoCheck { + name: "screen", + kind: StreamKind::Screen, + truth: &truth, + not_sent: &[], + may_skip: Box::new(|_| false), + allow_duplicates: true, + required: Vec::new(), + window: (0.0, 1.0), + observed, + }; + // Picture 4 is on screen from 0.20 to 0.25 s. + let around: Vec = (0..20) + .map(|k| f64::from(k) * 0.0167) + .filter(|t| !(0.199..=0.251).contains(t)) + .collect(); + assert!(check(around.clone()).dropped_for_every_client(4)); + assert!(!check(around.iter().copied().chain([0.22]).collect()).dropped_for_every_client(4)); + let only_before: Vec = around.iter().copied().filter(|t| *t < 0.2).collect(); + assert!( + !check(only_before).dropped_for_every_client(4), + "the observer must cover the window" + ); + assert!(!check(Vec::new()).dropped_for_every_client(4)); +} + +fn test_trace(name: &'static str, handoffs: Vec) -> SourceTrace { + let constraint = sched::Constraint { + period_ms: 33.333, + computation_ms: 16.667, + constraint_ms: 33.333, + preemptible: true, + }; + let state = sched::ThreadState { + policy: 2, + base_pri: 97, + cur_pri: 97, + max_pri: 63, + cpu_us: 0, + }; + SourceTrace { + name, + realtime: sched::Realtime { + requested: sched::Constraint { + computation_ms: 10.0, + ..constraint + }, + qos_status: 0, + policy_status: 0, + readback_status: 0, + readback_default: false, + granted: Some(constraint), + state: Some(state), + }, + end: Some(state), + handoffs, + } +} + +/// Camera hand-offs every 33 ms from `reference`; `late(slot)` gives the +/// wake delay and `push(slot)` the time inside the feed, in seconds. A +/// source behind schedule wakes as soon as its previous hand-off returns. +fn simulated_handoffs( + reference: u64, + slots: u32, + late: impl Fn(u32) -> f64, + push: impl Fn(u32) -> f64, + admission: impl Fn(u32) -> Admission, +) -> Vec { + let mut free = reference; + (0..slots) + .map(|slot| { + let due = host_add_secs(reference, f64::from(slot) / 30.0); + let wake = host_add_secs(due.max(free), late(slot)); + let built = host_add_secs(wake, 0.003); + let pushed = host_add_secs(built, push(slot)); + free = pushed; + Handoff { + index: slot, + due, + wake, + built, + pushed, + admission: admission(slot), + pri: 97, + } + }) + .collect() +} + +#[test] +fn source_lateness_is_attributed_to_the_source_or_to_cap() { + let reference = host_now(); + let window = (1.0, 8.0); + let ok = |_| Admission::Accepted; + // The source overslept 151 ms at slot 90 (3.0 s) and caught up in a + // burst: the host stalled it, inconclusive with its breakdown. + let overslept = test_trace( + "camera", + simulated_handoffs( + reference, + 300, + |s| if s == 90 { 0.151 } else { 0.0005 }, + |_| 0.00004, + ok, + ), + ); + let verdict = judge_source(&overslept, reference, window); + assert!(verdict.product.is_empty(), "{:?}", verdict.product); + let (late, reason) = verdict.stalled.expect("stall"); + assert!((late - 0.154).abs() < 0.002, "{late}"); + assert!( + reason.contains("woke 151.0 ms past its deadline"), + "{reason}" + ); + // Outside the recording window it does not count. + assert!( + judge_source(&overslept, reference, (3.5, 8.0)) + .stalled + .is_none() + ); + // Cap's feed held the source for 200 ms inside hand-off 90: the later + // hand-offs are late because of Cap, never inconclusive. + let blocked = test_trace( + "camera", + simulated_handoffs( + reference, + 300, + |_| 0.0005, + |s| if s == 90 { 0.2 } else { 0.00004 }, + ok, + ), + ); + let verdict = judge_source(&blocked, reference, window); + assert!(verdict.stalled.is_none(), "{:?}", verdict.stalled); + assert_eq!(verdict.product.len(), 1, "{:?}", verdict.product); + assert_eq!(verdict.product[0].0, "camera-ingress"); + assert!( + verdict.product[0].1.contains("hand-off 91"), + "{:?}", + verdict.product + ); + // A rejected hand-off fails as ingress inside the window and is only + // counted outside it. + let full = |s| { + if s == 15 || s == 150 { + Admission::Full + } else { + Admission::Accepted + } + }; + let rejected = test_trace( + "microphone", + simulated_handoffs(reference, 300, |_| 0.0005, |_| 0.00004, full), + ); + let verdict = judge_source(&rejected, reference, window); + assert_eq!(verdict.rejected_outside, 1); + assert_eq!(verdict.product.len(), 1); + assert_eq!(verdict.product[0].0, "mic-ingress"); + assert!( + verdict.product[0].1.contains("rejected 1 of"), + "{:?}", + verdict.product + ); + // A source below the real-time band is a harness failure. + let mut demoted = overslept.clone(); + demoted.handoffs[100].pri = 31; + assert!(source_scheduling(&demoted, reference, window).is_err()); + assert!(source_scheduling(&overslept, reference, window).is_ok()); +} + +#[test] +fn sentinel_freezes_excuse_only_failures_localized_inside_them() { + let reference = host_now(); + // A sentinel log with a 300 ms gap at 9.9 s and a 60 ms gap at 5 s. + let mut wakes = vec![host_add_secs(reference, 1.0)]; + let mut t: f64 = 1.0; + while t < 12.0 { + t += if (t - 9.9).abs() < 0.001 { + 0.3 + } else if (t - 5.0).abs() < 0.001 { + 0.06 + } else { + 0.002 + }; + t = (t * 1000.0).round() / 1000.0; + wakes.push(host_add_secs(reference, t)); + } + let realtime = test_trace("camera", vec![]).realtime; + let mut text = serde_json::json!({ + "period_secs": 0.002, + "first": wakes[0], + "realtime": realtime, + "end": realtime.state, + }) + .to_string(); + for w in wakes.windows(2) { + text.push_str(&format!("\n{}", w[1] - w[0])); + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SENTINEL_FILE); + std::fs::write(&path, text).unwrap(); + let log = SentinelLog::read(&path).unwrap(); + assert_eq!(log.wakes, wakes); + let freezes = machine_freezes(&log, reference, (2.0, 11.0)); + assert_eq!(freezes.len(), 1); + assert!((freezes[0].0 - 9.9).abs() < 1e-3 && (freezes[0].1 - 10.2).abs() < 1e-3); + assert!(machine_freezes(&log, reference, (2.0, 9.0)).is_empty()); + + let excuse = |problems: &[(&str, Option>)], freezes: &[(f64, f64)]| { + let mut case = Case::new("recording/instant/screen-mic-system"); + IMPLICATED.with(|m| m.borrow_mut().clear()); + for (class, times) in problems { + case.problem(class, format!("{class} problem")); + if let Some(times) = times { + implicate(&case, times.iter().copied()); + } + } + let implicated = IMPLICATED.with(|m| std::mem::take(&mut *m.borrow_mut())); + frozen_failures(&case, 0, &implicated, freezes) + }; + let lost = ("missing", Some(vec![9.8, 10.0, 10.4])); + // Expected classes (sync-screen-mic here) never need an excuse. + let clock = ("sync-screen-mic", None); + assert!(excuse(&[lost.clone(), clock.clone()], &freezes).is_some()); + // Without a freeze, or outside it, a loss stays a Cap failure. + assert!(excuse(&[lost.clone()], &[]).is_none()); + assert!(excuse(&[("missing", Some(vec![10.0, 10.6]))], &freezes).is_none()); + assert!(excuse(&[("missing", Some(vec![9.7]))], &freezes).is_none()); + // A failure that is not localized (a sync median) is never excused. + assert!(excuse(&[lost, ("sync-screen-system", None)], &freezes).is_none()); + // Only expected failures: nothing to excuse. + assert!(excuse(&[clock], &freezes).is_none()); +} + +/// A feed that stops taking frames (scripted slow consumer: its actor is +/// blocked for 4 s) must be reported as a product failure of class +/// `camera-ingress` / `mic-ingress`, never as a late, inconclusive source. +/// Runs the real synthetic sources, under the real-time constraint this +/// host grants, against the real feed actors. +#[test] +fn injected_feed_backpressure_is_ingress_not_inconclusive() { + let rt = runtime(); + rt.block_on(async { + let reference = host_now(); + let feeds = start_feeds(reference, 8.0, true, vec![], 0.0); + tokio::time::sleep(Duration::from_millis(500)).await; + let started = host_now(); + feeds.started.store(started, Ordering::Release); + let camera = feeds.camera.clone().expect("camera"); + assert_eq!(camera.stall(Duration::from_secs(4)), Admission::Accepted); + assert_eq!(feeds.mic.stall(Duration::from_secs(4)), Admission::Accepted); + tokio::time::sleep(Duration::from_secs(3)).await; + let stopped = host_now(); + feeds.stop.store(true, Ordering::Release); + let mic = tokio::task::block_in_place(|| feeds.mic_task.join()).expect("mic source"); + let (truth, _, cam) = + tokio::task::block_in_place(|| feeds.camera_task.expect("camera").join()) + .expect("camera source"); + let window = (host_secs(started, reference), host_secs(stopped, reference)); + let mut case = Case::new("recording/self-check/feed-backpressure"); + for trace in [&cam, &mic] { + source_scheduling(trace, reference, window).expect("real-time sources"); + let verdict = judge_source(trace, reference, window); + eprintln!( + " {}: {} hand-offs, worst {:.1} ms late, priority {:?}, stalled {:?}", + trace.name, + verdict.inside, + verdict.worst * 1000.0, + trace.realtime.state.map(|s| s.base_pri), + verdict.stalled + ); + // run_case reports product verdicts before (and instead of) + // any inconclusive retry. + assert!( + !verdict.product.is_empty(), + "{} rejections not reported", + trace.name + ); + for (class, text) in verdict.product { + eprintln!(" {class}: {text}"); + case.problem(class, text); + } + } + assert_eq!( + case.classes.iter().map(String::as_str).collect::>(), + ["camera-ingress", "mic-ingress"] + ); + let rejected: Vec = cam + .handoffs + .iter() + .filter(|h| h.admission == Admission::Full) + .map(|h| h.index) + .collect(); + assert!(!rejected.is_empty()); + assert!(rejected.iter().all(|slot| truth.contains_key(slot))); + }); +} + +#[test] +fn unscripted_stimulus_stalls_remain_failures() { + let spec = &specs()[0]; + let reference = host_now(); + let mut events = vec![StimulusEvent::Go { host: reference }]; + for index in 1..=(spec.seconds * spec.stimulus_fps).round() as u32 { + if (80..85).contains(&index) { + continue; + } + events.push(StimulusEvent::Frame { + index, + host: host_add_secs(reference, f64::from(index - 1) / spec.stimulus_fps), + render_started: None, + }); + } + assert!(starved(spec, &events).is_some_and(|reason| reason.starts_with("stalled"))); + events.retain(|event| !matches!(event, StimulusEvent::Frame { index: 90..110, .. })); + assert!(starved(spec, &events).is_some_and(|reason| reason.starts_with("presented"))); +} + +fn scratch_root() -> PathBuf { + std::env::var_os("CAP_VERIFY_SCRATCH") + .map(PathBuf::from) + .unwrap_or_else(|| std::env::temp_dir().join("cap-verify-recording")) +} + +async fn record( + spec: &Spec, + capture: CaptureMode, + _preflight: &SendableShareableContent, + scratch: &Path, + reference: u64, + case: &mut Case, +) -> anyhow::Result { + let stimulus_spec = StimulusSpec { + fps: spec.stimulus_fps, + seconds: spec.seconds, + holds: spec.holds.clone(), + busy: true, + audio: spec.system_audio && stimulus_plays_system_audio(), + warmup: match capture { + CaptureMode::Display => 2.0, + CaptureMode::Window => 2.0, + }, + audio_period: AUDIO_PERIOD, + audio_origin: reference, + window: match capture { + CaptureMode::Display => None, + CaptureMode::Window => Some((80, 120, 640, 400)), + }, + }; + let mut stimulus = tokio::task::block_in_place(|| { + StimulusProcess::spawn( + &stimulus_binary(), + &stimulus_spec, + &scratch.join("stimulus.jsonl"), + ) + })?; + tokio::task::block_in_place(|| stimulus.wait_for("ready", Duration::from_secs(30)))?; + let (target, excluded, shareable) = capture_setup(capture).await?; + + let camera = matches!(spec.mode, Mode::Studio { camera: true, .. }); + let feeds = start_feeds( + reference, + spec.seconds + 30.0, + camera, + spec.camera_drops_at.clone(), + spec.mic_delay, + ); + // The real-time source threads must not outlive a failed attempt. + let _stop_sources = StopOnDrop(feeds.stop.clone()); + let mic_lock = Arc::new(feeds.mic.lock().await?); + let camera_lock = match &feeds.camera { + Some(camera) => Some(Arc::new(camera.lock().await?)), + None => None, + }; + // Real cameras and microphones are already streaming when a recording + // starts. + tokio::time::sleep(Duration::from_millis(300)).await; + + let project = scratch.join("recording.cap"); + let defaults = RecordingDefaults::default(); + let gate = spec.gated.then(cap_recording::RecordingStartGate::new); + let (handle, max_fps) = match spec.mode { + Mode::Studio { fragmented, .. } => { + let (builder, max_fps) = studio_builder( + project.clone(), + target, + excluded, + fragmented, + camera, + spec.system_audio, + ); + let mut builder = builder; + if spec.mic { + builder = builder.with_mic_feed(mic_lock.clone()); + } + if let Some(gate) = &gate { + builder = builder.with_start_gate(gate.clone()); + } + if let Some(lock) = &camera_lock { + builder = builder.with_camera_feed(lock.clone()); + } + ( + Handle::Studio(builder.build(Some(shareable.clone())).await?), + max_fps, + ) + } + Mode::Instant => { + let builder = instant_recording::Actor::builder(project.clone(), target) + .with_system_audio(spec.system_audio) + .with_max_output_size(defaults.instant_mode_max_resolution) + .with_excluded_windows(excluded); + let builder = if spec.mic { + builder.with_mic_feed(mic_lock.clone()) + } else { + builder + }; + let builder = match &gate { + Some(gate) => builder.with_start_gate(gate.clone()), + None => builder, + }; + ( + Handle::Instant(builder.build(Some(shareable.clone())).await?), + cap_recording::DEFAULT_INSTANT_MODE_FPS, + ) + } + Mode::StudioCrashed => { + let (builder, max_fps) = studio_builder( + project.clone(), + target, + excluded, + true, + false, + spec.system_audio, + ); + let builder = builder.with_mic_feed(mic_lock.clone()); + ( + Handle::Studio(builder.build(Some(shareable.clone())).await?), + max_fps, + ) + } + }; + let started_host = match &gate { + Some(gate) => { + // The app arms the gate when its start cue finishes playing. + tokio::time::sleep(Duration::from_millis(START_CUE_MS)).await; + let at = host_now(); + gate.arm(); + at + } + None => host_now(), + }; + feeds.started.store(started_host, Ordering::Release); + tokio::task::block_in_place(|| stimulus.send("go"))?; + let waited = tokio::task::block_in_place(|| { + stimulus.wait_for("done", Duration::from_secs_f64(spec.seconds + 20.0)) + }); + tokio::time::sleep(Duration::from_millis(400)).await; + let stopped_host = host_now(); + let crashed = if spec.mode == Mode::StudioCrashed { + let snapshot = scratch.join("crash-snapshot.cap"); + let snapshot_started = host_now(); + let status = std::process::Command::new("cp") + .arg("-cR") + .arg(&project) + .arg(&snapshot) + .status()?; + anyhow::ensure!(status.success(), "snapshot of the live recording failed"); + case.note(format!( + "crash snapshot at {:.3}s, cloned in {:.3}s", + host_secs(snapshot_started, reference), + host_secs(host_now(), snapshot_started) + )); + Some(snapshot) + } else { + None + }; + let stop_started = std::time::Instant::now(); + let project_path = match handle { + Handle::Studio(handle) => { + let completed = handle.stop().await?; + let path = completed.project_path.clone(); + let finalize = std::time::Instant::now(); + let remux_path = path.clone(); + tokio::task::spawn_blocking(move || RecoveryManager::remux_if_needed(&remux_path)) + .await??; + case.note(format!( + "stop {:.2}s, remux {:.2}s", + (finalize - stop_started).as_secs_f64(), + finalize.elapsed().as_secs_f64() + )); + path + } + Handle::Instant(handle) => { + let mut completed = handle.stop().await?; + let finalize = std::time::Instant::now(); + finalize_instant(&mut completed).await?; + case.note(format!( + "stop {:.2}s, finalize {:.2}s", + (finalize - stop_started).as_secs_f64(), + finalize.elapsed().as_secs_f64() + )); + completed.project_path + } + }; + feeds.stop.store(true, Ordering::Release); + drop(mic_lock); + drop(camera_lock); + let join = |name: &str| anyhow::anyhow!("synthetic {name} source panicked"); + let mut sources = vec![ + tokio::task::block_in_place(|| feeds.mic_task.join()).map_err(|_| join("microphone"))?, + ]; + let mic_truth = if spec.mic { + feeds.mic_truth + } else { + Truth::new() + }; + let camera_truth = match feeds.camera_task { + Some(task) => { + let (truth, dropped, trace) = + tokio::task::block_in_place(|| task.join()).map_err(|_| join("camera"))?; + sources.push(trace); + Some((truth, dropped)) + } + None => None, + }; + drop(feeds.camera); + let events = tokio::task::block_in_place(|| stimulus.finish())?; + let sentinel = SentinelLog::read(&scratch.join(SENTINEL_FILE)).map_err(|e| format!("{e:#}")); + waited?; + let (project_path, loss_budget) = match crashed { + Some(snapshot) => { + let recovery_copy = scratch.join("crashed.cap"); + let status = std::process::Command::new("cp") + .arg("-cR") + .arg(&snapshot) + .arg(&recovery_copy) + .status()?; + anyhow::ensure!( + status.success(), + "copy of the preserved crash snapshot failed" + ); + let started = std::time::Instant::now(); + let recovered = tokio::task::spawn_blocking(move || { + let incomplete = RecoveryManager::inspect_recording(&recovery_copy) + .ok_or_else(|| anyhow::anyhow!("crash snapshot is not recoverable"))?; + RecoveryManager::recover(&incomplete).map_err(|e| anyhow::anyhow!("recover: {e}")) + }) + .await??; + case.note(format!( + "recovered the crash snapshot in {:.2}s", + started.elapsed().as_secs_f64() + )); + (recovered.project_path, Some(CRASH_LOSS_BUDGET)) + } + None => (project_path, None), + }; + Ok(Recorded { + project: project_path, + started: host_secs(started_host, reference), + stopped: host_secs(stopped_host, reference), + stimulus: events, + mic_truth, + camera: camera_truth, + max_fps, + mic_delay: spec.mic_delay, + holds: spec.holds.clone(), + stimulus_fps: spec.stimulus_fps, + sources, + sentinel, + loss_budget, + }) +} + +/// Seconds of capture a crash may cost: the fragment being written (Studio +/// fragments are 2 s, see `MacOSFragmentedM4SMuxerConfig`) plus the encoder +/// pipeline still in flight. +const CRASH_LOSS_BUDGET: f64 = 3.0; + +async fn finalize_instant( + recording: &mut instant_recording::CompletedRecording, +) -> anyhow::Result<()> { + let completion = recording.clean_completion.take(); + let project = recording.project_path.clone(); + let output = project.join("content/output.mp4"); + let audio_dir = project.join("content/audio"); + let display_dir = project.join("content/display"); + let already = std::fs::metadata(&output).is_ok_and(|m| m.len() > 0) && !audio_dir.exists(); + if !already { + tokio::task::spawn_blocking(move || match completion { + Some(completion) => RecoveryManager::finalize_completed_instant_output( + &display_dir, + &audio_dir, + &output, + completion, + ), + None => RecoveryManager::finalize_instant_output(&display_dir, &audio_dir, &output), + }) + .await??; + } + let meta = match &recording.meta { + InstantRecordingMeta::InProgress { .. } => InstantRecordingMeta::Failed { + error: "instant recording stopped before completion".to_string(), + }, + other => other.clone(), + }; + RecordingMeta { + platform: Some(cap_project::Platform::default()), + project_path: project.clone(), + pretty_name: "Cap Verify".to_string(), + sharing: None, + inner: RecordingMetaInner::Instant(meta), + upload: None, + } + .save_for_project() + .map_err(|e| anyhow::anyhow!("save meta: {e}"))?; + Ok(()) +} + +/// One recorded track located through recording-meta.json. +struct Track { + path: PathBuf, + /// `start_time` from the meta: the editor aligns tracks by it. + start: f64, +} + +struct Layout { + display: Track, + camera: Option, + mic: Option, + system: Option, + /// Instant mode muxes every source into the display file. + muxed_audio: bool, +} + +fn layout(project: &Path, case: &mut Case) -> anyhow::Result { + let meta = RecordingMeta::load_for_project(project) + .map_err(|e| anyhow::anyhow!("recording-meta.json: {e}"))?; + match &meta.inner { + RecordingMetaInner::Instant(instant) => { + if !matches!(instant, InstantRecordingMeta::Complete { .. }) { + case.problem( + "finalize", + format!("instant meta is not complete: {instant:?}"), + ); + } + Ok(Layout { + display: Track { + path: project.join("content/output.mp4"), + start: 0.0, + }, + camera: None, + mic: None, + system: None, + muxed_audio: true, + }) + } + RecordingMetaInner::Studio(studio) => { + let StudioRecordingMeta::MultipleSegments { inner } = &**studio else { + anyhow::bail!("expected a multi-segment studio recording"); + }; + if !matches!( + studio.status(), + cap_project::StudioRecordingStatus::Complete + ) { + case.problem( + "finalize", + format!("studio status after finalize: {:?}", studio.status()), + ); + } + if inner.segments.len() != 1 { + case.problem( + "finalize", + format!("{} segments, expected 1", inner.segments.len()), + ); + } + let segment = inner + .segments + .first() + .ok_or_else(|| anyhow::anyhow!("no segments"))?; + let mut display = meta.path(&segment.display.path); + if display.is_dir() { + case.problem( + "finalize", + format!( + "display path still points at fragments after finalize: {}", + display.display() + ), + ); + display = display.with_extension("mp4"); + } + // The editor places each track at its start_time; without one it + // aligns the track at 0, offsetting it by its start-up delay. + let unplaced: Vec<&str> = [ + ("display", Some(segment.display.start_time)), + ("camera", segment.camera.as_ref().map(|c| c.start_time)), + ("mic", segment.mic.as_ref().map(|m| m.start_time)), + ( + "system audio", + segment.system_audio.as_ref().map(|s| s.start_time), + ), + ] + .into_iter() + .filter_map(|(name, start)| matches!(start, Some(None)).then_some(name)) + .collect(); + if !unplaced.is_empty() { + case.problem( + "start-time", + format!( + "recording-meta.json has no start_time for {}: the editor aligns them at 0 instead of where they started", + unplaced.join(", ") + ), + ); + } + let track = |path: &relative_path::RelativePathBuf, start: Option| Track { + path: meta.path(path), + start: start.unwrap_or(0.0), + }; + Ok(Layout { + display: track(&segment.display.path, segment.display.start_time), + camera: segment + .camera + .as_ref() + .map(|c| track(&c.path, c.start_time)), + mic: segment.mic.as_ref().map(|m| track(&m.path, m.start_time)), + system: segment + .system_audio + .as_ref() + .map(|s| track(&s.path, s.start_time)), + muxed_audio: false, + }) + .map(|mut l| { + l.display.path = display; + l + }) + } + } +} + +/// Offsets of every observed marker: recorded (editor) time minus host truth. +#[derive(Default, Clone)] +struct Offsets(Vec<(f64, f64)>); + +impl Offsets { + fn median(&self) -> Option { + let mut v: Vec = self.0.iter().map(|(_, d)| *d).collect(); + if v.is_empty() { + return None; + } + v.sort_by(f64::total_cmp); + Some(v[v.len() / 2]) + } + + fn last_truth(&self) -> Option { + self.0 + .iter() + .map(|(t, _)| *t) + .fold(None, |m, t| Some(m.map_or(t, |m: f64| m.max(t)))) + } + + fn spread(&self) -> f64 { + let v: Vec = self.0.iter().map(|(_, d)| *d).collect(); + let max = v.iter().copied().fold(f64::MIN, f64::max); + let min = v.iter().copied().fold(f64::MAX, f64::min); + if v.is_empty() { 0.0 } else { max - min } + } + + /// Least-squares slope of offset over truth time, in ms per minute. + fn drift_ms_per_min(&self) -> f64 { + let n = self.0.len() as f64; + if n < 3.0 { + return 0.0; + } + let mx = self.0.iter().map(|(t, _)| t).sum::() / n; + let my = self.0.iter().map(|(_, d)| d).sum::() / n; + let sxy: f64 = self.0.iter().map(|(t, d)| (t - mx) * (d - my)).sum(); + let sxx: f64 = self.0.iter().map(|(t, _)| (t - mx).powi(2)).sum(); + if sxx == 0.0 { + 0.0 + } else { + sxy / sxx * 60_000.0 + } + } + + fn describe(&self) -> String { + format!( + "n={} median {:+.1} ms, spread {:.1} ms, drift {:+.2} ms/min", + self.0.len(), + self.median().unwrap_or(f64::NAN) * 1000.0, + self.spread() * 1000.0, + self.drift_ms_per_min() + ) + } +} + +/// Capture faults injected into a finished real recording's decoded screen +/// frames, after the fact, to prove on this very host and capture that the +/// oracle still reports them: what a pipeline that drops, freezes, re-emits +/// or delays frames would have written. Thresholds are the production ones. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Injection { + /// Three consecutive presented frames never reach the file. + DropRun, + /// The frame before them is repeated in their place (a frozen capture). + Freeze, + /// An already written frame is emitted again five frames later. + Stale, + /// The whole screen track lands 150 ms late against the audio: more than + /// the width of any screen sync window, so a working sync measurement + /// reports it from any in-window baseline. + Late, + /// The last second of the screen track is lost (a final segment that + /// never reached the file). + TruncateEnd, + /// The first second of the screen track is lost (a late start). + LateStart, +} + +thread_local! { + static INJECTION: std::cell::Cell> = const { std::cell::Cell::new(None) }; + /// Whether the last injection found frames to act on. + static INJECTED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +const INJECTIONS: [Injection; 6] = [ + Injection::DropRun, + Injection::Freeze, + Injection::Stale, + Injection::Late, + Injection::TruncateEnd, + Injection::LateStart, +]; +const INJECTED_LATENESS: f64 = 0.15; + +impl Injection { + fn label(self) -> &'static str { + match self { + Injection::DropRun => "drop 3 consecutive frames", + Injection::Freeze => "freeze over 3 frames", + Injection::Stale => "re-emit a stale frame", + Injection::Late => "screen 150 ms late", + Injection::TruncateEnd => "lose the last second", + Injection::LateStart => "start one second late", + } + } + + /// Class and problem prefix the oracle must report for this fault. + fn expected(self) -> (&'static str, &'static str) { + match self { + Injection::DropRun | Injection::Freeze => ("missing", "screen:"), + Injection::Stale => ("timestamps", "screen:"), + Injection::Late => ("sync-screen-mic", "screen vs"), + Injection::TruncateEnd | Injection::LateStart => ("coverage", "screen:"), + } + } + + /// First of three consecutive presented indices, around 60 % of the + /// recording, that were all captured and each stayed on screen long + /// enough that the capture rate guarantees a sample. + fn target(frames: &[(f64, Readout)], check: &VideoCheck) -> Option { + let mut seen: Vec = Vec::new(); + for (_, readout) in frames { + if let Readout::Marker { marker, .. } = readout + && marker.kind == StreamKind::Screen + && seen.last() != Some(&marker.index) + { + seen.push(marker.index); + } + } + let captured: std::collections::BTreeSet = seen.iter().copied().collect(); + let start = seen.len() * 3 / 5; + seen[start.min(seen.len())..].iter().copied().find(|&i| { + i > 1 + && captured.contains(&(i - 1)) + && (i..i + 3).all(|j| { + captured.contains(&j) && check.truth.contains_key(&j) && !(check.may_skip)(j) + }) + }) + } + + fn apply(self, frames: &mut Vec<(f64, Readout)>, check: &VideoCheck) { + let before = frames.clone(); + self.mutate(frames, check); + INJECTED.set(*frames != before); + } + + fn mutate(self, frames: &mut Vec<(f64, Readout)>, check: &VideoCheck) { + let index_of = |r: &Readout| match r { + Readout::Marker { marker, .. } => Some(marker.index), + _ => None, + }; + match self { + Injection::Late => { + for (pts, _) in frames.iter_mut() { + *pts += INJECTED_LATENESS; + } + } + Injection::TruncateEnd => { + if let Some(last) = frames.last().map(|(pts, _)| *pts) { + frames.retain(|(pts, _)| *pts < last - 1.0); + } + } + Injection::LateStart => { + if let Some(first) = frames.first().map(|(pts, _)| *pts) { + frames.retain(|(pts, _)| *pts > first + 1.0); + } + } + Injection::DropRun | Injection::Freeze => { + let Some(first) = Self::target(frames, check) else { + return; + }; + let run = first..first + 3; + if self == Injection::DropRun { + frames.retain(|(_, r)| !index_of(r).is_some_and(|i| run.contains(&i))); + } else if let Some(before) = frames + .iter() + .rev() + .find(|(_, r)| index_of(r) == Some(first - 1)) + .map(|(_, r)| r.clone()) + { + for (_, r) in frames.iter_mut() { + if index_of(r).is_some_and(|i| run.contains(&i)) { + *r = before.clone(); + } + } + } + } + Injection::Stale => { + let Some(first) = Self::target(frames, check) else { + return; + }; + let Some(stale) = frames + .iter() + .find(|(_, r)| index_of(r) == Some(first)) + .map(|(_, r)| r.clone()) + else { + return; + }; + let Some(after) = frames + .iter() + .position(|(_, r)| index_of(r).is_some_and(|i| i >= first + 5)) + else { + return; + }; + if after + 1 < frames.len() { + let pts = (frames[after].0 + frames[after + 1].0) / 2.0; + frames.insert(after + 1, (pts, stale)); + } + } + } + } +} + +fn self_check_enabled() -> bool { + match std::env::var("CAP_VERIFY_SELF_CHECK").as_deref() { + Ok("0") | Ok("false") => false, + Ok(_) => true, + Err(_) => std::env::var_os("CI").is_some(), + } +} + +/// Re-analyzes the real recording with each injected fault and fails the +/// case when the oracle does not report one: proof, per run, that a starved +/// or otherwise unusual capture has not blinded the checks. +fn self_check(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()> { + let mut caught = Vec::new(); + for injection in INJECTIONS { + let (class, prefix) = injection.expected(); + if injection == Injection::TruncateEnd && rec.loss_budget.is_some() { + caught.push(format!( + "{}: n/a (a crash case may lose its end within its loss budget)", + injection.label() + )); + continue; + } + if injection == Injection::Late && case.unobserved.contains_key(class) { + caught.push(format!("{}: unobserved (track absent)", injection.label())); + continue; + } + if injection == Injection::Late + && case.classes.iter().any(|c| c.starts_with("sync-screen-")) + { + caught.push(format!("{}: n/a (already out of sync)", injection.label())); + continue; + } + let mut probe = Case::new(format!("{} (self-check: {})", case.id, injection.label())); + INJECTED.set(false); + INJECTION.set(Some(injection)); + let result = analyze(rec, reference, &mut probe); + INJECTION.set(None); + result?; + let new: Vec<&String> = probe + .problems + .iter() + .filter(|p| p.starts_with(prefix) && !case.problems.contains(p)) + .collect(); + if !INJECTED.get() { + case.problem( + "oracle", + format!( + "self-check: no frames to inject '{}' into: no 3 consecutive pictures the capture rate must sample", + injection.label() + ), + ); + } else if probe.classes.contains(class) && !new.is_empty() { + caught.push(format!( + "{}: caught ({class}: {})", + injection.label(), + new[0].chars().take(110).collect::() + )); + } else { + case.problem( + "oracle", + format!( + "self-check: injected fault '{}' was not reported (expected class {class})", + injection.label() + ), + ); + } + } + case.note(format!("self-check: {}", caught.join("; "))); + Ok(()) +} + +fn first_presented(events: &[StimulusEvent], reference: u64) -> Truth { + let mut truth = Truth::new(); + for event in events { + if let StimulusEvent::Frame { index, host, .. } = event { + truth.entry(*index).or_insert(host_secs(*host, reference)); + } + } + truth +} + +struct VideoCheck<'a> { + name: &'a str, + kind: StreamKind, + truth: &'a Truth, + /// Indices that were never delivered to the recorder. + not_sent: &'a [u32], + /// Indices the source showed too briefly for the capture rate to + /// guarantee a sample (screen only). + may_skip: Box bool + 'a>, + allow_duplicates: bool, + /// Pictures that must each be in the recording, outside any loss budget: + /// the first picture after each scripted hold (a change after a static + /// screen, the moment a viewer is waiting for). + required: Vec<(u32, String)>, + /// (recording start, end of the kept span) in truth seconds: the + /// recording must hold the pictures shown from shortly after the start + /// to the end (see `START_COVERAGE_SECS`). + window: (f64, f64), + /// Frame times (truth seconds, sorted) of the stimulus's own, + /// independent ScreenCaptureKit stream (screen only; empty otherwise). + observed: Vec, +} + +/// Margin around a picture's time on screen within which the independent +/// observer must have frames for its silence in between to count. +const OBSERVER_COVER_SECS: f64 = 0.5; + +impl VideoCheck<'_> { + /// A picture the capture service never delivered to any client: the + /// stimulus's own capture stream received no frame while it was on + /// screen, although that stream delivered frames on both sides of it + /// (so it covered exactly that window). + fn dropped_for_every_client(&self, index: u32) -> bool { + let Some(&shown) = self.truth.get(&index) else { + return false; + }; + let Some((_, &replaced)) = self.truth.range(index + 1..).next() else { + return false; + }; + let before = self + .observed + .iter() + .any(|&t| t < shown && t >= shown - OBSERVER_COVER_SECS); + let after = self + .observed + .iter() + .any(|&t| t > replaced && t <= replaced + OBSERVER_COVER_SECS); + let during = self.observed.iter().any(|&t| t >= shown && t <= replaced); + before && after && !during + } +} + +/// How late after the recording starts its first picture may be: the +/// documented start-up drops (`STARTUP_DROPS` frames at the slowest capture +/// rate, 30 fps: 200 ms) plus the display's presentation latency (up to +/// 75 ms, see `sync_checks`). Measured on hosted macOS 15 and 26: the first +/// recorded picture is the first one presented, 9 to 69 ms after the start. +const START_COVERAGE_SECS: f64 = 0.3; + +fn check_video(case: &mut Case, track: &Track, check: VideoCheck) -> anyhow::Result { + let mut frames: Vec<(f64, Readout)> = Vec::new(); + media::decode_video(&track.path, |picture| { + let readout = video_code::read_stream(&picture.view(), check.kind); + frames.push((picture.pts_secs, readout)); + })?; + if check.kind == StreamKind::Screen + && let Some(injection) = INJECTION.get() + { + injection.apply(&mut frames, &check); + } + let name = check.name; + if frames.is_empty() { + case.problem("missing", format!("{name}: no frames")); + return Ok(Offsets::default()); + } + let mut bad = Vec::new(); + let mut indices = Vec::new(); + for (i, (pts, readout)) in frames.iter().enumerate() { + match readout { + Readout::Marker { marker, .. } if marker.kind == check.kind && marker.segment == 0 => { + indices.push((*pts, marker.index)); + } + other => bad.push(format!("frame {i} at {pts:.3}s: {other:?}")), + } + } + if !bad.is_empty() { + case.problem( + "unreadable", + format!( + "{name}: {} of {} frames carry no valid marker, first: {}", + bad.len(), + frames.len(), + bad.iter().take(3).cloned().collect::>().join("; ") + ), + ); + } + let mut backwards = Vec::new(); + let mut backwards_at = Vec::new(); + let mut duplicates = 0; + let mut pts_problems = Vec::new(); + for w in indices.windows(2) { + let ((pa, a), (pb, b)) = (w[0], w[1]); + if pb <= pa { + pts_problems.push(format!("{pa:.4}->{pb:.4}")); + } + if b < a { + backwards.push(format!("{a}->{b} at {pb:.3}s")); + backwards_at.extend([a, b].iter().filter_map(|i| check.truth.get(i).copied())); + } else if b == a { + duplicates += 1; + } + } + if !pts_problems.is_empty() { + case.problem( + "timestamps", + format!( + "{name}: non-increasing pts x{}: {}", + pts_problems.len(), + pts_problems + .iter() + .take(3) + .cloned() + .collect::>() + .join(", ") + ), + ); + } + if !backwards.is_empty() { + case.problem( + "timestamps", + format!( + "{name}: frame index went backwards x{}: {}", + backwards.len(), + backwards + .iter() + .take(3) + .cloned() + .collect::>() + .join(", ") + ), + ); + implicate(case, backwards_at); + } + if duplicates > 0 && !check.allow_duplicates { + case.problem( + "timestamps", + format!("{name}: {duplicates} duplicated frames"), + ); + } + let mut first_seen: BTreeMap = BTreeMap::new(); + for &(pts, index) in &indices { + first_seen.entry(index).or_insert(pts); + } + let unknown: Vec = first_seen + .keys() + .filter(|i| !check.truth.contains_key(i)) + .copied() + .collect(); + if !unknown.is_empty() { + case.problem( + "unexpected", + format!( + "{name}: {} recorded indices were never produced: {:?}", + unknown.len(), + &unknown[..unknown.len().min(8)] + ), + ); + } + let (Some(&lo), Some(&hi)) = (first_seen.keys().next(), first_seen.keys().next_back()) else { + return Ok(Offsets::default()); + }; + // Start and end coverage, from the stimulus's truth: missing pictures + // between the first and last recorded ones are counted below, so a late + // start or a lost final segment needs its own check. + let countable = |i: &u32| !check.not_sent.contains(i) && !(check.may_skip)(*i); + let (start, end) = check.window; + if let Some((&first_due, &due_at)) = check + .truth + .iter() + .find(|(i, t)| **t >= start + START_COVERAGE_SECS && countable(i)) + && lo > first_due + { + let late = check.truth.get(&lo).map_or(f64::NAN, |t| t - start); + case.measure( + "coverage", + &format!("{name}-start-late-ms"), + (late * 1000.0).round(), + ); + case.problem( + "coverage", + format!( + "{name}: the recording starts {:.0} ms after the recording start (first recorded index {lo}); index {first_due}, shown {:.0} ms after the start, is missing (allowed {:.0} ms)", + late * 1000.0, + (due_at - start) * 1000.0, + START_COVERAGE_SECS * 1000.0 + ), + ); + implicate(case, Some(due_at)); + } + if let Some((&last_due, &due_at)) = check + .truth + .iter() + .rev() + .find(|(i, t)| **t <= end && **t >= start && countable(i)) + && hi < last_due + { + let lost = check + .truth + .range(hi + 1..=last_due) + .filter(|(i, _)| countable(i)) + .count(); + case.measure("coverage", &format!("{name}-end-lost"), lost as f64); + case.problem( + "coverage", + format!( + "{name}: the recording ends at index {hi}; {lost} pictures shown before the stop are missing (last due: index {last_due}, {:.0} ms before the end of the kept span)", + (end - due_at) * 1000.0 + ), + ); + implicate(case, Some(due_at)); + } + let mut skipped = Vec::new(); + let mut startup = Vec::new(); + let mut explained = 0; + let startup_end = check.truth.get(&lo).copied().unwrap_or(f64::MIN) + STARTUP_SECS; + for (&index, &truth) in check.truth.range(lo + 1..hi) { + if first_seen.contains_key(&index) || check.not_sent.contains(&index) { + continue; + } + if (check.may_skip)(index) { + explained += 1; + } else if truth < startup_end { + startup.push(index); + } else { + skipped.push(index); + } + } + // Pictures the capture service dropped for every client (approved + // policy): counted and reported, never a threshold change. + let (for_every_client, rest): (Vec, Vec) = skipped + .into_iter() + .partition(|&index| check.dropped_for_every_client(index)); + skipped = rest; + if !for_every_client.is_empty() { + case.note(format!( + "capture-service drops: {name}: {} missing pictures also never reached the stimulus's own capture stream, which covered their window: {:?}", + for_every_client.len(), + for_every_client + )); + } + if startup.len() > STARTUP_DROPS { + skipped.extend(startup.iter().copied()); + } else if !startup.is_empty() { + case.note(format!( + "{name}: {} start-up drops {:?} (allowed {STARTUP_DROPS} in the first {STARTUP_SECS}s)", + startup.len(), + startup + )); + } + // The truth for a screen frame is when the stimulus handed it to the + // window server, not when the display showed it: on a loaded virtual + // display a frame can be replaced before it is ever composited. A small + // budget of isolated misses is that; a pipeline that drops frames loses + // runs of them or many. + let budget = if check.kind == StreamKind::Screen { + ((first_seen.len() as f64) * SCREEN_MISS_FRACTION).ceil() as usize + } else { + 0 + }; + let longest_run = skipped + .iter() + .fold((0usize, 0usize, None::), |(best, run, prev), &i| { + let run = if prev.is_some_and(|p| p + 1 == i) { + run + 1 + } else { + 1 + }; + (best.max(run), run, Some(i)) + }) + .0; + for (index, what) in &check.required { + if check.truth.contains_key(index) + && *index > lo + && *index < hi + && !first_seen.contains_key(index) + { + case.count("missing-after-hold", &format!("{name}-pictures"), 1.0); + case.problem( + "missing-after-hold", + format!("{name}: {what} (index {index}) is missing from the recording"), + ); + implicate(case, check.truth.get(index).copied()); + } + } + if skipped.len() > budget || longest_run >= 3 { + case.count("missing", &format!("{name}-frames"), skipped.len() as f64); + case.problem( + "missing", + format!( + "{name}: {} produced frames missing from the recording (budget {budget}, longest run {longest_run}): {:?}", + skipped.len(), + &skipped[..skipped.len().min(12)] + ), + ); + implicate( + case, + skipped.iter().filter_map(|i| check.truth.get(i).copied()), + ); + } else if !skipped.is_empty() { + case.note(format!( + "{name}: {} isolated misses within budget {budget}: {:?}", + skipped.len(), + skipped + )); + } + if let Some(dir) = std::env::var_os("CAP_VERIFY_DUMP").filter(|_| INJECTION.get().is_none()) { + let rows: Vec = indices + .iter() + .map(|(pts, i)| { + let truth = check.truth.get(i).copied().unwrap_or(f64::NAN); + format!("{pts:.6},{i},{truth:.6},{:.6}", pts + track.start - truth) + }) + .collect(); + let _ = std::fs::write( + Path::new(&dir).join(format!( + "{}-r{}-{name}.csv", + case.id.replace('/', "-"), + ATTEMPT.load(Ordering::Relaxed) + )), + format!("pts,index,truth,offset\n{}\n", rows.join("\n")), + ); + } + let mut offsets = Offsets::default(); + let mut snapped = Vec::new(); + for (&index, &pts) in &first_seen { + if index == 0 { + continue; + } + let Some(&truth) = check.truth.get(&index) else { + continue; + }; + // The documented cadence rule (#2573): after a source gap the first + // frame may stay on the skipped slot, so it can land up to the gap + // early. It is reported, and excluded from the steady-state offsets. + let gap = (0..index) + .rev() + .take_while(|i| check.not_sent.contains(i)) + .count(); + if gap > 0 { + snapped.push((index, pts + track.start - truth)); + continue; + } + offsets.0.push((truth, pts + track.start - truth)); + } + let first_truth = first_seen + .keys() + .filter_map(|i| check.truth.get(i)) + .copied() + .fold(f64::MAX, f64::min); + // Steady offsets on each side of the drift anchor (see sync_checks). + let side_median = |past: bool| { + Offsets( + offsets + .0 + .iter() + .copied() + .filter(|(t, _)| (*t >= first_truth + DRIFT_ANCHOR_SECS) == past) + .collect(), + ) + .median() + }; + if let Some(median) = offsets.median() { + let period = 1.0 / f64::from(CAMERA_FPS); + for &(index, offset) in &snapped { + let past = check + .truth + .get(&index) + .is_some_and(|t| *t >= first_truth + DRIFT_ANCHOR_SECS); + let median = side_median(past).unwrap_or(median); + let early = median - offset; + let gap = (0..index) + .rev() + .take_while(|i| check.not_sent.contains(i)) + .count() as f64; + let past_anchor = past; + // Late past the drift anchor is the processing-clock stamping + // (class camera-clock); early beyond the gap, or late before the + // anchor, is a timestamp defect. + let class = if early < -0.004 && past_anchor { + "camera-clock" + } else { + "timestamps" + }; + if early < -0.004 || early > gap * period + 0.004 { + case.problem(class, format!( + "{name}: frame {index} after a {gap}-frame source gap is {:+.1} ms off the steady offset (allowed 0..{:.1} early)", + -early * 1000.0, + gap * period * 1000.0 + )); + implicate(case, check.truth.get(&index).copied()); + } + } + if !snapped.is_empty() { + case.note(format!( + "{name}: first frames after source gaps: {}", + snapped + .iter() + .map(|(i, o)| format!("{i} {:+.1} ms", (o - median) * 1000.0)) + .collect::>() + .join(", ") + )); + } + } + case.note(format!( + "{name}: {} frames, indices {lo}..={hi}, {} distinct, {duplicates} repeats, {} skips explained by display timing, offset {}", + frames.len(), + first_seen.len(), + explained, + offsets.describe() + )); + Ok(offsets) +} + +struct AudioCheck<'a> { + name: &'a str, + source: AudioSource, + truth: &'a Truth, + /// Truth window inside which every marker must be present. + window: (f64, f64), +} + +fn check_audio( + case: &mut Case, + decoded: &media::DecodedAudio, + start: f64, + check: AudioCheck, +) -> Offsets { + let name = check.name; + let mono = decoded.mono(); + let located = audio_code::locate_markers(&mono, decoded.sample_rate, check.source); + if check.source == AudioSource::System && located.is_empty() && !require_system_audio() { + // Hosted runners' virtual output devices hand ScreenCaptureKit pure + // digital silence; a developer Mac checks this stream. + case.note(format!( + "{name}: no system audio reached the recording (runner output is not capturable); not checked" + )); + return Offsets::default(); + } + let mut found: BTreeMap = BTreeMap::new(); + let mut damaged = 0; + let mut repeated = Vec::new(); + let mut cut = Vec::new(); + let track_secs = mono.len() as f64 / f64::from(decoded.sample_rate); + for marker in &located { + let at = marker.time_secs + decoded.start_secs + start; + // Starting or stopping a recording may cut a marker; the rest of + // the track can be silence (encoder padding). A cut marker's payload + // is meaningless: undecodable, or a CRC-valid index of near-silence + // (index 0, chirp correlation 0.56, run 38780355968). Markers inside + // the track must decode to a known index. + let interior = marker.time_secs > 0.15 + && marker.time_secs + audio_code::MARKER_SECS < track_secs - 0.15; + let Some(index) = marker.index else { + if interior { + damaged += 1; + } + continue; + }; + if !interior && !check.truth.contains_key(&index) { + cut.push((index, marker.time_secs, marker.correlation)); + continue; + } + if found.insert(index, at).is_some() { + repeated.push(index); + } + } + if !cut.is_empty() { + case.note(format!( + "{name}: ignored markers cut by the track's start or end (index, at, chirp correlation): {}", + cut.iter() + .map(|(i, t, c)| format!("({i}, {t:.3}s, {c:.3})")) + .collect::>() + .join(", ") + )); + } + if damaged > 0 { + case.count("unreadable", &format!("{name}-markers"), f64::from(damaged)); + case.problem( + "unreadable", + format!("{name}: {damaged} markers inside the track located but undecodable"), + ); + } + if !repeated.is_empty() { + case.problem( + "timestamps", + format!("{name}: markers repeated {repeated:?}"), + ); + } + let unknown: Vec = found + .keys() + .filter(|i| !check.truth.contains_key(i)) + .copied() + .collect(); + if !unknown.is_empty() { + case.problem( + "unexpected", + format!("{name}: unexpected marker indices {unknown:?}"), + ); + } + let missing: Vec = check + .truth + .iter() + .filter(|(i, t)| **t > check.window.0 && **t < check.window.1 && !found.contains_key(i)) + .map(|(i, _)| *i) + .collect(); + if !missing.is_empty() { + case.count("missing", &format!("{name}-markers"), missing.len() as f64); + for i in &missing { + if let Some(t) = check.truth.get(i) { + case.measure( + "missing", + &format!("{name}-after-start-s"), + ((t - check.window.0) * 100.0).round() / 100.0, + ); + } + } + case.problem( + "missing", + format!( + "{name}: {} markers inside the recording window are missing: {:?}", + missing.len(), + &missing[..missing.len().min(12)] + ), + ); + implicate( + case, + missing.iter().filter_map(|i| check.truth.get(i).copied()), + ); + } + let mut order = found.iter().collect::>(); + order.sort_by(|a, b| a.1.total_cmp(b.1)); + if order.windows(2).any(|w| w[1].0 < w[0].0) { + case.problem("timestamps", format!("{name}: markers out of order")); + } + if let Some(dir) = std::env::var_os("CAP_VERIFY_DUMP").filter(|_| INJECTION.get().is_none()) { + let rows: Vec = located + .iter() + .map(|m| { + let at = m.time_secs + decoded.start_secs + start; + let truth = m + .index + .and_then(|i| check.truth.get(&i)) + .copied() + .unwrap_or(f64::NAN); + format!("{at:.6},{:?},{truth:.6},{:.3}", m.index, m.correlation) + }) + .chain( + check + .truth + .iter() + .map(|(i, t)| format!("truth,{i},{t:.6},")), + ) + .collect(); + let _ = std::fs::write( + Path::new(&dir).join(format!( + "{}-r{}-{name}.csv", + case.id.replace('/', "-"), + ATTEMPT.load(Ordering::Relaxed) + )), + format!( + "at,index,truth,correlation (start {start:.6}, file start {:.6}, window {:.3}..{:.3})\n{}\n", + decoded.start_secs, + check.window.0, + check.window.1, + rows.join("\n") + ), + ); + } + let mut offsets = Offsets::default(); + for (index, at) in &found { + if let Some(&truth) = check.truth.get(index) { + offsets.0.push((truth, at - truth)); + } + } + let peak = mono.iter().fold(0.0f32, |m, v| m.max(v.abs())); + case.note(format!( + "{name}: {} markers ({} expected in window), {:.2}s at peak {peak:.3}, offset {}", + found.len(), + check + .truth + .values() + .filter(|t| **t > check.window.0 && **t < check.window.1) + .count(), + mono.len() as f64 / f64::from(decoded.sample_rate), + offsets.describe() + )); + offsets +} + +/// The first presented picture after each scripted hold. +fn after_holds(rec: &Recorded, truth: &Truth) -> Vec<(u32, String)> { + rec.holds + .iter() + .filter_map(|&(start, length)| { + let first = ((start + length) * rec.stimulus_fps).ceil() as u32 + 1; + truth.range(first..).next().map(|(&index, _)| { + ( + index, + format!("first picture after the {length} s hold at {start} s"), + ) + }) + }) + .collect() +} + +fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<()> { + let layout = layout(&rec.project, case)?; + let screen_truth = first_presented(&rec.stimulus, reference); + let mut observed: Vec = rec + .stimulus + .iter() + .filter_map(|e| match e { + StimulusEvent::Observed { host } => Some(host_secs(*host, reference)), + _ => None, + }) + .collect(); + observed.sort_by(f64::total_cmp); + let presented: Vec<(u32, f64)> = screen_truth.iter().map(|(i, t)| (*i, *t)).collect(); + let capture_interval = 1.0 / f64::from(rec.max_fps); + // SCK delivers a changed display at most once per minimum frame + // interval. A picture is guaranteed to be sampled only when it stays on + // screen for a full interval plus one display refresh of phase. + let refresh = 1.0 / 60.0; + let on_screen: BTreeMap = presented + .windows(2) + .map(|w| (w[0].0, w[1].1 - w[0].1)) + .collect(); + // The span whose pictures and markers must all be in the recording. A + // crash-recovery case has its own end check against its loss budget + // (below), so the video end coverage does not apply there. + let end = rec.stopped - rec.loss_budget.unwrap_or(0.3); + let coverage_end = if rec.loss_budget.is_some() { + f64::NEG_INFINITY + } else { + end + }; + let screen = check_video( + case, + &layout.display, + VideoCheck { + name: "screen", + kind: StreamKind::Screen, + truth: &screen_truth, + not_sent: &[], + may_skip: Box::new(|index| { + on_screen + .get(&index) + .is_some_and(|d| *d < capture_interval + refresh) + }), + allow_duplicates: true, + required: after_holds(rec, &screen_truth), + window: (rec.started, coverage_end), + observed, + }, + )?; + let camera = match (&layout.camera, &rec.camera) { + (Some(track), Some((truth, dropped))) if track.path.is_file() => { + case.note(format!("camera: dropped at source {dropped:?}")); + Some(check_video( + case, + track, + VideoCheck { + name: "camera", + kind: StreamKind::Camera, + truth, + not_sent: dropped, + may_skip: Box::new(|_| false), + allow_duplicates: false, + required: Vec::new(), + window: (rec.started, coverage_end), + observed: Vec::new(), + }, + )?) + } + (_, Some(_)) => { + case.problem( + "missing", + "camera feed attached but no camera track recorded", + ); + case.required_track_absent("camera"); + None + } + _ => None, + }; + let window = (rec.started + 0.3, end); + if rec.loss_budget.is_some() { + let last_presented = screen_truth.range(1..).next_back().map(|(_, t)| *t); + match (screen.last_truth(), last_presented) { + (Some(last), Some(presented)) if last < end.min(presented) => { + case.measure( + "missing", + "screen-lost-s", + ((rec.stopped - last) * 100.0).round() / 100.0, + ); + case.problem( + "missing", + format!( + "screen: recovery kept frames up to {:.2}s before the crash, budget {:.1}s", + rec.stopped - last, + rec.loss_budget.unwrap_or_default() + ), + ); + } + _ => {} + } + } + // ScreenCaptureKit starts delivering system audio some time after the + // stream starts (measured 0.2-0.6 s); Cap fills that span with silence. + let system_window = (rec.started + SYSTEM_AUDIO_STARTUP, window.1); + let mic_window = (window.0 + rec.mic_delay, window.1); + let system_truth: Truth = rec + .stimulus + .iter() + .filter_map(|e| match e { + StimulusEvent::Audio { index, host } => Some((*index, host_secs(*host, reference))), + _ => None, + }) + .collect(); + let audio_unavailable = rec.stimulus.iter().find_map(|e| match e { + StimulusEvent::AudioUnavailable { reason } => Some(reason.clone()), + _ => None, + }); + if let Some(reason) = &audio_unavailable { + case.note(format!("system audio stimulus unavailable: {reason}")); + if require_system_audio() { + case.problem( + "missing", + format!("system audio required but unavailable: {reason}"), + ); + } + } + let (mic, system) = if layout.muxed_audio { + let has_audio = ffmpeg::format::input(&layout.display.path)? + .streams() + .best(ffmpeg::media::Type::Audio) + .is_some(); + if has_audio { + let decoded = media::decode_audio(&layout.display.path, Some(48_000))?; + let mic = (!rec.mic_truth.is_empty()).then(|| { + check_audio( + case, + &decoded, + 0.0, + AudioCheck { + name: "mic", + source: AudioSource::Mic, + truth: &rec.mic_truth, + window: mic_window, + }, + ) + }); + let system = (!system_truth.is_empty()).then(|| { + check_audio( + case, + &decoded, + 0.0, + AudioCheck { + name: "system", + source: AudioSource::System, + truth: &system_truth, + window: system_window, + }, + ) + }); + (mic, system) + } else { + for (source, required) in [ + ("mic", !rec.mic_truth.is_empty()), + ("system", !system_truth.is_empty()), + ] { + if required { + case.problem("missing", format!("no {source} track recorded")); + case.required_track_absent(source); + } + } + (None, None) + } + } else { + let mic = match &layout.mic { + Some(track) if track.path.is_file() => { + let decoded = media::decode_audio(&track.path, Some(48_000))?; + Some(check_audio( + case, + &decoded, + track.start, + AudioCheck { + name: "mic", + source: AudioSource::Mic, + truth: &rec.mic_truth, + window: mic_window, + }, + )) + } + _ if rec.mic_truth.is_empty() => None, + _ => { + case.problem("missing", "no mic track recorded"); + case.required_track_absent("mic"); + None + } + }; + let system = match (&layout.system, system_truth.is_empty()) { + (Some(track), false) if track.path.is_file() => { + let decoded = media::decode_audio(&track.path, Some(48_000))?; + Some(check_audio( + case, + &decoded, + track.start, + AudioCheck { + name: "system", + source: AudioSource::System, + truth: &system_truth, + window: system_window, + }, + )) + } + (_, false) => { + case.problem( + "missing", + "system audio was playing but no system audio track was recorded", + ); + case.required_track_absent("system"); + None + } + _ => None, + }; + (mic, system) + }; + sync_checks( + rec.max_fps, + case, + &screen, + camera.as_ref(), + mic.as_ref(), + system.as_ref(), + ); + Ok(()) +} + +/// Tolerances, from first principles: +/// - Synthetic camera and microphone carry exact capture times, so their +/// offsets must be constant (spread within one audio chunk / encoder +/// rounding) and must agree with each other to within one camera frame +/// period's rounding. +/// - The screen truth is the time the stimulus handed a frame to the window +/// server; the display shows it at a later refresh, so the screen offset may +/// exceed the exact sources' by up to a few refresh intervals, never less. +fn sync_checks( + max_fps: u32, + case: &mut Case, + screen: &Offsets, + camera: Option<&Offsets>, + mic: Option<&Offsets>, + system: Option<&Offsets>, +) { + const EXACT_SPREAD: f64 = 0.004; + const CAMERA_SUSTAINED_FRAMES: usize = 3; + const CAMERA_VS_MIC: f64 = 1.0 / CAMERA_FPS as f64; + // Screen: the recorded frame may trail its hand-off by the display + // latency (up to 4 refreshes plus half a refresh of phase: the + // paravirtual runner display measured up to +54 ms) and, at 60 fps, the cadence ladder may + // place a variable-rate frame (20 fps content) up to two 16.7 ms slots + // early (measured floor: -29 ms on a developer Mac, -17 ms on hosted + // runners; at 30 fps capture the floor is +5 ms). + let screen_latency: (f64, f64) = if max_fps > 30 { + (-0.040, 0.075) + } else { + (-0.010, 0.075) + }; + const SYSTEM_VS_MIC: f64 = 0.030; + // System audio truth adds the output device's latency (measured -4 to + // +21 ms against the microphone on a developer Mac). + let slack = |name: &str| if name == "mic" { 0.0 } else { 0.015 }; + + let ms = |s: f64| s * 1000.0; + if let Some(mic) = mic + && mic.spread() > EXACT_SPREAD + { + case.measure( + "timestamps", + "mic-spread-ms", + (ms(mic.spread()) * 10.0).round() / 10.0, + ); + case.problem("timestamps", format!( + "mic: marker offsets vary by {:.1} ms (limit {:.1}): audio timing is not continuous", + ms(mic.spread()), + ms(EXACT_SPREAD) + )); + } + // Camera frames carry exact capture times. Until the drift anchor the + // recorder stamps them with those times, so their offsets must be + // constant: a frame more than three periods off, or three consecutive + // frames more than a period off (a ladder that froze a whole-slot lag, + // #2573: 66.7 ms for 22 frames), is a timestamp defect. Past the anchor + // the recorder stamps frames from the processing wall clock instead + // (`VideoDriftTracker::calculate_timestamp`), so every frame carries its + // processing delay; that is reported as class camera-clock, together + // with the same frame checks applied to those frames. + let camera_parts = camera.map(|camera| { + let first = camera.0.iter().map(|(t, _)| *t).fold(f64::MAX, f64::min); + let (pre, post): (Vec<_>, Vec<_>) = camera + .0 + .iter() + .copied() + .partition(|(t, _)| *t < first + DRIFT_ANCHOR_SECS); + (Offsets(pre), Offsets(post)) + }); + if let Some((pre, post)) = &camera_parts { + let period = 1.0 / f64::from(CAMERA_FPS); + for (part, class, label) in [ + (pre, "timestamps", "before the drift anchor"), + (post, "camera-clock", "past the drift anchor"), + ] { + let Some(median) = part.median() else { + continue; + }; + let mut run = 0; + let mut worst_run = (0, 0.0f64, 0.0f64); + let mut isolated = Vec::new(); + let mut isolated_at = Vec::new(); + let mut off_at = Vec::new(); + for &(truth, offset) in &part.0 { + let d = offset - median; + if d.abs() > 3.0 * period { + isolated.push(format!("{:.2}s {:+.1} ms", truth, ms(d))); + isolated_at.push(truth); + } + if d.abs() > period { + off_at.push(truth); + run += 1; + if run > worst_run.0 { + worst_run = (run, truth, d); + } + } else { + run = 0; + } + } + if worst_run.0 >= CAMERA_SUSTAINED_FRAMES { + case.problem( + class, + format!( + "camera {label}: {} consecutive frames off the steady offset by more than a frame period (until {:.2}s, {:+.1} ms)", + worst_run.0, + worst_run.1, + ms(worst_run.2) + ), + ); + implicate(case, off_at); + } + if !isolated.is_empty() { + case.problem( + class, + format!( + "camera {label}: {} frames more than three periods off the steady offset: {}", + isolated.len(), + isolated + .iter() + .take(4) + .cloned() + .collect::>() + .join(", ") + ), + ); + implicate(case, isolated_at); + } + } + // Stamped with their capture times, camera frames keep one offset: + // before the anchor every one of 20 runs had a spread of exactly 0 + // (run 38780355968), so any spread of a file tick (1.7 ms) or more + // past it is the processing clock. The microphone's 4 ms limit + // missed it in 2 of those 20 runs once the synthetic camera ran on + // time (spread 3.3 ms; 3.3 to 30 ms in the others). + const CAMERA_EXACT_SPREAD: f64 = 0.001; + if post.0.len() > 1 && post.spread() > CAMERA_EXACT_SPREAD { + case.problem( + "camera-clock", + format!( + "camera past the drift anchor: frame offsets vary by {:.1} ms (limit {:.1}): frames are stamped from the processing clock, not their capture times", + ms(post.spread()), + ms(CAMERA_EXACT_SPREAD) + ), + ); + } + } + let mic_m = mic.and_then(Offsets::median); + let screen_m = screen.median(); + let system_m = system.and_then(Offsets::median); + // Camera against the microphone, separately before and past the drift + // anchor (past it the camera carries processing delays, class + // camera-clock). + if let (Some((pre, post)), Some(mic_m)) = (&camera_parts, mic_m) { + for (part, class, label) in [ + (pre, "sync-camera-mic", "before the drift anchor"), + (post, "camera-clock", "past the drift anchor"), + ] { + let Some(camera_m) = part.median() else { + continue; + }; + let d = camera_m - mic_m; + case.note(format!("sync camera-mic {label} {:+.1} ms", ms(d))); + if d.abs() > CAMERA_VS_MIC { + case.measure(class, "camera-mic-ms", (ms(d) * 10.0).round() / 10.0); + case.problem( + class, + format!( + "camera {label} vs mic out of sync: {:+.1} ms (limit +-{:.0})", + ms(d), + ms(CAMERA_VS_MIC) + ), + ); + } + } + } + if let (Some(system_m), Some(mic_m)) = (system_m, mic_m) { + // Both carry host-clock truth: the microphone's exactly, system + // audio from the output callback's host time. + let d = system_m - mic_m; + case.note(format!("sync system-mic {:+.1} ms", ms(d))); + if d.abs() > SYSTEM_VS_MIC { + case.measure( + "sync-system-mic", + "offset-ms", + (ms(d) * 10.0).round() / 10.0, + ); + case.problem( + "sync-system-mic", + format!( + "system audio vs mic out of sync: {:+.1} ms (limit +-{:.0})", + ms(d), + ms(SYSTEM_VS_MIC) + ), + ); + } + } + let Some(screen_m) = screen_m else { + return; + }; + for (name, audio_m) in [("mic", mic_m), ("system audio", system_m)] { + let Some(audio_m) = audio_m else { + continue; + }; + let d = screen_m - audio_m; + case.note(format!( + "sync screen-{} {:+.1} ms", + name.split(' ').next().unwrap_or(name), + ms(d) + )); + if d < screen_latency.0 - slack(name) || d > screen_latency.1 + slack(name) { + // One class per pair, so a known offset of one source cannot + // excuse another's. + let class = if name == "mic" { + "sync-screen-mic" + } else { + "sync-screen-system" + }; + case.measure(class, "offset-ms", (ms(d) * 10.0).round() / 10.0); + case.problem( + class, + format!( + "screen vs {name} out of sync: {:+.1} ms (allowed {:+.0}..{:+.0}: display latency)", + ms(d), + ms(screen_latency.0 - slack(name)), + ms(screen_latency.1 + slack(name)) + ), + ); + } + } +} diff --git a/crates/verify/tests/seek.rs b/crates/verify/tests/seek.rs new file mode 100644 index 00000000000..a13c2acb736 --- /dev/null +++ b/crates/verify/tests/seek.rs @@ -0,0 +1,362 @@ +//! Seek oracle: random seek storms, keyframe-exact seeks, scrubbing in both +//! directions, seeks into holds and cold opens, all checked against the +//! model. Seeds are fixed and printed; the full tier runs many more seeks. + +mod common; + +use cap_project::ProjectConfiguration; +use cap_verify::{ + catalog, + fixture::{FixtureSpec, Rng}, + media, + report::{self, Tier}, +}; +use common::{Decoder, Player}; + +const RES: (u32, u32) = (480, 270); + +/// One request: output frame and whether it is a seek (vs a follow-on +/// playback frame). +type Request = (u32, bool); + +fn run( + spec: FixtureSpec, + name: &str, + fps: u32, + edit: fn(&mut ProjectConfiguration), + make: &dyn Fn(u32, &common::Project) -> Vec, +) -> Vec { + run_project(common::project(&spec), name, fps, edit, make) +} + +fn run_project( + project: common::Project, + name: &str, + fps: u32, + edit: fn(&mut ProjectConfiguration), + make: &dyn Fn(u32, &common::Project) -> Vec, +) -> Vec { + let rt = common::runtime(); + let mut failures = Vec::new(); + for decoder in Decoder::all() { + let id = format!("seek/{}/{name}{fps}/{}", project.name, decoder.label()); + if !report::selected(&id) { + continue; + } + let mut config = project.config(); + edit(&mut config); + let model = project.model(&config); + let requests = make(model.output_frames(fps), &project); + let Some(mut case) = common::begin(&id, &project, decoder) else { + continue; + }; + case.note(format!("{} requests", requests.len())); + let result = rt.block_on(async { + let mut player = Player::open(&project, config, decoder, RES).await?; + let mut readouts = Vec::new(); + for &(frame, seek) in &requests { + readouts.push( + player + .frame(frame, fps, seek, Some(&model.expect(frame, fps))) + .await?, + ); + } + player.check_decoders(&mut case, decoder); + Ok::<_, String>(readouts) + }); + let frames: Vec = requests.iter().map(|r| r.0).collect(); + match result { + Ok(readouts) => common::check_requests(&mut case, &model, &frames, &readouts, fps), + Err(e) => case.problem("error", e), + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + failures +} + +fn no_edit(_: &mut ProjectConfiguration) {} + +fn seek_count(fast: usize, full: usize) -> usize { + match report::tier() { + Tier::Fast => fast, + Tier::Full => full, + } +} + +/// Random seeks, each followed by 0..=3 playback frames. +fn storm(seed: u64, count: usize) -> impl Fn(u32, &common::Project) -> Vec { + move |total, _| { + eprintln!("seek storm seed {seed:#x}, {count} seeks"); + let mut rng = Rng::new(seed); + let mut out = Vec::new(); + for _ in 0..count { + let frame = rng.range(0, u64::from(total - 4)) as u32; + out.push((frame, true)); + for k in 1..=rng.range(0, 3) as u32 { + out.push((frame + k, false)); + } + } + out + } +} + +// One test per fixture: a storm is the slowest seek workload (FULL tier on +// the macos-15 runner: 240 to 570 s per decoder), and as one test the three +// storms of a shard ran past nextest's per-test limit. +#[test] +fn storm_seek_gop() { + let count = seek_count(150, 3000); + common::assert_passed(run( + catalog::seek_gop(report::tier()), + "storm", + 30, + no_edit, + &storm(0x5EE4_0001, count), + )); +} + +#[test] +fn storm_vfr_holds() { + let count = seek_count(150, 3000); + common::assert_passed(run( + catalog::vfr_holds(report::tier()), + "storm", + 60, + no_edit, + &storm(0x5EE4_0002, count), + )); +} + +#[test] +fn storm_seek_native() { + let count = seek_count(150, 3000); + common::assert_passed(run( + catalog::seek_gop_native(report::tier()), + "storm", + 30, + no_edit, + &storm(0x5EE4_0004, count), + )); +} + +/// Seeks landing on and around every source keyframe, at 60 fps output so +/// requests also fall between the keyframe's pts and the next source slot, +/// in shuffled order: a reader started a hair before the keyframe decodes +/// the previous GOP and shows its last frame under the keyframe's number. +#[test] +fn keyframe_seeks() { + let mut failures: Vec = Vec::new(); + let keyframes = |total: u32, project: &common::Project| { + let display = project.path.join("content/segments/segment-0/display.mp4"); + let times = media::keyframe_times(&display).expect("keyframes"); + let mut frames: Vec = times + .iter() + .flat_map(|t| { + let f = (t * 60.0).round() as u32; + [f, f.saturating_sub(1), f + 1, f + 2] + }) + .filter(|&f| f < total) + .collect(); + let mut rng = Rng::new(0x5EE4_0003); + for i in (1..frames.len()).rev() { + let j = rng.range(0, i as u64) as usize; + frames.swap(i, j); + } + frames.into_iter().map(|f| (f, true)).collect() + }; + failures.extend(run( + catalog::seek_gop(report::tier()), + "keyframes", + 60, + no_edit, + &keyframes, + )); + failures.extend(run( + catalog::seek_gop_native(report::tier()), + "keyframes", + 60, + no_edit, + &keyframes, + )); + common::assert_passed(failures); +} + +/// The editor's "jump to start" after watching later in the file, then a +/// click 2 s in and playback: seek 6.1 s, seek 0, seek 2.0 s and play 4 s. +#[test] +fn home_then_play() { + let mut failures: Vec = Vec::new(); + let home = |total: u32, _: &common::Project| { + let fps = 30.0; + let mut out: Vec = vec![((6.1 * fps) as u32, true), (0, true)]; + let start = (2.0 * fps) as u32; + out.push((start, true)); + out.extend((1..=120).map(|k| (start + k, false))); + out.retain(|&(f, _)| f < total); + out + }; + failures.extend(run( + catalog::seek_gop(report::tier()), + "home-then-play", + 30, + no_edit, + &home, + )); + common::assert_passed(failures); +} + +/// Scrubbing: seek every 0.25 s forwards, then backwards. +#[test] +fn scrub() { + let mut failures: Vec = Vec::new(); + let scrub = |total: u32, _: &common::Project| { + let step = 7; + let mut out: Vec = (0..total).step_by(step).map(|f| (f, true)).collect(); + out.extend((0..total).rev().step_by(step).map(|f| (f, true))); + out + }; + failures.extend(run( + catalog::seek_gop(report::tier()), + "scrub", + 30, + no_edit, + &scrub, + )); + failures.extend(run( + catalog::vfr_holds(report::tier()), + "scrub", + 60, + no_edit, + &scrub, + )); + common::assert_passed(failures); +} + +/// Ad-hoc repro runner: `CAP_VERIFY_SEEK_SCRIPT="@:+,..."` +/// seeks to each time and plays `follow` frames after it (fixtures: +/// seek-gop, seek-native, vfr-holds). Ignored without the variable. +#[test] +fn script() { + let Ok(script) = std::env::var("CAP_VERIFY_SEEK_SCRIPT") else { + return; + }; + let (head, steps) = script.split_once(':').expect("@:"); + let (fixture, fps) = head.split_once('@').expect("@"); + let fps: u32 = fps.parse().expect("fps"); + let spec = match fixture { + "seek-gop" => catalog::seek_gop(report::tier()), + "seek-native" => catalog::seek_gop_native(report::tier()), + "vfr-holds" => catalog::vfr_holds(report::tier()), + other => panic!("unknown fixture {other}"), + }; + let steps: Vec<(f64, u32)> = steps + .split(',') + .map(|step| { + let (t, n) = step.split_once('+').unwrap_or((step, "0")); + (t.parse().expect("secs"), n.parse().expect("follow")) + }) + .collect(); + let make = move |total: u32, _: &common::Project| { + let mut out = Vec::new(); + for &(t, n) in &steps { + let f = ((t * f64::from(fps)) as u32).min(total - 1); + out.push((f, true)); + out.extend((1..=n).map(|k| (f + k, false)).filter(|&(f, _)| f < total)); + } + out + }; + common::assert_passed(run(spec, "script", fps, no_edit, &make)); +} + +/// Seeks into the middle of static holds and across them, then playback +/// out of the hold. +#[test] +fn seek_into_holds() { + let mut failures: Vec = Vec::new(); + let into_holds = |total: u32, _: &common::Project| { + let mut out = Vec::new(); + for t in [2.0, 3.5, 1.2, 9.0, 11.1, 5.2, 7.05, 2.3] { + let f = (t * 60.0) as u32; + if f + 30 < total { + out.push((f, true)); + out.extend((1..30).map(|k| (f + k, false))); + } + } + out + }; + failures.extend(run( + catalog::vfr_holds(report::tier()), + "into-holds", + 60, + no_edit, + &into_holds, + )); + // Minimal deterministic sequence for the AVAssetReader camera answering + // one frame stale after revisiting a region (found by into-holds). + let revisit = |total: u32, _: &common::Project| { + let mut out = Vec::new(); + for t in [2.0, 3.5, 1.2, 2.3] { + let f = (t * 60.0) as u32; + if f + 30 < total { + out.push((f, true)); + out.extend((1..30).map(|k| (f + k, false))); + } + } + out + }; + failures.extend(run( + catalog::vfr_holds(report::tier()), + "revisit", + 60, + no_edit, + &revisit, + )); + common::assert_passed(failures); +} + +/// The first frame after opening a project, at several positions. +#[test] +fn cold_open() { + let spec = catalog::seek_gop(report::tier()); + let project = common::project(&spec); + let rt = common::runtime(); + let mut failures = Vec::new(); + for decoder in Decoder::all() { + let id = format!("seek/{}/cold-open30/{}", project.name, decoder.label()); + if !report::selected(&id) { + continue; + } + let config = project.config(); + let model = project.model(&config); + let total = model.output_frames(30); + let frames: Vec = [0.0, 0.37, 0.71, 0.93] + .iter() + .map(|p| (f64::from(total - 1) * p) as u32) + .collect(); + let Some(mut case) = common::begin(&id, &project, decoder) else { + continue; + }; + let mut readouts = Vec::new(); + for &frame in &frames { + let result = rt.block_on(async { + let mut player = Player::open(&project, config.clone(), decoder, RES).await?; + let readout = player.frame(frame, 30, true, None).await; + player.check_decoders(&mut case, decoder); + readout + }); + match result { + Ok(readout) => readouts.push(readout), + Err(e) => case.problem("error", e), + } + } + if readouts.len() == frames.len() { + common::check_requests(&mut case, &model, &frames, &readouts, 30); + } + if let Err(message) = report::settle(case) { + failures.push(message); + } + } + common::assert_passed(failures); +} diff --git a/crates/video-decode/Cargo.toml b/crates/video-decode/Cargo.toml index f9a0a2be4d8..486da43a1a9 100644 --- a/crates/video-decode/Cargo.toml +++ b/crates/video-decode/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" [lints] workspace = true +[features] +# Test-only: lets decoder model tests register scripted media in place of files. +scripted = [] + [dependencies] cap-enc-ffmpeg = { path = "../enc-ffmpeg" } ffmpeg.workspace = true diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 1f2dfc7b02a..14f1c12051f 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -196,6 +196,11 @@ fn scan_keyframes( impl KeyframeIndex { pub fn build(path: &Path) -> Result { + #[cfg(feature = "scripted")] + if let Some(state) = crate::scripted::lookup(path) { + return Ok(Self::from_script(&state.script)); + } + let build_start = std::time::Instant::now(); let mut input = avformat::input(path) @@ -279,6 +284,28 @@ impl KeyframeIndex { }) } + #[cfg(feature = "scripted")] + fn from_script(script: &crate::scripted::Script) -> Self { + let fps = script.avg_fps(); + let keyframes = script + .frames + .iter() + .filter(|frame| frame.keyframe) + .map(|frame| { + let secs = script.secs(frame.pts); + ((secs * fps).round() as u32, secs) + }) + .collect(); + Self { + keyframes, + fps, + duration_secs: script.duration_secs(), + pixel_format: Some(cv::PixelFormat::_420V), + width: crate::scripted::FRAME_WIDTH, + height: crate::scripted::FRAME_HEIGHT, + } + } + pub fn nearest_keyframe_before(&self, target_frame: u32) -> Option<(u32, f64)> { if self.keyframes.is_empty() { return None; @@ -394,8 +421,7 @@ pub struct AVAssetReaderDecoder { path: PathBuf, pixel_format: cv::PixelFormat, tokio_handle: TokioHandle, - track_output: R, - reader: R, + source: ReaderSource, width: u32, height: u32, keyframe_index: Option>, @@ -462,7 +488,7 @@ impl AVAssetReaderDecoder { let seek_time = compute_seek_time(keyframe_index.as_ref(), start_time); - let (track_output, reader) = Self::get_reader_track_output( + let source = Self::get_reader_track_output( &path, seek_time, &tokio_handle, @@ -475,8 +501,7 @@ impl AVAssetReaderDecoder { path, pixel_format, tokio_handle, - track_output, - reader, + source, width, height, keyframe_index, @@ -485,11 +510,11 @@ impl AVAssetReaderDecoder { } pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { - self.reader.cancel_reading(); + self.source.cancel_reading(); let seek_time = compute_seek_time(self.keyframe_index.as_ref(), requested_time); - (self.track_output, self.reader) = Self::get_reader_track_output( + self.source = Self::get_reader_track_output( &self.path, seek_time, &self.tokio_handle, @@ -538,7 +563,20 @@ impl AVAssetReaderDecoder { pixel_format: cv::PixelFormat, width: u32, height: u32, - ) -> Result<(R, R), String> { + ) -> Result { + let time_range = cm::TimeRange { + start: cm::Time::with_secs(time as f64, 100), + duration: cm::Time::infinity(), + }; + + #[cfg(feature = "scripted")] + if let Some(state) = crate::scripted::lookup(path) { + return Ok(ReaderSource::Scripted(crate::scripted::AvfReader::start( + state, + time_range.start, + ))); + } + let asset = av::UrlAsset::with_url( &ns::Url::with_fs_path_str( path.to_str() @@ -552,11 +590,6 @@ impl AVAssetReaderDecoder { let mut reader = av::AssetReader::with_asset(&asset) .map_err(|e| format!("AssetReader::with_asset: {e}"))?; - let time_range = cm::TimeRange { - start: cm::Time::with_secs(time as f64, 100), - duration: cm::Time::infinity(), - }; - reader.set_time_range(time_range); let tracks = handle @@ -592,12 +625,15 @@ impl AVAssetReaderDecoder { .start_reading() .map_err(|e| format!("reader.start_reading: {e}"))?; - Ok((reader_track_output, reader)) + Ok(ReaderSource::Native { + track_output: reader_track_output, + reader, + }) } pub fn frames(&mut self) -> FramesIter<'_> { FramesIter { - track_output: &mut self.track_output, + source: &mut self.source, } } @@ -610,13 +646,32 @@ impl AVAssetReaderDecoder { } } +enum ReaderSource { + Native { + track_output: R, + reader: R, + }, + #[cfg(feature = "scripted")] + Scripted(crate::scripted::AvfReader), +} + +impl ReaderSource { + fn cancel_reading(&mut self) { + match self { + Self::Native { reader, .. } => reader.cancel_reading(), + #[cfg(feature = "scripted")] + Self::Scripted(_) => {} + } + } +} + pub struct FramesIter<'a> { - track_output: &'a mut av::AssetReaderTrackOutput, + source: &'a mut ReaderSource, } impl Drop for AVAssetReaderDecoder { fn drop(&mut self) { - self.reader.cancel_reading(); + self.source.cancel_reading(); } } @@ -624,7 +679,11 @@ impl<'a> Iterator for FramesIter<'a> { type Item = ns::ExResult<'static, arc::R>; fn next(&mut self) -> Option { - self.track_output.next_sample_buf().transpose() + match self.source { + ReaderSource::Native { track_output, .. } => track_output.next_sample_buf().transpose(), + #[cfg(feature = "scripted")] + ReaderSource::Scripted(reader) => reader.next_sample_buf().map(Ok), + } } } diff --git a/crates/video-decode/src/ffmpeg.rs b/crates/video-decode/src/ffmpeg.rs index b343f8544f4..75481f6292c 100644 --- a/crates/video-decode/src/ffmpeg.rs +++ b/crates/video-decode/src/ffmpeg.rs @@ -192,6 +192,8 @@ pub struct FFmpegDecoder { enum DecoderInput { File(avformat::context::Input), Segmented(cap_enc_ffmpeg::SegmentedInput), + #[cfg(feature = "scripted")] + Scripted(crate::scripted::Demuxer), } impl DecoderInput { @@ -199,6 +201,8 @@ impl DecoderInput { match self { Self::File(input) => input, Self::Segmented(input) => input.input(), + #[cfg(feature = "scripted")] + Self::Scripted(_) => unreachable!("scripted media has no container"), } } @@ -206,6 +210,8 @@ impl DecoderInput { match self { Self::File(input) => input.seek(position, ..position), Self::Segmented(input) => input.seek(position), + #[cfg(feature = "scripted")] + Self::Scripted(demuxer) => demuxer.seek(position), } } @@ -213,6 +219,8 @@ impl DecoderInput { match self { Self::File(input) => DecoderPackets::File(input.packets()), Self::Segmented(input) => DecoderPackets::Segmented(input), + #[cfg(feature = "scripted")] + Self::Scripted(demuxer) => DecoderPackets::Scripted(demuxer), } } } @@ -220,6 +228,8 @@ impl DecoderInput { enum DecoderPackets<'a> { File(PacketIter<'a>), Segmented(&'a mut cap_enc_ffmpeg::SegmentedInput), + #[cfg(feature = "scripted")] + Scripted(&'a mut crate::scripted::Demuxer), } impl Iterator for DecoderPackets<'_> { @@ -238,6 +248,8 @@ impl Iterator for DecoderPackets<'_> { Err(error) => Some(Err(error)), } } + #[cfg(feature = "scripted")] + Self::Scripted(_) => None, } } } @@ -278,6 +290,18 @@ impl FFmpegDecoder { ) -> Result { let path = path.into(); + #[cfg(feature = "scripted")] + if let Some(state) = crate::scripted::lookup(&path) { + let demuxer = crate::scripted::Demuxer::new(state); + return Ok(FFmpegDecoder { + decoder: demuxer.open_decoder()?, + start_time: demuxer.start_time(), + input: DecoderInput::Scripted(demuxer), + stream_index: 0, + hw_device: None, + }); + } + let input = if path.is_dir() { DecoderInput::Segmented(open_fragmented_input(&path)?) } else { @@ -430,6 +454,11 @@ impl<'a> Iterator for FramesIter<'a> { type Item = Result; fn next(&mut self) -> Option { + #[cfg(feature = "scripted")] + if let DecoderPackets::Scripted(demuxer) = &mut self.packets { + return demuxer.next_frame().map(Ok); + } + let mut frame = avframe::Video::empty(); loop { diff --git a/crates/video-decode/src/lib.rs b/crates/video-decode/src/lib.rs index 682df7a778c..110d4b68730 100644 --- a/crates/video-decode/src/lib.rs +++ b/crates/video-decode/src/lib.rs @@ -3,6 +3,8 @@ pub mod avassetreader; pub mod ffmpeg; #[cfg(target_os = "windows")] pub mod media_foundation; +#[cfg(feature = "scripted")] +pub mod scripted; #[cfg(target_os = "macos")] pub use avassetreader::{AVAssetReaderDecoder, KeyframeIndex}; diff --git a/crates/video-decode/src/scripted.rs b/crates/video-decode/src/scripted.rs new file mode 100644 index 00000000000..a44e2a929ff --- /dev/null +++ b/crates/video-decode/src/scripted.rs @@ -0,0 +1,415 @@ +//! Scripted media for decoder model tests, compiled only with the `scripted` +//! feature (enabled by test builds, never by the app). A script is a list of +//! video samples with exact presentation times and sync flags. Registering it +//! yields a path; opening that path with the real decoders reads the script +//! instead of a file, so everything above the container and the native reader +//! runs unchanged. +//! +//! Modelled container and reader behaviour: +//! - FFmpeg: `seek(ts)` lands on the newest sync sample at or before `ts` +//! (the first sample when there is none), as `avformat_seek_file` with +//! `max_ts = ts` does for MP4. Decoding then vends every sample from there in +//! presentation order. +//! - AVAssetReader: a reader started at time `T` vends the sample on screen at +//! `T` (the newest at or before it) and every later one. When that sample +//! starts before `T` its presentation time is trimmed to `T`. A reader started +//! at or after the end of the track vends nothing. +//! +//! Every vended frame carries its sample index in its first luma row, see +//! [`frame_id`]. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicU64, Ordering}, + }, +}; + +const MAGIC: [u8; 4] = *b"CAPS"; +pub const FRAME_WIDTH: u32 = 16; +pub const FRAME_HEIGHT: u32 = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScriptFrame { + pub pts: i64, + pub keyframe: bool, +} + +#[derive(Clone, Debug)] +pub struct Script { + /// Stream time base as (numerator, denominator). + pub time_base: (i32, i32), + /// Samples in strictly increasing presentation order. + pub frames: Vec, + /// Duration of the last sample, in time base ticks. + pub last_duration: i64, +} + +impl Script { + pub fn end_pts(&self) -> i64 { + self.frames.last().map_or(0, |frame| frame.pts) + self.last_duration + } + + pub fn secs(&self, ticks: i64) -> f64 { + ticks as f64 * self.time_base.0 as f64 / self.time_base.1 as f64 + } + + pub fn duration_secs(&self) -> f64 { + self.secs(self.end_pts() - self.frames.first().map_or(0, |frame| frame.pts)) + } + + /// Average frame rate as the MP4 demuxer reports it: sample count over + /// track duration. + pub fn avg_fps(&self) -> f64 { + let duration = self.duration_secs(); + if duration > 0.0 { + self.frames.len() as f64 / duration + } else { + 30.0 + } + } + + fn validate(&self) { + assert!(self.time_base.0 > 0 && self.time_base.1 > 0); + assert!(!self.frames.is_empty(), "scripts need at least one frame"); + assert!( + self.frames[0].keyframe, + "the first sample must be a sync sample" + ); + assert!(self.last_duration > 0); + assert!( + self.frames.windows(2).all(|pair| pair[0].pts < pair[1].pts), + "sample presentation times must increase" + ); + } +} + +#[derive(Default)] +struct Counters { + reader_starts: AtomicU64, + seeks: AtomicU64, +} + +pub(crate) struct ScriptState { + pub(crate) script: Script, + counters: Counters, + vended: Mutex>, + #[cfg(target_os = "macos")] + samples: Mutex>, +} + +impl ScriptState { + fn record_vend(&self, index: usize) { + self.vended.lock().unwrap().push(index as u32); + } + + /// Whether `pts` (in stream ticks) is at or before `value / scale` seconds. + #[cfg(target_os = "macos")] + fn at_or_before(&self, pts: i64, value: i64, scale: i32) -> bool { + let (num, den) = self.script.time_base; + (pts as i128) * (num as i128) * (scale as i128) <= (value as i128) * (den as i128) + } + + #[cfg(target_os = "macos")] + fn newest_at_or_before(&self, value: i64, scale: i32) -> Option { + self.script + .frames + .partition_point(|frame| self.at_or_before(frame.pts, value, scale)) + .checked_sub(1) + } +} + +fn registry() -> &'static Mutex>> { + static REGISTRY: OnceLock>>> = OnceLock::new(); + REGISTRY.get_or_init(Default::default) +} + +pub(crate) fn lookup(path: &Path) -> Option> { + registry().lock().unwrap().get(path).cloned() +} + +/// Frame dimensions of a registered script, for probes that would otherwise +/// open the path as a file. +pub fn dimensions(path: &Path) -> Option<(u32, u32)> { + lookup(path).map(|_| (FRAME_WIDTH, FRAME_HEIGHT)) +} + +/// A registered script. Dropping it unregisters the path. +pub struct Registration { + path: PathBuf, + state: Arc, +} + +impl Registration { + pub fn path(&self) -> &Path { + &self.path + } + + pub fn script(&self) -> &Script { + &self.state.script + } + + /// Sample indices vended by every reader of this script since the last call. + pub fn take_vended(&self) -> Vec { + std::mem::take(&mut *self.state.vended.lock().unwrap()) + } + + /// AVAssetReader starts: creations and resets. + pub fn reader_starts(&self) -> u64 { + self.state.counters.reader_starts.load(Ordering::Relaxed) + } + + /// FFmpeg demuxer seeks. + pub fn seeks(&self) -> u64 { + self.state.counters.seeks.load(Ordering::Relaxed) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + registry().lock().unwrap().remove(&self.path); + } +} + +pub fn register(script: Script) -> Registration { + static NEXT: AtomicU64 = AtomicU64::new(0); + script.validate(); + let path = PathBuf::from(format!( + "/nonexistent/cap-scripted-media/{}-{}.mp4", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let state = Arc::new(ScriptState { + script, + counters: Counters::default(), + vended: Mutex::new(Vec::new()), + #[cfg(target_os = "macos")] + samples: Mutex::new(HashMap::new()), + }); + registry() + .lock() + .unwrap() + .insert(path.clone(), state.clone()); + Registration { path, state } +} + +fn identity_row(index: u32) -> [u8; FRAME_WIDTH as usize] { + let mut row = [0u8; FRAME_WIDTH as usize]; + row[..4].copy_from_slice(&MAGIC); + row[4..8].copy_from_slice(&index.to_le_bytes()); + row[8..12].copy_from_slice(&(!index).to_le_bytes()); + row +} + +/// The sample index a decoded frame was made from: reads the first luma row of +/// the frame's raw planes. +pub fn frame_id(first_row: &[u8]) -> Option { + if first_row.len() < 12 || first_row[..4] != MAGIC { + return None; + } + let index = u32::from_le_bytes(first_row[4..8].try_into().ok()?); + let check = u32::from_le_bytes(first_row[8..12].try_into().ok()?); + (check == !index).then_some(index) +} + +pub(crate) struct Demuxer { + state: Arc, + next: usize, +} + +impl Demuxer { + pub(crate) fn new(state: Arc) -> Self { + Self { state, next: 0 } + } + + pub(crate) fn start_time(&self) -> i64 { + self.state.script.frames[0].pts + } + + pub(crate) fn time_base(&self) -> ffmpeg::Rational { + let (num, den) = self.state.script.time_base; + ffmpeg::Rational::new(num, den) + } + + /// `position` is in `AV_TIME_BASE` units, as `avformat_seek_file` receives + /// it with no stream index. + pub(crate) fn seek(&mut self, position: i64) -> Result<(), ffmpeg::Error> { + self.state.counters.seeks.fetch_add(1, Ordering::Relaxed); + let (num, den) = self.state.script.time_base; + let ticks = rescale_rounded(position, den as i64, 1_000_000 * num as i64); + let frames = &self.state.script.frames; + let at_or_before = frames.partition_point(|frame| frame.pts <= ticks); + self.next = frames[..at_or_before] + .iter() + .rposition(|frame| frame.keyframe) + .unwrap_or(0); + Ok(()) + } + + pub(crate) fn next_frame(&mut self) -> Option { + let frame = self.state.script.frames.get(self.next)?; + let index = self.next; + self.next += 1; + self.state.record_vend(index); + let mut video = + ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUV420P, FRAME_WIDTH, FRAME_HEIGHT); + video.data_mut(0).fill(16); + let row = identity_row(index as u32); + video.data_mut(0)[..row.len()].copy_from_slice(&row); + video.data_mut(1).fill(128); + video.data_mut(2).fill(128); + video.set_pts(Some(frame.pts)); + Some(video) + } + + pub(crate) fn open_decoder(&self) -> Result { + let codec = ffmpeg::decoder::find(ffmpeg::codec::Id::RAWVIDEO) + .or_else(|| ffmpeg::decoder::find(ffmpeg::codec::Id::H264)) + .ok_or("no decoder available for scripted media")?; + let mut context = ffmpeg::codec::Context::new_with_codec(codec); + unsafe { + let raw = context.as_mut_ptr(); + (*raw).width = FRAME_WIDTH as i32; + (*raw).height = FRAME_HEIGHT as i32; + (*raw).pix_fmt = ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_YUV420P; + } + let mut decoder = context + .decoder() + .video() + .map_err(|error| format!("scripted decoder / {error}"))?; + decoder.set_time_base(self.time_base()); + Ok(decoder) + } +} + +/// `a * b / c` rounded half away from zero, as `av_rescale` does. +fn rescale_rounded(a: i64, b: i64, c: i64) -> i64 { + let product = a as i128 * b as i128; + let c = c as i128; + let rounded = if product >= 0 { + (product + c / 2) / c + } else { + (product - c / 2) / c + }; + rounded as i64 +} + +#[cfg(target_os = "macos")] +pub(crate) use avf::AvfReader; + +#[cfg(target_os = "macos")] +mod avf { + use super::*; + use cidre::{arc::R, cm, cv}; + + /// Sample buffers are immutable once built and CoreFoundation reference + /// counting is thread safe, so a script shares them across its readers. + pub(crate) struct SharedSample(R); + + unsafe impl Send for SharedSample {} + + pub(crate) struct AvfReader { + state: Arc, + next: usize, + trimmed_start: Option, + } + + impl AvfReader { + pub(crate) fn start(state: Arc, start: cm::Time) -> Self { + state.counters.reader_starts.fetch_add(1, Ordering::Relaxed); + let script = &state.script; + let past_end = state.at_or_before(script.end_pts(), start.value, start.scale); + let (next, trimmed_start) = if past_end { + (script.frames.len(), None) + } else { + match state.newest_at_or_before(start.value, start.scale) { + None => (0, None), + Some(index) => { + let pts = script.frames[index].pts; + let (num, den) = script.time_base; + let exact = (pts as i128) * (num as i128) * (start.scale as i128) + == (start.value as i128) * (den as i128); + (index, (!exact).then_some(start)) + } + } + }; + Self { + state, + next, + trimmed_start, + } + } + + pub(crate) fn next_sample_buf(&mut self) -> Option> { + let script = &self.state.script; + let frame = script.frames.get(self.next)?; + let index = self.next; + self.next += 1; + self.state.record_vend(index); + + let trimmed = self.trimmed_start.take(); + if trimmed.is_none() + && let Some(SharedSample(sample)) = self.state.samples.lock().unwrap().get(&index) + { + return Some(sample.retained()); + } + let (num, den) = script.time_base; + let pts = trimmed.unwrap_or_else(|| cm::Time::new(frame.pts * num as i64, den)); + let end = script + .frames + .get(index + 1) + .map_or(script.end_pts(), |next| next.pts); + let duration = cm::Time::new((end - frame.pts) * num as i64, den); + + let mut pixel_buf = cv::PixelBuf::new( + FRAME_WIDTH as usize, + FRAME_HEIGHT as usize, + cv::PixelFormat::_420V, + None, + ) + .ok()?; + unsafe { + pixel_buf + .lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT) + .result() + .ok()?; + let row = identity_row(index as u32); + for plane in 0..2 { + let bytes = + pixel_buf.plane_bytes_per_row(plane) * pixel_buf.plane_height(plane); + let base = pixel_buf.plane_base_address(plane) as *mut u8; + std::ptr::write_bytes(base, if plane == 0 { 16 } else { 128 }, bytes); + if plane == 0 { + std::ptr::copy_nonoverlapping(row.as_ptr(), base, row.len()); + } + } + pixel_buf.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::DEFAULT); + } + let format = cm::VideoFormatDesc::with_image_buf(&pixel_buf).ok()?; + let timing = cm::SampleTimingInfo { + duration, + pts, + dts: cm::Time::invalid(), + }; + let sample = cm::SampleBuf::with_image_buf( + &pixel_buf, + true, + None, + std::ptr::null(), + &format, + &timing, + ) + .ok()?; + if trimmed.is_none() { + self.state + .samples + .lock() + .unwrap() + .insert(index, SharedSample(sample.retained())); + } + Some(sample) + } + } +} diff --git a/scripts/verify/bless.sh b/scripts/verify/bless.sh new file mode 100755 index 00000000000..c8f6c2f91f1 --- /dev/null +++ b/scripts/verify/bless.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Rewrites the strict expectations files so that a run passes exactly. +# +# scripts/verify/bless.sh [fast|full] [run.sh options] run here, then bless +# scripts/verify/bless.sh --run bless from the report +# artifacts of a CI run +# (every platform) +# scripts/verify/bless.sh --reports ... bless from report dirs +# +# Each platform only changes its own entries; reasons and links of existing +# entries are kept and new failures get a "TODO:" reason that must be +# replaced before CI accepts the file. Review the diff: a blessed failure is a +# known bug the suite will keep checking. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$root" + +reports=() +case "${1:-}" in + --run) + [[ -n "${2:-}" ]] || { echo "usage: $0 --run " >&2; exit 2; } + dir="$(mktemp -d)" + gh run download "$2" --pattern 'verify-report-*' --dir "$dir" + reports=("$dir") + ;; + --reports) + shift + reports=("$@") + ;; + *) + tier=fast + [[ "${1:-}" == fast || "${1:-}" == full ]] && tier="$1" + case "$(uname -s)" in + Darwin) platform=macos ;; + Linux) platform=linux ;; + *) platform=windows ;; + esac + export CAP_VERIFY_REPORT_DIR="${CAP_VERIFY_REPORT_DIR:-$root/target/verify-report/$platform-$tier}" + scripts/verify/run.sh "$@" || true + reports=("$CAP_VERIFY_REPORT_DIR") + ;; +esac + +cargo run -q -p cap-verify-expect -- bless "${reports[@]}" +git --no-pager diff --stat -- '*expectations*.toml' diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py new file mode 100644 index 00000000000..d9332ad199c --- /dev/null +++ b/scripts/verify/diff.py @@ -0,0 +1,406 @@ +import argparse +import fnmatch +import json +import math +import os +import re +import sys +import xml.etree.ElementTree as ET +from collections import defaultdict + + +def load_na(path): + + out = [] + if path: + for line in open(path, encoding="utf-8"): + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + glob, _, reason = line.partition("\t") + out.append((glob.strip(), reason.strip())) + return out + + +def na_reason(na, key): + case = key.split("::", 1)[-1] + for glob, reason in na: + if fnmatch.fnmatchcase(case, glob) or fnmatch.fnmatchcase(key, glob): + return reason + return None + + +def lane_of(artifact): + m = re.match(r"verify-report-(.+?)-(fast|full)(-\d+)?$", artifact) + if m: + return m.group(1) + m = re.match(r"verify-recording-(.+)$", artifact) + if m: + return "recording-" + m.group(1) + return artifact + + +def load(roots): + + + + records = defaultdict(lambda: defaultdict(list)) + junit = defaultdict(dict) + for root in roots.split(","): + if not os.path.isdir(root): + sys.exit(f"no such directory: {root}") + load_one(root, records, junit) + return records, junit + + +def load_one(root, records, junit): + for artifact in sorted(os.listdir(root)): + base = os.path.join(root, artifact) + if not os.path.isdir(base): + continue + lane = lane_of(artifact) + for dirpath, _, files in os.walk(base): + for name in sorted(files): + path = os.path.join(dirpath, name) + if name.endswith(".jsonl"): + with open(path, encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + if "case" not in r: + continue + key = f"{r.get('suite', '?')}::{r['case']}" + repeat = next((n for n in r.get("notes", []) if n.startswith("repeat ")), "single") + r["_sample"] = f"{os.path.realpath(root)}:{repeat}" + records[lane][key].append(r) + elif name.startswith("junit") and name.endswith(".xml"): + try: + tree = ET.parse(path) + except ET.ParseError: + continue + for case in tree.getroot().iter("testcase"): + test = f"{case.get('classname', '')} {case.get('name', '')}".strip() + status = "pass" + if case.find("failure") is not None: + status = "fail" + elif case.find("error") is not None: + status = "error" + elif case.find("skipped") is not None: + status = "skipped" + elif case.find("flakyFailure") is not None or case.find("rerunFailure") is not None: + status = "flaky-pass" + prev = junit[lane].get(test) + rank = {"pass": 0, "skipped": 0, "flaky-pass": 1, "fail": 2, "error": 3} + if prev is None or rank[status] > rank[prev]: + junit[lane][test] = status + + +def magnitude(rng): + try: + lo, hi = rng + return max(abs(float(lo)), abs(float(hi))) + except (TypeError, ValueError): + return None + + +def normalized_measures(record): + measures = {cls: dict(metrics or {}) for cls, metrics in (record.get("measures") or {}).items()} + if record.get("case") != "export/instant/output/mp4/native": + return measures + unobserved = record.get("unobserved") or {} + if "sync-mic" in unobserved or "sync-system" in unobserved: + return measures + mic = measures.get("sync-mic", {}).get("offset-ms") + system = measures.get("sync-system", {}).get("offset-ms") + if mic is not None and system is not None: + # Historical source offsets are already rounded to 0.1 ms. Subtract their + # signed intervals without inventing precision or replacing newer telemetry. + metrics = measures["sync-system"] + metrics.setdefault("mic-offset-ms", mic) + metrics.setdefault("system-minus-mic-ms", [round(system[0] - mic[1], 1), round(system[1] - mic[0], 1)]) + return measures + + +def fresh_fixture(case): + return case.startswith(("recording/", "crash/", "export/instant", "export/rec-")) + + +def timing_resolution(metric): + if metric.endswith("-start-late-ms"): + return 1.0 + if metric in {"secs", "screen-lost-s", "off-periods"} or metric.endswith("-after-start-s"): + return 0.01 + if metric.endswith("-ms"): + return 0.1 + if metric.endswith("-s"): + return 1e-9 + return None + + +def metric_resolution(metric): + resolution = timing_resolution(metric) + return resolution if resolution is not None else 1.0 + + +def summarize(recs): + ran = [r for r in recs if str(r.get("outcome", "")).lower() != "unsupported"] + decisive = [r for r in ran if str(r.get("outcome", "")).lower() != "inconclusive"] or ran + outcomes = sorted({str(r.get("outcome", "?")).lower() for r in recs}) + grouped = defaultdict(list) + for index, r in enumerate(decisive): + grouped[r.get("_sample", index)].append(r) + class_runs = defaultdict(int) + measures = defaultdict(dict) + samples = [] + for group in grouped.values(): + classes = set().union(*(set(r.get("failures") or {}) for r in group)) + unobserved = set.intersection(*(set(r.get("unobserved") or {}) for r in group)) - classes + sample = {"classes": classes, "measures": defaultdict(dict), "unobserved": unobserved} + for cls in classes: + class_runs[cls] += 1 + for r in group: + for cls, metrics in normalized_measures(r).items(): + for metric, rng in (metrics or {}).items(): + value = magnitude(rng) + if value is None: + continue + sample["measures"][cls][metric] = max(value, sample["measures"][cls].get(metric, 0)) + measures[cls][metric] = max(value, measures[cls].get(metric, 0)) + samples.append(sample) + unobserved = set.intersection(*(s["unobserved"] for s in samples)) if samples else set() + why = sorted({(r.get("problems") or {}).get("unsupported", "") or r.get("summary", "") for r in recs if str(r.get("outcome", "")).lower() == "unsupported"}) + return { + "why": why, + "case": next((r.get("case", "") for r in recs), ""), + "ran": bool(ran), + "runs": len(samples), + "outcomes": outcomes, + "classes": dict(class_runs), + "measures": {k: dict(v) for k, v in measures.items()}, + "unobserved": unobserved, + "samples": samples, + } + + +def coverage_gained(b, m): + return sorted(b["unobserved"] - m["unobserved"]) if m["ran"] else [] + + +def coverage_lost(b, m): + return sorted(m["unobserved"] - b["unobserved"]) if b["ran"] and m["ran"] else [] + + +def distribution(summary, cls, metric): + values = [] + for sample in summary["samples"]: + if cls in sample["unobserved"]: + continue + if cls not in sample["classes"]: + values.append(0) + elif metric in sample["measures"].get(cls, {}): + values.append(sample["measures"][cls][metric]) + return values + + +def larger(a, b, rel, abs_): + return a > b + max(abs_, rel * abs(b)) + + +def beyond_resolution(a, b, rel, abs_): + limit = b + max(abs_, rel * abs(b)) + return a > limit and not math.isclose(a, limit, rel_tol=1e-12, abs_tol=1e-12) + + +def compare_case(b, m, rel, abs_): + + worse, better, pending, overlap = [], [], [], [] + gained = coverage_gained(b, m) + if b["ran"] and not m["ran"]: + return "WORSE", [f"unsupported on main (baseline {'/'.join(b['outcomes'])})"] + if not b["ran"] and m["ran"]: + return "N/A", ["unsupported on baseline: " + "; ".join(b.get("why", [])[:1])] + if not b["ran"] and not m["ran"]: + return "EQUAL", [] + lost = coverage_lost(b, m) + worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost) + bc, mc = b["classes"], m["classes"] + fresh = fresh_fixture(b["case"]) or fresh_fixture(m["case"]) + if fresh and (b["runs"] < 3 or m["runs"] < 3): + pending.append("fresh fixture comparison requires at least 3 independent runs per side on this OS image") + sampled_classes = { + cls for cls in set(bc) | set(mc) + if fresh or any(timing_resolution(metric) is not None for metric in + set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {}))) + } + stochastic = "missing-after-hold" + distribution_classes = (set(bc) | set(mc)) & {stochastic} + if b["case"].startswith("recording/") or m["case"].startswith("recording/"): + distribution_classes.add(stochastic) + for cls in sorted(distribution_classes - set(lost)): + metrics = {"count"} | set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {})) + for metric in sorted(metrics): + bv, mv = distribution(b, cls, metric), distribution(m, cls, metric) + description = f"{cls}.{metric} baseline={bv}, main={mv}" + if len(bv) < 3 or len(mv) < 3: + pending.append(description + " (requires at least 3 independent runs per side on this OS image)") + elif min(mv) > max(bv): + worse.append(description + " (every main count above baseline range)") + elif max(mv) < min(bv): + better.append(description + " (every main count below baseline range)") + elif bv != mv: + overlap.append(description + " (ranges overlap)") + for cls in sorted(set(mc) - set(bc) - set(gained) - {stochastic}): + if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3): + pending.append(f"+{cls} (requires at least 3 independent runs per side on this OS image)") + elif cls in sampled_classes and mc[cls] != m["runs"]: + overlap.append(f"+{cls} in {mc[cls]}/{m['runs']} main runs (ranges overlap)") + else: + worse.append(f"+{cls}") + for cls in sorted(set(bc) - set(mc) - {stochastic}): + if cls in m["unobserved"]: + continue + if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3): + pending.append(f"-{cls} (requires at least 3 independent runs per side on this OS image)") + elif cls in sampled_classes and bc[cls] != b["runs"]: + overlap.append(f"-{cls} in {bc[cls]}/{b['runs']} baseline runs (ranges overlap)") + else: + better.append(f"-{cls}") + for cls in sorted((set(bc) & set(mc)) - {stochastic}): + if cls not in sampled_classes and b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1): + bf, mf = bc[cls] / b["runs"], mc[cls] / m["runs"] + if mf > bf: + worse.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}") + elif mf < bf: + better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}") + bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {}) + for metric in sorted(set(bm) | set(mm)): + if metric == "frame": + continue + if fresh or timing_resolution(metric) is not None: + bv, mv = distribution(b, cls, metric), distribution(m, cls, metric) + resolution = metric_resolution(metric) + description = f"{cls}.{metric} baseline={bv}, main={mv}, resolution={resolution:g}" + if len(bv) < 3 or len(mv) < 3: + pending.append(description + " (requires at least 3 independent runs per side on this OS image)") + elif beyond_resolution(min(mv), max(bv), rel, max(abs_, resolution)): + worse.append(description + " (every main value above baseline range by more than resolution)") + elif beyond_resolution(min(bv), max(mv), rel, max(abs_, resolution)): + better.append(description + " (every main value below baseline range by more than resolution)") + elif bv != mv: + overlap.append(description + " (ranges overlap within resolution)") + elif metric in bm and metric in mm: + if larger(mm[metric], bm[metric], rel, abs_): + worse.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}") + elif larger(bm[metric], mm[metric], rel, abs_): + better.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}") + elif metric in mm: + worse.append(f"{cls}.{metric} new on main ({fmt(mm[metric])})") + if worse: + return "WORSE", worse + ([f"(also better: {', '.join(better)})"] if better else []) + if pending: + return "INSUFFICIENT", pending + better + overlap + if better: + return "BETTER", better + overlap + if overlap: + return "OVERLAP", overlap + if gained: + return "COVERAGE-GAINED", [f"{cls}: baseline unobserved -> main observed" for cls in gained] + return "EQUAL", [] + + +def fmt(x): + return f"{x:.4g}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("base") + ap.add_argument("main") + ap.add_argument("--rel-tol", type=float, default=0.0) + ap.add_argument("--abs-tol", type=float, default=0.0) + ap.add_argument("--json") + ap.add_argument("--title", default="baseline vs main") + ap.add_argument("--na", help="file of 'case-globreason' lines: cases n/a on the baseline") + args = ap.parse_args() + + na = load_na(args.na) + brec, bjunit = load(args.base) + mrec, mjunit = load(args.main) + lanes = sorted(set(brec) | set(mrec) | set(bjunit) | set(mjunit)) + out = {} + print(f"# {args.title}\n") + print(f"tolerance: rel {args.rel_tol}, abs {args.abs_tol}\n") + print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | coverage lost | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |") + print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|") + detail = [] + for lane in lanes: + b, m = brec.get(lane, {}), mrec.get(lane, {}) + res = defaultdict(list) + for key in sorted(set(b) | set(m)): + if key in b and key not in m: + bs = summarize(b[key]) + if bs["ran"]: + res["WORSE"].append((key, ["missing on main"])) + res["ONLY-BASE"].append((key, [])) + continue + if key in m and key not in b: + reason = na_reason(na, key) + if reason: + res["N/A"].append((key, [reason])) + else: + res["ONLY-MAIN"].append((key, [])) + continue + bs, ms = summarize(b[key]), summarize(m[key]) + verdict, reasons = compare_case(bs, ms, args.rel_tol, args.abs_tol) + res[verdict].append((key, reasons)) + gained = coverage_gained(bs, ms) + if gained and verdict != "COVERAGE-GAINED": + res["COVERAGE-GAINED"].append((key, [f"{cls}: baseline unobserved -> main observed" for cls in gained])) + lost = coverage_lost(bs, ms) + if lost: + res["COVERAGE-LOST"].append((key, [f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost])) + bj, mj = bjunit.get(lane, {}), mjunit.get(lane, {}) + jworse, jbetter = [], [] + bad = {"fail", "error"} + for test in sorted(set(bj) | set(mj)): + bs, ms = bj.get(test), mj.get(test) + if bs is not None and ms is None: + if bs != "skipped": + jworse.append((test, [f"missing on main (baseline {bs})"])) + elif bs is None: + continue + elif ms in bad and bs not in bad: + jworse.append((test, [f"{bs} -> {ms}"])) + elif bs in bad and ms not in bad: + jbetter.append((test, [f"{bs} -> {ms}"])) + elif bs == "error" and ms == "fail": + jbetter.append((test, ["error -> fail"])) + elif bs == "fail" and ms == "error": + jworse.append((test, ["fail -> error (crash/timeout)"])) + print( + f"| {lane} | {len(b)}/{len(m)} | {len(res['WORSE'])} | {len(res['BETTER'])} | {len(res['EQUAL'])}" + f" | {len(res['COVERAGE-GAINED'])} | {len(res['COVERAGE-LOST'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |" + ) + out[lane] = {k: v for k, v in res.items()} + out[lane]["JUNIT-WORSE"] = jworse + out[lane]["JUNIT-BETTER"] = jbetter + for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "COVERAGE-LOST", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"): + items = out[lane].get(label) or [] + if not items: + continue + detail.append(f"\n## {lane}: {label} ({len(items)})\n") + for key, reasons in items: + detail.append(f"- `{key}`" + (f": {'; '.join(reasons)}" if reasons else "")) + print("\n".join(detail)) + if args.json: + with open(args.json, "w") as fh: + json.dump(out, fh, indent=1, default=list) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/floor.py b/scripts/verify/floor.py new file mode 100644 index 00000000000..a646b641646 --- /dev/null +++ b/scripts/verify/floor.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Executed-case floor for an engine-verify run. + + floor.py --matrix JSON --tier fast|full REPORTS_DIR + floor.py --recording --runners JSON --tier fast|full REPORTS_DIR + +REPORTS_DIR holds the downloaded report artifacts +(verify-report---/). For every platform and suite the +plan ran, the distinct case ids whose id starts with the suite's `cases.prefix` +(scripts/verify/suites.json) must reach the floor for this tier and platform, +and a suite with an exact `test` name must appear in the run's junit. A suite +that silently shrinks (a renamed test, a skip glob or catalog edit that drops +cases, a shard that never ran) fails the gate instead of passing vacuously. +With --recording, REPORTS_DIR holds the real-capture artifacts +(verify-recording-/verify-report): every runner must have recorded at +least suites.json's `recording.cases` floor of distinct real-capture cases. +Prints one line per check; exit status 1 when any check fails. +""" + +import argparse +import glob +import json +import os +import sys +import xml.etree.ElementTree as ET + +ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + + +def platform_reports(reports, platform, tier): + return sorted(glob.glob(os.path.join(reports, f"verify-report-{platform}-{tier}-*"))) + + +def case_ids(dirs): + ids = set() + for d in dirs: + # Retry and flaky-rerun records count too: a timeout can stop a test + # before its later cases write, and its retry then runs them. + for path in glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True): + with open(path, encoding="utf-8") as f: + for line in f: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if record.get("case"): + ids.add(record["case"]) + return ids + + +def junit_tests(dirs): + names = set() + for d in dirs: + path = os.path.join(d, "junit.xml") + if not os.path.exists(path): + continue + for case in ET.parse(path).getroot().iter("testcase"): + names.add(case.get("name", "")) + return names + + +def check(config, matrix, tier, reports): + lines, failed = [], 0 + suites = {s["name"]: s for s in config["suites"]} + for entry in matrix.get("include", []): + platform = entry["platform"] + dirs = platform_reports(reports, platform, tier) + expected_shards = int(entry.get("shards", 1)) + if len(dirs) < expected_shards: + failed += 1 + lines.append(f"FAIL {platform}: {len(dirs)} of {expected_shards} shard reports present") + ids = case_ids(dirs) + tests = junit_tests(dirs) + for name in [n for n in entry["suites"].split(",") if n]: + suite = suites[name] + floors = suite.get("cases", {}).get(tier, {}) + floor = floors.get(platform, floors.get(entry.get("os", platform))) + if floor is not None: + prefix = suite["cases"]["prefix"] + ran = sum(1 for i in ids if i.startswith(prefix)) + ok = ran >= floor + failed += not ok + lines.append( + f"{'ok ' if ok else 'FAIL'} {platform} {name}: {ran} cases ran (floor {floor}, prefix {prefix})" + ) + if suite.get("test"): + ok = suite["test"] in tests + failed += not ok + lines.append( + f"{'ok ' if ok else 'FAIL'} {platform} {name}: test {suite['test']} {'ran' if ok else 'is missing from the junit report'}" + ) + return lines, failed + + +def check_recording(config, runners, tier, reports): + lines, failed = [], 0 + floor = config.get("recording", {}).get("cases", {}).get(tier, 0) + for runner in runners: + d = os.path.join(reports, f"verify-recording-{runner}") + ids = { + i for i in case_ids([d]) + if i.startswith("recording/") and "(self-check" not in i + } + ok = os.path.isdir(d) and len(ids) >= floor + failed += not ok + lines.append(f"{'ok ' if ok else 'FAIL'} {runner}: {len(ids)} real-capture cases recorded (floor {floor})") + return lines, failed + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--matrix") + parser.add_argument("--recording", action="store_true") + parser.add_argument("--runners") + parser.add_argument("--tier", required=True) + parser.add_argument("reports") + args = parser.parse_args() + with open(os.path.join(ROOT, "scripts", "verify", "suites.json")) as f: + config = json.load(f) + if args.recording: + lines, failed = check_recording(config, json.loads(args.runners), args.tier, args.reports) + else: + lines, failed = check(config, json.loads(args.matrix), args.tier, args.reports) + print("\n".join(lines)) + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/guards.py b/scripts/verify/guards.py new file mode 100644 index 00000000000..89cd9f57dfa --- /dev/null +++ b/scripts/verify/guards.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Runs the guard sentinels listed in scripts/verify/guards.toml. + + guards.py [--profile PROFILE] [--summary FILE] + +A sentinel that fails, or that no longer exists, is reported as a WARNING +(a ::warning annotation and a "Guard changed: needs deep review" section in +the summary file), never as a failure: the guards are equivalent mutants, so +a change to them needs review rather than a red build. Exit status is 0 +unless cargo itself cannot run. +""" + +import argparse +import os +import subprocess +import sys +import tomllib + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--profile", default=os.environ.get("CAP_VERIFY_CARGO_PROFILE", "dev")) + parser.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY")) + args = parser.parse_args() + with open(os.path.join(ROOT, "scripts", "verify", "guards.toml"), "rb") as f: + guards = tomllib.load(f).get("guard", []) + changed = [] + for g in guards: + cmd = ["cargo", "test", "--profile", args.profile, "-p", g["package"], g["target"], "--", "--exact", g["test"]] + if os.environ.get("CAP_VERIFY_LOCKED") == "1": + cmd.insert(2, "--locked") + print(f"guard {g['id']}: {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + output = result.stdout + result.stderr + if result.returncode == 0 and "running 1 test" in output: + print(f"guard {g['id']}: sentinel holds") + continue + if result.returncode == 0: + reason = f"sentinel {g['test']} did not run (renamed or removed?)" + elif "test result: FAILED" in output: + reason = f"sentinel {g['test']} fails" + else: + sys.stdout.write(output[-4000:]) + sys.exit(f"guard {g['id']}: cargo failed before running the sentinel") + print(output[-3000:]) + print(f"::warning title=Guard changed: {g['id']} ({g['area']})::{reason}. {g['prevents']}") + changed.append((g, reason)) + if changed and args.summary: + lines = ["## Guard changed: needs deep review", ""] + lines.append("A sentinel of a guard whose removal changes no checked output now fails. This is a warning, not a failure: review the change to the guard closely.") + lines.append("") + for g, reason in changed: + lines.append(f"- **{g['id']}** ({g['area']}; {', '.join(g['symbols'])}): {reason}. {g['why']} Prevents: {g['prevents']} Repro: `cargo test -p {g['package']} {g['target']} -- --exact {g['test']}`") + with open(args.summary, "a") as f: + f.write("\n".join(lines) + "\n\n") + print(f"guards: {len(guards) - len(changed)} of {len(guards)} sentinels hold") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/guards.toml b/scripts/verify/guards.toml new file mode 100644 index 00000000000..e2f0cae7cac --- /dev/null +++ b/scripts/verify/guards.toml @@ -0,0 +1,100 @@ +# Code that protects users but that the engine suites cannot see change. +# +# [[region]]: high-risk code. A pull request whose diff touches one gets a +# "needs deep review" warning (job summary, annotation, and the +# needs-deep-review label where the token may add labels), and Engine verify +# plus real capture run the FULL tier (real capture on macos-15 and +# macos-26). Paths are globs relative to the repository root; `/**` matches +# everything below a directory. +# +# [[guard]]: a protective guard whose removal changes no output of the suites +# today (an equivalent mutant), so only a sentinel notices. The sentinel is a +# small behaviour test of the unit's own API. It runs on every Linux Engine +# verify job; a failing sentinel, or a diff touching the guard's files, is a +# WARNING ("Guard changed: needs deep review"), never a CI failure. To cover +# a new equivalent mutant, add a sentinel test and an entry here. +# +# Read by scripts/verify/plan.py (`risk`) and scripts/verify/guards.py. + +[[region]] +name = "encoder timestamps" +paths = [ + "crates/enc-ffmpeg/src/base.rs", + "crates/enc-ffmpeg/src/video/**", + "crates/enc-ffmpeg/src/mux/**", + "crates/enc-avfoundation/src/**", + "crates/enc-mediafoundation/src/**", +] + +[[region]] +name = "audio mixing and Studio Sound" +paths = [ + "crates/audio/src/**", + "crates/editor/src/audio.rs", + "crates/editor/src/audio_output.rs", + "crates/enc-ffmpeg/src/audio/**", +] + +[[region]] +name = "decoder seek and frame selection" +paths = [ + "crates/rendering/src/decoder/**", + "crates/video-decode/src/**", +] + +[[region]] +name = "recording timestamps and sync" +paths = [ + "crates/recording/src/output_pipeline/**", + "crates/recording/src/sources/**", + "crates/recording/src/feeds/**", + "crates/recording/src/sync_calibration.rs", + "crates/timestamp/src/**", +] + +[[region]] +name = "Instant finalize" +paths = [ + "crates/recording/src/instant_recording.rs", + "crates/recording/src/instant_recording/**", +] + +[[region]] +name = "recovery" +paths = [ + "crates/recording/src/recovery.rs", + "crates/recording/src/recovery/**", + "crates/recording/src/track_heal.rs", +] + +[[region]] +name = "upload retry" +paths = [ + "apps/desktop-gpui/src/upload.rs", + "apps/desktop-gpui/src/upload/**", + "crates/recording/src/upload_resume.rs", +] + +[[guard]] +id = "E02" +mutation = "E02" +area = "encoder timestamps" +files = ["crates/enc-ffmpeg/src/base.rs"] +symbols = ["EncoderBase::update_pts"] +package = "cap-enc-ffmpeg" +target = "--lib" +test = "base::tests::guard_e02_rebases_a_stream_whose_first_pts_is_not_zero" +why = "The encoder rebases a stream whose first frame has a non-zero pts so it starts at 0. Exports already start at 0, so removing the rebase changes no export the suites check." +prevents = "A recording or export whose first frame is not at 0 would start with a gap or frozen first frame and its audio would drift from the picture by that offset." + +[[guard]] +id = "A09" +mutation = "A09" +area = "audio mixing and Studio Sound" +files = ["crates/audio/src/voice.rs", "crates/editor/src/audio.rs"] +symbols = ["VoiceEnhancer::render", "VoiceEnhancementCache::render"] +package = "cap-audio" +target = "--lib" +test = "voice::tests::guard_a09_render_after_a_seek_equals_a_fresh_enhancer" +why = "A Studio Sound render that does not continue the previous one restarts the enhancer (the reset in VoiceEnhancer::render), and the editor cache never reuses an enhancer across a seek. Either alone keeps the output right, so removing one changes nothing the suites see. The sentinel checks the property itself (a render after a seek equals a fresh one); removing only the reset leaves its output bit-identical too, so that edit is flagged by the path warning, not by the sentinel." +prevents = "Studio Sound after a seek or scrub would carry denoiser and leveler state from another part of the recording: a burst of wrong gain or suppressed speech right after every seek, different from the export." diff --git a/scripts/verify/mtime.py b/scripts/verify/mtime.py new file mode 100644 index 00000000000..4de0229d654 --- /dev/null +++ b/scripts/verify/mtime.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Lets cargo reuse cached builds of unchanged workspace crates in CI. + +Cargo decides whether a workspace crate is fresh by comparing the mtimes of +its source files with its last build. A fresh checkout gives every file the +current time, so every workspace crate would rebuild even when the Rust cache +holds an up-to-date build. + + mtime.py record after a build: record the git blob id of every + tracked file (stored inside the cached target) + mtime.py apply before building: files whose blob id matches + the manifest get an old fixed mtime, so cargo + sees them as unchanged; every other file keeps + the checkout time and its crate rebuilds. With + no manifest (cold build) every file gets the + fixed time. + +Only content equality is trusted, never commit times, so a stale build cannot +be reused for changed code. +""" + +import os +import subprocess +import sys + +OLD = 946684800 # 2000-01-01 + + +def blobs(): + out = subprocess.run( + ["git", "ls-files", "-s", "-z"], check=True, capture_output=True + ).stdout.decode("utf-8", "surrogateescape") + entries = {} + for item in out.split("\0"): + if not item: + continue + meta, path = item.split("\t", 1) + entries[path] = meta.split()[1] + # The index ids only describe the files when the work tree matches it. + dirty = subprocess.run( + ["git", "diff", "--name-only", "-z"], check=True, capture_output=True + ).stdout.decode("utf-8", "surrogateescape") + for path in filter(None, dirty.split("\0")): + entries[path] = "dirty" + return entries + + +def main(): + if len(sys.argv) != 3 or sys.argv[1] not in ("record", "apply"): + sys.exit(__doc__) + command, manifest = sys.argv[1], sys.argv[2] + current = blobs() + if command == "record": + os.makedirs(os.path.dirname(os.path.abspath(manifest)), exist_ok=True) + with open(manifest, "w", encoding="utf-8", errors="surrogateescape") as f: + for path, blob in sorted(current.items()): + f.write(f"{blob} {path}\n") + print(f"recorded {len(current)} files in {manifest}") + return + saved = None + if os.path.exists(manifest): + saved = {} + with open(manifest, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + blob, _, path = line.rstrip("\n").partition(" ") + saved[path] = blob + else: + # Cold build: give every file the fixed time too. Build scripts + # without rerun-if-changed fingerprint the newest mtime of their + # package by equality, so the build that gets cached must see the + # same times the next run will set. + print(f"no manifest at {manifest}: cold build, normalising every file") + same = changed = 0 + for path, blob in current.items(): + if ( + blob != "dirty" + and (saved is None or saved.get(path) == blob) + and os.path.isfile(path) + and not os.path.islink(path) + ): + os.utime(path, (OLD, OLD)) + same += 1 + else: + changed += 1 + print(f"{same} files unchanged since the cached build, {changed} changed or new") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/plan.py b/scripts/verify/plan.py new file mode 100755 index 00000000000..83d06ca3152 --- /dev/null +++ b/scripts/verify/plan.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Plans engine verification runs from scripts/verify/suites.json. + + plan.py select --tier fast --platform linux [--suites a,b] [--local] + Prints four lines for run.sh: the suite names, the cargo target + selection, the nextest filter expression and the case globs to skip. + + plan.py changes --tier fast [--files FILE | --all] [--metadata FILE] + Maps changed paths to suites through the workspace dependency graph + (`cargo metadata`) and prints GitHub step outputs: run, reason, + suites, matrix. + plan.py recording [--files FILE | --all] + run=, reason= for the real-capture workflow: whether a change reaches + the recorder's or the stimulus's dependency closure or the harness. + plan.py risk --files F [--summary FILE] [--output FILE] + changed files -> high-risk regions and guards (scripts/verify/guards.toml): + escalate=, regions=, guards= lines plus ::warning annotations. + + plan.py test + Self-test of the change mapping. +""" + +import argparse +import fnmatch +import json +import os +import subprocess +import sys + +ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) +PLATFORMS = ["linux", "macos", "windows"] + + +def load_guards(): + import tomllib + + with open(os.path.join(ROOT, "scripts", "verify", "guards.toml"), "rb") as f: + return tomllib.load(f) + + +# Engine crates: a pull request that changes any of them runs the FULL tier +# (with Blacksmith runners there is no queue to save). +ENGINE_CRATES = [ + "crates/rendering/**", + "crates/editor/**", + "crates/export/**", + "crates/recording/**", + "crates/enc-*/**", + "crates/audio/**", + "crates/video-decode/**", + "crates/project/**", + "crates/scap-*/**", + "crates/camera*/**", + "crates/media/**", + "crates/media-info/**", + "crates/frame-converter/**", + "crates/gpu-converters/**", + "crates/cursor-capture/**", + "crates/timestamp/**", + "crates/cap-muxer/**", +] + + +def engine_files(files): + return sorted({p for p in files for pattern in ENGINE_CRATES if match(p, pattern)}) + + +def risk(files, guards=None): + """(regions touched, guards whose files are touched) for changed files.""" + guards = guards or load_guards() + regions = [] + for region in guards.get("region", []): + hits = sorted({p for p in files for pattern in region["paths"] if match(p, pattern)}) + if hits: + regions.append((region["name"], hits)) + touched = [ + g for g in guards.get("guard", []) if any(p in g["files"] for p in files) + ] + return regions, touched + + +def risk_report(regions, touched): + """Job-summary markdown and annotation lines for a risky diff.""" + if not regions and not touched: + return "", [] + md = ["## Needs deep review", ""] + notes = [] + if regions: + md.append("This change touches high-risk engine code. A pull request that does runs the FULL tier, with real capture on both macOS versions:") + md.append("") + for name, hits in regions: + md.append(f"- **{name}**: " + ", ".join(f"`{h}`" for h in hits[:6]) + (" ..." if len(hits) > 6 else "")) + notes.append(f"::warning title=Needs deep review ({name})::{', '.join(hits[:6])}") + md.append("") + if touched: + md.append("### Guard changed: needs deep review") + md.append("") + md.append("These guards protect users, but removing them changes no output the suites check (equivalent mutants), so only review can catch a regression:") + md.append("") + for g in touched: + md.append(f"- **{g['id']}** ({g['area']}; {', '.join(g['symbols'])}): {g['why']} Prevents: {g['prevents']}") + notes.append(f"::warning title=Guard changed: {g['id']} ({g['area']})::{g['prevents']}") + md.append("") + return "\n".join(md), notes + + +def load_config(): + with open(os.path.join(ROOT, "scripts", "verify", "suites.json")) as f: + return json.load(f) + + +def match(path, pattern): + if pattern.endswith("/**"): + prefix = pattern[:-2] + if any(c in prefix for c in "*?["): + return fnmatch.fnmatchcase(path, prefix + "*") + return path.startswith(prefix) + return fnmatch.fnmatchcase(path, pattern) + + +def suite_filter(suite): + parts = [] + for binary in suite["binaries"]: + expr = f"binary_id({binary})" + if suite.get("test"): + expr = f"({expr} and test(={suite['test']}))" + parts.append(expr) + return " or ".join(parts) + + +def select(config, tier, platform, names=None, local=False): + # Locally, platform-independent suites (CI runs them on linux only) run + # on every platform too. + suites = [ + s for s in config["suites"] + if platform in s["platforms"][tier] or (local and s["platforms"][tier] == ["linux"]) + ] + if names: + unknown = set(names) - {s["name"] for s in config["suites"]} + if unknown: + sys.exit(f"unknown suites: {', '.join(sorted(unknown))}") + suites = [s for s in suites if s["name"] in names] + return suites + + +def cargo_args(suites): + packages, flags = [], [] + for suite in suites: + args = suite["cargo"] + i = 0 + while i < len(args): + if args[i] == "-p": + if args[i + 1] not in packages: + packages.append(args[i + 1]) + i += 2 + continue + if args[i] == "--test": + flag = ["--test", args[i + 1]] + i += 2 + else: + flag = [args[i]] + i += 1 + if flag not in flags: + flags.append(flag) + out = [] + for package in packages: + out += ["-p", package] + for flag in flags: + out += flag + return out + + +def cmd_select(args): + config = load_config() + names = [n for n in (args.suites or "").split(",") if n] + suites = select(config, args.tier, args.platform, names or None, args.local) + print(" ".join(s["name"] for s in suites)) + print(" ".join(cargo_args(suites))) + print(" or ".join(f"({suite_filter(s)})" for s in suites)) + skips = [] + for suite in suites: + for glob in suite.get("skip", {}).get(args.tier, {}).get(args.platform, []): + if glob not in skips: + skips.append(glob) + print(",".join(skips)) + + +def cmd_env(args): + """Shell `export` lines for the env of the selected suites (suites.json + `env`: {"KEY": "value"} or {"KEY": {"": "value"}}) and the + platform's unsupported-case allowlist. Variables already set in the + environment win.""" + import shlex + config = load_config() + names = [n for n in (args.suites or "").split(",") if n] + allow = ",".join(config.get("unsupported", {}).get(args.platform, {}).keys()) + if allow and "CAP_VERIFY_UNSUPPORTED_ALLOW" not in os.environ: + print(f"export CAP_VERIFY_UNSUPPORTED_ALLOW={shlex.quote(allow)}") + for suite in select(config, args.tier, args.platform, names or None, args.local): + for key, value in suite.get("env", {}).items(): + if isinstance(value, dict): + value = value.get(args.platform) + if value is None or key in os.environ: + continue + print(f"export {key}={shlex.quote(str(value))}") + + +def workspace_graph(metadata): + """Package dirs and reverse dependencies (normal, dev and build).""" + dirs, deps = {}, {} + for package in metadata["packages"]: + name = package["name"] + dirs[name] = os.path.relpath(os.path.dirname(package["manifest_path"]), metadata["workspace_root"]).replace(os.sep, "/") + deps[name] = [ + (d["name"], d.get("kind") or "normal") + for d in package["dependencies"] + if d.get("path") + ] + return dirs, deps + + +def closure(root, deps): + """Packages whose code ends up in `root`'s tests: root's normal, dev and + build dependencies, then normal and build dependencies transitively.""" + seen = {root} + stack = [(name, kind) for name, kind in deps.get(root, [])] + while stack: + name, _kind = stack.pop() + if name in seen or name not in deps: + continue + seen.add(name) + stack.extend((n, k) for n, k in deps[name] if k != "dev") + return seen + + +def owning_package(path, dirs): + best = None + for name, d in dirs.items(): + if d == ".": + continue + if path == d or path.startswith(d + "/"): + if best is None or len(d) > len(dirs[best]): + best = name + return best + + +def affected(config, files, metadata, tier): + """Returns (suite names, reason).""" + all_names = [s["name"] for s in config["suites"]] + for path in files: + if path.startswith("apps/desktop-gpui/"): + return all_names, f"{path} changes the standalone GPUI application" + for pattern in config["always"]: + if match(path, pattern): + return all_names, f"{path} affects every suite" + dirs, deps = workspace_graph(metadata) + closures = {s["name"]: set().union(*(closure(r, deps) for r in s["roots"])) for s in config["suites"]} + owned = {} + for suite in config["suites"]: + for pattern in suite["own"]: + owned.setdefault(pattern, set()).add(suite["name"]) + selected, reasons = set(), [] + for path in files: + package = owning_package(path, dirs) + if package is None: + continue + owners = set() + for pattern, names in owned.items(): + if match(path, pattern): + owners |= names + hits = owners if owners else {n for n, c in closures.items() if package in c} + if hits - selected: + reasons.append(f"{path} ({package}) -> {', '.join(sorted(hits))}") + selected |= hits + names = [n for n in all_names if n in selected] + if not names: + return [], "no engine code changed" + return names, "; ".join(reasons[:8]) + (" ..." if len(reasons) > 8 else "") + + +def extra_runners(config, names, tier, files): + """Additional platform entries (suites.json `extra_runners`): the same + platform's suites on another machine class, for a path the main runner + cannot cover (macOS export on the VideoToolbox hardware encoder, which + Blacksmith VMs lack). Included for every changed path that matches the + entry's `paths`, and always when `files` is None (a full run).""" + out = [] + for name, entry in config.get("extra_runners", {}).items(): + suites = [ + s["name"] for s in config["suites"] + if s["name"] in names and s["name"] in entry["suites"] and entry["os"] in s["platforms"][tier] + ] + if not suites: + continue + if files is not None and not any(match(f, p) for f in files for p in entry["paths"]): + continue + shards = entry["shards"][tier] if isinstance(entry["shards"], dict) else entry["shards"] + out.append({ + "platform": name, + "os": entry["os"], + "runner": os.environ.get(f"ENGINE_VERIFY_{name.upper().replace('-', '_')}_RUNNER", "").strip() or entry["runner"], + "target": entry["target"], + "shards": shards, + "shard_list": json.dumps(list(range(1, shards + 1))), + "archive": entry.get("archive", False), + "suites": ",".join(suites), + }) + return out + + +def matrix(config, names, tier, files=None): + include = [] + for platform in PLATFORMS: + suites = [s["name"] for s in config["suites"] if s["name"] in names and platform in s["platforms"][tier]] + if not suites: + continue + runner = dict(config["runners"][platform]) + # One switch per platform without a code change: the repository + # variable ENGINE_VERIFY__RUNNER (passed in by + # engine-verify.yml) overrides suites.json, e.g. macos-15 to go back + # to GitHub-hosted runners. + override = os.environ.get(f"ENGINE_VERIFY_{platform.upper()}_RUNNER", "").strip() + if override: + runner["runner"] = override + if isinstance(runner["shards"], dict): + runner["shards"] = runner["shards"][tier] + include.append({ + "platform": platform, + "os": platform, + "runner": runner["runner"], + "target": runner["target"], + "shards": runner["shards"], + "shard_list": json.dumps(list(range(1, runner["shards"] + 1))), + "archive": runner.get("archive", False), + "suites": ",".join(suites), + }) + include += extra_runners(config, names, tier, files) + return {"include": include} + + +def load_metadata(path): + if path: + with open(path) as f: + return json.load(f) + env = dict(os.environ) + env.setdefault("RUSTUP_TOOLCHAIN", "stable") + out = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=ROOT, env=env, check=True, capture_output=True, text=True, + ).stdout + return json.loads(out) + + +def cmd_changes(args): + config = load_config() + files = None + if args.all: + names, reason = [s["name"] for s in config["suites"]], "full run" + else: + with open(args.files) as f: + files = [line.strip() for line in f if line.strip()] + if len(files) >= args.max_files: + names, reason = [s["name"] for s in config["suites"]], f"{len(files)} changed files (list may be truncated)" + files = None + else: + names, reason = affected(config, files, load_metadata(args.metadata), args.tier) + if args.suites: + wanted = [n for n in args.suites.split(",") if n] + unknown = set(wanted) - {s["name"] for s in config["suites"]} + if unknown: + sys.exit(f"unknown suites: {', '.join(sorted(unknown))}") + names = [n for n in names if n in wanted] + reason += f" (limited to {args.suites})" + m = matrix(config, names, args.tier, files) + lines = { + "run": "true" if m["include"] else "false", + "reason": reason.replace("\n", " "), + "suites": ",".join(names), + "matrix": json.dumps(m, separators=(",", ":")), + } + for key, value in lines.items(): + print(f"{key}={value}") + + +# Real capture (crates/verify/tests/recording.rs): the recorder and the +# stimulus, their whole dependency closure, and the harness itself. +RECORDING = { + "roots": ["cap-recording", "cap-verify-stimulus"], + "own": [ + "crates/verify/tests/recording.rs", + "crates/verify/src/**", + "crates/verify/Cargo.toml", + "crates/verify/expectations.toml", + "crates/verify-expect/**", + "crates/verify-stimulus/**", + ".github/workflows/verify-recording.yml", + ], +} + + +def recording_affected(config, files, metadata): + """(run, reason) for the real-capture workflow.""" + for path in files: + for pattern in config["always"] + RECORDING["own"]: + if match(path, pattern): + return True, f"{path} affects real capture" + dirs, deps = workspace_graph(metadata) + reach = set().union(*(closure(r, deps) for r in RECORDING["roots"])) + for path in files: + # Other suites' test files share the cap-verify package but not the + # recording test binary. + if path.startswith("crates/verify/tests/"): + continue + package = owning_package(path, dirs) + if package in reach: + return True, f"{path} ({package}) is in the recorder's dependency closure" + return False, "no recording code changed" + + +def cmd_recording(args): + config = load_config() + if args.all: + run, reason = True, "full run" + else: + with open(args.files) as f: + files = [line.strip() for line in f if line.strip()] + if len(files) >= args.max_files: + run, reason = True, f"{len(files)} changed files (list may be truncated)" + else: + run, reason = recording_affected(config, files, load_metadata(args.metadata)) + print(f"run={'true' if run else 'false'}") + print(f"reason={reason}") + + +def cmd_risk(args): + with open(args.files) as f: + files = [line.strip() for line in f if line.strip()] + regions, touched = risk(files) + md, notes = risk_report(regions, touched) + if args.summary and md: + with open(args.summary, "a") as f: + f.write(md + "\n") + for note in notes: + print(note) + engine = engine_files(files) + if engine: + print(f"::notice title=Engine crates changed::FULL tier for {', '.join(engine[:6])}{' ...' if len(engine) > 6 else ''}") + outputs = [ + f"escalate={'true' if regions or touched else 'false'}", + f"engine={'true' if engine else 'false'}", + f"regions={','.join(name for name, _ in regions)}", + f"guards={','.join(g['id'] for g in touched)}", + ] + if args.output: + with open(args.output, "a") as f: + f.write("\n".join(outputs) + "\n") + else: + print("\n".join(outputs)) + + +def cmd_test(_args): + config = load_config() + metadata = load_metadata(None) + cases = [ + (["apps/web/app/page.tsx", "README.md"], []), + (["crates/verify/tests/seek.rs"], ["seek"]), + (["crates/verify/tests/seek.rs", "crates/verify/tests/export.rs"], ["export", "seek"]), + (["crates/verify/src/oracle.rs"], [s["name"] for s in config["suites"] if "cap-verify" in s["roots"]]), + (["crates/verify/src/audio_model.rs"], ["audio"]), + (["crates/rendering/src/decoder/ffmpeg.rs"], [s["name"] for s in config["suites"]]), + (["crates/video-decode/src/ffmpeg.rs"], [s["name"] for s in config["suites"]]), + (["Cargo.lock"], [s["name"] for s in config["suites"]]), + (["scripts/verify/run.sh"], [s["name"] for s in config["suites"]]), + (["apps/desktop-gpui/src/main.rs"], [s["name"] for s in config["suites"]]), + (["apps/desktop-gpui/src/editor.rs"], [s["name"] for s in config["suites"]]), + (["apps/desktop-gpui/src/recording.rs"], [s["name"] for s in config["suites"]]), + (["apps/desktop-gpui/src/upload/verify_tests.rs"], [s["name"] for s in config["suites"]]), + (["apps/desktop-gpui/Cargo.toml"], [s["name"] for s in config["suites"]]), + ] + failed = 0 + for files, expected in cases: + names, reason = affected(config, files, metadata, "fast") + ok = sorted(names) == sorted(expected) + failed += not ok + print(f"{'ok ' if ok else 'BAD'} {files} -> {names} ({reason})") + guards = load_guards() + for g in guards.get("guard", []): + if not any(match(f, pattern) for f in g["files"] for r in guards["region"] for pattern in r["paths"]): + failed += 1 + print(f"BAD guard {g['id']}: its files lie outside every high-risk region") + for files, expected in [ + (["crates/enc-ffmpeg/src/base.rs"], (["encoder timestamps"], ["E02"])), + (["crates/editor/src/audio.rs", "README.md"], (["audio mixing and Studio Sound"], ["A09"])), + (["apps/desktop-gpui/src/upload/queue.rs"], (["upload retry"], [])), + (["crates/verify/src/oracle.rs", "apps/web/app/page.tsx"], ([], [])), + ]: + regions, touched = risk(files, guards) + got = ([n for n, _ in regions], [g["id"] for g in touched]) + ok = got == expected + failed += not ok + print(f"{'ok ' if ok else 'BAD'} risk {files} -> {got}") + for files, expected in [ + (["crates/rendering/src/lib.rs"], True), + (["crates/enc-ffmpeg/src/lib.rs"], True), + (["crates/camera-ffmpeg/src/lib.rs"], True), + (["apps/desktop-gpui/src/main.rs", "crates/verify/src/oracle.rs"], False), + ]: + ok = bool(engine_files(files)) == expected + failed += not ok + print(f"{'ok ' if ok else 'BAD'} engine {files} -> {bool(engine_files(files))}") + for files, expected in [ + (["crates/frame-converter/src/lib.rs"], True), + (["crates/recording/src/studio_recording.rs"], True), + (["crates/verify/tests/recording.rs"], True), + (["vendor/wgpu-hal/src/lib.rs"], True), + (["Cargo.toml"], True), + (["crates/verify/tests/export.rs", "apps/web/app/page.tsx"], False), + ]: + got = recording_affected(config, files, metadata)[0] + ok = got == expected + failed += not ok + print(f"{'ok ' if ok else 'BAD'} recording {files} -> {got}") + names = [s["name"] for s in config["suites"]] + for files, expected in [ + (["crates/enc-ffmpeg/src/video/h264.rs"], True), + (["crates/export/src/mp4.rs"], True), + (["crates/rendering/src/decoder/ffmpeg.rs"], False), + (["crates/editor/src/playback.rs"], False), + (None, True), + ]: + got = any(e["platform"] == "macos-vt" for e in matrix(config, names, "fast", files)["include"]) + ok = got == expected + failed += not ok + print(f"{'ok ' if ok else 'BAD'} macos-vt {files} -> {got}") + rendering_only = affected(config, ["crates/rendering/src/decoder/model/mod.rs"], metadata, "fast")[0] + if "decoder-model-ffmpeg" not in rendering_only: + failed += 1 + print(f"BAD decoder model change -> {rendering_only}") + sys.exit(1 if failed else 0) + + +def main(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("select") + p.add_argument("--tier", choices=["fast", "full"], required=True) + p.add_argument("--platform", choices=PLATFORMS, required=True) + p.add_argument("--suites") + p.add_argument("--local", action="store_true") + p.set_defaults(func=cmd_select) + p = sub.add_parser("env") + p.add_argument("--tier", choices=["fast", "full"], required=True) + p.add_argument("--platform", choices=PLATFORMS, required=True) + p.add_argument("--suites") + p.add_argument("--local", action="store_true") + p.set_defaults(func=cmd_env) + p = sub.add_parser("changes") + p.add_argument("--tier", choices=["fast", "full"], required=True) + p.add_argument("--files") + p.add_argument("--all", action="store_true") + p.add_argument("--metadata") + p.add_argument("--max-files", type=int, default=3000) + p.add_argument("--suites") + p.set_defaults(func=cmd_changes) + p = sub.add_parser("recording") + p.add_argument("--files") + p.add_argument("--all", action="store_true") + p.add_argument("--metadata") + p.add_argument("--max-files", type=int, default=3000) + p.set_defaults(func=cmd_recording) + p = sub.add_parser("risk") + p.add_argument("--files", required=True) + p.add_argument("--summary", help="append the markdown report to this file") + p.add_argument("--output", help="append the escalate/regions/guards lines to this file (GITHUB_OUTPUT)") + p.set_defaults(func=cmd_risk) + p = sub.add_parser("test") + p.set_defaults(func=cmd_test) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/retry.py b/scripts/verify/retry.py new file mode 100644 index 00000000000..684585f1c8e --- /dev/null +++ b/scripts/verify/retry.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Decides which failed engine-verify tests may be retried once. + +A failed test is retried only when the machine-starvation sentinel +(crates/verify/src/sentinel.rs) proves the runner starved it right where it +failed, and the failure is one that wall-clock timing can cause: + +- every case it failed is `inconclusive` (a wall-clock class such as + real-time playback `dropped`, under a sentinel wake gap of 100 ms or more + during that case, judged in cap_verify::report::settle), or +- nextest killed it at its timeout and its sentinel log shows the process + stalled for at least STALLED_MS in total (wake gaps of 100 ms or more) in + the TIMEOUT_WINDOW_SECS before it was killed, or +- it panicked in wgpu-core's queue drop ("We timed out while waiting on the + last successful submission", a 6.3 s limit) or on a lost GPU device + ("Device with 'cap-rendering-device' label is invalid") and its sentinel + log shows such stalls in the PANIC_WINDOW_SECS before the panic. + +A gap anywhere else in a test's life is not evidence (an intermittent hang +after an unrelated stall is not excused). At most MAX_RETRIES tests are +retried per job; beyond that the job fails as an unusable machine. + +A test with any FAIL or XPASS case, or that crashed or timed out without +starvation evidence, is not retried: it fails the run as before. The retry +runs with CAP_VERIFY_RETRY=1, so a case that is inconclusive again fails as +`environment`. + +Usage: retry.py (retry.py test: self-test) +Prints one "binarytestevidence" line per test to retry, then +"blocking" with the number of failed tests that cannot be retried. +""" + +import glob +import json +import os +import sys +import xml.etree.ElementTree as ET + +STARVATION_GAP_MS = 100.0 +# Stalled time (sum of wake gaps of at least STARVATION_GAP_MS) that must +# fall inside the failure window. +STALLED_MS = 1000.0 +TIMEOUT_WINDOW_SECS = 60.0 +PANIC_WINDOW_SECS = 15.0 +MAX_RETRIES = 3 +QUEUE_DRAIN_PANIC = "We timed out while waiting on the last successful submission" +# The GPU device was lost (WARP gives up on work that takes too long) and the +# next wgpu call on it panics instead of reporting an error. +DEVICE_LOST_PANIC = "Device with 'cap-rendering-device' label is invalid" + + +def records(report_dir): + out = [] + for path in glob.glob(os.path.join(report_dir, "*.jsonl")): + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + pass + return out + + +def sentinels(report_dir): + """(executable stem, argv, gaps [(start s, ms)], end s) per test process. + The end is the last heartbeat or gap: the sentinel writes one every 2 s, + so it is within about 2 s of when the process stopped.""" + out = [] + for path in glob.glob(os.path.join(report_dir, "sentinel-*.log")): + exe, args, gaps, end = "", [], [], 0.0 + with open(path, encoding="utf-8") as f: + for line in f: + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if "args" in entry: + args = entry["args"] + exe = entry.get("exe", "") + if "alive" in entry: + end = max(end, float(entry["alive"])) + if "gap_ms" in entry: + at, gap = float(entry.get("at", 0.0)), float(entry["gap_ms"]) + gaps.append((at, gap)) + end = max(end, at + gap / 1000.0) + out.append((exe, args, gaps, end)) + return out + + +def stalled(gaps, end, window): + """Total ms of starvation gaps overlapping (end - window, end].""" + start = end - window + total = 0.0 + for at, gap in gaps: + if gap < STARVATION_GAP_MS: + continue + overlap = min(at + gap / 1000.0, end) - max(at, start) + if overlap > 0: + total += overlap * 1000.0 + return total + + +def self_test(): + """Synthetic junit and sentinel logs: only a stall inside the failure + window is evidence, and the per-job cap holds.""" + import tempfile + + failed = 0 + with tempfile.TemporaryDirectory() as d: + def write(path, text): + with open(os.path.join(d, path), "w", encoding="utf-8") as f: + f.write(text) + + tests = [] + for i, (kind, gaps) in enumerate([ + ("timeout", [(890.0, 3000.0)]), + ("timeout", [(30.0, 5000.0)]), + ("timeout", [(880.0, 150.0)]), + ("lost", [(895.0, 2000.0)]), + ("lost", [(100.0, 9000.0)]), + ("timeout", [(890.0, 2000.0)]), + ("timeout", [(890.0, 2000.0)]), + ("timeout", [(890.0, 2000.0)]), + ]): + name = f"t{i}" + lines = [json.dumps({"pid": i, "exe": "export-abc", "args": [name, "--exact"]})] + lines += [json.dumps({"at": at, "gap_ms": gap}) for at, gap in gaps] + lines.append(json.dumps({"alive": 900.0})) + write(f"sentinel-{i}.log", "\n".join(lines) + "\n") + if kind == "timeout": + body = 'killed' + else: + body = f"{DEVICE_LOST_PANIC}" + tests.append(f'{body}') + write("junit.xml", "" + "".join(tests) + "") + import contextlib + import io + + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + sys.argv = ["retry.py", os.path.join(d, "junit.xml"), d] + main() + lines = out.getvalue().splitlines() + retried = {line.split("\t")[1] for line in lines if not line.startswith("blocking")} + blocking = next(int(line.split("\t")[1]) for line in lines if line.startswith("blocking")) + expected_retried = {"t0", "t3", "t5"} + if retried != expected_retried or blocking != 5: + failed += 1 + print(f"{'ok ' if not failed else 'BAD'} retried {sorted(retried)} (expected {sorted(expected_retried)}), blocking {blocking} (expected 5)") + sys.exit(1 if failed else 0) + + +def main(): + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(newline="\n") + if sys.argv[1:] == ["test"]: + self_test() + return + junit, report_dir = sys.argv[1], sys.argv[2] + recs = records(report_dir) + logs = sentinels(report_dir) + blocking = 0 + retried = 0 + for case in ET.parse(junit).getroot().iter("testcase"): + node = case.find("failure") + if node is None: + node = case.find("error") + if node is None: + continue + binary, name = case.get("classname", ""), case.get("name", "") + text = node.text or node.get("message") or "" + mine = [r for r in recs if r.get("case") and r["case"] in text] + bad = [r for r in mine if str(r.get("outcome", "")).lower() in ("fail", "xpass")] + inconclusive = [r for r in mine if str(r.get("outcome", "")).lower() == "inconclusive"] + timed_out = "timeout" in (node.get("type") or "").lower() + # Test binaries are named after their target (export-): match + # the target as well as the test name, which can repeat across them. + target = binary.split("::")[-1].replace("-", "_") + mine_logs = [ + (gaps, end) + for exe, args, gaps, end in logs + if name in args and exe.replace("-", "_").startswith(target) + ] + + def evidence(window): + return max((stalled(gaps, end, window) for gaps, end in mine_logs), default=0.0) + + reason = None + if bad: + pass + elif timed_out: + ms = evidence(TIMEOUT_WINDOW_SECS) + if ms >= STALLED_MS: + reason = f"timed out; the machine stalled the process {ms:.0f} ms in the {TIMEOUT_WINDOW_SECS:.0f} s before the kill" + elif DEVICE_LOST_PANIC in text or QUEUE_DRAIN_PANIC in text: + # Product hazards too (a lost device should be an error, not a + # panic; the renderer should drain before release), excused only + # when the sentinel proves the machine stalled right before. + ms = evidence(PANIC_WINDOW_SECS) + what = "wgpu device lost" if DEVICE_LOST_PANIC in text else "wgpu queue drain timed out" + if ms >= STALLED_MS: + reason = f"{what}; the machine stalled the process {ms:.0f} ms in the {PANIC_WINDOW_SECS:.0f} s before the panic" + elif inconclusive: + cases = ", ".join(r["case"] for r in inconclusive) + reason = f"inconclusive under starvation: {cases}" + if reason is None: + blocking += 1 + elif retried >= MAX_RETRIES: + blocking += 1 + print(f"::error title=Too many machine stalls::{binary} {name}: {reason} (more than {MAX_RETRIES} stalled tests in one job: the runner is unusable)", file=sys.stderr) + else: + retried += 1 + print(f"{binary}\t{name}\t{reason}") + print(f"blocking\t{blocking}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/run.sh b/scripts/verify/run.sh new file mode 100755 index 00000000000..e5145ffcba5 --- /dev/null +++ b/scripts/verify/run.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# Runs the engine verification suites, locally or in CI. +# +# scripts/verify/run.sh [fast|full] [options] [-- extra nextest args] +# +# --suites a,b only these suites (scripts/verify/suites.json); +# default: every suite of the tier on this platform +# --case GLOB only cases matching GLOB (CAP_VERIFY_CASE) +# --list print the plan and exit +# --check-rerun-selection check that a test name read back from Python +# (as the retry and flaky reruns do) selects exactly +# that test (CRLF on Windows) +# --no-run build only +# --archive FILE build a nextest archive (CI build job) +# --from-archive FILE run from a nextest archive (CI shard job) +# --partition hash:M/N run one shard +# +# Uses cargo-nextest when installed (one process per test, parallel), else +# runs `cargo test` per suite. Results: one JSON line per case in +# $CAP_VERIFY_REPORT_DIR (default target/verify-report/-), +# summarised at the end by scripts/verify/summary.py. +# +# Environment: CAP_VERIFY_CARGO_PROFILE (default dev; CI uses ci-verify), +# CAP_VERIFY_NEXTEST_PROFILE (default default; CI uses ci), +# CAP_VERIFY_FIXTURE_DIR (generated fixtures cache). +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$root" + +tier=fast +suites="" +mode=run +archive="" +partition="" +prepare_fixtures=0 +extra=() +while [[ $# -gt 0 ]]; do + case "$1" in + fast|full) tier="$1" ;; + --suites) suites="$2"; shift ;; + --case) export CAP_VERIFY_CASE="$2"; shift ;; + --list) mode=list ;; + --check-rerun-selection) mode=check-selection ;; + --no-run) mode=build ;; + --prepare-fixtures) prepare_fixtures=1 ;; + --archive) mode=archive; archive="$2"; shift ;; + --from-archive) mode=from-archive; archive="$2"; shift ;; + --partition) partition="$2"; shift ;; + --) shift; extra=("$@"); break ;; + -h|--help) sed -n '2,24p' "$0"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +case "$(uname -s)" in + Darwin) platform=macos ;; + Linux) platform=linux ;; + MINGW*|MSYS*|CYGWIN*) platform=windows ;; + *) echo "unsupported platform $(uname -s)" >&2; exit 2 ;; +esac + +py="$(command -v python3 || command -v python)" +local_flag="" +[[ -z "${CI:-}" ]] && local_flag=--local +plan="$("$py" scripts/verify/plan.py select --tier "$tier" --platform "$platform" ${suites:+--suites "$suites"} $local_flag)" +suite_names="$(sed -n 1p <<<"$plan")" +read -r -a cargo_args <<<"$(sed -n 2p <<<"$plan")" +filter="$(sed -n 3p <<<"$plan")" +skip="$(sed -n 4p <<<"$plan")" +eval "$("$py" scripts/verify/plan.py env --tier "$tier" --platform "$platform" ${suites:+--suites "$suites"} $local_flag)" +if [[ -n "$skip" ]]; then + export CAP_VERIFY_SKIP="${CAP_VERIFY_SKIP:+$CAP_VERIFY_SKIP,}$skip" +fi + +if [[ -z "$suite_names" ]]; then + echo "no $tier suites for $platform${suites:+ in $suites}" + exit 0 +fi + +# CAP_VERIFY_CARGO_CONFIG: extra `--config KEY=VALUE` overrides, `;` separated +# (profile experiments). CAP_VERIFY_LOCKED=1 adds --locked. +if [[ -n "${CAP_VERIFY_CARGO_CONFIG:-}" ]]; then + IFS=';' read -r -a overrides <<<"$CAP_VERIFY_CARGO_CONFIG" + for override in "${overrides[@]}"; do + [[ -n "$override" ]] && cargo_args+=(--config "$override") + done +fi +if [[ "${CAP_VERIFY_LOCKED:-}" == 1 ]]; then + cargo_args+=(--locked) +fi + +profile="${CAP_VERIFY_CARGO_PROFILE:-dev}" +nextest_profile="${CAP_VERIFY_NEXTEST_PROFILE:-default}" +export CAP_VERIFY_TIER="$tier" +export CAP_VERIFY_REPORT_DIR="${CAP_VERIFY_REPORT_DIR:-$root/target/verify-report/$platform-$tier}" + +echo "engine verify: $tier tier on $platform: $suite_names${CAP_VERIFY_SHARD:+ (case shard $CAP_VERIFY_SHARD)}" +if [[ "$mode" == list ]]; then + echo "cargo: ${cargo_args[*]}" + echo "filter: $filter" + echo "skip: ${CAP_VERIFY_SKIP:-}" + exit 0 +fi + +# The crash suite spawns the out-of-process muxer; build it with the same +# profile so it sits beside the test binaries (target//cap-muxer). +if [[ " ${suite_names//,/ } " == *" crash "* && "$mode" != archive ]]; then + cargo build --profile "$profile" -p cap-muxer ${CAP_VERIFY_LOCKED:+--locked} +fi + +have_nextest=0 +if command -v cargo-nextest >/dev/null 2>&1 || cargo nextest --version >/dev/null 2>&1; then + have_nextest=1 +fi + +if [[ "$mode" == archive || "$mode" == from-archive ]] && [[ $have_nextest == 0 ]]; then + echo "--archive and --from-archive need cargo-nextest (cargo install cargo-nextest --locked)" >&2 + exit 2 +fi + +if [[ "$mode" == check-selection ]]; then + list=(cargo nextest list --color never --cargo-profile "$profile" "${cargo_args[@]}" --message-format oneline) + line="$("${list[@]}" -E "$filter" 2>/dev/null | head -n 1)" + [[ -n "$line" ]] || { echo "::error::no test listed for $filter"; exit 1; } + selected() { "${list[@]}" -E "binary_id($1) and test(=$2)" 2>/dev/null | grep -c . || true; } + raw="$("$py" -c 'import sys; print(sys.argv[1], sys.argv[2], sep="\t")' "${line%% *}" "${line#* }")" + IFS=$'\t' read -r b n <<<"$raw" + echo "without CR stripping: $(selected "$b" "$n") test(s) selected" + IFS=$'\t' read -r b n <<<"$(tr -d '\r' <<<"$raw")" + count="$(selected "$b" "$n")" + if [[ "$count" != 1 ]]; then + echo "::error::the rerun filter for '$b' '$n' selects $count tests" + exit 1 + fi + echo "rerun selection round trip ok: $b $n" + exit 0 +fi + +case "$mode" in + build) + if [[ $have_nextest == 1 ]]; then + exec cargo nextest run --cargo-profile "$profile" "${cargo_args[@]}" --no-run + fi + exec cargo test --profile "$profile" "${cargo_args[@]}" --no-run + ;; + archive) + exec cargo nextest archive --cargo-profile "$profile" "${cargo_args[@]}" --archive-file "$archive" + ;; +esac + +if [[ -z "${CI:-}" ]]; then + rm -rf "$CAP_VERIFY_REPORT_DIR" +fi +mkdir -p "$CAP_VERIFY_REPORT_DIR" + +status=0 +started=$SECONDS +if [[ $have_nextest == 1 ]]; then + # --no-tests=fail: a filter that selects nothing (a renamed test, a typo) + # fails instead of passing. + run=(cargo nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail) + if [[ "$mode" == from-archive ]]; then + # Running from an archive needs no cargo (or toolchain install). + if command -v cargo-nextest >/dev/null; then + run=(cargo-nextest nextest run --profile "$nextest_profile" --no-fail-fast --no-tests=fail) + fi + run+=(--archive-file "$archive" --workspace-remap "$root") + else + run+=(--cargo-profile "$profile" "${cargo_args[@]}") + fi + if [[ $prepare_fixtures == 1 ]]; then + "${run[@]}" --run-ignored ignored-only -E 'binary_id(cap-verify::export) and test(=prepare_recorded_fixtures)' + exit + fi + shard=() + [[ -n "$partition" ]] && shard=(--partition "$partition") + "${run[@]}" ${shard[@]+"${shard[@]}"} -E "$filter" ${extra[@]+"${extra[@]}"} || status=$? + junit="${CARGO_TARGET_DIR:-$root/target}/nextest/$nextest_profile/junit.xml" + # Always kept: the gate checks that every planned test ran (floor.py). + if [[ -f "$junit" ]]; then cp "$junit" "$CAP_VERIFY_REPORT_DIR/junit.xml"; fi + # Starvation retry: a test whose only failures are wall-clock classes + # under a proven machine stall (inconclusive cases, or a timeout with the + # sentinel's evidence) runs once more with CAP_VERIFY_RETRY=1. The run + # passes only if every failed test was retryable and every retry passed; + # a case inconclusive again fails as `environment`. See retry.py. + if [[ $status != 0 && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then + # Python on Windows ends lines with CRLF: strip the CR, or the last + # field (and the test name in a filter) carries it. + retry_plan="$("$py" scripts/verify/retry.py "$CAP_VERIFY_REPORT_DIR/junit.xml" "$CAP_VERIFY_REPORT_DIR" | tr -d '\r')" + blocking="$(sed -n 's/^blocking\t//p' <<<"$retry_plan")" + retried=0 + retry_failed=0 + while IFS=$'\t' read -r binary name evidence; do + [[ -z "$binary" || "$binary" == blocking ]] && continue + retried=$((retried + 1)) + echo "retrying $binary $name once: $evidence" + echo "::warning title=Retried after a machine stall::$binary $name: $evidence" + echo "$binary $name: $evidence" >>"$CAP_VERIFY_REPORT_DIR/retried.txt" + if ! CAP_VERIFY_RETRY=1 CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR/retry" \ + "${run[@]}" -E "binary_id($binary) and test(=$name)"; then + retry_failed=$((retry_failed + 1)) + echo "$binary $name: failed again on retry" >>"$CAP_VERIFY_REPORT_DIR/retried.txt" + fi + done <<<"$retry_plan" + if [[ $retried -gt 0 && "${blocking:-1}" == 0 && $retry_failed == 0 ]]; then + echo "every failure was a proven machine stall and passed on retry" + status=0 + fi + fi + # No other retries: any other failed test is rerun once only to label it + # flaky in the summary. The run still fails. + if [[ $status != 0 && -n "${CAP_VERIFY_FLAKY_FILE:-}" && -f "$CAP_VERIFY_REPORT_DIR/junit.xml" ]]; then + : >"$CAP_VERIFY_FLAKY_FILE" + while IFS=$'\t' read -r binary name; do + [[ -z "$binary" ]] && continue + echo "rerunning $binary $name once to check for flakiness" + rerun=0 + CAP_VERIFY_REPORT_DIR="$CAP_VERIFY_REPORT_DIR.rerun" "${run[@]}" -E "binary_id($binary) and test(=$name)" || rerun=$? + if [[ $rerun == 0 ]]; then + echo "$binary $name" >>"$CAP_VERIFY_FLAKY_FILE" + elif [[ $rerun == 4 ]]; then + # nextest's NO_TESTS_RUN: the filter is wrong, not the test. + echo "::error title=Flaky rerun selected no test::$binary $name" + status=1 + fi + done < <("$py" -c 'import sys, xml.etree.ElementTree as ET +for c in ET.parse(sys.argv[1]).getroot().iter("testcase"): + if c.find("failure") is not None or c.find("error") is not None: + print(c.get("classname"), c.get("name"), sep="\t")' "$CAP_VERIFY_REPORT_DIR/junit.xml" | tr -d '\r' | head -n 5) + fi +else + if [[ $prepare_fixtures == 1 ]]; then + echo "--prepare-fixtures needs cargo-nextest" >&2 + exit 2 + fi + echo "cargo-nextest not found: running suites one by one with cargo test (cargo install cargo-nextest --locked for parallel runs)" + for name in $suite_names; do + plan_one="$("$py" scripts/verify/plan.py select --tier "$tier" --platform "$platform" --suites "$name" --local)" + read -r -a one_args <<<"$(sed -n 2p <<<"$plan_one")" + test_name="$("$py" -c 'import json,sys; c=json.load(open("scripts/verify/suites.json")); print(next(s.get("test","") for s in c["suites"] if s["name"]==sys.argv[1]))' "$name")" + echo "== $name" + if [[ -n "$test_name" ]]; then + cargo test --profile "$profile" "${one_args[@]}" -- --exact "$test_name" ${extra[@]+"${extra[@]}"} || status=$? + else + cargo test --profile "$profile" "${one_args[@]}" -- ${extra[@]+"${extra[@]}"} || status=$? + fi + done +fi + +echo +"$py" scripts/verify/summary.py --title "$platform $tier" --text \ + --junit "$CAP_VERIFY_REPORT_DIR/junit.xml" ${CAP_VERIFY_FLAKY_FILE:+--flaky "$CAP_VERIFY_FLAKY_FILE"} \ + "$CAP_VERIFY_REPORT_DIR" || true +echo "engine verify: $tier tier on $platform finished in $((SECONDS - started)) s (exit $status)" +exit $status diff --git a/scripts/verify/suites.json b/scripts/verify/suites.json new file mode 100644 index 00000000000..de61d65e3eb --- /dev/null +++ b/scripts/verify/suites.json @@ -0,0 +1,374 @@ +{ + "_doc": "Engine verification suites. Read by scripts/verify/plan.py (CI planning, change detection) and scripts/verify/run.sh (local and CI runs). Fields: cargo = the cargo target selection that builds the suite; test = optional exact test name inside that binary; platforms = where each tier runs it (platform-independent logic runs on linux only); roots = workspace packages whose dependency closure decides whether a change can affect the suite; own = files that only affect this suite (a change confined to them selects just this suite). skip = case-id globs left out of a tier on a platform (CAP_VERIFY_SKIP), to keep the FAST tier within its time budget; the FULL tier runs them. Exception: CI Windows VMs (Blacksmith windows-2025, formerly hosted windows-2022) have no D3D11 video device, so every Windows case decodes with FFmpeg (the code Linux runs) and renders on the WARP software rasterizer; on hosted windows-2022 the 120 s jitter-camera export and playback and the 3000-seek storms took 6 to more than 15 minutes each (killed at the 15 minute per-test limit in 4 of 5 full runs), so FULL leaves them to Linux and macOS. cases = the executed-case floor per tier and platform: the gate fails when fewer distinct cases whose id starts with prefix ran across all shards (scripts/verify/floor.py). unsupported = per-platform case globs that may report unsupported (with the reason); CI sets CAP_VERIFY_REQUIRE_SUPPORTED=1, so any other unsupported case fails.", + "suites": [ + { + "name": "verify-self", + "about": "marker codecs, reference timeline model, oracle and report self-tests", + "cargo": ["-p", "cap-verify", "--lib", "--test", "codec"], + "binaries": ["cap-verify", "cap-verify::codec"], + "platforms": { + "fast": ["linux"], + "full": ["linux", "macos"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/codec.rs"] + }, + { + "name": "export", + "about": "every exported frame against the reference model (MP4, GIF, both decoders on macOS)", + "cargo": ["-p", "cap-verify", "--test", "export"], + "binaries": ["cap-verify::export"], + "platforms": { + "fast": ["linux", "macos", "windows"], + "full": ["linux", "macos", "windows"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/export.rs"], + "skip": { + "fast": { + "macos": ["export/*/ffmpeg", "export/backward-hold/*/mp430/*"], + "windows": [ + "export/*/mp460/*", + "export/backward-hold/*", + "export/jitter-camera-*", + "export/*/uhd-*" + ] + }, + "full": { + "windows": ["export/jitter-camera-120s/*"] + } + }, + "cases": { + "prefix": "export/", + "fast": { + "linux": 42, + "macos": 41, + "windows": 43 + }, + "full": { + "linux": 43, + "macos": 69, + "windows": 67 + } + } + }, + { + "name": "playback", + "about": "offline-driven editor playback frames against the reference model", + "cargo": ["-p", "cap-verify", "--test", "playback"], + "binaries": ["cap-verify::playback"], + "platforms": { + "fast": ["linux", "macos"], + "full": ["linux", "macos", "windows"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/playback.rs"], + "skip": { + "fast": { + "macos": ["playback/*/ffmpeg"] + }, + "full": { + "windows": ["playback/jitter-camera-120s/*"] + } + }, + "cases": { + "prefix": "playback/", + "fast": { + "linux": 19, + "macos": 19 + }, + "full": { + "linux": 19, + "macos": 38, + "windows": 36 + } + } + }, + { + "name": "seek", + "about": "random, keyframe and scrub seeks against the reference model", + "cargo": ["-p", "cap-verify", "--test", "seek"], + "binaries": ["cap-verify::seek"], + "platforms": { + "fast": ["linux", "macos"], + "full": ["linux", "macos", "windows"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/seek.rs"], + "skip": { + "fast": { + "macos": ["seek/*/ffmpeg"] + }, + "full": { + "windows": ["seek/*/storm*"] + } + }, + "cases": { + "prefix": "seek/", + "fast": { + "linux": 11, + "macos": 11 + }, + "full": { + "linux": 11, + "macos": 22, + "windows": 16 + } + } + }, + { + "name": "crash", + "about": "crash safety of fragmented Studio recording: in-process vs out-of-process muxer equality, recorder and muxer kills, recovery against what is physically on disk (needs the cap-muxer binary, built by run.sh)", + "cargo": ["-p", "cap-verify", "--test", "crash"], + "binaries": ["cap-verify::crash"], + "test": "crash_safety", + "platforms": { + "fast": ["linux", "macos", "windows"], + "full": ["linux", "macos", "windows"] + }, + "roots": ["cap-verify", "cap-muxer"], + "own": ["crates/verify/tests/crash.rs"], + "cases": { + "prefix": "crash/", + "fast": { + "linux": 7, + "macos": 16, + "windows": 7 + }, + "full": { + "linux": 9, + "macos": 21, + "windows": 9 + } + } + }, + { + "name": "audio", + "about": "export and offline playback audio against sample-accurate markers (mic, system audio, cuts, speeds, mute, volume, A/V offset)", + "cargo": ["-p", "cap-verify", "--test", "audio"], + "binaries": ["cap-verify::audio"], + "platforms": { + "fast": ["linux", "macos", "windows"], + "full": ["linux", "macos", "windows"] + }, + "roots": ["cap-verify"], + "own": [ + "crates/verify/tests/audio.rs", + "crates/verify/tests/audio_support/**", + "crates/verify/src/audio_model.rs", + "crates/verify/src/audio_fixture.rs" + ], + "skip": { + "fast": { + "windows": ["audio/*/mp4-plain30", "audio/*/gif30", "audio/*/mov30"], + "macos": ["audio/*/mp4-plain30", "audio/*/gif30"] + } + }, + "cases": { + "prefix": "audio/", + "fast": { + "linux": 56, + "macos": 40, + "windows": 39 + }, + "full": { + "linux": 58, + "macos": 58, + "windows": 58 + } + } + }, + { + "name": "editing", + "about": "random edit sequences (split, delete, speed, trim, reorder, transitions) exported and checked frame by frame against the reference model", + "cargo": ["-p", "cap-verify", "--test", "editing"], + "binaries": ["cap-verify::editing"], + "platforms": { + "fast": ["linux"], + "full": ["linux"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/editing.rs", "crates/verify/tests/edits/**"], + "cases": { + "prefix": "editing/", + "fast": { + "linux": 22 + }, + "full": { + "linux": 78 + } + } + }, + { + "name": "captions", + "about": "fake-transcriber captions: caption segments and rendered captions on the right frames", + "cargo": ["-p", "cap-verify", "--test", "captions"], + "binaries": ["cap-verify::captions"], + "platforms": { + "fast": ["linux"], + "full": ["linux"] + }, + "roots": ["cap-verify"], + "own": ["crates/verify/tests/captions.rs"], + "cases": { + "prefix": "captions/", + "fast": { + "linux": 19 + }, + "full": { + "linux": 19 + } + } + }, + { + "name": "decoder-model-ffmpeg", + "about": "model-based fuzzing of the FFmpeg decoder's request decisions on scripted media", + "cargo": ["-p", "cap-rendering", "--lib"], + "binaries": ["cap-rendering"], + "test": "decoder::model::ffmpeg_answers_match_the_reference_model", + "platforms": { + "fast": ["linux"], + "full": ["linux"] + }, + "roots": ["cap-rendering"], + "own": [], + "cases": { + "prefix": "decoder-model/ffmpeg/", + "fast": { + "linux": 29 + }, + "full": { + "linux": 29 + } + } + }, + { + "name": "decoder-model-avf", + "about": "model-based fuzzing of the AVAssetReader decoder's request decisions on scripted media", + "cargo": ["-p", "cap-rendering", "--lib"], + "binaries": ["cap-rendering"], + "test": "decoder::model::avassetreader_answers_match_the_reference_model", + "platforms": { + "fast": ["macos"], + "full": ["macos"] + }, + "roots": ["cap-rendering"], + "own": [], + "cases": { + "prefix": "decoder-model/avf/", + "fast": { + "macos": 31 + }, + "full": { + "macos": 31 + } + } + }, + { + "name": "native-rendering", + "about": "native macOS rendering regressions (IOSurface source lifetime in the YUV converter)", + "cargo": ["-p", "cap-rendering", "--test", "iosurface-source-lifetime"], + "binaries": ["cap-rendering::iosurface-source-lifetime"], + "platforms": { + "fast": ["macos"], + "full": ["macos"] + }, + "roots": ["cap-rendering"], + "own": [ + "crates/rendering/tests/iosurface-source-lifetime.rs", + "crates/rendering/tests/expectations.toml" + ], + "cases": { + "prefix": "native/", + "fast": { + "macos": 1 + }, + "full": { + "macos": 1 + } + } + } + ], + "always": [ + ".cargo/**", + ".config/nextest.toml", + ".github/workflows/engine-verify*.yml", + ".github/actions/**", + "Cargo.lock", + "Cargo.toml", + "rust-toolchain.toml", + "scripts/setup.js", + "scripts/verify/**", + "vendor/**" + ], + "runners": { + "linux": { + "runner": "blacksmith-8vcpu-ubuntu-2404", + "target": "x86_64-unknown-linux-gnu", + "shards": { + "fast": 2, + "full": 3 + }, + "archive": false + }, + "macos": { + "runner": "blacksmith-6vcpu-macos-15", + "target": "aarch64-apple-darwin", + "shards": { + "fast": 1, + "full": 2 + }, + "archive": false + }, + "windows": { + "runner": "blacksmith-8vcpu-windows-2025", + "target": "x86_64-pc-windows-msvc", + "shards": { + "fast": 3, + "full": 4 + }, + "archive": false + } + }, + "unsupported": { + "windows": { + "*/mf": "Media Foundation decoding needs a D3D11 device with video support; CI Windows VMs render on the WARP software adapter, which has none, so the product falls back to FFmpeg there. Media Foundation is not exercised by CI." + } + }, + "recording": { + "_doc": "Real capture (verify-recording.yml): distinct case ids each runner must record per tier (crates/verify/tests/recording.rs specs; system-only reports unsupported when the stimulus is silent).", + "cases": { + "fast": 8, + "full": 12 + } + }, + "extra_runners": { + "macos-vt": { + "_doc": "macOS MP4 exports on the VideoToolbox hardware H.264 encoder. Blacksmith macOS VMs have none: Cap opens h264_videotoolbox with allow_sw=0 for exports, which fails there (\"Try -allow_sw 1\", probe run 38853844860), so Cap falls back to libx264, a path real Macs only take when the hardware encoder is busy or over capacity. The suites that encode (export, audio) run again on hosted macOS when a change reaches encoding (encoders, export, the renderer's output and RGBA to NV12 conversion, media and frame conversion) or the suite itself, and in every full run (nightly, dispatch).", + "os": "macos", + "runner": "macos-15", + "target": "aarch64-apple-darwin", + "shards": { + "fast": 1, + "full": 1 + }, + "suites": ["export", "audio"], + "paths": [ + "crates/enc-ffmpeg/**", + "crates/enc-avfoundation/**", + "crates/export/**", + "crates/rendering/src/lib.rs", + "crates/rendering/src/frame_pipeline.rs", + "crates/rendering/src/shaders/**", + "crates/media/**", + "crates/frame-converter/**", + "crates/gpu-converters/**", + "crates/verify/**", + "scripts/verify/**", + ".github/workflows/engine-verify*.yml", + ".github/actions/engine-verify-setup/**", + "Cargo.lock" + ] + } + } +} diff --git a/scripts/verify/summary.py b/scripts/verify/summary.py new file mode 100755 index 00000000000..02a749d4935 --- /dev/null +++ b/scripts/verify/summary.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Summarises engine verification records (JSON lines written to +CAP_VERIFY_REPORT_DIR by every suite) as GitHub job-summary markdown or +plain text. + + summary.py [--title T] [--text] [--junit FILE]... [--flaky FILE] DIR... + +Exit status is 0; the test runner's own exit status decides the job. +""" + +import argparse +import glob +import json +import os +import xml.etree.ElementTree as ET + +MAX_MISMATCHES = 12 +MAX_CASES = 40 + + +def load_records(dirs): + records = [] + for d in dirs: + for path in sorted(glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True)): + # Retry records repeat cases of the main pass (counted with the + # retries below), so they are not cases of their own. + rel = os.path.relpath(path, d).replace(os.sep, "/") + if rel.startswith("retry/"): + continue + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + r["case"] = r.get("case") or r.get("id", "?") + r["outcome"] = str(r.get("outcome", "?")).lower() + records.append(r) + return records + + +def junit_failures(paths): + """(test name, message) for failed or errored junit test cases.""" + out = [] + for path in paths: + if not os.path.exists(path): + continue + for case in ET.parse(path).getroot().iter("testcase"): + for tag in ("failure", "error"): + node = case.find(tag) + if node is not None: + text = (node.text or node.get("message") or "").strip() + kind = node.get("type") or "" + if "timeout" in kind.lower() and "timed out" not in text: + text = f"{text}\n(test timed out: {kind})" + out.append((f"{case.get('classname', '')} {case.get('name', '')}".strip(), text)) + break + return out + + +def crash_excerpt(msg, limit): + """The first panic (message and a few lines after it), else the tail.""" + lines = msg.splitlines() + for i, line in enumerate(lines): + if "timed out" in line: + return [line for line in lines if "[cap-verify]" in line or "timed out" in line][-limit:] + if "panicked at" in line or "error:" in line.lower(): + excerpt = [] + for follow in lines[i:i + 12]: + if "stack backtrace" in follow.lower(): + break + excerpt.append(follow) + return excerpt + return lines[-limit:] + + +def frame(m): + def fmt(v): + return "none" if v is None else f"seg {v[0]} frame {v[1]}" + delta = f", {m['delta']:+d} source frames" if m.get("delta") is not None else "" + detail = f" ({m['detail']})" if m.get("detail") else "" + return ( + f"output frame {m.get('output_frame')} at {m.get('time_secs', 0):.3f} s: {m.get('kind')} " + f"expected {fmt(m.get('expected'))}, got {fmt(m.get('observed'))}{delta}{detail}" + ) + + +def details(r): + lines = [] + summary = (r.get("summary") or "").strip() + if summary: + lines.extend(summary.splitlines()) + for stream in r.get("streams") or []: + mismatches = stream.get("mismatches") or [] + if not mismatches: + continue + name = stream.get("stream") or "stream" + lines.append(f"{name}: first {min(len(mismatches), MAX_MISMATCHES)} of {len(mismatches)} mismatches") + lines.extend(" " + frame(m) for m in mismatches[:MAX_MISMATCHES]) + if not r.get("streams"): + for class_name, problem in (r.get("problems") or {}).items(): + if problem not in summary: + lines.append(f"{class_name}: {problem}") + return lines + + +def retried(dirs): + """Tests retried after a proven machine stall -> (evidence, passed).""" + out = {} + for d in dirs: + path = os.path.join(d, "retried.txt") + if not os.path.exists(path): + continue + with open(path, encoding="utf-8") as f: + for line in f: + test, _, evidence = line.strip().partition(": ") + if evidence == "failed again on retry": + out[test] = (out.get(test, ("", True))[0], False) + elif test: + out[test] = (evidence, out.get(test, ("", True))[1]) + return out + + +def render(records, title, junit, flaky, text, retries=None): + retries = retries or {} + counts = {} + for r in records: + counts[r["outcome"]] = counts.get(r["outcome"], 0) + 1 + bad = [r for r in records if r["outcome"] in ("fail", "xpass")] + xfails = [r for r in records if r["outcome"] == "xfail"] + seconds = sum(float(r.get("seconds") or 0) for r in records) + head = ( + f"{len(records)} cases: {counts.get('pass', 0)} pass, {counts.get('xfail', 0)} expected failures, " + f"{counts.get('fail', 0)} FAILED, {counts.get('xpass', 0)} unexpectedly passing, " + f"{counts.get('unsupported', 0)} unsupported ({seconds:.0f} s of case time)" + ) + excused = {} + for r in records: + for kind in (r.get("excused") or {}).values(): + excused[kind] = excused.get(kind, 0) + 1 + if excused.get("flaky") or excused.get("flaky, starved"): + head += ( + f"; expected failures excused by flaky entries: {excused.get('flaky', 0)}, " + f"only under a proven machine stall: {excused.get('flaky, starved', 0)}" + ) + second = [ + r for r in records + if any(str(n).startswith("inconclusive, retrying") for n in r.get("notes") or []) + ] + if second: + head += f"; {len(second)} real-capture case(s) settled on a second attempt after a proven input fault" + service = 0 + for r in records: + for n in r.get("notes") or []: + n = str(n) + if n.startswith("capture-service drops:"): + try: + service += int(n.split(": ")[2].split(" ")[0]) + except (IndexError, ValueError): + service += 1 + if service: + head += f"; {service} missing picture(s) excused because the capture service dropped them for every client" + unmeasured = [r for r in records if r.get("unobserved")] + if unmeasured: + head += f"; {len(unmeasured)} with classes not measured because an earlier check failed" + # A failed test is explained only by a failing case it recorded. A test + # that timed out or crashed after recording passing or expected cases + # still failed: the cases after the last record never ran. + failing = {r["case"] for r in bad} + crashed = [ + (name, msg) + for name, msg in junit + if not any(c in msg for c in failing) and not retries.get(name, ("", False))[1] + ] + inconclusive = [r for r in records if r["outcome"] == "inconclusive"] + if inconclusive or retries: + passed = sum(1 for _, ok in retries.values() if ok) + head += f"; {len(inconclusive)} inconclusive under a proven machine stall, {passed} of {len(retries)} retried tests passed" + if crashed: + head += f"; {len(crashed)} test(s) FAILED without a failing case (timed out or crashed, later cases not run)" + out = [] + if text: + out.append(f"== engine verify {title}: {head}") + for r in bad: + out.append(f"{r['outcome'].upper()} {r['case']}") + out.extend(" " + line for line in details(r)) + if r.get("repro"): + out.append(f" repro: {r['repro']}") + for test, (evidence, ok) in retries.items(): + out.append(f"RETRIED {test}: {evidence}: {'passed on retry' if ok else 'FAILED again (environment)'}") + for name, msg in crashed: + out.append(f"TEST FAILED {name}") + out.extend(" " + line for line in crash_excerpt(msg, 15)) + for name in flaky: + out.append(f"FLAKY {name}: failed, then passed when rerun (still a failure: fix the cause)") + return "\n".join(out) + + icon = "FAILED" if bad or crashed else "passed" + out.append(f"## Engine verify, {title}: {icon}") + out.append("") + out.append(head) + out.append("") + if flaky: + out.append("### Flaky") + out.append("Failed, then passed when rerun once. The job still fails: find the nondeterminism.") + out.extend(f"- `{name}`" for name in flaky) + out.append("") + if bad: + out.append("### Failures") + for r in bad[:MAX_CASES]: + kind = "unexpectedly passes" if r["outcome"] == "xpass" else "FAILED" + out.append(f"**`{r['case']}`** {kind} ({float(r.get('seconds') or 0):.1f} s)") + out.append("```text") + out.extend(details(r)[:MAX_MISMATCHES * 3]) + out.append("```") + if r.get("repro"): + out.append(f"Repro: `{r['repro']}`") + out.append("") + if len(bad) > MAX_CASES: + out.append(f"... and {len(bad) - MAX_CASES} more (see the report artifact)") + if retries: + out.append("### Retried after a proven machine stall") + out.append("The starvation sentinel recorded the runner not scheduling the test; only wall-clock failures are retried, once.") + out.extend( + f"- `{test}`: {evidence}: {'passed on retry' if ok else '**failed again (environment)**'}" + for test, (evidence, ok) in retries.items() + ) + out.append("") + if crashed: + out.append("### Tests that failed without a case record") + for name, msg in crashed[:MAX_CASES]: + out.append(f"**`{name}`**") + out.append("```text") + out.extend(crash_excerpt(msg, 25)) + out.append("```") + unsupported = [r for r in records if r["outcome"] == "unsupported"] + if unsupported: + out.append(f"
Unsupported on this runner ({len(unsupported)})") + out.append("") + out.extend(f"- `{r['case']}`: {r.get('summary', '')}" for r in unsupported) + out.append("") + out.append("
") + if xfails: + out.append(f"
Expected failures ({len(xfails)})") + out.append("") + out.append("| Case | Classes | Summary |") + out.append("| --- | --- | --- |") + for r in xfails: + classes = ", ".join(sorted((r.get("failures") or {}).keys())) or "-" + unmeasured = sorted((r.get("unobserved") or {}).keys()) + if unmeasured: + classes += f" (not measured: {', '.join(unmeasured)})" + summary = (r.get("summary") or "").splitlines()[0:1] + summary = summary[0].replace("|", "\\|")[:200] if summary else "" + out.append(f"| `{r['case']}` | {classes} | {summary} |") + out.append("") + out.append("
") + slow = sorted(records, key=lambda r: -float(r.get("seconds") or 0))[:10] + if slow: + out.append("
Slowest cases") + out.append("") + out.extend(f"- {float(r.get('seconds') or 0):.1f} s `{r['case']}`" for r in slow) + out.append("") + out.append("
") + return "\n".join(out) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("dirs", nargs="+") + parser.add_argument("--title", default="") + parser.add_argument("--text", action="store_true") + parser.add_argument("--junit", action="append", default=[]) + parser.add_argument("--flaky", help="file with one flaky test name per line") + args = parser.parse_args() + flaky = [] + if args.flaky and os.path.exists(args.flaky): + with open(args.flaky) as f: + flaky = [line.strip() for line in f if line.strip()] + print( + render( + load_records(args.dirs), + args.title, + junit_failures(args.junit), + flaky, + args.text, + retried(args.dirs), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.py new file mode 100644 index 00000000000..83eb1100661 --- /dev/null +++ b/scripts/verify/test_diff.py @@ -0,0 +1,239 @@ +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +spec = importlib.util.spec_from_file_location("verify_diff", Path(__file__).with_name("diff.py")) +diff = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diff) + + +def record(count=0, unobserved=None, sample=None): + result = { + "outcome": "xfail" if count else "pass", + "failures": {"missing-after-hold": []} if count else {}, + "measures": {"missing-after-hold": {"count": [count, count], "pictures": [count, count]}} if count else {}, + "unobserved": unobserved or {}, + } + if sample is not None: + result["_sample"] = sample + return result + + +def compare(base, main): + return diff.compare_case(diff.summarize(base), diff.summarize(main), 0, 0) + + +def measured(value, case="export/instant-gated-mic-only/output/mp4/native", cls="sync-mic", metric="offset-ms", sample=None): + result = record(sample=sample) + result.update(case=case, failures={cls: []}, measures={cls: {metric: [value, value]}}) + return result + + +class ComparisonTests(unittest.TestCase): + def test_witness_frame_movement_does_not_change_severity(self): + for cls in ["stale", "future", "unreadable", "timestamps", "nondeterministic"]: + with self.subTest(cls=cls): + base = measured(4, case="editing/export/basic/seed-10", cls=cls, metric="screen-frames") + base["measures"][cls]["frame"] = [63, 63] + main = json.loads(json.dumps(base)) + main["measures"][cls]["frame"] = [109, 109] + self.assertEqual(compare([base], [main]), ("EQUAL", [])) + main["measures"][cls]["screen-frames"] = [3, 3] + self.assertEqual(compare([base], [main])[0], "BETTER") + main["measures"][cls]["screen-frames"] = [5, 5] + self.assertEqual(compare([base], [main])[0], "WORSE") + + def test_witness_frame_does_not_hide_new_classes_or_larger_deltas(self): + for cls in ["stale", "future", "unreadable", "grid-rounding"]: + with self.subTest(cls=cls): + main = measured(109, case="export/basic", cls=cls, metric="frame") + self.assertEqual(compare([dict(record(), case="export/basic")], [main])[0], "WORSE") + base = json.loads(json.dumps(main)) + base["measures"][cls]["delta"] = [1, 1] + main["measures"][cls]["delta"] = [2, 2] + self.assertEqual(compare([base], [main])[0], "WORSE") + + def test_continuous_metrics_require_three_independent_runs(self): + self.assertEqual(compare([measured(15.2)], [measured(15.3)])[0], "INSUFFICIENT") + self.assertEqual(compare([measured(15.2, sample="same") for _ in range(3)], [measured(16) for _ in range(3)])[0], "INSUFFICIENT") + + def test_continuous_overlap_and_one_resolution_step_are_not_worse(self): + base = [measured(v) for v in [15.1, 15.2, 15.3]] + for values in [[15.2, 15.3, 15.4], [15.4, 15.4, 15.4]]: + with self.subTest(values=values): + self.assertEqual(compare(base, [measured(v) for v in values])[0], "OVERLAP") + + def test_continuous_separated_ranges_are_worse(self): + self.assertEqual(compare([measured(v) for v in [15.1, 15.2, 15.3]], [measured(v) for v in [15.5, 15.6, 15.5]])[0], "WORSE") + + def test_signed_offsets_compare_magnitude(self): + self.assertEqual(compare([measured(-15) for _ in range(3)], [measured(-17) for _ in range(3)])[0], "WORSE") + + def test_timing_metric_outside_recording_cases_uses_distribution(self): + self.assertEqual(compare([measured(1, case="playback/basic", metric="warm-up-s")], [measured(2, case="playback/basic", metric="warm-up-s")])[0], "INSUFFICIENT") + + def test_reporter_precision_is_preserved_for_each_timing_unit(self): + for metric, base, within, beyond in [ + ("screen-start-late-ms", 200, 201, 202), + ("screen-lost-s", 3.25, 3.26, 3.27), + ("mic-after-start-s", 0.11, 0.12, 0.13), + ("off-periods", 1, 1.01, 1.02), + ]: + with self.subTest(metric=metric): + records = [measured(base, case="export/basic", metric=metric) for _ in range(3)] + self.assertEqual(compare(records[:1], [measured(within, case="export/basic", metric=metric)])[0], "INSUFFICIENT") + self.assertEqual(compare(records, [measured(within, case="export/basic", metric=metric) for _ in range(3)])[0], "OVERLAP") + self.assertEqual(compare(records, [measured(beyond, case="export/basic", metric=metric) for _ in range(3)])[0], "WORSE") + + def test_fresh_crash_counts_use_frame_resolution(self): + base = [measured(176, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)] + for value, expected in [(177, "OVERLAP"), (178, "WORSE")]: + main = [measured(value, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)] + self.assertEqual(compare(base, main)[0], expected) + + def test_missing_metric_is_insufficient_not_zero(self): + base = [measured(15) for _ in range(3)] + base[0]["measures"] = {} + self.assertEqual(compare(base, [measured(17) for _ in range(3)])[0], "INSUFFICIENT") + + def test_new_fresh_class_needs_consistent_occurrence(self): + base = [dict(record(), case="crash/example") for _ in range(3)] + main = [measured(1, case="crash/example", cls="unexpected", metric="count") for _ in range(3)] + self.assertEqual(compare(base[:1], main[:1])[0], "INSUFFICIENT") + self.assertEqual(compare(base, main)[0], "WORSE") + self.assertEqual(compare(base, main[:2] + base[:1])[0], "OVERLAP") + + def test_legacy_instant_telemetry_preserves_signed_interval(self): + rec = measured(-45.9, case="export/instant/output/mp4/native", cls="sync-system") + rec["measures"]["sync-mic"] = {"offset-ms": [14.0, 14.2]} + rec["measures"]["sync-system"]["offset-ms"] = [-46.0, -45.8] + actual = diff.normalized_measures(rec)["sync-system"] + self.assertEqual(actual["mic-offset-ms"], [14.0, 14.2]) + self.assertEqual(actual["system-minus-mic-ms"], [-60.2, -59.8]) + self.assertNotIn("mic-offset-ms", rec["measures"]["sync-system"]) + + def test_legacy_normalization_does_not_override_measurements_or_invent_sources(self): + rec = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system") + self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"]) + rec["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]} + rec["measures"]["sync-system"]["system-minus-mic-ms"] = [-6.7, -6.7] + self.assertEqual(diff.normalized_measures(rec)["sync-system"]["system-minus-mic-ms"], [-6.7, -6.7]) + rec["unobserved"] = {"sync-mic": "track absent"} + self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"]) + + def test_legacy_and_current_instant_records_compare_equally(self): + legacy = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system") + legacy["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]} + current = json.loads(json.dumps(legacy)) + current["measures"]["sync-system"].update({"mic-offset-ms": [15.1, 15.1], "system-minus-mic-ms": [-6.6, -6.6]}) + self.assertEqual(compare([legacy] * 3, [current] * 3), ("EQUAL", [])) + + def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self): + main = record(unobserved={"sync-screen-mic": "track absent"}) + for failed in [False, True]: + with self.subTest(failed=failed): + base = record() + if failed: + base["failures"] = {"sync-screen-mic": []} + verdict, reasons = compare([base], [main]) + self.assertEqual(verdict, "WORSE") + self.assertEqual(reasons, ["sync-screen-mic: baseline observed -> main unobserved (lost coverage)"]) + + def test_coverage_gain_does_not_hide_simultaneous_loss(self): + base = record(unobserved={"sync-screen-mic": "track absent"}) + main = record(unobserved={"sync-screen-system": "track absent"}) + bs, ms = diff.summarize([base]), diff.summarize([main]) + self.assertEqual(diff.compare_case(bs, ms, 0, 0)[0], "WORSE") + self.assertEqual(diff.coverage_gained(bs, ms), ["sync-screen-mic"]) + self.assertEqual(diff.coverage_lost(bs, ms), ["sync-screen-system"]) + + def test_partial_baseline_observation_still_detects_total_loss(self): + unseen = record(unobserved={"sync-screen-mic": "track absent"}) + self.assertEqual(compare([unseen, record()], [unseen, unseen])[0], "WORSE") + + def test_partial_main_observation_is_not_total_loss(self): + unseen = record(unobserved={"sync-screen-mic": "track absent"}) + self.assertEqual(diff.coverage_lost(diff.summarize([record()]), diff.summarize([unseen, record()])), []) + + def test_unobserved_on_both_sides_is_not_lost_coverage(self): + unseen = record(unobserved={"sync-screen-mic": "track absent"}) + self.assertEqual(compare([unseen], [unseen]), ("EQUAL", [])) + + def test_lost_distribution_coverage_is_worse_instead_of_insufficient(self): + unseen = record(unobserved={"missing-after-hold": "track absent"}) + self.assertEqual(compare([record(1)], [unseen])[0], "WORSE") + + def test_cli_reports_coverage_lost_and_gained_separately(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root) + base = record(unobserved={"sync-screen-mic": "track absent"}) + main = record(unobserved={"sync-screen-system": "track absent"}) + for side, rec in [("base", base), ("main", main)]: + path = root / side / "verify-report-macos-fast" + path.mkdir(parents=True) + rec.update({"suite": "export", "case": "a"}) + (path / "records.jsonl").write_text(json.dumps(rec) + "\n") + result = subprocess.run( + [sys.executable, "-B", str(Path(__file__).with_name("diff.py")), str(root / "base"), str(root / "main"), "--json", str(root / "diff.json")], + check=True, capture_output=True, text=True, + ) + self.assertIn("| coverage gained | coverage lost |", result.stdout) + output = json.loads((root / "diff.json").read_text())["macos"] + for category in ["WORSE", "COVERAGE-LOST", "COVERAGE-GAINED"]: + self.assertEqual(output[category][0][0], "export::a") + + def test_absent_track_is_coverage_gained_even_when_main_fails(self): + base = record(unobserved={"sync-screen-mic": "track absent"}) + main = record() + main["failures"] = {"sync-screen-mic": []} + self.assertEqual(compare([base], [main])[0], "COVERAGE-GAINED") + + def test_coverage_gained_does_not_hide_other_regression(self): + base = record(unobserved={"sync-screen-mic": "track absent"}) + main = record() + main["failures"] = {"sync-screen-mic": [], "timestamps": []} + verdict, reasons = compare([base], [main]) + self.assertEqual(verdict, "WORSE") + self.assertEqual(reasons, ["+timestamps"]) + + def test_single_capture_cannot_establish_distribution(self): + self.assertEqual(compare([record(1)], [record(2)])[0], "INSUFFICIENT") + + def test_passing_recording_still_requires_three_runs_for_baseline_gate(self): + base, main = record(), record() + base["case"] = main["case"] = "recording/instant/long-holds" + self.assertEqual(compare([base], [main])[0], "INSUFFICIENT") + + def test_consistently_above_baseline_range_is_worse(self): + self.assertEqual(compare([record(1), record(2), record(1)], [record(3), record(4), record(3)])[0], "WORSE") + + def test_overlap_is_explicit_not_equal_or_worse(self): + self.assertEqual(compare([record(1), record(2), record(1)], [record(2), record(3), record(2)])[0], "OVERLAP") + + def test_absence_of_class_counts_as_zero_per_run(self): + self.assertEqual(compare([record(), record(), record()], [record(1), record(1), record(1)])[0], "WORSE") + + def test_retries_do_not_count_as_independent_captures(self): + self.assertEqual(compare([record(1, sample="same") for _ in range(3)], [record(2), record(2), record(2)])[0], "INSUFFICIENT") + + def test_partial_observability_does_not_hide_measured_failures(self): + base = record(unobserved={"sync-screen-mic": "track absent"}) + observed = record() + main = record() + main["failures"] = {"sync-screen-mic": []} + self.assertEqual(compare([base, observed], [main])[0], "WORSE") + + def test_other_classes_keep_zero_tolerance(self): + base, main = record(), record() + for rec, value in [(base, 1), (main, 2)]: + rec["failures"] = {"timestamps": []} + rec["measures"] = {"timestamps": {"count": [value, value]}} + self.assertEqual(compare([base], [main])[0], "WORSE") + + +if __name__ == "__main__": + unittest.main()