fix(recorder): survive SQLite writer contention in the video writer

.github/workflows/live-qualification.yml

@@ -67,6 +67,7 @@ jobs:
 
   build-candidate:
     name: Build candidate distributions
+    if: vars.CAPTURE_SELF_HOSTED_QUALIFIED_RUNNERS == '1'
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -427,164 +428,3 @@ jobs:
           path: evidence/
           if-no-files-found: error
           retention-days: 30
-
-  # Hosted Windows runs the same four live recorder tests as the required
-  # macOS lane, but it does not gate a release. On 2026-08-28 it failed the
-  # same way in both qualification dispatches of ecdd9c02 (runs 33186627127
-  # and 33189604580): the video_writer child exits 1 during startup, the
-  # recorder reports "Recording tasks exited before readiness", and an
-  # orphaned child then holds the step open until the job timeout. The same
-  # four tests pass in test.yml's windows-latest leg on the same commit, so
-  # this reproduces only from an isolated wheel install. It is a real defect
-  # and it stays visible here until it is fixed, rather than blocking every
-  # release while it is open.
-  hosted-live-recorder-windows:
-    name: Hosted live recorder qualification (${{ matrix.os }})
-    needs: build-candidate
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [windows-latest]
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 35
-    steps:
-      - name: Checkout the exact candidate source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-
-      - name: Set up Python
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.12"
-
-      - name: Install exact uv
-        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
-        with:
-          version: "0.11.29"
-
-      - name: Download the exact candidate
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: capture-candidate-${{ github.sha }}
-          path: dist
-
-      - name: Install the exact wheel into an isolated environment
-        shell: bash
-        run: |
-          set -euo pipefail
-          qualification_root="${HOME}/capture-qualification-${GITHUB_RUN_ID}"
-          if [ "${RUNNER_OS}" = "Windows" ]; then
-            qualification_python="${qualification_root}/Scripts/python.exe"
-            workspace="$(cygpath -u "${GITHUB_WORKSPACE}")"
-          else
-            qualification_python="${qualification_root}/bin/python"
-            workspace="${GITHUB_WORKSPACE}"
-          fi
-          candidate_wheels=(dist/*.whl)
-          if [ "${#candidate_wheels[@]}" -ne 1 ]; then
-            echo "Expected exactly one candidate wheel."
-            exit 1
-          fi
-          uv venv --clear --python 3.12 "${qualification_root}"
-          uv pip install --python "${qualification_python}" \
-            "${candidate_wheels[0]}" \
-            pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2
-          echo "QUALIFICATION_PYTHON=${qualification_python}" >> "${GITHUB_ENV}"
-          echo "QUALIFICATION_WORKSPACE=${workspace}" >> "${GITHUB_ENV}"
-
-      - name: Install the reviewed external video tools
-        shell: bash
-        run: |
-          set -euo pipefail
-          if [ "${RUNNER_OS}" = "Windows" ]; then
-            choco install ffmpeg -y --no-progress
-          else
-            brew install ffmpeg
-          fi
-
-      - name: Require the reviewed external video tools
-        shell: bash
-        run: |
-          set -euo pipefail
-          mkdir -p evidence
-          command -v ffmpeg
-          command -v ffprobe
-          ffmpeg -version > evidence/ffmpeg-version.txt
-          ffprobe -version > evidence/ffprobe-version.txt
-
-      # A GitHub-hosted runner exposes exactly one virtual display. The
-      # multiple-monitor contract is proven only by the self-hosted lanes in
-      # live-qualification.yml. This step still records the exact topology the
-      # trials ran against, and still fails on a topology that changes during
-      # qualification.
-      - name: Record a stable single-monitor desktop
-        shell: bash
-        run: |
-          set -euo pipefail
-          "${QUALIFICATION_PYTHON}" scripts/check_display_topology.py \
-            --minimum-monitors 1 \
-            --output evidence/display-topology.json
-
-      # These four live tests drive the real recorder against the real display
-      # of the hosted runner: they start it, wait for a real first frame, prove
-      # the capture database is created, prove memory stays bounded, and prove
-      # a clean shutdown. They are named one by one, and the trial evidence is
-      # rejected below unless exactly four of them ran, so a rename or a
-      # deletion fails the gate instead of silently shrinking it.
-      #
-      # The three input-injection tests of this file are deliberately NOT here.
-      # Injected input does not reach the native low-level hooks in a hosted
-      # runner session, measured on all three hosted operating systems on
-      # 2026-08-28. live-qualification.yml runs them on a qualified host.
-      # A failing trial can leave a recorder child process holding this step's
-      # output pipe open, and the step then sits until the job timeout rather
-      # than reporting the failure. Three trials take under three minutes on a
-      # healthy runner, so bound the step itself.
-      - name: Run three counted live recorder trials
-        timeout-minutes: 12
-        shell: bash
-        env:
-          OPENADAPT_CAPTURE_PRODUCTION_QUALIFICATION: "1"
-        run: |
-          set -euo pipefail
-          cd "${HOME}"
-          for trial in 1 2 3; do
-            echo "::group::${RUNNER_OS} trial ${trial}"
-            "${QUALIFICATION_PYTHON}" -m pytest \
-              "${QUALIFICATION_WORKSPACE}/tests/test_performance.py" \
-              -m slow -v --timeout=300 --import-mode=importlib \
-              -k "test_initial_frame_ready_without_input or test_shutdown_time or test_memory_bounded or test_db_file_created" \
-              "--junitxml=${QUALIFICATION_WORKSPACE}/evidence/trial-${trial}-${{ matrix.os }}.xml"
-            echo "::endgroup::"
-          done
-
-      - name: Reject skipped, incomplete, or shrunken qualification trials
-        shell: bash
-        run: |
-          set -euo pipefail
-          for trial in 1 2 3; do
-            python scripts/check_junit_no_skips.py \
-              --expected-tests 4 \
-              "evidence/trial-${trial}-${{ matrix.os }}.xml"
-          done
-
-      - name: Aggregate the counted trial evidence
-        shell: bash
-        run: |
-          set -euo pipefail
-          python scripts/aggregate_qualification_trials.py \
-            --os "${{ matrix.os }}" \
-            --candidate-sha "${GITHUB_SHA}" \
-            --expected-trials 3 \
-            "evidence/trial-1-${{ matrix.os }}.xml" \
-            "evidence/trial-2-${{ matrix.os }}.xml" \
-            "evidence/trial-3-${{ matrix.os }}.xml" \
-            --output "evidence/qualification-summary-${{ matrix.os }}.json"
-
-      - name: Upload hosted live recorder evidence
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: capture-hosted-live-${{ matrix.os }}-${{ github.sha }}
-          path: evidence/
-          if-no-files-found: error
-          retention-days: 30

.github/workflows/production-qualification.yml

@@ -32,16 +32,10 @@
 #     boundary;
 #   - that exact wheel installs, imports, exposes its CLI, and uninstalls on
 #     clean Linux, macOS, and Windows machines;
-#   - the real recorder starts against a real display on hosted macOS,
-#     produces a real first frame, creates its capture database, holds
-#     memory bounded, and shuts down cleanly, in three counted trials.
-#
-# The same lane on hosted Windows is in live-qualification.yml, not here.
-# It reproduces a video_writer startup failure that test.yml's
-# windows-latest leg does not hit, and test.yml is already required on the
-# same exact commit, so Windows live recorder coverage stays in the required
-# evidence while that defect is open. The comment above that lane has the
-# run ids and the failure signature.
+#   - the real recorder starts against a real display on hosted macOS and
+#     Windows, produces a real first frame, creates its capture database,
+#     holds memory bounded, and shuts down cleanly, three counted trials per
+#     operating system.
 #
 # scripts/check_release_ci.py additionally requires a successful test.yml run
 # on the same exact commit, which is where the complete headless suite and the
@@ -176,7 +170,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest]
+        os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 35
     steps:
@@ -267,10 +261,10 @@ jobs:
       # Injected input does not reach the native low-level hooks in a hosted
       # runner session, measured on all three hosted operating systems on
       # 2026-08-28. live-qualification.yml runs them on a qualified host.
-      # A failing trial can leave a recorder child process holding this step's
-      # output pipe open, and the step then sits until the job timeout rather
-      # than reporting the failure. Three trials take under three minutes on a
-      # healthy runner, so bound the step itself.
+      # Three trials take under three minutes on a healthy runner, so the step
+      # carries its own bound well under the job's. A step that stops making
+      # progress reports a failure in twelve minutes instead of holding the
+      # runner for the job's full thirty-five.
       - name: Run three counted live recorder trials
         timeout-minutes: 12
         shell: bash

docs/DESIGN.md

@@ -235,9 +235,10 @@ commit. Every job runs on a GitHub-hosted runner. It must:
 - build and validate one wheel and sdist;
 - install and uninstall that exact wheel in a clean environment on Linux,
   macOS, and Windows;
-- start the real recorder against the real display of a hosted macOS runner,
-  prove a real first frame, prove the capture database, prove bounded memory,
-  and prove a clean shutdown, in three counted trials with no skip;
+- start the real recorder against the real display of a hosted macOS and a
+  hosted Windows runner, prove a real first frame, prove the capture database,
+  prove bounded memory, and prove a clean shutdown, in three counted trials per
+  operating system with no skip;
 - record the exact display topology each trial ran against; and
 - retain machine-readable test and topology evidence.
 
@@ -253,14 +254,7 @@ commit. Missing, stale, skipped, partial, or failed evidence blocks publication.
 `live-qualification.yml` runs the interactive lanes that need a physical
 desktop. It runs weekly and on demand, and it is not a release gate.
 
-It also carries the same live recorder lane on hosted Windows. That lane
-reproduces a `video_writer` startup failure that `test.yml` does not hit on
-the same commit, so it does not gate a release while that defect is open.
-`test.yml` runs the same four tests on `windows-latest` and is required on
-the exact commit, so Windows live recorder coverage stays in the required
-evidence.
-
-The self-hosted lanes prove, and the release gate therefore does not prove:
+Those lanes prove, and the release gate therefore does not prove:
 
 - that global input injected through the operating system reaches the native
   listeners and is written into the capture;

openadapt_capture/db/crud.py

@@ -6,8 +6,8 @@
 
 import json
 import sqlite3
-from time import sleep
-from typing import Any, TypeVar
+from time import monotonic, sleep
+from typing import Any, Callable, TypeVar
 
 import sqlalchemy as sa
 from loguru import logger
@@ -26,6 +26,7 @@
 
 # Type variable for generic model queries
 BaseModelType = TypeVar("BaseModelType")
+T = TypeVar("T")
 
 BATCH_SIZE = 1
 
@@ -69,36 +70,68 @@ def _is_sqlite_lock_error(error: sa.exc.OperationalError) -> bool:
     )
 
 
-def _execute_insert_with_lock_retry(
+def _write_with_lock_retry(
     session: SaSession,
-    table: sa.Table,
-    to_insert: list[dict[str, Any]],
-) -> sa.engine.Result:
-    """Commit one insert, with bounded recovery from SQLite writer contention."""
+    write: Callable[[], T],
+    statement_label: str,
+) -> T:
+    """Run one write transaction, recovering from bounded SQLite contention.
+
+    ``write`` must perform every statement of the transaction and must be safe
+    to run again from the start: a rollback discards its partial work before
+    each retry.
+
+    Every recorder writer process writes the one per-capture database file, so
+    each of them competes for the single SQLite write lock. A connection whose
+    bounded wait expires reports "database is locked", which is contention, not
+    corruption. Each log line carries how long that attempt waited, because the
+    wait is the only measure of how close a capture is to losing this race.
+    """
     for attempt in range(len(SQLITE_LOCK_RETRY_DELAYS_SECONDS) + 1):
+        started_at = monotonic()
         try:
-            result = session.execute(sa.insert(table), to_insert)
+            result = write()
             session.commit()
             return result
         except sa.exc.OperationalError as exc:
             if not _is_sqlite_lock_error(exc):
                 raise
+            waited = monotonic() - started_at
 
             # A failed execute or commit can leave the Session transaction
             # unusable. Roll it back before either retrying or failing loud.
             session.rollback()
             if attempt == len(SQLITE_LOCK_RETRY_DELAYS_SECONDS):
+                logger.error(
+                    f"SQLite writer lock during {statement_label} did not clear: "
+                    f"attempt {attempt + 1} waited {waited:.2f}s and every retry "
+                    "is spent"
+                )
                 raise
 
             delay = SQLITE_LOCK_RETRY_DELAYS_SECONDS[attempt]
             logger.warning(
-                "SQLite writer lock during insert; retrying in "
+                f"SQLite writer lock during {statement_label} after waiting "
+                f"{waited:.2f}s; retrying in "
                 f"{delay:.2f}s ({attempt + 1}/"
                 f"{len(SQLITE_LOCK_RETRY_DELAYS_SECONDS)})"
             )
             sleep(delay)
 
-    raise AssertionError("unreachable SQLite insert retry state")
+    raise AssertionError("unreachable SQLite write retry state")
+
+
+def _execute_insert_with_lock_retry(
+    session: SaSession,
+    table: sa.Table,
+    to_insert: list[dict[str, Any]],
+) -> sa.engine.Result:
+    """Commit one insert, with bounded recovery from SQLite writer contention."""
+    return _write_with_lock_retry(
+        session,
+        lambda: session.execute(sa.insert(table), to_insert),
+        "insert",
+    )
 
 
 def _insert(
@@ -334,22 +367,36 @@ def update_video_start_time(
         recording (Recording): The recording object to update.
         video_start_time (float): The new video start time to set.
     """
-    # Find the recording by its timestamp
-    recording = session.query(Recording).filter(Recording.id == recording.id).first()
-
-    if not recording:
-        logger.error(f"No recording found with id {recording.id}.")
-        return
+    recording_id = recording.id
+
+    # This is the first thing the video writer process does, and it runs while
+    # the screen, action, performance and memory writer processes are already
+    # committing to the same database file. It therefore has to wait for the
+    # one SQLite write lock like every other writer, and it fails the whole
+    # recording if that wait is not bounded and retried.
+    #
+    # Read nothing first: a read that finds no row is answered below by the
+    # affected row count, and a session that has already read holds a
+    # transaction the retry would have to unwind.
+    session.rollback()
+
+    def _write() -> int:
+        result = session.execute(
+            sa.update(Recording)
+            .where(Recording.id == recording_id)
+            .values(video_start_time=video_start_time)
+        )
+        return result.rowcount
 
-    # Update the video start time
-    recording.video_start_time = video_start_time
+    updated_rows = _write_with_lock_retry(
+        session,
+        _write,
+        "video start time update",
+    )
 
-    # the function is called from a different process which uses a different
-    # session from the one used to create the recording object, so we need to
-    # add the recording object to the session
-    session.add(recording)
-    # Commit the changes to the database
-    session.commit()
+    if not updated_rows:
+        logger.error(f"No recording found with id {recording_id}.")
+        return
 
     logger.info(
         f"Updated video start time for recording {recording.timestamp} to"