ci(tests): cut idle waits from the python suite and skip OS-independent tests on windows

.github/workflows/python-tests.yml

@@ -72,6 +72,9 @@ jobs:
             # runner with no spare core and the deadline-sensitive tests miss their budgets
             # non-deterministically. Two workers keep that headroom.
             pytest-workers: '2'
+            # Windows installs linux's dependency set, so a `platform_independent` test cannot fail differently
+            # here than on linux, which runs it. macOS runs a different torch without bitsandbytes and keeps them.
+            pytest-markers: 'not slow and not platform_independent'
             # Roughly 1.5x the pytest step's four-worker wall time (~13.5 min), plus the job's
             # ~1.5 min of checkout and install, plus margin.
             timeout-minutes: 45
@@ -155,7 +158,7 @@ jobs:
         # the vCPUs a standard runner exposes. Much of the suite waits on timers and polls, so
         # one worker per vCPU is the right level where the runner has the headroom for it. The
         # matrix carries the count, because windows needs a lower one.
-        run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0
+        run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0 -m "${{ matrix.pytest-markers || 'not slow' }}"
 
   # Decides whether the external-database job runs. That job's service containers start before any of its
   # steps, so the decision has to come from a job of its own: otherwise every pull request, frontend-only ones

invokeai/app/services/download/download_default.py

@@ -45,6 +45,9 @@
 # Maximum number of bytes to download during each call to requests.iter_content()
 DOWNLOAD_CHUNK_SIZE = 100000
 
+# How long an idle worker blocks on the queue before rechecking the stop event, which bounds how long stop() waits.
+_QUEUE_POLL_SECONDS = 1.0
+
 
 class DownloadQueueService(DownloadQueueServiceBase):
     """Class for queued download of models."""
@@ -389,7 +392,7 @@ def _download_next_item(self) -> None:
                 done = True
                 continue
             try:
-                job = self._queue.get(timeout=1)
+                job = self._queue.get(timeout=_QUEUE_POLL_SECONDS)
             except Empty:
                 continue
             try:

invokeai/app/services/model_install/model_install_default.py

@@ -109,6 +109,13 @@ class _InstallCancelledBeforeTransfer(Exception):
 # created; the combined name is only a label, so it is shortened past this point with a count.
 _MAX_COMBINED_SUBFOLDER_NAME = 96
 
+# How long the idle install thread blocks on its queue before rechecking the stop event, which bounds how long
+# stop() waits.
+_INSTALL_QUEUE_POLL_SECONDS = 1.0
+
+# First wait before retrying a move that hit a locked file; each further retry doubles it.
+_MOVE_RETRY_DELAY_SECONDS = 0.5
+
 
 def _combined_subfolder_name(subfolder_names: List[str]) -> str:
     combined = "_".join(subfolder_names)
@@ -800,8 +807,9 @@ def register_path(
     # TODO: Replace this with a proper fix for underlying problem of Windows holding open
     # the file when it needs to be moved.
     @staticmethod
-    def _move_with_retries(src: Path, dst: Path, attempts: int = 5, delay: float = 0.5) -> None:
+    def _move_with_retries(src: Path, dst: Path, attempts: int = 5) -> None:
         """Workaround for Windows file-handle issues when moving files."""
+        delay = _MOVE_RETRY_DELAY_SECONDS
         for tries_left in range(attempts, 0, -1):
             try:
                 move(src, dst)
@@ -1591,7 +1599,7 @@ def _install_next_item(self) -> None:
                 break
             self._logger.debug(f"Installer thread {threading.get_ident()} polling")
             try:
-                job = self._install_queue.get(timeout=1)
+                job = self._install_queue.get(timeout=_INSTALL_QUEUE_POLL_SECONDS)
             except Empty:
                 continue
             assert job.local_path is not None

pyproject.toml

@@ -579,6 +579,7 @@ markers = [
   "timeout: Marks the timeout override.",
   "uses_database: Runs on the `empty_database` fixture, which targets INVOKEAI_TEST_DB_URL when it is set and SQLite otherwise. Applied by the collection hook in tests/conftest.py.",
   "sqlite_only: Depends on SQLite internals (query plans, PRAGMAs, the single connection); skipped when INVOKEAI_TEST_DB_URL targets another backend.",
+  "platform_independent: The outcome cannot depend on the operating system for the same dependency set: import-graph boundaries in a fresh interpreter, static source scans, deterministic counts and shapes. Windows CI, which installs the linux dependency set, deselects them.",
 ]
 # A cache-key or other SQLAlchemy warning is a bug in the query layer, not noise.
 filterwarnings = ["error::sqlalchemy.exc.SAWarning"]

tests/AGENTS.md

@@ -7,5 +7,11 @@
 - Cover affected rollback, revisions, cross-user isolation, retries, cancellation, bounds, and cleanup. IndexedDB, Web Locks, focus, and rendering belong in webv2 browser tests.
 - Use root pytest commands. CI runs xdist workers per vCPU, except windows, which runs fewer; `--dist loadfile` in `addopts` keeps each file on one worker, with arbitrary file assignment/order. Each file must establish its own process-global state (e.g. JWT secret) and pass alone; verify new files independently.
 - Default tests assert behavior, not throughput/latency under runner contention. Generous hang ceilings are acceptable; CPU time tolerates contention better than wall time. Hardware/quiet-machine measurements require `slow`, excluded by default and CI. On development machines, use `-m slow` for that lane or `-m ""` for everything.
+- Keep a test's run time to the work it checks; windows runs the suite on two workers, so idle seconds there set the CI wall clock. Check new files with `--durations=10`, which splits setup, call and teardown.
+  - No real waits. Poll intervals, retry backoffs and grace periods belong in module-level constants that a fixture can shorten. The root `conftest.py` already shortens the download, installer and image-index queue polls, so a service's `stop()` in teardown returns at once. Shorten a retry backoff only in the tests that script its failures, as `_fake_move` in `test_model_install.py` does: the backoff is what lets a real Windows file handle close.
+  - Passwords hash at bcrypt cost 4 in tests. Request `production_password_hashing` to assert or measure the production cost.
+  - A fresh interpreter costs about ten seconds of imports. Spawn one only when isolation is the point (import order, `sys.modules`, process-global state), and check many cases inside one subprocess unless each case needs its own cold start.
+  - A full `gc.collect()` walks the worker's whole heap, a third of a second late in a session. Tests that collect repeatedly freeze the pre-existing heap first, as `test_model_cache_ram_budget.py` does.
+- Mark a test `platform_independent` only when no operating system can change its outcome for the same dependency set: import-graph boundaries in a fresh interpreter, static source scans, deterministic counts or shapes. Windows CI, which installs the linux dependency set, deselects them; macOS runs a different torch and keeps them. Numerics (BLAS kernels differ per platform), process-spawning behavior (start methods, helper executables), filesystem paths and locking, and dependency smoke tests stay unmarked.
 - State hardware prerequisites and skipped/unavailable devices. CPU/mocked passes do not establish CUDA/ROCm/MPS/XPU behavior.
 - Do not run a CPU bf16 matmul unless low precision is the behavior under test: torch's CPU bf16 GEMM faults with an illegal instruction (`0xc000001d`) on part of the hosted windows fleet, which aborts the worker and fails the job. Run the forward at float32 and assert the width separately, as the nvfp4 loader tests do; skip on win32 only when bf16 itself is the subject.

tests/app/invocations/test_anima_qwen3_5_conditioning.py

@@ -75,6 +75,7 @@ def test_qwen3_5_is_read_at_the_trained_layers_with_an_attention_only_last_layer
     assert mask.tolist() == [True, True, True]
 
 
+@pytest.mark.platform_independent
 def test_the_prompt_node_does_not_import_transformers_qwen3_5_at_startup() -> None:
     """Every app start imports every node module; transformers' qwen3_5 modeling costs ~0.25 s of it."""
     program = (

tests/app/invocations/test_invocation_api_imports_alone.py

@@ -13,6 +13,10 @@
 import subprocess
 import sys
 
+import pytest
+
+pytestmark = pytest.mark.platform_independent
+
 PROGRAM = """
 import sys
 

tests/app/invocations/test_module_compatibility.py

@@ -8,6 +8,8 @@
 
 from invokeai.app.invocations._legacy_module_compatibility import LEGACY_MODULE_MAP
 
+pytestmark = pytest.mark.platform_independent
+
 
 def _run(program: str) -> dict[str, object]:
     result = subprocess.run([sys.executable, "-c", program], capture_output=True, text=True, timeout=300)
@@ -20,7 +22,29 @@ def test_compatibility_map_covers_all_moved_modules() -> None:
     assert len(set(LEGACY_MODULE_MAP.values())) == 106
 
 
+# A public symbol of each of these must be reachable through the legacy path, and be the canonical object.
+_REPRESENTATIVE_SYMBOLS = [
+    ("anima_denoise", "loglinear_timestep_shift"),
+    ("cogview4_denoise", "CogView4DenoiseInvocation"),
+    ("ernie_image_denoise", "ErnieImageDenoiseInvocation"),
+    ("flux_denoise", "FluxDenoiseInvocation"),
+    ("flux2_denoise", "Flux2DenoiseInvocation"),
+    ("ideogram4_caption", "Ideogram4Region"),
+    ("krea2_denoise", "Krea2DenoiseInvocation"),
+    ("minimax_h3_denoise", "MiniMaxH3DenoiseOutput"),
+    ("flux2_pid_decode", "Flux2PiDDecodeInvocation"),
+    ("qwen_image_denoise", "QwenImageDenoiseInvocation"),
+    ("controlnet", "ControlField"),
+    ("sd3_denoise", "SD3DenoiseInvocation"),
+    ("anima_text_encoder", "AnimaTextEncoderInvocation"),
+    ("anima_image_to_latents", "AnimaImageToLatentsInvocation"),
+    ("wan_denoise", "WanDenoiseInvocation"),
+    ("z_image_denoise", "ZImageDenoiseInvocation"),
+]
+
+
 def test_every_legacy_module_imports_and_aliases_canonical() -> None:
+    symbols = {f"invokeai.app.invocations.{legacy}": symbol for legacy, symbol in _REPRESENTATIVE_SYMBOLS}
     result = _run(
         f"""
 import importlib
@@ -35,20 +59,27 @@ def test_every_legacy_module_imports_and_aliases_canonical() -> None:
 logger.addHandler(handler)
 logger.setLevel(logging.INFO)
 mapping = {dict(LEGACY_MODULE_MAP)!r}
+symbols = {symbols!r}
 with warnings.catch_warnings(record=True) as caught:
     warnings.simplefilter("always")
     aliases = {{}}
+    broken_symbols = []
     for legacy, canonical in mapping.items():
         old = importlib.import_module(legacy)
         repeat = importlib.import_module(legacy)
-        aliases[legacy] = [old is importlib.import_module(canonical), repeat is old]
-print(json.dumps({{"aliases": aliases, "records": records, "deprecations": [
+        new = importlib.import_module(canonical)
+        aliases[legacy] = [old is new, repeat is old]
+        symbol = symbols.get(legacy)
+        if symbol is not None and getattr(old, symbol, None) is not getattr(new, symbol, object()):
+            broken_symbols.append(legacy + "." + symbol)
+print(json.dumps({{"aliases": aliases, "broken_symbols": broken_symbols, "records": records, "deprecations": [
     {{"message": str(w.message), "filename": str(w.filename)}}
     for w in caught if issubclass(w.category, DeprecationWarning)
 ]}}))
 """
     )
     assert result["aliases"] == {legacy: [True, True] for legacy in LEGACY_MODULE_MAP}
+    assert result["broken_symbols"] == []
     assert len(result["records"]) == len(LEGACY_MODULE_MAP)
     for legacy, canonical in LEGACY_MODULE_MAP.items():
         assert sum(legacy in record and canonical in record for record in result["records"]) == 1
@@ -81,36 +112,6 @@ def test_canonical_import_silent_unknown_path_stays_missing() -> None:
     assert result == {"missing": True, "records": []}
 
 
-@pytest.mark.parametrize(
-    ("legacy", "symbol"),
-    [
-        ("anima_denoise", "loglinear_timestep_shift"),
-        ("cogview4_denoise", "CogView4DenoiseInvocation"),
-        ("ernie_image_denoise", "ErnieImageDenoiseInvocation"),
-        ("flux_denoise", "FluxDenoiseInvocation"),
-        ("flux2_denoise", "Flux2DenoiseInvocation"),
-        ("ideogram4_caption", "Ideogram4Region"),
-        ("krea2_denoise", "Krea2DenoiseInvocation"),
-        ("minimax_h3_denoise", "MiniMaxH3DenoiseOutput"),
-        ("flux2_pid_decode", "Flux2PiDDecodeInvocation"),
-        ("qwen_image_denoise", "QwenImageDenoiseInvocation"),
-        ("controlnet", "ControlField"),
-        ("sd3_denoise", "SD3DenoiseInvocation"),
-        ("anima_text_encoder", "AnimaTextEncoderInvocation"),
-        ("anima_image_to_latents", "AnimaImageToLatentsInvocation"),
-        ("wan_denoise", "WanDenoiseInvocation"),
-        ("z_image_denoise", "ZImageDenoiseInvocation"),
-    ],
-)
-def test_representative_public_symbols_keep_identity(legacy: str, symbol: str) -> None:
-    old = f"invokeai.app.invocations.{legacy}"
-    canonical = LEGACY_MODULE_MAP[old]
-    result = _run(
-        f"import importlib, json; old = importlib.import_module({old!r}); new = importlib.import_module({canonical!r}); assert getattr(old, {symbol!r}) is getattr(new, {symbol!r}); print(json.dumps({{}}))"
-    )
-    assert result == {}
-
-
 def test_flux2_klein_text_encoder_legacy_import_explicitly_works() -> None:
     legacy = "invokeai.app.invocations.flux2_klein_text_encoder"
     canonical = LEGACY_MODULE_MAP[legacy]

tests/app/invocations/test_video_frame_extract_range.py

@@ -7,19 +7,22 @@
 stops decoding as soon as the range is written.
 """
 
+import re
 import shutil
+import subprocess
 from pathlib import Path
 from typing import Iterator
 from unittest.mock import MagicMock, patch
 
 import imageio.v2 as iio2
+import imageio_ffmpeg
 import numpy as np
 import pytest
 
 from invokeai.app.invocations.fields import VideoField
 from invokeai.app.invocations.video_frame_extract_range import ExtractVideoRangeInvocation, _write_frame_range
 from invokeai.app.services.session_processor.session_processor_common import CanceledException
-from invokeai.app.util.h264_splice import read_h264_layout
+from invokeai.app.util.h264_splice import MIN_ENCODED_FRAMES, read_h264_layout
 from invokeai.app.util.mp4_metadata import read_mp4_tags, write_mp4_tags
 from invokeai.app.util.video_encoding import make_mp4_writer
 from invokeai.app.util.video_thumbnails import probe_video
@@ -150,6 +153,19 @@ def test_invocation_echoes_the_source_video_it_was_given() -> None:
     assert output.video.video_name == "output.mp4"
 
 
+def _key_frames(path: Path) -> list[int]:
+    """Display indices of the key frames, as the decoder reports them."""
+    result = subprocess.run(
+        [imageio_ffmpeg.get_ffmpeg_exe(), "-hide_banner", "-i", str(path), "-vf", "showinfo", "-f", "null", "-"],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        check=True,
+    )
+    return [int(n) for n, key in re.findall(r"\bn:\s*(\d+)\b.*?\biskey:(\d)", result.stderr) if key == "1"]
+
+
 def _decode(path: Path) -> list[np.ndarray]:
     reader = iio2.get_reader(str(path), format="FFMPEG")
     try:
@@ -314,8 +330,19 @@ def test_open_gop_source_trim_keeps_every_frame(tmp_path: Path) -> None:
         writer.append_data(frame)
     writer.close()
 
-    # Every start in the last two GOPs, so one lands on an open-GOP I-frame wherever x264 put it.
-    for start in range(60, 100):
+    layout = read_h264_layout(source)
+    assert layout is not None
+    open_gop_frames = [n for n in _key_frames(source) if n >= 60]
+    assert open_gop_frames, "x264 placed no keyframe in the last two GOPs"
+    # Only the IDR is a clean cut; offering an open-GOP I-frame as one is the regression this guards.
+    assert layout.sync_samples == (0,), f"cut points {layout.sync_samples} include an open-GOP I-frame"
+
+    # Around each open-GOP I-frame of the last two GOPs: a cut on it, a head of MIN_ENCODED_FRAMES
+    # or more before it, a head too short to keep, and a start just past it. The window is wide
+    # enough for the I-frame's decode position, which leading B-frames put up to two before its
+    # display position, so every way of treating it as a cut point is exercised.
+    starts = sorted({s for p in open_gop_frames for s in range(p - MIN_ENCODED_FRAMES - 2, p + 2) if s < 100})
+    for start in starts:
         saved, _ = _invoke_trim(tmp_path, source, start, 99)
         assert len(_decode(saved)) == 100 - start, f"trim from {start}"
 

tests/app/services/auth/test_password_utils.py

@@ -286,6 +286,7 @@ class TestPasswordSecurityProperties:
     """Tests for security properties of password handling."""
 
     @pytest.mark.slow
+    @pytest.mark.usefixtures("production_password_hashing")
     def test_timing_attack_resistance_same_length(self):
         """Test that password verification takes similar time for correct and incorrect passwords.
 
@@ -334,6 +335,7 @@ def test_hash_output_format(self):
         # Bcrypt hashes are 60 characters long
         assert len(hashed) == 60
 
+    @pytest.mark.usefixtures("production_password_hashing")
     def test_hashing_cost_stays_at_a_defensible_work_factor(self):
         """The cost factor is what makes a stolen hash expensive to attack.
 

tests/app/services/auth/test_performance.py

@@ -32,6 +32,7 @@ def user_service(database: Database) -> UserService:
 
 
 @pytest.mark.slow
+@pytest.mark.usefixtures("production_password_hashing")
 class TestPasswordPerformance:
     """Tests for password hashing and verification performance."""
 
@@ -128,6 +129,7 @@ def test_token_verification_performance(self):
 
 
 @pytest.mark.slow
+@pytest.mark.usefixtures("production_password_hashing")
 class TestAuthenticationOverhead:
     """Tests for overall authentication system overhead."""
 
@@ -204,6 +206,7 @@ def test_token_verification_overhead(self):
 
 
 @pytest.mark.slow
+@pytest.mark.usefixtures("production_password_hashing")
 class TestUserServicePerformance:
     """Tests for user service performance."""
 
@@ -367,6 +370,7 @@ def authenticate_user(user_index: int):
 
 
 @pytest.mark.slow
+@pytest.mark.usefixtures("production_password_hashing")
 class TestScalabilityBenchmarks:
     """Sustained-load scalability benchmarks."""
 

tests/app/services/image_index/test_image_index_service.py

@@ -22,7 +22,6 @@
     _ACTIVATION_RETRY_INTERVAL_S,
     _MAX_ATTEMPTS,
     _MAX_BACKOFF_SECONDS,
-    _POLL_SECONDS,
     ImageIndexService,
 )
 from invokeai.app.services.image_index.image_index_records_default import ImageIndexRecords
@@ -1188,11 +1187,11 @@ def encode(images: list[Image.Image]) -> np.ndarray:
 
 def test_systemic_backoff_grows_and_is_capped(service: ImageIndexService) -> None:
     service._systemic_failures = 0
-    assert service._backoff_seconds() == _POLL_SECONDS
+    assert service._backoff_seconds() == image_index_default._POLL_SECONDS
     service._systemic_failures = 1
-    assert service._backoff_seconds() == _POLL_SECONDS
+    assert service._backoff_seconds() == image_index_default._POLL_SECONDS
     service._systemic_failures = 3
-    assert service._backoff_seconds() == _POLL_SECONDS * 4
+    assert service._backoff_seconds() == image_index_default._POLL_SECONDS * 4
     # Capped, and no overflow for an outage that lasts a very long time.
     service._systemic_failures = 10_000
     assert service._backoff_seconds() == _MAX_BACKOFF_SECONDS
@@ -1399,7 +1398,7 @@ def flaky_upsert(item, model_id, embedding):
             assert service._process_batch(imgs("stored.png", "locked.png")) is False
 
         assert service._systemic_failures == 0, "progress must clear the outage counter"
-        assert service._backoff_seconds() == _POLL_SECONDS, "a draining index must not back off"
+        assert service._backoff_seconds() == image_index_default._POLL_SECONDS, "a draining index must not back off"
         # The half that stored is stored, and no image was charged an attempt.
         assert index_records.get_embeddings(imgs("stored.png"), MODEL_ID)[0] == imgs("stored.png")
         assert service._failed == set()