From 1e43b1da090ee058047dcef2eb7c71f19aeed17b Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Thu, 15 Oct 2026 02:36:07 +0200 Subject: [PATCH 1/3] ci(tests): cut idle waits from the python suite and skip OS-independent tests on windows Tests hash at bcrypt cost 4, shorten service queue polls, scope the move backoff to scripted failures, and freeze the heap before repeated gc passes. Legacy-module symbol checks share one subprocess; a new platform_independent marker is deselected on windows only. tests/AGENTS.md documents test cost and when the marker applies. --- .github/workflows/python-tests.yml | 5 +- .../app/services/download/download_default.py | 5 +- .../model_install/model_install_default.py | 12 +++- pyproject.toml | 1 + tests/AGENTS.md | 6 ++ .../test_anima_qwen3_5_conditioning.py | 1 + .../test_invocation_api_imports_alone.py | 4 ++ .../invocations/test_module_compatibility.py | 65 ++++++++++--------- .../app/services/auth/test_password_utils.py | 2 + tests/app/services/auth/test_performance.py | 4 ++ .../image_index/test_image_index_service.py | 9 ++- .../model_install/test_model_install.py | 29 +++++---- .../shared/test_for_scheduler_performance.py | 1 + .../shared/test_graph_execution_runtime.py | 3 + .../shared/test_graph_if_activation.py | 1 + .../shared/test_graph_if_dependencies.py | 1 + .../services/shared/test_graph_if_runtime.py | 1 + .../shared/test_graph_iterate_planner.py | 1 + .../shared/test_graph_materializer.py | 1 + .../app/services/shared/test_graph_models.py | 1 + .../shared/test_graph_runtime_records.py | 3 + .../services/shared/test_graph_scheduler.py | 3 + tests/app/test_database_access_guard.py | 4 ++ .../architectures/test_import_isolation.py | 4 ++ tests/backend/architectures/test_layering.py | 4 ++ .../test_model_cache_ram_budget.py | 10 +++ tests/backend/util/test_vae_tiling_scope.py | 4 ++ tests/conftest.py | 31 +++++++++ tests/test_imports.py | 3 + tests/test_node_graph.py | 1 + 30 files changed, 168 insertions(+), 52 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 11c8d591584..7eed10c0178 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -72,6 +72,9 @@ jobs: # runner with no spare core and the deadline-sensitive tests miss their budgets # non-deterministically. Two workers keep that headroom. pytest-workers: '2' + # Windows installs linux's dependency set, so a `platform_independent` test cannot fail differently + # here than on linux, which runs it. macOS runs a different torch without bitsandbytes and keeps them. + pytest-markers: 'not slow and not platform_independent' # Roughly 1.5x the pytest step's four-worker wall time (~13.5 min), plus the job's # ~1.5 min of checkout and install, plus margin. timeout-minutes: 45 @@ -155,7 +158,7 @@ jobs: # the vCPUs a standard runner exposes. Much of the suite waits on timers and polls, so # one worker per vCPU is the right level where the runner has the headroom for it. The # matrix carries the count, because windows needs a lower one. - run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0 + run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0 -m "${{ matrix.pytest-markers || 'not slow' }}" # Decides whether the external-database job runs. That job's service containers start before any of its # steps, so the decision has to come from a job of its own: otherwise every pull request, frontend-only ones diff --git a/invokeai/app/services/download/download_default.py b/invokeai/app/services/download/download_default.py index 7fe1eceb9b1..4848f1935a8 100644 --- a/invokeai/app/services/download/download_default.py +++ b/invokeai/app/services/download/download_default.py @@ -45,6 +45,9 @@ # Maximum number of bytes to download during each call to requests.iter_content() DOWNLOAD_CHUNK_SIZE = 100000 +# How long an idle worker blocks on the queue before rechecking the stop event, which bounds how long stop() waits. +_QUEUE_POLL_SECONDS = 1.0 + class DownloadQueueService(DownloadQueueServiceBase): """Class for queued download of models.""" @@ -389,7 +392,7 @@ def _download_next_item(self) -> None: done = True continue try: - job = self._queue.get(timeout=1) + job = self._queue.get(timeout=_QUEUE_POLL_SECONDS) except Empty: continue try: diff --git a/invokeai/app/services/model_install/model_install_default.py b/invokeai/app/services/model_install/model_install_default.py index 651ed29a9aa..20e6eebfcc0 100644 --- a/invokeai/app/services/model_install/model_install_default.py +++ b/invokeai/app/services/model_install/model_install_default.py @@ -109,6 +109,13 @@ class _InstallCancelledBeforeTransfer(Exception): # created; the combined name is only a label, so it is shortened past this point with a count. _MAX_COMBINED_SUBFOLDER_NAME = 96 +# How long the idle install thread blocks on its queue before rechecking the stop event, which bounds how long +# stop() waits. +_INSTALL_QUEUE_POLL_SECONDS = 1.0 + +# First wait before retrying a move that hit a locked file; each further retry doubles it. +_MOVE_RETRY_DELAY_SECONDS = 0.5 + def _combined_subfolder_name(subfolder_names: List[str]) -> str: combined = "_".join(subfolder_names) @@ -800,8 +807,9 @@ def register_path( # TODO: Replace this with a proper fix for underlying problem of Windows holding open # the file when it needs to be moved. @staticmethod - def _move_with_retries(src: Path, dst: Path, attempts: int = 5, delay: float = 0.5) -> None: + def _move_with_retries(src: Path, dst: Path, attempts: int = 5) -> None: """Workaround for Windows file-handle issues when moving files.""" + delay = _MOVE_RETRY_DELAY_SECONDS for tries_left in range(attempts, 0, -1): try: move(src, dst) @@ -1591,7 +1599,7 @@ def _install_next_item(self) -> None: break self._logger.debug(f"Installer thread {threading.get_ident()} polling") try: - job = self._install_queue.get(timeout=1) + job = self._install_queue.get(timeout=_INSTALL_QUEUE_POLL_SECONDS) except Empty: continue assert job.local_path is not None diff --git a/pyproject.toml b/pyproject.toml index c2617206361..6e0595aeb1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -578,6 +578,7 @@ markers = [ "timeout: Marks the timeout override.", "uses_database: Runs on the `empty_database` fixture, which targets INVOKEAI_TEST_DB_URL when it is set and SQLite otherwise. Applied by the collection hook in tests/conftest.py.", "sqlite_only: Depends on SQLite internals (query plans, PRAGMAs, the single connection); skipped when INVOKEAI_TEST_DB_URL targets another backend.", + "platform_independent: The outcome cannot depend on the operating system for the same dependency set: import-graph boundaries in a fresh interpreter, static source scans, deterministic counts and shapes. Windows CI, which installs the linux dependency set, deselects them.", ] # A cache-key or other SQLAlchemy warning is a bug in the query layer, not noise. filterwarnings = ["error::sqlalchemy.exc.SAWarning"] diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 2ea50aedf67..b75684872b9 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -7,5 +7,11 @@ - Cover affected rollback, revisions, cross-user isolation, retries, cancellation, bounds, and cleanup. IndexedDB, Web Locks, focus, and rendering belong in webv2 browser tests. - Use root pytest commands. CI runs xdist workers per vCPU, except windows, which runs fewer; `--dist loadfile` in `addopts` keeps each file on one worker, with arbitrary file assignment/order. Each file must establish its own process-global state (e.g. JWT secret) and pass alone; verify new files independently. - Default tests assert behavior, not throughput/latency under runner contention. Generous hang ceilings are acceptable; CPU time tolerates contention better than wall time. Hardware/quiet-machine measurements require `slow`, excluded by default and CI. On development machines, use `-m slow` for that lane or `-m ""` for everything. +- Keep a test's run time to the work it checks; windows runs the suite on two workers, so idle seconds there set the CI wall clock. Check new files with `--durations=10`, which splits setup, call and teardown. + - No real waits. Poll intervals, retry backoffs and grace periods belong in module-level constants that a fixture can shorten. The root `conftest.py` already shortens the download, installer and image-index queue polls, so a service's `stop()` in teardown returns at once. Shorten a retry backoff only in the tests that script its failures, as `_fake_move` in `test_model_install.py` does: the backoff is what lets a real Windows file handle close. + - Passwords hash at bcrypt cost 4 in tests. Request `production_password_hashing` to assert or measure the production cost. + - A fresh interpreter costs about ten seconds of imports. Spawn one only when isolation is the point (import order, `sys.modules`, process-global state), and check many cases inside one subprocess unless each case needs its own cold start. + - A full `gc.collect()` walks the worker's whole heap, a third of a second late in a session. Tests that collect repeatedly freeze the pre-existing heap first, as `test_model_cache_ram_budget.py` does. +- Mark a test `platform_independent` only when no operating system can change its outcome for the same dependency set: import-graph boundaries in a fresh interpreter, static source scans, deterministic counts or shapes. Windows CI, which installs the linux dependency set, deselects them; macOS runs a different torch and keeps them. Numerics (BLAS kernels differ per platform), process-spawning behavior (start methods, helper executables), filesystem paths and locking, and dependency smoke tests stay unmarked. - State hardware prerequisites and skipped/unavailable devices. CPU/mocked passes do not establish CUDA/ROCm/MPS/XPU behavior. - Do not run a CPU bf16 matmul unless low precision is the behavior under test: torch's CPU bf16 GEMM faults with an illegal instruction (`0xc000001d`) on part of the hosted windows fleet, which aborts the worker and fails the job. Run the forward at float32 and assert the width separately, as the nvfp4 loader tests do; skip on win32 only when bf16 itself is the subject. diff --git a/tests/app/invocations/test_anima_qwen3_5_conditioning.py b/tests/app/invocations/test_anima_qwen3_5_conditioning.py index 34fb231a0c6..b450fd58d53 100644 --- a/tests/app/invocations/test_anima_qwen3_5_conditioning.py +++ b/tests/app/invocations/test_anima_qwen3_5_conditioning.py @@ -75,6 +75,7 @@ def test_qwen3_5_is_read_at_the_trained_layers_with_an_attention_only_last_layer assert mask.tolist() == [True, True, True] +@pytest.mark.platform_independent def test_the_prompt_node_does_not_import_transformers_qwen3_5_at_startup() -> None: """Every app start imports every node module; transformers' qwen3_5 modeling costs ~0.25 s of it.""" program = ( diff --git a/tests/app/invocations/test_invocation_api_imports_alone.py b/tests/app/invocations/test_invocation_api_imports_alone.py index 7aca72c1bd3..88100be974b 100644 --- a/tests/app/invocations/test_invocation_api_imports_alone.py +++ b/tests/app/invocations/test_invocation_api_imports_alone.py @@ -13,6 +13,10 @@ import subprocess import sys +import pytest + +pytestmark = pytest.mark.platform_independent + PROGRAM = """ import sys diff --git a/tests/app/invocations/test_module_compatibility.py b/tests/app/invocations/test_module_compatibility.py index 647becb0148..a04d1f801a1 100644 --- a/tests/app/invocations/test_module_compatibility.py +++ b/tests/app/invocations/test_module_compatibility.py @@ -8,6 +8,8 @@ from invokeai.app.invocations._legacy_module_compatibility import LEGACY_MODULE_MAP +pytestmark = pytest.mark.platform_independent + def _run(program: str) -> dict[str, object]: result = subprocess.run([sys.executable, "-c", program], capture_output=True, text=True, timeout=300) @@ -20,7 +22,29 @@ def test_compatibility_map_covers_all_moved_modules() -> None: assert len(set(LEGACY_MODULE_MAP.values())) == 106 +# A public symbol of each of these must be reachable through the legacy path, and be the canonical object. +_REPRESENTATIVE_SYMBOLS = [ + ("anima_denoise", "loglinear_timestep_shift"), + ("cogview4_denoise", "CogView4DenoiseInvocation"), + ("ernie_image_denoise", "ErnieImageDenoiseInvocation"), + ("flux_denoise", "FluxDenoiseInvocation"), + ("flux2_denoise", "Flux2DenoiseInvocation"), + ("ideogram4_caption", "Ideogram4Region"), + ("krea2_denoise", "Krea2DenoiseInvocation"), + ("minimax_h3_denoise", "MiniMaxH3DenoiseOutput"), + ("flux2_pid_decode", "Flux2PiDDecodeInvocation"), + ("qwen_image_denoise", "QwenImageDenoiseInvocation"), + ("controlnet", "ControlField"), + ("sd3_denoise", "SD3DenoiseInvocation"), + ("anima_text_encoder", "AnimaTextEncoderInvocation"), + ("anima_image_to_latents", "AnimaImageToLatentsInvocation"), + ("wan_denoise", "WanDenoiseInvocation"), + ("z_image_denoise", "ZImageDenoiseInvocation"), +] + + def test_every_legacy_module_imports_and_aliases_canonical() -> None: + symbols = {f"invokeai.app.invocations.{legacy}": symbol for legacy, symbol in _REPRESENTATIVE_SYMBOLS} result = _run( f""" import importlib @@ -35,20 +59,27 @@ def test_every_legacy_module_imports_and_aliases_canonical() -> None: logger.addHandler(handler) logger.setLevel(logging.INFO) mapping = {dict(LEGACY_MODULE_MAP)!r} +symbols = {symbols!r} with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") aliases = {{}} + broken_symbols = [] for legacy, canonical in mapping.items(): old = importlib.import_module(legacy) repeat = importlib.import_module(legacy) - aliases[legacy] = [old is importlib.import_module(canonical), repeat is old] -print(json.dumps({{"aliases": aliases, "records": records, "deprecations": [ + new = importlib.import_module(canonical) + aliases[legacy] = [old is new, repeat is old] + symbol = symbols.get(legacy) + if symbol is not None and getattr(old, symbol, None) is not getattr(new, symbol, object()): + broken_symbols.append(legacy + "." + symbol) +print(json.dumps({{"aliases": aliases, "broken_symbols": broken_symbols, "records": records, "deprecations": [ {{"message": str(w.message), "filename": str(w.filename)}} for w in caught if issubclass(w.category, DeprecationWarning) ]}})) """ ) assert result["aliases"] == {legacy: [True, True] for legacy in LEGACY_MODULE_MAP} + assert result["broken_symbols"] == [] assert len(result["records"]) == len(LEGACY_MODULE_MAP) for legacy, canonical in LEGACY_MODULE_MAP.items(): assert sum(legacy in record and canonical in record for record in result["records"]) == 1 @@ -81,36 +112,6 @@ def test_canonical_import_silent_unknown_path_stays_missing() -> None: assert result == {"missing": True, "records": []} -@pytest.mark.parametrize( - ("legacy", "symbol"), - [ - ("anima_denoise", "loglinear_timestep_shift"), - ("cogview4_denoise", "CogView4DenoiseInvocation"), - ("ernie_image_denoise", "ErnieImageDenoiseInvocation"), - ("flux_denoise", "FluxDenoiseInvocation"), - ("flux2_denoise", "Flux2DenoiseInvocation"), - ("ideogram4_caption", "Ideogram4Region"), - ("krea2_denoise", "Krea2DenoiseInvocation"), - ("minimax_h3_denoise", "MiniMaxH3DenoiseOutput"), - ("flux2_pid_decode", "Flux2PiDDecodeInvocation"), - ("qwen_image_denoise", "QwenImageDenoiseInvocation"), - ("controlnet", "ControlField"), - ("sd3_denoise", "SD3DenoiseInvocation"), - ("anima_text_encoder", "AnimaTextEncoderInvocation"), - ("anima_image_to_latents", "AnimaImageToLatentsInvocation"), - ("wan_denoise", "WanDenoiseInvocation"), - ("z_image_denoise", "ZImageDenoiseInvocation"), - ], -) -def test_representative_public_symbols_keep_identity(legacy: str, symbol: str) -> None: - old = f"invokeai.app.invocations.{legacy}" - canonical = LEGACY_MODULE_MAP[old] - result = _run( - f"import importlib, json; old = importlib.import_module({old!r}); new = importlib.import_module({canonical!r}); assert getattr(old, {symbol!r}) is getattr(new, {symbol!r}); print(json.dumps({{}}))" - ) - assert result == {} - - def test_flux2_klein_text_encoder_legacy_import_explicitly_works() -> None: legacy = "invokeai.app.invocations.flux2_klein_text_encoder" canonical = LEGACY_MODULE_MAP[legacy] diff --git a/tests/app/services/auth/test_password_utils.py b/tests/app/services/auth/test_password_utils.py index 4f080c442d4..65f246faa31 100644 --- a/tests/app/services/auth/test_password_utils.py +++ b/tests/app/services/auth/test_password_utils.py @@ -286,6 +286,7 @@ class TestPasswordSecurityProperties: """Tests for security properties of password handling.""" @pytest.mark.slow + @pytest.mark.usefixtures("production_password_hashing") def test_timing_attack_resistance_same_length(self): """Test that password verification takes similar time for correct and incorrect passwords. @@ -334,6 +335,7 @@ def test_hash_output_format(self): # Bcrypt hashes are 60 characters long assert len(hashed) == 60 + @pytest.mark.usefixtures("production_password_hashing") def test_hashing_cost_stays_at_a_defensible_work_factor(self): """The cost factor is what makes a stolen hash expensive to attack. diff --git a/tests/app/services/auth/test_performance.py b/tests/app/services/auth/test_performance.py index 63c4bc6d5e5..f77483a1445 100644 --- a/tests/app/services/auth/test_performance.py +++ b/tests/app/services/auth/test_performance.py @@ -32,6 +32,7 @@ def user_service(database: Database) -> UserService: @pytest.mark.slow +@pytest.mark.usefixtures("production_password_hashing") class TestPasswordPerformance: """Tests for password hashing and verification performance.""" @@ -128,6 +129,7 @@ def test_token_verification_performance(self): @pytest.mark.slow +@pytest.mark.usefixtures("production_password_hashing") class TestAuthenticationOverhead: """Tests for overall authentication system overhead.""" @@ -204,6 +206,7 @@ def test_token_verification_overhead(self): @pytest.mark.slow +@pytest.mark.usefixtures("production_password_hashing") class TestUserServicePerformance: """Tests for user service performance.""" @@ -367,6 +370,7 @@ def authenticate_user(user_index: int): @pytest.mark.slow +@pytest.mark.usefixtures("production_password_hashing") class TestScalabilityBenchmarks: """Sustained-load scalability benchmarks.""" diff --git a/tests/app/services/image_index/test_image_index_service.py b/tests/app/services/image_index/test_image_index_service.py index 36a3d201baa..4d0bcc5b702 100644 --- a/tests/app/services/image_index/test_image_index_service.py +++ b/tests/app/services/image_index/test_image_index_service.py @@ -22,7 +22,6 @@ _ACTIVATION_RETRY_INTERVAL_S, _MAX_ATTEMPTS, _MAX_BACKOFF_SECONDS, - _POLL_SECONDS, ImageIndexService, ) from invokeai.app.services.image_index.image_index_records_default import ImageIndexRecords @@ -1188,11 +1187,11 @@ def encode(images: list[Image.Image]) -> np.ndarray: def test_systemic_backoff_grows_and_is_capped(service: ImageIndexService) -> None: service._systemic_failures = 0 - assert service._backoff_seconds() == _POLL_SECONDS + assert service._backoff_seconds() == image_index_default._POLL_SECONDS service._systemic_failures = 1 - assert service._backoff_seconds() == _POLL_SECONDS + assert service._backoff_seconds() == image_index_default._POLL_SECONDS service._systemic_failures = 3 - assert service._backoff_seconds() == _POLL_SECONDS * 4 + assert service._backoff_seconds() == image_index_default._POLL_SECONDS * 4 # Capped, and no overflow for an outage that lasts a very long time. service._systemic_failures = 10_000 assert service._backoff_seconds() == _MAX_BACKOFF_SECONDS @@ -1399,7 +1398,7 @@ def flaky_upsert(item, model_id, embedding): assert service._process_batch(imgs("stored.png", "locked.png")) is False assert service._systemic_failures == 0, "progress must clear the outage counter" - assert service._backoff_seconds() == _POLL_SECONDS, "a draining index must not back off" + assert service._backoff_seconds() == image_index_default._POLL_SECONDS, "a draining index must not back off" # The half that stored is stored, and no image was charged an attempt. assert index_records.get_embeddings(imgs("stored.png"), MODEL_ID)[0] == imgs("stored.png") assert service._failed == set() diff --git a/tests/app/services/model_install/test_model_install.py b/tests/app/services/model_install/test_model_install.py index 2639a1d3f51..d9b15f9e3b2 100644 --- a/tests/app/services/model_install/test_model_install.py +++ b/tests/app/services/model_install/test_model_install.py @@ -13,7 +13,7 @@ import uuid from pathlib import Path from types import SimpleNamespace -from typing import Any, Dict +from typing import Any, Callable, Dict from unittest.mock import MagicMock import pytest @@ -72,6 +72,13 @@ OS = platform.uname().system +def _fake_move(monkeypatch: pytest.MonkeyPatch, move: Callable[[Path, Path], object]) -> None: + """Install a move that fails on purpose. Its failures are scripted, so the retry backoff, which gives a real + Windows file handle time to close, would only add sleep.""" + monkeypatch.setattr(model_install_default, "move", move) + monkeypatch.setattr(model_install_default, "_MOVE_RETRY_DELAY_SECONDS", 0.0) + + def test_registration(mm2_installer: ModelInstallServiceBase, embedding_file: Path) -> None: store = mm2_installer.record_store matches = store.search_by_attr(model_name="test_embedding") @@ -179,7 +186,7 @@ def copy_then_fail_unlink(src: Path, dst: Path): raise PermissionError("simulated Windows source unlink failure") return real_move(src, dst) - monkeypatch.setattr(model_install_default, "move", copy_then_fail_unlink) + _fake_move(monkeypatch, copy_then_fail_unlink) key = mm2_installer.install_path(embedding_file) @@ -484,7 +491,7 @@ def flaky_move(src: Path, dst: Path): raise PermissionError("simulated Windows file-handle race") return real_move(src, dst) - monkeypatch.setattr(model_install_default, "move", flaky_move) + _fake_move(monkeypatch, flaky_move) key = mm2_installer.install_path(diffusers_dir) @@ -508,7 +515,7 @@ def test_directory_install_failure_does_not_leave_partial_destination( def fail_first_move(_src: Path, _dst: Path): raise PermissionError("simulated Windows file-handle race") - monkeypatch.setattr(model_install_default, "move", fail_first_move) + _fake_move(monkeypatch, fail_first_move) with pytest.raises(PermissionError, match="simulated Windows file-handle race"): mm2_installer.install_path(diffusers_dir) @@ -543,7 +550,7 @@ def move_with_locked_source(src: Path, dst: Path): moved_first_item = True return result - monkeypatch.setattr(model_install_default, "move", move_with_locked_source) + _fake_move(monkeypatch, move_with_locked_source) with pytest.raises(PermissionError, match="source file remains locked"): mm2_installer.install_path(diffusers_dir) @@ -588,7 +595,7 @@ def move_with_recreated_source(src: Path, dst: Path): raise FileExistsError(dst) return real_move(src, dst) - monkeypatch.setattr(model_install_default, "move", move_with_recreated_source) + _fake_move(monkeypatch, move_with_recreated_source) with pytest.raises(RuntimeError, match="recovery required") as exc_info: mm2_installer.install_path(diffusers_dir) @@ -631,7 +638,7 @@ def partial_copy_then_fail(src: Path, dst: Path) -> None: dst.write_bytes(partial_destination_bytes) raise OSError(errno.ENOSPC, "simulated disk full") - monkeypatch.setattr(model_install_default, "move", partial_copy_then_fail) + _fake_move(monkeypatch, partial_copy_then_fail) job = installer.import_model(LocalModelSource(path=source_path, inplace=False)) installer.wait_for_job(job, timeout=10) @@ -692,7 +699,7 @@ def recreate_after_absence_check(src: Path, dst: Path) -> None: dst.write_bytes(b"new user bytes") real_rename_noreplace(src, dst) - monkeypatch.setattr(model_install_default, "move", fail_later_move) + _fake_move(monkeypatch, fail_later_move) monkeypatch.setattr(ModelInstallService, "_rename_noreplace", staticmethod(recreate_after_absence_check)) with pytest.raises(model_install_default.InstallRecoveryRequiredError, match="recovery required"): @@ -722,7 +729,7 @@ def partial_copy_then_fail(src: Path, dst: Path): raise PermissionError("copy failed after destination creation") return shutil.move(src, dst) - monkeypatch.setattr(model_install_default, "move", partial_copy_then_fail) + _fake_move(monkeypatch, partial_copy_then_fail) with pytest.raises(RuntimeError, match="recovery required") as exc_info: mm2_installer.install_path(diffusers_dir) @@ -747,7 +754,7 @@ def fail_after_unowned_destination_artifact(src: Path, dst: Path): raise PermissionError("later source remains locked") return shutil.move(src, dst) - monkeypatch.setattr(model_install_default, "move", fail_after_unowned_destination_artifact) + _fake_move(monkeypatch, fail_after_unowned_destination_artifact) with pytest.raises(RuntimeError, match="recovery required") as exc_info: mm2_installer.install_path(diffusers_dir) @@ -803,7 +810,7 @@ def sorted_source_files(path: Path): return iter(sorted(entries, key=lambda entry: entry.name)) if path == tmpdir else iter(entries) monkeypatch.setattr(Path, "iterdir", sorted_source_files) - monkeypatch.setattr(model_install_default, "move", fail_after_partial_transfer) + _fake_move(monkeypatch, fail_after_partial_transfer) real_write_install_marker = mm2_installer._write_install_marker def fail_recovery_marker(job_arg, *args, **kwargs): diff --git a/tests/app/services/shared/test_for_scheduler_performance.py b/tests/app/services/shared/test_for_scheduler_performance.py index c2de4d3706c..d58b2c0f5a3 100644 --- a/tests/app/services/shared/test_for_scheduler_performance.py +++ b/tests/app/services/shared/test_for_scheduler_performance.py @@ -146,6 +146,7 @@ def _run_trivial_loop( return clock() - started +@pytest.mark.platform_independent @pytest.mark.parametrize( ("loop_type", "include_if", "external_condition"), [ diff --git a/tests/app/services/shared/test_graph_execution_runtime.py b/tests/app/services/shared/test_graph_execution_runtime.py index d5b0ee11fd9..b0c00e531dc 100644 --- a/tests/app/services/shared/test_graph_execution_runtime.py +++ b/tests/app/services/shared/test_graph_execution_runtime.py @@ -2,10 +2,13 @@ import sys import textwrap +import pytest + from invokeai.app.services.shared import graph as graph_facade from invokeai.app.services.shared import graph_execution_runtime +@pytest.mark.platform_independent def test_runtime_module_imports_without_graph_cycle() -> None: script = textwrap.dedent( """ diff --git a/tests/app/services/shared/test_graph_if_activation.py b/tests/app/services/shared/test_graph_if_activation.py index f2cfc39deed..90827e48521 100644 --- a/tests/app/services/shared/test_graph_if_activation.py +++ b/tests/app/services/shared/test_graph_if_activation.py @@ -74,6 +74,7 @@ def _is_activation_dependency_rejected(self, dependency: ActivationDependency) - return dependency in self.rejected +@pytest.mark.platform_independent def test_standalone_import_does_not_import_graph() -> None: assert_module_imports_without_graph("invokeai.app.services.shared.graph_if_activation") diff --git a/tests/app/services/shared/test_graph_if_dependencies.py b/tests/app/services/shared/test_graph_if_dependencies.py index b77f7f90206..a1004553b3f 100644 --- a/tests/app/services/shared/test_graph_if_dependencies.py +++ b/tests/app/services/shared/test_graph_if_dependencies.py @@ -227,6 +227,7 @@ def _indirectly_connected_if_graph() -> Graph: return graph +@pytest.mark.platform_independent def test_leaf_import_does_not_import_graph() -> None: assert_module_imports_without_graph("invokeai.app.services.shared.graph_if_dependencies") diff --git a/tests/app/services/shared/test_graph_if_runtime.py b/tests/app/services/shared/test_graph_if_runtime.py index ad596f2812a..7a8c72abbe7 100644 --- a/tests/app/services/shared/test_graph_if_runtime.py +++ b/tests/app/services/shared/test_graph_if_runtime.py @@ -20,6 +20,7 @@ def state() -> GraphExecutionState: return state +@pytest.mark.platform_independent def test_runtime_import_without_graph() -> None: assert_module_imports_without_graph("invokeai.app.services.shared.graph_if_runtime") diff --git a/tests/app/services/shared/test_graph_iterate_planner.py b/tests/app/services/shared/test_graph_iterate_planner.py index 9e4e1016276..e98e430c389 100644 --- a/tests/app/services/shared/test_graph_iterate_planner.py +++ b/tests/app/services/shared/test_graph_iterate_planner.py @@ -6,6 +6,7 @@ import pytest +@pytest.mark.platform_independent @pytest.mark.parametrize("planner_first", [True, False]) def test_planner_import_boundary(planner_first: bool) -> None: script = textwrap.dedent( diff --git a/tests/app/services/shared/test_graph_materializer.py b/tests/app/services/shared/test_graph_materializer.py index e407af81998..f5c28118e86 100644 --- a/tests/app/services/shared/test_graph_materializer.py +++ b/tests/app/services/shared/test_graph_materializer.py @@ -9,6 +9,7 @@ from invokeai.app.services.shared import graph as graph_facade +@pytest.mark.platform_independent def test_materializer_import_is_leaf_and_keeps_networkx_lazy() -> None: script = textwrap.dedent( """ diff --git a/tests/app/services/shared/test_graph_models.py b/tests/app/services/shared/test_graph_models.py index f6211956c81..9ab948b1db7 100644 --- a/tests/app/services/shared/test_graph_models.py +++ b/tests/app/services/shared/test_graph_models.py @@ -293,6 +293,7 @@ def test_graph_models_preserve_heterogeneous_control_flow_roundtrip() -> None: assert isinstance(restored_state.execution_graph.nodes["return"], ForReturnInvocation) +@pytest.mark.platform_independent def test_graph_models_preserve_lazy_networkx_and_registry_behavior() -> None: script = """ import builtins diff --git a/tests/app/services/shared/test_graph_runtime_records.py b/tests/app/services/shared/test_graph_runtime_records.py index 82d7bd45e2a..689018af729 100644 --- a/tests/app/services/shared/test_graph_runtime_records.py +++ b/tests/app/services/shared/test_graph_runtime_records.py @@ -2,10 +2,13 @@ import sys import textwrap +import pytest + from invokeai.app.services.shared import graph as graph_facade from invokeai.app.services.shared import graph_runtime_records +@pytest.mark.platform_independent def test_runtime_records_import_without_graph_cycle() -> None: script = textwrap.dedent( """ diff --git a/tests/app/services/shared/test_graph_scheduler.py b/tests/app/services/shared/test_graph_scheduler.py index da40f6b90da..f5984381dba 100644 --- a/tests/app/services/shared/test_graph_scheduler.py +++ b/tests/app/services/shared/test_graph_scheduler.py @@ -1,6 +1,8 @@ import subprocess import sys +import pytest + from invokeai.app.services.shared import graph as graph_module from invokeai.app.services.shared import graph_scheduler @@ -10,6 +12,7 @@ def test_graph_facade_reexports_scheduler_classes() -> None: assert graph_module._GenericGraphSchedulerAdapter is graph_scheduler._GenericGraphSchedulerAdapter +@pytest.mark.platform_independent def test_graph_scheduler_import_does_not_import_graph_facade() -> None: result = subprocess.run( [ diff --git a/tests/app/test_database_access_guard.py b/tests/app/test_database_access_guard.py index 59e113d4fc3..43dc4e0c9c6 100644 --- a/tests/app/test_database_access_guard.py +++ b/tests/app/test_database_access_guard.py @@ -11,6 +11,10 @@ from dataclasses import dataclass from pathlib import Path +import pytest + +pytestmark = pytest.mark.platform_independent + REPO_ROOT = Path(__file__).resolve().parents[2] SCANNED_ROOTS = ("invokeai", "scripts") DATABASE_LAYER = ( diff --git a/tests/backend/architectures/test_import_isolation.py b/tests/backend/architectures/test_import_isolation.py index e2db7e9d9e0..d31a1379a75 100644 --- a/tests/backend/architectures/test_import_isolation.py +++ b/tests/backend/architectures/test_import_isolation.py @@ -13,8 +13,12 @@ mean this never runs in CI. """ +import pytest + from tests.dangerously_run_function_in_subprocess import dangerously_run_function_in_subprocess +pytestmark = pytest.mark.platform_independent + def _registry_is_full_after_importing_dependencies() -> None: # No arguments and no closure: the whole function body is re-executed in a fresh interpreter, so diff --git a/tests/backend/architectures/test_layering.py b/tests/backend/architectures/test_layering.py index b2f9ca0f67b..c4e426fdcc6 100644 --- a/tests/backend/architectures/test_layering.py +++ b/tests/backend/architectures/test_layering.py @@ -18,6 +18,10 @@ from collections.abc import Iterable from pathlib import Path +import pytest + +pytestmark = pytest.mark.platform_independent + REPO_ROOT = Path(__file__).parents[3] ARCH = "invokeai.backend.architectures" ARCH_DIR = "invokeai/backend/architectures" diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_ram_budget.py b/tests/backend/model_manager/load/model_cache/test_model_cache_ram_budget.py index bc186af4794..646445fb2f1 100644 --- a/tests/backend/model_manager/load/model_cache/test_model_cache_ram_budget.py +++ b/tests/backend/model_manager/load/model_cache/test_model_cache_ram_budget.py @@ -41,6 +41,16 @@ def mock_logger(): return logger +@pytest.fixture(autouse=True) +def _collect_only_this_tests_objects(): + """These tests call gc.collect() to run finalizers, often once per loop iteration. A full collection walks + the whole heap of the worker process, a third of a second deep into a session. Freezing what existed before + the test leaves every collection to the objects the test created, which is where its garbage lives.""" + gc.freeze() + yield + gc.unfreeze() + + def _make_cache(store, budget, logger, keep_ram_copy=True) -> ModelCache: return ModelCache( execution_device_working_mem_gb=1.0, diff --git a/tests/backend/util/test_vae_tiling_scope.py b/tests/backend/util/test_vae_tiling_scope.py index d944ec94a3a..669db6d713d 100644 --- a/tests/backend/util/test_vae_tiling_scope.py +++ b/tests/backend/util/test_vae_tiling_scope.py @@ -111,6 +111,7 @@ def test_entering_with_tiling_already_on_turns_it_off_and_puts_it_back(self): assert (vae.use_tiling, vae.tile_sample_min_size, vae.tile_latent_min_size) == before + @pytest.mark.platform_independent def test_every_tile_size_the_field_accepts_assembles_the_full_latent(self): """The decode sweep below, pointed at the encode. @@ -141,6 +142,7 @@ def test_the_vaes_own_geometry_is_restored(self): assert (vae.use_tiling, vae.tile_sample_min_size, vae.tile_latent_min_size) == before + @pytest.mark.platform_independent def test_every_tile_size_the_field_accepts_assembles_the_full_image(self): """A sweep over every value the node field accepts, on the class that can fail it. @@ -164,6 +166,7 @@ def test_every_tile_size_the_field_accepts_assembles_the_full_image(self): assert not wrong + @pytest.mark.platform_independent @pytest.mark.parametrize("overlap_factor", [0.125, 0.2, 0.3, 1 / 3, 0.35]) def test_a_non_stock_overlap_factor_is_honoured_rather_than_assumed(self, overlap_factor): """Which tiles a factor admits depends on how it rounds in binary, not on its value. @@ -183,6 +186,7 @@ def test_a_non_stock_overlap_factor_is_honoured_rather_than_assumed(self, overla with torch.no_grad(), scoped_vae_tiling(vae, tile_size): assert vae.decode(latent, return_dict=False)[0].shape == (1, 3, 512, 512) + @pytest.mark.platform_independent def test_a_decode_at_the_vaes_default_resolution_actually_tiles(self): """The case the missing size costs: a 1024px decode against a VAE whose default tile is also 1024 stays in the single-pass path, which is the one the reservation was sized against.""" diff --git a/tests/conftest.py b/tests/conftest.py index badeaca0e79..9bf75bc752a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ import pytest +from invokeai.app.services.auth import password_utils from invokeai.app.services.board_image_records.board_image_records_default import BoardImageRecordStorage from invokeai.app.services.board_records.board_records_default import BoardRecordStorage from invokeai.app.services.board_video_records.board_video_records_default import BoardVideoRecordStorage @@ -18,8 +19,10 @@ from invokeai.app.services.bulk_download.bulk_download_default import BulkDownloadService from invokeai.app.services.client_state_persistence.client_state_persistence_default import ClientStatePersistence from invokeai.app.services.config.config_default import InvokeAIAppConfig +from invokeai.app.services.download import download_default from invokeai.app.services.external_generation.external_generation_default import ExternalGenerationService from invokeai.app.services.gallery.gallery_default import GalleryService +from invokeai.app.services.image_index import image_index_default from invokeai.app.services.image_index.image_index_default import ImageIndexService from invokeai.app.services.image_index.image_index_records_default import ImageIndexRecords from invokeai.app.services.image_records.image_records_default import ImageRecordStorage @@ -30,6 +33,7 @@ from invokeai.app.services.invocation_services import InvocationServices from invokeai.app.services.invocation_stats.invocation_stats_default import InvocationStatsService from invokeai.app.services.invoker import Invoker +from invokeai.app.services.model_install import model_install_default from invokeai.app.services.project_records.project_records_default import ProjectRecordsStorage from invokeai.app.services.shared.database.database import Database from invokeai.app.services.system_prompt_records.system_prompt_records_default import ( @@ -79,6 +83,33 @@ def _clear_deferred_empty_cache(): TorchDevice._empty_cache_deferred.clear() +_PRODUCTION_PASSWORD_CONTEXT = password_utils.pwd_context +_TEST_PASSWORD_CONTEXT = _PRODUCTION_PASSWORD_CONTEXT.copy(bcrypt__rounds=4) + + +@pytest.fixture(autouse=True) +def _cheap_password_hashing(monkeypatch: pytest.MonkeyPatch) -> None: + """At the production cost factor every user a test creates or logs in costs a fifth of a second, which + was most of the multiuser router tests' run time. Cost-4 hashes still round-trip, and stored cost-12 + hashes still verify. Tests about the cost itself request `production_password_hashing`.""" + monkeypatch.setattr(password_utils, "pwd_context", _TEST_PASSWORD_CONTEXT) + + +@pytest.fixture +def production_password_hashing(monkeypatch: pytest.MonkeyPatch) -> None: + """Hash at the production cost factor, for tests that assert or measure it.""" + monkeypatch.setattr(password_utils, "pwd_context", _PRODUCTION_PASSWORD_CONTEXT) + + +@pytest.fixture(autouse=True) +def _fast_service_polling(monkeypatch: pytest.MonkeyPatch) -> None: + """Idle service workers poll their queues once a second, so every stop() in a fixture teardown waited up to + that long. Shortening the intervals keeps the same loops and orderings; it only removes the idle wait.""" + monkeypatch.setattr(download_default, "_QUEUE_POLL_SECONDS", 0.02) + monkeypatch.setattr(model_install_default, "_INSTALL_QUEUE_POLL_SECONDS", 0.02) + monkeypatch.setattr(image_index_default, "_POLL_SECONDS", 0.02) + + @pytest.fixture def mock_sqlite_database() -> Database: """The in-memory database behind `mock_services`, for tests that need SQL of their own or a service that diff --git a/tests/test_imports.py b/tests/test_imports.py index 48889bf3dd8..e464f19cbaf 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -4,6 +4,8 @@ import sys import textwrap +import pytest + import invokeai KNOWN_IMPORT_ERRORS = { @@ -38,6 +40,7 @@ def test_invokeai_imports(): assert not failed_to_import, f"Modules failed to import: {failed_to_import}" +@pytest.mark.platform_independent def test_graph_module_import_does_not_require_networkx(): script = """ import builtins diff --git a/tests/test_node_graph.py b/tests/test_node_graph.py index d06a1905cb5..6efc2fcfcb8 100644 --- a/tests/test_node_graph.py +++ b/tests/test_node_graph.py @@ -2743,6 +2743,7 @@ def invoke(self) -> str: return "foo" +@pytest.mark.platform_independent def test_nodes_must_return_invocation_output_under_optimized_python(): result = subprocess.run( [ From d213bf5e4fa807546d5ff4727e0bde1b36e2e518 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Thu, 15 Oct 2026 02:50:12 +0200 Subject: [PATCH 2/3] test(video): trim the open-GOP source only around its I-frames Reads the source's key frames and trims around each open-GOP I-frame instead of from all 40 starts. Asserts read_h264_layout offers no open-GOP I-frame as a cut point, which the old loop did not catch. --- .../test_video_frame_extract_range.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/app/invocations/test_video_frame_extract_range.py b/tests/app/invocations/test_video_frame_extract_range.py index 1cd2dec8dfd..6bec7b32fa9 100644 --- a/tests/app/invocations/test_video_frame_extract_range.py +++ b/tests/app/invocations/test_video_frame_extract_range.py @@ -7,19 +7,22 @@ stops decoding as soon as the range is written. """ +import re import shutil +import subprocess from pathlib import Path from typing import Iterator from unittest.mock import MagicMock, patch import imageio.v2 as iio2 +import imageio_ffmpeg import numpy as np import pytest from invokeai.app.invocations.fields import VideoField from invokeai.app.invocations.video_frame_extract_range import ExtractVideoRangeInvocation, _write_frame_range from invokeai.app.services.session_processor.session_processor_common import CanceledException -from invokeai.app.util.h264_splice import read_h264_layout +from invokeai.app.util.h264_splice import MIN_ENCODED_FRAMES, read_h264_layout from invokeai.app.util.mp4_metadata import read_mp4_tags, write_mp4_tags from invokeai.app.util.video_encoding import make_mp4_writer from invokeai.app.util.video_thumbnails import probe_video @@ -150,6 +153,19 @@ def test_invocation_echoes_the_source_video_it_was_given() -> None: assert output.video.video_name == "output.mp4" +def _key_frames(path: Path) -> list[int]: + """Display indices of the key frames, as the decoder reports them.""" + result = subprocess.run( + [imageio_ffmpeg.get_ffmpeg_exe(), "-hide_banner", "-i", str(path), "-vf", "showinfo", "-f", "null", "-"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=True, + ) + return [int(n) for n, key in re.findall(r"\bn:\s*(\d+)\b.*?\biskey:(\d)", result.stderr) if key == "1"] + + def _decode(path: Path) -> list[np.ndarray]: reader = iio2.get_reader(str(path), format="FFMPEG") try: @@ -314,8 +330,19 @@ def test_open_gop_source_trim_keeps_every_frame(tmp_path: Path) -> None: writer.append_data(frame) writer.close() - # Every start in the last two GOPs, so one lands on an open-GOP I-frame wherever x264 put it. - for start in range(60, 100): + layout = read_h264_layout(source) + assert layout is not None + open_gop_frames = [n for n in _key_frames(source) if n >= 60] + assert open_gop_frames, "x264 placed no keyframe in the last two GOPs" + # Only the IDR is a clean cut; offering an open-GOP I-frame as one is the regression this guards. + assert layout.sync_samples == (0,), f"cut points {layout.sync_samples} include an open-GOP I-frame" + + # Around each open-GOP I-frame of the last two GOPs: a cut on it, a head of MIN_ENCODED_FRAMES + # or more before it, a head too short to keep, and a start just past it. The window is wide + # enough for the I-frame's decode position, which leading B-frames put up to two before its + # display position, so every way of treating it as a cut point is exercised. + starts = sorted({s for p in open_gop_frames for s in range(p - MIN_ENCODED_FRAMES - 2, p + 2) if s < 100}) + for start in starts: saved, _ = _invoke_trim(tmp_path, source, start, 99) assert len(_decode(saved)) == 100 - start, f"trim from {start}" From 0a54b0b99e6d440142463845a1672d48c4fda729 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Thu, 15 Oct 2026 21:59:40 +0200 Subject: [PATCH 3/3] test(session-processor): give the woken worker time for its gc pass A woken worker runs a full gc.collect() before taking its item, which late in an xdist session can exceed the 2 s join. Polls every 60 s with a 10 s deadline, so a pass still requires the release listener. --- .../test_lent_device_scheduling.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/app/services/session_processor/test_lent_device_scheduling.py b/tests/app/services/session_processor/test_lent_device_scheduling.py index d9eb2cf2060..032ff35b61e 100644 --- a/tests/app/services/session_processor/test_lent_device_scheduling.py +++ b/tests/app/services/session_processor/test_lent_device_scheduling.py @@ -24,6 +24,12 @@ GPU0 = torch.device("cuda:0") GPU1 = torch.device("cuda:1") +# A woken worker still runs a full gc.collect() before it takes its item, which late in an xdist +# session can take seconds. The poll interval stays far above this deadline, so a pass still +# needs the release listener rather than the poll. +_POLLING_INTERVAL_S = 60 +_WAKE_DEADLINE_S = 10 + @pytest.fixture(autouse=True) def pool() -> Iterator[None]: @@ -65,7 +71,7 @@ def get_queue_item(self, item_id: int): configuration=SimpleNamespace(multiuser=False, clear_vram_after_session=False), ) ) - processor._polling_interval = 5 # long enough that only the release listener can wake it in time + processor._polling_interval = _POLLING_INTERVAL_S processor._thread_semaphore = BoundedSemaphore(1) processor._poll_now_event = Event() GENERATION_DEVICE_POOL.set_release_listener(processor._poll_now_event.set) @@ -93,8 +99,8 @@ def test_worker_leaves_items_alone_while_its_gpu_is_lent_and_resumes_on_release( assert claims == [] # nothing claimed onto the lent GPU GENERATION_DEVICE_POOL.release_borrow(GPU0) - # The release wakes the worker at once; its 5 s poll interval would fail this join. - thread.join(timeout=2) + # The release wakes the worker at once; waiting for its poll interval would fail this join. + thread.join(timeout=_WAKE_DEADLINE_S) assert not thread.is_alive() assert claims == ["cuda:0"] finally: @@ -108,7 +114,7 @@ def test_worker_on_a_free_gpu_claims_while_another_is_lent() -> None: thread, stop_event, claims, processor = _start_worker(GPU1) try: - thread.join(timeout=2) + thread.join(timeout=_WAKE_DEADLINE_S) assert not thread.is_alive() assert claims == ["cuda:1"] finally: