diff --git a/docs/source/getting_started/quick_start.rst b/docs/source/getting_started/quick_start.rst index 792401e39..abc96464e 100644 --- a/docs/source/getting_started/quick_start.rst +++ b/docs/source/getting_started/quick_start.rst @@ -108,9 +108,10 @@ The first launch downloads the CloudXR Web Client SDK and asks you to review and accept the EULA on the terminal; answer the prompt once and the acceptance is remembered for subsequent runs. -The CloudXR runtime uses the ``auto-webrtc`` device profile by default -(Pico & Quest). For Apple Vision Pro it defaults to ``auto-native``. To -override settings, write a ``KEY=value`` env file and pass it to the example +The CloudXR runtime uses the ``Quest3`` device profile by default; Apple Vision +Pro needs ``auto-native``. The profile is set on the *runtime*, so applications +inherit whatever the runtime they connect to was started with. To override it, +or any other setting, write a ``KEY=value`` env file and pass it to the example with ``--cloudxr-env-config``: .. code-block:: bash @@ -145,7 +146,7 @@ To inspect the resolved settings after startup: - Description - Values * - ``NV_DEVICE_PROFILE`` - - ``auto-webrtc`` + - ``Quest3`` - Device profile - ``auto-webrtc``, ``auto-native``, ``Quest3``, ``AppleVisionPro`` * - ``NV_CXR_ENABLE_PUSH_DEVICES`` diff --git a/docs/source/references/cloudxr.rst b/docs/source/references/cloudxr.rst index 80791bccd..99cdb1856 100644 --- a/docs/source/references/cloudxr.rst +++ b/docs/source/references/cloudxr.rst @@ -193,3 +193,30 @@ To inspect the active settings after startup: .. code-block:: bash cat ~/.cloudxr/run/cloudxr.env + +Troubleshooting +--------------- + +.. _cloudxr-form-factor-unavailable: + +``xrGetSystem`` fails with ``XR_ERROR_FORM_FACTOR_UNAVAILABLE`` (-35) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +-35 is returned *after* ``xrCreateInstance`` succeeded, so the runtime was +found and loaded — a missing runtime gives -51 (``XR_ERROR_RUNTIME_UNAVAILABLE``) +instead. It means the runtime is up but no headset session is attached to it +yet. Check, in order: + +1. A client is connected. The headset must have loaded the web client and + clicked CONNECT; until then there is no system to return. +2. The device profile matches the device: check ``NV_DEVICE_PROFILE`` in + ``~/.cloudxr/run/cloudxr.env``, which the launcher also prints at startup. +3. The application is talking to the runtime you think it is. Applications that + do not embed ``CloudXRLauncher`` need ``source ~/.cloudxr/run/cloudxr.env`` + first, so that ``XR_RUNTIME_JSON`` and ``NV_CXR_RUNTIME_DIR`` point at it. + +Library defaults are fail-fast: ``OpenXRSession`` uses ``wait_for_system=false`` +and ``VizSessionConfig`` uses ``xr_system_wait_seconds = 0``, so -35 is raised +on the first call rather than waited out. Set either one to block until a +headset connects — that is what the examples do when they let you start the app +before putting the headset on. diff --git a/src/core/cloudxr_tests/python/test_launcher.py b/src/core/cloudxr_tests/python/test_launcher.py index 711cb389a..f5172c144 100644 --- a/src/core/cloudxr_tests/python/test_launcher.py +++ b/src/core/cloudxr_tests/python/test_launcher.py @@ -4,8 +4,11 @@ """Tests for isaacteleop.cloudxr.launcher — CloudXRLauncher lifecycle.""" import argparse +import contextlib +import logging import os import signal +import socket import subprocess import sys from contextlib import contextmanager @@ -50,6 +53,30 @@ def ensure_logs_dir(self) -> Path: self._logs_dir.mkdir(parents=True, exist_ok=True) return self._logs_dir + def env_filepath(self) -> str: + return os.path.join(self._run_dir, "cloudxr.env") + + +@contextmanager +def _live_ipc_socket(run_dir: str): + """Serve ``run_dir``'s IPC socket for the duration of the block. + + Binds relative from a chdir: AF_UNIX ``sun_path`` caps at 108 bytes, + which pytest's tmp_path can exceed. + """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + cwd = os.getcwd() + try: + os.chdir(run_dir) + with contextlib.suppress(FileNotFoundError): + os.remove("ipc_cloudxr") + sock.bind("ipc_cloudxr") + sock.listen(1) + yield sock + finally: + os.chdir(cwd) + sock.close() + def _make_mock_popen(pid: int = 12345, poll_returns: list | None = None) -> MagicMock: """Create a mock subprocess.Popen with configurable poll() behaviour.""" @@ -297,31 +324,33 @@ def test_context_manager_stops_on_exit(self, tmp_path): # ============================================================================ +@_posix_only class TestCleanupStaleRuntime: """Tests for CloudXRLauncher._cleanup_stale_runtime.""" - def test_removes_stale_sentinel_files(self, tmp_path): - """Stale ipc_cloudxr, runtime_started, and pidfiles are removed.""" + @staticmethod + def _stale_run_dir(tmp_path) -> tuple[str, list[str]]: + """Create a run dir holding every sentinel file; return it and their paths.""" run_dir = str(tmp_path / "run") os.makedirs(run_dir) - ipc_socket = os.path.join(run_dir, "ipc_cloudxr") - sentinel = os.path.join(run_dir, "runtime_started") - cloudxr_pid = os.path.join(run_dir, "cloudxr.pid") - Path(ipc_socket).touch() - Path(sentinel).touch() - Path(cloudxr_pid).touch() - + paths = [ + os.path.join(run_dir, name) + for name in ("ipc_cloudxr", "runtime_started", "cloudxr.pid") + ] + for path in paths: + Path(path).touch() + return run_dir, paths + + def test_removes_stale_sentinel_files(self, tmp_path, caplog): + """A socket file nobody is serving is stale: removed, at WARNING.""" + run_dir, paths = self._stale_run_dir(tmp_path) fake_cfg = _FakeEnvConfig(run_dir, tmp_path / "logs") - with patch( - "isaacteleop.cloudxr.launcher.subprocess.run", - return_value=MagicMock(returncode=1), - ): + with caplog.at_level(logging.WARNING, logger="isaacteleop.cloudxr.launcher"): CloudXRLauncher._cleanup_stale_runtime(fake_cfg) - assert not os.path.exists(ipc_socket) - assert not os.path.exists(sentinel) - assert not os.path.exists(cloudxr_pid) + assert not any(os.path.exists(p) for p in paths) + assert len(caplog.records) == len(paths) def test_noop_when_no_stale_files(self, tmp_path): """No errors when the run directory has no stale files.""" @@ -331,28 +360,19 @@ def test_noop_when_no_stale_files(self, tmp_path): fake_cfg = _FakeEnvConfig(run_dir, tmp_path / "logs") CloudXRLauncher._cleanup_stale_runtime(fake_cfg) - def test_handles_missing_fuser(self, tmp_path): - """Sentinel files are still cleaned up when fuser is not found.""" - run_dir = str(tmp_path / "run") - os.makedirs(run_dir) - ipc_socket = os.path.join(run_dir, "ipc_cloudxr") - sentinel = os.path.join(run_dir, "runtime_started") - cloudxr_pid = os.path.join(run_dir, "cloudxr.pid") - Path(ipc_socket).touch() - Path(sentinel).touch() - Path(cloudxr_pid).touch() - + def test_refuses_when_runtime_is_live(self, tmp_path): + """A served socket is a live runtime: refuse, and keep its files.""" + run_dir, paths = self._stale_run_dir(tmp_path) fake_cfg = _FakeEnvConfig(run_dir, tmp_path / "logs") - with patch( - "isaacteleop.cloudxr.launcher.subprocess.run", - side_effect=FileNotFoundError("fuser not found"), - ): - CloudXRLauncher._cleanup_stale_runtime(fake_cfg) + with _live_ipc_socket(run_dir): + with pytest.raises(RuntimeError, match="already serving") as exc_info: + CloudXRLauncher._cleanup_stale_runtime(fake_cfg) + assert all(os.path.exists(p) for p in paths) - assert not os.path.exists(ipc_socket) - assert not os.path.exists(sentinel) - assert not os.path.exists(cloudxr_pid) + message = str(exc_info.value) + assert fake_cfg.env_filepath() in message + assert "--no-launch-cloudxr-runtime" in message class TestLaunchArgumentHelpers: @@ -537,6 +557,22 @@ def test_launcher_defaults_apply_when_unset(self, tmp_path, monkeypatch): assert cfg._resolved_env is not None assert cfg._resolved_env["NV_DEVICE_PROFILE"] == "Quest3" + def test_resolved_reads_back_the_applied_value(self, tmp_path, monkeypatch): + """resolved() is what the startup banner prints the device profile from.""" + monkeypatch.delenv("NV_DEVICE_PROFILE", raising=False) + + from isaacteleop.cloudxr.env_config import EnvConfig + + assert EnvConfig().resolved("NV_DEVICE_PROFILE") is None + + cfg = EnvConfig.from_args( + str(tmp_path), + launcher_defaults={"NV_DEVICE_PROFILE": "auto-native"}, + ) + + assert cfg.resolved("NV_DEVICE_PROFILE") == "auto-native" + assert cfg.resolved("NOT_A_KEY") is None + def test_env_file_overrides_launcher_defaults(self, tmp_path, monkeypatch): monkeypatch.delenv("NV_DEVICE_PROFILE", raising=False) env_file = tmp_path / "custom.env" diff --git a/src/core/cloudxr_tests/python/test_runtime.py b/src/core/cloudxr_tests/python/test_runtime.py index 42dbfd39e..66e12dc62 100644 --- a/src/core/cloudxr_tests/python/test_runtime.py +++ b/src/core/cloudxr_tests/python/test_runtime.py @@ -6,6 +6,7 @@ import importlib.util import os +import socket import threading import time from types import SimpleNamespace @@ -18,6 +19,7 @@ _should_join_main, _should_use_exp, get_sdk_path, + is_runtime_live, resolve_cloudxr_runtime_module, terminate_or_kill_runtime, wait_for_runtime_ready_sync, @@ -442,5 +444,55 @@ def test_noop_if_already_dead(self): proc.kill.assert_not_called() +@pytest.mark.skipif( + not hasattr(socket, "AF_UNIX"), reason="AF_UNIX sockets are POSIX-only" +) +class TestIsRuntimeLive: + """Tests for the IPC-socket liveness probe.""" + + def test_false_when_socket_missing(self, tmp_path): + """An empty run directory has no runtime.""" + run_dir = str(tmp_path) + assert is_runtime_live(run_dir) is False + + def test_false_for_leftover_socket_file(self, tmp_path): + """A socket file nobody listens on is the post-crash case: not live.""" + run_dir = str(tmp_path) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + cwd = os.getcwd() + try: + os.chdir(run_dir) + sock.bind("ipc_cloudxr") + finally: + os.chdir(cwd) + sock.close() + + assert os.path.exists(os.path.join(run_dir, "ipc_cloudxr")) + assert is_runtime_live(run_dir) is False + + def test_true_while_socket_is_served(self, tmp_path): + """A listening peer means a runtime owns the run directory.""" + run_dir = str(tmp_path) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + cwd = os.getcwd() + try: + os.chdir(run_dir) + sock.bind("ipc_cloudxr") + sock.listen(1) + os.chdir(cwd) + assert is_runtime_live(run_dir) is True + finally: + os.chdir(cwd) + sock.close() + + def test_ambiguous_errors_count_as_live(self, tmp_path): + """Errors that do not prove absence must not license a takeover.""" + run_dir = str(tmp_path) + (tmp_path / "ipc_cloudxr").touch() + with patch("socket.socket") as mock_socket: + mock_socket.return_value.connect.side_effect = PermissionError + assert is_runtime_live(run_dir) is True + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/src/core/oxr/cpp/oxr_session.cpp b/src/core/oxr/cpp/oxr_session.cpp index 2d57c815b..9f6aa5e95 100644 --- a/src/core/oxr/cpp/oxr_session.cpp +++ b/src/core/oxr/cpp/oxr_session.cpp @@ -146,11 +146,23 @@ void OpenXRSession::create_system() break; } - if (result != XR_ERROR_FORM_FACTOR_UNAVAILABLE || !wait_for_system_) + if (result != XR_ERROR_FORM_FACTOR_UNAVAILABLE) { throw std::runtime_error("Failed to get OpenXR system: " + std::to_string(result)); } + if (!wait_for_system_) + { + // xrCreateInstance already succeeded, so the runtime was found + // (a missing one gives -51). No headset is attached to it. + throw std::runtime_error( + "Failed to get OpenXR system: XR_ERROR_FORM_FACTOR_UNAVAILABLE (-35). " + "The OpenXR runtime is up but no headset is connected to it. " + "Connect the headset, check NV_DEVICE_PROFILE matches it, or pass " + "wait_for_system=true to block until it connects. " + "See docs/source/references/cloudxr.rst, Troubleshooting."); + } + if (!logged_waiting) { std::cout << "OpenXR HMD form factor is unavailable; waiting for a system..." << std::endl; diff --git a/src/python/isaacteleop/cloudxr/__main__.py b/src/python/isaacteleop/cloudxr/__main__.py index 59a01337e..54c5134f9 100644 --- a/src/python/isaacteleop/cloudxr/__main__.py +++ b/src/python/isaacteleop/cloudxr/__main__.py @@ -195,14 +195,22 @@ def main() -> None: raise SystemExit(1) from exc oob_progress("setup-oob", "preflight OK") - with CloudXRLauncher( - install_dir=args.cloudxr_install_dir, - env_config=args.cloudxr_env_config, - accept_eula=args.accept_eula, - setup_oob=args.setup_oob, - usb_local=args.usb_local, - host_client=args.host_client, - ) as launcher: + try: + launcher_ctx = CloudXRLauncher( + install_dir=args.cloudxr_install_dir, + env_config=args.cloudxr_env_config, + accept_eula=args.accept_eula, + setup_oob=args.setup_oob, + usb_local=args.usb_local, + host_client=args.host_client, + ) + except RuntimeError as exc: + # Operator-facing conditions (a live runtime, a rejected EULA); the + # message is the whole point, so don't bury it in a traceback. + print(f"\n\033[31m{exc}\033[0m\n", file=sys.stderr) + raise SystemExit(1) from None + + with launcher_ctx as launcher: cxr_ver = runtime_version() print( f"Running Isaac Teleop \033[36m{isaacteleop_version}\033[0m, CloudXR Runtime \033[36m{cxr_ver}\033[0m" @@ -218,6 +226,13 @@ def main() -> None: print( f"CloudXR WSS proxy: \033[36mrunning\033[0m, log file: \033[90m{wss_log}\033[0m" ) + # A profile that does not match the connecting device is the usual + # cause of XR_ERROR_FORM_FACTOR_UNAVAILABLE (-35) in clients. + profile = env_cfg.resolved("NV_DEVICE_PROFILE") + print( + f"device profile: \033[36m{profile}\033[0m " + "\033[90m(NV_DEVICE_PROFILE)\033[0m" + ) if args.usb_local: _hosted_client_url = f"https://127.0.0.1:{usb_ui_port()}/" diff --git a/src/python/isaacteleop/cloudxr/env_config.py b/src/python/isaacteleop/cloudxr/env_config.py index 4408f6809..de30b922f 100644 --- a/src/python/isaacteleop/cloudxr/env_config.py +++ b/src/python/isaacteleop/cloudxr/env_config.py @@ -14,6 +14,9 @@ import warnings from pathlib import Path +DEFAULT_DEVICE_PROFILE = "Quest3" +"""``NV_DEVICE_PROFILE`` used when no env file, process env, or caller sets one.""" + class EnvConfig: """Singleton holding CloudXR env configuration and resolved state. @@ -50,7 +53,7 @@ class EnvConfig: "NV_CXR_ENABLE_PUSH_DEVICES": "true", "NV_CXR_ENABLE_TENSOR_DATA": "true", "NV_CXR_FILE_LOGGING": "true", - "NV_DEVICE_PROFILE": "auto-webrtc", + "NV_DEVICE_PROFILE": DEFAULT_DEVICE_PROFILE, } def __new__(cls) -> "EnvConfig": @@ -99,6 +102,12 @@ def env_filepath(self) -> str: """Return the path to the env file.""" return os.path.join(self.openxr_run_dir(), self._env_filename()) + def resolved(self, key: str) -> str | None: + """Return the resolved value of ``key``, or ``None`` before resolution.""" + if self._resolved_env is None: + return None + return self._resolved_env.get(key) + # ------------------------------------------------------------------------- # Private instance methods # ------------------------------------------------------------------------- diff --git a/src/python/isaacteleop/cloudxr/launcher.py b/src/python/isaacteleop/cloudxr/launcher.py index 0236b8e15..2f742cdac 100644 --- a/src/python/isaacteleop/cloudxr/launcher.py +++ b/src/python/isaacteleop/cloudxr/launcher.py @@ -20,15 +20,15 @@ import subprocess import sys import threading -import time from datetime import datetime, timezone from pathlib import Path -from .env_config import EnvConfig +from .env_config import DEFAULT_DEVICE_PROFILE, EnvConfig from .runtime import ( RUNTIME_STARTUP_TIMEOUT_SEC, RUNTIME_TERMINATE_TIMEOUT_SEC, get_sdk_path, + is_runtime_live, resolve_cloudxr_runtime_module, check_eula, wait_for_runtime_ready_sync, @@ -36,8 +36,6 @@ logger = logging.getLogger(__name__) -DEFAULT_DEVICE_PROFILE = "Quest3" - _RUNTIME_WORKER_CODE = """\ import sys, os sys.path = [p for p in sys.path if p] @@ -120,8 +118,9 @@ def __init__( subprocess is needed. Raises: - RuntimeError: If the EULA is not accepted or the runtime - fails to start within the timeout. + RuntimeError: If the EULA is not accepted, another runtime is + already serving *install_dir*, or the runtime fails to + start within the timeout. ValueError: If *start_wss_proxy* is ``False`` while any WSS-only option (*setup_oob*, *usb_local*, or *host_client*) is set. """ @@ -569,43 +568,33 @@ def _restore_signal_handlers(self) -> None: @staticmethod def _cleanup_stale_runtime(env_cfg: EnvConfig) -> None: - """Remove stale sentinel files from a previous runtime that wasn't cleaned up. + """Refuse to start over a live runtime; otherwise clear stale sentinels. + + A run directory holds one runtime. Liveness is decided by + connecting to the IPC socket, not by its existence — the file + routinely outlives the process that made it. - If the ``ipc_cloudxr`` socket still exists in the run directory, a - previous Monado/CloudXR process is likely still alive. We send - SIGTERM to the process group that owns the socket, giving it a - chance to exit cleanly before we start a fresh runtime. + Raises: + RuntimeError: If a runtime is already serving the run directory. """ run_dir = env_cfg.openxr_run_dir() - ipc_socket = os.path.join(run_dir, "ipc_cloudxr") - if os.path.exists(ipc_socket): - logger.warning( - "Stale CloudXR IPC socket found at %s; attempting cleanup of previous runtime", - ipc_socket, + if is_runtime_live(run_dir): + raise RuntimeError( + f"A CloudXR runtime is already serving {run_dir}; starting a " + "second one would drop the live session. To use the running " + f"runtime: source {env_cfg.env_filepath()} and pass " + "--no-launch-cloudxr-runtime. To replace it, stop that runtime " + "first (Ctrl+C in its terminal)." ) - try: - result = subprocess.run( - ["fuser", "-k", "-TERM", ipc_socket], - capture_output=True, - timeout=5, - ) - if result.returncode == 0: - time.sleep(1) - logger.info("Sent SIGTERM to processes holding stale IPC socket") - except (FileNotFoundError, subprocess.TimeoutExpired): - pass + for name in ("ipc_cloudxr", "runtime_started", "monado.pid", "cloudxr.pid"): + path = os.path.join(run_dir, name) try: - os.remove(ipc_socket) + os.remove(path) except FileNotFoundError: - pass - - for name in ("runtime_started", "monado.pid", "cloudxr.pid"): - try: - os.remove(os.path.join(run_dir, name)) - except FileNotFoundError: - pass + continue + logger.warning("Removed stale CloudXR runtime file %s", path) def _collect_startup_failure_detail(self, logs_dir: Path) -> str: """Build a diagnostic string after a failed runtime startup. diff --git a/src/python/isaacteleop/cloudxr/runtime.py b/src/python/isaacteleop/cloudxr/runtime.py index d0ea5b4a6..4e3c3bf0b 100644 --- a/src/python/isaacteleop/cloudxr/runtime.py +++ b/src/python/isaacteleop/cloudxr/runtime.py @@ -8,6 +8,7 @@ import os import shutil import signal +import socket import sys import threading import time @@ -30,6 +31,9 @@ RUNTIME_POLL_INTERVAL_SEC: float = 0.5 """Polling interval [s] used by :func:`wait_for_runtime_ready_sync`.""" +IPC_PROBE_TIMEOUT_SEC: float = 0.5 +"""Connect timeout [s] for :func:`is_runtime_live`.""" + _CLOUDXR_EXP_ENV = "ISAAC_TELEOP_CLOUDXR_EXP" _CLOUDXR_JOIN_MAIN_ENV = "ISAAC_TELEOP_CLOUDXR_JOIN_MAIN" _CLOUDXR_MODULE = "isaacteleop.cloudxr" @@ -254,6 +258,30 @@ def wait_for_runtime_ready_sync( return False +def is_runtime_live(run_dir: str) -> bool: + """Return whether a CloudXR runtime is currently serving ``run_dir``. + + The socket file outliving its process is the common case, so existence + proves nothing; only a successful ``connect()`` does. Ambiguous errors + (permissions, timeout) count as live: refusing to start is recoverable, + tearing down someone else's session is not. + """ + path = os.path.join(run_dir, "ipc_cloudxr") + if not os.path.exists(path): + return False + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.settimeout(IPC_PROBE_TIMEOUT_SEC) + sock.connect(path) + return True + except (ConnectionRefusedError, FileNotFoundError): + return False + except OSError: + return True + finally: + sock.close() + + def _load_libcloudxr(sdk_path: str) -> ctypes.CDLL: """Load libcloudxr.so with RTLD_DEEPBIND so the native stack resolves symbols from its own packaged libraries.""" deepbind = getattr(os, "RTLD_DEEPBIND", 0) diff --git a/src/viz/xr/cpp/openxr_session.cpp b/src/viz/xr/cpp/openxr_session.cpp index fb0f1ce06..576454cc8 100644 --- a/src/viz/xr/cpp/openxr_session.cpp +++ b/src/viz/xr/cpp/openxr_session.cpp @@ -185,8 +185,12 @@ void OpenXrSession::wait_for_system(int system_wait_seconds) { throw std::runtime_error( "OpenXrSession: xrGetSystem timed out waiting for HMD " - "(XR_ERROR_FORM_FACTOR_UNAVAILABLE) after " + - std::to_string(system_wait_seconds) + "s"); + "(XR_ERROR_FORM_FACTOR_UNAVAILABLE, -35) after " + + std::to_string(system_wait_seconds) + + "s. The OpenXR runtime is up but no headset is connected to it. " + "Connect the headset, check NV_DEVICE_PROFILE matches it, or raise " + "xr_system_wait_seconds to block until it connects. " + "See docs/source/references/cloudxr.rst, Troubleshooting."); } if (!announced || (now - last_log) >= kLogEvery) {