From af5a2c733cae5cb7ab7ce7bd493054c6acaaf3c0 Mon Sep 17 00:00:00 2001 From: Martin Sawczynski Date: Fri, 8 May 2026 17:00:13 +0100 Subject: [PATCH 1/2] fix(workflow): poll for WS_STARTED after checkout to break ready_to_start loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `check_workflow_for_launch` calls `start_workflow_for_record` and then immediately re-validates via `validator.validate()`. Because the router processes `start_workflow` asynchronously, the validate call can return `WS_READY_TO_START` before the `WS_STARTED` transition is visible. At that point `handled_ready_to_start` is already `True`, so the outer loop falls to the "already-handled" guard and returns `WorkflowGate(allowed=False)`. The user sees "Workflow approved but not yet checked out" a second time and is prompted to check out again, creating an infinite loop until the session is restarted. Fix: after a successful `start_workflow_for_record`, poll with exponential back-off (0.5 → 1 → 2 → 2 s, ≤5.5 s total) until the router confirms `WS_STARTED` or retries are exhausted. On success the outer loop breaks immediately; on failure the outer loop continues to its normal handling. Confirmed in lab: `pam launch ` with auto-checkout no longer loops; gateway reaches "Establishing secure session" without error. Co-authored-by: Cursor --- keepercommander/commands/workflow/mfa.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/keepercommander/commands/workflow/mfa.py b/keepercommander/commands/workflow/mfa.py index d9a977eac..0d96355af 100644 --- a/keepercommander/commands/workflow/mfa.py +++ b/keepercommander/commands/workflow/mfa.py @@ -12,6 +12,7 @@ import datetime import getpass import logging +import time from typing import NamedTuple, Optional from ..pam.router_helper import _post_request_to_router @@ -742,6 +743,31 @@ def check_workflow_for_launch( except Exception as e: logging.error("Failed to check out: %s", sanitize_router_error(e)) return WorkflowGate(allowed=False) + + # Router propagation delay: the start_workflow call is processed + # asynchronously, so a validate() immediately after checkout can + # still return WS_READY_TO_START before the router has written the + # WS_STARTED transition. When that happens and handled_ready_to_start + # is already True the outer loop falls to the "already-handled" guard + # and returns WorkflowGate(allowed=False) — the user sees "Workflow + # approved but not yet checked out" again and re-triggers checkout, + # producing an infinite prompt loop. + # + # Fix: poll with exponential back-off (0.5 → 1 → 2 → 2 s, ≤5.5 s + # total) until the router confirms WS_STARTED or we exhaust retries. + # On success we break out of the outer loop immediately; on failure + # we fall through to let the outer loop decide. + _poll_delays = (0.5, 1.0, 2.0, 2.0) + for _delay in _poll_delays: + time.sleep(_delay) + _probe = validator.validate(silent_actionable=True) + if _probe.get('allowed') or _probe.get('block_reason') != 'ready_to_start': + result = _probe + break + else: + result = validator.validate(silent_actionable=True) + if result.get('allowed'): + break continue if block_reason == 'waiting' and not handled_waiting and wait: From c0118147515e4e4cd3b80b8b46c1fe7c392b7875 Mon Sep 17 00:00:00 2001 From: Martin Sawczynski Date: Fri, 8 May 2026 17:11:18 +0100 Subject: [PATCH 2/2] fix(pam-launch): raise WebRTC connect timeout default from 24s to 60s The 24s default causes pam launch to abort with "WebRTC connection not established within timeout" on JIT-enabled environments, even though the connection succeeds ~10 s later. The symptom is visible as the "Connection established successfully" ghost message that appears in the terminal after the error. Root cause: JIT (ephemeral) account provisioning on the gateway adds 30-45s of wall-clock time on top of the standard TURN-relay ICE path. The old default was sized for non-JIT TURN relay (~15-20s observed), so JIT connections reliably hit the ceiling. Fix: raise the default to 60s, which is well above the worst observed JIT+TURN path (~45s) and still bounds genuinely stuck sessions (ICE state permanently Connecting / tube_status connecting). PAM_WEBRTC_CONNECT_TIMEOUT_SEC is unchanged and still overrides the default for diagnostics or unusually slow environments. Co-authored-by: Cursor --- .../commands/pam_launch/connect_timing.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/keepercommander/commands/pam_launch/connect_timing.py b/keepercommander/commands/pam_launch/connect_timing.py index 6e5bf6e62..2188e3cd2 100644 --- a/keepercommander/commands/pam_launch/connect_timing.py +++ b/keepercommander/commands/pam_launch/connect_timing.py @@ -135,24 +135,29 @@ def webrtc_connection_poll_sec() -> float: _PAM_WEBRTC_CONNECT_TIMEOUT_ENV = 'PAM_WEBRTC_CONNECT_TIMEOUT_SEC' -_PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT = 24.0 # seconds — see note below +_PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT = 60.0 # seconds — see note below def webrtc_connect_timeout_sec() -> float: """Maximum wall-clock to wait for the WebRTC data plane to reach ``connected`` after ``OpenConnection`` is sent. - Default 24s — observed ICE completion sometimes lands a few seconds - past the prior 16s bound (notably on TURN-relay fallback paths), so - the client was aborting just before the connection would have come - up. JIT ephemeral accounts need more time to create and connect, so - the extra headroom also covers gateway-side account provisioning - before the data plane comes up. 24s keeps us clearly above the - gateway/guacd-side 15s connect timeout while absorbing that jitter. - If ICE really is stuck (state staying at ``Connecting`` / tube_status - ``connecting`` indefinitely) we still fail and the user can re-run — - the retry typically succeeds on a fresh ICE gathering pass. Set - ``PAM_WEBRTC_CONNECT_TIMEOUT_SEC`` to override for targeted diagnostics. + Default 60s — raised from 24s because JIT (ephemeral) account + provisioning on the gateway adds significant latency on top of the + standard TURN-relay ICE round-trip. In environments with JIT + enabled, ICE completion regularly lands between 30–45s; the old 24s + default caused the client to abort with "WebRTC connection not + established within timeout" even though the connection succeeded + ~10s later (visible as the "Connection established successfully" + ghost message that appears after the error). + + 60s keeps us well above the worst observed JIT+TURN path (~45s) while + still bounding truly stuck sessions (ICE state permanently + ``Connecting`` / tube_status ``connecting``). The retry path is + unchanged — a stuck session can always be re-run and the fresh ICE + gathering pass typically succeeds immediately. + + Set ``PAM_WEBRTC_CONNECT_TIMEOUT_SEC`` to override. """ return _env_float(_PAM_WEBRTC_CONNECT_TIMEOUT_ENV, _PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT)