Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions keepercommander/commands/pam_launch/connect_timing.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,24 +135,29 @@ def webrtc_connection_poll_sec() -> float:


_PAM_WEBRTC_CONNECT_TIMEOUT_ENV = 'PAM_WEBRTC_CONNECT_TIMEOUT_SEC'
_PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT = 24.0 # seconds — see note below
_PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT = 60.0 # seconds — see note below


def webrtc_connect_timeout_sec() -> float:
"""Maximum wall-clock to wait for the WebRTC data plane to reach
``connected`` after ``OpenConnection`` is sent.

Default 24s — observed ICE completion sometimes lands a few seconds
past the prior 16s bound (notably on TURN-relay fallback paths), so
the client was aborting just before the connection would have come
up. JIT ephemeral accounts need more time to create and connect, so
the extra headroom also covers gateway-side account provisioning
before the data plane comes up. 24s keeps us clearly above the
gateway/guacd-side 15s connect timeout while absorbing that jitter.
If ICE really is stuck (state staying at ``Connecting`` / tube_status
``connecting`` indefinitely) we still fail and the user can re-run —
the retry typically succeeds on a fresh ICE gathering pass. Set
``PAM_WEBRTC_CONNECT_TIMEOUT_SEC`` to override for targeted diagnostics.
Default 60s — raised from 24s because JIT (ephemeral) account
provisioning on the gateway adds significant latency on top of the
standard TURN-relay ICE round-trip. In environments with JIT
enabled, ICE completion regularly lands between 30–45s; the old 24s
default caused the client to abort with "WebRTC connection not
established within timeout" even though the connection succeeded
~10s later (visible as the "Connection established successfully"
ghost message that appears after the error).

60s keeps us well above the worst observed JIT+TURN path (~45s) while
still bounding truly stuck sessions (ICE state permanently
``Connecting`` / tube_status ``connecting``). The retry path is
unchanged — a stuck session can always be re-run and the fresh ICE
gathering pass typically succeeds immediately.

Set ``PAM_WEBRTC_CONNECT_TIMEOUT_SEC`` to override.
"""
return _env_float(_PAM_WEBRTC_CONNECT_TIMEOUT_ENV, _PAM_WEBRTC_CONNECT_TIMEOUT_DEFAULT)

Expand Down
26 changes: 26 additions & 0 deletions keepercommander/commands/workflow/mfa.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import datetime
import getpass
import logging
import time
from typing import NamedTuple, Optional

from ..pam.router_helper import _post_request_to_router
Expand Down Expand Up @@ -742,6 +743,31 @@ def check_workflow_for_launch(
except Exception as e:
logging.error("Failed to check out: %s", sanitize_router_error(e))
return WorkflowGate(allowed=False)

# Router propagation delay: the start_workflow call is processed
# asynchronously, so a validate() immediately after checkout can
# still return WS_READY_TO_START before the router has written the
# WS_STARTED transition. When that happens and handled_ready_to_start
# is already True the outer loop falls to the "already-handled" guard
# and returns WorkflowGate(allowed=False) — the user sees "Workflow
# approved but not yet checked out" again and re-triggers checkout,
# producing an infinite prompt loop.
#
# Fix: poll with exponential back-off (0.5 → 1 → 2 → 2 s, ≤5.5 s
# total) until the router confirms WS_STARTED or we exhaust retries.
# On success we break out of the outer loop immediately; on failure
# we fall through to let the outer loop decide.
_poll_delays = (0.5, 1.0, 2.0, 2.0)
for _delay in _poll_delays:
time.sleep(_delay)
_probe = validator.validate(silent_actionable=True)
if _probe.get('allowed') or _probe.get('block_reason') != 'ready_to_start':
result = _probe
break
else:
result = validator.validate(silent_actionable=True)
if result.get('allowed'):
break
continue

if block_reason == 'waiting' and not handled_waiting and wait:
Expand Down
Loading