From c34ae46684cef2d46d7cc548606d8606148d1fe8 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:19:48 +0000 Subject: [PATCH 1/2] [SSH] Report port forwarding refusals as their own telemetry categories Both refusals from the previous changes were reported as SERVER_START_TIMEOUT (the driver proxy's 403) or UNKNOWN (the up-front setting check), so they looked like slow server starts or CLI bugs. They now report PORT_FORWARDING_DISABLED when the setting check stops the connection before any compute is started, and DRIVER_PROXY_FORBIDDEN when the driver proxy refuses the server after its run was submitted. Keeping them apart shows how often the setting can't be read and the proxy has to catch it. The values need the matching universe enum change to be stored. DECO-28889 Co-authored-by: Isaac --- .../ssh/connect-serverless-feature-disabled/output.txt | 5 +++++ .../ssh/connect-serverless-feature-disabled/script | 3 +++ .../output.txt | 5 +++++ .../connect-serverless-port-forwarding-disabled/script | 3 +++ experimental/ssh/internal/client/client.go | 4 ++++ libs/telemetry/protos/ssh_tunnel.go | 9 +++++++++ 6 files changed, 29 insertions(+) diff --git a/acceptance/ssh/connect-serverless-feature-disabled/output.txt b/acceptance/ssh/connect-serverless-feature-disabled/output.txt index bef259c7ff2..678cdab2437 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/output.txt +++ b/acceptance/ssh/connect-serverless-feature-disabled/output.txt @@ -3,6 +3,11 @@ === CLI ssh connect +=== Reported error category + +>>> jq -r select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category [OUTPUT_DIR]/out.requests.txt +DRIVER_PROXY_FORBIDDEN + >>> print_requests.py //api/2.2/jobs/runs/cancel { "method": "POST", diff --git a/acceptance/ssh/connect-serverless-feature-disabled/script b/acceptance/ssh/connect-serverless-feature-disabled/script index 9fb5bb05b06..68bc8b23528 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/script +++ b/acceptance/ssh/connect-serverless-feature-disabled/script @@ -15,4 +15,7 @@ title "CLI ssh connect\n" musterr $CLI ssh connect --name feature-disabled --releases-dir="$PWD/releases" -- "echo 'Connection successful'" >out.connect.txt 2>&1 add_repl "$(grep -o 'run ID: [0-9]*' out.connect.txt | grep -o '[0-9]*$')" RUN_ID +title "Reported error category\n" +trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" + trace print_requests.py //api/2.2/jobs/runs/cancel diff --git a/acceptance/ssh/connect-serverless-port-forwarding-disabled/output.txt b/acceptance/ssh/connect-serverless-port-forwarding-disabled/output.txt index 3b3f5fea6f9..26b382fefe6 100644 --- a/acceptance/ssh/connect-serverless-port-forwarding-disabled/output.txt +++ b/acceptance/ssh/connect-serverless-port-forwarding-disabled/output.txt @@ -7,4 +7,9 @@ Connecting to port-forwarding-disabled... Error: port forwarding to serverless compute is disabled for this workspace (setting port_fwd_serverless_enabled), and the SSH tunnel needs it. Ask a workspace or account admin to enable it, or connect to a dedicated cluster with --cluster +=== Reported error category + +>>> jq -r select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category [OUTPUT_DIR]/out.requests.txt +PORT_FORWARDING_DISABLED + >>> print_requests.py //api diff --git a/acceptance/ssh/connect-serverless-port-forwarding-disabled/script b/acceptance/ssh/connect-serverless-port-forwarding-disabled/script index 6c250774c4b..2dbc12b6ac0 100644 --- a/acceptance/ssh/connect-serverless-port-forwarding-disabled/script +++ b/acceptance/ssh/connect-serverless-port-forwarding-disabled/script @@ -9,5 +9,8 @@ print_requests.py //api/2.1/settings >/dev/null title "CLI ssh connect\n" trace musterr $CLI ssh connect --name port-forwarding-disabled -- "echo 'Connection successful'" +title "Reported error category\n" +trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" + # Expect no requests other than reads: the connection stops at the settings check. trace print_requests.py //api diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 28197d143c5..3ac21fab8f9 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -359,6 +359,7 @@ func Run(ctx context.Context, client *databricks.WorkspaceClient, opts ClientOpt // running, so there is no compute left to spare, and every IDE reconnect would repeat the read. if opts.IsServerlessMode() && opts.ServerMetadata == "" { if err := checkPortForwardingToServerless(ctx, client); err != nil { + outcome.errorCategory = protos.SshTunnelErrorCategoryPortForwardingDisabled return err } } @@ -467,6 +468,9 @@ func Run(ctx context.Context, client *databricks.WorkspaceClient, opts ClientOpt userName, serverPort, clusterID, err = ensureSSHServerIsRunning(ctx, client, version, secretScopeName, opts) if err != nil { outcome.errorCategory = protos.SshTunnelErrorCategoryServerStartTimeout + if errors.Is(err, errDriverProxyForbidden) { + outcome.errorCategory = protos.SshTunnelErrorCategoryDriverProxyForbidden + } return fmt.Errorf("failed to ensure that ssh server is running: %w", err) } outcome.serverStartTimeMs = time.Since(serverStartTime).Milliseconds() diff --git a/libs/telemetry/protos/ssh_tunnel.go b/libs/telemetry/protos/ssh_tunnel.go index 55d8ca30fda..1ec469e914c 100644 --- a/libs/telemetry/protos/ssh_tunnel.go +++ b/libs/telemetry/protos/ssh_tunnel.go @@ -80,6 +80,15 @@ const ( // its metadata never appeared before the timeout. SshTunnelErrorCategoryServerStartTimeout SshTunnelErrorCategory = "SERVER_START_TIMEOUT" + // An administrator turned off port forwarding to serverless compute + // (port_fwd_serverless_enabled), read from the setting before any compute was started. + SshTunnelErrorCategoryPortForwardingDisabled SshTunnelErrorCategory = "PORT_FORWARDING_DISABLED" + + // The driver proxy refused the SSH server with a 403 after its run was submitted. Usually + // port forwarding to serverless compute is off where the CLI cannot read that setting, but + // the 403 is not specific to it, so this names what happened rather than the likely cause. + SshTunnelErrorCategoryDriverProxyForbidden SshTunnelErrorCategory = "DRIVER_PROXY_FORBIDDEN" + // The user interrupted the connection (Ctrl-C or a termination signal). Takes precedence // over the category of whichever step observed the interruption, including steps that shell // out to the IDE and see only a killed child process. From b38d473e547c441eacad229368f60088c665b1b8 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:32:25 +0000 Subject: [PATCH 2/2] Describe DRIVER_PROXY_FORBIDDEN as also covering an already-running server Co-authored-by: Isaac --- libs/telemetry/protos/ssh_tunnel.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/telemetry/protos/ssh_tunnel.go b/libs/telemetry/protos/ssh_tunnel.go index 1ec469e914c..708a453ac63 100644 --- a/libs/telemetry/protos/ssh_tunnel.go +++ b/libs/telemetry/protos/ssh_tunnel.go @@ -84,9 +84,10 @@ const ( // (port_fwd_serverless_enabled), read from the setting before any compute was started. SshTunnelErrorCategoryPortForwardingDisabled SshTunnelErrorCategory = "PORT_FORWARDING_DISABLED" - // The driver proxy refused the SSH server with a 403 after its run was submitted. Usually - // port forwarding to serverless compute is off where the CLI cannot read that setting, but - // the 403 is not specific to it, so this names what happened rather than the likely cause. + // The driver proxy refused a request to the SSH server with a 403, for a server that was + // already running or for one that this connection started. Usually port forwarding to + // serverless compute is off where the CLI cannot read that setting, but the 403 is not + // specific to it, so this names what happened rather than the likely cause. SshTunnelErrorCategoryDriverProxyForbidden SshTunnelErrorCategory = "DRIVER_PROXY_FORBIDDEN" // The user interrupted the connection (Ctrl-C or a termination signal). Takes precedence