From f3c8d7e5231262233ba68401850fc49b2d156d92 Mon Sep 17 00:00:00 2001 From: Mihalache Marius Date: Mon, 7 Sep 2026 23:26:50 +0300 Subject: [PATCH 1/4] fix(agent): classify LLM provider 400s as USER with an actionable detail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An LLM gateway 400 was falling into the unclassified branch of `_classify`, so it reached telemetry as HTTP_ERROR / UNKNOWN with `detail` set to the HTTP reason phrase — the two words "Bad Request". That is the largest slice of the fleet's two-word failure messages (PC-5002), and it hides causes the customer can actually fix, such as the `max_tokens=65535` that Agent Builder itself wrote into the model settings. 400 now maps to LLM_PROVIDER_BAD_REQUEST / USER. Where the gateway supplies a first-party ProblemDetails `detail`, that still wins; otherwise the error carries a canned message pointing at the agent's model settings. The provider body is deliberately not read out — it may carry customer PII and is already recorded on the tenant-scoped LLM call span. 404 is left in UNKNOWN on purpose: every LLM-gateway 404 seen in prod over 30 days was a missing or unreachable deployment (BYO relay not connected, Azure DeploymentNotFound, a retired Bedrock model), which is Deployment rather than User, and deserves its own decision. Also fixes a stale file reference in the develop-agent-module skill (`exceptions/licensing.py` → `exceptions/llm.py`). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01UZ5LHC1Xhp1zBRfYnYENwv --- .claude/skills/develop-agent-module/SKILL.md | 2 +- .../agent/exceptions/exceptions.py | 3 + src/uipath_langchain/agent/exceptions/llm.py | 133 +++++++- tests/agent/react/test_llm_node.py | 53 ++++ tests/agent/test_llm.py | 289 +++++++++++++++++- 5 files changed, 463 insertions(+), 17 deletions(-) diff --git a/.claude/skills/develop-agent-module/SKILL.md b/.claude/skills/develop-agent-module/SKILL.md index 66a66736f..262624119 100644 --- a/.claude/skills/develop-agent-module/SKILL.md +++ b/.claude/skills/develop-agent-module/SKILL.md @@ -113,7 +113,7 @@ Use the structured error types in `exceptions/` — never raise raw `Exception`, - **Runtime errors** (during execution): `AgentRuntimeError(code=AgentRuntimeErrorCode.X, title=..., detail=..., category=...)` - **Startup errors** (during init): `AgentStartupError(code=AgentStartupErrorCode.X, title=..., detail=..., category=...)` - **HTTP errors from platform calls**: catch `EnrichedException`, map via `raise_for_enriched()` in `exceptions/helpers.py` -- **LLM provider errors**: handled by `raise_for_provider_http_error()` in `exceptions/licensing.py` +- **LLM provider errors**: handled by `raise_for_provider_http_error()` in `exceptions/llm.py` - Always chain exceptions: `raise AgentRuntimeError(...) from e` ## Testing diff --git a/src/uipath_langchain/agent/exceptions/exceptions.py b/src/uipath_langchain/agent/exceptions/exceptions.py index afd5c5fac..1db7bee50 100644 --- a/src/uipath_langchain/agent/exceptions/exceptions.py +++ b/src/uipath_langchain/agent/exceptions/exceptions.py @@ -31,6 +31,9 @@ class AgentRuntimeErrorCode(str, Enum): HTTP_ERROR = "HTTP_ERROR" LICENSE_NOT_AVAILABLE = "LICENSE_NOT_AVAILABLE" LLM_PROVIDER_FORBIDDEN = "LLM_PROVIDER_FORBIDDEN" + LLM_PROVIDER_BAD_REQUEST = "LLM_PROVIDER_BAD_REQUEST" + LLM_PROVIDER_NOT_FOUND = "LLM_PROVIDER_NOT_FOUND" + LLM_BYO_CONNECTION_UNAVAILABLE = "LLM_BYO_CONNECTION_UNAVAILABLE" # Routing ROUTING_ERROR = "ROUTING_ERROR" diff --git a/src/uipath_langchain/agent/exceptions/llm.py b/src/uipath_langchain/agent/exceptions/llm.py index 287b2f7cd..9f9327002 100644 --- a/src/uipath_langchain/agent/exceptions/llm.py +++ b/src/uipath_langchain/agent/exceptions/llm.py @@ -21,6 +21,55 @@ _LICENSE_ERROR_CODE = 10000 _LICENSE_TITLE = "license not available" +# A canned, provider-free replacement for the useless HTTP reason phrase. The +# relayed provider message is deliberately NOT read out of the body: +# it may carry customer PII, and it is already recorded on the LLM call span, +# which is tenant-scoped. It has to stand on its own -- USER is not in +# _SHOULD_WRAP_CATEGORIES, so nothing else is prepended to it. +_BAD_REQUEST_DETAIL = ( + "The model provider rejected the request as invalid. Review the agent's model " + "settings (output-token limit, temperature, effort). The provider's own message " + "is recorded on the LLM call span for this run." +) + +_Verdict = tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str] + +_NOT_FOUND_SIGNATURES: tuple[tuple[tuple[str, ...], _Verdict], ...] = ( + ( + ("no active connection found for endpoint",), + ( + AgentRuntimeErrorCode.LLM_BYO_CONNECTION_UNAVAILABLE, + UiPathErrorCategory.DEPLOYMENT, + "The agent's model connection is not available", + "The model this agent uses is served through a bring-your-own-model " + "connection whose relay is not connected. Start the relay client for " + "that connection, or reload the relay on your nodes if you recently " + "changed its configuration, then run the agent again.", + ), + ), + ( + ("deploymentnotfound",), + ( + AgentRuntimeErrorCode.LLM_PROVIDER_NOT_FOUND, + UiPathErrorCategory.DEPLOYMENT, + "The agent's model deployment does not exist", + "If you are using a Bring Your Own configuration " + "make sure it is correctly configured. If the error " + "persists, contact your administrator.", + ), + ), + ( + ("reached the end of its life",), + ( + AgentRuntimeErrorCode.LLM_PROVIDER_NOT_FOUND, + UiPathErrorCategory.DEPLOYMENT, + "The agent's model has been retired", + "The provider has retired the model version this agent is configured " + "to use. Point the agent at a currently supported model.", + ), + ), +) + def raise_for_llm_client_error(error: UiPathError) -> None: """Raise a structured agent error for known LLM-client error codes.""" @@ -60,13 +109,55 @@ def _is_license_error(body: object) -> bool: return isinstance(title, str) and title.strip().lower() == _LICENSE_TITLE +def _body_fields(body: object) -> list[str]: + """The body's free-text fields, lowercased, for marker matching only.""" + if isinstance(body, str): + return [body.lower()] + if not isinstance(body, dict): + return [] + + sources: list[object] = [body] + error = body.get("error") + if isinstance(error, dict): + sources.append(error) + elif isinstance(error, str): + sources.append({"message": error}) + + return [ + value.lower() + for source in sources + if isinstance(source, dict) + for key in ("message", "code", "detail", "title") + if isinstance(value := source.get(key), str) + ] + + +def _match_not_found_signature(body: object) -> _Verdict | None: + """The verdict for a 404 whose body names its own cause, else ``None``.""" + fields = _body_fields(body) + for markers, verdict in _NOT_FOUND_SIGNATURES: + if any(all(marker in field for marker in markers) for field in fields): + return verdict + return None + + def _classify( status_code: int, body: object -) -> tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str]: - """Map an LLM provider HTTP status onto (code, category, title). +) -> tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str | None]: + """Map an LLM provider HTTP status onto (code, category, title, fallback_detail). - 403 is the only status whose meaning depends on the body; keeping the code, - category and title decided in one place stops them drifting apart. + Only 400, 403 and 404 are classified beyond the 5xx/other split. + + 403 and 404 are the statuses whose meaning depends on the body; keeping the + code, category, title and fallback detail decided in one place stops them + drifting apart. Both name a cause only for a body that names its own -- + ``_is_license_error`` for 403, ``_NOT_FOUND_SIGNATURES`` for 404 -- and + leave the rest unnamed rather than guessing. + + ``fallback_detail`` is the customer-facing text to use when the gateway + supplied no ProblemDetails ``detail`` of its own. ``None`` means "fall back + to the HTTP reason phrase" -- the useless two-word message, so only + statuses whose cause we cannot name are left with it. """ if status_code == 403: if _is_license_error(body): @@ -77,17 +168,41 @@ def _classify( title if isinstance(title, str) and title.strip() else "License not available", + None, ) return ( AgentRuntimeErrorCode.LLM_PROVIDER_FORBIDDEN, UiPathErrorCategory.DEPLOYMENT, "LLM provider returned HTTP 403", + None, ) + if status_code == 400: + return ( + AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST, + UiPathErrorCategory.USER, + "LLM provider rejected the request", + _BAD_REQUEST_DETAIL, + ) + + if status_code == 404: + if (verdict := _match_not_found_signature(body)) is not None: + return verdict + title = f"LLM provider returned HTTP {status_code}" if status_code >= 500: - return AgentRuntimeErrorCode.HTTP_ERROR, UiPathErrorCategory.SYSTEM, title - return AgentRuntimeErrorCode.HTTP_ERROR, UiPathErrorCategory.UNKNOWN, title + return ( + AgentRuntimeErrorCode.HTTP_ERROR, + UiPathErrorCategory.SYSTEM, + title, + None, + ) + return ( + AgentRuntimeErrorCode.HTTP_ERROR, + UiPathErrorCategory.UNKNOWN, + title, + None, + ) def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: @@ -98,13 +213,13 @@ def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: """ status_code = error.status_code body = error.body - code, category, title = _classify(status_code, body) - detail = error.body.get("detail") if isinstance(error.body, dict) else None + code, category, title, fallback_detail = _classify(status_code, body) + gateway_detail = body.get("detail") if isinstance(body, dict) else None raise AgentRuntimeError( code=code, title=title, - detail=detail or error.message or str(error), + detail=gateway_detail or fallback_detail or error.message or str(error), category=category, status=status_code, ) from error diff --git a/tests/agent/react/test_llm_node.py b/tests/agent/react/test_llm_node.py index 861679061..8d1d46976 100644 --- a/tests/agent/react/test_llm_node.py +++ b/tests/agent/react/test_llm_node.py @@ -429,6 +429,59 @@ async def test_legacy_raw_provider_error_is_normalized_and_mapped(self): assert info.status == 403 assert info.code.endswith(AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE.value) + @staticmethod + def _http_400() -> httpx.Response: + """The 400 from job 1fab7e97-...: max_tokens written by Agent Builder.""" + request = httpx.Request("POST", "http://gateway/") + return httpx.Response( + 400, + request=request, + json={ + "error": { + "message": ( + "max_tokens is too large: 65535. This model supports at " + "most 32768 completion tokens." + ), + "code": "invalid_value", + "param": "max_tokens", + } + }, + ) + + @pytest.mark.asyncio + async def test_new_client_400_maps_to_user_without_the_provider_body(self): + # Previously this reached telemetry as the two words "Bad Request", + # categorized Unknown. It is now User, with a canned actionable detail + # and no provider text. + node = self._node_raising(UiPathAPIError.from_response(self._http_400())) + + with pytest.raises(AgentRuntimeError) as exc_info: + await node(self.state) + + info = exc_info.value.error_info + assert info.status == 400 + assert info.category == UiPathErrorCategory.USER + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST.value) + assert info.detail != "Bad Request" + assert "65535" not in info.detail + + @pytest.mark.asyncio + async def test_legacy_400_maps_to_user(self): + raw = openai.BadRequestError( + "Bad Request", + response=self._http_400(), + body={"error": {"message": "max_tokens is too large: 65535."}}, + ) + node = self._node_raising(raw) + + with pytest.raises(AgentRuntimeError) as exc_info: + await node(self.state) + + info = exc_info.value.error_info + assert info.status == 400 + assert info.category == UiPathErrorCategory.USER + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST.value) + @pytest.mark.asyncio async def test_unmarked_403_maps_to_provider_forbidden_not_license(self): # Regression for PC-5000 / SRE-654983: an edge or BYOM endpoint refuses diff --git a/tests/agent/test_llm.py b/tests/agent/test_llm.py index 2829a3115..9d88c0d0f 100644 --- a/tests/agent/test_llm.py +++ b/tests/agent/test_llm.py @@ -10,11 +10,19 @@ customer as LICENSE_NOT_AVAILABLE, sending them to look for AGU they already had. -Note on ``detail``: the mapper reads the gateway's ProblemDetails ``detail`` key -and otherwise falls back to ``UiPathAPIError.message`` -- the HTTP reason phrase. -A passthrough provider body is therefore *not* quoted back to the customer; an -unmarked 403 reports "Forbidden" whatever the upstream body contained. The tests -below pin that down as the current contract. +Note on ``detail``: the mapper quotes only the gateway's ProblemDetails +``detail`` key -- first-party UiPath text -- and never the vendor envelope. A +passthrough provider body is therefore *not* quoted back to the customer +whatever it contained. 404 reads the envelope to *classify* -- a body that names +a missing model, deployment or relay is Deployment, and anything else keeps the +5xx/other split -- but still emits its own text. +Where the gateway supplied no ``detail``, 400 and a named 404 fall back to a canned, +actionable message and everything else falls back to ``UiPathAPIError.message``, +the HTTP reason phrase (an unmarked 403 reports "Forbidden"). The reason-phrase +fallback is what made 49% of fleet failures two words long, so the statuses that +dominate that bucket now carry real text that is still free of provider content. + +The tests below pin all of that down as the current contract. """ import httpx @@ -221,12 +229,90 @@ def test_5xx_maps_to_system_http_error(): assert "boom" in info.detail -def test_unclassified_status_remains_unknown(): - err = _api_error(400, {"status": 400, "detail": "bad request"}) +@pytest.mark.parametrize("status_code", [408, 413, 422, 429]) +def test_unclassified_4xx_remains_unknown(status_code: int): + """Only 400, 403 and 404 are classified; the rest of 4xx stays UNKNOWN.""" + err = _api_error(status_code, {"status": status_code, "detail": "nope"}) info = _raise(err).error_info assert info.category == UiPathErrorCategory.UNKNOWN assert info.code.endswith(AgentRuntimeErrorCode.HTTP_ERROR.value) + assert info.title == f"LLM provider returned HTTP {status_code}" + + +# -------------------------------------------------------------------------- +# 400: User, with a canned detail instead of the reason phrase +# -------------------------------------------------------------------------- + +# The body of the 400 that failed 192/192 runs on gpt-4.1-mini-e2e-custom +# (job 1fab7e97-...): max_tokens=65535 written by Agent Builder itself. +_MAX_TOKENS_BODY: dict[str, object] = { + "error": { + "message": ( + "max_tokens is too large: 65535. This model supports at most 32768 " + "completion tokens, whereas you provided 65535." + ), + "code": "invalid_value", + "param": "max_tokens", + } +} + + +@pytest.mark.parametrize( + "err_factory", + [ + pytest.param(lambda: _api_error(400, _MAX_TOKENS_BODY), id="vendor-envelope"), + pytest.param( + lambda: _api_error(400, {"message": "Malformed input request."}), + id="bedrock-envelope", + ), + pytest.param(lambda: _api_error_text(400, _EDGE_HTML), id="raw-html"), + pytest.param(lambda: _api_error(400, {}), id="empty-body"), + ], +) +def test_400_maps_to_user_with_a_canned_detail(err_factory): + info = _raise(err_factory()).error_info + + assert info.status == 400 + assert info.category == UiPathErrorCategory.USER + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST.value) + assert info.title == "LLM provider rejected the request" + # The bare reason phrase is the failure mode being fixed -- it must be gone. + assert info.detail != "Bad Request" + assert "model settings" in info.detail + + +@pytest.mark.parametrize( + "err_factory", + [ + pytest.param(lambda: _api_error(400, _MAX_TOKENS_BODY), id="vendor-envelope"), + pytest.param(lambda: _api_error_text(400, _EDGE_HTML), id="raw-html"), + ], +) +def test_400_does_not_quote_the_provider_body(err_factory): + error = _raise(err_factory()) + + for rendered in (error.error_info.detail, str(error), repr(error)): + assert "65535" not in rendered + assert "doctype" not in rendered.lower() + + +def test_400_prefers_the_gateway_detail_over_the_canned_text(): + """A ProblemDetails ``detail`` is first-party UiPath text and more specific.""" + err = _api_error(400, {"status": 400, "detail": "Model not enabled."}) + info = _raise(err).error_info + + assert info.detail == "Model not enabled." + assert info.category == UiPathErrorCategory.USER + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST.value) + + +def test_user_category_is_not_wrapped_in_the_generic_prefix(): + """USER is outside _SHOULD_WRAP_CATEGORIES, so the canned detail stands alone.""" + info = _raise(_api_error(400, _MAX_TOKENS_BODY)).error_info + + assert not info.detail.startswith("An unexpected error occurred") + assert info.detail.startswith("The model provider rejected the request") def test_legacy_raw_provider_error_is_normalized_and_mapped(): @@ -248,3 +334,192 @@ def test_legacy_raw_provider_error_is_normalized_and_mapped(): assert info.status == 403 assert info.code.endswith(AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE.value) assert info.detail == _DETAIL + + +# -------------------------------------------------------------------------- +# 404: named only where the body names itself, and never User +# -------------------------------------------------------------------------- +# +# The bodies below are the whole agent-attributable 404 population of prd over +# 30 days (24 events on Agents.* / AgentHub.LLM / ConversationalAgents.* +# operation codes). Three signatures cover 17 of them; the remaining 7 are left +# UNKNOWN rather than given a cause the response never claimed -- the mistake +# PC-5000 and SRE-654983 were about. + +# Integration Service, when a bring-your-own-model connection's relay client is +# not connected. 7 events, on gpt-4.1-AMD-LLMGateway / gpt-5.4-AMD-LLMGateway. +# It flaps: the same connection answered OK 21 seconds before returning this. +_RELAY_DOWN_BODY: dict[str, object] = { + "error": { + "message": ( + "No active connection found for endpoint. Ensure the relay client is " + "running and connected for this endpoint. If you recently updated the " + "relay configuration, perform relay reload on your nodes to pick up " + "the changes." + ) + } +} + +# Azure OpenAI, when the deployment behind a BYO model is gone. 6 events, the +# largest single signature. The marker is in ``error.code``, not the message. +_AZURE_DEPLOYMENT_BODY: dict[str, object] = { + "error": { + "type": "invalid_request_error", + "code": "DeploymentNotFound", + "message": ( + "The API deployment for this resource does not exist. If you created " + "the deployment within the last 5 minutes, please wait a moment and " + "try again." + ), + } +} + +# Bedrock, when the configured model version has been retired. 4 events. The +# marker is a top-level ``message`` with no ``error`` envelope at all. +_BEDROCK_RETIRED_BODY: dict[str, object] = { + "message": ( + "This model version has reached the end of its life. Please refer to the " + "AWS documentation for more details." + ) +} + +# The gateway logs "No llm configuration found" and forwards anyway, to a +# publisher that does not host the model -- gemini-3.7-flash to +# publishers/anthropic, claude-opus-5 to publishers/google. 1 event in this +# window, non-BYO. Deliberately *not* a signature: it names a publisher, not a +# model deployment, and the marker would be a Vertex-specific phrase we would be +# guessing at from a single event. +_VERTEX_PUBLISHER_BODY: dict[str, object] = { + "error": { + "code": 404, + "message": ( + "Publisher model `projects/uipath-llm-gateway-prd/locations/us/" + "publishers/anthropic/models/gemini-3.7-flash` was not found or your " + "project does not have access to it." + ), + "status": "NOT_FOUND", + } +} + +# The 6 that name nothing: an empty body (3), "The operation was canceled." (2), +# and a bare "Resource not found" (1). +_UNNAMEABLE_404_BODIES = [ + pytest.param(lambda: _api_error_text(404, ""), id="empty-body"), + pytest.param(lambda: _api_error_text(404, " "), id="whitespace-body"), + pytest.param( + lambda: _api_error_text(404, "The operation was canceled."), id="canceled" + ), + pytest.param( + lambda: _api_error( + 404, {"error": {"code": "404", "message": "Resource not found"}} + ), + id="bare-resource-not-found", + ), + pytest.param(lambda: _api_error_text(404, _EDGE_HTML), id="raw-html"), +] + +# Everything the mapper leaves UNKNOWN: the bodies that name nothing, plus the +# publisher mismatch, which names something we deliberately do not key on. +_UNCLASSIFIED_404_BODIES = [ + *_UNNAMEABLE_404_BODIES, + pytest.param( + lambda: _api_error(404, _VERTEX_PUBLISHER_BODY), id="vertex-publisher" + ), +] + +_ALL_404_BODIES = [ + pytest.param(lambda: _api_error(404, _RELAY_DOWN_BODY), id="relay-down"), + pytest.param( + lambda: _api_error(404, _AZURE_DEPLOYMENT_BODY), id="azure-deployment" + ), + pytest.param(lambda: _api_error(404, _BEDROCK_RETIRED_BODY), id="bedrock-retired"), + *_UNCLASSIFIED_404_BODIES, +] + + +def test_404_with_relay_marker_maps_to_deployment(): + info = _raise(_api_error(404, _RELAY_DOWN_BODY)).error_info + + assert info.status == 404 + assert info.category == UiPathErrorCategory.DEPLOYMENT + assert info.code.endswith( + AgentRuntimeErrorCode.LLM_BYO_CONNECTION_UNAVAILABLE.value + ) + assert "relay" in info.detail.lower() + + +def test_404_with_azure_deployment_marker_maps_to_deployment(): + """The marker is ``error.code``; the message never says "not found".""" + info = _raise(_api_error(404, _AZURE_DEPLOYMENT_BODY)).error_info + + assert info.category == UiPathErrorCategory.DEPLOYMENT + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_NOT_FOUND.value) + assert "deployment" in info.title.lower() + assert "bring your own" in info.detail.lower() + + +def test_404_with_retired_model_marker_maps_to_deployment(): + """A top-level ``message`` with no ``error`` envelope -- Bedrock's shape.""" + info = _raise(_api_error(404, _BEDROCK_RETIRED_BODY)).error_info + + assert info.category == UiPathErrorCategory.DEPLOYMENT + assert info.code.endswith(AgentRuntimeErrorCode.LLM_PROVIDER_NOT_FOUND.value) + assert "retired" in info.detail.lower() + + +def test_404_marker_is_matched_in_a_non_json_body(): + err = _api_error_text( + 404, "No active connection found for endpoint. Ensure the relay client" + ) + info = _raise(err).error_info + + assert info.category == UiPathErrorCategory.DEPLOYMENT + + +@pytest.mark.parametrize("err_factory", _UNCLASSIFIED_404_BODIES) +def test_404_without_a_recognized_marker_stays_unknown(err_factory): + """No marker we classify, no cause. UNKNOWN is honest, a guess is not.""" + info = _raise(err_factory()).error_info + + assert info.status == 404 + assert info.category == UiPathErrorCategory.UNKNOWN + assert info.code.endswith(AgentRuntimeErrorCode.HTTP_ERROR.value) + assert info.title == "LLM provider returned HTTP 404" + + +@pytest.mark.parametrize("err_factory", _ALL_404_BODIES) +def test_404_is_never_a_user_error(err_factory): + """Not one agent-attributable LLM 404 in prd was caused by the user. + + A missing BYO deployment, a retired model and a disconnected relay are all + things an administrator fixes; the bodies left UNKNOWN must not be pinned on + the user either. + """ + info = _raise(err_factory()).error_info + + assert info.category != UiPathErrorCategory.USER + + +@pytest.mark.parametrize("err_factory", _ALL_404_BODIES) +def test_404_does_not_quote_the_provider_body(err_factory): + """The body is read to classify, never to quote. + + A BYO passthrough relays a customer-controlled endpoint, so this position is + third-party content of unknown sensitivity whatever it happens to say. + """ + error = _raise(err_factory()) + + for rendered in (error.error_info.detail, str(error), repr(error)): + assert "uipath-llm-gateway-prd" not in rendered + assert "within the last 5 minutes" not in rendered + assert "AWS documentation" not in rendered + assert "perform relay reload on your nodes" not in rendered + assert "doctype" not in rendered.lower() + + +def test_404_prefers_the_gateway_detail_over_the_canned_text(): + """A ProblemDetails ``detail`` is first-party UiPath text and more specific.""" + err = _api_error(404, {"status": 404, "detail": "Model not enabled for tenant."}) + info = _raise(err).error_info + + assert "Model not enabled for tenant." in info.detail From dc48d7fc1b2f9fb1495c42b266921c41a8666d39 Mon Sep 17 00:00:00 2001 From: Mihalache Marius Date: Thu, 17 Sep 2026 17:47:30 +0300 Subject: [PATCH 2/4] refactor(agent): fold the detail precedence into _classify Review follow-ups on #1072: - _classify now reads the gateway's ProblemDetails detail itself and returns the final detail, so the gateway-wins-over-canned precedence no longer leaks into raise_for_provider_http_error. None still means "fall back to the HTTP reason phrase". - Trim the comment on _BAD_REQUEST_DETAIL to the part that carries motivation -- the provider body is not read out because it may carry customer PII -- and move it to the 400 branch it explains. - Drop the job key from the _http_400 fixture docstring. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FcQT9jLym8DrdpTamH38Yy --- src/uipath_langchain/agent/exceptions/llm.py | 50 ++++++++++---------- tests/agent/react/test_llm_node.py | 2 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/uipath_langchain/agent/exceptions/llm.py b/src/uipath_langchain/agent/exceptions/llm.py index 9f9327002..0c044f3ed 100644 --- a/src/uipath_langchain/agent/exceptions/llm.py +++ b/src/uipath_langchain/agent/exceptions/llm.py @@ -21,11 +21,6 @@ _LICENSE_ERROR_CODE = 10000 _LICENSE_TITLE = "license not available" -# A canned, provider-free replacement for the useless HTTP reason phrase. The -# relayed provider message is deliberately NOT read out of the body: -# it may carry customer PII, and it is already recorded on the LLM call span, -# which is tenant-scoped. It has to stand on its own -- USER is not in -# _SHOULD_WRAP_CATEGORIES, so nothing else is prepended to it. _BAD_REQUEST_DETAIL = ( "The model provider rejected the request as invalid. Review the agent's model " "settings (output-token limit, temperature, effort). The provider's own message " @@ -144,21 +139,24 @@ def _match_not_found_signature(body: object) -> _Verdict | None: def _classify( status_code: int, body: object ) -> tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str | None]: - """Map an LLM provider HTTP status onto (code, category, title, fallback_detail). + """Map an LLM provider HTTP status onto (code, category, title, detail). Only 400, 403 and 404 are classified beyond the 5xx/other split. 403 and 404 are the statuses whose meaning depends on the body; keeping the - code, category, title and fallback detail decided in one place stops them - drifting apart. Both name a cause only for a body that names its own -- + code, category, title and detail decided in one place stops them drifting + apart. Both name a cause only for a body that names its own -- ``_is_license_error`` for 403, ``_NOT_FOUND_SIGNATURES`` for 404 -- and leave the rest unnamed rather than guessing. - ``fallback_detail`` is the customer-facing text to use when the gateway - supplied no ProblemDetails ``detail`` of its own. ``None`` means "fall back - to the HTTP reason phrase" -- the useless two-word message, so only - statuses whose cause we cannot name are left with it. + The gateway's own ProblemDetails ``detail`` is first-party UiPath text and + more specific, so it wins over anything decided here. A ``detail`` of + ``None`` means "fall back to the HTTP reason phrase" -- the useless + two-word message, so only statuses whose cause we cannot name are left + with it. """ + gateway_detail = body.get("detail") if isinstance(body, dict) else None + if status_code == 403: if _is_license_error(body): title = body.get("title") if isinstance(body, dict) else None @@ -168,26 +166,30 @@ def _classify( title if isinstance(title, str) and title.strip() else "License not available", - None, + gateway_detail, ) return ( AgentRuntimeErrorCode.LLM_PROVIDER_FORBIDDEN, UiPathErrorCategory.DEPLOYMENT, "LLM provider returned HTTP 403", - None, + gateway_detail, ) if status_code == 400: + # The relayed provider message is deliberately not read out of the + # body: it may carry customer PII, and it is already recorded on the + # LLM call span, which is tenant-scoped. return ( AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST, UiPathErrorCategory.USER, "LLM provider rejected the request", - _BAD_REQUEST_DETAIL, + gateway_detail or _BAD_REQUEST_DETAIL, ) if status_code == 404: if (verdict := _match_not_found_signature(body)) is not None: - return verdict + code, category, title, signature_detail = verdict + return code, category, title, gateway_detail or signature_detail title = f"LLM provider returned HTTP {status_code}" if status_code >= 500: @@ -195,31 +197,31 @@ def _classify( AgentRuntimeErrorCode.HTTP_ERROR, UiPathErrorCategory.SYSTEM, title, - None, + gateway_detail, ) return ( AgentRuntimeErrorCode.HTTP_ERROR, UiPathErrorCategory.UNKNOWN, title, - None, + gateway_detail, ) def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: """Convert a normalized ``UiPathAPIError`` into a structured ``AgentRuntimeError``. - Reads the HTTP status code and the gateway's ``detail`` (from ``error.body``) - and re-raises as an ``AgentRuntimeError`` chained on the original. + Reads the HTTP status code and ``error.body``, and re-raises as an + ``AgentRuntimeError`` chained on the original. When ``_classify`` names no + detail, the error's own message -- the HTTP reason phrase -- is all that is + left. """ status_code = error.status_code - body = error.body - code, category, title, fallback_detail = _classify(status_code, body) - gateway_detail = body.get("detail") if isinstance(body, dict) else None + code, category, title, detail = _classify(status_code, error.body) raise AgentRuntimeError( code=code, title=title, - detail=gateway_detail or fallback_detail or error.message or str(error), + detail=detail or error.message or str(error), category=category, status=status_code, ) from error diff --git a/tests/agent/react/test_llm_node.py b/tests/agent/react/test_llm_node.py index 8d1d46976..68279d300 100644 --- a/tests/agent/react/test_llm_node.py +++ b/tests/agent/react/test_llm_node.py @@ -431,7 +431,7 @@ async def test_legacy_raw_provider_error_is_normalized_and_mapped(self): @staticmethod def _http_400() -> httpx.Response: - """The 400 from job 1fab7e97-...: max_tokens written by Agent Builder.""" + """A gateway 400 as seen in prod: max_tokens above the model's limit.""" request = httpx.Request("POST", "http://gateway/") return httpx.Response( 400, From 85704856901f580d8ceb3f47242fa760da92deba Mon Sep 17 00:00:00 2001 From: Mihalache Marius Date: Thu, 17 Sep 2026 17:57:30 +0300 Subject: [PATCH 3/4] test(agent): drop the remaining job key from the 400 fixture comment Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FcQT9jLym8DrdpTamH38Yy --- tests/agent/test_llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/agent/test_llm.py b/tests/agent/test_llm.py index 9d88c0d0f..5264e168a 100644 --- a/tests/agent/test_llm.py +++ b/tests/agent/test_llm.py @@ -244,8 +244,8 @@ def test_unclassified_4xx_remains_unknown(status_code: int): # 400: User, with a canned detail instead of the reason phrase # -------------------------------------------------------------------------- -# The body of the 400 that failed 192/192 runs on gpt-4.1-mini-e2e-custom -# (job 1fab7e97-...): max_tokens=65535 written by Agent Builder itself. +# The body of the 400 that failed 192/192 runs on gpt-4.1-mini-e2e-custom: +# max_tokens=65535 written by Agent Builder itself. _MAX_TOKENS_BODY: dict[str, object] = { "error": { "message": ( From a4dbaee86e2f03a09c14f5ad7ae8b2aee71e289c Mon Sep 17 00:00:00 2001 From: Mihalache Marius Date: Thu, 17 Sep 2026 18:24:14 +0300 Subject: [PATCH 4/4] refactor(agent): split _classify to cut its cognitive complexity Sonar flagged _classify at 18 against a limit of 15: applying the gateway-detail precedence inside every branch repeated a conditional per status. The precedence now applies once. _status_verdict decides code, category, title and the detail this mapping names, _forbidden_verdict holds the 403 body check that was the nested branch, and _classify overlays the gateway's ProblemDetails detail on the result. Complexity per function is now 5, 4 and 2. Behavior is unchanged -- gateway detail still wins over both the canned 400 text and the 404 signature details. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FcQT9jLym8DrdpTamH38Yy --- src/uipath_langchain/agent/exceptions/llm.py | 90 ++++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/uipath_langchain/agent/exceptions/llm.py b/src/uipath_langchain/agent/exceptions/llm.py index 0c044f3ed..623643d48 100644 --- a/src/uipath_langchain/agent/exceptions/llm.py +++ b/src/uipath_langchain/agent/exceptions/llm.py @@ -27,7 +27,7 @@ "is recorded on the LLM call span for this run." ) -_Verdict = tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str] +_Verdict = tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str | None] _NOT_FOUND_SIGNATURES: tuple[tuple[tuple[str, ...], _Verdict], ...] = ( ( @@ -136,10 +136,27 @@ def _match_not_found_signature(body: object) -> _Verdict | None: return None -def _classify( - status_code: int, body: object -) -> tuple[AgentRuntimeErrorCode, UiPathErrorCategory, str, str | None]: - """Map an LLM provider HTTP status onto (code, category, title, detail). +def _forbidden_verdict(body: object) -> _Verdict: + """The verdict for a 403, whose meaning is in the body rather than the status.""" + if not _is_license_error(body): + return ( + AgentRuntimeErrorCode.LLM_PROVIDER_FORBIDDEN, + UiPathErrorCategory.DEPLOYMENT, + "LLM provider returned HTTP 403", + None, + ) + + title = body.get("title") if isinstance(body, dict) else None + return ( + AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE, + UiPathErrorCategory.DEPLOYMENT, + title if isinstance(title, str) and title.strip() else "License not available", + None, + ) + + +def _status_verdict(status_code: int, body: object) -> _Verdict: + """The verdict this mapping decides for a status, before the gateway's detail. Only 400, 403 and 404 are classified beyond the 5xx/other split. @@ -148,32 +165,9 @@ def _classify( apart. Both name a cause only for a body that names its own -- ``_is_license_error`` for 403, ``_NOT_FOUND_SIGNATURES`` for 404 -- and leave the rest unnamed rather than guessing. - - The gateway's own ProblemDetails ``detail`` is first-party UiPath text and - more specific, so it wins over anything decided here. A ``detail`` of - ``None`` means "fall back to the HTTP reason phrase" -- the useless - two-word message, so only statuses whose cause we cannot name are left - with it. """ - gateway_detail = body.get("detail") if isinstance(body, dict) else None - if status_code == 403: - if _is_license_error(body): - title = body.get("title") if isinstance(body, dict) else None - return ( - AgentRuntimeErrorCode.LICENSE_NOT_AVAILABLE, - UiPathErrorCategory.DEPLOYMENT, - title - if isinstance(title, str) and title.strip() - else "License not available", - gateway_detail, - ) - return ( - AgentRuntimeErrorCode.LLM_PROVIDER_FORBIDDEN, - UiPathErrorCategory.DEPLOYMENT, - "LLM provider returned HTTP 403", - gateway_detail, - ) + return _forbidden_verdict(body) if status_code == 400: # The relayed provider message is deliberately not read out of the @@ -183,30 +177,36 @@ def _classify( AgentRuntimeErrorCode.LLM_PROVIDER_BAD_REQUEST, UiPathErrorCategory.USER, "LLM provider rejected the request", - gateway_detail or _BAD_REQUEST_DETAIL, + _BAD_REQUEST_DETAIL, ) - if status_code == 404: - if (verdict := _match_not_found_signature(body)) is not None: - code, category, title, signature_detail = verdict - return code, category, title, gateway_detail or signature_detail + if status_code == 404 and (verdict := _match_not_found_signature(body)) is not None: + return verdict - title = f"LLM provider returned HTTP {status_code}" - if status_code >= 500: - return ( - AgentRuntimeErrorCode.HTTP_ERROR, - UiPathErrorCategory.SYSTEM, - title, - gateway_detail, - ) return ( AgentRuntimeErrorCode.HTTP_ERROR, - UiPathErrorCategory.UNKNOWN, - title, - gateway_detail, + UiPathErrorCategory.SYSTEM + if status_code >= 500 + else UiPathErrorCategory.UNKNOWN, + f"LLM provider returned HTTP {status_code}", + None, ) +def _classify(status_code: int, body: object) -> _Verdict: + """Map an LLM provider HTTP status onto (code, category, title, detail). + + The gateway's own ProblemDetails ``detail`` is first-party UiPath text and + more specific, so it wins over the detail ``_status_verdict`` decided. A + ``detail`` of ``None`` means "fall back to the HTTP reason phrase" -- the + useless two-word message, so only statuses whose cause neither the gateway + nor this mapping can name are left with it. + """ + code, category, title, own_detail = _status_verdict(status_code, body) + gateway_detail = body.get("detail") if isinstance(body, dict) else None + return code, category, title, gateway_detail or own_detail + + def raise_for_provider_http_error(error: UiPathAPIError) -> NoReturn: """Convert a normalized ``UiPathAPIError`` into a structured ``AgentRuntimeError``.