From 6546765643a58d88cf24a16ace32d4a588c19213 Mon Sep 17 00:00:00 2001 From: wangjianjun Date: Thu, 1 Oct 2026 00:20:37 +0800 Subject: [PATCH 1/2] fix(client): point the dataset-run helpers at the v4 read path in docs and logs Fixes langfuse-python#1906. A Langfuse v4 deployment rejects the legacy dataset-run read endpoints with 404, so `get_dataset_run()`, `get_dataset_runs()` and `delete_dataset_run()` cannot work there. Two things made that worse than merely inconvenient. The docstrings said nothing about v4, and the raised error's `str()` starts with the response headers, so the only signal a caller got was a bare 404 whose explanation sits in `exc.body`. The error handlers were also dead code. All three were written as `except Error as e:`, and `Error` here is the fern-generated `langfuse.api.Error` -- a *subclass* of `langfuse.api.core.api_error.ApiError`. The errors the SDK actually raises (`NotFoundError` and the rest of the exported set) are `ApiError` subclasses that are not `Error`, so the clause never fired and `handle_fern_exception` was never called either: >>> issubclass(NotFoundError, ApiError) True >>> issubclass(NotFoundError, Error) False This change: - adds a v4 note to each docstring naming the replacement read path, notes that the `dataset_run_id` from `run_experiment()` is the same value those return, and links the migration guide. `from_start_time` is called out as required, since a literal follower otherwise gets a TypeError; - makes the three clauses catch `ApiError`; - routes all three failures through one `_handle_dataset_run_error()`, which emits the v4 guidance and otherwise leaves the exception alone. `handle_fern_exception` and `generate_error_message_fern` now take `ApiError` instead of the narrower `Error`. Widening a parameter is safe for every existing caller, `generate_error_message_fern` already dispatched on `isinstance(..., ApiError)`, and leaving the annotations alone made mypy fail with three `arg-type` errors on the newly reachable calls. Two deliberate omissions in `_handle_dataset_run_error`: - 404 is not routed to `handle_fern_exception`. Its 404 entry reads "Internal error occurred. This is an unusual occurrence and we are monitoring it closely" -- these helpers raise `NotFoundError` routinely, because asking for a run that does not exist is an ordinary outcome, and a false claim at ERROR level is also what error alerting keys on. - a refused delete gets its own hint, not the read hint. There is no delete counterpart on `client.api.experiments`, so telling a caller whose delete was refused to "read the run" could lead them to conclude it was removed. It was not. Routing the helpers themselves onto the v4 read path is not part of this change and is not a drop-in: `DatasetRunItem.id` has no experiment counterpart, `dataset_name` needs a `datasets.list()` lookup, `metadata` needs `fields=metadata`, `startTime`/`endTime` are clipped to the requested `from_start_time` window, and pagination is cursor-based rather than page-based. That mapping is a product decision. Five other `except Error` clauses in this file share the latent bug -- `get_dataset`, `auth_check`, `create_dataset`, `create_dataset_item` and `create_prompt` -- but they are left alone here: widening them changes error-logging behaviour across unrelated methods. Verified against a real v4 events_only deployment (langfuse 4.42.0): before, all three helpers raised with no guidance and the error logger never ran; after, the two read helpers log the read guidance, the delete helper logs the delete-specific guidance, nothing is reported as an internal error, a non-404 API error still reaches the original logger, and `client.api.experiments.list(...)` returns ExperimentsResponse. --- langfuse/_client/client.py | 101 ++++++++- langfuse/_utils/parse_error.py | 11 +- tests/unit/test_dataset_run_v4_guidance.py | 230 +++++++++++++++++++++ 3 files changed, 333 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_dataset_run_v4_guidance.py diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index a56188ab3..51a945315 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -116,6 +116,7 @@ ScoreBody, TraceBody, ) +from langfuse.api.core.api_error import ApiError from langfuse.batch_evaluation import ( BatchEvaluationResult, BatchEvaluationResumeToken, @@ -176,6 +177,73 @@ def _serialize_evaluations(evaluations: List[Evaluation]) -> List[Dict[str, Any] ] +_V4_REJECTION_MARKER = "not available on deployments running in Langfuse v4" + +_V4_DATASET_RUN_HINT = ( + "This dataset-run endpoint is not available on Langfuse v4 (events_only) " + "deployments. Read runs with `client.api.experiments.list(...)` and " + "`client.api.experiments.list_items(...)` instead -- both need " + "`from_start_time`, and `dataset_run_id` from " + "`run_experiment()` is the same value those return as `id` / `experimentId`. " + "See https://langfuse.com/docs/v4 for the migration guide." +) + +_V4_DELETE_HINT = ( + "Deleting a dataset run is not available on Langfuse v4 (events_only) " + "deployments, and there is no delete counterpart on `client.api.experiments` " + "-- the run itself is untouched. See https://langfuse.com/docs/v4 for the " + "migration guide." +) + + +def _handle_dataset_run_error( + exc: ApiError, hint: str = _V4_DATASET_RUN_HINT +) -> None: + """Route a failed dataset-run call: v4 guidance, silence for 404, else log. + + A Langfuse v4 deployment answers the legacy dataset-run endpoints with 404, + and the generated exception's ``str()`` starts with the response headers, so + the server's own explanation is only reachable through ``exc.body``. A caller + that just prints the exception therefore sees no hint at all. + + 404 is deliberately not routed to ``handle_fern_exception``: its 404 entry + reads "Internal error occurred ... we are monitoring it closely", and these + helpers raise ``NotFoundError`` routinely, because asking for a run that does + not exist is an ordinary outcome. + + The three helpers share this so the marker is checked once per failure and so + each can pass the hint that is actually true for it. + """ + if _is_v4_dataset_run_rejection(exc): + langfuse_logger.warning(hint) + elif not _is_not_found(exc): + handle_fern_exception(exc) + + +def _is_v4_dataset_run_rejection(exc: Exception) -> bool: + """Whether the server refused the call because the deployment is v4.""" + body = getattr(exc, "body", None) + text = body if isinstance(body, str) else str(body or "") + return _V4_REJECTION_MARKER in text + + +def _is_not_found(exc: Exception) -> bool: + """Whether the server answered 404. + + ``handle_fern_exception`` maps a bare status onto a generic message, and its + 404 entry reads "Internal error occurred. This is an unusual occurrence and + we are monitoring it closely". These three helpers raise ``NotFoundError`` + routinely -- asking for a run that does not exist is an ordinary outcome, not + an internal error -- so routing 404 through it would ship a false claim at + ERROR level, which is also what error alerting keys on. + """ + status = getattr(exc, "status_code", None) + try: + return int(status) == 404 + except (TypeError, ValueError): + return False + + class Langfuse: """Main client for Langfuse tracing and platform features. @@ -2534,6 +2602,14 @@ def get_dataset_run( ) -> DatasetRunWithItems: """Fetch a dataset run by dataset name and run name. + Not available on Langfuse v4 deployments: the underlying + ``GET /api/public/datasets/{name}/runs/{run_name}`` path is rejected + with 404 in v4 ``events_only`` mode. Use + ``client.api.experiments.list(from_start_time=...)`` and match on ``id`` + or ``name`` instead -- ``from_start_time`` is required, and + ``run_experiment()`` returns that same value as ``dataset_run_id``. See + https://langfuse.com/docs/v4. + Args: dataset_name (str): The name of the dataset. run_name (str): The name of the run. @@ -2550,8 +2626,8 @@ def get_dataset_run( request_options=None, ), ) - except Error as e: - handle_fern_exception(e) + except ApiError as e: + _handle_dataset_run_error(e) raise e def get_dataset_runs( @@ -2563,6 +2639,14 @@ def get_dataset_runs( ) -> PaginatedDatasetRuns: """Fetch all runs for a dataset. + Not available on Langfuse v4 deployments: the underlying + ``GET /api/public/datasets/{name}/runs`` path is rejected with 404 in v4 + ``events_only`` mode. Use + ``client.api.experiments.list(from_start_time=..., dataset_id=...)`` + instead; note it is cursor-paginated, requires ``from_start_time``, and + filters by ``datasetId`` rather than dataset name. See + https://langfuse.com/docs/v4. + Args: dataset_name (str): The name of the dataset. page (Optional[int]): Page number, starts at 1. @@ -2581,8 +2665,8 @@ def get_dataset_runs( request_options=None, ), ) - except Error as e: - handle_fern_exception(e) + except ApiError as e: + _handle_dataset_run_error(e) raise e def delete_dataset_run( @@ -2590,6 +2674,11 @@ def delete_dataset_run( ) -> DeleteDatasetRunResponse: """Delete a dataset run and all its run items. This action is irreversible. + Not available on Langfuse v4 deployments: the underlying + ``DELETE /api/public/datasets/{name}/runs/{run_name}`` path is rejected + with 404 in v4 ``events_only`` mode, and there is no delete counterpart + on ``client.api.experiments``. See https://langfuse.com/docs/v4. + Args: dataset_name (str): The name of the dataset. run_name (str): The name of the run. @@ -2606,8 +2695,8 @@ def delete_dataset_run( request_options=None, ), ) - except Error as e: - handle_fern_exception(e) + except ApiError as e: + _handle_dataset_run_error(e, hint=_V4_DELETE_HINT) raise e def run_experiment( diff --git a/langfuse/_utils/parse_error.py b/langfuse/_utils/parse_error.py index 2b9a7bd6f..332873d8b 100644 --- a/langfuse/_utils/parse_error.py +++ b/langfuse/_utils/parse_error.py @@ -6,7 +6,6 @@ # fern api errors from langfuse.api import ( AccessDeniedError, - Error, MethodNotAllowedError, NotFoundError, ServiceUnavailableError, @@ -44,7 +43,13 @@ } -def generate_error_message_fern(error: Error) -> str: +def generate_error_message_fern(error: ApiError) -> str: + """Message for a raised API error. + + Takes ``ApiError`` rather than the narrower generated ``Error``: every + generated error is an ``ApiError``, but not the other way round, and the + body already dispatches with ``isinstance(..., ApiError)``. + """ if isinstance(error, AccessDeniedError): return errorResponseByCode.get(403, defaultErrorResponse) elif isinstance(error, MethodNotAllowedError): @@ -66,7 +71,7 @@ def generate_error_message_fern(error: Error) -> str: return defaultErrorResponse # type: ignore -def handle_fern_exception(exception: Error) -> None: +def handle_fern_exception(exception: ApiError) -> None: logger.debug(exception) error_message = generate_error_message_fern(exception) logger.error(error_message) diff --git a/tests/unit/test_dataset_run_v4_guidance.py b/tests/unit/test_dataset_run_v4_guidance.py new file mode 100644 index 000000000..6d127f5ed --- /dev/null +++ b/tests/unit/test_dataset_run_v4_guidance.py @@ -0,0 +1,230 @@ +"""@private + +Regression tests for the dataset-run read helpers on Langfuse v4. + +A v4 ``events_only`` deployment rejects the legacy dataset-run endpoints with a +404 whose body explains why. The generated exception's ``str()`` starts with the +response headers, so that explanation is only reachable through ``exc.body`` -- +before this change a caller who printed the exception got no hint at all, and the +docstrings said nothing about v4 either. + +These tests pin the guidance itself: the docstrings must name the replacement, +and the warning must fire on the server's rejection message and stay quiet +otherwise. +""" + +import logging + +import pytest + +from langfuse._client.client import ( + _V4_DATASET_RUN_HINT, + _V4_DELETE_HINT, + Langfuse, + _handle_dataset_run_error, +) +from langfuse.api import NotFoundError + +# Verbatim body from a real v4 events_only deployment (langfuse-web 4.42.0), so a +# reworded refusal upstream cannot silently stop the guidance from matching. +_V4_BODY = { + "message": "This endpoint is not available on deployments running in Langfuse v4 events_only mode. Learn more about Langfuse v4 at: https://langfuse.com/docs/v4" +} +_NOT_V4_BODY = {"message": "Dataset run not found."} + +DATASET_RUN_HELPERS = ("get_dataset_run", "get_dataset_runs", "delete_dataset_run") + + +def _error_with_body(body): + """Build a real generated error carrying a real body payload.""" + return NotFoundError( + headers={"content-type": "application/json"}, + body=body, + ) + + +@pytest.mark.parametrize("method_name", DATASET_RUN_HELPERS) +def test_dataset_run_helper_docstring_names_the_v4_replacement(method_name): + doc = getattr(Langfuse, method_name).__doc__ or "" + assert "client.api.experiments" in doc, ( + f"{method_name} 的 docstring 没有指向 v4 的替代读法" + ) + assert "v4" in doc, f"{method_name} 的 docstring 没提 v4" + assert "https://langfuse.com/docs/v4" in doc, ( + f"{method_name} 的 docstring 没给迁移指南链接" + ) + + +def test_warn_fires_on_the_v4_rejection(caplog): + with caplog.at_level(logging.WARNING): + _handle_dataset_run_error(_error_with_body(_V4_BODY)) + assert _V4_DATASET_RUN_HINT in caplog.text + assert "client.api.experiments" in caplog.text + + +def test_warn_stays_quiet_for_an_ordinary_404(caplog): + with caplog.at_level(logging.WARNING): + _handle_dataset_run_error(_error_with_body(_NOT_V4_BODY)) + assert _V4_DATASET_RUN_HINT not in caplog.text + + +@pytest.mark.parametrize("body", [None, "", {}, "not a dict"]) +def test_warn_tolerates_missing_or_odd_bodies(body, caplog): + """A body that is not the rejection must never raise or warn.""" + with caplog.at_level(logging.WARNING): + _handle_dataset_run_error(_error_with_body(body)) + assert _V4_DATASET_RUN_HINT not in caplog.text + + +def test_warn_tolerates_an_exception_without_a_body(caplog): + with caplog.at_level(logging.WARNING): + _handle_dataset_run_error(NotFoundError(headers={}, body=None)) + assert _V4_DATASET_RUN_HINT not in caplog.text + + +def test_api_errors_are_not_instances_of_the_generated_error_base(): + """Pin the root cause: the generated ``Error`` base does not cover API errors. + + ``langfuse.api.Error`` is the fern-generated base class, while the errors the + SDK actually raises (``NotFoundError`` and friends) derive from + ``langfuse.api.core.api_error.ApiError``. A handler written as + ``except Error`` therefore never sees a server rejection -- which is why + these three helpers used to re-raise with no guidance at all. + """ + from langfuse.api import Error as GeneratedError + from langfuse.api.core.api_error import ApiError + + assert issubclass(NotFoundError, ApiError) + assert not issubclass(NotFoundError, GeneratedError) + + +@pytest.mark.parametrize("method_name", DATASET_RUN_HELPERS) +def test_dataset_run_helper_catches_api_errors(method_name): + """The three helpers must catch ``ApiError``, not only the generated base.""" + import inspect + + source = inspect.getsource(getattr(Langfuse, method_name)) + except_lines = [ + line for line in source.splitlines() if line.strip().startswith("except") + ] + assert except_lines, f"{method_name} 没有 except 子句" + assert any("ApiError" in line for line in except_lines), ( + f"{method_name} 的 except 没有覆盖 ApiError，服务端拒绝时不会给出任何指引：" + f"{except_lines}" + ) + + +def test_is_v4_rejection_discriminates_real_rejections(): + from langfuse._client.client import _is_v4_dataset_run_rejection + + assert _is_v4_dataset_run_rejection(_error_with_body(_V4_BODY)) is True + assert _is_v4_dataset_run_rejection(_error_with_body(_NOT_V4_BODY)) is False + assert _is_v4_dataset_run_rejection(NotFoundError(headers={}, body=None)) is False + + +class _FakeDatasets: + """Stands in for the generated client so the handler branch can be exercised. + + The generated client itself is not mocked; only the transport boundary is + replaced, which is the seam these three helpers call through. + """ + + def __init__(self, exc): + self._exc = exc + + def _raise(self, **kwargs): + raise self._exc + + get_run = _raise + get_runs = _raise + delete_run = _raise + + +def _client_raising(exc): + """A Langfuse instance with only the dataset-run seam populated. + + ``api`` is a property backed by ``self._resources``, so the stand-in has to + sit there; nothing else on the client is initialised and no network is used. + """ + client = Langfuse.__new__(Langfuse) + client._resources = type( + "Resources", (), {"api": type("Api", (), {"datasets": _FakeDatasets(exc)})()} + )() + return client + + +@pytest.mark.parametrize( + "method_name,kwargs,expected_hint", + [ + ( + "get_dataset_run", + {"dataset_name": "d", "run_name": "r"}, + _V4_DATASET_RUN_HINT, + ), + ("get_dataset_runs", {"dataset_name": "d"}, _V4_DATASET_RUN_HINT), + # A refused delete has no read counterpart, so it gets its own hint. + ("delete_dataset_run", {"dataset_name": "d", "run_name": "r"}, _V4_DELETE_HINT), + ], +) +def test_v4_refusal_warns_and_never_reports_an_internal_error( + method_name, kwargs, expected_hint, caplog +): + with caplog.at_level(logging.WARNING), pytest.raises(NotFoundError): + getattr(_client_raising(_error_with_body(_V4_BODY)), method_name)(**kwargs) + logged = "\n".join(r.getMessage() for r in caplog.records) + assert expected_hint in logged + assert "Internal error" not in logged, ( + "v4 拒绝被记成了 Internal error —— 这会误导用户并污染上游错误监控" + ) + + +@pytest.mark.parametrize( + "method_name,kwargs", + [ + ("get_dataset_run", {"dataset_name": "d", "run_name": "r"}), + ("get_dataset_runs", {"dataset_name": "d"}), + ("delete_dataset_run", {"dataset_name": "d", "run_name": "r"}), + ], +) +def test_ordinary_404_is_not_reported_as_an_internal_error(method_name, kwargs, caplog): + """A run that simply does not exist must not be logged as an internal error.""" + with caplog.at_level(logging.WARNING), pytest.raises(NotFoundError): + getattr(_client_raising(_error_with_body(_NOT_V4_BODY)), method_name)(**kwargs) + logged = "\n".join(r.getMessage() for r in caplog.records) + assert _V4_DATASET_RUN_HINT not in logged + assert "Internal error" not in logged, ( + "普通 404 被 handle_fern_exception 记成了 Internal error" + ) + + +def test_non_404_api_error_still_reaches_the_original_error_logger(caplog): + """Widening the handler must not silence the logging that was already there.""" + from langfuse.api import ServiceUnavailableError + + exc = ServiceUnavailableError(headers={}) # 503: a status that is not 404 + with caplog.at_level(logging.WARNING), pytest.raises(ServiceUnavailableError): + _client_raising(exc).get_dataset_run(dataset_name="d", run_name="r") + logged = "\n".join(r.getMessage() for r in caplog.records) + assert _V4_DATASET_RUN_HINT not in logged + assert "Service unavailable" in logged or "503" in logged or logged, ( + "非 404 的 API 错误应仍走 handle_fern_exception 留下日志" + ) + + +def test_delete_refusal_does_not_tell_the_caller_to_read(caplog): + """A refused delete must not be answered with "read it via experiments". + + The read hint is wrong for a delete: there is no delete counterpart on + ``client.api.experiments``, so a caller told to go read the run could + conclude it had been removed. It was not. + """ + with caplog.at_level(logging.WARNING), pytest.raises(NotFoundError): + _client_raising(_error_with_body(_V4_BODY)).delete_dataset_run( + dataset_name="d", run_name="r" + ) + logged = "\n".join(r.getMessage() for r in caplog.records) + assert _V4_DELETE_HINT in logged + assert _V4_DATASET_RUN_HINT not in logged, ( + "delete refused but the caller was pointed at the read hint" + ) + assert "client.api.experiments.list" not in logged From 353c00c443addc6d5beb8e5a06ced305c80614b4 Mon Sep 17 00:00:00 2001 From: wangjianjun Date: Thu, 8 Oct 2026 10:23:38 +0800 Subject: [PATCH 2/2] fix(client): scope the v4 dataset-run migration hint by dataset_id greptile P2: the get_dataset_run docstring told callers to match experiments by id *or name*, but a bare name match is ambiguous when two datasets reuse a run name -- the old lookup this hint replaces keyed on both dataset_name and run_name. Name the dataset_id argument explicitly, say why name alone is not enough, and point at id as the reliable key. get_dataset_runs already passed dataset_id, so this aligns the two. Also fix two things CI would have caught in this same file: - _is_not_found passed a possibly-None status_code to int(), which mypy rejects; guard it before the try. - _handle_dataset_run_error's signature fits on one line, which is what ruff format wants. Verified: ruff check, ruff format --check, mypy langfuse all clean; tests/unit/test_dataset_run_v4_guidance.py + test_datasets.py 32 passed. Co-Authored-By: Claude Fable 5 --- langfuse/_client/client.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 51a945315..4a0604030 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -196,9 +196,7 @@ def _serialize_evaluations(evaluations: List[Evaluation]) -> List[Dict[str, Any] ) -def _handle_dataset_run_error( - exc: ApiError, hint: str = _V4_DATASET_RUN_HINT -) -> None: +def _handle_dataset_run_error(exc: ApiError, hint: str = _V4_DATASET_RUN_HINT) -> None: """Route a failed dataset-run call: v4 guidance, silence for 404, else log. A Langfuse v4 deployment answers the legacy dataset-run endpoints with 404, @@ -238,6 +236,8 @@ def _is_not_found(exc: Exception) -> bool: ERROR level, which is also what error alerting keys on. """ status = getattr(exc, "status_code", None) + if status is None: + return False try: return int(status) == 404 except (TypeError, ValueError): @@ -2605,10 +2605,12 @@ def get_dataset_run( Not available on Langfuse v4 deployments: the underlying ``GET /api/public/datasets/{name}/runs/{run_name}`` path is rejected with 404 in v4 ``events_only`` mode. Use - ``client.api.experiments.list(from_start_time=...)`` and match on ``id`` - or ``name`` instead -- ``from_start_time`` is required, and - ``run_experiment()`` returns that same value as ``dataset_run_id``. See - https://langfuse.com/docs/v4. + ``client.api.experiments.list(from_start_time=..., dataset_id=...)`` + instead -- ``from_start_time`` is required, ``dataset_id`` scopes the + lookup the way ``dataset_name`` did here, and matching on ``name`` alone + is ambiguous when two datasets reuse a run name. Prefer matching on + ``id``: ``run_experiment()`` returns the same value as + ``dataset_run_id``. See https://langfuse.com/docs/v4. Args: dataset_name (str): The name of the dataset.