Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 18 additions & 22 deletions langfuse/_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@
AsyncLangfuseAPI,
CreateChatPromptRequest,
CreateChatPromptType,
CreateScoreRequest,
CreateTextPromptRequest,
Dataset,
DatasetItem,
Expand Down Expand Up @@ -1946,29 +1945,26 @@ def create_score(
timestamp = timestamp.replace(tzinfo=timezone.utc)

try:
request = CreateScoreRequest(
id=score_id,
session_id=session_id,
dataset_run_id=dataset_run_id,
trace_id=trace_id,
observation_id=observation_id,
name=name,
value=value,
data_type=data_type, # type: ignore
comment=comment,
config_id=config_id,
environment=environment or self._environment,
timestamp=timestamp or _get_timestamp(),
)
# Fern's dict() keeps fields explicitly set to None, so drop them here.
# The model turned bools and ints into floats (True -> 1.0); keep that.
if isinstance(value, (bool, int)):
value = float(value)

score = {
key: value
for key, value in request.dict(by_alias=True).items()
if value is not None
"id": score_id,
"traceId": trace_id,
"sessionId": session_id,
"observationId": observation_id,
"datasetRunId": dataset_run_id,
"name": name,
"value": value,
"comment": comment,
"environment": environment or self._environment,
"dataType": data_type,
"configId": config_id,
"timestamp": timestamp or _get_timestamp(),
"metadata": metadata,
}
# The API accepts any JSON value as metadata, the generated model only dicts.
if metadata is not None:
score["metadata"] = metadata
score = {key: field for key, field in score.items() if field is not None}

if self._resources is not None:
# Force the score to be in sample if it was for a legacy trace ID, i.e. non-32 hexchar
Expand Down
14 changes: 7 additions & 7 deletions tests/unit/test_resource_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,9 +409,7 @@ def test_at_fork_reinit_new_httpx_client_uses_configured_timeout_and_headers(
client.shutdown()


def test_create_score_with_invalid_input_logs_error_without_enqueueing(
monkeypatch, caplog
):
def test_create_score_logs_error_instead_of_raising(monkeypatch, caplog):
monkeypatch.setenv("LANGFUSE_MEDIA_UPLOAD_ENABLED", "false")

with LangfuseResourceManager._lock:
Expand All @@ -424,13 +422,15 @@ def test_create_score_with_invalid_input_logs_error_without_enqueueing(
)
rm = client._resources
assert rm is not None
enqueued = []
monkeypatch.setattr(rm, "add_score_task", lambda event, **_: enqueued.append(event))

def fail(event, **_):
raise RuntimeError("enqueue failed")

monkeypatch.setattr(rm, "add_score_task", fail)

with caplog.at_level(logging.ERROR, logger="langfuse"):
client.create_score(name="invalid", value=object(), trace_id="a" * 32)
client.create_score(name="invalid", value=1.0, trace_id="a" * 32)

assert enqueued == []
assert rm._score_queue.empty()
error_records = [r for r in caplog.records if r.levelno == logging.ERROR]
assert len(error_records) == 1
Expand Down
71 changes: 70 additions & 1 deletion tests/unit/test_score_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
from langfuse._client.resource_manager import LangfuseResourceManager
from langfuse._task_manager import score_consumer as score_consumer_module
from langfuse._task_manager.score_consumer import ScoreConsumer
from langfuse.api import LangfuseAPI
from langfuse._utils.serializer import EventSerializer
from langfuse.api import CreateScoreRequest, LangfuseAPI

ACCEPTED = (202, {"message": "Accepted"})

Expand Down Expand Up @@ -288,6 +289,8 @@ def client_factory():
def create(recorder: Recorder, **kwargs: Any) -> Langfuse:
with LangfuseResourceManager._lock:
LangfuseResourceManager._instances.clear()
# The global provider may carry a sampler set by another test.
kwargs.setdefault("tracer_provider", TracerProvider())
client = Langfuse(
public_key="pk-score-consumer",
secret_key="sk-score-consumer",
Expand Down Expand Up @@ -348,6 +351,72 @@ def test_create_score_posts_to_scores_endpoint(client_factory):
assert sent[1]["timestamp"].endswith("Z")


def test_create_score_payload_matches_request_model_serialization(client_factory):
recorder = Recorder(_responder(ACCEPTED))
client = client_factory(recorder, environment="staging")
timestamp = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
cases: List[Dict[str, Any]] = [
dict(
name="numeric",
value=0.9,
trace_id="a" * 32,
observation_id="b" * 16,
data_type="NUMERIC",
comment="fine",
),
dict(name="int-value", value=1, trace_id="a" * 32),
dict(name="boolean", value=True, trace_id="a" * 32, data_type="BOOLEAN"),
dict(
name="categorical",
value="good",
session_id="session-1",
data_type="CATEGORICAL",
config_id="config-1",
),
dict(
name="text",
value="free text",
dataset_run_id="run-1",
data_type="TEXT",
environment="override",
),
dict(
name="correction",
value="corrected output",
trace_id="a" * 32,
observation_id="b" * 16,
data_type="CORRECTION",
),
dict(
name="dict-metadata",
value=0.5,
trace_id="a" * 32,
metadata={"evaluator": "judge", "nested": {"steps": [1, 2]}},
),
dict(name="list-metadata", value=0.5, trace_id="a" * 32, metadata=["a", 1]),
]

for i, case in enumerate(cases):
client.create_score(score_id=f"score-{i}", timestamp=timestamp, **case)
client.flush()

expected = []
for i, case in enumerate(cases):
fields = dict(case)
metadata = fields.pop("metadata", None)
fields.setdefault("environment", "staging")
request = CreateScoreRequest(id=f"score-{i}", timestamp=timestamp, **fields)
payload = {
k: v for k, v in request.dict(by_alias=True).items() if v is not None
}
if metadata is not None:
payload["metadata"] = metadata
expected.append(json.dumps(payload, cls=EventSerializer))
# Compare JSON text, because 1 == 1.0 == True as Python values.
sent = [json.dumps(score) for body in recorder.bodies for score in body]
assert sent == expected


@pytest.fixture
def non_utc_local_timezone(monkeypatch):
# Makes treating naive values as local time observable.
Expand Down
Loading