feat(api): update API spec from langfuse/langfuse 1ea859c

langfuse/api/__init__.py

@@ -253,6 +253,7 @@
         Evaluator,
         EvaluatorBase,
         EvaluatorChatMessage,
+        EvaluatorChatMessageRole,
         EvaluatorChatPrompt,
         EvaluatorChatPromptInput,
         EvaluatorModelConfig,
@@ -612,6 +613,7 @@
     "EvaluatorAssignment": ".evaluation_rules",
     "EvaluatorBase": ".evaluators",
     "EvaluatorChatMessage": ".evaluators",
+    "EvaluatorChatMessageRole": ".evaluators",
     "EvaluatorChatPrompt": ".evaluators",
     "EvaluatorChatPromptInput": ".evaluators",
     "EvaluatorModelConfig": ".evaluators",
@@ -1066,6 +1068,7 @@ def __dir__():
     "EvaluatorAssignment",
     "EvaluatorBase",
     "EvaluatorChatMessage",
+    "EvaluatorChatMessageRole",
     "EvaluatorChatPrompt",
     "EvaluatorChatPromptInput",
     "EvaluatorModelConfig",

langfuse/api/evaluators/__init__.py

@@ -21,6 +21,7 @@
         Evaluator,
         EvaluatorBase,
         EvaluatorChatMessage,
+        EvaluatorChatMessageRole,
         EvaluatorChatPrompt,
         EvaluatorChatPromptInput,
         EvaluatorModelConfig,
@@ -55,6 +56,7 @@
     "Evaluator": ".types",
     "EvaluatorBase": ".types",
     "EvaluatorChatMessage": ".types",
+    "EvaluatorChatMessageRole": ".types",
     "EvaluatorChatPrompt": ".types",
     "EvaluatorChatPromptInput": ".types",
     "EvaluatorModelConfig": ".types",
@@ -118,6 +120,7 @@ def __dir__():
     "Evaluator",
     "EvaluatorBase",
     "EvaluatorChatMessage",
+    "EvaluatorChatMessageRole",
     "EvaluatorChatPrompt",
     "EvaluatorChatPromptInput",
     "EvaluatorModelConfig",

langfuse/api/evaluators/types/__init__.py

@@ -22,6 +22,7 @@
     from .evaluator import Evaluator, Evaluator_Code, Evaluator_LlmAsJudge
     from .evaluator_base import EvaluatorBase
     from .evaluator_chat_message import EvaluatorChatMessage
+    from .evaluator_chat_message_role import EvaluatorChatMessageRole
     from .evaluator_chat_prompt import EvaluatorChatPrompt
     from .evaluator_chat_prompt_input import EvaluatorChatPromptInput
     from .evaluator_model_config import EvaluatorModelConfig
@@ -55,6 +56,7 @@
     "Evaluator": ".evaluator",
     "EvaluatorBase": ".evaluator_base",
     "EvaluatorChatMessage": ".evaluator_chat_message",
+    "EvaluatorChatMessageRole": ".evaluator_chat_message_role",
     "EvaluatorChatPrompt": ".evaluator_chat_prompt",
     "EvaluatorChatPromptInput": ".evaluator_chat_prompt_input",
     "EvaluatorModelConfig": ".evaluator_model_config",
@@ -118,6 +120,7 @@ def __dir__():
     "Evaluator",
     "EvaluatorBase",
     "EvaluatorChatMessage",
+    "EvaluatorChatMessageRole",
     "EvaluatorChatPrompt",
     "EvaluatorChatPromptInput",
     "EvaluatorModelConfig",

langfuse/api/evaluators/types/create_llm_as_judge_evaluator_request.py

@@ -34,7 +34,7 @@ class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel):
 
     prompt: EvaluatorChatPromptInput = pydantic.Field()
     """
-    User prompt string shortcut or a list containing exactly one user chat message.
+    User prompt string shortcut or an ordered list of chat messages.
     """
 
     model_config_: typing_extensions.Annotated[

langfuse/api/evaluators/types/evaluator_chat_message.py

@@ -4,14 +4,15 @@
 
 import pydantic
 from ...core.pydantic_utilities import UniversalBaseModel
+from .evaluator_chat_message_role import EvaluatorChatMessageRole
 
 
 class EvaluatorChatMessage(UniversalBaseModel):
     """
-    One user chat message in an evaluator prompt.
+    One chat message in an evaluator prompt.
     """
 
-    role: typing.Literal["user"] = "user"
+    role: EvaluatorChatMessageRole
     content: str = pydantic.Field()
     """
     Message content. Evaluator variables use `{{variable}}` syntax.

langfuse/api/evaluators/types/evaluator_chat_message_role.py

@@ -0,0 +1,30 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ...core import enum
+
+T_Result = typing.TypeVar("T_Result")
+
+
+class EvaluatorChatMessageRole(enum.StrEnum):
+    """
+    Role of an evaluator prompt message.
+    """
+
+    SYSTEM = "system"
+    USER = "user"
+    ASSISTANT = "assistant"
+
+    def visit(
+        self,
+        system: typing.Callable[[], T_Result],
+        user: typing.Callable[[], T_Result],
+        assistant: typing.Callable[[], T_Result],
+    ) -> T_Result:
+        if self is EvaluatorChatMessageRole.SYSTEM:
+            return system()
+        if self is EvaluatorChatMessageRole.USER:
+            return user()
+        if self is EvaluatorChatMessageRole.ASSISTANT:
+            return assistant()

langfuse/api/evaluators/types/llm_as_judge_evaluator.py

@@ -24,7 +24,7 @@ class LlmAsJudgeEvaluator(EvaluatorBase):
 
     prompt: EvaluatorChatPrompt = pydantic.Field()
     """
-    The single user chat message used by the latest evaluator version.
+    Ordered chat messages used by the latest evaluator version.
     """
 
     variables: typing.List[str] = pydantic.Field()

langfuse/api/evaluators/types/llm_as_judge_evaluator_version.py

@@ -24,7 +24,7 @@ class LlmAsJudgeEvaluatorVersion(EvaluatorVersionBase):
 
     prompt: EvaluatorChatPrompt = pydantic.Field()
     """
-    The single user chat message used during evaluation.
+    Ordered chat messages used during evaluation.
     """
 
     variables: typing.List[str] = pydantic.Field()

langfuse/api/evaluators/types/update_llm_as_judge_evaluator_request.py

@@ -34,7 +34,7 @@ class UpdateLlmAsJudgeEvaluatorRequest(UniversalBaseModel):
 
     prompt: EvaluatorChatPromptInput = pydantic.Field()
     """
-    Complete replacement user prompt string or a list containing exactly one user chat message.
+    Complete replacement user prompt string or an ordered list of chat messages.
     """
 
     model_config_: typing_extensions.Annotated[