Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions packages/sdk/server-ai/src/ldai/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
from ldai.providers import (
AgentGraphResult,
AgentGraphRunner,
AgentResult,
AgentRunner,
ManagedResult,
Runner,
Expand All @@ -49,7 +48,6 @@
'Evaluator',
'AgentRunner',
'AgentGraphRunner',
'AgentResult',
'AgentGraphResult',
'ManagedResult',
'Runner',
Expand Down
2 changes: 1 addition & 1 deletion packages/sdk/server-ai/src/ldai/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ def create_judge(
if not provider:
return None

return Judge(judge_config, provider) # type: ignore[arg-type]
return Judge(judge_config, provider)
except Exception as error:
return None

Expand Down
24 changes: 12 additions & 12 deletions packages/sdk/server-ai/src/ldai/judge/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
from ldai import log
from ldai.judge.evaluation_schema_builder import EvaluationSchemaBuilder
from ldai.models import AIJudgeConfig, LDMessage
from ldai.providers.model_runner import ModelRunner
from ldai.providers.types import JudgeResult, ModelResponse
from ldai.providers.runner import Runner
from ldai.providers.types import JudgeResult, RunnerResult


class Judge:
Expand All @@ -23,13 +23,13 @@ class Judge:
def __init__(
self,
ai_config: AIJudgeConfig,
model_runner: ModelRunner,
model_runner: Runner,
):
"""
Initialize the Judge.

:param ai_config: The judge AI configuration
:param model_runner: The model runner to use for evaluation
:param model_runner: The runner to use for evaluation
"""
self._ai_config = ai_config
self._model_runner = model_runner
Expand Down Expand Up @@ -76,10 +76,10 @@ async def evaluate(

response = await tracker.track_metrics_of_async(
lambda result: result.metrics,
lambda: self._model_runner.invoke_structured_model(messages, self._evaluation_response_structure),
lambda: self._model_runner.run(messages, output_type=self._evaluation_response_structure),
)

parsed = self._parse_evaluation_response(response.data)
parsed = self._parse_evaluation_response(response.parsed)

if parsed is None:
log.warning('Judge evaluation did not return the expected evaluation')
Expand All @@ -99,7 +99,7 @@ async def evaluate(
async def evaluate_messages(
self,
messages: list[LDMessage],
response: ModelResponse,
response: RunnerResult,
sampling_ratio: float = 1.0,
) -> JudgeResult:
"""
Expand All @@ -111,7 +111,7 @@ async def evaluate_messages(
:return: The result of the judge evaluation.
"""
input_text = '\r\n'.join([msg.content for msg in messages]) if messages else ''
output_text = response.message.content
output_text = response.content

return await self.evaluate(input_text, output_text, sampling_ratio)

Expand All @@ -123,11 +123,11 @@ def get_ai_config(self) -> AIJudgeConfig:
"""
return self._ai_config

def get_model_runner(self) -> ModelRunner:
def get_model_runner(self) -> Runner:
"""
Returns the model runner used by this judge.
Returns the runner used by this judge.

:return: The model runner
:return: The runner
"""
return self._model_runner

Expand Down Expand Up @@ -164,7 +164,7 @@ def _interpolate_message(self, content: str, variables: Dict[str, str]) -> str:
# Use chevron (Mustache) for templating, with no escaping
return chevron.render(content, variables)

def _parse_evaluation_response(self, data: Dict[str, Any]) -> Optional[Tuple[float, str]]:
def _parse_evaluation_response(self, data: Optional[Dict[str, Any]]) -> Optional[Tuple[float, str]]:
"""
Parses the structured evaluation response. Expects {"score": n, "reasoning": "..."}.

Expand Down
4 changes: 2 additions & 2 deletions packages/sdk/server-ai/src/ldai/managed_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from ldai import log
from ldai.models import AICompletionConfig, LDMessage
from ldai.providers.runner import Runner
from ldai.providers.types import JudgeResult, ManagedResult
from ldai.providers.types import JudgeResult, ManagedResult, RunnerResult
from ldai.tracker import LDAIConfigTracker


Expand Down Expand Up @@ -46,7 +46,7 @@ async def run(self, prompt: str) -> ManagedResult:
config_messages = self._ai_config.messages or []
all_messages = config_messages + self._messages

result = await tracker.track_metrics_of_async(
result: RunnerResult = await tracker.track_metrics_of_async(
lambda r: r.metrics,
lambda: self._model_runner.run(all_messages),
)
Expand Down
6 changes: 0 additions & 6 deletions packages/sdk/server-ai/src/ldai/providers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,24 @@
from ldai.providers.runner_factory import RunnerFactory
from ldai.providers.types import (
AgentGraphResult,
AgentResult,
JudgeResult,
LDAIMetrics,
ManagedResult,
ModelResponse,
RunnerResult,
StructuredResponse,
ToolRegistry,
)

__all__ = [
'AIProvider',
'AgentGraphResult',
'AgentGraphRunner',
'AgentResult',
'AgentRunner',
'JudgeResult',
'LDAIMetrics',
'ManagedResult',
'ModelResponse',
'ModelRunner',
'Runner',
'RunnerFactory',
'RunnerResult',
'StructuredResponse',
'ToolRegistry',
]
6 changes: 3 additions & 3 deletions packages/sdk/server-ai/src/ldai/providers/agent_runner.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Any, Protocol, runtime_checkable

from ldai.providers.types import AgentResult
from ldai.providers.types import RunnerResult


@runtime_checkable
Expand All @@ -18,11 +18,11 @@ class AgentRunner(Protocol):
the caller just passes input.
"""

async def run(self, input: Any) -> AgentResult:
async def run(self, input: Any) -> RunnerResult:
"""
Run the agent with the given input.

:param input: The input to the agent (string prompt or structured input)
:return: AgentResult containing the output, raw response, and metrics
:return: RunnerResult containing the agent's content, metrics, and optional raw/parsed fields
"""
...
50 changes: 2 additions & 48 deletions packages/sdk/server-ai/src/ldai/providers/ai_provider.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
from abc import ABC
from typing import Any, Dict, List, Optional
from typing import Any, Optional

from ldai import log
from ldai.models import LDMessage
from ldai.providers.types import ModelResponse, StructuredResponse, ToolRegistry
from ldai.providers.types import ToolRegistry


class AIProvider(ABC):
Expand All @@ -16,51 +15,6 @@ class AIProvider(ABC):
create_model(), create_agent(), and create_agent_graph().
"""

async def invoke_model(self, messages: List[LDMessage]) -> ModelResponse:
"""
Invoke the chat model with an array of messages.

Default implementation takes no action and returns a placeholder response.
Provider implementations should override this method.

:param messages: Array of LDMessage objects representing the conversation
:return: ModelResponse containing the model's response
"""
log.warning('invoke_model not implemented by this provider')

from ldai.models import LDMessage
from ldai.providers.types import LDAIMetrics

return ModelResponse(
message=LDMessage(role='assistant', content=''),
metrics=LDAIMetrics(success=False, usage=None),
)

async def invoke_structured_model(
self,
messages: List[LDMessage],
response_structure: Dict[str, Any],
) -> StructuredResponse:
"""
Invoke the chat model with structured output support.

Default implementation takes no action and returns a placeholder response.
Provider implementations should override this method.

:param messages: Array of LDMessage objects representing the conversation
:param response_structure: Dictionary of output configurations keyed by output name
:return: StructuredResponse containing the structured data
"""
log.warning('invoke_structured_model not implemented by this provider')

from ldai.providers.types import LDAIMetrics

return StructuredResponse(
data={},
raw_response='',
metrics=LDAIMetrics(success=False, usage=None),
)

def create_model(self, config: Any) -> Optional[Any]:
"""
Create a configured model executor for the given AI config.
Expand Down
22 changes: 4 additions & 18 deletions packages/sdk/server-ai/src/ldai/providers/model_runner.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import Any, Dict, List, Protocol, runtime_checkable
from typing import List, Protocol, runtime_checkable

from ldai.models import LDMessage
from ldai.providers.types import ModelResponse, StructuredResponse
from ldai.providers.types import RunnerResult


@runtime_checkable
Expand All @@ -14,25 +14,11 @@ class ModelRunner(Protocol):
and with what parameters — the caller just passes messages.
"""

async def invoke_model(self, messages: List[LDMessage]) -> ModelResponse:
async def invoke_model(self, messages: List[LDMessage]) -> RunnerResult:
"""
Invoke the model with an array of messages.

:param messages: Array of LDMessage objects representing the conversation
:return: ModelResponse containing the model's response and metrics
"""
...

async def invoke_structured_model(
self,
messages: List[LDMessage],
response_structure: Dict[str, Any],
) -> StructuredResponse:
"""
Invoke the model with structured output support.

:param messages: Array of LDMessage objects representing the conversation
:param response_structure: Dictionary defining the JSON schema for output structure
:return: StructuredResponse containing the structured data
:return: RunnerResult containing the model's response and metrics
"""
...
42 changes: 0 additions & 42 deletions packages/sdk/server-ai/src/ldai/providers/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional

from ldai.models import LDMessage
from ldai.tracker import LDAIMetricSummary, TokenUsage

# Type alias for a registry of tools available to an agent.
Expand Down Expand Up @@ -87,33 +86,6 @@ class ManagedResult:
"""Optional asyncio Task that resolves to the list of :class:`JudgeResult` instances when awaited."""


@dataclass
class ModelResponse:
"""
Response from a model invocation.

.. deprecated::
Use :class:`RunnerResult` (from a runner) and :class:`ManagedResult`
(from the managed layer) instead.
"""
message: LDMessage
metrics: LDAIMetrics
evaluations: Optional[asyncio.Task[List[JudgeResult]]] = None


@dataclass
class StructuredResponse:
"""
Structured response from AI models.

.. deprecated::
Structured output is now represented by :attr:`RunnerResult.parsed`.
"""
data: Dict[str, Any]
raw_response: str
metrics: LDAIMetrics


@dataclass
class JudgeResult:
"""Contains the result of a single judge evaluation."""
Expand Down Expand Up @@ -160,20 +132,6 @@ def to_dict(self) -> Dict[str, Any]:
return result


@dataclass
class AgentResult:
"""
Result from a single-agent run.

.. deprecated::
Use :class:`ManagedResult` (managed layer) or :class:`RunnerResult`
(runner layer) instead.
"""
output: str
raw: Any
metrics: LDAIMetrics


@dataclass
class AgentGraphResult:
"""Contains the result of an agent graph run."""
Expand Down
Loading
Loading