Skip to content

Commit a2147d1

Browse files
feat(gooddata-eval): send the document format on agentic document runs
gen-ai picks the document content format per turn from `options.documentFormat` (`aac-v1` or `aac-v2`) and otherwise from the `enablePublisherV2` flag, which is off for the evaluation org, so the `agent_document_skill` items always ran on the v1 Report names. `ChatClient` now takes `document_format`, validated by `normalize_document_format` when the client is built, and sends it beside `reasoningEffort` in one `options` object that is omitted when both are unset. `run_agentic_document_skill` and `evaluate_agentic_document_skill` take the same argument and validate it before any request. Every turn carries it, since gen-ai keeps a draft in the format of the turn that wrote it. `evaluate_agentic_document_skill` also records it as `document_format` in the dataset-run metadata. It is not part of the run name, so it separates runs in Langfuse only when their names already differ. jira: LX-3326 risk: low
1 parent 243ec3e commit a2147d1

5 files changed

Lines changed: 185 additions & 10 deletions

File tree

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from gooddata_eval.core.agentic.dashboard_skill import _extract_tool_result, _skill_activated
3333
from gooddata_eval.core.chat.render import render_answer_text
3434
from gooddata_eval.core.chat.sse_client import ChatClient
35-
from gooddata_eval.core.config import ReasoningEffort
35+
from gooddata_eval.core.config import DocumentFormat, ReasoningEffort, normalize_document_format
3636
from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
3737
from gooddata_eval.core.langfuse.item_scope import item_scope
3838
from gooddata_eval.core.models import (
@@ -801,18 +801,24 @@ def run_agentic_document_skill(
801801
agent_id: str | None = None,
802802
judge: LLMJudge | None = None,
803803
user_context: dict | None = None,
804+
document_format: DocumentFormat | None = None,
804805
) -> AgenticDocumentSummary:
805806
"""Run the document-skill agentic evaluation K times and return a summary.
806807
807808
The narrative judge is built only for a fixture that states a ``narrative``, so an item
808809
without one needs neither the llm-judge extra nor ``OPENAI_API_KEY``.
809810
811+
``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves
812+
the format to the server's ``enablePublisherV2`` feature flag.
813+
810814
Raises:
811815
ValueError: the fixture is unusable — see ``_validate_expectation`` and
812-
``_validate_user_context``.
816+
``_validate_user_context``, or ``document_format`` is not a format gen-ai
817+
accepts. Raised before any chat request.
813818
"""
814819
_validate_expectation(expected_output)
815820
_validate_user_context(user_context)
821+
document_format = normalize_document_format(document_format)
816822
if judge is None and _has_narrative(expected_output):
817823
judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS)
818824
run_results: list[DocumentRunResult] = []
@@ -823,6 +829,7 @@ def run_agentic_document_skill(
823829
reasoning_effort=reasoning_effort,
824830
agent_id=agent_id,
825831
user_context=user_context,
832+
document_format=document_format,
826833
)
827834

828835
try:
@@ -882,20 +889,32 @@ def evaluate_agentic_document_skill(
882889
gate: EvalGate = DEFAULT_GATE,
883890
judge: LLMJudge | None = None,
884891
user_context: dict | None = None,
892+
document_format: DocumentFormat | None = None,
885893
) -> AgenticEvalOutcome:
886894
"""Run document-skill evaluation, log to Langfuse, and raise on failure.
887895
888896
Returns the best run's outcome on success; on failure the same values are attached to the
889897
raised ``DocumentSkillAssertionError`` so callers can retrieve them either way.
890898
899+
``document_format`` (``aac-v1``/``aac-v2``) asks gen-ai for that document content format
900+
on every turn and is recorded as ``document_format`` in the dataset-run metadata; None
901+
leaves it to the server's ``enablePublisherV2`` feature flag.
902+
891903
Raises:
892904
DocumentSkillAssertionError: the gate did not pass.
893-
ValueError: the fixture is unusable — see ``_validate_expectation`` and
894-
``_validate_user_context``. Raised before any
895-
request, so it means a fixture to fix rather than a result to read.
905+
ValueError: the fixture is unusable, see ``_validate_expectation`` and
906+
``_validate_user_context``. Raised before any chat request, so it means a
907+
fixture to fix rather than a result to read. Also raised, equally before any
908+
request, when the caller passes a ``document_format`` gen-ai does not accept.
896909
JudgeResponseError: the fixture states a narrative and the judge returned no readable
897910
verdict for any run -- an item without a result, not K failures.
898911
"""
912+
document_format = normalize_document_format(document_format)
913+
if document_format is not None:
914+
# Metadata only, not part of the run name: it tells runs apart only when their names
915+
# already differ, so two runs that share a run_timestamp, dataset and model but not the
916+
# format still collide on one dataset-run name.
917+
run_metadata_extra = {**(run_metadata_extra or {}), "document_format": document_format}
899918
identity = RunIdentity(
900919
host,
901920
token,
@@ -921,6 +940,7 @@ def evaluate_agentic_document_skill(
921940
agent_id=agent_id,
922941
judge=judge,
923942
user_context=user_context,
943+
document_format=document_format,
924944
)
925945

926946
if langfuse is not None and dataset_item_id:

‎packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@
2626
import httpx
2727

2828
from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient
29-
from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
29+
from gooddata_eval.core.config import (
30+
DocumentFormat,
31+
ReasoningEffort,
32+
normalize_document_format,
33+
normalize_reasoning_effort,
34+
)
3035
from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
3136
from gooddata_eval.core.models import ChatResult, DatasetItem
3237
from gooddata_eval.core.timing import note_trace_latency
@@ -460,6 +465,7 @@ def __init__(
460465
reasoning_effort: ReasoningEffort | None = None,
461466
agent_id: str | None = None,
462467
user_context: dict[str, Any] | None = None,
468+
document_format: DocumentFormat | None = None,
463469
):
464470
"""Create a chat client bound to one workspace.
465471
@@ -476,6 +482,13 @@ def __init__(
476482
back to it: a caller that changes or clears the context per message builds the
477483
client without one. The server grounds answers in it only while
478484
the ``enableAiContextSetup`` feature flag is on for the organization.
485+
486+
``document_format`` (``aac-v1``/``aac-v2``) is sent as ``options.documentFormat``
487+
on every message and selects the document content format gen-ai drafts in for
488+
that turn. gen-ai keeps a draft in the format of the turn that wrote it and a
489+
turn in the other format cannot continue it, so follow-up and clarification
490+
turns carry it too. When None the key is omitted and the server picks the format
491+
from the ``enablePublisherV2`` feature flag. Any other value raises ``ValueError``.
479492
"""
480493
self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
481494
self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
@@ -494,6 +507,7 @@ def __init__(
494507
self._client = httpx.Client(timeout=http_timeout)
495508
self._preserve_failed = preserve_failed
496509
self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
510+
self._document_format = normalize_document_format(document_format)
497511
self._agent_id = agent_id
498512
self._user_context = user_context
499513
# Run index of each conversation this client has sent to, in first-send order: every
@@ -531,8 +545,13 @@ def send_message(
531545
join_headers, joined = self._join_item_trace(conversation_id)
532546
headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers}
533547
body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
548+
options: dict[str, str] = {}
534549
if self._reasoning_effort is not None:
535-
body["options"] = {"reasoningEffort": self._reasoning_effort}
550+
options["reasoningEffort"] = self._reasoning_effort
551+
if self._document_format is not None:
552+
options["documentFormat"] = self._document_format
553+
if options:
554+
body["options"] = options
536555
# A per-call context overrides the client's. Only sent when there is one: gen-ai
537556
# accepts an explicit null, so assigning unconditionally would quietly change every
538557
# request that has no attachment.

‎packages/gooddata-eval/src/gooddata_eval/core/config.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ def judge_model() -> str:
3333
ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"]
3434
"""Effort values the AI chat endpoint accepts, uppercase as the server enum requires."""
3535

36+
DocumentFormat = Literal["aac-v1", "aac-v2"]
37+
"""Document content formats the AI chat endpoint accepts as ``options.documentFormat``."""
38+
3639
EvalGate = Literal["any", "power"]
3740
"""`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here
3841
because putting it there would make config import the agentic package, whose __init__ imports
@@ -73,6 +76,22 @@ def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
7376
return cast("ReasoningEffort", candidate)
7477

7578

79+
def normalize_document_format(value: str | None) -> DocumentFormat | None:
80+
"""Canonical document format, or None when unset.
81+
82+
gen-ai rejects any value outside `DocumentFormat`, so an unknown value raises here,
83+
before a run sends its first request.
84+
"""
85+
if value is None:
86+
return None
87+
candidate = value.strip().lower()
88+
if not candidate:
89+
return None
90+
if candidate not in get_args(DocumentFormat):
91+
raise ValueError(f"Invalid document format {value!r}; expected one of {', '.join(get_args(DocumentFormat))}.")
92+
return cast("DocumentFormat", candidate)
93+
94+
7695
@dataclass
7796
class RunConfig:
7897
host: str

‎packages/gooddata-eval/tests/test_agentic_document_skill.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,74 @@ def test_a_user_context_with_view_report_fails_before_any_request(monkeypatch: p
536536
assert client.sent == []
537537

538538

539+
def test_the_document_format_reaches_the_chat_client(monkeypatch: pytest.MonkeyPatch) -> None:
540+
client = _ScriptedChatClient([_drafting_turn()])
541+
built: list[dict[str, Any]] = []
542+
543+
def _build(**kwargs: Any) -> _ScriptedChatClient:
544+
built.append(kwargs)
545+
return client
546+
547+
monkeypatch.setattr(document_skill, "ChatClient", _build)
548+
evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="aac-v2")
549+
assert [kwargs["document_format"] for kwargs in built] == ["aac-v2"]
550+
551+
552+
def test_an_unknown_document_format_fails_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None:
553+
client = _ScriptedChatClient([_drafting_turn()])
554+
_install_client(monkeypatch, client)
555+
with pytest.raises(ValueError, match="Invalid document format 'v2'"):
556+
evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="v2")
557+
assert client.created == client.sent == []
558+
559+
560+
def test_an_unknown_document_format_fails_before_the_narrative_judge_is_built(
561+
monkeypatch: pytest.MonkeyPatch,
562+
) -> None:
563+
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
564+
client = _ScriptedChatClient([_drafting_turn()])
565+
_install_client(monkeypatch, client)
566+
with pytest.raises(ValueError, match="Invalid document format 'v2'"):
567+
run_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, document_format="v2")
568+
assert client.created == client.sent == []
569+
570+
571+
@pytest.mark.parametrize(
572+
("document_format", "extra", "expected"),
573+
[
574+
("AAC-V2", {"tags": ["ci"]}, {"tags": ["ci"], "document_format": "aac-v2"}),
575+
(None, {"tags": ["ci"]}, {"tags": ["ci"]}),
576+
(None, None, None),
577+
],
578+
)
579+
def test_the_document_format_is_recorded_in_the_run_metadata(
580+
monkeypatch: pytest.MonkeyPatch,
581+
document_format: str | None,
582+
extra: dict[str, Any] | None,
583+
expected: dict[str, Any] | None,
584+
) -> None:
585+
_install_client(monkeypatch, _ScriptedChatClient([_drafting_turn()]))
586+
identities: list[Any] = []
587+
monkeypatch.setattr(
588+
document_skill, "submit_trace_scoring", lambda _link, identity, **_kw: identities.append(identity)
589+
)
590+
try:
591+
evaluate_agentic_document_skill(
592+
"https://h",
593+
"tok",
594+
"ws",
595+
"Create a document",
596+
{},
597+
langfuse=object(),
598+
dataset_item_id="item-1",
599+
run_metadata_extra=extra,
600+
document_format=document_format,
601+
)
602+
except DocumentSkillAssertionError:
603+
pass # the identity is handed to scoring before the gate raises
604+
assert [identity.run_metadata_extra for identity in identities] == [expected]
605+
606+
539607
# ── fixture validation ──────────────────────────────────────────────────────
540608

541609

‎packages/gooddata-eval/tests/test_sse_client.py‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import json
33
import logging
44
from collections.abc import Callable
5+
from typing import Any
56
from urllib.parse import unquote
67

78
import httpx
@@ -951,12 +952,23 @@ def handler(request):
951952
assert ei.value.conversation_id == "conv-exc"
952953

953954

954-
def _capture_body_client(captured, *, reasoning_effort=None):
955-
def handler(request):
955+
def _capture_body_client(
956+
captured: list[dict[str, Any]],
957+
*,
958+
reasoning_effort: str | None = None,
959+
document_format: str | None = None,
960+
) -> ChatClient:
961+
def handler(request: httpx.Request) -> httpx.Response:
956962
captured.append(json.loads(request.content))
957963
return httpx.Response(200, content=_OK_SSE)
958964

959-
client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort=reasoning_effort)
965+
client = ChatClient(
966+
host="https://example.invalid",
967+
token="t",
968+
workspace_id="w",
969+
reasoning_effort=reasoning_effort,
970+
document_format=document_format,
971+
)
960972
client._client = httpx.Client(transport=httpx.MockTransport(handler))
961973
return client
962974

@@ -1026,6 +1038,43 @@ def test_invalid_reasoning_effort_fails_at_construction():
10261038
ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum")
10271039

10281040

1041+
@pytest.mark.parametrize("document_format", ["aac-v1", "aac-v2"])
1042+
def test_send_message_sends_document_format_on_every_message(document_format: str) -> None:
1043+
"""gen-ai keeps a draft in the format of the turn that wrote it, so follow-ups need it too."""
1044+
captured: list[dict[str, Any]] = []
1045+
client = _capture_body_client(captured, document_format=document_format)
1046+
client.send_message("conv", "first")
1047+
client.send_message("conv", "follow-up")
1048+
assert [c["options"] for c in captured] == [{"documentFormat": document_format}] * 2
1049+
1050+
1051+
def test_send_message_sends_document_format_with_reasoning_effort() -> None:
1052+
captured: list[dict[str, Any]] = []
1053+
_capture_body_client(captured, reasoning_effort="LOW", document_format="aac-v2").send_message("conv", "q")
1054+
assert captured[0]["options"] == {"reasoningEffort": "LOW", "documentFormat": "aac-v2"}
1055+
1056+
1057+
@pytest.mark.parametrize(("given", "sent"), [("AAC-V2", "aac-v2"), (" aac-v1 ", "aac-v1")])
1058+
def test_send_message_normalizes_document_format(given: str, sent: str) -> None:
1059+
captured: list[dict[str, Any]] = []
1060+
_capture_body_client(captured, document_format=given).send_message("conv", "q")
1061+
assert captured[0]["options"] == {"documentFormat": sent}
1062+
1063+
1064+
@pytest.mark.parametrize("blank", ["", " "])
1065+
def test_blank_document_format_is_treated_as_unset(blank: str) -> None:
1066+
captured: list[dict[str, Any]] = []
1067+
_capture_body_client(captured, document_format=blank).send_message("conv", "q")
1068+
assert "options" not in captured[0]
1069+
1070+
1071+
@pytest.mark.parametrize("bad", ["v2", "aac-v3", "aac_v2"])
1072+
def test_invalid_document_format_fails_at_construction(bad: str) -> None:
1073+
"""gen-ai rejects any other value; fail before the first request instead."""
1074+
with pytest.raises(ValueError, match="Invalid document format"):
1075+
ChatClient(host="https://example.invalid", token="t", workspace_id="w", document_format=bad)
1076+
1077+
10291078
def test_turn_timeout_aborts_a_streaming_turn_and_is_not_retried(monkeypatch):
10301079
"""A chatty-but-slow agent must be cut off: httpx's per-read timeout never fires
10311080
for one, so only the wall-clock budget bounds the item."""

0 commit comments

Comments
 (0)