From f7b3c0414b227f99dd7346726cd759b112d9e83c Mon Sep 17 00:00:00 2001 From: zzkamzn Date: Wed, 30 Sep 2026 08:30:24 +0000 Subject: [PATCH 1/8] Show application outcomes on the Observability page from evaluation runs Evaluation datasets now carry a free-form scenario name and a scoring config: json_exact compares one (optionally dotted) JSON output field with the expected value deterministically, llm_judge keeps the judge with an optional business rubric. Runs pin the published agent's version and system prompt at start and fail if the agent is republished mid-run, so a run never mixes versions. Rows keep the full prompt, expectation, answer and predicted/expected values. Observability discovers scenarios from the runs and shows pass rate, label distribution, invalid outputs, low-score cases and a same-dataset regression comparison. scripts/run_observability_demo.py seeds two synthetic benchmarks against a deployed portal with real model calls. Co-Authored-By: Claude Opus 5.5 (1M context) --- EXTENDING.md | 10 + backend/app/api/evals.py | 14 +- backend/app/models/schemas.py | 15 + backend/app/services/eval_service.py | 197 ++++++++++--- backend/tests/test_eval_scenarios.py | 178 ++++++++++++ docs/observability.md | 74 +++++ frontend/src/pages/EvalPage.tsx | 62 ++++- frontend/src/pages/ObservabilityPage.tsx | 336 +++++++++++++++++++---- frontend/src/services/api.ts | 29 +- scripts/run_observability_demo.py | 178 ++++++++++++ 10 files changed, 988 insertions(+), 105 deletions(-) create mode 100644 backend/tests/test_eval_scenarios.py create mode 100644 scripts/run_observability_demo.py diff --git a/EXTENDING.md b/EXTENDING.md index d839930..ee4e0c2 100644 --- a/EXTENDING.md +++ b/EXTENDING.md @@ -77,6 +77,16 @@ MCP URL, a plain streamable-HTTP URL) or a skill is a registry write: portal UI, API call, or a new entry in `seed_data.py`. No mechanism changes, nothing to rebuild. +### Add an application-effect dashboard scenario — data only + +Create an Evaluation dataset with a custom `scenario` name and `scoring` +configuration. Use `json_exact` with an `output_field` (including a dotted +field such as `decision.intent`) for deterministic classification, or +`llm_judge` with a business rubric for answer quality. Once the dataset runs, +Observability discovers the scenario and charts its actual results. See +[docs/observability.md](docs/observability.md#application-effect-dashboard) +for an API example and the code touch points for adding a new scoring method. + ### Add an integration *kind* — a known multi-file change The registry classifies each MCP entry by `kind` (`agentcore-runtime` | diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index b94d835..55057f3 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -1,6 +1,6 @@ """Evaluation endpoints.""" -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from app.dependencies import get_current_user, require_admin from app.models.schemas import EvalDatasetCreateRequest, EvalRunRequest @@ -22,6 +22,9 @@ def create_dataset(req: EvalDatasetCreateRequest, user: str = Depends(get_curren user=user, name=req.name, description=req.description, + scenario=req.scenario, + scoring=req.scoring.model_dump() if req.scoring else None, + synthetic=req.synthetic, cases=[c.model_dump() for c in req.cases], ) except ValueError as e: @@ -39,8 +42,11 @@ def delete_dataset(dataset_id: str, user: str = Depends(get_current_user)): @router.get("/runs") -def list_runs(user: str = Depends(get_current_user)): - return eval_service.list_runs() +def list_runs( + limit: int = Query(20, ge=1, le=50), + user: str = Depends(get_current_user), +): + return eval_service.list_runs(limit) @router.get("/runs/{run_id}") @@ -57,6 +63,6 @@ async def start_run(req: EvalRunRequest, user: str = Depends(get_current_user)): try: run = eval_service.start_run(user=user, dataset_id=req.dataset_id, target=req.target) except KeyError: - raise HTTPException(status_code=404, detail="Dataset not found") + raise HTTPException(status_code=404, detail="Dataset or agent not found") audit_service.record(user, "eval.run.start", f"run:{run['id']}", f"dataset {run['dataset_name']} → {req.target}") return run diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 16c21ac..7816270 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -1,5 +1,7 @@ """API request/response models.""" +from typing import Literal + from pydantic import BaseModel, Field @@ -183,9 +185,22 @@ class EvalCase(BaseModel): expected: str = Field(default="", max_length=1000) +class EvalScoringConfig(BaseModel): + method: Literal["json_exact", "llm_judge"] = "llm_judge" + output_field: str = Field( + default="category", + max_length=80, + pattern=r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*){0,3}$", + ) + rubric: str = Field(default="", max_length=1000) + + class EvalDatasetCreateRequest(BaseModel): name: str = Field(min_length=1, max_length=120) description: str = Field(default="", max_length=400) + scenario: str = Field(default="general", max_length=48, pattern=r"^[a-z][a-z0-9_-]*$") + scoring: EvalScoringConfig | None = None + synthetic: bool = False cases: list[EvalCase] = Field(min_length=1, max_length=20) diff --git a/backend/app/services/eval_service.py b/backend/app/services/eval_service.py index 0affd75..077829d 100644 --- a/backend/app/services/eval_service.py +++ b/backend/app/services/eval_service.py @@ -1,11 +1,11 @@ -"""Evaluation: fixed task suites scored by an LLM judge. +"""Evaluation: fixed task suites with dataset-selected scoring. Datasets (``PK=EVAL``) hold up to :data:`MAX_CASES` cases inline — each a prompt plus free-text expectation. A *run* executes every case against a -chosen target (kernel or published agent) through the invocation pipeline, -then asks the same headless kernel to act as a strict judge, scoring each -answer against the expectation. Runs (``PK=EVALRUN``) update progressively so -the portal can poll while a run executes in the background. +chosen target (kernel or published agent) through the invocation pipeline. +JSON field datasets use deterministic exact-match scoring; other datasets ask +the same headless kernel to act as a strict judge. Runs (``PK=EVALRUN``) update +progressively so the portal can poll while a run executes in the background. Judging with the platform's own kernel keeps the sample dependency-free; the judge system prompt pins the output to a JSON verdict for parsing. @@ -47,8 +47,10 @@ def _parse_verdict(text: str) -> dict: if match: try: v = json.loads(match.group(0)) + if not isinstance(v, dict) or not isinstance(v.get("pass"), bool): + raise ValueError("judge pass must be a boolean") return { - "pass": bool(v.get("pass")), + "pass": v["pass"], "score": max(0, min(10, int(v.get("score", 0)))), "reason": str(v.get("reason", ""))[:300], } @@ -57,6 +59,53 @@ def _parse_verdict(text: str) -> dict: return {"pass": False, "score": 0, "reason": f"unparseable judge output: {text[:120]}"} +def _default_scoring(scenario: str) -> dict: + # Existing datasets predate scoring configuration. + return { + "method": "json_exact" if scenario == "classification" else "llm_judge", + "output_field": "category", + "rubric": "", + } + + +def _parse_json_field(text: str, field: str) -> str: + """Read one bounded dotted field from a JSON object returned by the agent.""" + cleaned = text.strip() + if cleaned.startswith("```"): + cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", cleaned, flags=re.IGNORECASE).strip() + try: + value = json.loads(cleaned) + except json.JSONDecodeError: + return "" + for part in field.split("."): + if not isinstance(value, dict): + return "" + value = value.get(part) + return value.strip()[:120] if isinstance(value, str) else "" + + +def _parse_category(text: str) -> str: + """Compatibility helper for the original classifier contract.""" + return _parse_json_field(text, "category") + + +def _validated_scoring(scenario: str, scoring: dict | None) -> dict: + config = {**_default_scoring(scenario), **(scoring or {})} + if config["method"] not in ("json_exact", "llm_judge"): + raise ValueError("unknown scoring method") + if not isinstance(config["output_field"], str) or not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*){0,3}", config["output_field"] + ): + raise ValueError("invalid scoring output_field") + if not isinstance(config["rubric"], str) or len(config["rubric"]) > 1000: + raise ValueError("invalid scoring rubric") + return { + "method": config["method"], + "output_field": config["output_field"], + "rubric": config["rubric"], + } + + class EvalService: def __init__(self) -> None: dynamodb = boto3.resource("dynamodb", region_name=settings.aws_region) @@ -74,6 +123,9 @@ def _ds_public(item: dict) -> dict: "id": item["SK"].partition("#")[2], "name": item.get("name", ""), "description": item.get("description", ""), + "scenario": item.get("scenario", "general"), + "scoring": item.get("scoring") or _default_scoring(item.get("scenario", "general")), + "synthetic": bool(item.get("synthetic", False)), "cases": item.get("cases", []), "created_by": item.get("created_by", ""), "created_at": item.get("created_at", ""), @@ -90,7 +142,14 @@ def list_datasets(self) -> list[dict]: reverse=True, ) - def create_dataset(self, *, user: str, name: str, description: str, cases: list[dict]) -> dict: + def create_dataset( + self, *, user: str, name: str, description: str, + cases: list[dict], scenario: str = "general", synthetic: bool = False, + scoring: dict | None = None, + ) -> dict: + if not re.fullmatch(r"[a-z][a-z0-9_-]{0,47}", scenario): + raise ValueError("invalid evaluation scenario") + scoring = _validated_scoring(scenario, scoring) cleaned = [ {"prompt": str(c.get("prompt", ""))[:2000], "expected": str(c.get("expected", ""))[:1000]} for c in cases[:MAX_CASES] @@ -103,6 +162,9 @@ def create_dataset(self, *, user: str, name: str, description: str, cases: list[ "SK": f"DS#{uuid.uuid4().hex[:12]}", "name": name[:120] or "dataset", "description": description[:400], + "scenario": scenario, + "scoring": scoring, + "synthetic": synthetic, "cases": cleaned, "created_by": user, "created_at": _now(), @@ -125,7 +187,12 @@ def _run_public(item: dict) -> dict: "id": item.get("run_id", ""), "dataset_id": item.get("dataset_id", ""), "dataset_name": item.get("dataset_name", ""), + "scenario": item.get("scenario", "general"), + "scoring": item.get("scoring") or _default_scoring(item.get("scenario", "general")), + "synthetic": bool(item.get("synthetic", False)), "target": item.get("target", ""), + "agent_version": int(item["agent_version"]) if item.get("agent_version") is not None else None, + "system_prompt": item.get("system_prompt", ""), "status": item.get("status", ""), "started_by": item.get("started_by", ""), "started_at": item.get("started_at", ""), @@ -176,13 +243,28 @@ def start_run(self, *, user: str, dataset_id: str, target: str) -> dict: raise KeyError("dataset not found") run_id = uuid.uuid4().hex[:12] sk = f"{_now()}#{run_id}" + agent_version = None + system_prompt = "" + if target.startswith("agent:"): + from app.services.agent_service import agent_service + + agent = agent_service.get_agent(target.partition(":")[2]) + if not agent: + raise KeyError("agent not found") + agent_version = agent["version"] + system_prompt = agent["system_prompt"] item = { "PK": PK_RUN, "SK": sk, "run_id": run_id, "dataset_id": dataset_id, "dataset_name": ds.get("name", ""), + "scenario": ds.get("scenario", "general"), + "scoring": ds.get("scoring") or _default_scoring(ds.get("scenario", "general")), + "synthetic": bool(ds.get("synthetic", False)), "target": target, + "agent_version": agent_version, + "system_prompt": system_prompt, "status": "running", "started_by": user, "started_at": _now(), @@ -194,18 +276,30 @@ def start_run(self, *, user: str, dataset_id: str, target: str) -> dict: # fire-and-forget; progress lands in DDB. Requires a running event # loop, so the API route calling this must be ``async def`` (sync # routes run in a threadpool thread with no loop). - task = asyncio.get_running_loop().create_task(self._execute(sk, user, ds, target)) + task = asyncio.get_running_loop().create_task( + self._execute(sk, user, ds, target, agent_version=agent_version) + ) self._tasks.add(task) task.add_done_callback(self._tasks.discard) return self._run_public(item) - async def _execute(self, sk: str, user: str, ds: dict, target: str) -> None: + async def _execute( + self, sk: str, user: str, ds: dict, target: str, agent_version: int | None = None, + ) -> None: from app.services.invocation_service import invoke # avoid import cycle run_id = sk.partition("#")[2] + scenario = ds.get("scenario", "general") + scoring = ds.get("scoring") or _default_scoring(scenario) results: list[dict] = [] try: for idx, case in enumerate(ds.get("cases", [])): + if agent_version is not None: + from app.services.agent_service import agent_service + + current = agent_service.get_agent(target.partition(":")[2]) + if not current or current["version"] != agent_version: + raise RuntimeError("agent version changed during evaluation") answer_res = await asyncio.to_thread( invoke, user=user, @@ -214,35 +308,68 @@ async def _execute(self, sk: str, user: str, ds: dict, target: str) -> None: prompt=case["prompt"], ref=f"eval:{run_id}", ) + if agent_version is not None: + current = agent_service.get_agent(target.partition(":")[2]) + if not current or current["version"] != agent_version: + raise RuntimeError("agent version changed during evaluation") answer = (answer_res.get("result") or "").strip() - judge_prompt = ( - f"Task prompt:\n{case['prompt']}\n\n" - f"Expected outcome:\n{case['expected'] or '(none given — judge general correctness)'}\n\n" - f"Candidate answer:\n{answer or '(empty answer)'}" - ) - judge_res = await asyncio.to_thread( - invoke, - user=user, - source="eval", - target="agent-sdk", - prompt=judge_prompt, - system=JUDGE_SYSTEM, - max_turns=1, - ref=f"eval-judge:{run_id}", - ) - verdict = _parse_verdict(judge_res.get("result") or "") - results.append( - { - "case": idx, - "prompt": case["prompt"][:200], - "expected": case["expected"][:200], - "answer": answer[:500], - "pass": verdict["pass"], - "score": verdict["score"], - "reason": verdict["reason"], + if scoring["method"] == "json_exact": + predicted = ( + _parse_json_field(answer, scoring["output_field"]) + if answer_res.get("ok") else "" + ) + expected = case["expected"].strip() + passed = bool(predicted) and predicted == expected + verdict = { + "pass": passed, + "score": 10 if passed else 0, + "reason": ( + f"{scoring['output_field']} matches expected value" if passed else + "agent invocation failed" if not answer_res.get("ok") else + f"invalid or missing {scoring['output_field']} in JSON output" if not predicted else + f"expected {expected}, got {predicted}" + ), } - ) + elif not answer_res.get("ok"): + predicted = "" + verdict = {"pass": False, "score": 0, "reason": "agent invocation failed"} + else: + predicted = "" + judge_prompt = ( + f"Task prompt:\n{case['prompt']}\n\n" + f"Expected outcome:\n{case['expected'] or '(none given — judge general correctness)'}\n\n" + f"Evaluation rubric:\n{scoring.get('rubric') or '(use expected outcome)'}\n\n" + f"Candidate answer:\n{answer or '(empty answer)'}" + ) + judge_res = await asyncio.to_thread( + invoke, + user=user, + source="eval", + target="agent-sdk", + prompt=judge_prompt, + system=JUDGE_SYSTEM, + max_turns=1, + ref=f"eval-judge:{run_id}", + ) + verdict = ( + _parse_verdict(judge_res.get("result") or "") + if judge_res.get("ok") + else {"pass": False, "score": 0, "reason": "judge invocation failed"} + ) + row = { + "case": idx, + "prompt": case["prompt"], + "expected": case["expected"], + "answer": answer[:8000], + "pass": verdict["pass"], + "score": verdict["score"], + "reason": verdict["reason"], + } + if scoring["method"] == "json_exact": + row["predicted_value"] = predicted + row["expected_value"] = case["expected"].strip() + results.append(row) self._update_run( sk, results=results, diff --git a/backend/tests/test_eval_scenarios.py b/backend/tests/test_eval_scenarios.py new file mode 100644 index 0000000..926f5a9 --- /dev/null +++ b/backend/tests/test_eval_scenarios.py @@ -0,0 +1,178 @@ +"""Focused tests for application scoring and persisted case evidence.""" + +import asyncio +import importlib +import sys +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +import boto3 + + +class FakeTable: + def __init__(self): + self.fields = {} + self.item = None + + def put_item(self, **kwargs): + self.item = kwargs["Item"] + + def update_item(self, **kwargs): + names = kwargs["ExpressionAttributeNames"] + values = kwargs["ExpressionAttributeValues"] + for index in range(len(names)): + self.fields[names[f"#f{index}"]] = values[f":v{index}"] + + +def load_eval_module(): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + config = types.ModuleType("app.config") + config.settings = types.SimpleNamespace(aws_region="us-east-1", dynamo_table="test") + resource = types.SimpleNamespace(Table=lambda name: FakeTable()) + with patch.dict(sys.modules, {"app.config": config}), patch.object(boto3, "resource", return_value=resource): + return importlib.import_module("app.services.eval_service") + + +class EvalScenarioTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_eval_module() + + @classmethod + def tearDownClass(cls): + sys.modules.pop("app.services.eval_service", None) + package = sys.modules.get("app.services") + if package is not None: + package.__dict__.pop("eval_service", None) + + def setUp(self): + self.service = self.module.EvalService.__new__(self.module.EvalService) + self.service.table = FakeTable() + + def execute(self, dataset, invoke, *, agent_version=None, agent_service=None): + invocation_module = types.ModuleType("app.services.invocation_service") + invocation_module.invoke = invoke + async def call_direct(function, **kwargs): + return function(**kwargs) + + modules = {"app.services.invocation_service": invocation_module} + if agent_service is not None: + agent_module = types.ModuleType("app.services.agent_service") + agent_module.agent_service = agent_service + modules["app.services.agent_service"] = agent_module + with patch.dict(sys.modules, modules), patch.object( + self.module.asyncio, "to_thread", call_direct + ): + asyncio.run(self.service._execute( + "ts#run1", "tester", dataset, "agent:demo", agent_version=agent_version, + )) + return self.service.table.fields + + def test_category_contract(self): + parse = self.module._parse_category + self.assertEqual(parse('{"category":"billing"}'), "billing") + self.assertEqual(parse('```json\n{"category":"technical"}\n```'), "technical") + self.assertEqual(parse('{"category":42}'), "") + self.assertEqual(parse("technical"), "") + self.assertEqual( + self.module._parse_json_field('{"decision":{"intent":"refund"}}', "decision.intent"), + "refund", + ) + + def test_judge_string_false_is_not_a_pass(self): + verdict = self.module._parse_verdict('{"pass":"false","score":8,"reason":"wrong type"}') + self.assertFalse(verdict["pass"]) + self.assertEqual(verdict["score"], 0) + + def test_classification_uses_exact_match_without_judge(self): + answers = { + "a": '{"category":"billing"}', + "b": '{"category":"billing"}', + "c": "not JSON", + } + calls = [] + + def invoke(**kwargs): + calls.append(kwargs) + self.assertEqual(kwargs["target"], "agent:demo") + return {"ok": True, "result": answers[kwargs["prompt"]]} + + fields = self.execute({ + "scenario": "classification", + "cases": [ + {"prompt": "a", "expected": "billing"}, + {"prompt": "b", "expected": "technical"}, + {"prompt": "c", "expected": "account"}, + ], + }, invoke) + self.assertEqual(len(calls), 3) + self.assertEqual(fields["status"], "completed") + self.assertEqual(fields["passed"], 1) + self.assertEqual([r["predicted_value"] for r in fields["results"]], ["billing", "billing", ""]) + self.assertEqual([r["score"] for r in fields["results"]], [10, 0, 0]) + + def test_support_uses_real_judge_verdict(self): + calls = [] + + def invoke(**kwargs): + calls.append(kwargs) + if kwargs["target"] == "agent:demo": + return {"ok": True, "result": "Please contact human support."} + return {"ok": True, "result": '{"pass":true,"score":8,"reason":"Escalated appropriately"}'} + + fields = self.execute({ + "scenario": "support", + "scoring": {"method": "llm_judge", "output_field": "category", "rubric": "Escalate unsupported requests."}, + "cases": [{"prompt": "Can I get a refund after 15 days?", "expected": "Escalate to human support"}], + }, invoke) + self.assertEqual(len(calls), 2) + self.assertIn("Escalate unsupported requests.", calls[1]["prompt"]) + self.assertEqual(fields["passed"], 1) + self.assertEqual(fields["results"][0]["answer"], "Please contact human support.") + self.assertEqual(fields["results"][0]["reason"], "Escalated appropriately") + + def test_custom_scenario_reads_configured_json_field(self): + fields = self.execute({ + "scenario": "claims-routing", + "scoring": {"method": "json_exact", "output_field": "decision.intent", "rubric": ""}, + "cases": [{"prompt": "route this", "expected": "refund"}], + }, lambda **kwargs: {"ok": True, "result": '{"decision":{"intent":"refund"}}'}) + self.assertEqual(fields["passed"], 1) + self.assertEqual(fields["results"][0]["predicted_value"], "refund") + + def test_user_defined_scenario_is_saved_with_scoring_contract(self): + dataset = self.service.create_dataset( + user="tester", + name="claims-routing", + description="Claim intent", + scenario="claims-routing", + scoring={"method": "json_exact", "output_field": "decision.intent", "rubric": ""}, + cases=[{"prompt": "Refund this", "expected": "refund"}], + ) + self.assertEqual(dataset["scoring"]["output_field"], "decision.intent") + self.assertEqual(self.service.table.item["scenario"], "claims-routing") + with self.assertRaises(ValueError): + self.service.create_dataset( + user="tester", name="bad", description="", scenario="claims-routing", + scoring={"method": "json_exact", "output_field": "decision[0]", "rubric": ""}, + cases=[{"prompt": "Refund this", "expected": "refund"}], + ) + + def test_version_change_fails_run(self): + versions = iter((1, 2)) + agent = types.SimpleNamespace(get_agent=lambda agent_id: {"version": next(versions)}) + with patch.object(self.module.logger, "exception"): + fields = self.execute( + {"scenario": "classification", "cases": [{"prompt": "a", "expected": "billing"}]}, + lambda **kwargs: {"ok": True, "result": '{"category":"billing"}'}, + agent_version=1, + agent_service=agent, + ) + self.assertEqual(fields["status"], "failed") + self.assertIn("version changed", fields["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/observability.md b/docs/observability.md index 09defa5..ab9a7fa 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -153,3 +153,77 @@ correlated by `spanId`; the kernel's stdout lines in the same group also carry - Business counters that only the workflow engine knows (funnel counts per phase) are still application-side; publish them as EMF metrics in the same `bedrock-agentcore` namespace to put them on the same dashboard. + +## Application-effect dashboard + +The portal's **Observability** page reads evaluation runs and displays +application outcomes separately from execution health. It discovers scenario +tabs from the datasets that have actually run; no scenario name is built into +the dashboard. Each dataset selects one of two scoring methods: + +- **`json_exact`** reads a configured string field from the agent's JSON + response (for example `category` or `decision.intent`) and compares it + exactly with the case's expected value. The dashboard counts expected and + actual values, exact-match rate, invalid outputs and failures. It makes no + judge-model call. +- **`llm_judge`** scores each answer against the case's expectation and an + optional dataset rubric. The dashboard shows pass rate, mean score, + low-score cases, full answers and reasons. Judge scores are screening + signals; calibrate them against a small human-reviewed set before treating + them as a quality target. + +Users can create a custom scenario and choose its scoring contract in the +**Evaluation** page. The equivalent API request is: + +```json +{ + "name": "claims-routing-v1", + "scenario": "claims-routing", + "scoring": {"method": "json_exact", "output_field": "decision.intent", "rubric": ""}, + "cases": [{"prompt": "I need a refund for order A", "expected": "refund"}] +} +``` + +The agent must then return a JSON object with `decision.intent`. A new +scenario appears in Observability when its first evaluation run starts. +For an LLM-judged scenario, use `method = llm_judge` and set `rubric` to the +business-specific judging criteria. The original `classification` and +`support` datasets still work without a `scoring` field. + +This is the no-code extension path. For a new scoring algorithm, extend +`EvalScoringConfig` in `backend/app/models/schemas.py`, the scoring dispatch +in `backend/app/services/eval_service.py`, and the method selector in +`frontend/src/pages/EvalPage.tsx`. The dashboard draws a value distribution +for exact JSON scoring and a score breakdown for judge scoring. A custom +algorithm may also need a corresponding dashboard component. The platform +does not execute arbitrary user-uploaded evaluator code. + +Each run captures the published agent's version and configured system prompt +at start, together with the case prompts and actual answers. A version change +while a run is executing fails the run so it cannot be presented as a +single-version comparison. The dashboard compares completed runs of the +**same dataset** and flags a drop in pass rate. This is an offline benchmark: +the general invocation ledger still stores only a 200-character user-prompt +preview and execution metrics. It has no automatic business label or +production-conversation quality score. + +To generate results without customer data, run the synthetic benchmark against +a **deployed** portal: + +```bash +PORTAL_URL=https:// \ +PORTAL_TOKEN= \ +python3 scripts/run_observability_demo.py +``` + +For local open-auth development, omit `PORTAL_TOKEN`. The script publishes a +classifier and a support agent, creates two reusable synthetic datasets, +starts real AgentCore invocations, waits for scored results, and checks that +the case calls landed in the invocation ledger. Re-running bumps the agents' +versions and adds comparable runs to the same datasets. It does not fabricate +predictions, answers, scores, token usage, latency, or cost. Model calls incur +the platform's normal charges and quota. + +The page stays empty until runs actually return results. A workspace with no +portal URL or AWS credentials can build and test the feature but cannot make +claims about CloudWatch trace delivery or measured agent quality. diff --git a/frontend/src/pages/EvalPage.tsx b/frontend/src/pages/EvalPage.tsx index c7c22fb..3e668d1 100644 --- a/frontend/src/pages/EvalPage.tsx +++ b/frontend/src/pages/EvalPage.tsx @@ -1,7 +1,7 @@ import { useEffect, useRef, useState } from 'react' import { FlaskConical, Loader2, Play, Plus, RefreshCw, Trash2 } from 'lucide-react' import { Modal, SectionTitle } from '@/components/common/ui' -import { api, type EvalDataset, type EvalRun, type PublishedAgent } from '@/services/api' +import { api, type EvalDataset, type EvalRun, type EvalScoring, type PublishedAgent } from '@/services/api' import { fmtTs } from '@/services/format' export default function EvalPage() { @@ -15,6 +15,11 @@ export default function EvalPage() { // create form const [name, setName] = useState('') + const [scenarioChoice, setScenarioChoice] = useState('general') + const [customScenario, setCustomScenario] = useState('') + const [scoringMethod, setScoringMethod] = useState('llm_judge') + const [outputField, setOutputField] = useState('category') + const [rubric, setRubric] = useState('') const [casesText, setCasesText] = useState('What is 2+2? => 4\nCapital of France? => Paris') const [creating, setCreating] = useState(false) const [createError, setCreateError] = useState('') @@ -57,9 +62,20 @@ export default function EvalPage() { return { prompt, expected } }) .filter((c) => c.prompt) - await api.createEvalDataset({ name, cases }) + const scenario = scenarioChoice === 'custom' ? customScenario.trim() : scenarioChoice + await api.createEvalDataset({ + name, + scenario, + scoring: { method: scoringMethod, output_field: outputField.trim() || 'category', rubric }, + cases, + }) setShowCreate(false) setName('') + setScenarioChoice('general') + setCustomScenario('') + setScoringMethod('llm_judge') + setOutputField('category') + setRubric('') refresh() } catch (e) { setCreateError(String(e)) @@ -85,7 +101,7 @@ export default function EvalPage() {
@@ -105,7 +121,7 @@ export default function EvalPage() {

{d.name}

-

{d.cases.length} cases

+

{d.cases.length} cases · {d.scenario || 'general'} · {d.scoring?.method || 'llm_judge'}{d.synthetic ? ' · synthetic' : ''}