Observability: customizable application-layer outcomes from evaluation runs

EXTENDING.md

@@ -78,6 +78,16 @@ MCP URL, a plain streamable-HTTP URL) or a skill is a registry write: portal UI,
 API call, or a new entry in `seed_data.py`. No mechanism changes, nothing to
 rebuild.
 
+### Add an application-effect dashboard scenario — data only
+
+Create an Evaluation dataset with a custom `scenario` name and `scoring`
+configuration. Use `json_exact` with an `output_field` (including a dotted
+field such as `decision.intent`) for deterministic classification, or
+`llm_judge` with a business rubric for answer quality. Once the dataset runs,
+Observability discovers the scenario and charts its actual results. See
+[docs/observability.md](docs/observability.md#application-effect-dashboard)
+for an API example and the code touch points for adding a new scoring method.
+
 ### Add an integration *kind* — a known multi-file change
 
 The registry classifies each MCP entry by `kind` (`agentcore-runtime` |

backend/app/api/evals.py

@@ -1,6 +1,6 @@
 """Evaluation endpoints."""
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
 
 from app.dependencies import get_current_user, require_admin
 from app.models.schemas import EvalDatasetCreateRequest, EvalRunRequest
@@ -22,6 +22,9 @@ def create_dataset(req: EvalDatasetCreateRequest, user: str = Depends(get_curren
             user=user,
             name=req.name,
             description=req.description,
+            scenario=req.scenario,
+            scoring=req.scoring.model_dump() if req.scoring else None,
+            synthetic=req.synthetic,
             cases=[c.model_dump() for c in req.cases],
         )
     except ValueError as e:
@@ -30,6 +33,16 @@ def create_dataset(req: EvalDatasetCreateRequest, user: str = Depends(get_curren
     return ds
 
 
+@router.get("/datasets/{dataset_id}/runs")
+def list_dataset_runs(
+    dataset_id: str,
+    limit: int = Query(20, ge=1, le=50),
+    user: str = Depends(get_current_user),
+):
+    """Run summaries of one dataset, newest first (for version comparison)."""
+    return eval_service.list_dataset_runs(dataset_id, limit)
+
+
 @router.delete("/datasets/{dataset_id}")
 def delete_dataset(dataset_id: str, user: str = Depends(get_current_user)):
     if not eval_service.delete_dataset(dataset_id):
@@ -39,8 +52,11 @@ def delete_dataset(dataset_id: str, user: str = Depends(get_current_user)):
 
 
 @router.get("/runs")
-def list_runs(user: str = Depends(get_current_user)):
-    return eval_service.list_runs()
+def list_runs(
+    limit: int = Query(20, ge=1, le=50),
+    user: str = Depends(get_current_user),
+):
+    return eval_service.list_runs(limit)
 
 
 @router.get("/runs/{run_id}")
@@ -57,6 +73,6 @@ async def start_run(req: EvalRunRequest, user: str = Depends(get_current_user)):
     try:
         run = eval_service.start_run(user=user, dataset_id=req.dataset_id, target=req.target)
     except KeyError:
-        raise HTTPException(status_code=404, detail="Dataset not found")
+        raise HTTPException(status_code=404, detail="Dataset or agent not found")
     audit_service.record(user, "eval.run.start", f"run:{run['id']}", f"dataset {run['dataset_name']} → {req.target}")
     return run

backend/app/models/schemas.py

@@ -1,5 +1,7 @@
 """API request/response models."""
 
+from typing import Literal
+
 from pydantic import BaseModel, Field
 
 
@@ -183,9 +185,22 @@ class EvalCase(BaseModel):
     expected: str = Field(default="", max_length=1000)
 
 
+class EvalScoringConfig(BaseModel):
+    method: Literal["json_exact", "llm_judge"] = "llm_judge"
+    output_field: str = Field(
+        default="category",
+        max_length=80,
+        pattern=r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*){0,3}$",
+    )
+    rubric: str = Field(default="", max_length=1000)
+
+
 class EvalDatasetCreateRequest(BaseModel):
     name: str = Field(min_length=1, max_length=120)
     description: str = Field(default="", max_length=400)
+    scenario: str = Field(default="general", max_length=48, pattern=r"^[a-z][a-z0-9_-]*$")
+    scoring: EvalScoringConfig | None = None
+    synthetic: bool = False
     cases: list[EvalCase] = Field(min_length=1, max_length=20)