Sync from internal repositories: sample agents and tools, application-layer outcomes, QA hardening

.gitignore

@@ -55,3 +55,6 @@ CLAUDE.md
 .kiro/
 .cfn-archive/
 .qa-runs/
+
+# build artefacts of the sample tool packages (terraform/workloads/modules/sample_tools)
+terraform/workloads/modules/sample_tools/.build/

EXTENDING.md

@@ -78,6 +78,16 @@ MCP URL, a plain streamable-HTTP URL) or a skill is a registry write: portal UI,
 API call, or a new entry in `seed_data.py`. No mechanism changes, nothing to
 rebuild.
 
+### Add an application-effect dashboard scenario — data only
+
+Create an Evaluation dataset with a custom `scenario` name and `scoring`
+configuration. Use `json_exact` with an `output_field` (including a dotted
+field such as `decision.intent`) for deterministic classification, or
+`llm_judge` with a business rubric for answer quality. Once the dataset runs,
+Observability discovers the scenario and charts its actual results. See
+[docs/observability.md](docs/observability.md#application-effect-dashboard)
+for an API example and the code touch points for adding a new scoring method.
+
 ### Add an integration *kind* — a known multi-file change
 
 The registry classifies each MCP entry by `kind` (`agentcore-runtime` |

backend/.env.example

@@ -6,4 +6,6 @@ PLATFORM_SDK_RUNTIME_ARN=arn:aws:bedrock-agentcore:REGION:ACCOUNT:runtime/RUNTIM
 PLATFORM_WORKSPACE_BUCKET=agent-platform-workspaces-ACCOUNT-REGION
 # Optional static bearer token; empty = open (local dev only)
 PLATFORM_API_TOKEN=
+# Local development without any IdP or token: must be asked for explicitly.
+PLATFORM_ALLOW_OPEN_AUTH=1
 PLATFORM_CORS_ORIGINS=http://localhost:5173

backend/app/api/agents.py

@@ -47,10 +47,11 @@ def publish_agent(req: AgentPublishRequest, user: Principal = Depends(get_curren
             model_backend=req.model_backend,
             model=req.model,
             platform_version=req.platform_version,
+            runtime=req.runtime,
         )
     except PermissionError as e:
         raise HTTPException(status_code=403, detail=str(e))
-    except ValueError as e:
+    except (ValueError, KeyError) as e:
         raise HTTPException(status_code=400, detail=str(e))
     audit_service.record(user, "agent.publish", f"agent:{agent['name']}", f"v{agent['version']}")
     return agent

backend/app/api/ecosystem.py

@@ -8,6 +8,7 @@
     McpServerCreateRequest,
     SkillCreateRequest,
 )
+from app.services.audit_service import audit_service
 from app.services.ecosystem_service import ecosystem_service
 
 router = APIRouter(prefix="/api/v1/ecosystem", tags=["ecosystem"])
@@ -21,11 +22,17 @@ def list_mcp_servers(user: str = Depends(get_current_user)):
 @router.post("/mcp-servers", response_model=EcosystemEntry)
 def create_mcp_server(req: McpServerCreateRequest, user: str = Depends(require_admin)):
     try:
-        return ecosystem_service.create_mcp_server(
+        entry = ecosystem_service.create_mcp_server(
             req.name, req.description, req.kind, req.target, headers=req.headers, auth=req.auth
         )
+    except KeyError as e:
+        raise HTTPException(status_code=409, detail=str(e).strip("'"))
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
+    # registry writes change what every attached agent runs against: audited
+    # like the other mutating admin actions
+    audit_service.record(user, "mcp.create", f"mcp:{entry['name']}", f"{entry['kind']} -> {entry['target']}")
+    return entry
 
 
 @router.delete("/mcp-servers/{server_id}")
@@ -35,6 +42,7 @@ def delete_mcp_server(server_id: str, user: str = Depends(require_admin)):
             raise HTTPException(status_code=404, detail="Not found")
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
+    audit_service.record(user, "mcp.delete", f"mcp:{server_id}")
     return {"ok": True}
 
 
@@ -45,7 +53,12 @@ def list_skills(user: str = Depends(get_current_user)):
 
 @router.post("/skills", response_model=EcosystemEntry)
 def create_skill(req: SkillCreateRequest, user: str = Depends(require_admin)):
-    return ecosystem_service.create_skill(req.name, req.description, req.skill_md)
+    try:
+        entry = ecosystem_service.create_skill(req.name, req.description, req.skill_md)
+    except KeyError as e:
+        raise HTTPException(status_code=409, detail=str(e).strip("'"))
+    audit_service.record(user, "skill.create", f"skill:{entry['name']}")
+    return entry
 
 
 @router.delete("/skills/{skill_id}")
@@ -55,4 +68,5 @@ def delete_skill(skill_id: str, user: str = Depends(require_admin)):
             raise HTTPException(status_code=404, detail="Not found")
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
+    audit_service.record(user, "skill.delete", f"skill:{skill_id}")
     return {"ok": True}

backend/app/api/evals.py

@@ -1,6 +1,6 @@
 """Evaluation endpoints."""
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
 
 from app.dependencies import get_current_user, require_admin
 from app.models.schemas import EvalDatasetCreateRequest, EvalRunRequest
@@ -22,6 +22,9 @@ def create_dataset(req: EvalDatasetCreateRequest, user: str = Depends(get_curren
             user=user,
             name=req.name,
             description=req.description,
+            scenario=req.scenario,
+            scoring=req.scoring.model_dump() if req.scoring else None,
+            synthetic=req.synthetic,
             cases=[c.model_dump() for c in req.cases],
         )
     except ValueError as e:
@@ -30,6 +33,16 @@ def create_dataset(req: EvalDatasetCreateRequest, user: str = Depends(get_curren
     return ds
 
 
+@router.get("/datasets/{dataset_id}/runs")
+def list_dataset_runs(
+    dataset_id: str,
+    limit: int = Query(20, ge=1, le=50),
+    user: str = Depends(get_current_user),
+):
+    """Run summaries of one dataset, newest first (for version comparison)."""
+    return eval_service.list_dataset_runs(dataset_id, limit)
+
+
 @router.delete("/datasets/{dataset_id}")
 def delete_dataset(dataset_id: str, user: str = Depends(get_current_user)):
     if not eval_service.delete_dataset(dataset_id):
@@ -39,8 +52,11 @@ def delete_dataset(dataset_id: str, user: str = Depends(get_current_user)):
 
 
 @router.get("/runs")
-def list_runs(user: str = Depends(get_current_user)):
-    return eval_service.list_runs()
+def list_runs(
+    limit: int = Query(20, ge=1, le=50),
+    user: str = Depends(get_current_user),
+):
+    return eval_service.list_runs(limit)
 
 
 @router.get("/runs/{run_id}")
@@ -57,6 +73,6 @@ async def start_run(req: EvalRunRequest, user: str = Depends(get_current_user)):
     try:
         run = eval_service.start_run(user=user, dataset_id=req.dataset_id, target=req.target)
     except KeyError:
-        raise HTTPException(status_code=404, detail="Dataset not found")
+        raise HTTPException(status_code=404, detail="Dataset or agent not found")
     audit_service.record(user, "eval.run.start", f"run:{run['id']}", f"dataset {run['dataset_name']} → {req.target}")
     return run

backend/app/api/governance.py

@@ -1,6 +1,6 @@
 """Governance endpoints: usage policy + audit log."""
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 
 from app.dependencies import get_current_user, require_admin
 from app.models.schemas import GovernancePolicyUpdate
@@ -28,5 +28,5 @@ def usage(user: str = Depends(get_current_user)):
 
 
 @router.get("/audit")
-def audit(limit: int = 100, user: str = Depends(get_current_user)):
+def audit(limit: int = Query(100, ge=1, le=200), user: str = Depends(get_current_user)):
     return audit_service.list_events(limit)

backend/app/api/observability.py

@@ -1,6 +1,6 @@
 """Observability endpoints: the platform invocation ledger."""
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 
 from app.dependencies import get_current_user, require_admin
 from app.services.observability_service import observability_service
@@ -9,7 +9,7 @@
 
 
 @router.get("/invocations")
-def list_invocations(limit: int = 100, user: str = Depends(get_current_user)):
+def list_invocations(limit: int = Query(100, ge=1, le=200), user: str = Depends(get_current_user)):
     return observability_service.list_invocations(limit)
 
 

backend/app/api/pipelines.py

@@ -6,8 +6,9 @@
 """
 
 from fastapi import APIRouter, Depends, HTTPException, Query
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
+from app.config import settings
 from app.dependencies import get_current_user, require_admin
 from app.services.audit_service import audit_service
 from app.services.pipeline_service import pipeline_service
@@ -23,6 +24,9 @@ class PipelineUpsert(BaseModel):
 
 class PipelineRunRequest(BaseModel):
     args: dict | list | str | None = None
+    # Set only by the schedule-runner Lambda (settings.pipeline_delegation_users):
+    # the schedule's created_by, so the run is attributed to its owner.
+    on_behalf_of: str | None = Field(default=None, min_length=1, max_length=120)
 
 
 @router.get("/pipelines")
@@ -61,14 +65,28 @@ def delete_pipeline(pipe_id: str, user: str = Depends(get_current_user)):
 @router.post("/pipelines/{name}/runs")
 async def start_run(name: str, req: PipelineRunRequest, user: str = Depends(get_current_user)):
     # async: start_run schedules the background executor on the event loop
+    run_as = str(user)
+    if req.on_behalf_of:
+        # A scheduled pipeline fires through the Lambda, which signs in as the
+        # delegation user; the run must still belong to the schedule's owner.
+        # Nobody else may name another user here: an administrator who could
+        # would run pipelines under a colleague's quota and audit trail.
+        if str(user) not in delegation_users():
+            raise HTTPException(status_code=403, detail="on_behalf_of is reserved for the schedule-runner delegation user")
+        run_as = req.on_behalf_of
     try:
-        run = pipeline_service.start_run(name=name, user=user, args=req.args)
+        run = pipeline_service.start_run(name=name, user=run_as, args=req.args)
     except KeyError:
         raise HTTPException(status_code=404, detail="Pipeline not found")
-    audit_service.record(user, "pipeline.run.start", f"pipeline:{name}", f"run:{run['id']}")
+    detail = f"run:{run['id']}" + (f" delegated_by:{user}" if run_as != str(user) else "")
+    audit_service.record(run_as, "pipeline.run.start", f"pipeline:{name}", detail)
     return run
 
 
+def delegation_users() -> set[str]:
+    return {u.strip() for u in settings.pipeline_delegation_users.split(",") if u.strip()}
+
+
 @router.get("/pipeline-runs")
 def list_runs(
     pipeline: str | None = None,

backend/app/api/samples.py

@@ -0,0 +1,110 @@
+"""Sample Agents: browse, try, clone.
+
+Open to every authenticated user: a sample is published to be read and
+copied. What a user cannot do is reach a sample's runtime by any other path:
+the clone is the only publish that sets runtime = sample:<id>, and
+agent_service refuses the field on any agent not cloned from that sample.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from app.dependencies import Principal, get_current_user, require_admin
+from app.models.schemas import InvokeResponse, SampleCloneRequest, SampleSeedRequest, SampleTryRequest
+from app.services import invocation_service
+from app.services.audit_service import audit_service
+from app.services.governance_service import QuotaExceeded, SourceDisabled
+from app.services.agent_service import agent_service
+from app.services.samples_service import SEED_USER, samples_service
+
+router = APIRouter(prefix="/api/v1/samples", tags=["samples"])
+
+
+@router.get("")
+def list_samples(user: Principal = Depends(get_current_user)):
+    out = []
+    for s in samples_service.list_samples():
+        public = {k: v for k, v in s.items() if k not in ("readme", "agent", "eval_cases")}
+        public["status"] = samples_service.status(s)
+        out.append(public)
+    return out
+
+
+@router.get("/{sample_id}")
+def get_sample(sample_id: str, user: Principal = Depends(get_current_user)):
+    s = samples_service.get_sample(sample_id)
+    if not s:
+        raise HTTPException(status_code=404, detail="Sample not found")
+    return {**s, "status": samples_service.status(s)}
+
+
+@router.post("/{sample_id}/clone")
+def clone_sample(sample_id: str, req: SampleCloneRequest, user: Principal = Depends(get_current_user)):
+    s = samples_service.get_sample(sample_id)
+    if not s:
+        raise HTTPException(status_code=404, detail="Sample not found")
+    try:
+        agent = samples_service.clone(user=user, sample=s, name=req.name)
+    except PermissionError as e:
+        raise HTTPException(status_code=403, detail=str(e))
+    except (ValueError, KeyError) as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    audit_service.record(user, "sample.clone", f"sample:{sample_id}", f"agent:{agent['name']} v{agent['version']}")
+    return agent
+
+
+@router.post("/{sample_id}/try", response_model=InvokeResponse)
+def try_sample(sample_id: str, req: SampleTryRequest, user: Principal = Depends(get_current_user)):
+    """Invoke the sample's shared instance as the caller, through the governed
+    pipeline (the caller's quota, the ledger)."""
+    s = samples_service.get_sample(sample_id)
+    if not s:
+        raise HTTPException(status_code=404, detail="Sample not found")
+    shared = s.get("shared_agent_id")
+    if not shared:
+        raise HTTPException(status_code=409, detail="this sample has no shared instance yet (seed not run)")
+    try:
+        return invocation_service.invoke(
+            user=user,
+            source="api",
+            target=f"agent:{shared}",
+            prompt=req.prompt,
+            runtime_session_id=invocation_service.resolve_session_id(user, req.session_id),
+            memory_actor_id=invocation_service.resolve_memory_actor(user, ""),
+            ref=f"sample:{sample_id}",
+        )
+    except KeyError:
+        raise HTTPException(status_code=404, detail="shared instance not found; re-run the seed")
+    except (QuotaExceeded, SourceDisabled) as e:
+        raise HTTPException(status_code=429, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=f"Runtime invocation failed: {e}")
+
+
+@router.put("/{sample_id}")
+def seed_sample(sample_id: str, req: SampleSeedRequest, user: Principal = Depends(require_admin)):
+    """Seed or refresh one sample from its repository content (administrators).
+    The shared instance is published as the seed user on the sample's runtime,
+    so Try has something to call; a republish bumps its version, the id stays."""
+    if str(req.sample.get("id", "")) != sample_id:
+        raise HTTPException(status_code=400, detail="sample.id must match the path")
+    try:
+        item = samples_service.upsert(req.sample, agent=req.agent, readme=req.readme, eval_cases=req.eval_cases)
+    except (ValueError, KeyError) as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    status = samples_service.status(item)
+    shared = None
+    if req.publish_shared and status["cloneable"]:
+        a = item["agent"]
+        ref = f"sample:{sample_id}"
+        try:
+            shared = agent_service.publish(
+                user=SEED_USER, name=a["name"], description=a["description"], system_prompt=a["system_prompt"],
+                max_turns=a["max_turns"], mcp_server_names=a["mcp_server_names"], skill_names=a["skill_names"],
+                platform_version=a["platform_version"], source=ref, runtime=ref,
+            )
+        except (PermissionError, ValueError, KeyError) as e:
+            raise HTTPException(status_code=400, detail=f"shared instance: {e}")
+        samples_service.set_shared_agent(sample_id, shared["id"])
+        item["shared_agent_id"] = shared["id"]
+    audit_service.record(user, "sample.seed", f"sample:{sample_id}", f"shared={shared['id'] if shared else '-'}")
+    return {**item, "status": samples_service.status(item), "shared_instance": shared}

backend/app/api/sessions.py

@@ -54,7 +54,13 @@ def _session_config(item: dict) -> dict | None:
     ids_s = list(item.get("skill_ids", []))
     if not ids_m and not ids_s:
         return None
-    cfg = ecosystem_service.resolve_session_config(ids_m, ids_s)
+    try:
+        cfg = ecosystem_service.resolve_session_config(ids_m, ids_s)
+    except ValueError as e:
+        # An attachment the registry no longer knows (deleted after the
+        # session was created) is the caller's 400, not a 500: the ids were
+        # validated at creation, so only the registry can have moved.
+        raise HTTPException(status_code=400, detail=str(e))
     # Identity-forwarding attachments get the caller's token resolved at
     # warmup. Note the token has the IdP's normal lifetime — a workspace that
     # outlives it must be restarted to pick up a fresh one.
@@ -239,8 +245,10 @@ def stop_session(session_id: str, user: str = Depends(get_current_user)):
     session_service.set_status(user, session_id, "dormant")
     # The next connect mints a fresh grant, so drop this one now: a token
     # scraped out of the container stops working when the session does rather
-    # than lingering until its hour is up.
+    # than lingering until its hour is up. Same for the workspace refresh
+    # token, which otherwise had no end of life at all.
     llm_credentials_service.revoke(item["runtime_session_id"])
+    workspace_credentials_service.revoke(user, session_id, item["runtime_session_id"])
     item["status"] = "dormant"
     return _to_response(item)
 
@@ -252,6 +260,7 @@ def delete_session(session_id: str, user: str = Depends(get_current_user)):
         raise HTTPException(status_code=404, detail="Session not found")
     session_service.set_status(user, session_id, "terminated")
     llm_credentials_service.revoke(item["runtime_session_id"])
+    workspace_credentials_service.revoke(user, session_id, item["runtime_session_id"])
     audit_service.record(user, "session.delete", f"session:{item.get('name', session_id)}")
     return {"ok": True}