From 6a56e1c35b4952a684f7eb6d6071772a1b6541ee Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 29 Sep 2026 17:30:34 -0700 Subject: [PATCH 01/24] Simplify custom metric capture --- .../protocol/CreateCustomMetricDialog.tsx | 5 +- .../protocol/CustomMetricDialogs.test.tsx | 8 +- .../protocol/CustomMetricDialogs.tsx | 4 +- .../src/components/protocol/MetricsEditor.tsx | 15 ++- frontend/src/lib/agentOutputMetrics.ts | 6 - frontend/src/lib/experiment.test.ts | 15 +-- frontend/src/lib/experiment.ts | 2 +- frontend/src/lib/mcpToolMetrics.ts | 2 +- frontend/src/lib/measurementPlan.ts | 1 + frontend/src/lib/metricCatalog.ts | 29 +++-- frontend/src/lib/pythonScriptMetrics.ts | 6 - frontend/src/lib/reportedMetrics.test.ts | 8 +- frontend/src/types/experiments.ts | 14 +-- src/asaree/services/csv_export.py | 10 +- .../services/experiment_measurements.py | 29 ++--- src/asaree/services/experiment_run_results.py | 11 +- src/asaree/services/measurement_engine.py | 21 ++-- src/asaree/services/measurement_migration.py | 28 +++-- src/asaree/services/metrics.py | 44 ++----- src/asaree/services/reported_metrics.py | 107 ++---------------- tests/test_bdm_use_cases.py | 22 ++-- tests/test_experiment_run_results.py | 21 ++-- tests/test_measurement_engine.py | 2 +- tests/test_measurement_migration.py | 12 +- tests/test_metrics.py | 33 ++---- tests/test_reported_metrics.py | 18 ++- 26 files changed, 165 insertions(+), 308 deletions(-) diff --git a/frontend/src/components/protocol/CreateCustomMetricDialog.tsx b/frontend/src/components/protocol/CreateCustomMetricDialog.tsx index 52922b3b..cde4599e 100644 --- a/frontend/src/components/protocol/CreateCustomMetricDialog.tsx +++ b/frontend/src/components/protocol/CreateCustomMetricDialog.tsx @@ -40,10 +40,7 @@ export function CreateCustomMetricDialog({ const existingBinding = agentOutputBindingForMetric(experiment?.measurement_plan ?? null, metricId) ?? pythonScriptBindingForMetric(experiment?.measurement_plan ?? null, metricId) ?? mcpToolBindingForMetric(experiment?.measurement_plan ?? null, metricId) - const [draft] = useState(() => makeCustomMetric({ - name: '', - description: '', - })) + const [draft] = useState(() => makeCustomMetric({ name: '' })) const flushAutosaveRef = useRef<(() => void) | null>(null) const createMutation = useMutation({ mutationFn: async ({ metric, config }: { metric: typeof draft; config: CustomMetricProducerConfig }) => { diff --git a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx index cd6f7572..985b79cf 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx @@ -14,10 +14,6 @@ function renderFlow(graph: ProtocolGraph, onSave = vi.fn(), sourceContext?: Cust id: 'metric-1', name: '', kind: 'custom', - valueType: 'number', - direction: 'neutral', - aggregation: 'mean', - primary: false, } render( @@ -103,7 +99,7 @@ describe('CustomMetricFlow', () => { } as unknown as ProtocolGraph const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) render( { } as unknown as ProtocolGraph const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) render(

Custom metrics

Create metrics from eligible nodes on this canvas.

- {!customMetricDraft && } + {!customMetricDraft && }
{visibleCustomMetrics.length > 0 &&
{visibleCustomMetrics.map((metric) => { const binding = stagedBindingForMetric(metric.id) @@ -355,7 +358,7 @@ function MetricsDialog({ - {binding && metric.valueType === 'opaque' && } + {binding && }
})}} @@ -502,7 +505,7 @@ export function MetricsEditor({ ...nextPlan, metrics: nextPlan.metrics.map((definition) => { const designMetric = byId.get(definition.id) - return designMetric ? { ...definition, primary: designMetric.primary } : definition + return designMetric?.kind === 'runtime' ? { ...definition, primary: designMetric.primary } : definition }), } return { metrics: nextMetrics, measurementPlan: synchronizedPlan } diff --git a/frontend/src/lib/agentOutputMetrics.ts b/frontend/src/lib/agentOutputMetrics.ts index 14c799ca..76ca1886 100644 --- a/frontend/src/lib/agentOutputMetrics.ts +++ b/frontend/src/lib/agentOutputMetrics.ts @@ -39,12 +39,6 @@ export function upsertAgentOutputMetric( { id: metric.id, name: metric.name, - value_type: 'opaque', - direction: 'neutral', - aggregation: 'none', - primary: false, - description: metric.description, - unit: metric.unit, }, ], producers: [ diff --git a/frontend/src/lib/experiment.test.ts b/frontend/src/lib/experiment.test.ts index 56fc1be1..fee701c8 100644 --- a/frontend/src/lib/experiment.test.ts +++ b/frontend/src/lib/experiment.test.ts @@ -36,8 +36,8 @@ describe('displayFactorLevel', () => { }) }) -describe('reported scalar metrics', () => { - it('preserves an explicitly typed custom metric and uses it as primary', () => { +describe('custom metric observations', () => { + it('removes imported value semantics and does not use a custom metric as primary', () => { const metrics = normalizeDesignMetrics([{ id: 'pr-auc', name: 'pr_auc', @@ -49,16 +49,17 @@ describe('reported scalar metrics', () => { }]) const experiment = { design_spec: { metrics } } as Experiment - expect(metrics[0]).toMatchObject({ valueType: 'number', direction: 'maximize', primary: true }) - expect(primaryMetric(experiment)).toEqual({ key: 'pr_auc', direction: 'maximize', valueType: 'number' }) + expect(metrics[0]).toEqual({ id: 'pr-auc', name: 'pr_auc', kind: 'custom' }) + expect(primaryMetric(experiment)).toBeNull() }) - it('keeps an ordinary custom draft opaque and non-primary', () => { - expect(normalizeDesignMetrics([{ + it('keeps only the identity and label of a custom draft', () => { + const metric = normalizeDesignMetrics([{ name: 'Reviewer report', kind: 'custom', direction: 'maximize', primary: true, - }])[0]).toMatchObject({ valueType: 'opaque', direction: 'neutral', primary: false }) + }])[0] + expect(metric).toEqual({ id: metric.id, name: 'Reviewer report', kind: 'custom' }) }) }) diff --git a/frontend/src/lib/experiment.ts b/frontend/src/lib/experiment.ts index 811d8d44..b593dd81 100644 --- a/frontend/src/lib/experiment.ts +++ b/frontend/src/lib/experiment.ts @@ -303,7 +303,7 @@ export function pickMetricColumns(experiment: Experiment | undefined, replicates * must never choose the canvas top-bar result. */ export function primaryMetric(experiment: Experiment | undefined): { key: string; direction: 'maximize' | 'minimize'; valueType?: 'number' | 'boolean' | 'string' } | null { const metric = experiment?.design_spec?.metrics?.find((candidate) => candidate.primary && candidate.name) - if (!metric || metric.direction === 'neutral' || metric.valueType === 'string' || metric.valueType === 'opaque') return null + if (!metric || (metric.direction !== 'maximize' && metric.direction !== 'minimize') || metric.valueType === 'string' || metric.valueType === 'opaque') return null const key = metric.kind === 'runtime' && typeof metric.catalogKey === 'string' ? metric.catalogKey : metric.name return { key, direction: metric.direction, valueType: metric.valueType } } diff --git a/frontend/src/lib/mcpToolMetrics.ts b/frontend/src/lib/mcpToolMetrics.ts index fecdc979..cf4954bf 100644 --- a/frontend/src/lib/mcpToolMetrics.ts +++ b/frontend/src/lib/mcpToolMetrics.ts @@ -42,5 +42,5 @@ export function mcpToolBindingForMetric(plan: MeasurementPlan | null, metricId: export function upsertMcpToolMetric(plan: MeasurementPlan | null, metric: DesignMetric, source: McpToolSourceOption | undefined, toolName = ''): MeasurementPlan { if (!metric.id) throw new Error('An MCP Tool metric needs a stable id.') const current = plan ?? { metrics: [], producers: [], inputs: [] }; const id = `mcp-${metric.id}` - return { metrics: [...current.metrics.filter((item) => item.id !== metric.id), { id: metric.id, name: metric.name, value_type: 'opaque', direction: 'neutral', aggregation: 'none', primary: false, description: metric.description, unit: metric.unit }], producers: [...current.producers.filter((item) => item.id !== id), { id, producer_id: MCP_TOOL_PRODUCER_ID, kind: 'reported', outputs: { value: metric.id }, artifacts: [], config: { agent_node_id: source?.agentNodeId ?? '', mcp_node_id: source?.mcpNodeId ?? '', server_id: source?.serverId ?? '', tool_name: toolName } }], inputs: current.inputs.filter((item) => item.producer_binding_id !== id) } + return { metrics: [...current.metrics.filter((item) => item.id !== metric.id), { id: metric.id, name: metric.name }], producers: [...current.producers.filter((item) => item.id !== id), { id, producer_id: MCP_TOOL_PRODUCER_ID, kind: 'reported', outputs: { value: metric.id }, artifacts: [], config: { agent_node_id: source?.agentNodeId ?? '', mcp_node_id: source?.mcpNodeId ?? '', server_id: source?.serverId ?? '', tool_name: toolName } }], inputs: current.inputs.filter((item) => item.producer_binding_id !== id) } } diff --git a/frontend/src/lib/measurementPlan.ts b/frontend/src/lib/measurementPlan.ts index 27fc069c..1da046f6 100644 --- a/frontend/src/lib/measurementPlan.ts +++ b/frontend/src/lib/measurementPlan.ts @@ -157,6 +157,7 @@ export function localMetricReadinessPreview( } export function metricMetadata(metric: DesignMetric): string { + if (metric.kind === 'custom') return 'captured output' const catalog = metricCatalogEntry(metric.catalogKey) const unit = metric.unit || catalog?.unit || 'unitless' const aggregation = metric.valueType === 'boolean' ? 'rate' : (metric.aggregation ?? catalog?.aggregation ?? 'mean') diff --git a/frontend/src/lib/metricCatalog.ts b/frontend/src/lib/metricCatalog.ts index 06255364..c096d7c8 100644 --- a/frontend/src/lib/metricCatalog.ts +++ b/frontend/src/lib/metricCatalog.ts @@ -66,25 +66,28 @@ export function normalizeDesignMetrics(metrics: DesignMetric[] | undefined): Des const catalog = metricCatalogEntry(raw.catalogKey) if (raw.kind && !['runtime', 'custom'].includes(raw.kind)) return [] const kind = raw.kind ?? catalog?.kind ?? 'custom' - const scalarCustom = kind === 'custom' && raw.kind === 'custom' && (raw.valueType === 'number' || raw.valueType === 'boolean') - return [{ + const id = typeof raw.id === 'string' && raw.id ? raw.id : legacyId(raw) + const name = raw.name.trim() + if (kind === 'custom') return [{ id, name, kind }] + const metric: DesignMetric = { ...raw, - id: typeof raw.id === 'string' && raw.id ? raw.id : legacyId(raw), + id, catalogKey: catalog ? catalog.key : raw.catalogKey, - name: raw.name.trim(), + name, description: typeof raw.description === 'string' && raw.description.trim() ? raw.description.trim() : catalog?.shortDescription ?? `Legacy metric declaration for ${raw.name.trim()}.`, kind, - valueType: kind === 'custom' && !scalarCustom ? 'opaque' : raw.valueType ?? catalog?.valueType ?? 'number', - direction: kind === 'custom' && !scalarCustom ? 'neutral' : raw.direction === 'maximize' || raw.direction === 'minimize' || raw.direction === 'neutral' + valueType: raw.valueType ?? catalog?.valueType ?? 'number', + direction: raw.direction === 'maximize' || raw.direction === 'minimize' || raw.direction === 'neutral' ? raw.direction : (catalog?.defaultDirection ?? 'maximize'), - aggregation: kind === 'custom' && !scalarCustom ? 'none' : raw.valueType === 'boolean' ? 'mean' : (raw.aggregation === 'sum' ? 'sum' : raw.aggregation === 'none' ? 'none' : (catalog?.aggregation ?? 'mean')), - primary: kind === 'custom' && !scalarCustom ? false : Boolean(raw.primary), + aggregation: raw.valueType === 'boolean' ? 'mean' : (raw.aggregation === 'sum' ? 'sum' : raw.aggregation === 'none' ? 'none' : (catalog?.aggregation ?? 'mean')), + primary: Boolean(raw.primary), unit: raw.unit ?? catalog?.unit, scoring: undefined, - }] + } + return [metric] }) } @@ -103,17 +106,11 @@ export function makeCatalogMetric(entry: MetricCatalogEntry, isPrimary: boolean) } } -export function makeCustomMetric(input: { name: string; description: string; unit?: string }): DesignMetric { +export function makeCustomMetric(input: { name: string }): DesignMetric { return { id: newMetricId(), name: input.name.trim(), - description: input.description.trim(), kind: 'custom', - valueType: 'opaque', - direction: 'neutral', - aggregation: 'none', - primary: false, - unit: input.unit?.trim() || undefined, } } diff --git a/frontend/src/lib/pythonScriptMetrics.ts b/frontend/src/lib/pythonScriptMetrics.ts index f91ec81c..bdf5c8c8 100644 --- a/frontend/src/lib/pythonScriptMetrics.ts +++ b/frontend/src/lib/pythonScriptMetrics.ts @@ -52,12 +52,6 @@ export function upsertPythonScriptMetric( { id: metric.id, name: metric.name, - value_type: 'opaque', - direction: 'neutral', - aggregation: 'none', - primary: false, - description: metric.description, - unit: metric.unit, }, ], producers: [ diff --git a/frontend/src/lib/reportedMetrics.test.ts b/frontend/src/lib/reportedMetrics.test.ts index 2e495d88..13ffb6b9 100644 --- a/frontend/src/lib/reportedMetrics.test.ts +++ b/frontend/src/lib/reportedMetrics.test.ts @@ -10,10 +10,6 @@ const metric: DesignMetric = { id: 'quality', name: 'Quality', kind: 'custom', - valueType: 'opaque', - direction: 'neutral', - aggregation: 'none', - primary: false, } const graph = { @@ -29,10 +25,10 @@ const graph = { } as unknown as ProtocolGraph describe('reported custom metric plans', () => { - it('stores Agent output as an opaque, display-only metric', () => { + it('stores an Agent output metric without value semantics', () => { const plan = upsertAgentOutputMetric(null, metric, 'agent') - expect(plan.metrics[0]).toMatchObject({ value_type: 'opaque', direction: 'neutral', aggregation: 'none', primary: false }) + expect(plan.metrics[0]).toEqual({ id: 'quality', name: 'Quality' }) expect(plan.producers[0]).toMatchObject({ producer_id: 'asaree.agent_output', kind: 'reported', config: { agent_node_id: 'agent' } }) expect(plan.inputs).toEqual([]) expect(localMetricReadinessPreview(metric, plan, graph).ready).toBe(true) diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 5e745691..92fe91e5 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -33,8 +33,8 @@ export interface DesignMetric { kind?: 'runtime' | 'custom' valueType?: 'number' | 'boolean' | 'string' | 'opaque' unit?: string - primary: boolean - direction: 'maximize' | 'minimize' | 'neutral' + primary?: boolean + direction?: 'maximize' | 'minimize' | 'neutral' aggregation?: 'mean' | 'sum' | 'none' [key: string]: unknown } @@ -47,10 +47,10 @@ export interface MeasurementPlan { metrics: Array<{ id: string name: string - value_type: 'number' | 'boolean' | 'opaque' - direction: MeasurementDirection - aggregation: MeasurementAggregation - primary: boolean + value_type?: 'number' | 'boolean' | 'opaque' + direction?: MeasurementDirection + aggregation?: MeasurementAggregation + primary?: boolean description?: string unit?: string }> @@ -315,7 +315,7 @@ export interface ResultNodeRun { export interface MetricObservation { metric_id: string metric_name: string - value_type: 'number' | 'boolean' | 'opaque' + value_type?: 'number' | 'boolean' | 'opaque' status: ObservationStatus value: unknown error: string | null diff --git a/src/asaree/services/csv_export.py b/src/asaree/services/csv_export.py index d60f659a..289255d6 100644 --- a/src/asaree/services/csv_export.py +++ b/src/asaree/services/csv_export.py @@ -420,15 +420,13 @@ def result_rows_schema( { "name": key, "role": "reported" if key in reported_keys else "outcome", - "value_type": ( - "string" - if key in script_columns - else (metric_types or {}).get(key, "opaque" if key in reported_keys else "number") - ), **( {} if key in reported_keys - else {"cell_aggregation": (metric_aggregations or {}).get(key, "mean")} + else { + "value_type": (metric_types or {}).get(key, "number"), + "cell_aggregation": (metric_aggregations or {}).get(key, "mean"), + } ), }, *( diff --git a/src/asaree/services/experiment_measurements.py b/src/asaree/services/experiment_measurements.py index 3671f0f8..2f7115a4 100644 --- a/src/asaree/services/experiment_measurements.py +++ b/src/asaree/services/experiment_measurements.py @@ -121,20 +121,21 @@ async def validate_experiment_measurement_plan( ) ) continue - selected_semantics = ( - str(metric.get("name") or metric_id), - metric.get("valueType", "number"), - metric.get("direction", "maximize"), - "rate" if metric.get("valueType") == "boolean" else metric.get("aggregation", "mean"), - bool(metric.get("primary")), - ) - declared_semantics = ( - declared.name, - declared.value_type, - declared.direction, - declared.aggregation, - declared.primary, - ) + selected_semantics = (str(metric.get("name") or metric_id),) + declared_semantics = (declared.name,) + if metric.get("kind") == "runtime": + selected_semantics += ( + metric.get("valueType", "number"), + metric.get("direction", "maximize"), + "rate" if metric.get("valueType") == "boolean" else metric.get("aggregation", "mean"), + bool(metric.get("primary")), + ) + declared_semantics += ( + declared.value_type, + declared.direction, + declared.aggregation, + bool(declared.primary), + ) if selected_semantics != declared_semantics: reconciliation_issues.append( ValidationIssue( diff --git a/src/asaree/services/experiment_run_results.py b/src/asaree/services/experiment_run_results.py index 42641f58..cbd71933 100644 --- a/src/asaree/services/experiment_run_results.py +++ b/src/asaree/services/experiment_run_results.py @@ -95,8 +95,8 @@ def _merge_legacy_facets( ) -> LegacyResultFacets: """Add compatibility facts without duplicating current measurements. - Opaque reported metrics are JSON values, which legacy migration also - recognizes as non-scalar historical data. A matching current observation + Reported metrics may contain arbitrary JSON values, which legacy migration + also recognizes as non-scalar historical data. A matching current observation is authoritative, so it must not leave a duplicate ``legacy_values`` entry that would hide the declared Results column. """ @@ -139,6 +139,7 @@ def _primary_metric(design_spec: dict[str, Any] | None) -> tuple[str | None, str for metric in metrics: if ( isinstance(metric, dict) + and metric.get("kind") == "runtime" and metric.get("primary") and isinstance(metric.get("name"), str) and metric.get("valueType", "number") in {"number", "boolean"} @@ -178,7 +179,7 @@ def _declared_metric_types(design_spec: dict[str, Any] | None) -> dict[str, str] types: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") - if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: + if isinstance(key, str) and metric.get("valueType") in {"number", "boolean"}: types[key] = metric["valueType"] return types @@ -188,7 +189,7 @@ def _declared_metric_aggregations(design_spec: dict[str, Any] | None) -> dict[st aggregations: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") - if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: + if isinstance(key, str) and metric.get("valueType") in {"number", "boolean"}: aggregations[key] = metric["aggregation"] return aggregations @@ -198,7 +199,7 @@ def _declared_metric_directions(design_spec: dict[str, Any] | None) -> dict[str, directions: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") - if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: + if isinstance(key, str) and metric.get("valueType") in {"number", "boolean"}: directions[key] = metric["direction"] return directions diff --git a/src/asaree/services/measurement_engine.py b/src/asaree/services/measurement_engine.py index 9a36618f..3f706b4d 100644 --- a/src/asaree/services/measurement_engine.py +++ b/src/asaree/services/measurement_engine.py @@ -39,10 +39,10 @@ class MetricDefinition: id: str name: str - value_type: MetricValueType - direction: MetricDirection - aggregation: MetricAggregation - primary: bool = False + value_type: MetricValueType | None = None + direction: MetricDirection | None = None + aggregation: MetricAggregation | None = None + primary: bool | None = None description: str | None = None unit: str | None = None @@ -96,7 +96,7 @@ def parse_measurement_plan(document: Any) -> MeasurementPlan: def normalize_measurement_plan(document: Any) -> dict[str, Any]: """Return a JSON-compatible canonical plan document.""" plan = parse_measurement_plan(document) - ensure_at_most_one_primary(tuple(metric.primary for metric in plan.metrics)) + ensure_at_most_one_primary(tuple(bool(metric.primary) for metric in plan.metrics)) return cast( dict[str, Any], _MEASUREMENT_PLAN_ADAPTER.dump_python(plan, mode="json", exclude_none=True), @@ -203,7 +203,7 @@ class ProducerProvenance: class MetricObservation: metric_id: str metric_name: str - value_type: MetricValueType + value_type: MetricValueType | None status: ObservationStatus value: ScalarValue | None error: str | None @@ -238,7 +238,7 @@ def to_document(self) -> dict[str, Any]: { "metric_id": item.metric_id, "metric_name": item.metric_name, - "value_type": item.value_type, + **({"value_type": item.value_type} if item.value_type is not None else {}), "status": item.status, "value": item.value, "error": item.error, @@ -415,7 +415,7 @@ def validate_measurement_plan_structure( ) ) - primary_count = sum(metric.primary for metric in plan.metrics) + primary_count = sum(bool(metric.primary) for metric in plan.metrics) if require_primary and plan.metrics and primary_count == 0: issues.append( ValidationIssue( @@ -716,9 +716,10 @@ def normalize_binding(binding: ProducerBinding, produced: ProducerResult) -> Mea output_key, ProducedObservation.unavailable(f"Producer did not return output {output_key!r}."), ) - if outcome.status == "measured" and not is_scalar_value(outcome.value, metric.value_type): + value_type = metric.value_type or "opaque" + if outcome.status == "measured" and not is_scalar_value(outcome.value, value_type): outcome = ProducedObservation.failed( - f"Producer returned an invalid {metric.value_type} for output {output_key!r}." + f"Producer returned an invalid {value_type} for output {output_key!r}." ) normalized_observations.append( MetricObservation( diff --git a/src/asaree/services/measurement_migration.py b/src/asaree/services/measurement_migration.py index 053dee18..5b7e89cd 100644 --- a/src/asaree/services/measurement_migration.py +++ b/src/asaree/services/measurement_migration.py @@ -59,23 +59,31 @@ def normalize_experiment_measurement_plan(document: Any, metrics: Any) -> dict[s reported_bindings = tuple(binding for binding in plan.producers if binding.producer_id in reported_ids) reported_binding_ids = {binding.id for binding in reported_bindings} reported_metric_ids = {metric_id for binding in reported_bindings for metric_id in binding.outputs.values()} - projected_metric_ids = { - metric_id - for binding in reported_bindings - for output_key, metric_id in binding.outputs.items() - if isinstance(binding.config.get("projections"), Mapping) - and output_key in binding.config["projections"] - } normalized = replace( plan, metrics=tuple( - replace(metric, value_type="opaque", direction="neutral", aggregation="none", primary=False) - if metric.id in reported_metric_ids and metric.id not in projected_metric_ids + replace( + metric, + value_type=None, + direction=None, + aggregation=None, + primary=None, + description=None, + unit=None, + ) + if metric.id in reported_metric_ids else metric for metric in plan.metrics ), producers=tuple( - replace(binding, kind="reported", artifacts=()) if binding.id in reported_binding_ids else binding + replace( + binding, + kind="reported", + artifacts=(), + config={key: value for key, value in binding.config.items() if key != "projections"}, + ) + if binding.id in reported_binding_ids + else binding for binding in plan.producers ), inputs=tuple(item for item in plan.inputs if item.producer_binding_id not in reported_binding_ids), diff --git a/src/asaree/services/metrics.py b/src/asaree/services/metrics.py index c2621b7b..f57cd2a8 100644 --- a/src/asaree/services/metrics.py +++ b/src/asaree/services/metrics.py @@ -4,7 +4,6 @@ import uuid from copy import deepcopy -from math import isfinite from typing import Any from asaree.services.measurement_engine import ensure_at_most_one_primary, parse_measurement_plan @@ -203,13 +202,13 @@ def design_metrics_from_measurement_plan(document: Any) -> list[dict[str, Any]]: "id": metric.id, "name": metric.name, "kind": "runtime" if catalog_key is not None else "custom", - "valueType": metric.value_type, - "direction": metric.direction, - "aggregation": metric.aggregation, - "primary": metric.primary, } if catalog_key is not None: declaration["catalogKey"] = catalog_key + declaration["valueType"] = metric.value_type + declaration["direction"] = metric.direction + declaration["aggregation"] = metric.aggregation + declaration["primary"] = bool(metric.primary) if metric.description is not None: declaration["description"] = metric.description if metric.unit is not None: @@ -276,6 +275,9 @@ def normalize_metrics(metrics: Any, *, validate_custom_names: bool = False) -> l metric["kind"] = ( metric.get("kind") if metric.get("kind") in _KINDS else catalog["kind"] if catalog else "custom" ) + if metric["kind"] == "custom": + normalized.append({"id": metric["id"], "name": name, "kind": "custom"}) + continue metric["valueType"] = ( metric.get("valueType") if metric.get("valueType") in _VALUE_TYPES @@ -300,20 +302,6 @@ def normalize_metrics(metrics: Any, *, validate_custom_names: bool = False) -> l if catalog else "mean" ) - # Explicit scalar custom declarations are produced by structured - # reported-output projections. Legacy/custom drafts without an - # explicit scalar type keep the original opaque display-only - # semantics. - scalar_custom = ( - metric["kind"] == "custom" - and raw.get("kind") == "custom" - and raw.get("valueType") in {"number", "boolean"} - ) - if metric["kind"] == "custom" and not scalar_custom: - metric["valueType"] = "opaque" - metric["direction"] = "neutral" - metric["aggregation"] = "none" - metric["primary"] = False if metric.get("unit") is None and catalog and catalog.get("unit"): metric["unit"] = catalog["unit"] metric.pop("scoring", None) @@ -322,7 +310,7 @@ def normalize_metrics(metrics: Any, *, validate_custom_names: bool = False) -> l # "Untitled custom metric"), so the measurement-plan validator blocks # production while still preserving every draft for repair. normalized.append(metric) - ensure_at_most_one_primary(tuple(bool(metric["primary"]) for metric in normalized)) + ensure_at_most_one_primary(tuple(bool(metric.get("primary")) for metric in normalized)) return normalized @@ -369,16 +357,15 @@ def normalize_design_spec( def declared_primary_metric(metrics: Any) -> dict[str, Any] | None: """Return the one explicitly primary design metric, if one is declared.""" - return next((metric for metric in normalize_metrics(metrics) if metric["primary"]), None) + return next((metric for metric in normalize_metrics(metrics) if metric.get("primary")), None) def validate_metric_values(metrics: Any, values: dict[str, Any] | None) -> dict[str, Any]: """Validate declared outcome values at the API boundary. - Metric declarations are dynamic JSONB, so this is intentionally a - declaration-driven validation rather than a database-column constraint. + Custom metrics capture producer output without imposing a value type. Unknown keys stay allowed for backwards-compatible externally reported - metrics; declared custom metrics must honor their selected value type. + metrics. """ declared = {metric["name"]: metric for metric in normalize_metrics(metrics) if metric["kind"] != "runtime"} normalized: dict[str, Any] = {} @@ -387,14 +374,5 @@ def validate_metric_values(metrics: Any, values: dict[str, Any] | None) -> dict[ if metric is None: normalized[key] = value continue - value_type = metric["valueType"] - if value_type == "boolean": - if not isinstance(value, bool): - raise ValueError(f"Metric {key!r} must be Boolean.") - elif value_type == "number": - if isinstance(value, bool) or not isinstance(value, int | float) or not isfinite(float(value)): - raise ValueError(f"Metric {key!r} must be a finite number.") - elif value_type == "string" and not isinstance(value, str): - raise ValueError(f"Metric {key!r} must be text.") normalized[key] = value return normalized diff --git a/src/asaree/services/reported_metrics.py b/src/asaree/services/reported_metrics.py index 214c8028..6dd83afe 100644 --- a/src/asaree/services/reported_metrics.py +++ b/src/asaree/services/reported_metrics.py @@ -7,7 +7,6 @@ from __future__ import annotations -import json import uuid from collections.abc import Mapping, Sequence from copy import deepcopy @@ -26,7 +25,6 @@ ProducerProvenance, ValidationIssue, ValidationReport, - is_scalar_value, preserved_binding_severity, ) from asaree.services.protocol_graph import directly_connected_tool_pair, node_map @@ -40,52 +38,6 @@ _SCRIPT_TOOL = "run_wired_script" -def _structured_value(value: Any) -> Any: - """Decode a JSON transport string while preserving already-typed values.""" - if not isinstance(value, str): - return value - try: - return json.loads(value) - except json.JSONDecodeError: - return value - - -def _project_reported_value(value: Any, projection: Any) -> tuple[Any, str | None]: - """Select one typed value from a reported JSON document. - - Projection paths use dot-separated object keys (and integer list indexes). - ``length`` is intentionally the only transform: it covers collection counts - without turning measurement plans into an expression language. - """ - if not isinstance(projection, Mapping): - return None, "The reported metric projection is invalid." - path = projection.get("path") - if not isinstance(path, str): - return None, "The reported metric projection has no path." - current = _structured_value(value) - segments = ( - [part.replace("~1", "/").replace("~0", "~") for part in path.removeprefix("/").split("/")] - if path.startswith("/") - else [part for part in path.split(".") if part] - ) - for segment in segments: - if isinstance(current, Mapping) and segment in current: - current = current[segment] - elif isinstance(current, Sequence) and not isinstance(current, str): - try: - current = current[int(segment)] - except (ValueError, IndexError): - return None, f"Reported metric path {path!r} is unavailable." - else: - return None, f"Reported metric path {path!r} is unavailable." - transform = projection.get("transform") - if transform is None: - return current, None - if transform == "length" and isinstance(current, Mapping | Sequence) and not isinstance(current, str): - return len(current), None - return None, f"Reported metric transform {transform!r} cannot be applied." - - def _tool_matches(recorded: Any, configured: str) -> bool: return isinstance(recorded, str) and (recorded == configured or recorded.endswith(f".{configured}")) @@ -190,16 +142,11 @@ async def collect_reported_metrics( if isinstance(node_run, Mapping) else None ) - structured_agent_output = ( - node_run.get("last_successful_payload", node_run.get("payload")) - if isinstance(node_run, Mapping) - else None - ) agent_output_available = ( binding.producer_id == AGENT_OUTPUT_PRODUCER_ID and isinstance(node_run, Mapping) and (node_run.get("status") == "completed" or "last_successful_output_text" in node_run) - and (successful_output is not None or structured_agent_output is not None) + and successful_output is not None ) call = ( None if binding.producer_id == AGENT_OUTPUT_PRODUCER_ID else await _last_matching_call(run, binding, graph) @@ -225,37 +172,24 @@ async def collect_reported_metrics( ), }, ) - projections = binding.config.get("projections") - projections = projections if isinstance(projections, Mapping) else {} - for output_key, metric_id in binding.outputs.items(): + for _output_key, metric_id in binding.outputs.items(): metric = metrics.get(metric_id) if metric is None: continue raw_value = ( - structured_agent_output - if agent_output_available and output_key in projections and structured_agent_output is not None - else successful_output + successful_output if agent_output_available else call.get("result") if call is not None else None ) - value, projection_error = ( - _project_reported_value(raw_value, projections[output_key]) - if output_key in projections - else (raw_value, None) - ) - type_error = ( - None - if projection_error is not None or not measured or is_scalar_value(value, metric.value_type) - else f"Projected value does not match declared type {metric.value_type!r}." - ) - observation_error = projection_error or type_error + value = raw_value + observation_error = None observations.append( MetricObservation( metric_id=metric.id, metric_name=metric.name, - value_type=metric.value_type, + value_type=None, value=value if observation_error is None else None, status="measured" if measured and observation_error is None else "unavailable", error=( @@ -361,39 +295,16 @@ async def validate_reported_measurement_plan( if binding.producer_id not in REPORTED_PRODUCER_IDS: continue path = f"producers[{index}]" - raw_projections = binding.config.get("projections") - projections = raw_projections if isinstance(raw_projections, Mapping) else {} - if raw_projections is not None and not isinstance(raw_projections, Mapping): - issues.append( - ValidationIssue( - "reported_projections_invalid", - "Reported metric projections must be an object keyed by producer output.", - f"{path}.config.projections", - ) - ) for output_key, metric_id in binding.outputs.items(): metric = metrics.get(metric_id) - projection = projections.get(output_key) - if metric is not None and metric.value_type != "opaque" and projection is None: + if metric is not None and metric.value_type is not None: issues.append( ValidationIssue( - "reported_projection_missing", - f"Scalar reported metric {metric.name!r} requires a structured field projection.", + "reported_metric_semantics_not_supported", + f"Custom metric {metric.name!r} captures the producer output without value semantics.", f"{path}.outputs.{output_key}", ) ) - if projection is not None and ( - not isinstance(projection, Mapping) - or not isinstance(projection.get("path"), str) - or projection.get("transform") not in {None, "length"} - ): - issues.append( - ValidationIssue( - "reported_projection_invalid", - "A reported metric projection needs a path and an optional supported transform.", - f"{path}.config.projections.{output_key}", - ) - ) if agent_issue := _agent_issue(binding, index, graph, preserved_binding_ids): issues.append(agent_issue) if binding.producer_id == AGENT_OUTPUT_PRODUCER_ID: diff --git a/tests/test_bdm_use_cases.py b/tests/test_bdm_use_cases.py index b93e8b10..0319bf26 100644 --- a/tests/test_bdm_use_cases.py +++ b/tests/test_bdm_use_cases.py @@ -14,7 +14,7 @@ USE_CASES = sorted(BDM.glob("myocardial-*-v0.8.0.json")) -def test_myocardial_use_cases_declare_the_same_analysis_ready_measurements() -> None: +def test_myocardial_use_cases_normalize_reported_metrics_as_named_observations() -> None: documents = [json.loads(path.read_text()) for path in USE_CASES] expected_metrics = documents[0]["design_spec"]["metrics"] expected_plan_metrics = documents[0]["measurement_plan"]["metrics"] @@ -26,10 +26,19 @@ def test_myocardial_use_cases_declare_the_same_analysis_ready_measurements() -> assert metrics == expected_metrics assert plan["metrics"] == expected_plan_metrics assert normalize_metrics(metrics) == design_metrics_from_measurement_plan(plan) - assert normalize_experiment_measurement_plan(plan, metrics) == plan + normalized_plan = normalize_experiment_measurement_plan(plan, metrics) + reported = next( + producer for producer in normalized_plan["producers"] if producer["id"] == "model-evaluation-source" + ) + reported_metric_ids = set(reported["outputs"].values()) + assert "projections" not in reported["config"] + assert all( + set(metric) <= {"id", "name"} + for metric in normalized_plan["metrics"] + if metric["id"] in reported_metric_ids + ) + assert not any(metric.get("primary") for metric in normalize_metrics(metrics) if metric["kind"] == "custom") - primary = [metric for metric in metrics if metric["primary"]] - assert [(metric["name"], metric["direction"]) for metric in primary] == [("pr_auc", "maximize")] runtime = next(producer for producer in plan["producers"] if producer["producer_id"] == "asaree.runtime") assert set(runtime["outputs"]) == { "cost_usd", @@ -43,11 +52,6 @@ def test_myocardial_use_cases_declare_the_same_analysis_ready_measurements() -> "critic_rejections", "critic_approvals", } - score = next(producer for producer in plan["producers"] if producer["id"] == "model-evaluation-source") - assert score["config"]["projections"]["pr_auc"] == {"path": "test_metrics.average_precision"} - assert score["config"]["projections"]["accuracy_at_0_5"] == { - "path": "/test_metrics/metrics_at_0.5/accuracy" - } def test_myocardial_output_parser_types_match_the_prompted_payloads() -> None: diff --git a/tests/test_experiment_run_results.py b/tests/test_experiment_run_results.py index 2010d73b..51760a67 100644 --- a/tests/test_experiment_run_results.py +++ b/tests/test_experiment_run_results.py @@ -78,7 +78,7 @@ def test_declared_runtime_metrics_are_projected_from_execution_telemetry() -> No assert _primary_metric(spec) == ("cost_usd", "maximize") -def test_builtin_and_reported_scalar_metrics_declare_aggregations() -> None: +def test_only_builtin_metrics_declare_aggregations() -> None: assert _declared_metric_aggregations( { "metrics": [ @@ -92,10 +92,10 @@ def test_builtin_and_reported_scalar_metrics_declare_aggregations() -> None: }, ] } - ) == {"Quality": "mean", "cost_usd": "sum"} + ) == {"cost_usd": "sum"} -def test_builtin_and_reported_scalar_metrics_declare_ranking_directions() -> None: +def test_only_builtin_metrics_declare_ranking_directions() -> None: assert _declared_metric_directions( { "metrics": [ @@ -109,7 +109,7 @@ def test_builtin_and_reported_scalar_metrics_declare_ranking_directions() -> Non }, ] } - ) == {"Quality": "maximize", "cost_usd": "minimize"} + ) == {"cost_usd": "minimize"} def test_cell_metric_aggregations_apply_the_declared_operation() -> None: @@ -166,10 +166,10 @@ def test_results_csv_keeps_an_empty_column_for_an_unreported_custom_metric() -> design_spec=design_spec, ) reported = next(column for column in schema["columns"] if column["name"] == "Reviewer report") - assert reported == {"name": "Reviewer report", "role": "reported", "value_type": "opaque"} + assert reported == {"name": "Reviewer report", "role": "reported"} -def test_results_csv_describes_a_reported_scalar_as_numeric() -> None: +def test_results_csv_ignores_legacy_custom_metric_semantics() -> None: design_spec = { "metrics": [ { @@ -195,12 +195,8 @@ def test_results_csv_describes_a_reported_scalar_as_numeric() -> None: schema = result_rows_schema(rows, metric_types={"pr_auc": "number"}, design_spec=design_spec) reported = next(column for column in schema["columns"] if column["name"] == "pr_auc") - assert reported == { - "name": "pr_auc", - "role": "reported", - "value_type": "number", - } - assert _primary_metric(design_spec) == ("pr_auc", "maximize") + assert reported == {"name": "pr_auc", "role": "reported"} + assert _primary_metric(design_spec) == (None, None) def test_results_csv_projects_script_stdout_and_execution_metadata() -> None: @@ -255,7 +251,6 @@ def test_results_csv_projects_script_stdout_and_execution_metadata() -> None: assert columns["Script custom metric"] == { "name": "Script custom metric", "role": "reported", - "value_type": "string", } assert columns["Script custom metric__raw_result"]["value_type"] == "json" assert columns["Script custom metric__exit_code"]["value_type"] == "integer" diff --git a/tests/test_measurement_engine.py b/tests/test_measurement_engine.py index 0abb89fd..f8a6b140 100644 --- a/tests/test_measurement_engine.py +++ b/tests/test_measurement_engine.py @@ -108,7 +108,7 @@ def test_measurement_plan_document_round_trips_in_normalized_collections() -> No **document, "metrics": [{key: value for key, value in document["metrics"][0].items() if key != "primary"}], } - assert normalize_measurement_plan(without_primary)["metrics"][0]["primary"] is False + assert "primary" not in normalize_measurement_plan(without_primary)["metrics"][0] multiple_primary = { **document, diff --git a/tests/test_measurement_migration.py b/tests/test_measurement_migration.py index 3bc6f7e0..d5652902 100644 --- a/tests/test_measurement_migration.py +++ b/tests/test_measurement_migration.py @@ -16,7 +16,7 @@ def test_legacy_declarations_are_not_migrated_into_measurement_plans() -> None: assert normalize_experiment_measurement_plan(None, metrics) == {"metrics": [], "producers": [], "inputs": []} -def test_projected_reported_metrics_keep_their_scalar_contract() -> None: +def test_reported_metrics_drop_scalar_semantics_and_projections() -> None: plan = normalize_experiment_measurement_plan( { "metrics": [ @@ -46,14 +46,8 @@ def test_projected_reported_metrics_keep_their_scalar_contract() -> None: (), ) - assert plan["metrics"][0] == { - "id": "pr-auc", - "name": "pr_auc", - "value_type": "number", - "direction": "maximize", - "aggregation": "mean", - "primary": True, - } + assert plan["metrics"][0] == {"id": "pr-auc", "name": "pr_auc"} + assert plan["producers"][0]["config"] == {"agent_node_id": "agent"} def test_historical_values_and_artifacts_gain_legacy_unknown_provenance() -> None: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index befbefb1..ed85c276 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -44,17 +44,12 @@ def test_frontend_metric_catalog_matches_the_backend_contract() -> None: ) -def test_legacy_metric_is_normalized_as_an_opaque_display_only_custom_metric() -> None: +def test_legacy_metric_is_normalized_as_a_named_custom_observation() -> None: legacy = [{"name": "Accuracy", "primary": True, "direction": "maximize"}] first = normalize_metrics(legacy) second = normalize_metrics(legacy) assert first[0]["id"] == second[0]["id"] - assert first[0]["kind"] == "custom" - assert first[0]["description"] == "Legacy metric declaration for Accuracy." - assert first[0]["valueType"] == "opaque" - assert first[0]["direction"] == "neutral" - assert first[0]["aggregation"] == "none" - assert first[0]["primary"] is False + assert first[0] == {"id": first[0]["id"], "name": "Accuracy", "kind": "custom"} def test_legacy_metric_id_is_stable_when_an_unrelated_declaration_is_inserted() -> None: @@ -73,7 +68,7 @@ def test_normalization_preserves_an_explicitly_primary_less_metric_set() -> None ] ) - assert [metric["primary"] for metric in normalized] == [False, False] + assert all("primary" not in metric for metric in normalized) def test_normalization_rejects_multiple_primary_metrics() -> None: @@ -169,7 +164,7 @@ def test_design_metrics_are_derived_from_a_caller_supplied_plan() -> None: ] -def test_explicit_scalar_custom_metric_values_are_type_checked() -> None: +def test_custom_metric_values_are_preserved_without_type_checks() -> None: metrics = [ {"name": "Passed", "kind": "custom", "valueType": "boolean", "primary": True}, {"name": "Score", "kind": "custom", "valueType": "number", "primary": False}, @@ -178,26 +173,22 @@ def test_explicit_scalar_custom_metric_values_are_type_checked() -> None: "Passed": True, "Score": 0.8, } - with pytest.raises(ValueError, match="must be Boolean"): - validate_metric_values(metrics, {"Passed": 1}) - with pytest.raises(ValueError, match="finite number"): - validate_metric_values(metrics, {"Score": {"grade": "A"}}) + assert validate_metric_values(metrics, {"Passed": 1, "Score": {"grade": "A"}}) == { + "Passed": 1, + "Score": {"grade": "A"}, + } -def test_explicit_custom_boolean_declarations_remain_scalar() -> None: +def test_explicit_custom_semantics_are_removed() -> None: metrics = normalize_metrics( [{"name": "Passed", "kind": "custom", "valueType": "boolean", "aggregation": "sum", "primary": True}] ) - assert metrics[0]["valueType"] == "boolean" - assert metrics[0]["aggregation"] == "mean" - assert metrics[0]["primary"] is True + assert metrics[0] == {"id": metrics[0]["id"], "name": "Passed", "kind": "custom"} -def test_custom_draft_without_an_explicit_scalar_type_remains_opaque() -> None: +def test_custom_draft_has_no_value_semantics() -> None: metrics = normalize_metrics([{"name": "Reviewer report", "kind": "custom", "primary": True}]) - assert metrics[0]["valueType"] == "opaque" - assert metrics[0]["aggregation"] == "none" - assert metrics[0]["primary"] is False + assert metrics[0] == {"id": metrics[0]["id"], "name": "Reviewer report", "kind": "custom"} def test_duplicate_custom_labels_remain_persistable_drafts() -> None: diff --git a/tests/test_reported_metrics.py b/tests/test_reported_metrics.py index d8958dac..a3819261 100644 --- a/tests/test_reported_metrics.py +++ b/tests/test_reported_metrics.py @@ -14,10 +14,6 @@ def _plan(producer_id: str, config: dict) -> object: { "id": "quality", "name": "Quality", - "value_type": "opaque", - "direction": "neutral", - "aggregation": "none", - "primary": False, } ], "producers": [ @@ -199,7 +195,7 @@ async def test_agent_output_report_captures_completed_final_output(output: str) @pytest.mark.asyncio -async def test_agent_output_projection_reads_the_structured_parser_payload() -> None: +async def test_agent_output_captures_the_complete_text_without_projection() -> None: plan = parse_measurement_plan( { "metrics": [ @@ -243,11 +239,12 @@ async def test_agent_output_projection_reads_the_structured_parser_payload() -> result = await collect_reported_metrics(run, plan, {"nodes": [], "edges": []}) assert result.observations[0].status == "measured" - assert result.observations[0].value == 2 + assert result.observations[0].value == "summary" + assert result.observations[0].value_type is None @pytest.mark.asyncio -async def test_mcp_projection_supports_json_pointer_keys_containing_dots( +async def test_mcp_metric_captures_the_complete_tool_result_without_projection( monkeypatch: pytest.MonkeyPatch, ) -> None: plan = parse_measurement_plan( @@ -304,7 +301,8 @@ async def test_mcp_projection_supports_json_pointer_keys_containing_dots( result = await collect_reported_metrics(run, plan, {"nodes": [], "edges": []}) assert result.observations[0].status == "measured" - assert result.observations[0].value == 0.81 + assert result.observations[0].value == '{"test_metrics":{"metrics_at_0.5":{"accuracy":0.81}}}' + assert result.observations[0].value_type is None @pytest.mark.asyncio @@ -374,7 +372,7 @@ async def test_agent_output_plan_requires_an_active_agent() -> None: @pytest.mark.asyncio -async def test_scalar_reported_metric_requires_a_projection() -> None: +async def test_reported_metric_rejects_value_semantics() -> None: plan = parse_measurement_plan( { "metrics": [ @@ -404,7 +402,7 @@ async def test_scalar_reported_metric_requires_a_projection() -> None: {"nodes": [{"id": "agent", "type": "agent", "data": {}}]}, ) - assert [issue.code for issue in report.issues] == ["reported_projection_missing"] + assert [issue.code for issue in report.issues] == ["reported_metric_semantics_not_supported"] async def _async_value(value): From 966270d70a35db55f2963a28e30c521a9199015e Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 30 Sep 2026 09:50:52 -0700 Subject: [PATCH 02/24] feat: score replicates with a deterministic Tool Step and code-computed metrics - Add a tool_step protocol node that calls one MCP tool directly with the upstream payload and wired Script code: optional XGBoost hyperparameter sanitizer, script/payload hash verification, provenance recorded before the call. - Add derived metric producers: asaree.tool_step (scalar result fields by dotted path, e.g. test_metrics.*) and asaree.feature_pipeline (DC/FTE/FS feature counts computed like the notebook's process_metrics). - Frontend: Tool Step node, inspector, connection rules, layout, accent (--node-11), and metrics-editor display for derived producers. - BDM myocardial publications: replace the score agent/parser with the Tool Step, ReAct max_iterations 12, 20 replicates; update README/tests. --- .../src/components/protocol/AddNodePanel.tsx | 9 +- .../protocol/CustomMetricDialogs.tsx | 2 +- .../src/components/protocol/MetricsEditor.tsx | 13 +- .../components/protocol/ProtocolCanvas.tsx | 25 +- .../components/protocol/TestRunResults.tsx | 8 +- .../protocol/ToolStepNodeInspector.tsx | 184 +++++++ .../protocol/connectionValidation.test.ts | 14 + .../protocol/connectionValidation.ts | 14 +- frontend/src/components/protocol/layout.ts | 21 +- .../protocol/nodes/ToolStepNode.tsx | 86 ++++ frontend/src/index.css | 5 + frontend/src/lib/measurementPlan.ts | 23 + frontend/src/lib/nodeAccent.ts | 4 + frontend/src/lib/nodeNames.ts | 1 + frontend/src/types/protocols.ts | 36 ++ publications/BDM/README.md | 59 ++- publications/BDM/import_use_case.py | 2 +- .../BDM/myocardial-anthropic-v0.8.0.json | 480 ++++++------------ .../BDM/myocardial-azure-foundry-v0.8.0.json | 480 ++++++------------ .../BDM/myocardial-openai-v0.8.0.json | 480 ++++++------------ src/asaree/services/derived_metrics.py | 262 ++++++++++ .../services/experiment_measurements.py | 4 + src/asaree/services/measurement_migration.py | 8 +- src/asaree/services/protocol_execution.py | 54 ++ src/asaree/services/reported_metrics.py | 32 +- src/asaree/services/tool_steps.py | 464 +++++++++++++++++ tests/test_bdm_use_cases.py | 37 +- tests/test_tool_steps.py | 242 +++++++++ 28 files changed, 2018 insertions(+), 1031 deletions(-) create mode 100644 frontend/src/components/protocol/ToolStepNodeInspector.tsx create mode 100644 frontend/src/components/protocol/nodes/ToolStepNode.tsx create mode 100644 src/asaree/services/derived_metrics.py create mode 100644 src/asaree/services/tool_steps.py create mode 100644 tests/test_tool_steps.py diff --git a/frontend/src/components/protocol/AddNodePanel.tsx b/frontend/src/components/protocol/AddNodePanel.tsx index d7b72186..fbaa9f33 100644 --- a/frontend/src/components/protocol/AddNodePanel.tsx +++ b/frontend/src/components/protocol/AddNodePanel.tsx @@ -1,5 +1,5 @@ import { useState } from 'react' -import { ArrowRight, Atom, BookMarked, Bot, Braces, BrainCircuit, Cloud, Code2, Database, FileText, HardDrive, Repeat2, Route, ScrollText, ShieldCheck, Server, Sparkles, X } from 'lucide-react' +import { ArrowRight, Atom, BookMarked, Bot, Braces, BrainCircuit, Cloud, Code2, Cog, Database, FileText, HardDrive, Repeat2, Route, ScrollText, ShieldCheck, Server, Sparkles, X } from 'lucide-react' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { DATASET_BROWSE } from './datasetCatalog' @@ -56,6 +56,13 @@ const NODE_CATALOG = [ description: "Reviews an upstream Agent's output, requests revisions", icon: ShieldCheck, }, + { + type: 'tool_step', + category: 'tools', + label: 'Tool Step', + description: "Calls one MCP tool directly with the upstream node's payload -- deterministic, no model", + icon: Cog, + }, { type: 'model_anthropic', category: 'models', label: 'Anthropic', description: "An Agent or Critic Gate's model, temperature, and parameters", icon: Sparkles }, { type: 'model_openai', category: 'models', label: 'OpenAI', description: "An Agent or Critic Gate's model, temperature, and parameters", icon: Atom }, { diff --git a/frontend/src/components/protocol/CustomMetricDialogs.tsx b/frontend/src/components/protocol/CustomMetricDialogs.tsx index 25c57afe..ca348b95 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.tsx @@ -15,7 +15,7 @@ import type { ProtocolGraph } from '@/types/protocols' import { useDialogAutosave, type DialogAutosaveStatus } from './useDialogAutosave' type Binding = MeasurementPlan['producers'][number] | undefined -export type MetricNodeDisplay = { label: string; type: 'Agent' | 'Script' | 'MCP Tool' } +export type MetricNodeDisplay = { label: string; type: 'Agent' | 'Script' | 'MCP Tool' | 'Tool Step' | 'Feature pipeline' } export function MetricNodeLabel({ display, reason }: { display: MetricNodeDisplay; reason?: string }) { return ( diff --git a/frontend/src/components/protocol/MetricsEditor.tsx b/frontend/src/components/protocol/MetricsEditor.tsx index 227c6361..24041bfb 100644 --- a/frontend/src/components/protocol/MetricsEditor.tsx +++ b/frontend/src/components/protocol/MetricsEditor.tsx @@ -10,7 +10,14 @@ import { AGENT_OUTPUT_PRODUCER_ID, agentOutputBindingForMetric, agentOutputSourc import { applyCustomMetricChange, type CustomMetricProducerConfig } from '@/lib/customMetrics' import { pythonScriptBindingForMetric, pythonScriptSourceOptions } from '@/lib/pythonScriptMetrics' import { mcpToolBindingForMetric, mcpToolSourceOptions } from '@/lib/mcpToolMetrics' -import { localMetricReadinessPreview, removeMetricFromMeasurementPlan, upsertRuntimeMetric } from '@/lib/measurementPlan' +import { + DERIVED_PRODUCER_IDS, + FEATURE_PIPELINE_PRODUCER_ID, + TOOL_STEP_PRODUCER_ID, + localMetricReadinessPreview, + removeMetricFromMeasurementPlan, + upsertRuntimeMetric, +} from '@/lib/measurementPlan' import { METRIC_CATALOG, makeCatalogMetric, makeCustomMetric, normalizeDesignMetrics, type MetricCatalogEntry } from '@/lib/metricCatalog' import type { ProtocolGraph } from '@/types/protocols' import type { DesignMetric, MeasurementPlan } from '@/types/experiments' @@ -58,6 +65,8 @@ function customMetricProducerDisplay( if (binding.producer_id === 'asaree.python_script') { return { label: `${agent}:${nodeLabel(binding.config.script_node_id)}`, type: 'Script' } } + if (binding.producer_id === TOOL_STEP_PRODUCER_ID) return { label: nodeLabel(binding.config.node_id), type: 'Tool Step' } + if (binding.producer_id === FEATURE_PIPELINE_PRODUCER_ID) return { label: 'DC → FTE → FS', type: 'Feature pipeline' } if (binding.producer_id === 'asaree.mcp_tool') { const toolName = typeof binding.config.tool_name === 'string' ? binding.config.tool_name : null return { @@ -358,7 +367,7 @@ function MetricsDialog({ - {binding && } + {binding && !DERIVED_PRODUCER_IDS.has(binding.producer_id) && } })}} diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 41db3cee..68b6bc11 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -33,6 +33,7 @@ import { defaultAnthropicModelNodeData, defaultAzureFoundryModelNodeData, defaultCriticGateNodeData, + defaultToolStepNodeData, defaultDatasetNodeData, defaultLocalModelNodeData, defaultMcpToolNodeData, @@ -47,6 +48,7 @@ import { import type { AgentNodeData, CriticGateNodeData, + ToolStepNodeData, DatasetNodeData, ModelNodeData, McpToolNodeData, @@ -75,6 +77,7 @@ import { AgentNodeInspector } from './AgentNodeInspector' import { agentTracedLabel, factorBoundField, revealsHiddenMcpServers, toolFactorServerId, unboundBindableFields, type UnboundField } from './bindableFields' import { CanvasControls } from './CanvasControls' import { CriticGateNodeInspector } from './CriticGateNodeInspector' +import { ToolStepNodeInspector } from './ToolStepNodeInspector' import { DatasetNodeInspector } from './DatasetNodeInspector' import { DeleteNodeConfirmDialog } from './DeleteNodeConfirmDialog' import { DEFAULT_ZOOM } from './constants' @@ -126,6 +129,7 @@ import { ConversationTranscript } from './ConversationTranscript' import { InteractEdge } from './edges/InteractEdge' import { AgentNode } from './nodes/AgentNode' import { CriticGateNode } from './nodes/CriticGateNode' +import { ToolStepNode } from './nodes/ToolStepNode' import { DatasetNode } from './nodes/DatasetNode' import { ModelNode } from './nodes/ModelNode' import { McpClientToolNode } from './nodes/McpClientToolNode' @@ -167,6 +171,7 @@ const NODE_TYPES = { // that distinguishes it. Same data, same inspector. mcp_client_tool: McpClientToolNode, critic_gate: CriticGateNode, + tool_step: ToolStepNode, // All five LLM provider types render through the same component -- it // derives icon/accent/placeholder from data.config.provider, not from // which of these five keys it was registered under. @@ -217,6 +222,7 @@ function defaultDataFor(nodeType: string): ProtocolNode['data'] { // it. if (nodeType === 'mcp_tool') return defaultMcpToolNodeData() if (nodeType === 'critic_gate') return defaultCriticGateNodeData() + if (nodeType === 'tool_step') return defaultToolStepNodeData() if (nodeType === 'model_anthropic') return defaultAnthropicModelNodeData() if (nodeType === 'model_openai') return defaultOpenAiModelNodeData() if (nodeType === 'model_azure_foundry') return defaultAzureFoundryModelNodeData() @@ -1295,7 +1301,9 @@ export const ProtocolCanvas = forwardRef() for (const metricId of Object.values(producer.outputs)) { @@ -1908,6 +1916,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> + ) : selectedNode?.type === 'tool_step' ? ( + edge.target === selectedNode.id && edge.targetHandle === 'tool') + .map((edge) => nodes.find((node) => node.id === edge.source)) + .filter((node) => MCP_TOOL_NODE_TYPES.includes(node?.type ?? '')) + .flatMap((node) => (node!.data as McpToolNodeData).config?.tool_names ?? [])} + nodeRun={latestNodeRuns?.[selectedNode.id]} + onChange={updateNodeData} + onDelete={requestDeleteNode} + onClose={() => setSelectedNodeId(null)} + /> ) : MODEL_NODE_TYPES.includes(selectedNode?.type ?? '') ? ( nodeTypes.get(nodeId) !== 'critic_gate') }, + { + label: 'Agents', + entries: nodeRuns.filter(([nodeId]) => !['critic_gate', 'tool_step'].includes(nodeTypes.get(nodeId) ?? '')), + }, + { label: 'Tool steps', entries: nodeRuns.filter(([nodeId]) => nodeTypes.get(nodeId) === 'tool_step') }, { label: 'Critic gates', entries: nodeRuns.filter(([nodeId]) => nodeTypes.get(nodeId) === 'critic_gate') }, ].filter((group) => group.entries.length > 0) const timestamp = new Date(run.created_at) diff --git a/frontend/src/components/protocol/ToolStepNodeInspector.tsx b/frontend/src/components/protocol/ToolStepNodeInspector.tsx new file mode 100644 index 00000000..5c814196 --- /dev/null +++ b/frontend/src/components/protocol/ToolStepNodeInspector.tsx @@ -0,0 +1,184 @@ +import { useState } from 'react' +import { Cog } from 'lucide-react' +import { nodeAccent } from '@/lib/nodeAccent' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { Switch } from '@/components/ui/switch' +import { Textarea } from '@/components/ui/textarea' +import { EditableNodeTitle } from './EditableNodeTitle' +import { NodeInspectorDialog } from './NodeInspectorDialog' +import { NodeRunOutputPanel } from './NodeRunOutputPanel' +import type { NodeRunState, ProtocolNode, ToolStepNodeConfig, ToolStepNodeData } from '@/types/protocols' + +const ACCENT = nodeAccent('tool_step') + +const SANITIZER_LABELS: Record = { + none: 'None -- send the payload unchanged', + xgboost_hyperparameters: 'XGBoost hyperparameters', +} + +// Same NodeInspectorDialog shell as CriticGateNodeInspector. `toolOptions` is +// the wired MCP Tool node's own enabled tools: a Tool Step may only call one +// its MCP node allow-lists (services/tool_steps.py's validate_tool_step), so +// the picker offers exactly those rather than a free-text name. +export function ToolStepNodeInspector({ + node, + toolOptions, + nodeRun, + onChange, + onDelete, + onClose, +}: { + node: (ProtocolNode & { data: ToolStepNodeData }) | null + toolOptions: string[] + nodeRun?: NodeRunState + onChange: (nodeId: string, data: ToolStepNodeData) => void + onDelete: (nodeId: string) => void + onClose: () => void +}) { + const [argumentsText, setArgumentsText] = useState(() => JSON.stringify(node?.data.config.arguments ?? {}, null, 2)) + const [argumentsError, setArgumentsError] = useState(null) + if (!node) return null + const data = node.data + const config = data.config + const options = config.tool_name && !toolOptions.includes(config.tool_name) ? [config.tool_name, ...toolOptions] : toolOptions + + function patchConfig(patch: Partial) { + onChange(node!.id, { ...data, config: { ...config, ...patch } }) + } + + function commitArguments(text: string) { + setArgumentsText(text) + try { + const parsed: unknown = JSON.parse(text || '{}') + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + setArgumentsError('Arguments must be a JSON object.') + return + } + setArgumentsError(null) + patchConfig({ arguments: parsed as Record }) + } catch { + setArgumentsError('Not valid JSON yet -- the last valid arguments are kept.') + } + } + + return ( + { + if (!open) onClose() + }} + accent={ACCENT} + title={ + <> + + onChange(node.id, { ...data, label })} /> + + } + onDelete={() => onDelete(node.id)} + onClose={onClose} + > +
+
+

+ Calls one MCP tool directly with the upstream node's JSON payload -- no model decides whether or how + to call it. A wired Script's code is sent as {config.code_argument || 'code'}, + the payload as {config.payload_argument || 'payload_json'}. +

+ +
+ + +
+ +
+ + +

+ XGBoost hyperparameters drops every suggestion outside the fixed search space (unknown or harness-fixed + params, out-of-bound ranges, malformed entries) so the replicate still scores; each drop is recorded as a + note on the step. +

+
+ +
+ +