From dd6ffa341259db894a11191f90fe5ccae4754252 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 15:35:50 -0700 Subject: [PATCH 01/63] feat: compare connected skills as factor levels --- frontend/src/api/client.ts | 2 +- .../src/components/protocol/DesignTab.tsx | 10 +- .../protocol/FactorEditorDialog.tsx | 8 +- .../protocol/ProtocolCanvas.test.tsx | 33 ++++- .../components/protocol/ProtocolCanvas.tsx | 116 +++++++++++++++--- .../protocol/ProtocolCanvasContext.tsx | 1 + .../components/protocol/SkillFactorEditor.tsx | 75 +++++++++++ .../protocol/SkillNodeInspector.tsx | 7 +- .../protocol/SkillTestSelectors.tsx | 19 +++ .../src/components/protocol/bindableFields.ts | 1 + .../src/components/protocol/factorLevels.ts | 5 +- .../components/protocol/nodes/AgentNode.tsx | 2 +- .../protocol/nodes/ConnectorAddStub.tsx | 26 +++- .../protocol/nodes/NodeHoverToolbar.tsx | 2 +- .../components/protocol/nodes/SkillNode.tsx | 5 +- .../protocol/useSkillTestSelection.test.tsx | 31 +++++ .../protocol/useSkillTestSelection.ts | 20 +++ frontend/src/lib/factorBindings.ts | 8 +- frontend/src/lib/skillFactors.test.ts | 31 +++++ frontend/src/lib/skillFactors.ts | 54 ++++++++ frontend/src/types/experiments.ts | 1 + frontend/src/types/protocols.ts | 2 + src/asaree/api/protocols.py | 20 ++- src/asaree/services/design_generation.py | 17 ++- src/asaree/services/factor_bindings.py | 13 +- src/asaree/services/protocol_execution.py | 17 ++- src/asaree/services/protocol_revisions.py | 15 ++- src/asaree/services/protocol_runs.py | 2 + src/asaree/services/skill_factors.py | 111 +++++++++++++++++ tests/test_skill_factors.py | 116 ++++++++++++++++++ 30 files changed, 722 insertions(+), 48 deletions(-) create mode 100644 frontend/src/components/protocol/SkillFactorEditor.tsx create mode 100644 frontend/src/components/protocol/SkillTestSelectors.tsx create mode 100644 frontend/src/components/protocol/useSkillTestSelection.test.tsx create mode 100644 frontend/src/components/protocol/useSkillTestSelection.ts create mode 100644 frontend/src/lib/skillFactors.test.ts create mode 100644 frontend/src/lib/skillFactors.ts create mode 100644 src/asaree/services/skill_factors.py create mode 100644 tests/test_skill_factors.py diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index ea31d26c..a306d30a 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -187,7 +187,7 @@ export interface ResultsScope { protocol_revision_id?: string } -export interface RowSelection { row_index?: number | null } +export interface RowSelection { row_index?: number | null; skill_selections?: Record } function scopeQuery(scope?: ResultsScope): string { const params = new URLSearchParams() diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 7a1abb93..bfca950d 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -9,9 +9,10 @@ import { Label } from '@/components/ui/label' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { Textarea } from '@/components/ui/textarea' -import { ApiError, experimentsApi, protocolsApi } from '@/api/client' +import { ApiError, experimentsApi, protocolsApi, skillsApi } from '@/api/client' import { coordinationStrategyIssues } from '@/lib/coordinationStrategy' import { unboundFactorNames } from '@/lib/factorBindings' +import { skillFactorIssues } from '@/lib/skillFactors' import { promptReferenceScope } from '@/lib/promptReferences' import { protocolGraphQueryKey } from '@/lib/protocolGraph' import { @@ -256,6 +257,7 @@ function FactorsEditor({ const editMutation = useMutation({ mutationFn: async ({ oldName, next }: { oldName: string; next: DesignFactor }) => { const fresh = await experimentsApi.get(experiment.id) + if (next.name !== oldName && (fresh.design_spec?.factors ?? []).some((factor) => factor.name === next.name)) throw new Error('A factor with this name already exists.') const nextFactors = (fresh.design_spec?.factors ?? []).map((f) => (f.name === oldName ? next : f)) await experimentsApi.update(experiment.id, { design_spec: { ...fresh.design_spec, factors: nextFactors } }) return { oldName, next } @@ -296,6 +298,7 @@ function FactorsEditor({ variant="ghost" size="icon-sm" aria-label="Remove factor" + title={factor.level_type === 'skill_selection' ? 'Remove factor and enable all connected skills' : 'Remove factor'} disabled={disabled || deleteMutation.isPending} onClick={() => deleteMutation.mutate(factor.name)} > @@ -490,6 +493,8 @@ export function DesignTab({ ? ({ nodes: graphQuery.data.nodes, edges: graphQuery.data.edges } as unknown as ProtocolGraph) : undefined const unboundFactors = unboundFactorNames(experiment.design_spec, draftGraph) + const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => factor.level_type === 'skill_selection') }) + const skillIssues = skillFactorIssues(draftGraph, factors, skillLibrary.data ? new Set(skillLibrary.data.map((skill) => skill.id)) : undefined) const impact = impactQuery.data // A design-time mirror of the backend's own strategy validation, so an @@ -858,13 +863,14 @@ export function DesignTab({ Unbound factor{unboundFactors.length === 1 ? '' : 's'}: {unboundFactors.join(', ')}. Rebind on the canvas or remove from this design.

)} + {skillIssues.map((issue) =>

{issue}

)} } + {skillFactorDraft && !open && setSkillFactorDraft(null)} + onSave={async (factor) => { + const graph = toPersistedGraph(nodes, edges) + if (connectedSkills(graph, skillFactorDraft.nodeId).some((skill) => Object.keys(skill.data.factor_bindings ?? {}).length)) throw new Error('Remove individual skill factor bindings first.') + const fresh = await experimentsApi.get(experimentId!) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== skillFactorDraft.oldName)) throw new Error('A factor with this name already exists.') + const next = reconcileSkillFactor(factor, graph, skillFactorDraft.nodeId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== skillFactorDraft.oldName), next] } }) + setNodes((nodes) => nodes.map((node) => node.id === skillFactorDraft.nodeId ? { ...node, data: { ...node.data, skill_selection: next.levels[0] ?? [], factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [SKILL_FACTOR_PATH]: next.name } } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={skillFactorDraft.oldName ? async () => { + const fresh = await experimentsApi.get(experimentId!) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== skillFactorDraft.oldName) } }) + removeFactorBindings(skillFactorDraft.oldName!) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + } : undefined} + />} {editingFactor && ( void + requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). requestMainEdgeAdd: (request: MainEdgeAddRequest) => void diff --git a/frontend/src/components/protocol/SkillFactorEditor.tsx b/frontend/src/components/protocol/SkillFactorEditor.tsx new file mode 100644 index 00000000..6c6a4cae --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { skillsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove }: { + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const skills = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list() }) + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the skill factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + Skills as levels + Each cell receives exactly one connected skill. Add or disconnect Skill nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different skills before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const skill = skills.data?.find((skill) => skill.id === id) + return
+
+

{skill?.name ?? labels[index] ?? 'Unavailable skill'}{index === 0 ? ' · default test selection' : ''}

+ {skills.isSuccess && !skill &&

Unavailable: disconnect or restore this skill.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+ {skills.isError &&

Could not load the skill library.

} +

Individual skill switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected skills and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/SkillNodeInspector.tsx b/frontend/src/components/protocol/SkillNodeInspector.tsx index 2d9939d2..ec7582fa 100644 --- a/frontend/src/components/protocol/SkillNodeInspector.tsx +++ b/frontend/src/components/protocol/SkillNodeInspector.tsx @@ -2,6 +2,7 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useQuery } from '@tanstack/react-query' import { ScrollText } from 'lucide-react' import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' @@ -32,6 +33,7 @@ export function SkillNodeInspector({ onChange, onDelete, onClose, + connectorFactorName, }: { node: (ProtocolNode & { data: SkillNodeData }) | null experimentId: string | null @@ -41,6 +43,7 @@ export function SkillNodeInspector({ onChange: (nodeId: string, data: SkillNodeData) => void onDelete: (nodeId: string) => void onClose: () => void + connectorFactorName?: string }) { const skillsQuery = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list() }) @@ -80,7 +83,7 @@ export function SkillNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > -

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)} : patchConfig({ enabled: checked })} /> )} - + }
diff --git a/frontend/src/components/protocol/SkillTestSelectors.tsx b/frontend/src/components/protocol/SkillTestSelectors.tsx new file mode 100644 index 00000000..2a816523 --- /dev/null +++ b/frontend/src/components/protocol/SkillTestSelectors.tsx @@ -0,0 +1,19 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedSkills, skillId } from '@/lib/skillFactors' +import type { useSkillTestSelection } from './useSkillTestSelection' + +export function SkillTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedSkills(selection.graph!, node.id) + return
+

{node.data.label}: skill for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 1f2f1933..5aba885a 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -444,6 +444,7 @@ export function agentTracedLabel(node: Node, edges: Edge[], nodes: Node[]): stri export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundField[] { const result: UnboundField[] = [] for (const node of nodes) { + if (node.type === 'skill' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'skill' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.skill_selection)) continue const bindings = (node.data as { factor_bindings?: Record })?.factor_bindings ?? {} const label = agentTracedLabel(node, edges, nodes) const wholeConfigBound = !!bindings.config diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index 3828c7cb..3a73e8cf 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -21,6 +21,7 @@ export type LevelType = | 'script_config' | 'dataset_config' | 'tool_names' + | 'skill_selection' export const LEVEL_TYPE_LABELS: Record = { string: 'String', @@ -33,6 +34,7 @@ export const LEVEL_TYPE_LABELS: Record = { script_config: 'Script', dataset_config: 'Dataset', tool_names: 'Tools allowed', + skill_selection: 'Skills as levels', } // Whether a level of this kind is a structured value (an object, or -- for @@ -47,7 +49,8 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'pattern' || type === 'script_config' || type === 'dataset_config' || - type === 'tool_names' + type === 'tool_names' || + type === 'skill_selection' ) } diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index cef07cb9..a0833e64 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -287,7 +287,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Skill - + {/* Dataset -- the data an agent works ON, as opposed to the Tool connector's "capabilities it works WITH". UNCAPPED, like Skill/Knowledge/Tool: a cell's workspace holds one dataset per named diff --git a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx index 513b2fbf..6b161a90 100644 --- a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx +++ b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx @@ -1,6 +1,8 @@ import type { MouseEvent } from 'react' import { useNodeConnections } from '@xyflow/react' import { Plus } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { FACTOR_TRIGGER_CLASSNAME } from '../factorLevels' import { CONNECTOR_SLOT_LABELS, useProtocolCanvasActions, type ConnectorSlot } from '../ProtocolCanvasContext' // The affordance for an empty sub-connector: a short stub line ending @@ -21,15 +23,19 @@ export function ConnectorAddStub({ left, side = 'bottom', alwaysVisible = false, + factorName, + factorLevelCount, }: { nodeId: string slot: ConnectorSlot left?: string side?: 'bottom' | 'right' | 'top' alwaysVisible?: boolean + factorName?: string + factorLevelCount?: number }) { const connections = useNodeConnections({ id: nodeId, handleType: 'target', handleId: slot }) - const { requestConnectorAdd } = useProtocolCanvasActions() + const { requestConnectorAdd, requestSkillFactor, experimentLocked } = useProtocolCanvasActions() if (connections.length > 0 && !alwaysVisible) return null @@ -58,6 +64,16 @@ export function ConnectorAddStub({ // node, so the line needs to be its LAST child to land in the same // "closest to the node" spot. return ( +
+ {slot === 'skill' && requestSkillFactor && !experimentLocked && } + {factorName && Skills · {factorLevelCount ?? 0} levels} +
) } diff --git a/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx b/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx index 39c5757c..5ebea87d 100644 --- a/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx +++ b/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx @@ -96,7 +96,7 @@ export function NodeHoverToolbar({ // ProtocolCanvasPage.tsx). runAlone.canRun && runAlone.onRun()} > diff --git a/frontend/src/components/protocol/nodes/SkillNode.tsx b/frontend/src/components/protocol/nodes/SkillNode.tsx index 8e952a8c..48d1c6fe 100644 --- a/frontend/src/components/protocol/nodes/SkillNode.tsx +++ b/frontend/src/components/protocol/nodes/SkillNode.tsx @@ -16,7 +16,8 @@ const ACCENT = nodeAccent('skill') export function SkillNode({ id, data, selected }: NodeProps & { data: SkillNodeData }) { const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const controlled = !!data.skillFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) return ( updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/useSkillTestSelection.test.tsx b/frontend/src/components/protocol/useSkillTestSelection.test.tsx new file mode 100644 index 00000000..2be87aa3 --- /dev/null +++ b/frontend/src/components/protocol/useSkillTestSelection.test.tsx @@ -0,0 +1,31 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, expect, it, vi } from 'vitest' +import { skillsApi } from '@/api/client' +import type { ProtocolGraph } from '@/types/protocols' +import { useSkillTestSelection } from './useSkillTestSelection' + +afterEach(() => vi.restoreAllMocks()) +it('keeps preview selections scoped to the agent and does not alter the first level', async () => { + vi.spyOn(skillsApi, 'list').mockResolvedValue([{ id: 'a' }, { id: 'b' }] as Awaited>) + const graph = { + nodes: [ + { id: 'one', type: 'agent', data: { skill_selection: ['a'], factor_bindings: { skill_selection: 'One' } } }, + { id: 'two', type: 'agent', data: { skill_selection: ['b'], factor_bindings: { skill_selection: 'Two' } } }, + ...['a', 'b'].map((id) => ({ id, type: 'skill', data: { config: { skill_id: id } } })), + ], + edges: ['one', 'two'].flatMap((target) => ['a', 'b'].map((source) => ({ source, target, targetHandle: 'skill' }))), + } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result, rerender } = renderHook(({ nodeId }) => useSkillTestSelection(graph, nodeId), { wrapper, initialProps: { nodeId: undefined as string | undefined } }) + await waitFor(() => expect(result.current.error).toBeNull()) + expect(result.current.selections).toEqual({ one: 'a', two: 'b' }) + act(() => result.current.setChoice('one', 'b')) + expect(result.current.options).toEqual({ skill_selections: { one: 'b', two: 'b' } }) + rerender({ nodeId: 'one' }) + expect(result.current.selections).toEqual({ one: 'a' }) + expect(JSON.stringify(graph)).toBe(original) +}) diff --git a/frontend/src/components/protocol/useSkillTestSelection.ts b/frontend/src/components/protocol/useSkillTestSelection.ts new file mode 100644 index 00000000..41c5a291 --- /dev/null +++ b/frontend/src/components/protocol/useSkillTestSelection.ts @@ -0,0 +1,20 @@ +import { useEffect, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { skillsApi } from '@/api/client' +import { connectedSkills, skillId, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import type { ProtocolGraph } from '@/types/protocols' + +export function useSkillTestSelection(graph: ProtocolGraph | undefined, nodeId?: string) { + const [choices, setChoices] = useState>({}) + const library = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: !!graph?.nodes.some((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH]) }) + const owners = graph?.nodes.filter((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] && (!nodeId || node.id === nodeId)) ?? [] + const signature = JSON.stringify(owners.map((node) => [node.id, node.data.skill_selection, connectedSkills(graph!, node.id).map(skillId)])) + useEffect(() => { setChoices({}) }, [signature]) + const selections = Object.fromEntries(owners.map((node) => { + const ids = connectedSkills(graph!, node.id).map(skillId) + const baseline = (node.data.skill_selection as string[] | undefined)?.[0] + return [node.id, choices[node.id] ?? (baseline && ids.includes(baseline) ? baseline : ids[0] ?? '')] + })) + const error = owners.length === 0 ? null : library.isError ? 'Could not load available skills.' : !library.data ? 'Loading skills…' : Object.values(selections).some((id) => !library.data.some((skill) => skill.id === id)) ? 'Select an available skill for each agent.' : null + return { graph, owners, library, selections, options: owners.length ? { skill_selections: selections } : {}, error, setChoice: (agentId: string, id: string) => setChoices((previous) => ({ ...previous, [agentId]: id })) } +} diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index ebd369cd..2117d302 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -1,5 +1,6 @@ import type { DesignFactor, DesignSpec } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' +import { reconcileSkillFactor, SKILL_FACTOR_PATH } from './skillFactors' export interface FactorBindingDiscrepancy { nodeId: string @@ -59,6 +60,7 @@ function withPathValue(root: Record, dottedPath: string, nextVa * as an extra level would turn each pause mid-edit into a new treatment. */ export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { + if (factor.level_type === 'skill_selection') return factor if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) @@ -92,6 +94,10 @@ export function reconcileFactorBaselines(designSpec: DesignSpec | null | undefin } return factors.map((factor) => { + if (factor.level_type === 'skill_selection' && graph) { + const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] === factor.name) + return owner ? reconcileSkillFactor(factor, graph, owner.id) : factor + } const values = valuesByFactor.get(factor.name.trim()) ?? [] if (values.length === 0 || values.some((value) => !valuesEqual(value, values[0]))) return factor return factorWithCanvasBaseline(factor, values[0]) @@ -143,7 +149,7 @@ export function factorBindingDiscrepancies( ? 'factor is no longer declared' : value === MISSING ? 'field no longer exists on the canvas' - : isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) + : factor.level_type === 'skill_selection' || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) ? null : 'published value does not match the first (canvas baseline) level' if (reason) discrepancies.push({ nodeId: node.id, nodeLabel, fieldPath, factorName, reason }) diff --git a/frontend/src/lib/skillFactors.test.ts b/frontend/src/lib/skillFactors.test.ts new file mode 100644 index 00000000..ff0471ba --- /dev/null +++ b/frontend/src/lib/skillFactors.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { reconcileFactorBaselines } from './factorBindings' +import { reconcileSkillFactor, skillFactorIssues, skillFactorOwner } from './skillFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Agent', skill_selection: ['a'], factor_bindings: { skill_selection: 'Skills' } } }, + ...['a', 'b', 'c'].map((id) => ({ id, type: 'skill', data: { label: id, config: { skill_id: id, skill_name: `Skill ${id}` } } })), + ], + edges: ['a', 'b', 'c'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'skill' })), +} as unknown as ProtocolGraph +const factor: DesignFactor = { name: 'Skills', level_type: 'skill_selection', levels: [['b'], ['a']], level_labels: ['Custom B', 'Custom A'] } + +describe('skill connector factors', () => { + it('preserves ordered labels and appends newly connected skills', () => { + expect(reconcileSkillFactor(factor, graph, 'agent')).toEqual({ ...factor, levels: [['b'], ['a'], ['c']], level_labels: ['Custom B', 'Custom A', 'Skill c'] }) + }) + it('removes disconnected levels without replacing the first level from canvas data', () => { + const changed = { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') } + expect(reconcileFactorBaselines({ factors: [factor] }, changed)[0]).toEqual({ ...factor, levels: [['a'], ['c']], level_labels: ['Custom A', 'Skill c'] }) + }) + it('keeps an empty factor visible and reports incomplete and unavailable skills', () => { + const empty = { ...graph, edges: [] } + expect(reconcileSkillFactor(factor, empty, 'agent').levels).toEqual([]) + expect(skillFactorIssues(empty, [factor])[0]).toContain('at least two') + expect(skillFactorIssues(graph, [factor], new Set(['a', 'c']))[0]).toContain('unavailable') + expect(skillFactorOwner(graph, 'b')).toBe('Skills') + }) +}) diff --git a/frontend/src/lib/skillFactors.ts b/frontend/src/lib/skillFactors.ts new file mode 100644 index 00000000..2e565c08 --- /dev/null +++ b/frontend/src/lib/skillFactors.ts @@ -0,0 +1,54 @@ +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +export const SKILL_FACTOR_PATH = 'skill_selection' + +export function connectedSkills(graph: ProtocolGraph, agentId: string): ProtocolNode[] { + return graph.edges.filter((edge) => edge.target === agentId && edge.targetHandle === 'skill') + .flatMap((edge) => { + const node = graph.nodes.find((node) => node.id === edge.source && node.type === 'skill') + return node ? [node] : [] + }) +} + +export function skillId(node: ProtocolNode): string { + return String((node.data.config as { skill_id?: string } | undefined)?.skill_id ?? '') +} + +export function reconcileSkillFactor(factor: DesignFactor, graph: ProtocolGraph, agentId: string): DesignFactor { + const skills = connectedSkills(graph, agentId) + const ids = [...new Set(skills.map(skillId))] + const oldIds = factor.levels.map((level) => Array.isArray(level) ? String(level[0] ?? '') : '') + const ordered = [...new Set([...oldIds.filter((id) => ids.includes(id)), ...ids.filter((id) => !oldIds.includes(id))])] + return { + ...factor, + levels: ordered.map((id) => [id]), + level_labels: ordered.map((id) => { + const index = oldIds.indexOf(id) + if (index >= 0 && factor.level_labels?.[index]) return factor.level_labels[index] + const node = skills.find((node) => skillId(node) === id) + const label = String((node?.data.config as { skill_name?: string } | undefined)?.skill_name ?? node?.data.label ?? 'Unavailable skill') + return label || 'Unavailable skill' + }), + } +} + +export function skillFactorOwner(graph: ProtocolGraph | undefined, skillNodeId: string): string | undefined { + return graph?.edges.filter((edge) => edge.source === skillNodeId && edge.targetHandle === 'skill') + .map((edge) => graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[SKILL_FACTOR_PATH]) + .find(Boolean) +} + +export function skillFactorIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[], availableIds?: Set): string[] { + return factors.filter((factor) => factor.level_type === 'skill_selection').flatMap((factor) => { + const owner = graph?.nodes.find((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] === factor.name) + if (!owner || !graph) return [`${factor.name}: rebind or remove this skill factor.`] + const skills = connectedSkills(graph, owner.id) + const ids = [...new Set(skills.map(skillId))] + const conflicts = skills.some((node) => Object.keys(node.data.factor_bindings ?? {}).length > 0) + if (conflicts) return [`${factor.name}: remove individual skill factor bindings first.`] + if (ids.length < 2) return [`${factor.name}: connect at least two different skills.`] + if (ids.some((id) => !id || (availableIds && !availableIds.has(id)))) return [`${factor.name}: disconnect or restore unavailable skills.`] + return [] + }) +} diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 29365ed1..7e6010ed 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -22,6 +22,7 @@ export interface DesignFactor { | 'script_config' | 'dataset_config' | 'tool_names' + | 'skill_selection' } export interface DesignMetric { diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index d76664fa..366ce209 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -357,6 +357,8 @@ export interface AgentNodeConfig { } export interface AgentNodeData { + // Connector-owned selection; comparison levels contain one skill ID. + skill_selection?: string[] label: string config: AgentNodeConfig // field path (e.g. "model_config_data.temperature") -> factor name, for diff --git a/src/asaree/api/protocols.py b/src/asaree/api/protocols.py index 329a4ae5..1b19f370 100644 --- a/src/asaree/api/protocols.py +++ b/src/asaree/api/protocols.py @@ -66,6 +66,7 @@ list_protocols, update_protocol, ) +from asaree.services.skill_factors import test_skill_factor_values from asaree.services.test_run_results import FreshnessReason, project_test_run_result from asaree.worker.enqueue import enqueue_protocol_run @@ -275,12 +276,23 @@ class TestRunRequest(BaseModel): """Optional original Dataset row selection for a row-mode Test Run.""" row_index: StrictInt | None = Field(default=None, ge=0) + skill_selections: dict[str, str] | None = None class NodePlayRequest(BaseModel): """Optional original Dataset row selection for an eligible node Play.""" row_index: StrictInt | None = Field(default=None, ge=0) + skill_selections: dict[str, str] | None = None + + +async def _node_test_skill_values( + graph: dict[str, Any], body: NodePlayRequest | None, owner_id: uuid.UUID, node_id: str +) -> dict[str, Any]: + try: + return await test_skill_factor_values(graph, body.skill_selections if body else None, owner_id, node_id) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc class CellRunBatchRequest(BaseModel): @@ -436,7 +448,9 @@ async def publish_protocol_endpoint( validate_stage_plan(design_spec) validate_prompt_references(graph=protocol.graph) topological_order(protocol.graph, require_acyclic=not is_conversation_strategy(design_spec)) - validate_factor_bindings(design_spec, protocol.graph) + # An incomplete skill comparison can still be published for a selected + # test. Generation and production planning enforce complete choices. + validate_factor_bindings(design_spec, protocol.graph, complete_skills=False) if experiment is not None: report = await validate_experiment_measurement_plan( db, @@ -662,6 +676,9 @@ async def create_test_run_endpoint( owner_id=user.id, protocol_revision_id=revision.id, dataset_row=dataset_row, + factor_values=await test_skill_factor_values( + revision.graph, body.skill_selections if body else None, user.id + ), ) except (ProtocolValidationError, ValueError) as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc @@ -751,6 +768,7 @@ async def run_single_node_endpoint( target_node_id=node_id, protocol_revision_id=revision.id, dataset_row=dataset_row, + factor_values=await _node_test_skill_values(revision.graph, body, user.id, node_id), ) await db.commit() await enqueue_protocol_run(run.id) diff --git a/src/asaree/services/design_generation.py b/src/asaree/services/design_generation.py index 52b24d72..0ef4fa6a 100644 --- a/src/asaree/services/design_generation.py +++ b/src/asaree/services/design_generation.py @@ -30,10 +30,12 @@ from asaree.models.experiment import ResearchExperiment from asaree.models.experiment_design_revision import ExperimentDesignRevision from asaree.models.factorial_replicate_result import FactorialReplicateResult +from asaree.models.protocol import Protocol from asaree.models.protocol_revision import ProtocolRevision from asaree.services.coordination import coordination_strategy_slug from asaree.services.design_revisions import get_current_revision, supersede_and_create from asaree.services.factorial_cells import list_replicates, upsert_replicate +from asaree.services.skill_factors import validate_skill_factors class DesignValidationError(ValueError): @@ -283,7 +285,14 @@ async def get_design_impact( ) -> DesignImpact: """Compare the declared factorial matrix to its materialized revision.""" material = material_design_spec(design_spec) - planned = _planned_replicates(material["factors"], material["replicates"]) if material["factors"] else [] + incomplete_skills = any( + factor.get("level_type") == "skill_selection" and len(factor.get("levels") or []) < 2 + for factor in material["factors"] + ) + planned = ( + _planned_replicates(material["factors"], material["replicates"]) + if material["factors"] and not incomplete_skills else [] + ) planned_labels = {label for label, _, _ in planned} planned_cell_keys = {_cell_key(combo, label) for label, combo, _ in planned} current = await get_current_revision(db, experiment_id) @@ -404,6 +413,12 @@ async def generate_design_cells( ).scalar_one_or_none() if experiment is None: raise DesignValidationError("Experiment not found") + if any(factor.get("level_type") == "skill_selection" for factor in factors): + protocol = await db.scalar(select(Protocol).where(Protocol.experiment_id == experiment_id)) + try: + await validate_skill_factors({"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id) + except ValueError as exc: + raise DesignValidationError(str(exc)) from exc # An empty factor declaration is a meaningful replacement when an # existing design's final factor was removed: it retires every current # cell into history and leaves an empty current revision. Do not call diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index e6192a9b..c1e921dc 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -4,6 +4,8 @@ from typing import Any +from asaree.services.skill_factors import validate_skill_factor_structure + _MISSING = object() @@ -26,7 +28,10 @@ def unbound_factor_names(design_spec: dict[str, Any] | None, graph: dict[str, An return [name for name in declared if name and name not in bound] -def validate_factor_bindings(design_spec: dict[str, Any] | None, graph: dict[str, Any]) -> None: +def validate_factor_bindings( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete_skills: bool = True +) -> None: + validate_skill_factor_structure(design_spec, graph, complete=complete_skills) missing = unbound_factor_names(design_spec, graph) if missing: labels = ", ".join(repr(name) for name in missing) @@ -61,7 +66,11 @@ def validate_factor_bindings(design_spec: dict[str, Any] | None, graph: dict[str legacy_boolean = ( "level_type" not in factor and len(levels) == 2 and all(isinstance(level, bool) for level in levels) ) - if level_type != "boolean" and not legacy_boolean and (not levels or published_value != levels[0]): + if ( + level_type not in {"boolean", "skill_selection"} + and not legacy_boolean + and (not levels or published_value != levels[0]) + ): raise ValueError( f"{label!r} field {field_path!r} is bound to factor {factor_name!r}, but its published value " "does not match the first (canvas baseline) level. Update the factor levels and regenerate " diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 82472965..890b324d 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -100,6 +100,7 @@ from asaree.services.protocols import get_protocol from asaree.services.run_tools import gather_tools from asaree.services.runtime_metrics import finalize_attempt_measurement +from asaree.services.skill_factors import validate_skill_factors from asaree.services.system_mcp_servers import ( DATASET_DICTIONARY_AGENT_TOOLS, EDA_SERVER_NAME, @@ -2031,14 +2032,20 @@ def _resolve_skill_config(graph: dict[str, Any], node_id: str) -> dict[str, Any] resolved ``skill_id`` is skipped, matching ``_resolve_tool_config``.""" nodes, _downstream, _upstream = _adjacency(graph) skill_ids: list[str] = [] + agent_data = (nodes.get(node_id) or {}).get("data") or {} + selection = ( + agent_data.get("skill_selection") if (agent_data.get("factor_bindings") or {}).get("skill_selection") else None + ) for edge in _edges_with_handle(graph, node_id, "skill", direction="incoming"): source = nodes.get(edge["source"]) if source is None or source.get("type") not in _SKILL_NODE_TYPES: continue skill_node_config = (source.get("data") or {}).get("config") or {} - if not skill_node_config.get("enabled", True): + if selection is None and not skill_node_config.get("enabled", True): continue skill_id = skill_node_config.get("skill_id") + if selection is not None and skill_id not in selection: + continue if skill_id and str(skill_id) not in skill_ids: skill_ids.append(str(skill_id)) return {"skill_ids": skill_ids} if skill_ids else {} @@ -4725,6 +4732,7 @@ async def plan_cell_runs( ) try: validate_factor_bindings(design_spec, graph) + await validate_skill_factors(design_spec, graph, owner_id) except ValueError as exc: raise ProtocolValidationError(str(exc)) from exc measurement_report = await validate_experiment_measurement_plan( @@ -5111,6 +5119,7 @@ async def plan_single_replicate_run( ) try: validate_factor_bindings(design_spec, graph) + await validate_skill_factors(design_spec, graph, owner_id) except ValueError as exc: raise ProtocolValidationError(str(exc)) from exc measurement_report = await validate_experiment_measurement_plan( @@ -5289,8 +5298,8 @@ async def _run_single_node( dataset_row: dict[str, Any] | None = None, ) -> None: """The canvas's per-node Play run: one Agent node, no upstream, no gated - pair, no factor substitution, no coordination-strategy check -- none of - those concepts apply to a single node run in isolation. A deliberately + pair and no coordination-strategy check. A preview skill choice has + already been substituted in the supplied graph. A deliberately separate path from the main topological walk below, not a special case bolted onto it.""" try: @@ -5521,6 +5530,8 @@ async def run_protocol(protocol_run_id: uuid.UUID) -> None: return if target_node_id: + if factor_values: + graph = apply_factor_bindings(graph, factor_values) await _run_single_node( protocol_run_id, protocol_id=protocol_id, diff --git a/src/asaree/services/protocol_revisions.py b/src/asaree/services/protocol_revisions.py index 66596876..915a5127 100644 --- a/src/asaree/services/protocol_revisions.py +++ b/src/asaree/services/protocol_revisions.py @@ -149,9 +149,18 @@ async def publish_protocol( raise ValueError("Experiment is locked. Unlock it before publishing a changed experiment.") if experiment is not None: from asaree.services.design_generation import get_design_impact - impact = await get_design_impact(db, experiment_id=experiment.id, design_spec=experiment.design_spec) - if impact.regeneration_required: - raise ValueError("Generate cells for the draft design before publishing the experiment.") + incomplete_skills = any( + factor.get("level_type") == "skill_selection" and len(factor.get("levels") or []) < 2 + for factor in (experiment.design_spec or {}).get("factors") or [] + ) + if incomplete_skills: + # This publication can be tested with an available choice, but it + # owns no generated design and cannot create production cell runs. + design = None + else: + impact = await get_design_impact(db, experiment_id=experiment.id, design_spec=experiment.design_spec) + if impact.regeneration_required: + raise ValueError("Generate cells for the draft design before publishing the experiment.") highest = ( await db.execute( diff --git a/src/asaree/services/protocol_runs.py b/src/asaree/services/protocol_runs.py index 2ced07e3..a28d0822 100644 --- a/src/asaree/services/protocol_runs.py +++ b/src/asaree/services/protocol_runs.py @@ -326,6 +326,7 @@ async def create_test_run( owner_id: uuid.UUID, protocol_revision_id: uuid.UUID, dataset_row: dict[str, Any] | None = None, + factor_values: dict[str, Any] | None = None, ) -> ProtocolRun: """Create the experiment's next canvas validation attempt. @@ -352,6 +353,7 @@ async def create_test_run( protocol_revision_id=protocol_revision_id, dataset_row=dataset_row, is_test_run=True, + factor_values=factor_values, ) experiment.latest_test_run_id = run.id await db.flush() diff --git a/src/asaree/services/skill_factors.py b/src/asaree/services/skill_factors.py new file mode 100644 index 00000000..29cfcc51 --- /dev/null +++ b/src/asaree/services/skill_factors.py @@ -0,0 +1,111 @@ +"""Connector-owned skill choices shared by design validation and test runs.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from motoro.services import skill_service + +SKILL_SELECTION = "skill_selection" + + +def connected_skill_nodes(graph: dict[str, Any], agent_id: str) -> list[dict[str, Any]]: + nodes = {node["id"]: node for node in graph.get("nodes") or []} + return [ + nodes[edge["source"]] + for edge in graph.get("edges") or [] + if edge.get("target") == agent_id + and edge.get("targetHandle") == "skill" + and edge.get("source") in nodes + and nodes[edge["source"]].get("type") == "skill" + ] + + +def validate_skill_factor_structure( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete: bool = True +) -> None: + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + bound = { + (node.get("data") or {}).get("factor_bindings", {}).get(SKILL_SELECTION) for node in graph.get("nodes") or [] + } + for name, factor in factors.items(): + if factor.get("level_type") == SKILL_SELECTION and name not in bound: + raise ValueError(f"Rebind or remove skill factor {name!r}.") + for node in graph.get("nodes") or []: + data = node.get("data") or {} + name = (data.get("factor_bindings") or {}).get(SKILL_SELECTION) + if not name: + continue + factor = factors.get(name) + if node.get("type") not in {"agent", "sub_agent"}: + raise ValueError("Only an Agent skill connector can own a skill factor.") + if not factor or factor.get("level_type") != SKILL_SELECTION: + raise ValueError(f"Skill connector factor {name!r} must declare skills as levels.") + skills = connected_skill_nodes(graph, node["id"]) + ids = {(skill.get("data") or {}).get("config", {}).get("skill_id") for skill in skills} + if complete and (None in ids or "" in ids): + raise ValueError(f"Skill factor {name!r}: disconnect or restore unavailable skills.") + if complete and len(ids) < 2: + raise ValueError(f"Skill factor {name!r}: connect at least two different skills.") + if any((skill.get("data") or {}).get("factor_bindings") for skill in skills): + raise ValueError(f"Skill factor {name!r}: remove individual skill factor bindings first.") + levels = factor.get("levels") or [] + if any(not isinstance(level, list) or len(level) != 1 or level[0] not in ids for level in levels): + raise ValueError(f"Skill factor {name!r}: each level must name one connected skill.") + if len(levels) != len(ids) or {level[0] for level in levels} != ids: + raise ValueError(f"Skill factor {name!r}: levels must include every connected skill exactly once.") + + +async def require_available_skills(ids: list[str], owner_id: uuid.UUID) -> None: + for skill_id in dict.fromkeys(ids): + try: + skill = await skill_service.get_skill(uuid.UUID(skill_id)) + except (ValueError, TypeError, AttributeError): + skill = None + if skill is None or not (skill.owner_id == owner_id or skill.is_system): + raise ValueError("A skill factor references an unavailable skill. Disconnect or restore it.") + + +async def validate_skill_factors( + design_spec: dict[str, Any] | None, graph: dict[str, Any], owner_id: uuid.UUID +) -> None: + validate_skill_factor_structure(design_spec, graph) + ids = [ + level[0] + for factor in (design_spec or {}).get("factors") or [] + if factor.get("level_type") == SKILL_SELECTION + for level in factor.get("levels") or [] + ] + await require_available_skills(ids, owner_id) + + +async def test_skill_factor_values( + graph: dict[str, Any], selections: dict[str, str] | None, owner_id: uuid.UUID, node_id: str | None = None +) -> dict[str, Any]: + """Validate preview choices against the publication and retain run provenance.""" + selections = selections or {} + owners = { + node["id"]: node + for node in graph.get("nodes") or [] + if (node.get("data") or {}).get("factor_bindings", {}).get(SKILL_SELECTION) + and (node_id is None or node["id"] == node_id) + } + if set(selections) - owners.keys(): + raise ValueError("Skill selection must belong to a skill factor in this run.") + values: dict[str, Any] = {} + for agent_id, node in owners.items(): + data = node.get("data") or {} + connected = [ + (skill.get("data") or {}).get("config", {}).get("skill_id") + for skill in connected_skill_nodes(graph, agent_id) + ] + if any((skill.get("data") or {}).get("factor_bindings") for skill in connected_skill_nodes(graph, agent_id)): + raise ValueError("Remove individual skill factor bindings before testing this connector factor.") + baseline = data.get(SKILL_SELECTION) or [] + selected = selections.get(agent_id, baseline[0] if baseline else None) + if not selected or selected not in connected: + raise ValueError("Select an available connected skill for the test run.") + await require_available_skills([selected], owner_id) + values[data["factor_bindings"][SKILL_SELECTION]] = [selected] + return values diff --git a/tests/test_skill_factors.py b/tests/test_skill_factors.py new file mode 100644 index 00000000..eb76901b --- /dev/null +++ b/tests/test_skill_factors.py @@ -0,0 +1,116 @@ +"""Skill connector factors select one skill without changing reusable nodes.""" + +from __future__ import annotations + +import copy +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from asaree.services.design_generation import get_design_impact +from asaree.services.factor_bindings import validate_factor_bindings +from asaree.services.protocol_execution import _resolve_skill_config, apply_factor_bindings +from asaree.services.skill_factors import test_skill_factor_values as preview_values +from asaree.services.skill_factors import validate_skill_factors + + +def skill_graph(): + ids = [str(uuid.uuid4()), str(uuid.uuid4())] + graph = { + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": { + "factor_bindings": {"skill_selection": "Skills"}, + "skill_selection": [ids[0]], + }, + }, + *[ + { + "id": f"skill-{index}", + "type": "skill", + "data": { + "config": {"skill_id": skill_id, "enabled": False}, + }, + } + for index, skill_id in enumerate(ids) + ], + ], + "edges": [{"source": f"skill-{index}", "target": "agent", "targetHandle": "skill"} for index in range(2)], + } + spec = {"factors": [{"name": "Skills", "level_type": "skill_selection", "levels": [[id] for id in ids]}]} + return graph, spec, ids + + +def test_cell_selection_overrides_individual_switches_without_mutating_graph(): + graph, spec, ids = skill_graph() + original = copy.deepcopy(graph) + validate_factor_bindings(spec, graph) + assert _resolve_skill_config(graph, "agent") == {"skill_ids": [ids[0]]} + patched = apply_factor_bindings(graph, {"Skills": [ids[1]]}) + assert _resolve_skill_config(patched, "agent") == {"skill_ids": [ids[1]]} + assert graph == original + + +@pytest.mark.parametrize("change", ["one", "missing-level", "duplicate", "conflict"]) +def test_incomplete_or_conflicting_factor_blocks_cells(change): + graph, spec, ids = skill_graph() + if change == "one": + graph["edges"].pop() + spec["factors"][0]["levels"] = [[ids[0]]] + elif change == "missing-level": + spec["factors"][0]["levels"].pop() + elif change == "duplicate": + spec["factors"][0]["levels"] = [[ids[0]], [ids[0]]] + else: + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "On"} + with pytest.raises(ValueError): + validate_factor_bindings(spec, graph) + + +@pytest.mark.asyncio +async def test_preview_accepts_valid_alternative_even_when_other_skill_unavailable(monkeypatch): + graph, _, ids = skill_graph() + owner = uuid.uuid4() + get_skill = AsyncMock( + side_effect=lambda id: SimpleNamespace(owner_id=owner, is_system=False) if str(id) == ids[1] else None + ) + monkeypatch.setattr("asaree.services.skill_factors.skill_service.get_skill", get_skill) + assert await preview_values(graph, {"agent": ids[1]}, owner) == {"Skills": [ids[1]]} + assert get_skill.await_count == 1 + with pytest.raises(ValueError, match="unavailable"): + await preview_values(graph, None, owner) + with pytest.raises(ValueError, match="belong"): + await preview_values(graph, {"other-agent": ids[1]}, owner) + + +@pytest.mark.asyncio +async def test_generation_checks_every_skill_but_one_skill_can_be_published_and_tested(monkeypatch): + graph, spec, ids = skill_graph() + owner = uuid.uuid4() + monkeypatch.setattr("asaree.services.skill_factors.skill_service.get_skill", AsyncMock(return_value=None)) + with pytest.raises(ValueError, match="unavailable"): + await validate_skill_factors(spec, graph, owner) + graph["edges"].pop() + spec["factors"][0]["levels"] = [[ids[0]]] + validate_factor_bindings(spec, graph, complete_skills=False) + monkeypatch.setattr( + "asaree.services.skill_factors.skill_service.get_skill", + AsyncMock(return_value=SimpleNamespace(owner_id=owner, is_system=False)), + ) + assert await preview_values(graph, None, owner, "agent") == {"Skills": [ids[0]]} + + +@pytest.mark.asyncio +async def test_empty_skill_factor_has_readable_design_impact_instead_of_error(monkeypatch): + monkeypatch.setattr("asaree.services.design_generation.get_current_revision", AsyncMock(return_value=None)) + impact = await get_design_impact( + AsyncMock(), + experiment_id=uuid.uuid4(), + design_spec={"factors": [{"name": "Skills", "level_type": "skill_selection", "levels": []}]}, + ) + assert impact.proposed_cell_count == 0 + assert impact.proposed_replicate_count == 0 From 7f8c585fc2d024954a5f4410e87aee37dc82f3fe Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 15:39:13 -0700 Subject: [PATCH 02/63] fix: size script editor to available inspector height --- frontend/src/components/protocol/PythonCodeEditor.tsx | 7 ++++++- frontend/src/components/protocol/ScriptNodeInspector.tsx | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/protocol/PythonCodeEditor.tsx b/frontend/src/components/protocol/PythonCodeEditor.tsx index c813927a..d9bc030c 100644 --- a/frontend/src/components/protocol/PythonCodeEditor.tsx +++ b/frontend/src/components/protocol/PythonCodeEditor.tsx @@ -62,13 +62,15 @@ export function PythonCodeEditor({ onChange, rows = 16, resizable = false, + height: editorHeight, }: { value: string onChange: (value: string) => void rows?: number resizable?: boolean + height?: string }) { - const height = `${rows * 1.35}em` + const height = editorHeight ?? `${rows * 1.35}em` return (
- patchConfig({ code })} rows={16} resizable /> + patchConfig({ code })} + height="max(12rem, calc(100vh - 22rem))" + />
)} From c05155019d3338823d990e0a8fa9ebe1b80f3830 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 15:48:25 -0700 Subject: [PATCH 03/63] Clarify per-row dataset factor binding restriction --- .../components/protocol/edges/DatasetInputPanel.tsx | 11 +++++++---- frontend/src/lib/datasetRows.ts | 9 +++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx index d4e95444..fafb0e52 100644 --- a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx +++ b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx @@ -5,7 +5,7 @@ import { datasetsApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog' -import { datasetRowBindings, datasetRowTopologyError } from '@/lib/datasetRows' +import { datasetConfigIsFactorBound, datasetRowBindings, datasetRowFactorBindingError, datasetRowTopologyError } from '@/lib/datasetRows' import type { DatasetInput, ProtocolEdge, ProtocolNode } from '@/types/protocols' export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId: string; source: string; disabled?: boolean }) { @@ -22,7 +22,9 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId ? ['per_row', [...saved.columns].sort()] : ['whole_dataset'])) const inputKey = JSON.stringify(mode === 'per_row' ? [mode, [...columns].sort()] : [mode]) const hasChanges = inputKey !== appliedKey - const id = (nodes.find(node => node.id === source)?.data.config as { dataset_id?: string })?.dataset_id + const sourceNode = nodes.find(node => node.id === source) + const id = (sourceNode?.data.config as { dataset_id?: string })?.dataset_id + const factorBound = datasetConfigIsFactorBound(sourceNode) const otherRowDriver = datasetRowBindings({ nodes, edges }).find(binding => binding.edge.id !== edgeId && binding.input.mode === 'per_row' && binding.config?.dataset_id?.toLowerCase() !== id?.toLowerCase(), @@ -59,11 +61,12 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId

Dataset input

{(['whole_dataset', 'per_row'] as const).map(value => { - const optionDisabled = disabled || (value === 'per_row' && !!otherRowDriver) + const optionDisabled = disabled || (value === 'per_row' && (!!otherRowDriver || factorBound)) return })}
{otherRowDriver &&

Per row is unavailable because another dataset already uses it. All per-row inputs must use the same dataset. Use Whole dataset here, or switch the other dataset to Whole dataset first.

} + {factorBound &&

{datasetRowFactorBindingError}

} {schema.data &&

{schema.data.row_count} original rows. {mode === 'per_row' ? 'Each row runs one complete protocol; Agents share one driver and select their own columns.' : 'Each replicate runs the protocol with the whole dataset.'}

} {mode === 'per_row' && (displayedColumns.length <= 10 ? columnPicker :

{columns.length} of {availableColumns.length} columns selected

@@ -77,7 +80,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId
)} - {error &&

{error}

} + {error && !(factorBound && error === datasetRowFactorBindingError) &&

{error}

} {applied ?

{applied === 'per_row' ? 'Per row' : 'Whole dataset'} applied to the draft. Publish the experiment to update production runs and results.

:

Apply changes the draft connection. Publish the experiment to use it for production runs and results.

}
diff --git a/frontend/src/lib/datasetRows.ts b/frontend/src/lib/datasetRows.ts index 57aff1ef..74d9c523 100644 --- a/frontend/src/lib/datasetRows.ts +++ b/frontend/src/lib/datasetRows.ts @@ -1,6 +1,12 @@ import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' const handles = new Set(['dataset', 'resource', 'tool']) +export const datasetRowFactorBindingError = 'Per row is unavailable because this Dataset has configuration bound to an experimental factor. Per-row runs require the same dataset across all cells. Use Whole dataset here, or remove the factor bindings from this Dataset’s configuration first.' + +export function datasetConfigIsFactorBound(node?: ProtocolNode) { + return Object.keys(node?.data.factor_bindings ?? {}).some(path => path === 'config' || path.startsWith('config.')) +} + export function datasetRowBindings(graph: ProtocolGraph) { const nodes = new Map(graph.nodes.map(node => [node.id, node])) return graph.edges.flatMap(edge => { @@ -32,8 +38,7 @@ export function datasetRowTopologyError(graph: ProtocolGraph): string | null { const key = JSON.stringify(binding.input.columns) if (agents.has(binding.edge.target) && agents.get(binding.edge.target) !== key) return 'Duplicate Dataset connections must select identical columns.' agents.set(binding.edge.target, key) - const factors = (binding.source.data as ProtocolNode['data']).factor_bindings - if (Object.keys(factors ?? {}).some(path => path === 'config' || path.startsWith('config.'))) return 'The row driver cannot be factor-bound.' + if (datasetConfigIsFactorBound(binding.source)) return datasetRowFactorBindingError } return null } From d28dd7e0df48a01b6ba915e43c45d6b47d82d4b9 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 16:56:40 -0700 Subject: [PATCH 04/63] Show dataset IDs and make split action available after registration --- .../protocol/DatasetNodeInspector.tsx | 7 +++ .../protocol/RegisterDatasetDialog.test.tsx | 63 +++++++++++++++++++ .../protocol/RegisterDatasetDialog.tsx | 6 ++ 3 files changed, 76 insertions(+) create mode 100644 frontend/src/components/protocol/RegisterDatasetDialog.test.tsx diff --git a/frontend/src/components/protocol/DatasetNodeInspector.tsx b/frontend/src/components/protocol/DatasetNodeInspector.tsx index b210b05a..811b27a9 100644 --- a/frontend/src/components/protocol/DatasetNodeInspector.tsx +++ b/frontend/src/components/protocol/DatasetNodeInspector.tsx @@ -180,6 +180,13 @@ export function DatasetNodeInspector({ )} +
+ +

+ {config.dataset_id || 'No dataset selected.'} +

+
+ {datasetsQuery.isLoading ? ( ) : datasetsQuery.isError ? ( diff --git a/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx b/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx new file mode 100644 index 00000000..051a11a5 --- /dev/null +++ b/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx @@ -0,0 +1,63 @@ +import { useState, type ReactNode } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen } from '@testing-library/react' +import { afterEach, expect, it, vi } from 'vitest' +import { datasetsApi } from '@/api/client' +import type { Dataset } from '@/types/datasets' +import { nodeDataForDataset } from './datasetCatalog' +import { DatasetNodeInspector } from './DatasetNodeInspector' +import { RegisterDatasetDialog } from './RegisterDatasetDialog' + +vi.mock('@/components/ui/dialog', () => { + const Container = ({ children }: { children: ReactNode }) =>
{children}
+ return { + Dialog: ({ open, children }: { open: boolean; children: ReactNode }) => open ?
{children}
: null, + DialogContent: Container, DialogDescription: Container, DialogFooter: Container, + DialogHeader: Container, DialogTitle: Container, + } +}) +vi.mock('./NodeInspectorDialog', () => ({ + NodeInspectorDialog: ({ children }: { children: ReactNode }) =>
{children}
, +})) +vi.mock('./FactorBindableField', () => ({ + FactorBindableField: ({ children }: { children: (trigger: ReactNode) => ReactNode }) => children(null), +})) +vi.mock('./SplitDatasetDialog', () => ({ SplitDatasetDialog: () => null })) + +afterEach(() => vi.restoreAllMocks()) + +it('shows the split action immediately after registration while the list refresh is pending', async () => { + const dataset: Dataset = { + id: 'dc6a940a-2de2-4639-b58d-17bd5ac9c045', name: 'New dataset', + raw_path: '/data/datasets/new/raw.csv', raw_sha256: 'a'.repeat(64), + train_path: null, test_path: null, train_sha256: null, test_sha256: null, + split_method: null, split_group_column: null, split_test_size: null, split_seed: null, + target_column: null, description: null, dictionary_json: null, created_at: null, + } + vi.spyOn(datasetsApi, 'create').mockResolvedValue(dataset) + vi.spyOn(datasetsApi, 'list').mockImplementation(() => new Promise(() => {})) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + client.setQueryData(['datasets'], [{ ...dataset, id: 'existing', name: 'Existing dataset' }]) + + function Handoff() { + const [created, setCreated] = useState(null) + return created ? ( + {}} onDelete={() => {}} onClose={() => {}} + /> + ) : {}} onCreated={setCreated} /> + } + + render() + fireEvent.change(screen.getByLabelText('Name'), { target: { value: dataset.name } }) + fireEvent.change(document.getElementById('dataset-file')!, { + target: { files: [new File(['value\n1\n'], 'source.csv', { type: 'text/csv' })] }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Register dataset' })) + + expect(await screen.findByRole('button', { name: 'Split dataset' })).toBeTruthy() + expect(client.getQueryData(['datasets'])?.map((d) => d.id)).toEqual(['existing', dataset.id]) + client.clear() +}) diff --git a/frontend/src/components/protocol/RegisterDatasetDialog.tsx b/frontend/src/components/protocol/RegisterDatasetDialog.tsx index 7f267c9a..81b863d3 100644 --- a/frontend/src/components/protocol/RegisterDatasetDialog.tsx +++ b/frontend/src/components/protocol/RegisterDatasetDialog.tsx @@ -83,6 +83,12 @@ export function RegisterDatasetDialog({ }) }, onSuccess: (dataset) => { + // The inspector opens immediately and reads this list. Seed it before + // the handoff so its actions don't depend on the background refresh. + queryClient.setQueryData(['datasets'], (datasets = []) => [ + ...datasets.filter((existing) => existing.id !== dataset.id), + dataset, + ]) queryClient.invalidateQueries({ queryKey: ['datasets'] }) onCreated?.(dataset) reset() From 40b55af3d4bea6f0383af5a8d0e7c59d70fb2cff Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 17:03:49 -0700 Subject: [PATCH 05/63] Remove skill factors from design when their last binding is removed --- .../protocol/ProtocolCanvas.test.tsx | 40 +++++++++++++++++++ .../components/protocol/ProtocolCanvas.tsx | 4 +- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 52ec809d..28eee2f2 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -6,6 +6,7 @@ import { MemoryRouter } from 'react-router-dom' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { datasetsApi, experimentsApi, protocolsApi, skillsApi } from '@/api/client' import { protocolGraphQueryKey } from '@/lib/protocolGraph' +import type { Experiment } from '@/types/experiments' import { defaultAgentNodeData, defaultAnthropicModelNodeData, @@ -407,6 +408,45 @@ describe('ProtocolCanvas connector adds', () => { expect(screen.queryByRole('button', { name: /Create (another )?metric/ })).not.toBeInTheDocument() }) + it.each([false, true])('removes a Skill factor declaration only after its last binding is removed (shared: %s)', async (shared) => { + const factorName = 'Skill enabled' + const experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [{ name: factorName, levels: [true, false], level_type: 'boolean' }], metrics: [] }, + measurement_plan: null, dataset_ids: [], dataset_id: null, locked_at: null, + locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(skillsApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: (shared ? ['a', 'b'] : ['a']).map((id) => ({ + id, type: 'skill', position: { x: id === 'a' ? 100 : 300, y: 100 }, + data: { + label: `Skill ${id}`, config: { skill_id: id, skill_name: `Skill ${id}`, enabled: true }, + factor_bindings: { 'config.enabled': factorName }, + }, + })), + edges: [], + } + const { client } = renderCanvas(graph, experiment.id) + fireEvent.doubleClick(await screen.findByText('Skill a')) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor binding' })) + + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'a')?.data.factor_bindings).toEqual({}) + }) + if (shared) { + expect(save).not.toHaveBeenCalled() + } else { + await waitFor(() => expect(save).toHaveBeenCalledWith(experiment.id, { + design_spec: { factors: [], metrics: [] }, + })) + } + }) + it('opens a bound factor from the Script inspector', async () => { const user = userEvent.setup() const scriptData = defaultScriptNodeData('Score script') diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index e351fc22..ec64af41 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -2013,9 +2013,7 @@ export const ProtocolCanvas = forwardRef name) const nextNodes = nodes.map((node) => (node.id === nodeId ? { ...node, data } : node)) setNodes(nextNodes) - // Keep individual skill declarations visible for explicit removal or - // rebinding when the user switches to connector ownership. - if (previous?.type !== 'skill') void removeUnboundFactors(nextNodes, removedNames) + void removeUnboundFactors(nextNodes, removedNames) } // Client-side guardrail mirroring the backend's own connector validation From 5bfa287dbbeaed20495abf14ffa097b223855ef2 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 17:12:45 -0700 Subject: [PATCH 06/63] Consolidate skill factor creation and add all-or-none mode --- .../src/components/protocol/DesignTab.tsx | 6 +- .../protocol/FactorEditorDialog.tsx | 2 +- .../protocol/ProtocolCanvas.test.tsx | 13 +- .../components/protocol/ProtocolCanvas.tsx | 82 +++++------ .../protocol/ProtocolCanvasContext.tsx | 1 + .../protocol/SkillFactorDialog.test.tsx | 46 +++++++ .../components/protocol/SkillFactorDialog.tsx | 130 ++++++++++++++++++ .../components/protocol/SkillFactorEditor.tsx | 10 +- .../protocol/SkillNodeInspector.tsx | 38 +---- .../protocol/SkillTestSelectors.tsx | 7 + .../src/components/protocol/factorLevels.ts | 5 +- .../components/protocol/nodes/CircleNode.tsx | 4 +- .../protocol/nodes/ConnectorAddStub.tsx | 9 +- .../components/protocol/nodes/SkillNode.tsx | 5 +- .../protocol/useSkillTestSelection.test.tsx | 20 +++ .../protocol/useSkillTestSelection.ts | 8 +- frontend/src/lib/factorBindings.ts | 6 +- frontend/src/lib/skillFactors.test.ts | 7 + frontend/src/lib/skillFactors.ts | 16 ++- frontend/src/types/experiments.ts | 1 + frontend/src/types/protocols.ts | 3 +- src/asaree/services/design_generation.py | 5 +- src/asaree/services/factor_bindings.py | 2 +- src/asaree/services/protocol_revisions.py | 3 +- src/asaree/services/skill_factors.py | 38 ++++- tests/test_skill_factors.py | 65 ++++++++- 26 files changed, 411 insertions(+), 121 deletions(-) create mode 100644 frontend/src/components/protocol/SkillFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/SkillFactorDialog.tsx diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index bfca950d..413dab24 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -298,7 +298,7 @@ function FactorsEditor({ variant="ghost" size="icon-sm" aria-label="Remove factor" - title={factor.level_type === 'skill_selection' ? 'Remove factor and enable all connected skills' : 'Remove factor'} + title={(factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') ? 'Remove factor and enable all connected skills' : 'Remove factor'} disabled={disabled || deleteMutation.isPending} onClick={() => deleteMutation.mutate(factor.name)} > @@ -493,7 +493,7 @@ export function DesignTab({ ? ({ nodes: graphQuery.data.nodes, edges: graphQuery.data.edges } as unknown as ProtocolGraph) : undefined const unboundFactors = unboundFactorNames(experiment.design_spec, draftGraph) - const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => factor.level_type === 'skill_selection') }) + const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) }) const skillIssues = skillFactorIssues(draftGraph, factors, skillLibrary.data ? new Set(skillLibrary.data.map((skill) => skill.id)) : undefined) const impact = impactQuery.data @@ -870,7 +870,7 @@ export function DesignTab({ generateMutation.isPending || isAutosavingMetadata || !canGenerate || - unboundFactors.length > 0 || skillIssues.length > 0 || (factors.some((factor) => factor.level_type === 'skill_selection') && !skillLibrary.isSuccess) + unboundFactors.length > 0 || skillIssues.length > 0 || (factors.some((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) && !skillLibrary.isSuccess) } onClick={() => generateMutation.mutate()} > diff --git a/frontend/src/components/protocol/FactorEditorDialog.tsx b/frontend/src/components/protocol/FactorEditorDialog.tsx index b8fdb95f..946ab828 100644 --- a/frontend/src/components/protocol/FactorEditorDialog.tsx +++ b/frontend/src/components/protocol/FactorEditorDialog.tsx @@ -500,7 +500,7 @@ function ToolNamesLevelRow({ // remove button on DesignTab's own summary row), so this dialog only ever // edits, never deletes. export function FactorEditorDialog(props: Parameters[0]) { - if (props.factor.level_type === 'skill_selection') return + if ((props.factor.level_type === 'skill_selection' || props.factor.level_type === 'skill_toggle')) return return } diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 28eee2f2..047e7f8a 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -144,7 +144,7 @@ describe('ProtocolCanvas connector adds', () => { })) }) - it('creates a connector skill factor and preserves the connected skill nodes', async () => { + it.each(['skill_selection', 'skill_toggle'] as const)('creates a %s skill factor from the node hover menu and preserves connected nodes', async (mode) => { const experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, @@ -162,15 +162,17 @@ describe('ProtocolCanvas connector adds', () => { edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'skill' })), } const { client } = renderCanvas(graph, 'experiment-1') - const makeFactor = document.querySelector('[data-testid="rf__node-agent"] button.bg-chart-2') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') expect(makeFactor).not.toBeNull() fireEvent.click(makeFactor!) await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'skill_toggle' ? 'All agent skills on/off' : 'Skill levels' })) fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) - await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Skills', level_type: 'skill_selection', levels: [['a'], ['b']], level_labels: ['Skill a', 'Skill b'] }] }) }))) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Skills', level_type: mode, levels: mode === 'skill_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'skill_toggle' ? ['All enabled', 'All disabled'] : ['Skill a', 'Skill b'] }] }) }))) await waitFor(() => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ skill_selection: 'Writer:Skills' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.skill_factor_mode).toBe(mode) expect(persisted?.nodes.filter((node) => node.type === 'skill')).toHaveLength(2) }) }) @@ -431,8 +433,9 @@ describe('ProtocolCanvas connector adds', () => { edges: [], } const { client } = renderCanvas(graph, experiment.id) - fireEvent.doubleClick(await screen.findByText('Skill a')) - fireEvent.click(await screen.findByRole('button', { name: 'Remove factor binding' })) + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + fireEvent.click(makeFactor!) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) await waitFor(() => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index ec64af41..9d295823 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -1,11 +1,10 @@ import { DatasetRowSelector } from './DatasetRowSelector' import { useDatasetRowSelection } from './useDatasetRowSelection' import { rowBindingForNode } from '@/lib/datasetRows' -import { connectedSkills, reconcileSkillFactor, skillFactorOwner, SKILL_FACTOR_PATH } from '@/lib/skillFactors' -import { SkillFactorEditor } from './SkillFactorEditor' +import { reconcileSkillFactor, skillFactorOwner, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import { SkillFactorDialog } from './SkillFactorDialog' import { SkillTestSelectors } from './SkillTestSelectors' import { useSkillTestSelection } from './useSkillTestSelection' -import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { @@ -415,7 +414,10 @@ export const ProtocolCanvas = forwardRef name !== factorName)) const data: Record = { ...n.data, factor_bindings: next } - if (bindings[SKILL_FACTOR_PATH] === factorName) delete data.skill_selection + if (bindings[SKILL_FACTOR_PATH] === factorName) { + delete data.skill_selection + delete data.skill_factor_mode + } return { ...n, data } }) }) @@ -470,8 +472,7 @@ export const ProtocolCanvas = forwardRef(null) const [editingFactorName, setEditingFactorName] = useState(null) - const [skillFactorDraft, setSkillFactorDraft] = useState<{ nodeId: string; factor: DesignFactor; oldName?: string } | null>(null) - const [skillFactorError, setSkillFactorError] = useState(null) + const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) const [addPanelOpen, setAddPanelOpen] = useState(false) // The second level of the add-node panel: AddNodePanel's "MCP Servers" // entry swaps the browser in over it, and its Back button returns. Only @@ -1326,22 +1327,9 @@ export const ProtocolCanvas = forwardRef { if (!experimentId || experimentLocked) return - const graph = toPersistedGraph(nodes, edges) - const node = graph.nodes.find((node) => node.id === nodeId) - if (!node) return - const connected = connectedSkills(graph, nodeId) - if (connected.some((skill) => Object.keys(skill.data.factor_bindings ?? {}).length > 0)) { - setSkillFactorError('Remove individual skill factor bindings before making a connector factor. Their declarations must then be removed or rebound in Design.') - return - } - const oldName = node.data.factor_bindings?.[SKILL_FACTOR_PATH] - const existing = experimentQuery.data?.design_spec?.factors?.find((factor) => factor.name === oldName) - const names = new Set(experimentQuery.data?.design_spec?.factors?.map((factor) => factor.name)) - const base = `${node.data.label || 'Agent'}:Skills` - let name = base - for (let suffix = 2; names.has(name); suffix++) name = `${base} (${suffix})` - setSkillFactorDraft({ nodeId, oldName, factor: reconcileSkillFactor(existing ?? { name, level_type: 'skill_selection', levels: [], level_labels: [] }, graph, nodeId) }) - }, [experimentId, experimentLocked, nodes, edges, experimentQuery.data]) + setSelectedNodeId(null) + setSkillFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) const metricsByNode = useMemo(() => { const definitions = new Map((experimentQuery.data?.measurement_plan?.metrics ?? []).map((metric) => [metric.id, metric])) const bindings = new Map>() @@ -1885,7 +1873,7 @@ export const ProtocolCanvas = forwardRef { if (experimentLocked) return setNodes((current) => current.map((node) => { - const factor = factors.find((factor) => factor.level_type === 'skill_selection' && (node.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factor.name) + const factor = factors.find((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') && (node.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factor.name) if (!factor || JSON.stringify(node.data.skill_selection) === JSON.stringify(factor.levels[0] ?? [])) return node return { ...node, data: { ...node.data, skill_selection: factor.levels[0] ?? [] } } })) @@ -2336,8 +2324,6 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2488,29 +2474,43 @@ export const ProtocolCanvas = forwardRef )} - {skillFactorError && setSkillFactorError(null)}>Individual skill factors must be removed{skillFactorError}} - {skillFactorDraft && !open && setSkillFactorDraft(null)} - onSave={async (factor) => { - const graph = toPersistedGraph(nodes, edges) - if (connectedSkills(graph, skillFactorDraft.nodeId).some((skill) => Object.keys(skill.data.factor_bindings ?? {}).length)) throw new Error('Remove individual skill factor bindings first.') + {skillFactorNodeId && setSkillFactorNodeId(null)} + onSave={async (factor, ownerId, removeNames) => { const fresh = await experimentsApi.get(experimentId!) - if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== skillFactorDraft.oldName)) throw new Error('A factor with this name already exists.') - const next = reconcileSkillFactor(factor, graph, skillFactorDraft.nodeId) - await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== skillFactorDraft.oldName), next] } }) - setNodes((nodes) => nodes.map((node) => node.id === skillFactorDraft.nodeId ? { ...node, data: { ...node.data, skill_selection: next.levels[0] ?? [], factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [SKILL_FACTOR_PATH]: next.name } } } : node)) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && !removeNames.includes(existing.name))) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileSkillFactor(factor, toPersistedGraph(nodes, edges), ownerId) + const oldIndividualName = nodes.find((node) => node.id === ownerId)?.data.factor_bindings as Record | undefined + const preserveIndividual = next.level_type === 'boolean' ? oldIndividualName?.['config.enabled'] : undefined + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => !removeNames.includes(existing.name)), next] } }) + removeNames.filter((name) => name !== preserveIndividual).forEach(removeFactorBindings) + if (preserveIndividual) { + renameFactorBindings(preserveIndividual, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { skill_selection: next.levels[0] ?? [], skill_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH]: next.name }, + } } : node)) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} - onRemove={skillFactorDraft.oldName ? async () => { + onRemove={async (name) => { const fresh = await experimentsApi.get(experimentId!) - await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== skillFactorDraft.oldName) } }) - removeFactorBindings(skillFactorDraft.oldName!) + const graph = toPersistedGraph(nodes, edges) + const individual = graph.nodes.find((node) => node.id === skillFactorNodeId)?.data.factor_bindings?.['config.enabled'] === name + const shared = individual && graph.nodes.some((node) => node.id !== skillFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === skillFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== 'config.enabled')) } } : node)) + else removeFactorBindings(name) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) - } : undefined} + }} />} {editingFactor && ( void + // Opens the consolidated factor dialog from a Skill node's hover toolbar. requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). diff --git a/frontend/src/components/protocol/SkillFactorDialog.test.tsx b/frontend/src/components/protocol/SkillFactorDialog.test.tsx new file mode 100644 index 00000000..fe76db33 --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorDialog.test.tsx @@ -0,0 +1,46 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { SkillFactorDialog } from './SkillFactorDialog' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'skill', type: 'skill', data: { label: 'Summarize', config: { skill_id: 'a' }, factor_bindings: { 'config.enabled': 'Skill enabled' } } }, + ], + edges: [{ source: 'skill', target: 'agent', targetHandle: 'skill' }], +} as unknown as ProtocolGraph + +it('requires explicit confirmation before replacing individual factors with a group factor', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.click(screen.getByRole('button', { name: 'All agent skills on/off' })) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByText(/removes these factor declarations/)).toHaveTextContent('Skill enabled') + fireEvent.click(screen.getByRole('checkbox', { name: 'Replace these factors' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Writer:Skills', level_type: 'skill_toggle', levels: [['a'], []], level_labels: ['All enabled', 'All disabled'] }, 'agent', ['Skill enabled'])) +}) + +it('only offers individual factors for disconnected skills', () => { + render() + expect(screen.getByRole('button', { name: 'Skill levels' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'All agent skills on/off' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'This skill on/off' })).toHaveAttribute('aria-pressed', 'true') +}) + +it('opens the existing group factor when a shared skill has multiple connected agents', () => { + const sharedGraph = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'First agent' } }, + { id: 'second', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Second agent', skill_selection: ['a'], skill_factor_mode: 'skill_toggle', factor_bindings: { skill_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { skill_id: 'a' } } }, + ], + edges: [...graph.edges, { id: 'second-edge', source: 'skill', target: 'second', targetHandle: 'skill' }], + } as ProtocolGraph + render() + expect(screen.getByRole('combobox')).toHaveTextContent('Second agent') + expect(screen.getByRole('button', { name: 'All agent skills on/off' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByLabelText('Factor name')).toHaveValue('Group') +}) diff --git a/frontend/src/components/protocol/SkillFactorDialog.tsx b/frontend/src/components/protocol/SkillFactorDialog.tsx new file mode 100644 index 00000000..06713b1d --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorDialog.tsx @@ -0,0 +1,130 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedSkills, reconcileSkillFactor, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +type Mode = 'boolean' | 'skill_selection' | 'skill_toggle' +const modes: Record = { + boolean: 'This skill on/off', + skill_selection: 'Skill levels', + skill_toggle: 'All agent skills on/off', +} + +export function SkillFactorDialog({ skillNodeId, graph, factors, onClose, onSave, onRemove }: { + skillNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, removeNames: string[]) => Promise + onRemove: (name: string) => Promise +}) { + const skill = graph.nodes.find((node) => node.id === skillNodeId)! + const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === skillNodeId && edge.target === node.id && edge.targetHandle === 'skill')) + const [agentId, setAgentId] = useState(agents.find((agent) => agent.data.factor_bindings?.[SKILL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, SKILL_FACTOR_PATH) ?? existingFor(skillNodeId, 'config.enabled') + const initial = initialFactor && initialFactor.level_type !== 'boolean' && initialFactor.level_type ? reconcileSkillFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState((initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = nextMode === 'boolean' ? skillNodeId : nextAgentId + const existing = existingFor(ownerId, nextMode === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level) => level ? 'Enabled' : 'Disabled'), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Skill'), nextMode === 'boolean' ? 'Enabled' : 'Skills', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return nextMode === 'boolean' ? factor : reconcileSkillFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level) => level ? 'Enabled' : 'Disabled') } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const [confirmed, setConfirmed] = useState(false) + const ownerId = mode === 'boolean' ? skillNodeId : agentId + const current = existingFor(ownerId, mode === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH) + const affected = mode === 'boolean' ? [skill] : connectedSkills(graph, agentId) + const conflicts = [...new Set(mode === 'boolean' + ? agents.flatMap((agent) => agent.data.factor_bindings?.[SKILL_FACTOR_PATH] ? [agent.data.factor_bindings[SKILL_FACTOR_PATH]] : []) + : affected.flatMap((node) => Object.values(node.data.factor_bindings ?? {})))] + const changingType = !!current && (current.level_type ?? 'boolean') !== mode + const replacements = [...new Set([...(current ? [current.name] : []), ...conflicts])] + const needsConfirmation = changingType || conflicts.length > 0 + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setAgentId(nextAgentId) + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setConfirmed(false) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, replacements) + onClose() + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the skill factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Skill factor + Choose how skills vary across experimental cells. + +
+ {agents.length > 1 &&
} +
{(Object.keys(modes) as Mode[]).map((value) => )}
+ {!agentId &&

Connect this Skill to an Agent to use Agent-level factors.

} +

{mode === 'boolean' ? 'Enable or disable this Skill node.' : mode === 'skill_selection' ? 'Each cell receives exactly one connected skill.' : 'Each cell enables every connected skill, including deactivated skills, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {mode !== 'boolean' && affected.length < (mode === 'skill_toggle' ? 1 : 2) &&

Connect at least {mode === 'skill_toggle' ? 'one skill' : 'two different skills'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'skill_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => (node.data.config as { skill_id?: string })?.skill_id === (level as string[])[0])?.data.label ?? 'Unavailable skill')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+ {needsConfirmation &&
+

Saving replaces {changingType ? `the current ${modes[(current!.level_type ?? 'boolean') as Mode]} factor` : 'conflicting factors'}{conflicts.length ? ` and removes these factor declarations and all their bindings: ${conflicts.join(', ')}` : ''}.

+ +
} +

Agent-level factors follow current Skill connections. Changes require design review and regeneration.

+ {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/SkillFactorEditor.tsx b/frontend/src/components/protocol/SkillFactorEditor.tsx index 6c6a4cae..0063a604 100644 --- a/frontend/src/components/protocol/SkillFactorEditor.tsx +++ b/frontend/src/components/protocol/SkillFactorEditor.tsx @@ -42,8 +42,8 @@ export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove return !saving && onOpenChange(open)}> - Skills as levels - Each cell receives exactly one connected skill. Add or disconnect Skill nodes on the canvas to change the levels. + {factor.level_type === 'skill_toggle' ? 'All agent skills on/off' : 'Skills as levels'} + {factor.level_type === 'skill_toggle' ? 'Each cell receives all connected skills or none.' : 'Each cell receives exactly one connected skill.'} Add or disconnect Skill nodes on the canvas to change the levels.
@@ -53,10 +53,10 @@ export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove {draft.levels.map((level, index) => { const id = (level as string[])[0] const skill = skills.data?.find((skill) => skill.id === id) - return
+ return
-

{skill?.name ?? labels[index] ?? 'Unavailable skill'}{index === 0 ? ' · default test selection' : ''}

- {skills.isSuccess && !skill &&

Unavailable: disconnect or restore this skill.

} +

{factor.level_type === 'skill_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : skill?.name ?? labels[index] ?? 'Unavailable skill'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'skill_toggle' && skills.isSuccess && !skill &&

Unavailable: disconnect or restore this skill.

} setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} />
diff --git a/frontend/src/components/protocol/SkillNodeInspector.tsx b/frontend/src/components/protocol/SkillNodeInspector.tsx index ec7582fa..dd3188f2 100644 --- a/frontend/src/components/protocol/SkillNodeInspector.tsx +++ b/frontend/src/components/protocol/SkillNodeInspector.tsx @@ -8,7 +8,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { skillsApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { ProtocolNode, SkillNodeData } from '@/types/protocols' @@ -28,18 +27,12 @@ const ACCENT = nodeAccent('skill') // across many protocols, so an inline edit would silently change all of them. export function SkillNodeInspector({ node, - experimentId, - factorNodeLabel, onChange, onDelete, onClose, connectorFactorName, }: { node: (ProtocolNode & { data: SkillNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, this node's own plain header title. - factorNodeLabel: string onChange: (nodeId: string, data: SkillNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -57,10 +50,6 @@ export function SkillNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -83,29 +72,10 @@ export function SkillNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
: bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

Off: the wired agent never sees this skill at all.

-
- patchConfig({ enabled: checked })} /> -
- )} - } + {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this skill at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
}
diff --git a/frontend/src/components/protocol/SkillTestSelectors.tsx b/frontend/src/components/protocol/SkillTestSelectors.tsx index 2a816523..ea2b239e 100644 --- a/frontend/src/components/protocol/SkillTestSelectors.tsx +++ b/frontend/src/components/protocol/SkillTestSelectors.tsx @@ -5,6 +5,13 @@ import type { useSkillTestSelection } from './useSkillTestSelection' export function SkillTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedSkills(selection.graph!, node.id) + if (node.data.skill_factor_mode === 'skill_toggle') return
+

{node.data.label}: skills for this test

+ +
return

{node.data.label}: skill for this test

} -
{(Object.keys(modes) as Mode[]).map((value) => )}
- {!agentId &&

Connect this Skill to an Agent to use Agent-level factors.

} +
{(Object.keys(modes) as Mode[]).map((value) => )}
+ {blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Skill node.' : mode === 'skill_selection' ? 'Each cell receives exactly one connected skill.' : 'Each cell enables every connected skill, including deactivated skills, or disables them all.'}

setDraft({ ...draft, name: event.target.value })} />
{mode !== 'boolean' && affected.length < (mode === 'skill_toggle' ? 1 : 2) &&

Connect at least {mode === 'skill_toggle' ? 'one skill' : 'two different skills'} before generating cells.

} @@ -116,15 +106,12 @@ export function SkillFactorDialog({ skillNodeId, graph, factors, onClose, onSave
{mode !== 'boolean' && <>}
)}
- {needsConfirmation &&
-

Saving replaces {changingType ? `the current ${modes[(current!.level_type ?? 'boolean') as Mode]} factor` : 'conflicting factors'}{conflicts.length ? ` and removes these factor declarations and all their bindings: ${conflicts.join(', ')}` : ''}.

- -
}

Agent-level factors follow current Skill connections. Changes require design review and regeneration.

{current && } + } {error &&

{error}

}
- +
} diff --git a/frontend/src/components/protocol/SkillNodeInspector.tsx b/frontend/src/components/protocol/SkillNodeInspector.tsx index dd3188f2..796a1998 100644 --- a/frontend/src/components/protocol/SkillNodeInspector.tsx +++ b/frontend/src/components/protocol/SkillNodeInspector.tsx @@ -1,6 +1,6 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useQuery } from '@tanstack/react-query' -import { ScrollText } from 'lucide-react' +import { Power, ScrollText, Split, Trash2 } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -72,6 +72,19 @@ export function SkillNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > +
+

+ Make factor has moved to the node toolbar. Hover over the Skill node and click the

+
{connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:

Off: the wired agent never sees this skill at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
patchConfig({ enabled: checked })} /> diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index a0833e64..acf2dcb8 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -287,7 +287,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Skill - + {/* Dataset -- the data an agent works ON, as opposed to the Tool connector's "capabilities it works WITH". UNCAPPED, like Skill/Knowledge/Tool: a cell's workspace holds one dataset per named diff --git a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx index 37863b40..40654fea 100644 --- a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx +++ b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx @@ -23,6 +23,7 @@ export function ConnectorAddStub({ alwaysVisible = false, factorName, factorLevelCount, + factorOnOff = false, }: { nodeId: string slot: ConnectorSlot @@ -31,6 +32,7 @@ export function ConnectorAddStub({ alwaysVisible?: boolean factorName?: string factorLevelCount?: number + factorOnOff?: boolean }) { const connections = useNodeConnections({ id: nodeId, handleType: 'target', handleId: slot }) const { requestConnectorAdd } = useProtocolCanvasActions() @@ -66,7 +68,7 @@ export function ConnectorAddStub({ className={side === 'right' ? 'group/stub absolute top-1/2 -right-11 -translate-y-1/2' : side === 'top' ? 'group/stub absolute -top-11 -translate-x-1/2' : 'group/stub absolute -bottom-11 -translate-x-1/2'} style={side !== 'right' ? { left } : undefined} > - {factorName && Skills · {factorLevelCount ?? 0} levels} + {factorName && Skills · {factorOnOff ? 'on/off' : `${factorLevelCount ?? 0} levels`}} )}
+ {legacy &&

This Dataset has an existing configuration factor: {legacy.name}. Remove it before creating a Dataset factor.

} + {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Dataset node.' : mode === 'dataset_selection' ? 'Each cell receives exactly one connected dataset.' : 'Each cell enables every connected dataset, including deactivated datasets, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {mode !== 'boolean' && affected.length < (mode === 'dataset_toggle' ? 1 : 2) &&

Connect at least {mode === 'dataset_toggle' ? 'one dataset' : 'two different datasets'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'dataset_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => (node.data.config as { dataset_id?: string })?.dataset_id === (level as string[])[0])?.data.label ?? 'Unavailable dataset')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+

Agent-level factors follow current Dataset connections. Changes require design review and regeneration.

+ {current && } + } + {error &&

{error}

} + + + + +} diff --git a/frontend/src/components/protocol/DatasetFactorEditor.tsx b/frontend/src/components/protocol/DatasetFactorEditor.tsx new file mode 100644 index 00000000..8bf38aa9 --- /dev/null +++ b/frontend/src/components/protocol/DatasetFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { datasetsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +export function DatasetFactorEditor({ factor, open, onOpenChange, onSave, onRemove }: { + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const datasets = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list() }) + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the dataset factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'dataset_toggle' ? 'All datasets on/off' : 'Datasets as levels'} + {factor.level_type === 'dataset_toggle' ? 'Each cell receives all connected datasets or none.' : 'Each cell receives exactly one connected dataset.'} Add or disconnect Dataset nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different datasets before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const dataset = datasets.data?.find((dataset) => dataset.id === id) + return
+
+

{factor.level_type === 'dataset_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : dataset?.name ?? labels[index] ?? 'Unavailable dataset'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'dataset_toggle' && datasets.isSuccess && !dataset &&

Unavailable: disconnect or restore this dataset.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+ {datasets.isError &&

Could not load the dataset library.

} +

Individual dataset switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected datasets and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/DatasetNodeInspector.tsx b/frontend/src/components/protocol/DatasetNodeInspector.tsx index 811b27a9..ad33503f 100644 --- a/frontend/src/components/protocol/DatasetNodeInspector.tsx +++ b/frontend/src/components/protocol/DatasetNodeInspector.tsx @@ -1,7 +1,7 @@ import { useState } from 'react' import { nodeAccent } from '@/lib/nodeAccent' import { useQuery } from '@tanstack/react-query' -import { Database } from 'lucide-react' +import { Database, Power, Split, Trash2 } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -9,7 +9,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { datasetsApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import { SplitDatasetDialog } from './SplitDatasetDialog' import type { DatasetNodeData, ProtocolNode } from '@/types/protocols' @@ -52,25 +51,7 @@ function formatTestSize(fraction: number): string { return `${+(fraction * 100).toFixed(2)}%` } -// Same floating-dialog shell as McpToolNodeInspector, but with NO "which -// dataset" picker: the dataset IS the node, chosen when it was added from the -// canvas's Datasets browser (DatasetBrowserPanel/nodeDataForDataset), the same -// model as Skill and the server-dedicated MCP node types. So this is a -// read-out of the bound dataset -- description, target column, hashes, split -// state, data dictionary -- plus the two things that ARE this node's to -// change: its Enabled switch (factor-bindable) and its split. -// -// Which is also why the experiment's own attached-dataset list is synced from -// the canvas's Dataset nodes rather than here (ProtocolCanvas's -// syncExperimentDatasets effect): with no picker, the binding can no longer -// change from inside this dialog. -// -// The one exception to "no picker here" is the title row's "Make factor" -// button: a dataset_config factor's levels are whole Dataset configs, so -// picking among registered datasets happens in FactorEditorDialog's own -// DatasetConfigLevelRow. That's the supported way to run one experiment -// across several datasets -- the Dataset connector itself is capped at one -// node per agent, since a cell's workspace holds exactly one dataset. +// Dataset identity is pinned by the browser; factor controls live in the hover toolbar. // // A dataset_id in an imported protocol JSON is per-account/environment (same // reasoning as an MCP Tool node's server_id), so an imported node can name a @@ -79,18 +60,18 @@ function formatTestSize(fraction: number): string { // dataset from the browser -- there's nothing here to repoint. export function DatasetNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: DatasetNodeData }) | null - experimentId: string | null + experimentId?: string | null // The agent-traced display label (see bindableFields.ts's // agentTracedLabel) -- distinct from data.label, which is this node's own // plain label shown in the header title. - factorNodeLabel: string + factorNodeLabel?: string + connectorFactorName?: string onChange: (nodeId: string, data: DatasetNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -109,10 +90,6 @@ export function DatasetNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -130,55 +107,28 @@ export function DatasetNodeInspector({ <> onChange(node.id, { ...data, label })} /> - {/* In the title row rather than beside a field, because there is no - "which dataset" field here to sit beside -- the dataset IS the - node. Same position and visual identity as the Agent inspector's - own Active factor control, but this one - binds one specific field (`config`, the whole node) directly - instead of opening a per-node field picker: a Dataset node has - exactly one whole-node factor worth making, so a picker listing - one entry would be a step with no choice in it. */} - bindFactor('config', name)} - onUnbind={() => unbindFactor('config')} - > - {(trigger) => trigger} - } onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

Off: no dataset context is given to the wired agent.

-
- patchConfig({ enabled: checked })} /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Dataset node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this dataset at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
}
diff --git a/frontend/src/components/protocol/DatasetTestSelectors.tsx b/frontend/src/components/protocol/DatasetTestSelectors.tsx new file mode 100644 index 00000000..03d43c31 --- /dev/null +++ b/frontend/src/components/protocol/DatasetTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedDatasets, datasetId } from '@/lib/datasetFactors' +import type { useDatasetTestSelection } from './useDatasetTestSelection' + +export function DatasetTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedDatasets(selection.graph!, node.id) + if (node.data.dataset_factor_mode === 'dataset_toggle') return
+

{node.data.label}: datasets for this test

+ +
+ return
+

{node.data.label}: dataset for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 413dab24..d62b5c5f 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -1,3 +1,5 @@ +import { datasetsApi } from '@/api/client' +import { datasetFactorIssues } from '@/lib/datasetFactors' import { useCallback, useEffect, useRef, useState, type RefObject } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' @@ -494,6 +496,8 @@ export function DesignTab({ : undefined const unboundFactors = unboundFactorNames(experiment.design_spec, draftGraph) const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) }) + const datasetLibrary = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list(), enabled: factors.some((factor) => factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') }) + const datasetIssues = datasetFactorIssues(draftGraph, factors, datasetLibrary.data ? new Set(datasetLibrary.data.map((dataset) => dataset.id)) : undefined) const skillIssues = skillFactorIssues(draftGraph, factors, skillLibrary.data ? new Set(skillLibrary.data.map((skill) => skill.id)) : undefined) const impact = impactQuery.data @@ -863,14 +867,14 @@ export function DesignTab({ Unbound factor{unboundFactors.length === 1 ? '' : 's'}: {unboundFactors.join(', ')}. Rebind on the canvas or remove from this design.

)} - {skillIssues.map((issue) =>

{issue}

)} + {[...skillIssues, ...datasetIssues].map((issue) =>

{issue}

)}
} -
{(Object.keys(modes) as Mode[]).map((value) => )}
+
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Skill node.' : mode === 'skill_selection' ? 'Each cell receives exactly one connected skill.' : 'Each cell enables every connected skill, including deactivated skills, or disables them all.'}

setDraft({ ...draft, name: event.target.value })} />
diff --git a/frontend/src/components/protocol/SkillFactorEditor.tsx b/frontend/src/components/protocol/SkillFactorEditor.tsx index 0063a604..afb725f7 100644 --- a/frontend/src/components/protocol/SkillFactorEditor.tsx +++ b/frontend/src/components/protocol/SkillFactorEditor.tsx @@ -42,7 +42,7 @@ export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove return !saving && onOpenChange(open)}> - {factor.level_type === 'skill_toggle' ? 'All agent skills on/off' : 'Skills as levels'} + {factor.level_type === 'skill_toggle' ? 'All skills on/off' : 'Skills as levels'} {factor.level_type === 'skill_toggle' ? 'Each cell receives all connected skills or none.' : 'Each cell receives exactly one connected skill.'} Add or disconnect Skill nodes on the canvas to change the levels.
diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 5aba885a..d465cf66 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -445,6 +445,7 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel const result: UnboundField[] = [] for (const node of nodes) { if (node.type === 'skill' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'skill' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.skill_selection)) continue + if (node.type === 'dataset' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'dataset' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.dataset_selection)) continue const bindings = (node.data as { factor_bindings?: Record })?.factor_bindings ?? {} const label = agentTracedLabel(node, edges, nodes) const wholeConfigBound = !!bindings.config diff --git a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx index fafb0e52..7df79e61 100644 --- a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx +++ b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx @@ -24,7 +24,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId const hasChanges = inputKey !== appliedKey const sourceNode = nodes.find(node => node.id === source) const id = (sourceNode?.data.config as { dataset_id?: string })?.dataset_id - const factorBound = datasetConfigIsFactorBound(sourceNode) + const factorBound = datasetConfigIsFactorBound(sourceNode) || !!nodes.find(node => node.id === edge?.target)?.data.factor_bindings?.dataset_selection const otherRowDriver = datasetRowBindings({ nodes, edges }).find(binding => binding.edge.id !== edgeId && binding.input.mode === 'per_row' && binding.config?.dataset_id?.toLowerCase() !== id?.toLowerCase(), diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index 8ed84d6e..c91ffff8 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -23,6 +23,8 @@ export type LevelType = | 'tool_names' | 'skill_selection' | 'skill_toggle' + | 'dataset_selection' + | 'dataset_toggle' export const LEVEL_TYPE_LABELS: Record = { string: 'String', @@ -36,7 +38,9 @@ export const LEVEL_TYPE_LABELS: Record = { dataset_config: 'Dataset', tool_names: 'Tools allowed', skill_selection: 'Skills as levels', - skill_toggle: 'All agent skills on/off', + skill_toggle: 'All skills on/off', + dataset_selection: 'Dataset levels', + dataset_toggle: 'All datasets on/off', } // Whether a level of this kind is a structured value (an object, or -- for @@ -53,7 +57,9 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'dataset_config' || type === 'tool_names' || type === 'skill_selection' || - type === 'skill_toggle' + type === 'skill_toggle' || + type === 'dataset_selection' || + type === 'dataset_toggle' ) } diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index acf2dcb8..01cbf2af 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -321,7 +321,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Dataset - + {/* Knowledge -- registered OKF bundles, each a directory of Markdown concepts on the SERVER's disk that the agent reads AND writes as it works (see OkfBundleNodeData). Its own slot rather than sharing diff --git a/frontend/src/components/protocol/nodes/DatasetNode.tsx b/frontend/src/components/protocol/nodes/DatasetNode.tsx index 2874a544..362b6b65 100644 --- a/frontend/src/components/protocol/nodes/DatasetNode.tsx +++ b/frontend/src/components/protocol/nodes/DatasetNode.tsx @@ -4,6 +4,7 @@ import { Database } from 'lucide-react' import type { DatasetNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' import { CircleNode } from './CircleNode' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' // Declares which registered dataset an Agent's workspace tools operate on -- // a real runtime effect once wired (see DatasetNodeData's own comment in @@ -22,7 +23,9 @@ const ACCENT = nodeAccent('dataset') export function DatasetNode({ id, data, selected }: NodeProps & { data: DatasetNodeData }) { const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestDatasetFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.datasetFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) return ( requestDatasetFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/useDatasetTestSelection.test.tsx b/frontend/src/components/protocol/useDatasetTestSelection.test.tsx new file mode 100644 index 00000000..49cff115 --- /dev/null +++ b/frontend/src/components/protocol/useDatasetTestSelection.test.tsx @@ -0,0 +1,51 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, expect, it, vi } from 'vitest' +import { datasetsApi } from '@/api/client' +import type { ProtocolGraph } from '@/types/protocols' +import { useDatasetTestSelection } from './useDatasetTestSelection' + +afterEach(() => vi.restoreAllMocks()) +it('uses all-or-none defaults per agent without altering shared dataset switches', async () => { + vi.spyOn(datasetsApi, 'list').mockResolvedValue([{ id: 'a' }] as Awaited>) + const graph = { + nodes: [ + { id: 'one', type: 'agent', data: { dataset_factor_mode: 'dataset_toggle', dataset_selection: ['a'], factor_bindings: { dataset_selection: 'One' } } }, + { id: 'two', type: 'agent', data: { dataset_factor_mode: 'dataset_toggle', dataset_selection: [], factor_bindings: { dataset_selection: 'Two' } } }, + { id: 'a', type: 'dataset', data: { config: { dataset_id: 'a', enabled: false } } }, + ], + edges: ['one', 'two'].map((target) => ({ source: 'a', target, targetHandle: 'dataset' })), + } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result } = renderHook(() => useDatasetTestSelection(graph), { wrapper }) + await waitFor(() => expect(result.current.error).toBeNull()) + expect(result.current.selections).toEqual({ one: 'all', two: 'none' }) + act(() => result.current.setChoice('one', 'none')) + expect(result.current.options).toEqual({ dataset_selections: { one: 'none', two: 'none' } }) + expect(JSON.stringify(graph)).toBe(original) +}) +it('keeps preview selections scoped to the agent and does not alter the first level', async () => { + vi.spyOn(datasetsApi, 'list').mockResolvedValue([{ id: 'a' }, { id: 'b' }] as Awaited>) + const graph = { + nodes: [ + { id: 'one', type: 'agent', data: { dataset_selection: ['a'], factor_bindings: { dataset_selection: 'One' } } }, + { id: 'two', type: 'agent', data: { dataset_selection: ['b'], factor_bindings: { dataset_selection: 'Two' } } }, + ...['a', 'b'].map((id) => ({ id, type: 'dataset', data: { config: { dataset_id: id } } })), + ], + edges: ['one', 'two'].flatMap((target) => ['a', 'b'].map((source) => ({ source, target, targetHandle: 'dataset' }))), + } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result, rerender } = renderHook(({ nodeId }) => useDatasetTestSelection(graph, nodeId), { wrapper, initialProps: { nodeId: undefined as string | undefined } }) + await waitFor(() => expect(result.current.error).toBeNull()) + expect(result.current.selections).toEqual({ one: 'a', two: 'b' }) + act(() => result.current.setChoice('one', 'b')) + expect(result.current.options).toEqual({ dataset_selections: { one: 'b', two: 'b' } }) + rerender({ nodeId: 'one' }) + expect(result.current.selections).toEqual({ one: 'a' }) + expect(JSON.stringify(graph)).toBe(original) +}) diff --git a/frontend/src/components/protocol/useDatasetTestSelection.ts b/frontend/src/components/protocol/useDatasetTestSelection.ts new file mode 100644 index 00000000..b796f929 --- /dev/null +++ b/frontend/src/components/protocol/useDatasetTestSelection.ts @@ -0,0 +1,24 @@ +import { useEffect, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { datasetsApi } from '@/api/client' +import { connectedDatasets, datasetId, DATASET_FACTOR_PATH } from '@/lib/datasetFactors' +import type { ProtocolGraph } from '@/types/protocols' + +export function useDatasetTestSelection(graph: ProtocolGraph | undefined, nodeId?: string) { + const [choices, setChoices] = useState>({}) + const library = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list(), enabled: !!graph?.nodes.some((node) => node.data.factor_bindings?.[DATASET_FACTOR_PATH]) }) + const owners = graph?.nodes.filter((node) => node.data.factor_bindings?.[DATASET_FACTOR_PATH] && (!nodeId || node.id === nodeId)) ?? [] + const signature = JSON.stringify(owners.map((node) => [node.id, node.data.dataset_factor_mode, node.data.dataset_selection, connectedDatasets(graph!, node.id).map(datasetId)])) + useEffect(() => { setChoices({}) }, [signature]) + const selections = Object.fromEntries(owners.map((node) => { + if (node.data.dataset_factor_mode === 'dataset_toggle') return [node.id, choices[node.id] ?? ((node.data.dataset_selection as string[] | undefined)?.length ? 'all' : 'none')] + const ids = connectedDatasets(graph!, node.id).map(datasetId) + const baseline = (node.data.dataset_selection as string[] | undefined)?.[0] + return [node.id, choices[node.id] ?? (baseline && ids.includes(baseline) ? baseline : ids[0] ?? '')] + })) + const error = owners.length === 0 ? null : library.isError ? 'Could not load available datasets.' : !library.data ? 'Loading datasets…' : owners.some((node) => { + const ids = node.data.dataset_factor_mode === 'dataset_toggle' ? (selections[node.id] === 'all' ? connectedDatasets(graph!, node.id).map(datasetId) : []) : [selections[node.id]] + return ids.some((id) => !library.data.some((dataset) => dataset.id === id)) + }) ? 'Select an available dataset for each agent.' : null + return { graph, owners, library, selections, options: owners.length ? { dataset_selections: selections } : {}, error, setChoice: (agentId: string, id: string) => setChoices((previous) => ({ ...previous, [agentId]: id })) } +} diff --git a/frontend/src/lib/datasetFactors.test.ts b/frontend/src/lib/datasetFactors.test.ts new file mode 100644 index 00000000..af19e2cc --- /dev/null +++ b/frontend/src/lib/datasetFactors.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { reconcileFactorBaselines } from './factorBindings' +import { reconcileDatasetFactor, datasetFactorIssues, datasetFactorOwner } from './datasetFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Agent', dataset_selection: ['a'], factor_bindings: { dataset_selection: 'Datasets' } } }, + ...['a', 'b', 'c'].map((id) => ({ id, type: 'dataset', data: { label: id, config: { dataset_id: id, dataset_name: `Dataset ${id}` } } })), + ], + edges: ['a', 'b', 'c'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'dataset' })), +} as unknown as ProtocolGraph +const factor: DesignFactor = { name: 'Datasets', level_type: 'dataset_selection', levels: [['b'], ['a']], level_labels: ['Custom B', 'Custom A'] } + +describe('dataset connector factors', () => { + it('keeps all-or-none levels in their chosen order as connections change', () => { + const toggle: DesignFactor = { name: 'Datasets', level_type: 'dataset_toggle', levels: [['a', 'b'], []], level_labels: ['With datasets', 'Without datasets'] } + expect(reconcileDatasetFactor(toggle, graph, 'agent')).toEqual({ ...toggle, levels: [['a', 'b', 'c'], []] }) + const noneFirst = { ...toggle, levels: [[], ['a', 'b']], level_labels: ['Without datasets', 'With datasets'] } + expect(reconcileDatasetFactor(noneFirst, { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') }, 'agent')).toEqual({ ...noneFirst, levels: [[], ['a', 'c']] }) + expect(datasetFactorIssues({ ...graph, edges: [] }, [toggle])[0]).toContain('at least one') + }) + it('preserves ordered labels and appends newly connected datasets', () => { + expect(reconcileDatasetFactor(factor, graph, 'agent')).toEqual({ ...factor, levels: [['b'], ['a'], ['c']], level_labels: ['Custom B', 'Custom A', 'Dataset c'] }) + }) + it('removes disconnected levels without replacing the first level from canvas data', () => { + const changed = { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') } + expect(reconcileFactorBaselines({ factors: [factor] }, changed)[0]).toEqual({ ...factor, levels: [['a'], ['c']], level_labels: ['Custom A', 'Dataset c'] }) + }) + it('keeps an empty factor visible and reports incomplete and unavailable datasets', () => { + const empty = { ...graph, edges: [] } + expect(reconcileDatasetFactor(factor, empty, 'agent').levels).toEqual([]) + expect(datasetFactorIssues(empty, [factor])[0]).toContain('at least two') + expect(datasetFactorIssues(graph, [factor], new Set(['a', 'c']))[0]).toContain('unavailable') + expect(datasetFactorOwner(graph, 'b')).toBe('Datasets') + }) +}) diff --git a/frontend/src/lib/datasetFactors.ts b/frontend/src/lib/datasetFactors.ts new file mode 100644 index 00000000..12d70e65 --- /dev/null +++ b/frontend/src/lib/datasetFactors.ts @@ -0,0 +1,96 @@ +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +export const DATASET_FACTOR_PATH = 'dataset_selection' + +export const datasetFactorModes = { + boolean: 'This dataset on/off', + dataset_selection: 'Dataset levels', + dataset_toggle: 'All datasets on/off', +} +export type DatasetFactorMode = keyof typeof datasetFactorModes + +export function datasetFactorConflict(graph: ProtocolGraph, factors: DesignFactor[], datasetNodeId: string, mode: DatasetFactorMode, agentId: string): string | undefined { + const affectedIds = new Set(mode === 'boolean' ? [datasetNodeId] : connectedDatasets(graph, agentId).map((node) => node.id)) + if (graph.edges.some((edge) => affectedIds.has(edge.source) && edge.data?.dataset_input?.mode === 'per_row')) return 'Dataset factors require Whole dataset inputs. Change Per row to Whole dataset first.' + const owner = graph.nodes.find((node) => node.id === (mode === 'boolean' ? datasetNodeId : agentId)) + const currentName = owner?.data.factor_bindings?.[mode === 'boolean' ? 'config.enabled' : DATASET_FACTOR_PATH] + const current = factors.find((factor) => factor.name === currentName) + if (current && (current.level_type ?? 'boolean') !== mode) { + return `${datasetFactorModes[mode]} is unavailable because this Agent already uses the factor ${current.name} (${datasetFactorModes[(current.level_type ?? 'boolean') as DatasetFactorMode]}). Keep the current factor type, or select its factor type and remove it first.` + } + const conflicts = [...new Set(mode === 'boolean' + ? [...(owner?.data.factor_bindings?.config ? [owner.data.factor_bindings.config] : []), ...graph.edges.filter((edge) => edge.source === datasetNodeId && ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '')) + .flatMap((edge) => { + const name = graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[DATASET_FACTOR_PATH] + return name ? [name] : [] + })] + : connectedDatasets(graph, agentId).flatMap((node) => Object.values(node.data.factor_bindings ?? {})))] + if (!conflicts.length) return undefined + return mode === 'boolean' + ? `${datasetFactorModes[mode]} is unavailable because a connected Agent already controls this Dataset with the factor ${conflicts.join(', ')}. Use the existing Agent-level factor, or select its factor type and remove it first.` + : `${datasetFactorModes[mode]} is unavailable because connected Datasets already have factor bindings: ${conflicts.join(', ')}. Agent-level dataset factors cannot be combined with individual Dataset factor bindings. Use This dataset on/off, or remove those bindings in each Dataset’s factor dialog first.` +} + +export function isDatasetFactor(factor: DesignFactor): boolean { + return factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' +} + +export function connectedDatasets(graph: ProtocolGraph, agentId: string): ProtocolNode[] { + return graph.edges.filter((edge) => edge.target === agentId && ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '')) + .flatMap((edge) => { + const node = graph.nodes.find((node) => node.id === edge.source && node.type === 'dataset') + return node ? [node] : [] + }) +} + +export function datasetId(node: ProtocolNode): string { + return String((node.data.config as { dataset_id?: string } | undefined)?.dataset_id ?? '') +} + +export function reconcileDatasetFactor(factor: DesignFactor, graph: ProtocolGraph, agentId: string): DesignFactor { + const datasets = connectedDatasets(graph, agentId) + const ids = [...new Set(datasets.map(datasetId))] + if (factor.level_type === 'dataset_toggle') { + const noneFirst = factor.levels.length === 2 && Array.isArray(factor.levels[0]) && factor.levels[0].length === 0 + return { + ...factor, + levels: noneFirst ? [[], ids] : [ids, []], + level_labels: factor.level_labels?.length === 2 ? factor.level_labels : noneFirst ? ['All disabled', 'All enabled'] : ['All enabled', 'All disabled'], + } + } + const oldIds = factor.levels.map((level) => Array.isArray(level) ? String(level[0] ?? '') : '') + const ordered = [...new Set([...oldIds.filter((id) => ids.includes(id)), ...ids.filter((id) => !oldIds.includes(id))])] + return { + ...factor, + levels: ordered.map((id) => [id]), + level_labels: ordered.map((id) => { + const index = oldIds.indexOf(id) + if (index >= 0 && factor.level_labels?.[index]) return factor.level_labels[index] + const node = datasets.find((node) => datasetId(node) === id) + const label = String((node?.data.config as { dataset_name?: string } | undefined)?.dataset_name ?? node?.data.label ?? 'Unavailable dataset') + return label || 'Unavailable dataset' + }), + } +} + +export function datasetFactorOwner(graph: ProtocolGraph | undefined, datasetNodeId: string): string | undefined { + return graph?.edges.filter((edge) => edge.source === datasetNodeId && ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '')) + .map((edge) => graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[DATASET_FACTOR_PATH]) + .find(Boolean) +} + +export function datasetFactorIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[], availableIds?: Set): string[] { + return factors.filter(isDatasetFactor).flatMap((factor) => { + const owner = graph?.nodes.find((node) => node.data.factor_bindings?.[DATASET_FACTOR_PATH] === factor.name) + if (!owner || !graph) return [`${factor.name}: rebind or remove this dataset factor.`] + const datasets = connectedDatasets(graph, owner.id) + if (graph.edges.some((edge) => edge.target === owner.id && datasets.some((node) => node.id === edge.source) && edge.data?.dataset_input?.mode === 'per_row')) return [`${factor.name}: use Whole dataset inputs before factorizing this connector.`] + const ids = [...new Set(datasets.map(datasetId))] + const conflicts = datasets.some((node) => Object.keys(node.data.factor_bindings ?? {}).length > 0) + if (conflicts) return [`${factor.name}: remove individual dataset factor bindings first.`] + if (ids.length < (factor.level_type === 'dataset_toggle' ? 1 : 2)) return [`${factor.name}: ${factor.level_type === 'dataset_toggle' ? 'connect at least one dataset' : 'connect at least two different datasets'}.`] + if (ids.some((id) => !id || (availableIds && !availableIds.has(id)))) return [`${factor.name}: disconnect or restore unavailable datasets.`] + return [] + }) +} diff --git a/frontend/src/lib/datasetRows.ts b/frontend/src/lib/datasetRows.ts index 74d9c523..fbfe88b4 100644 --- a/frontend/src/lib/datasetRows.ts +++ b/frontend/src/lib/datasetRows.ts @@ -1,7 +1,7 @@ import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' const handles = new Set(['dataset', 'resource', 'tool']) -export const datasetRowFactorBindingError = 'Per row is unavailable because this Dataset has configuration bound to an experimental factor. Per-row runs require the same dataset across all cells. Use Whole dataset here, or remove the factor bindings from this Dataset’s configuration first.' +export const datasetRowFactorBindingError = 'Per row is unavailable because this Dataset or its Agent connector is bound to an experimental factor. Per-row runs require the same dataset across all cells. Use Whole dataset here, or remove the Dataset factor bindings first.' export function datasetConfigIsFactorBound(node?: ProtocolNode) { return Object.keys(node?.data.factor_bindings ?? {}).some(path => path === 'config' || path.startsWith('config.')) @@ -38,7 +38,7 @@ export function datasetRowTopologyError(graph: ProtocolGraph): string | null { const key = JSON.stringify(binding.input.columns) if (agents.has(binding.edge.target) && agents.get(binding.edge.target) !== key) return 'Duplicate Dataset connections must select identical columns.' agents.set(binding.edge.target, key) - if (datasetConfigIsFactorBound(binding.source)) return datasetRowFactorBindingError + if (datasetConfigIsFactorBound(binding.source) || graph.nodes.find((node) => node.id === binding.edge.target)?.data.factor_bindings?.dataset_selection) return datasetRowFactorBindingError } return null } diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index 53480466..a1cf9eb1 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -1,3 +1,4 @@ +import { reconcileDatasetFactor, DATASET_FACTOR_PATH } from './datasetFactors' import type { DesignFactor, DesignSpec } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' import { reconcileSkillFactor, SKILL_FACTOR_PATH } from './skillFactors' @@ -60,7 +61,7 @@ function withPathValue(root: Record, dottedPath: string, nextVa * as an extra level would turn each pause mid-edit into a new treatment. */ export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { - if ((factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) return factor + if ((factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle')) return factor if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) @@ -98,6 +99,10 @@ export function reconcileFactorBaselines(designSpec: DesignSpec | null | undefin const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] === factor.name) return owner ? reconcileSkillFactor(factor, graph, owner.id) : factor } + if ((factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') && graph) { + const owner = graph.nodes.find((node) => node.data.factor_bindings?.[DATASET_FACTOR_PATH] === factor.name) + return owner ? reconcileDatasetFactor(factor, graph, owner.id) : factor + } const values = valuesByFactor.get(factor.name.trim()) ?? [] if (values.length === 0 || values.some((value) => !valuesEqual(value, values[0]))) return factor return factorWithCanvasBaseline(factor, values[0]) @@ -149,7 +154,7 @@ export function factorBindingDiscrepancies( ? 'factor is no longer declared' : value === MISSING ? 'field no longer exists on the canvas' - : (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) + : (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) ? null : 'published value does not match the first (canvas baseline) level' if (reason) discrepancies.push({ nodeId: node.id, nodeLabel, fieldPath, factorName, reason }) diff --git a/frontend/src/lib/skillFactors.ts b/frontend/src/lib/skillFactors.ts index 3a6d527f..309ce18e 100644 --- a/frontend/src/lib/skillFactors.ts +++ b/frontend/src/lib/skillFactors.ts @@ -6,7 +6,7 @@ export const SKILL_FACTOR_PATH = 'skill_selection' export const skillFactorModes = { boolean: 'This skill on/off', skill_selection: 'Skill levels', - skill_toggle: 'All agent skills on/off', + skill_toggle: 'All skills on/off', } export type SkillFactorMode = keyof typeof skillFactorModes diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 82a4c9b4..3616b4e7 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -24,6 +24,8 @@ export interface DesignFactor { | 'tool_names' | 'skill_selection' | 'skill_toggle' + | 'dataset_selection' + | 'dataset_toggle' } export interface DesignMetric { diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index 6edef30f..d51afbeb 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -358,6 +358,8 @@ export interface AgentNodeConfig { export interface AgentNodeData { // Connector-owned selection: one skill per level, or every skill versus none. + dataset_selection?: string[] + dataset_factor_mode?: 'dataset_selection' | 'dataset_toggle' skill_selection?: string[] skill_factor_mode?: 'skill_selection' | 'skill_toggle' label: string diff --git a/src/asaree/api/protocols.py b/src/asaree/api/protocols.py index 403d7de1..b94065c3 100644 --- a/src/asaree/api/protocols.py +++ b/src/asaree/api/protocols.py @@ -17,11 +17,13 @@ from pydantic import BaseModel, ConfigDict, Field, StrictInt, field_validator from sqlalchemy import cast, func, or_, select from sqlalchemy.dialects.postgresql import JSONPATH +from sqlalchemy.ext.asyncio import AsyncSession from asaree.deps import CurrentUser, DbSession from asaree.models.dataset import RegisteredDataset from asaree.models.protocol_revision import ProtocolRevision from asaree.models.protocol_run import ProtocolRun +from asaree.services.dataset_factors import test_dataset_factor_values from asaree.services.dataset_row_csv import DatasetRowCsvError, project_row, read_row_source from asaree.services.dataset_row_inputs import DatasetRowInputError, resolve_dataset_row_plan from asaree.services.experiment_measurements import ( @@ -277,6 +279,7 @@ class TestRunRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) skill_selections: dict[str, str] | None = None + dataset_selections: dict[str, str] | None = None class NodePlayRequest(BaseModel): @@ -284,13 +287,17 @@ class NodePlayRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) skill_selections: dict[str, str] | None = None + dataset_selections: dict[str, str] | None = None -async def _node_test_skill_values( - graph: dict[str, Any], body: NodePlayRequest | None, owner_id: uuid.UUID, node_id: str +async def _node_test_factor_values( + graph: dict[str, Any], body: NodePlayRequest | None, owner_id: uuid.UUID, node_id: str, db: AsyncSession ) -> dict[str, Any]: try: - return await test_skill_factor_values(graph, body.skill_selections if body else None, owner_id, node_id) + return { + **await test_skill_factor_values(graph, body.skill_selections if body else None, owner_id, node_id), + **await test_dataset_factor_values(graph, body.dataset_selections if body else None, owner_id, db, node_id), + } except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc @@ -678,9 +685,12 @@ async def create_test_run_endpoint( owner_id=user.id, protocol_revision_id=revision.id, dataset_row=dataset_row, - factor_values=await test_skill_factor_values( - revision.graph, body.skill_selections if body else None, user.id - ), + factor_values={ + **await test_skill_factor_values(revision.graph, body.skill_selections if body else None, user.id), + **await test_dataset_factor_values( + revision.graph, body.dataset_selections if body else None, user.id, db + ), + }, ) except (ProtocolValidationError, ValueError) as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc @@ -770,7 +780,7 @@ async def run_single_node_endpoint( target_node_id=node_id, protocol_revision_id=revision.id, dataset_row=dataset_row, - factor_values=await _node_test_skill_values(revision.graph, body, user.id, node_id), + factor_values=await _node_test_factor_values(revision.graph, body, user.id, node_id, db), ) await db.commit() await enqueue_protocol_run(run.id) diff --git a/src/asaree/services/dataset_factors.py b/src/asaree/services/dataset_factors.py new file mode 100644 index 00000000..b3fe5ef1 --- /dev/null +++ b/src/asaree/services/dataset_factors.py @@ -0,0 +1,150 @@ +"""Connector-owned dataset choices shared by design validation and test runs.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from asaree.services.datasets import get_dataset + +DATASET_SELECTION = "dataset_selection" +DATASET_TOGGLE = "dataset_toggle" +DATASET_FACTOR_TYPES = {DATASET_SELECTION, DATASET_TOGGLE} + + +def connected_dataset_nodes(graph: dict[str, Any], agent_id: str) -> list[dict[str, Any]]: + nodes = {node["id"]: node for node in graph.get("nodes") or []} + return [ + nodes[edge["source"]] + for edge in graph.get("edges") or [] + if edge.get("target") == agent_id + and edge.get("targetHandle") in {"dataset", "resource", "tool"} + and edge.get("source") in nodes + and nodes[edge["source"]].get("type") == "dataset" + ] + + +def validate_dataset_factor_structure( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete: bool = True +) -> None: + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + bound = { + (node.get("data") or {}).get("factor_bindings", {}).get(DATASET_SELECTION) for node in graph.get("nodes") or [] + } + for name, factor in factors.items(): + if factor.get("level_type") in DATASET_FACTOR_TYPES and name not in bound: + raise ValueError(f"Rebind or remove dataset factor {name!r}.") + for node in graph.get("nodes") or []: + data = node.get("data") or {} + name = (data.get("factor_bindings") or {}).get(DATASET_SELECTION) + if not name: + continue + factor = factors.get(name) + if node.get("type") not in {"agent", "sub_agent"}: + raise ValueError("Only an Agent dataset connector can own a dataset factor.") + if not factor or factor.get("level_type") not in DATASET_FACTOR_TYPES: + raise ValueError(f"Dataset connector factor {name!r} must declare a dataset factor type.") + toggle = factor.get("level_type") == DATASET_TOGGLE + if data.get("dataset_factor_mode", DATASET_SELECTION) != factor.get("level_type"): + raise ValueError(f"Dataset factor {name!r}: canvas mode does not match the declared factor.") + datasets = connected_dataset_nodes(graph, node["id"]) + if any( + edge.get("target") == node["id"] + and edge.get("source") in {dataset["id"] for dataset in datasets} + and (edge.get("data") or {}).get("dataset_input", {}).get("mode") == "per_row" + for edge in graph.get("edges") or [] + ): + raise ValueError(f"Dataset factor {name!r}: use Whole dataset inputs before factorizing this connector.") + ids = {(dataset.get("data") or {}).get("config", {}).get("dataset_id") for dataset in datasets} + if complete and (None in ids or "" in ids): + raise ValueError(f"Dataset factor {name!r}: disconnect or restore unavailable datasets.") + if complete and len(ids) < (1 if toggle else 2): + minimum = "one dataset" if toggle else "two different datasets" + raise ValueError(f"Dataset factor {name!r}: connect at least {minimum}.") + if any((dataset.get("data") or {}).get("factor_bindings") for dataset in datasets): + raise ValueError(f"Dataset factor {name!r}: remove individual dataset factor bindings first.") + levels = factor.get("levels") or [] + if toggle: + if ( + len(levels) != 2 + or any(not isinstance(level, list) for level in levels) + or sorted(len(level) for level in levels) != [0, len(ids)] + or any(level and (set(level) != ids or len(level) != len(ids)) for level in levels) + ): + raise ValueError(f"Dataset factor {name!r}: levels must enable every connected dataset or none.") + continue + if any(not isinstance(level, list) or len(level) != 1 or level[0] not in ids for level in levels): + raise ValueError(f"Dataset factor {name!r}: each level must name one connected dataset.") + if len(levels) != len(ids) or {level[0] for level in levels} != ids: + raise ValueError(f"Dataset factor {name!r}: levels must include every connected dataset exactly once.") + + +async def require_available_datasets(ids: list[str], owner_id: uuid.UUID, db: AsyncSession) -> None: + for dataset_id in dict.fromkeys(ids): + try: + dataset = await get_dataset(db, uuid.UUID(dataset_id)) + except (ValueError, TypeError, AttributeError): + dataset = None + if dataset is None or dataset.owner_id != owner_id: + raise ValueError("A dataset factor references an unavailable dataset. Disconnect or restore it.") + + +async def validate_dataset_factors( + design_spec: dict[str, Any] | None, graph: dict[str, Any], owner_id: uuid.UUID, db: AsyncSession +) -> None: + validate_dataset_factor_structure(design_spec, graph) + ids = [ + dataset_id + for factor in (design_spec or {}).get("factors") or [] + if factor.get("level_type") in DATASET_FACTOR_TYPES + for level in factor.get("levels") or [] + for dataset_id in level + ] + await require_available_datasets(ids, owner_id, db) + + +async def test_dataset_factor_values( + graph: dict[str, Any], + selections: dict[str, str] | None, + owner_id: uuid.UUID, + db: AsyncSession, + node_id: str | None = None, +) -> dict[str, Any]: + """Validate preview choices against the publication and retain run provenance.""" + selections = selections or {} + owners = { + node["id"]: node + for node in graph.get("nodes") or [] + if (node.get("data") or {}).get("factor_bindings", {}).get(DATASET_SELECTION) + and (node_id is None or node["id"] == node_id) + } + if set(selections) - owners.keys(): + raise ValueError("Dataset selection must belong to a dataset factor in this run.") + values: dict[str, Any] = {} + for agent_id, node in owners.items(): + data = node.get("data") or {} + connected = [ + (dataset.get("data") or {}).get("config", {}).get("dataset_id") + for dataset in connected_dataset_nodes(graph, agent_id) + ] + if any( + (dataset.get("data") or {}).get("factor_bindings") for dataset in connected_dataset_nodes(graph, agent_id) + ): + raise ValueError("Remove individual dataset factor bindings before testing this connector factor.") + baseline = data.get(DATASET_SELECTION) or [] + if data.get("dataset_factor_mode") == DATASET_TOGGLE: + selected = selections.get(agent_id, "all" if baseline else "none") + if selected not in {"all", "none"}: + raise ValueError("Select all enabled or all disabled for the test run.") + chosen = list(dict.fromkeys(connected)) if selected == "all" else [] + await require_available_datasets(chosen, owner_id, db) + values[data["factor_bindings"][DATASET_SELECTION]] = chosen + continue + selected = selections.get(agent_id, baseline[0] if baseline else None) + if not selected or selected not in connected: + raise ValueError("Select an available connected dataset for the test run.") + await require_available_datasets([selected], owner_id, db) + values[data["factor_bindings"][DATASET_SELECTION]] = [selected] + return values diff --git a/src/asaree/services/dataset_row_inputs.py b/src/asaree/services/dataset_row_inputs.py index dc4eeccf..6ec1a6f5 100644 --- a/src/asaree/services/dataset_row_inputs.py +++ b/src/asaree/services/dataset_row_inputs.py @@ -118,6 +118,10 @@ def resolve_dataset_row_plan(graph: dict[str, Any], design_spec: dict[str, Any] raise DatasetRowInputError("invalid_dataset_config", "dataset_name must be a string") if normalized["mode"] == "per_row": + if ((target.get("data") or {}).get("factor_bindings") or {}).get("dataset_selection"): + raise DatasetRowInputError( + "factorized_driver", "a row-driving Dataset connector cannot be factor-bound" + ) if not isinstance(dataset_id, str): raise DatasetRowInputError( "invalid_dataset_id", diff --git a/src/asaree/services/design_generation.py b/src/asaree/services/design_generation.py index e06bd8b1..e937d92a 100644 --- a/src/asaree/services/design_generation.py +++ b/src/asaree/services/design_generation.py @@ -33,6 +33,7 @@ from asaree.models.protocol import Protocol from asaree.models.protocol_revision import ProtocolRevision from asaree.services.coordination import coordination_strategy_slug +from asaree.services.dataset_factors import validate_dataset_factors from asaree.services.design_revisions import get_current_revision, supersede_and_create from asaree.services.factorial_cells import list_replicates, upsert_replicate from asaree.services.skill_factors import validate_skill_factors @@ -286,8 +287,8 @@ async def get_design_impact( """Compare the declared factorial matrix to its materialized revision.""" material = material_design_spec(design_spec) incomplete_skills = any( - (factor.get("level_type") == "skill_selection" and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") == "skill_toggle" and not any(factor.get("levels") or [])) + (factor.get("level_type") in {"skill_selection", "dataset_selection"} and len(factor.get("levels") or []) < 2) + or (factor.get("level_type") in {"skill_toggle", "dataset_toggle"} and not any(factor.get("levels") or [])) for factor in material["factors"] ) planned = ( @@ -414,10 +415,16 @@ async def generate_design_cells( ).scalar_one_or_none() if experiment is None: raise DesignValidationError("Experiment not found") - if any(factor.get("level_type") in {"skill_selection", "skill_toggle"} for factor in factors): + if any( + factor.get("level_type") in {"skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle"} + for factor in factors + ): protocol = await db.scalar(select(Protocol).where(Protocol.experiment_id == experiment_id)) try: await validate_skill_factors({"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id) + await validate_dataset_factors( + {"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id, db + ) except ValueError as exc: raise DesignValidationError(str(exc)) from exc # An empty factor declaration is a meaningful replacement when an diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index bb768e7e..f98ec88d 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -4,6 +4,7 @@ from typing import Any +from asaree.services.dataset_factors import validate_dataset_factor_structure from asaree.services.skill_factors import validate_skill_factor_structure _MISSING = object() @@ -32,6 +33,7 @@ def validate_factor_bindings( design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete_skills: bool = True ) -> None: validate_skill_factor_structure(design_spec, graph, complete=complete_skills) + validate_dataset_factor_structure(design_spec, graph, complete=complete_skills) missing = unbound_factor_names(design_spec, graph) if missing: labels = ", ".join(repr(name) for name in missing) @@ -67,7 +69,7 @@ def validate_factor_bindings( "level_type" not in factor and len(levels) == 2 and all(isinstance(level, bool) for level in levels) ) if ( - level_type not in {"boolean", "skill_selection", "skill_toggle"} + level_type not in {"boolean", "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle"} and not legacy_boolean and (not levels or published_value != levels[0]) ): diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 7f778744..463f2042 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -63,6 +63,7 @@ from asaree.services import prompt_references from asaree.services.agent_cards import AgentCard, build_agent_card from asaree.services.coordination import coordination_strategy_slug +from asaree.services.dataset_factors import validate_dataset_factors from asaree.services.dataset_row_csv import DatasetRowCsvError, project_row, read_row_source from asaree.services.dataset_row_inputs import DatasetRowInputError, resolve_dataset_row_plan from asaree.services.dataset_row_planning import enumerate_row_candidates @@ -1973,6 +1974,11 @@ def _resolve_dataset_configs( nodes, _downstream, _upstream = _adjacency(graph) configs: list[dict[str, Any]] = [] seen: set[str] = set() + agent_data = (nodes.get(node_id) or {}).get("data") or {} + selection = ( + agent_data.get("dataset_selection") + if (agent_data.get("factor_bindings") or {}).get("dataset_selection") else None + ) # Edges first, handles second: an agent's Dataset nodes should come out in # the order they were wired, not grouped by which handle spelling they # happen to be saved under. @@ -1983,7 +1989,9 @@ def _resolve_dataset_configs( if source is None or source.get("type") not in _DATASET_NODE_TYPES: continue config = (source.get("data") or {}).get("config") or {} - if not config.get("enabled", True) or not config.get("dataset_name"): + if (selection is None and not config.get("enabled", True)) or not config.get("dataset_name"): + continue + if selection is not None and config.get("dataset_id") not in selection: continue key = str(config.get("dataset_id") or config["dataset_name"]) if key in seen: @@ -4741,6 +4749,7 @@ async def plan_cell_runs( try: validate_factor_bindings(design_spec, graph) await validate_skill_factors(design_spec, graph, owner_id) + await validate_dataset_factors(design_spec, graph, owner_id, db) except ValueError as exc: raise ProtocolValidationError(str(exc)) from exc measurement_report = await validate_experiment_measurement_plan( @@ -5129,6 +5138,7 @@ async def plan_single_replicate_run( try: validate_factor_bindings(design_spec, graph) await validate_skill_factors(design_spec, graph, owner_id) + await validate_dataset_factors(design_spec, graph, owner_id, db) except ValueError as exc: raise ProtocolValidationError(str(exc)) from exc measurement_report = await validate_experiment_measurement_plan( diff --git a/src/asaree/services/protocol_revisions.py b/src/asaree/services/protocol_revisions.py index 889a11b8..a2425248 100644 --- a/src/asaree/services/protocol_revisions.py +++ b/src/asaree/services/protocol_revisions.py @@ -150,8 +150,9 @@ async def publish_protocol( if experiment is not None: from asaree.services.design_generation import get_design_impact incomplete_skills = any( - (factor.get("level_type") == "skill_selection" and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") == "skill_toggle" and not any(factor.get("levels") or [])) + (factor.get("level_type") in {"skill_selection", "dataset_selection"} + and len(factor.get("levels") or []) < 2) + or (factor.get("level_type") in {"skill_toggle", "dataset_toggle"} and not any(factor.get("levels") or [])) for factor in (experiment.design_spec or {}).get("factors") or [] ) if incomplete_skills: diff --git a/tests/test_dataset_factors.py b/tests/test_dataset_factors.py new file mode 100644 index 00000000..4f43f73c --- /dev/null +++ b/tests/test_dataset_factors.py @@ -0,0 +1,164 @@ +"""Dataset connector factors control run inputs without changing reusable nodes.""" + +import copy +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from asaree.services.dataset_factors import test_dataset_factor_values as preview_values +from asaree.services.dataset_factors import validate_dataset_factors +from asaree.services.factor_bindings import validate_factor_bindings +from asaree.services.protocol_execution import ( + _ambient_meta_for, + _build_user_input, + _resolve_dataset_configs, + _resolve_dataset_tool_config, + apply_factor_bindings, +) + + +def dataset_graph(mode="dataset_selection"): + ids = [str(uuid.uuid4()), str(uuid.uuid4())] + levels = [[value] for value in ids] if mode == "dataset_selection" else [ids, []] + graph = { + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": { + "factor_bindings": {"dataset_selection": "Datasets"}, + "dataset_selection": levels[0], + "dataset_factor_mode": mode, + }, + }, + *[ + { + "id": f"dataset-{index}", + "type": "dataset", + "data": { + "config": { + "dataset_id": value, + "dataset_name": f"Dataset {index}", + "enabled": False, + } + }, + } + for index, value in enumerate(ids) + ], + ], + "edges": [{"source": f"dataset-{index}", "target": "agent", "targetHandle": "dataset"} for index in range(2)], + } + spec = {"factors": [{"name": "Datasets", "level_type": mode, "levels": levels}]} + return graph, spec, ids + + +@pytest.mark.parametrize("mode", ["dataset_selection", "dataset_toggle"]) +def test_cell_inputs_override_disabled_nodes_without_mutating_graph(mode): + graph, spec, ids = dataset_graph(mode) + original = copy.deepcopy(graph) + validate_factor_bindings(spec, graph) + for selected in [[ids[1]]] if mode == "dataset_selection" else [ids, []]: + patched = apply_factor_bindings(graph, {"Datasets": selected}) + assert [config["dataset_id"] for config in _resolve_dataset_configs(patched, "agent")] == selected + assert graph == original + + +def test_shared_dataset_remains_independently_disabled_for_other_agent(): + graph, spec, ids = dataset_graph("dataset_toggle") + graph["nodes"].append({"id": "other", "type": "agent", "data": {}}) + graph["edges"].append({"source": "dataset-0", "target": "other", "targetHandle": "resource"}) + validate_factor_bindings(spec, graph) + patched = apply_factor_bindings(graph, {"Datasets": ids}) + assert len(_resolve_dataset_configs(patched, "agent")) == 2 + assert _resolve_dataset_configs(patched, "other") == [] + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_individual_on_off_controls_dataset_prompt_metadata_and_implicit_tools(enabled): + graph, _, _ = dataset_graph() + graph["nodes"][0]["data"] = {"label": "Agent", "config": {"goal": "Analyze the available data"}} + graph["edges"] = graph["edges"][:1] + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "Enabled"} + patched = apply_factor_bindings(graph, {"Enabled": enabled}) + prompt = _build_user_input( + patched["nodes"][0], patched, {}, experiment_id=uuid.uuid4(), effective_cell_label="cell" + ) + assert ("Dataset 0" in prompt) is enabled + assert ("Dataset context:" in prompt) is enabled + assert bool(_ambient_meta_for(patched, "agent").get("dataset_names")) is enabled + assert bool(_resolve_dataset_tool_config(patched, "agent")["tool_names"]) is enabled + + +@pytest.mark.parametrize("mode", ["dataset_selection", "dataset_toggle"]) +def test_connector_selection_filters_dataset_prompt_metadata_and_tools(mode): + graph, _, ids = dataset_graph(mode) + graph["nodes"][0]["data"]["config"] = {"goal": "Analyze the available data"} + choices = [[ids[0]], [ids[1]]] if mode == "dataset_selection" else [ids, []] + for selected in choices: + patched = apply_factor_bindings(graph, {"Datasets": selected}) + prompt = _build_user_input( + patched["nodes"][0], patched, {}, experiment_id=uuid.uuid4(), effective_cell_label="cell" + ) + expected_names = [f"Dataset {index}" for index, value in enumerate(ids) if value in selected] + assert _ambient_meta_for(patched, "agent").get("dataset_names", []) == expected_names + for index, value in enumerate(ids): + assert (f"Dataset {index}" in prompt) is (value in selected) + assert bool(_resolve_dataset_tool_config(patched, "agent")["tool_names"]) is bool(selected) + + +@pytest.mark.parametrize("mode", ["dataset_selection", "dataset_toggle"]) +def test_rejects_individual_bindings_and_row_inputs(mode): + graph, spec, _ = dataset_graph(mode) + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "Enabled"} + with pytest.raises(ValueError, match="individual dataset factor bindings"): + validate_factor_bindings(spec, graph) + graph["nodes"][1]["data"].pop("factor_bindings") + graph["edges"][0]["data"] = {"dataset_input": {"mode": "per_row", "columns": ["question"]}} + with pytest.raises(ValueError, match="Whole dataset"): + validate_factor_bindings(spec, graph) + + +@pytest.mark.parametrize("levels", ["partial", "duplicate", "foreign"]) +def test_rejects_invalid_toggle_levels(levels): + graph, spec, ids = dataset_graph("dataset_toggle") + spec["factors"][0]["levels"] = { + "partial": [[ids[0]], []], + "duplicate": [ids + [ids[0]], []], + "foreign": [[ids[0], str(uuid.uuid4())], []], + }[levels] + with pytest.raises(ValueError, match="every connected dataset or none"): + validate_factor_bindings(spec, graph) + + +@pytest.mark.asyncio +async def test_previews_preserve_choices_and_reject_unavailable_or_foreign_datasets(monkeypatch): + graph, spec, ids = dataset_graph() + owner = uuid.uuid4() + db = AsyncMock() + lookup = AsyncMock(return_value=SimpleNamespace(owner_id=owner)) + monkeypatch.setattr("asaree.services.dataset_factors.get_dataset", lookup) + await validate_dataset_factors(spec, graph, owner, db) + assert await preview_values(graph, {"agent": ids[1]}, owner, db) == {"Datasets": [ids[1]]} + with pytest.raises(ValueError, match="available connected dataset"): + await preview_values(graph, {"agent": str(uuid.uuid4())}, owner, db) + lookup.return_value = SimpleNamespace(owner_id=uuid.uuid4()) + with pytest.raises(ValueError, match="unavailable dataset"): + await preview_values(graph, None, owner, db) + lookup.return_value = None + with pytest.raises(ValueError, match="unavailable dataset"): + await validate_dataset_factors(spec, graph, owner, db) + + +@pytest.mark.asyncio +async def test_all_or_none_preview_uses_baseline_and_explicit_choices(monkeypatch): + graph, _, ids = dataset_graph("dataset_toggle") + owner, db = uuid.uuid4(), AsyncMock() + monkeypatch.setattr( + "asaree.services.dataset_factors.get_dataset", AsyncMock(return_value=SimpleNamespace(owner_id=owner)) + ) + assert await preview_values(graph, None, owner, db) == {"Datasets": ids} + assert await preview_values(graph, {"agent": "none"}, owner, db) == {"Datasets": []} + graph["nodes"][0]["data"]["dataset_selection"] = [] + assert await preview_values(graph, None, owner, db) == {"Datasets": []} diff --git a/tests/test_skill_factors.py b/tests/test_skill_factors.py index f26800a4..44ca8733 100644 --- a/tests/test_skill_factors.py +++ b/tests/test_skill_factors.py @@ -55,6 +55,16 @@ def test_cell_selection_overrides_individual_switches_without_mutating_graph(): assert graph == original +@pytest.mark.parametrize("enabled", [False, True]) +def test_individual_on_off_excludes_or_includes_only_the_bound_skill(enabled): + graph, _, ids = skill_graph() + graph["nodes"][0]["data"] = {} + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "Enabled"} + graph["nodes"][2]["data"]["config"]["enabled"] = True + patched = apply_factor_bindings(graph, {"Enabled": enabled}) + assert _resolve_skill_config(patched, "agent") == {"skill_ids": ids if enabled else [ids[1]]} + + @pytest.mark.parametrize("none_first", [False, True]) def test_all_or_none_runs_override_disabled_skills_and_preserve_graph(none_first): graph, spec, ids = skill_graph() From f65f7e27a071485734ad2610ab6c2815c2362ef4 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 13:33:06 -0700 Subject: [PATCH 09/63] feat: group factor and metric selectors by agent and component --- .../protocol/CustomMetricDialogs.test.tsx | 22 +++- .../protocol/CustomMetricDialogs.tsx | 79 ++++++++++---- .../protocol/DatasetFactorDialog.tsx | 10 +- .../components/protocol/DesignTab.test.tsx | 35 ++++++ .../src/components/protocol/DesignTab.tsx | 8 +- .../protocol/FactorEditorDialog.tsx | 31 ++++-- .../components/protocol/ProtocolCanvas.tsx | 28 ++++- .../components/protocol/SkillFactorDialog.tsx | 10 +- .../src/components/protocol/bindableFields.ts | 100 +++++++++++++++--- .../protocol/factorCreationFields.test.ts | 23 ++++ 10 files changed, 284 insertions(+), 62 deletions(-) create mode 100644 frontend/src/components/protocol/factorCreationFields.test.ts diff --git a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx index e4b0b406..2d509c71 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx @@ -32,6 +32,26 @@ function renderFlow(graph: ProtocolGraph, onSave = vi.fn(), sourceContext?: Cust } describe('CustomMetricFlow', () => { + it('groups nodes alphabetically by agent and supports collapsing and search', async () => { + const user = userEvent.setup() + const graph = { + nodes: ['Zebra', 'Alpha'].map(label => ({ id: label, type: 'agent', position: { x: 0, y: 0 }, data: { label } })), + edges: [], + } as unknown as ProtocolGraph + renderFlow(graph) + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + const list = screen.getByRole('listbox', { name: 'Metric nodes' }) + const summaries = [...list.querySelectorAll('summary')] + expect(summaries.map(summary => summary.textContent)).toEqual(['Alpha', 'Zebra', 'Runtime metrics for selected nodes']) + await user.click(summaries[0]) + expect(summaries[0].parentElement).not.toHaveAttribute('open') + await user.type(screen.getByRole('textbox', { name: 'Search metric nodes' }), 'Alpha') + expect(screen.queryByRole('option', { name: 'Zebra, Agent' })).not.toBeInTheDocument() + await user.click(screen.getByRole('option', { name: 'Alpha, Agent' })) + expect(screen.queryByRole('textbox', { name: 'Search metric nodes' })).not.toBeInTheDocument() + expect(screen.getByRole('combobox', { name: 'Metric node' })).toHaveTextContent('Alpha') + }) + it('sums a runtime measure over several chosen nodes', async () => { const user = userEvent.setup() const graph = { @@ -45,7 +65,7 @@ describe('CustomMetricFlow', () => { const onSave = renderFlow(graph) await user.click(screen.getByRole('combobox', { name: 'Metric node' })) - await user.click(screen.getByRole('option', { name: 'Runs of chosen nodes, Node runtime' })) + await user.click(screen.getByRole('option', { name: 'Runtime metrics for selected nodes, Node runtime' })) expect(screen.getByRole('button', { name: 'Add custom metric' })).toBeDisabled() await user.click(screen.getByRole('checkbox', { name: /^Critic \(FS\)/ })) await user.click(screen.getByRole('checkbox', { name: /^Critic \(DC\)/ })) diff --git a/frontend/src/components/protocol/CustomMetricDialogs.tsx b/frontend/src/components/protocol/CustomMetricDialogs.tsx index 15884935..11fc68d0 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.tsx @@ -1,11 +1,14 @@ import { useEffect, useRef, useState } from 'react' import { useQuery } from '@tanstack/react-query' +import { ChevronsUpDown } from 'lucide-react' import { mcpServersApi, protocolsApi } from '@/api/client' import { Badge } from '@/components/ui/badge' import { Checkbox } from '@/components/ui/checkbox' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { HUD_ACCENT_RING_CLASSNAME, cn } from '@/lib/utils' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { AGENT_OUTPUT_PRODUCER_ID, agentOutputSourceOptions } from '@/lib/agentOutputMetrics' import { mcpToolSourceOptions, type McpToolSourceOption } from '@/lib/mcpToolMetrics' @@ -50,16 +53,50 @@ export function MetricNodeLabel({ display, reason }: { display: MetricNodeDispla ) } -function MetricNodeOption({ value, display, reason }: { value: string; display: MetricNodeDisplay; reason?: string }) { - return ( - - - - ) +type MetricNodeChoice = { value: string; display: MetricNodeDisplay; reason?: string; agentId?: string; category: string } + +function MetricNodePicker({ value, display, choices, graph, onChange }: { + value: string + display: MetricNodeDisplay | null | undefined + choices: MetricNodeChoice[] + graph?: ProtocolGraph + onChange: (value: string) => void +}) { + const [open, setOpen] = useState(false) + const [search, setSearch] = useState('') + const groups = new Map }>() + for (const choice of choices) { + const key = choice.agentId ?? (choice.value === 'node_runtime' ? 'runtime' : 'shared') + const label = choice.agentId ? String(graph?.nodes.find(node => node.id === choice.agentId)?.data.label || 'Agent') + : key === 'runtime' ? 'Runtime metrics for selected nodes' : 'Shared or unconnected components' + if (!`${label} ${choice.category} ${choice.display.label} ${choice.display.type}`.toLowerCase().includes(search.trim().toLowerCase())) continue + if (!groups.has(key)) groups.set(key, { label, categories: new Map() }) + const categories = groups.get(key)!.categories + categories.set(choice.category, [...(categories.get(choice.category) ?? []), choice]) + } + const groupOrder = (key: string) => key === 'runtime' ? 2 : key === 'shared' ? 1 : 0 + const categoryOrder = ['Agent output', 'Scripts', 'MCP tools', 'Tool steps', 'Runtime'] + return { setOpen(next); if (next) setSearch('') }}> + }> + {display ? : 'Select a node…'} + + + setSearch(event.target.value)} /> +
+ {groups.size === 0 &&

No matching nodes.

} + {[...groups.entries()].sort(([a, ga], [b, gb]) => groupOrder(a) - groupOrder(b) || ga.label.localeCompare(gb.label)).map(([id, group]) =>
+ {group.label} +
{[...group.categories.entries()].sort(([a], [b]) => categoryOrder.indexOf(a) - categoryOrder.indexOf(b)).map(([category, options]) =>
+

{category}

+ {options.sort((a, b) => a.display.label.localeCompare(b.display.label)).map(choice => )} +
)}
+
)} +
+ + } function duplicateName(metric: DesignMetric, name: string, metrics: DesignMetric[]) { @@ -206,7 +243,7 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM label: source.label, type: 'Tool Step', }) - const nodeRuntimeDisplay: MetricNodeDisplay = { label: 'Runs of chosen nodes', type: 'Node runtime' } + const nodeRuntimeDisplay: MetricNodeDisplay = { label: 'Runtime metrics for selected nodes', type: 'Node runtime' } const selectedNodeDisplay = producer === 'node_runtime' ? nodeRuntimeDisplay : selectedAgentSource ? agentNodeDisplay(selectedAgentSource) : selectedPythonSource @@ -380,17 +417,13 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM Metric node {!sourceContext ?
- + ({ value: `agent:${source.agentNodeId}`, display: agentNodeDisplay(source), reason: agentSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Agent output' })), + ...pythonSources.map(source => ({ value: `python:${source.key}`, display: pythonNodeDisplay(source), reason: pythonSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Scripts' })), + ...mcpSources.map(source => ({ value: `mcp:${source.key}`, display: mcpNodeDisplay(source), reason: mcpSourceDisabledReason(source), agentId: source.agentNodeId, category: 'MCP tools' })), + ...toolStepSources.map(source => ({ value: `tool_step:${source.nodeId}`, display: toolStepNodeDisplay(source), reason: source.disabledReason, category: 'Tool steps' })), + ...(offerNodeRuntime ? [{ value: 'node_runtime', display: nodeRuntimeDisplay, category: 'Runtime' }] : []), + ]} />
: <>

{producer === 'agent' ? 'Agent output' : producer === 'tool_step' ? 'Tool Step' : producer === 'node_runtime' ? 'Node runtime' : producer === 'python' ? 'Python Script' : 'MCP Tool'}

{producer === 'tool_step' &&

{selectedToolStepSource?.label}

{toolStepSourceError &&

{toolStepSourceError}

}
} @@ -402,7 +435,7 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM {producer === 'node_runtime' &&
- Runtime total + Runtime metrics for selected nodes
{runtimeNodeSources.map((source) =>
-
+
{needsFieldPick ? ( -
+
setSearch(e.target.value)} /> -
+
{filteredFields.length === 0 && (

{pickableFields?.length === 0 @@ -713,21 +720,27 @@ function StandardFactorEditorDialog({ : 'No matching fields.'}

)} - {filteredFields.map((field) => ( + {fieldGroups.map((group) =>
+ {group.label}{group.components.reduce((count, component) => count + component.fields.length, 0)} options +
{group.components.map((component) =>
+

{component.category}{component.category === 'Agent fields' ? '' : ` · ${component.label}`}

+ {component.fields.map((field) => ( - ))} + ))} +
)}
+
)}
) : ( diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 9f7186d2..703643a0 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -85,7 +85,7 @@ import type { OkfBundle, OkfDocument } from '@/types/okf' import type { Skill } from '@/types/skills' import { AddNodePanel } from './AddNodePanel' import { AgentNodeInspector } from './AgentNodeInspector' -import { agentTracedLabel, factorBoundField, revealsHiddenMcpServers, toolFactorServerId, unboundBindableFields, type UnboundField } from './bindableFields' +import { agentTracedLabel, factorBoundField, factorCreationFields, revealsHiddenMcpServers, toolFactorServerId, type UnboundField } from './bindableFields' import { CanvasControls } from './CanvasControls' import { CriticGateNodeInspector } from './CriticGateNodeInspector' import { ToolStepNodeInspector } from './ToolStepNodeInspector' @@ -364,6 +364,7 @@ export interface ProtocolCanvasHandle { // level from the sibling Design tab must update every field controlled by // the factor, not leave the graph and design as two sources of truth. setFactorBaseline: (factorName: string, baseline: unknown) => void + openConnectorFactor: (field: UnboundField) => void } export const ProtocolCanvas = forwardRef { + if (!experimentId || experimentLocked || !field.connectorFactor) return + setSelectedNodeId(null) + setConnectorFactorChoice(field.connectorFactor) + if (field.connectorFactor.kind === 'skill') setSkillFactorNodeId(field.connectorFactor.nodeId) + else setDatasetFactorNodeId(field.connectorFactor.nodeId) + }, }), - [bindFactorOnNode, removeFactorBindings, renameFactorBindings, setFactorBaseline], + [bindFactorOnNode, removeFactorBindings, renameFactorBindings, setFactorBaseline, experimentId, experimentLocked], ) const [selectedNodeId, setSelectedNodeId] = useState(null) useEffect(() => { @@ -483,6 +491,7 @@ export const ProtocolCanvas = forwardRef(null) const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) const [datasetFactorNodeId, setDatasetFactorNodeId] = useState(null) + const [connectorFactorChoice, setConnectorFactorChoice] = useState() const [addPanelOpen, setAddPanelOpen] = useState(false) // The second level of the add-node panel: AddNodePanel's "MCP Servers" // entry swaps the browser in over it, and its Back button returns. Only @@ -1345,11 +1354,13 @@ export const ProtocolCanvas = forwardRef { if (!experimentId || experimentLocked) return setSelectedNodeId(null) + setConnectorFactorChoice(undefined) setSkillFactorNodeId(nodeId) }, [experimentId, experimentLocked]) const requestDatasetFactor = useCallback((nodeId: string) => { if (!experimentId || experimentLocked) return setSelectedNodeId(null) + setConnectorFactorChoice(undefined) setDatasetFactorNodeId(nodeId) }, [experimentId, experimentLocked]) const metricsByNode = useMemo(() => { @@ -1465,7 +1476,7 @@ export const ProtocolCanvas = forwardRef n.id === factorPickerNodeId) ?? null const factorPickerFields: UnboundField[] = factorPickerNodeId - ? unboundBindableFields(nodes, edges).filter((f) => f.nodeId === factorPickerNodeId) + ? factorCreationFields(nodes, edges).filter((f) => f.nodeId === factorPickerNodeId) : [] const factorPickerExistingNames = factorPickerExperimentQuery.data?.design_spec?.factors?.map((f) => f.name) ?? [] @@ -2500,6 +2511,13 @@ export const ProtocolCanvas = forwardRef { + if (!field.connectorFactor) return + setFactorPickerNodeId(null) + setConnectorFactorChoice(field.connectorFactor) + if (field.connectorFactor.kind === 'skill') setSkillFactorNodeId(field.connectorFactor.nodeId) + else setDatasetFactorNodeId(field.connectorFactor.nodeId) + }} existingNames={factorPickerExistingNames} emptyPickerMessage={`${(factorPickerNode?.data as { label?: string })?.label || 'This node'} has no fields that can be turned into a factor.`} revealHiddenServers={revealsHiddenMcpServers(nodes)} @@ -2512,6 +2530,8 @@ export const ProtocolCanvas = forwardRef setSkillFactorNodeId(null)} @@ -2550,6 +2570,8 @@ export const ProtocolCanvas = forwardRef setDatasetFactorNodeId(null)} diff --git a/frontend/src/components/protocol/SkillFactorDialog.tsx b/frontend/src/components/protocol/SkillFactorDialog.tsx index bf17f6ad..dcbc614c 100644 --- a/frontend/src/components/protocol/SkillFactorDialog.tsx +++ b/frontend/src/components/protocol/SkillFactorDialog.tsx @@ -10,21 +10,23 @@ import type { DesignFactor } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' import { computeFactorName } from './factorLevels' -export function SkillFactorDialog({ skillNodeId, graph, factors, onClose, onSave, onRemove }: { +export function SkillFactorDialog({ skillNodeId, graph, factors, initialMode, initialAgentId, onClose, onSave, onRemove }: { skillNodeId: string graph: ProtocolGraph factors: DesignFactor[] + initialMode?: Mode + initialAgentId?: string onClose: () => void onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise onRemove: (name: string) => Promise }) { const skill = graph.nodes.find((node) => node.id === skillNodeId)! const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === skillNodeId && edge.target === node.id && edge.targetHandle === 'skill')) - const [agentId, setAgentId] = useState(agents.find((agent) => agent.data.factor_bindings?.[SKILL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((agent) => agent.data.factor_bindings?.[SKILL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) const initialFactor = existingFor(agentId, SKILL_FACTOR_PATH) ?? existingFor(skillNodeId, 'config.enabled') const initial = initialFactor && initialFactor.level_type !== 'boolean' && initialFactor.level_type ? reconcileSkillFactor(initialFactor, graph, agentId) : initialFactor - const [mode, setMode] = useState((initial?.level_type as Mode) ?? 'boolean') + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') function seed(nextMode: Mode, nextAgentId: string): DesignFactor { const ownerId = nextMode === 'boolean' ? skillNodeId : nextAgentId const existing = existingFor(ownerId, nextMode === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH) @@ -43,7 +45,7 @@ export function SkillFactorDialog({ skillNodeId, graph, factors, onClose, onSave return nextMode === 'boolean' ? factor : reconcileSkillFactor(factor, graph, ownerId) } const [draft, setDraft] = useState(() => { - const factor = initial ?? seed(mode, agentId) + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level) => level ? 'Enabled' : 'Disabled') } }) const [saving, setSaving] = useState(false) diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index d465cf66..eebe8e75 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -251,16 +251,11 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { ] case 'dataset': return [ - // No runtime effect beyond skipping the Dataset-context block in - // _build_user_input. + // Individual availability controls Dataset context and implicit tools. { fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }, // The whole node as a factor -- levels are entirely different - // datasets, which is the ONLY supported way to run one experiment - // across several of them. The Dataset connector is capped at one - // node per agent (see AgentNode.tsx) precisely because a cell's - // workspace is keyed by experiment_id/cell_label and therefore holds - // exactly one dataset; varying it per CELL is the shape that fits, - // and wiring several at once never was. + // datasets. Keep this field for editing legacy bindings; new creation + // uses factorCreationFields and the Dataset connector's levels. // // Unlike skill/okf_bundle above -- whose configs are also just // per-account pointers -- this one earns a structured level type @@ -271,14 +266,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { { fieldPath: 'config', label: 'Dataset', levelType: 'dataset_config' }, ] case 'skill': - // Only `enabled` -- the natural "which skill" factor is the WHOLE node - // (levels = different skills), but unlike script/llm/tool_config, a - // skill node's config is just an id pointing at a row in the skill - // library, so a level would carry a per-account id rather than the - // thing itself. Comparing two skills today means two nodes, one - // enabled per cell; a real skill_config level type is deferred until - // the level editor can pick from the library the way the inspector - // does. + // Connector choices are listed by factorCreationFields. return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'okf_bundle': case 'okf_document': @@ -301,6 +289,8 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { } export interface UnboundField { + pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } + connectorFactor?: { kind: 'skill' | 'dataset'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -474,3 +464,81 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel } return result } + +/** Creation entries route connector choices through their dedicated factor dialogs. */ +export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField[] { + const fields = unboundBindableFields(nodes, edges).filter((field) => { + const kind = nodes.find((node) => node.id === field.nodeId)?.type + return kind !== 'skill' && kind !== 'dataset' + }) + const connectorOwners = new Set() + for (const node of nodes) { + if (node.type !== 'skill' && node.type !== 'dataset') continue + const kind = node.type + const handles = kind === 'skill' ? ['skill'] : ['dataset', 'resource', 'tool'] + const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && handles.includes(edge.targetHandle ?? ''))) + if (!(node.data.factor_bindings as Record | undefined)?.['config.enabled']) fields.push({ + nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.enabled', + fieldLabel: `This ${kind} on/off`, levelType: 'boolean', currentValue: undefined, + connectorFactor: { kind, nodeId: node.id, mode: 'boolean' }, + }) + for (const owner of owners) { + const key = `${owner.id}:${kind}` + if (connectorOwners.has(key)) continue + connectorOwners.add(key) + for (const mode of ['selection', 'toggle'] as const) { + const levelType = `${kind}_${mode}` as LevelType + fields.push({ + nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: `${kind}_selection`, + fieldLabel: mode === 'selection' ? `${kind === 'skill' ? 'Skill' : 'Dataset'} levels` : `All ${kind}s on/off`, + levelType, currentValue: undefined, + connectorFactor: { kind, nodeId: node.id, mode: levelType, agentId: owner.id }, + }) + } + } + } + return fields.map((field) => { + const node = nodes.find((candidate) => candidate.id === field.nodeId)! + const kind = field.connectorFactor?.kind ?? node.type ?? 'node' + const agentKinds = ['agent', 'sub_agent'] + const targets = nodes.filter((candidate) => agentKinds.includes(candidate.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === candidate.id)) + const owner = agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined + const category = field.connectorFactor?.kind === 'skill' ? 'Skills' + : field.connectorFactor?.kind === 'dataset' ? 'Datasets' + : agentKinds.includes(kind) ? 'Agent fields' + : kind.startsWith('llm_') ? 'Model' + : kind.startsWith('pattern_') ? 'Execution pattern' + : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? 'Other components' + const connectorChoice = field.connectorFactor && field.connectorFactor.mode !== 'boolean' + return { + ...field, + pickerGroup: { + id: owner?.id ?? 'shared-unconnected', + label: owner ? String(owner.data.label || 'Agent') : 'Shared or unconnected components', + category, + componentId: connectorChoice ? `${field.nodeId}:${kind}` : node.id, + componentLabel: connectorChoice ? `${category} on this agent` : String(node.data.label || kind), + }, + } + }) +} + +export function groupFactorCreationFields(fields: UnboundField[]) { + const agents = new Map }>() + for (const field of fields) { + const group = field.pickerGroup ?? { id: 'fields', label: 'Canvas fields', category: 'Fields', componentId: field.nodeId, componentLabel: field.nodeLabel } + if (!agents.has(group.id)) agents.set(group.id, { id: group.id, label: group.label, components: new Map() }) + const agent = agents.get(group.id)! + if (!agent.components.has(group.componentId)) agent.components.set(group.componentId, { id: group.componentId, category: group.category, label: group.componentLabel, fields: [] }) + agent.components.get(group.componentId)!.fields.push(field) + } + const categories = ['Agent fields', 'Skills', 'Datasets', 'Model', 'Execution pattern', 'Tools', 'Knowledge', 'Memory', 'Output parser', 'Critic gate', 'Other components'] + return [...agents.values()] + .sort((a, b) => a.id === 'shared-unconnected' ? 1 : b.id === 'shared-unconnected' ? -1 : a.label.localeCompare(b.label)) + .map((agent) => ({ ...agent, components: [...agent.components.values()] + .sort((a, b) => categories.indexOf(a.category) - categories.indexOf(b.category) || a.label.localeCompare(b.label)) + .map((component) => ({ ...component, fields: component.fields.sort((a, b) => { + const modeOrder = (field: UnboundField) => field.connectorFactor?.mode === 'boolean' ? 0 : field.connectorFactor?.mode.endsWith('_selection') ? 1 : 2 + return a.connectorFactor && b.connectorFactor ? modeOrder(a) - modeOrder(b) : a.fieldLabel.localeCompare(b.fieldLabel) + }) })) })) +} diff --git a/frontend/src/components/protocol/factorCreationFields.test.ts b/frontend/src/components/protocol/factorCreationFields.test.ts new file mode 100644 index 00000000..8b9adf0d --- /dev/null +++ b/frontend/src/components/protocol/factorCreationFields.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from 'vitest' +import type { Edge, Node } from '@xyflow/react' +import { factorCreationFields, groupFactorCreationFields } from './bindableFields' + +it('groups by owning agent and component, putting shared nodes last and keeping connector modes together', () => { + const nodes: Node[] = [ + { id: 'z', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Zebra', config: {} } }, + { id: 'a', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Alpha', config: {} } }, + { id: 'shared', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Shared skill', config: { skill_id: 's' } } }, + { id: 'data', type: 'dataset', position: { x: 0, y: 0 }, data: { label: 'Cohort', config: { dataset_id: 'd' } } }, + { id: 'loose', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Unconnected skill', config: { skill_id: 'u' } } }, + ] + const edges: Edge[] = [ + { id: 's-z', source: 'shared', target: 'z', targetHandle: 'skill' }, + { id: 's-a', source: 'shared', target: 'a', targetHandle: 'skill' }, + { id: 'd-a', source: 'data', target: 'a', targetHandle: 'dataset' }, + ] + const groups = groupFactorCreationFields(factorCreationFields(nodes, edges)) + expect(groups.map(group => group.label)).toEqual(['Alpha', 'Zebra', 'Shared or unconnected components']) + expect(groups[0].components.map(component => component.category)).toEqual(['Agent fields', 'Skills', 'Datasets', 'Datasets']) + expect(groups[0].components.find(component => component.category === 'Skills')?.fields.map(field => field.fieldLabel)).toEqual(['Skill levels', 'All skills on/off']) + expect(groups[2].components.map(component => component.label)).toEqual(['Shared skill', 'Unconnected skill']) +}) From 09cc451e5229b82e88e4cbd952132155e496065d Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 13:44:34 -0700 Subject: [PATCH 10/63] feat: add Knowledge node factor controls and execution --- frontend/src/api/client.ts | 2 +- .../src/components/protocol/DesignTab.tsx | 8 +- .../protocol/FactorEditorDialog.tsx | 2 + .../protocol/KnowledgeFactorDialog.test.tsx | 82 ++++++++++ .../protocol/KnowledgeFactorDialog.tsx | 119 ++++++++++++++ .../protocol/KnowledgeFactorEditor.tsx | 74 +++++++++ .../protocol/KnowledgeTestSelectors.tsx | 26 ++++ .../protocol/OkfBundleNodeInspector.tsx | 62 +++----- .../protocol/OkfDocumentNodeInspector.tsx | 60 +++----- .../components/protocol/ProtocolCanvas.tsx | 99 ++++++++++-- .../protocol/ProtocolCanvasContext.tsx | 1 + .../src/components/protocol/bindableFields.ts | 22 ++- .../protocol/factorCreationFields.test.ts | 13 ++ .../src/components/protocol/factorLevels.ts | 8 +- .../components/protocol/nodes/AgentNode.tsx | 2 +- .../protocol/nodes/ConnectorAddStub.tsx | 2 +- .../protocol/nodes/OkfBundleNode.tsx | 10 +- .../protocol/nodes/OkfDocumentNode.tsx | 10 +- .../protocol/useKnowledgeLibrary.ts | 9 ++ .../protocol/useKnowledgeTestSelection.ts | 23 +++ frontend/src/lib/factorBindings.ts | 9 +- frontend/src/lib/knowledgeFactors.test.ts | 38 +++++ frontend/src/lib/knowledgeFactors.ts | 94 ++++++++++++ frontend/src/types/experiments.ts | 2 + frontend/src/types/protocols.ts | 2 + src/asaree/api/protocols.py | 7 + src/asaree/services/design_generation.py | 15 +- src/asaree/services/factor_bindings.py | 7 +- src/asaree/services/knowledge_factors.py | 145 ++++++++++++++++++ src/asaree/services/protocol_execution.py | 12 +- src/asaree/services/protocol_revisions.py | 5 +- tests/test_knowledge_factors.py | 116 ++++++++++++++ 32 files changed, 961 insertions(+), 125 deletions(-) create mode 100644 frontend/src/components/protocol/KnowledgeFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/KnowledgeFactorDialog.tsx create mode 100644 frontend/src/components/protocol/KnowledgeFactorEditor.tsx create mode 100644 frontend/src/components/protocol/KnowledgeTestSelectors.tsx create mode 100644 frontend/src/components/protocol/useKnowledgeLibrary.ts create mode 100644 frontend/src/components/protocol/useKnowledgeTestSelection.ts create mode 100644 frontend/src/lib/knowledgeFactors.test.ts create mode 100644 frontend/src/lib/knowledgeFactors.ts create mode 100644 src/asaree/services/knowledge_factors.py create mode 100644 tests/test_knowledge_factors.py diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 32c6ec04..97a88575 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -187,7 +187,7 @@ export interface ResultsScope { protocol_revision_id?: string } -export interface RowSelection { row_index?: number | null; skill_selections?: Record; dataset_selections?: Record } +export interface RowSelection { row_index?: number | null; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } function scopeQuery(scope?: ResultsScope): string { const params = new URLSearchParams() diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index c8460ad8..2b3f3ace 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -1,3 +1,5 @@ +import { knowledgeFactorIssues, isKnowledgeFactor } from '@/lib/knowledgeFactors' +import { useKnowledgeLibrary } from './useKnowledgeLibrary' import { datasetsApi } from '@/api/client' import { datasetFactorIssues } from '@/lib/datasetFactors' import { useCallback, useEffect, useRef, useState, type RefObject } from 'react' @@ -499,6 +501,8 @@ export function DesignTab({ ? ({ nodes: graphQuery.data.nodes, edges: graphQuery.data.edges } as unknown as ProtocolGraph) : undefined const unboundFactors = unboundFactorNames(experiment.design_spec, draftGraph) + const knowledgeLibrary = useKnowledgeLibrary(factors.some(isKnowledgeFactor)) + const knowledgeIssues = knowledgeFactorIssues(draftGraph, factors, knowledgeLibrary.data ? new Set(knowledgeLibrary.data.map((item) => item.id)) : undefined) const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) }) const datasetLibrary = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list(), enabled: factors.some((factor) => factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') }) const datasetIssues = datasetFactorIssues(draftGraph, factors, datasetLibrary.data ? new Set(datasetLibrary.data.map((dataset) => dataset.id)) : undefined) @@ -871,14 +875,14 @@ export function DesignTab({ Unbound factor{unboundFactors.length === 1 ? '' : 's'}: {unboundFactors.join(', ')}. Rebind on the canvas or remove from this design.

)} - {[...skillIssues, ...datasetIssues].map((issue) =>

{issue}

)} + {[...skillIssues, ...datasetIssues, ...knowledgeIssues].map((issue) =>

{issue}

)} )}
+ {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Knowledge node.' : mode === 'knowledge_selection' ? 'Each cell receives exactly one connected knowledge source.' : 'Each cell enables every connected knowledge source, including deactivated sources, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {mode !== 'boolean' && affected.length < (mode === 'knowledge_toggle' ? 1 : 2) &&

Connect at least {mode === 'knowledge_toggle' ? 'one knowledge source' : 'two different knowledge sources'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'knowledge_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => knowledgeId(node) === (level as string[])[0])?.data.label ?? 'Unavailable knowledge')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+

Agent-level factors follow current Knowledge connections. Changes require design review and regeneration.

+ {current && } + } + {error &&

{error}

} +
+ + +
+} diff --git a/frontend/src/components/protocol/KnowledgeFactorEditor.tsx b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx new file mode 100644 index 00000000..bcd27137 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx @@ -0,0 +1,74 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { useKnowledgeLibrary } from './useKnowledgeLibrary' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +export function KnowledgeFactorEditor({ factor, open, onOpenChange, onSave, onRemove }: { + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const knowledge = useKnowledgeLibrary() + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the knowledge factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'knowledge_toggle' ? 'All knowledge on/off' : 'Knowledge as levels'} + {factor.level_type === 'knowledge_toggle' ? 'Each cell receives all connected knowledge or none.' : 'Each cell receives exactly one connected knowledge source.'} Add or disconnect Knowledge nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different knowledge sources before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const source = knowledge.data?.find((item) => item.id === id) + return
+
+

{factor.level_type === 'knowledge_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : source?.name ?? labels[index] ?? 'Unavailable knowledge'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'knowledge_toggle' && knowledge.isSuccess && !source &&

Unavailable: disconnect or restore this knowledge.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+ {knowledge.isError &&

Could not load the knowledge library.

} +

Individual knowledge switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected knowledge and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/KnowledgeTestSelectors.tsx b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx new file mode 100644 index 00000000..9b736a24 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedKnowledge, knowledgeId } from '@/lib/knowledgeFactors' +import type { useKnowledgeTestSelection } from './useKnowledgeTestSelection' + +export function KnowledgeTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedKnowledge(selection.graph!, node.id) + if (node.data.knowledge_factor_mode === 'knowledge_toggle') return
+

{node.data.label}: knowledge for this test

+ +
+ return
+

{node.data.label}: knowledge for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/OkfBundleNodeInspector.tsx b/frontend/src/components/protocol/OkfBundleNodeInspector.tsx index 0be12a05..dcdb1c7a 100644 --- a/frontend/src/components/protocol/OkfBundleNodeInspector.tsx +++ b/frontend/src/components/protocol/OkfBundleNodeInspector.tsx @@ -1,6 +1,6 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { BookMarked, RefreshCw } from 'lucide-react' +import { BookMarked, RefreshCw, Power, Trash2, Split } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -8,7 +8,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { okfApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import { OkfConceptList } from './OkfConceptList' import type { OkfBundleNodeData, ProtocolNode } from '@/types/protocols' @@ -28,17 +27,13 @@ const ACCENT = nodeAccent('okf_bundle') // bundle, not a useful configuration. export function OkfBundleNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: OkfBundleNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, this node's own plain header title. - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: OkfBundleNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -81,10 +76,6 @@ export function OkfBundleNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -111,35 +102,24 @@ export function OkfBundleNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

- Off: the wired agent gets none of this bundle’s tools, so it can neither read nor write it. -

-
- patchConfig({ enabled: checked })} - /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Knowledge node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this knowledge at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} +
diff --git a/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx b/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx index 73ca36a0..e7e17d5c 100644 --- a/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx +++ b/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx @@ -1,6 +1,6 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { FileText, RefreshCw } from 'lucide-react' +import { FileText, RefreshCw, Power, Trash2, Split } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -8,7 +8,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { okfApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { OkfDocumentNodeData, ProtocolNode } from '@/types/protocols' @@ -26,15 +25,13 @@ const ACCENT = nodeAccent('okf_document') // document as it works, and this is where you see what it became. export function OkfDocumentNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: OkfDocumentNodeData }) | null - experimentId: string | null - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: OkfDocumentNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -72,10 +69,6 @@ export function OkfDocumentNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -102,35 +95,24 @@ export function OkfDocumentNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

- Off: the wired agent gets none of this document’s tools, so it can neither read nor write it. -

-
- patchConfig({ enabled: checked })} - /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Knowledge node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this knowledge at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} +
diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 703643a0..e48a00e3 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -1,3 +1,7 @@ +import { reconcileKnowledgeFactor, knowledgeFactorConflict, knowledgeFactorOwner, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' +import { KnowledgeFactorDialog } from './KnowledgeFactorDialog' +import { KnowledgeTestSelectors } from './KnowledgeTestSelectors' +import { useKnowledgeTestSelection } from './useKnowledgeTestSelection' import { reconcileDatasetFactor, datasetFactorConflict, datasetFactorOwner, DATASET_FACTOR_PATH } from '@/lib/datasetFactors' import { DatasetFactorDialog } from './DatasetFactorDialog' import { DatasetTestSelectors } from './DatasetTestSelectors' @@ -412,14 +416,19 @@ export const ProtocolCanvas = forwardRef { setNodes((nds) => { + const knowledgeIds = new Set(edges.filter((edge) => edge.targetHandle === 'knowledge' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH] === factorName).map((edge) => edge.source)) const skillIds = new Set(edges.filter((edge) => edge.targetHandle === 'skill' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factorName).map((edge) => edge.source)) const datasetIds = new Set(edges.filter((edge) => ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '') && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factorName).map((edge) => edge.source)) return nds.map((n) => { - if (skillIds.has(n.id) || datasetIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } + if (skillIds.has(n.id) || datasetIds.has(n.id) || knowledgeIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } const bindings = n.data.factor_bindings as Record | undefined if (!bindings || !Object.values(bindings).includes(factorName)) return n const next = Object.fromEntries(Object.entries(bindings).filter(([, name]) => name !== factorName)) const data: Record = { ...n.data, factor_bindings: next } + if (bindings[KNOWLEDGE_FACTOR_PATH] === factorName) { + delete data.knowledge_selection + delete data.knowledge_factor_mode + } if (bindings[SKILL_FACTOR_PATH] === factorName) { delete data.skill_selection delete data.skill_factor_mode @@ -470,7 +479,8 @@ export const ProtocolCanvas = forwardRef(null) const [editingFactorName, setEditingFactorName] = useState(null) + const [knowledgeFactorNodeId, setKnowledgeFactorNodeId] = useState(null) const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) const [datasetFactorNodeId, setDatasetFactorNodeId] = useState(null) const [connectorFactorChoice, setConnectorFactorChoice] = useState() @@ -617,10 +628,11 @@ export const ProtocolCanvas = forwardRef { if (draftRow.error) throw new Error(draftRow.error) if (draftSkills.error) throw new Error(draftSkills.error) + if (draftKnowledge.error) throw new Error(draftKnowledge.error) if (draftDatasets.error) throw new Error(draftDatasets.error) await protocolsApi.update(protocolId, { graph: toPersistedGraph(nodes, edges) }) const published = await protocolsApi.publish(protocolId) - return { published, scope, rowIndex, skillOptions: { ...draftSkills.options, ...draftDatasets.options } } + return { published, scope, rowIndex, skillOptions: { ...draftSkills.options, ...draftKnowledge.options, ...draftDatasets.options } } }, onSuccess: async ({ published, scope, rowIndex, skillOptions }) => { queryClient.invalidateQueries({ queryKey: ['protocols', protocolId, 'revisions'] }) @@ -653,6 +665,8 @@ export const ProtocolCanvas = forwardRef { if (publishedSourceError || (publishedRow.binding && publishedRow.error)) throw new Error(publishedSourceError ?? publishedRow.error!) if (publishedSkills.error) throw new Error(publishedSkills.error) + if (publishedKnowledge.error) throw new Error(publishedKnowledge.error) if (publishedDatasets.error) throw new Error(publishedDatasets.error) - return protocolsApi.testRun(protocolId, publishedRow.binding || publishedSkills.owners.length || publishedDatasets.owners.length ? { ...(publishedRow.binding ? { row_index: rowIndex } : {}), ...publishedSkills.options, ...publishedDatasets.options } : undefined) + return protocolsApi.testRun(protocolId, publishedRow.binding || publishedSkills.owners.length || publishedKnowledge.owners.length || publishedDatasets.owners.length ? { ...(publishedRow.binding ? { row_index: rowIndex } : {}), ...publishedSkills.options, ...publishedKnowledge.options, ...publishedDatasets.options } : undefined) }, onSuccess: (run) => { setRunId(run.id) @@ -685,8 +700,9 @@ export const ProtocolCanvas = forwardRef { setRunId(run.id) @@ -1143,6 +1159,8 @@ export const ProtocolCanvas = forwardRef factor.name === (n.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH])?.levels.length, + knowledgeFactorControlled: ['okf_bundle', 'okf_document'].includes(n.type ?? '') && !!knowledgeFactorOwner(toPersistedGraph(nodes, edges), n.id), skillFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH])?.levels.length, skillFactorControlled: n.type === 'skill' && !!skillFactorOwner(toPersistedGraph(nodes, edges), n.id), datasetFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH])?.levels.length, @@ -1357,6 +1375,12 @@ export const ProtocolCanvas = forwardRef { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setKnowledgeFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) const requestDatasetFactor = useCallback((nodeId: string) => { if (!experimentId || experimentLocked) return setSelectedNodeId(null) @@ -1447,6 +1471,7 @@ export const ProtocolCanvas = forwardRef { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') && (node.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.knowledge_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, knowledge_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) useEffect(() => { if (experimentLocked) return setNodes((current) => current.map((node) => { @@ -2374,18 +2408,16 @@ export const ProtocolCanvas = forwardRef ) : selectedNode?.type === 'okf_bundle' ? ( setSelectedNodeId(null)} /> ) : selectedNode?.type === 'okf_document' ? ( setSelectedNodeId(null)} @@ -2491,11 +2523,13 @@ export const ProtocolCanvas = forwardRef publishAndRunMutation.mutate({ scope: pendingRunConfirm, rowIndex: draftRow.rowIndex })} confirmLabel={pendingRunConfirm.type === 'graph' ? 'Test Run' : undefined} - confirmDisabled={!!publishedSourceError || !!publishedSkills.error || !!publishedDatasets.error || (pendingHasRow && !!publishedRow.error)} - publishDisabled={!!draftRow.error || !!draftSkills.error || !!draftDatasets.error} + confirmDisabled={!!publishedSourceError || !!publishedSkills.error || !!publishedKnowledge.error || !!publishedDatasets.error || (pendingHasRow && !!publishedRow.error)} + publishDisabled={!!draftRow.error || !!draftSkills.error || !!draftKnowledge.error || !!draftDatasets.error} additionalContent={
{publishedSkills.owners.length > 0 &&

Published skills · run published v{publishedRevision}

} {showDraftInputs && draftSkills.owners.length > 0 &&

Draft skills · publish and run

} + {publishedKnowledge.owners.length > 0 &&

Published knowledge · run published v{publishedRevision}

} + {showDraftInputs && draftKnowledge.owners.length > 0 &&

Draft knowledge · publish and run

} {publishedDatasets.owners.length > 0 &&

Published datasets · run published v{publishedRevision}

} {showDraftInputs && draftDatasets.owners.length > 0 &&

Draft datasets · publish and run

} {publishedRevision !== null && (pendingHasRow ?

Published source · run published v{publishedRevision}

{publishedRow.error &&

{publishedRow.error}

}
: publishedSourceError ?

{publishedSourceError}

: null)} @@ -2516,7 +2550,8 @@ export const ProtocolCanvas = forwardRef} + {knowledgeFactorNodeId && setKnowledgeFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = knowledgeFactorConflict(graph, fresh.design_spec?.factors ?? [], knowledgeFactorNodeId, factor.level_type as 'boolean' | 'knowledge_selection' | 'knowledge_toggle', ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileKnowledgeFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { knowledge_selection: next.levels[0] ?? [], knowledge_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'config.enabled' : KNOWLEDGE_FACTOR_PATH]: next.name }, + } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individual = graph.nodes.find((node) => node.id === knowledgeFactorNodeId)?.data.factor_bindings?.['config.enabled'] === name + const shared = individual && graph.nodes.some((node) => node.id !== knowledgeFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === knowledgeFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== 'config.enabled')) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} {datasetFactorNodeId && void // Opens the consolidated factor dialog from a Skill node's hover toolbar. requestDatasetFactor?: (nodeId: string) => void + requestKnowledgeFactor?: (nodeId: string) => void requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index eebe8e75..a81dfbd4 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -270,12 +270,8 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'okf_bundle': case 'okf_document': - // Only `enabled`, same reasoning as skill above: the node's config is a - // pointer at a per-account registration (a server name plus a path on - // the server), not the knowledge itself, so a level would carry a - // reference rather than the thing being compared. Comparing "with the - // knowledge vs. without" -- the question actually worth an experiment - // here -- is exactly what toggling `enabled` per cell does. + // Individual availability binds here; Knowledge connector choices are + // listed by factorCreationFields and edited in the dedicated dialog. return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'script': // The whole node as a factor -- levels are entirely different scripts. @@ -290,7 +286,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { export interface UnboundField { pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } - connectorFactor?: { kind: 'skill' | 'dataset'; nodeId: string; mode: LevelType; agentId?: string } + connectorFactor?: { kind: 'skill' | 'dataset' | 'knowledge'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -434,6 +430,7 @@ export function agentTracedLabel(node: Node, edges: Edge[], nodes: Node[]): stri export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundField[] { const result: UnboundField[] = [] for (const node of nodes) { + if (['okf_bundle', 'okf_document'].includes(node.type ?? '') && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'knowledge' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.knowledge_selection)) continue if (node.type === 'skill' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'skill' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.skill_selection)) continue if (node.type === 'dataset' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'dataset' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.dataset_selection)) continue const bindings = (node.data as { factor_bindings?: Record })?.factor_bindings ?? {} @@ -469,13 +466,13 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField[] { const fields = unboundBindableFields(nodes, edges).filter((field) => { const kind = nodes.find((node) => node.id === field.nodeId)?.type - return kind !== 'skill' && kind !== 'dataset' + return kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' }) const connectorOwners = new Set() for (const node of nodes) { - if (node.type !== 'skill' && node.type !== 'dataset') continue - const kind = node.type - const handles = kind === 'skill' ? ['skill'] : ['dataset', 'resource', 'tool'] + if (!['skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue + const kind = (node.type === 'okf_bundle' || node.type === 'okf_document' ? 'knowledge' : node.type) as 'skill' | 'dataset' | 'knowledge' + const handles = kind === 'skill' ? ['skill'] : kind === 'knowledge' ? ['knowledge'] : ['dataset', 'resource', 'tool'] const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && handles.includes(edge.targetHandle ?? ''))) if (!(node.data.factor_bindings as Record | undefined)?.['config.enabled']) fields.push({ nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.enabled', @@ -490,7 +487,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const levelType = `${kind}_${mode}` as LevelType fields.push({ nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: `${kind}_selection`, - fieldLabel: mode === 'selection' ? `${kind === 'skill' ? 'Skill' : 'Dataset'} levels` : `All ${kind}s on/off`, + fieldLabel: mode === 'selection' ? `${kind === 'skill' ? 'Skill' : kind === 'knowledge' ? 'Knowledge' : 'Dataset'} levels` : `All ${kind === 'knowledge' ? 'knowledge' : `${kind}s`} on/off`, levelType, currentValue: undefined, connectorFactor: { kind, nodeId: node.id, mode: levelType, agentId: owner.id }, }) @@ -505,6 +502,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const owner = agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined const category = field.connectorFactor?.kind === 'skill' ? 'Skills' : field.connectorFactor?.kind === 'dataset' ? 'Datasets' + : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' : agentKinds.includes(kind) ? 'Agent fields' : kind.startsWith('llm_') ? 'Model' : kind.startsWith('pattern_') ? 'Execution pattern' diff --git a/frontend/src/components/protocol/factorCreationFields.test.ts b/frontend/src/components/protocol/factorCreationFields.test.ts index 8b9adf0d..12fe4539 100644 --- a/frontend/src/components/protocol/factorCreationFields.test.ts +++ b/frontend/src/components/protocol/factorCreationFields.test.ts @@ -21,3 +21,16 @@ it('groups by owning agent and component, putting shared nodes last and keeping expect(groups[0].components.find(component => component.category === 'Skills')?.fields.map(field => field.fieldLabel)).toEqual(['Skill levels', 'All skills on/off']) expect(groups[2].components.map(component => component.label)).toEqual(['Shared skill', 'Unconnected skill']) }) + +it('routes both knowledge kinds through the dialog and offers connector modes once per agent', () => { + const nodes: Node[] = [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Researcher' } }, + { id: 'bundle', type: 'okf_bundle', position: { x: 0, y: 0 }, data: { label: 'Bundle', config: { bundle_id: 'b' } } }, + { id: 'doc', type: 'okf_document', position: { x: 0, y: 0 }, data: { label: 'Document', config: { document_id: 'd' } } }, + ] + const edges: Edge[] = ['bundle', 'doc'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'knowledge' })) + const fields = factorCreationFields(nodes, edges).filter((field) => field.connectorFactor?.kind === 'knowledge') + expect(fields.map((field) => field.fieldLabel)).toEqual(['This knowledge on/off', 'Knowledge levels', 'All knowledge on/off', 'This knowledge on/off']) + expect(fields.every((field) => field.pickerGroup?.category === 'Knowledge')).toBe(true) + expect(fields.filter((field) => field.connectorFactor?.agentId === 'agent')).toHaveLength(2) +}) diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index c91ffff8..08d3e418 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -25,6 +25,8 @@ export type LevelType = | 'skill_toggle' | 'dataset_selection' | 'dataset_toggle' + | 'knowledge_selection' + | 'knowledge_toggle' export const LEVEL_TYPE_LABELS: Record = { string: 'String', @@ -41,6 +43,8 @@ export const LEVEL_TYPE_LABELS: Record = { skill_toggle: 'All skills on/off', dataset_selection: 'Dataset levels', dataset_toggle: 'All datasets on/off', + knowledge_selection: 'Knowledge levels', + knowledge_toggle: 'All knowledge on/off', } // Whether a level of this kind is a structured value (an object, or -- for @@ -59,7 +63,9 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'skill_selection' || type === 'skill_toggle' || type === 'dataset_selection' || - type === 'dataset_toggle' + type === 'dataset_toggle' || + type === 'knowledge_selection' || + type === 'knowledge_toggle' ) } diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index 01cbf2af..3af87b6a 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -351,7 +351,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Knowledge - + {/* Handle id `model`; graphs saved before the rename carry these edges on `ai` or `llm` -- ProtocolCanvas.tsx rewrites those on load (migrateLegacyHandles) and the backend keeps accepting both (see diff --git a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx index 40654fea..980aa25e 100644 --- a/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx +++ b/frontend/src/components/protocol/nodes/ConnectorAddStub.tsx @@ -68,7 +68,7 @@ export function ConnectorAddStub({ className={side === 'right' ? 'group/stub absolute top-1/2 -right-11 -translate-y-1/2' : side === 'top' ? 'group/stub absolute -top-11 -translate-x-1/2' : 'group/stub absolute -bottom-11 -translate-x-1/2'} style={side !== 'right' ? { left } : undefined} > - {factorName && Skills · {factorOnOff ? 'on/off' : `${factorLevelCount ?? 0} levels`}} + {factorName && {slot === 'knowledge' ? 'Knowledge' : slot === 'dataset' ? 'Datasets' : 'Skills'} · {factorOnOff ? 'on/off' : `${factorLevelCount ?? 0} levels`}} )} -
)}
+ onClick={() => { onChange(choice.value); setOpen(false) }}>)}
+ )} + )} )} @@ -420,7 +425,7 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM ({ value: `agent:${source.agentNodeId}`, display: agentNodeDisplay(source), reason: agentSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Agent output' })), ...pythonSources.map(source => ({ value: `python:${source.key}`, display: pythonNodeDisplay(source), reason: pythonSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Scripts' })), - ...mcpSources.map(source => ({ value: `mcp:${source.key}`, display: mcpNodeDisplay(source), reason: mcpSourceDisabledReason(source), agentId: source.agentNodeId, category: 'MCP tools' })), + ...mcpSources.map(source => ({ value: `mcp:${source.key}`, display: mcpNodeDisplay(source), reason: mcpSourceDisabledReason(source), agentId: source.agentNodeId, category: 'MCP Tools' })), ...toolStepSources.map(source => ({ value: `tool_step:${source.nodeId}`, display: toolStepNodeDisplay(source), reason: source.disabledReason, category: 'Tool steps' })), ...(offerNodeRuntime ? [{ value: 'node_runtime', display: nodeRuntimeDisplay, category: 'Runtime' }] : []), ]} /> diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 2b3f3ace..ccf1146f 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -192,6 +192,7 @@ function AddFactorButton({ revealHiddenServers={revealsHiddenMcpServers(graphQuery.data?.nodes ?? [])} promptScopeFor={promptScopeFor} pickableFields={fields} + canvasAgentCount={(graphQuery.data?.nodes ?? []).filter(node => node.type === 'agent' || node.type === 'sub_agent').length} onPickConnectorFactor={(field) => { canvasRef.current?.openConnectorFactor(field) setDialogOpen(false) diff --git a/frontend/src/components/protocol/FactorEditorDialog.tsx b/frontend/src/components/protocol/FactorEditorDialog.tsx index 8ae8c46a..5585470d 100644 --- a/frontend/src/components/protocol/FactorEditorDialog.tsx +++ b/frontend/src/components/protocol/FactorEditorDialog.tsx @@ -11,7 +11,7 @@ import { Switch } from '@/components/ui/switch' import { Textarea } from '@/components/ui/textarea' import { datasetsApi, mcpServersApi } from '@/api/client' import { isPromptReferenceField, type PromptReferenceScope } from '@/lib/promptReferences' -import { cardAccent, cn, hashToChartHue, HUD_ACCENT_RING_CLASSNAME } from '@/lib/utils' +import { cardAccent, cn, hashToChartHue, HUD_ACCENT_RING_CLASSNAME, PICKER_GROUP_CLASSNAME } from '@/lib/utils' import { groupFactorCreationFields, pickToolNamesForServer, selectableMcpServers, type UnboundField } from './bindableFields' import { useProviderModels } from './useProviderModels' import { @@ -513,6 +513,7 @@ function StandardFactorEditorDialog({ onOpenChange, factor, pickableFields, + canvasAgentCount = 1, existingNames, emptyPickerMessage, toolServerId, @@ -546,6 +547,7 @@ function StandardFactorEditorDialog({ // large number of fields to search through, and this dialog already has // the room a cramped popover wouldn't. pickableFields?: UnboundField[] + canvasAgentCount?: number onPickConnectorFactor?: (field: UnboundField) => void // Needed to dedupe the computed name once a field is picked -- only // meaningful alongside pickableFields. @@ -722,16 +724,18 @@ function StandardFactorEditorDialog({ : 'No matching fields.'}

)} - {fieldGroups.map((group) =>
- {group.label}{group.components.reduce((count, component) => count + component.fields.length, 0)} options -
{group.components.map((component) =>
-

{component.category}{component.category === 'Agent fields' ? '' : ` · ${component.label}`}

- {component.fields.map((field) => ( + {fieldGroups.map((group) =>
+ {group.label}{group.connectors.reduce((count, connector) => count + connector.components.reduce((total, component) => total + component.fields.length, 0), 0)} options +
{group.connectors.map((connector) =>
+

{connector.label}

+
{connector.components.map((component) =>
+ {connector.label !== 'Agent fields' &&

{component.label}

} +
{component.fields.map((field) => ( - ))} + ))}
+
)}
)}
)}
diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index e48a00e3..a720c9ab 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -2545,6 +2545,7 @@ export const ProtocolCanvas = forwardRef node.type === 'agent' || node.type === 'sub_agent').length} onPickConnectorFactor={(field) => { if (!field.connectorFactor) return setFactorPickerNodeId(null) diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index a81dfbd4..3210116e 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -504,9 +504,9 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField : field.connectorFactor?.kind === 'dataset' ? 'Datasets' : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' : agentKinds.includes(kind) ? 'Agent fields' - : kind.startsWith('llm_') ? 'Model' - : kind.startsWith('pattern_') ? 'Execution pattern' - : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? 'Other components' + : MODEL_NODE_TYPES.has(kind) ? 'Model' + : kind.startsWith('pattern_') ? 'Pattern' + : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', mcp_scikit_learn: 'Tools', mcp_client_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? String(node.data.label || kind) const connectorChoice = field.connectorFactor && field.connectorFactor.mode !== 'boolean' return { ...field, @@ -522,21 +522,23 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField } export function groupFactorCreationFields(fields: UnboundField[]) { - const agents = new Map }>() + type ComponentGroup = { id: string; label: string; fields: UnboundField[] } + type ConnectorGroup = { label: string; components: Map } + const agents = new Map }>() for (const field of fields) { const group = field.pickerGroup ?? { id: 'fields', label: 'Canvas fields', category: 'Fields', componentId: field.nodeId, componentLabel: field.nodeLabel } - if (!agents.has(group.id)) agents.set(group.id, { id: group.id, label: group.label, components: new Map() }) + if (!agents.has(group.id)) agents.set(group.id, { id: group.id, label: group.label, connectors: new Map() }) const agent = agents.get(group.id)! - if (!agent.components.has(group.componentId)) agent.components.set(group.componentId, { id: group.componentId, category: group.category, label: group.componentLabel, fields: [] }) - agent.components.get(group.componentId)!.fields.push(field) + if (!agent.connectors.has(group.category)) agent.connectors.set(group.category, { label: group.category, components: new Map() }) + const connector = agent.connectors.get(group.category)! + if (!connector.components.has(group.componentId)) connector.components.set(group.componentId, { id: group.componentId, label: group.componentLabel, fields: [] }) + connector.components.get(group.componentId)!.fields.push(field) } - const categories = ['Agent fields', 'Skills', 'Datasets', 'Model', 'Execution pattern', 'Tools', 'Knowledge', 'Memory', 'Output parser', 'Critic gate', 'Other components'] return [...agents.values()] - .sort((a, b) => a.id === 'shared-unconnected' ? 1 : b.id === 'shared-unconnected' ? -1 : a.label.localeCompare(b.label)) - .map((agent) => ({ ...agent, components: [...agent.components.values()] - .sort((a, b) => categories.indexOf(a.category) - categories.indexOf(b.category) || a.label.localeCompare(b.label)) - .map((component) => ({ ...component, fields: component.fields.sort((a, b) => { - const modeOrder = (field: UnboundField) => field.connectorFactor?.mode === 'boolean' ? 0 : field.connectorFactor?.mode.endsWith('_selection') ? 1 : 2 - return a.connectorFactor && b.connectorFactor ? modeOrder(a) - modeOrder(b) : a.fieldLabel.localeCompare(b.fieldLabel) - }) })) })) + .sort((a, b) => a.label.localeCompare(b.label)) + .map((agent) => ({ ...agent, connectors: [...agent.connectors.values()] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((connector) => ({ ...connector, components: [...connector.components.values()] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((component) => ({ ...component, fields: component.fields.sort((a, b) => a.fieldLabel.localeCompare(b.fieldLabel)) })) })) })) } diff --git a/frontend/src/components/protocol/factorCreationFields.test.ts b/frontend/src/components/protocol/factorCreationFields.test.ts index 12fe4539..f9f143b8 100644 --- a/frontend/src/components/protocol/factorCreationFields.test.ts +++ b/frontend/src/components/protocol/factorCreationFields.test.ts @@ -2,7 +2,7 @@ import { expect, it } from 'vitest' import type { Edge, Node } from '@xyflow/react' import { factorCreationFields, groupFactorCreationFields } from './bindableFields' -it('groups by owning agent and component, putting shared nodes last and keeping connector modes together', () => { +it('sorts agent groups, component groups, and factor choices alphabetically', () => { const nodes: Node[] = [ { id: 'z', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Zebra', config: {} } }, { id: 'a', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Alpha', config: {} } }, @@ -16,10 +16,25 @@ it('groups by owning agent and component, putting shared nodes last and keeping { id: 'd-a', source: 'data', target: 'a', targetHandle: 'dataset' }, ] const groups = groupFactorCreationFields(factorCreationFields(nodes, edges)) - expect(groups.map(group => group.label)).toEqual(['Alpha', 'Zebra', 'Shared or unconnected components']) - expect(groups[0].components.map(component => component.category)).toEqual(['Agent fields', 'Skills', 'Datasets', 'Datasets']) - expect(groups[0].components.find(component => component.category === 'Skills')?.fields.map(field => field.fieldLabel)).toEqual(['Skill levels', 'All skills on/off']) - expect(groups[2].components.map(component => component.label)).toEqual(['Shared skill', 'Unconnected skill']) + expect(groups.map(group => group.label)).toEqual(['Alpha', 'Shared or unconnected components', 'Zebra']) + expect(groups[0].connectors.map(connector => connector.label)).toEqual(['Agent fields', 'Datasets', 'Skills']) + expect(groups[0].connectors.find(connector => connector.label === 'Datasets')?.components.map(component => component.label)).toEqual(['Cohort', 'Datasets on this agent']) + expect(groups[0].connectors.find(connector => connector.label === 'Skills')?.components[0].fields.map(field => field.fieldLabel)).toEqual(['All skills on/off', 'Skill levels']) + expect(groups[1].connectors[0].components.map(component => component.label)).toEqual(['Shared skill', 'Unconnected skill']) +}) + +it('uses connector labels for every model and MCP family and orders components and fields by name', () => { + const kinds = ['mcp_tool', 'model_openai', 'mcp_client_tool', 'model_local', 'mcp_scikit_learn', 'model_anthropic', 'model_openrouter', 'model_azure_foundry', 'pattern_reason_act'] + const nodes: Node[] = [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + ...kinds.map((kind) => ({ id: kind, type: kind, position: { x: 0, y: 0 }, data: { label: kind, config: {} } })), + ] + const edges: Edge[] = kinds.map((kind) => ({ id: kind, source: kind, target: 'agent' })) + const groups = groupFactorCreationFields(factorCreationFields(nodes, edges)) + const connectors = groups[0].connectors + expect(connectors.map((connector) => connector.label)).toEqual(['Agent fields', 'Model', 'Pattern', 'Tools']) + expect(connectors.find((connector) => connector.label === 'Model')?.components.map((component) => component.label)).toEqual(['model_anthropic', 'model_azure_foundry', 'model_local', 'model_openai', 'model_openrouter']) + expect(connectors.find((connector) => connector.label === 'Pattern')?.components[0].fields.map((field) => field.fieldLabel)).toEqual(['Include scratchpad', 'Max iterations', 'Observation format', 'Scratchpad window']) }) it('routes both knowledge kinds through the dialog and offers connector modes once per agent', () => { diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 2d0de9f2..47f5db8d 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -31,6 +31,10 @@ export function cardAccent(color: string): CSSProperties { export const HUD_ACCENT_RING_CLASSNAME = "ring-1 ring-[color:var(--card-accent,var(--primary))]/15 shadow-[0_0_24px_-12px_var(--card-accent,var(--primary))] before:pointer-events-none before:absolute before:top-1 before:left-1 before:size-3 before:border-t-2 before:border-l-2 before:border-[color:var(--card-accent,var(--primary))]/60 before:content-[''] after:pointer-events-none after:absolute after:right-1 after:bottom-1 after:size-3 after:border-r-2 after:border-b-2 after:border-[color:var(--card-accent,var(--primary))]/60 after:content-['']" +/** Shared outline for collapsible agent groups in the factor and metric pickers. */ +export const PICKER_GROUP_CLASSNAME = + 'mb-2 rounded-md border border-primary/40 shadow-[0_0_16px_-6px_var(--primary)]' + const CHART_HUES = ['var(--chart-1)', 'var(--chart-2)', 'var(--chart-3)', 'var(--chart-4)', 'var(--chart-5)'] /** Deterministically maps a string (e.g. a model name) to one of the theme's From 2a08b96f0d611b6b9055910adb0eaf06d46e58ee Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 15:39:04 -0700 Subject: [PATCH 12/63] feat: consolidate model and pattern factor dialogs --- .../protocol/DatasetFactorDialog.test.tsx | 16 + .../protocol/DatasetFactorDialog.tsx | 8 +- .../protocol/FactorEditorDialog.tsx | 17 +- .../protocol/ModelFactorDialog.test.tsx | 141 +++++++ .../components/protocol/ModelFactorDialog.tsx | 126 ++++++ .../src/components/protocol/ModelField.tsx | 6 +- .../protocol/ModelNodeInspector.tsx | 377 +++++++----------- .../protocol/PatternFactorDialog.test.tsx | 77 ++++ .../protocol/PatternFactorDialog.tsx | 107 +++++ .../protocol/ProtocolCanvas.test.tsx | 43 ++ .../components/protocol/ProtocolCanvas.tsx | 126 +++++- .../protocol/ProtocolCanvasContext.tsx | 4 + .../ReasonActPatternNodeInspector.tsx | 218 ++++------ ...ingleAgentBaselinePatternNodeInspector.tsx | 101 ++--- .../protocol/SkillFactorDialog.test.tsx | 16 + .../components/protocol/SkillFactorDialog.tsx | 6 +- .../src/components/protocol/bindableFields.ts | 31 +- .../protocol/nodes/ModelFactorGroup.test.tsx | 50 +++ .../protocol/nodes/ModelFactorGroup.tsx | 92 +++++ .../components/protocol/nodes/ModelNode.tsx | 7 + .../protocol/nodes/ReasonActPatternNode.tsx | 5 +- .../nodes/SingleAgentBaselinePatternNode.tsx | 5 +- .../src/components/protocol/patternFactors.ts | 31 ++ 23 files changed, 1132 insertions(+), 478 deletions(-) create mode 100644 frontend/src/components/protocol/ModelFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/ModelFactorDialog.tsx create mode 100644 frontend/src/components/protocol/PatternFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/PatternFactorDialog.tsx create mode 100644 frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx create mode 100644 frontend/src/components/protocol/nodes/ModelFactorGroup.tsx create mode 100644 frontend/src/components/protocol/patternFactors.ts diff --git a/frontend/src/components/protocol/DatasetFactorDialog.test.tsx b/frontend/src/components/protocol/DatasetFactorDialog.test.tsx index e9273b22..1e87886b 100644 --- a/frontend/src/components/protocol/DatasetFactorDialog.test.tsx +++ b/frontend/src/components/protocol/DatasetFactorDialog.test.tsx @@ -11,6 +11,22 @@ const graph = { edges: [{ source: 'dataset', target: 'agent', targetHandle: 'dataset' }], } as unknown as ProtocolGraph +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Dataset enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Dataset levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + it.each(['Dataset levels', 'All datasets on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { const save = vi.fn().mockResolvedValue(undefined) render() diff --git a/frontend/src/components/protocol/DatasetFactorDialog.tsx b/frontend/src/components/protocol/DatasetFactorDialog.tsx index 619c645f..bdd17860 100644 --- a/frontend/src/components/protocol/DatasetFactorDialog.tsx +++ b/frontend/src/components/protocol/DatasetFactorDialog.tsx @@ -78,8 +78,10 @@ export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode setError('') try { if (remove && current) await onRemove(current.name) - else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) - onClose() + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the dataset factor.') } finally { setSaving(false) } @@ -100,7 +102,7 @@ export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode
{(Object.keys(modes) as Mode[]).map((value) => )}
{legacy &&

This Dataset has an existing configuration factor: {legacy.name}. Remove it before creating a Dataset factor.

} {blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Dataset node.' : mode === 'dataset_selection' ? 'Each cell receives exactly one connected dataset.' : 'Each cell enables every connected dataset, including deactivated datasets, or disables them all.'}

diff --git a/frontend/src/components/protocol/FactorEditorDialog.tsx b/frontend/src/components/protocol/FactorEditorDialog.tsx index 5585470d..9b9db72f 100644 --- a/frontend/src/components/protocol/FactorEditorDialog.tsx +++ b/frontend/src/components/protocol/FactorEditorDialog.tsx @@ -50,10 +50,11 @@ type StructuredLevel = Record // own Provider/Model/Temperature/Effort/Max tokens fields exactly, since a // level here IS a whole Model node's config (protocol_execution.py's // _resolve_model_config reads it verbatim, never the node's xyflow type). -function LlmConfigLevelRow({ value, onChange }: { value: StructuredLevel; onChange: (next: StructuredLevel) => void }) { +export function LlmConfigLevelRow({ value, onChange }: { value: StructuredLevel; onChange: (next: StructuredLevel) => void }) { const provider = (value.provider as string) || 'anthropic' const { modelsQuery, models } = useProviderModels(provider) const selectedModelInfo = models.find((m) => m.id === value.model) + const showTemperature = selectedModelInfo?.supports_temperature ?? true const showEffort = selectedModelInfo?.supports_effort ?? false const effortLevels = selectedModelInfo?.effort_levels.length ? selectedModelInfo.effort_levels : EFFORT_LEVELS_FALLBACK @@ -87,7 +88,7 @@ function LlmConfigLevelRow({ value, onChange }: { value: StructuredLevel; onChan onChange={(model) => patch({ model })} /> -
+ {showTemperature &&
patch({ temperature: e.target.value === '' ? null : Number(e.target.value) })} /> -
+
} {showEffort && (
@@ -221,7 +222,7 @@ function ToolConfigLevelRow({ // slug. Written onto the agent's own synthetic data.pattern_override (see // bindableFields.ts) -- protocol_execution.py's _resolve_pattern_config // checks this before falling back to the wired connector node. -function PatternLevelRow({ value, onChange }: { value: StructuredLevel; onChange: (next: StructuredLevel) => void }) { +export function PatternLevelRow({ value, onChange }: { value: StructuredLevel; onChange: (next: StructuredLevel) => void }) { const slug = (value.execution_pattern as string) || 'reason_act' const allParams = (value.pattern_params as Record | undefined) ?? {} const params = allParams[slug] ?? {} @@ -231,7 +232,7 @@ function PatternLevelRow({ value, onChange }: { value: StructuredLevel; onChange } function changeSlug(nextSlug: string) { - onChange({ execution_pattern: nextSlug, pattern_params: { [nextSlug]: allParams[nextSlug] ?? {} } }) + onChange({ execution_pattern: nextSlug, pattern_params: { [nextSlug]: allParams[nextSlug] ?? (nextSlug === 'reason_act' ? { max_iterations: 30, include_scratchpad: true, scratchpad_window: 10, observation_format: 'raw' } : { max_iterations: 10, stop_on_first_success: true }) } }) } return ( @@ -258,7 +259,7 @@ function PatternLevelRow({ value, onChange }: { value: StructuredLevel; onChange type="number" min="1" value={(params.max_iterations as number | undefined) ?? ''} - onChange={(e) => patchParams({ max_iterations: Number(e.target.value) })} + onChange={(e) => patchParams({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} />
{slug === 'reason_act' && ( @@ -277,6 +278,10 @@ function PatternLevelRow({ value, onChange }: { value: StructuredLevel; onChange /> )} + {slug === 'reason_act' && <> + {(params.include_scratchpad ?? true) &&
patchParams({ scratchpad_window: event.target.value === '' ? null : Number(event.target.value) })} />
} +
+ } ) } diff --git a/frontend/src/components/protocol/ModelFactorDialog.test.tsx b/frontend/src/components/protocol/ModelFactorDialog.test.tsx new file mode 100644 index 00000000..5dd161a3 --- /dev/null +++ b/frontend/src/components/protocol/ModelFactorDialog.test.tsx @@ -0,0 +1,141 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ModelNodeData, ProtocolNode } from '@/types/protocols' +import { factorCreationFields } from './bindableFields' +import { ModelFactorDialog } from './ModelFactorDialog' +import { LlmConfigLevelRow } from './FactorEditorDialog' + +vi.mock('./useProviderModels', () => ({ useProviderModels: () => ({ models: [ + { id: 'gpt-5', label: 'GPT 5', supports_temperature: true, supports_effort: false, effort_levels: [] }, + { id: 'reasoning', label: 'Reasoning model', supports_temperature: false, supports_effort: true, effort_levels: ['low', 'high'] }, +], modelsQuery: { isLoading: false } }) })) + +const node: ProtocolNode & { data: ModelNodeData } = { + id: 'model', type: 'model_openai', position: { x: 0, y: 0 }, + data: { label: 'OpenAI', config: { provider: 'openai', model: 'gpt-5', temperature: 0.7, max_tokens: 1000 } }, +} + +it('routes New Factor model choices to the dedicated dialog while retaining their node grouping', () => { + const fields = factorCreationFields([node], []) + expect(fields).toHaveLength(5) + expect(fields.find((field) => field.fieldPath === 'config')).toMatchObject({ + fieldLabel: 'Model levels', connectorFactor: { kind: 'model', nodeId: 'model', mode: 'model_config' }, + pickerGroup: { category: 'Model', componentId: 'model', componentLabel: 'OpenAI' }, + }) + expect(fields.every((field) => field.connectorFactor?.kind === 'model')).toBe(true) +}) + +it('blocks whole-model factors when individual fields are already bound, and allows editing that field', () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.temperature': 'Heat' } } } + render() + fireEvent.click(screen.getByRole('button', { name: 'Model levels' })) + expect(screen.getByRole('status')).toHaveTextContent('cannot be combined') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'Temperature' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Heat') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('saves reordered numeric levels with their labels and previous factor name', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.temperature': 'Heat' } } } + render() + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Temperature' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Temperature', level_type: 'number', levels: [1, 0.7], level_labels: ['High', 'Low'] }, 'config.temperature', 'Heat')) +}) + +it('removes an existing model factor through its bound path', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Models', 'config')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + rerender() + expect(screen.getByRole('dialog')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Temperature' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() +}) + +it('uses the shared model dropdown and disables models selected in other levels', async () => { + render() + expect(screen.getByRole('combobox', { name: 'Level 1 model' })).toHaveTextContent('GPT 5') + expect(screen.getByRole('combobox', { name: 'Level 2 model' })).toHaveTextContent('Select a model') + expect(screen.getByRole('button', { name: 'Temperature' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Effort' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('combobox', { name: 'Level 2 model' })) + expect(await screen.findByRole('option', { name: 'GPT 5' })).toHaveAttribute('aria-disabled', 'true') + expect(screen.getByRole('option', { name: 'Reasoning model' })).not.toHaveAttribute('aria-disabled', 'true') +}) + +it('offers only Effort for a reasoning model and disables efforts selected in other levels', async () => { + const reasoning = { ...node, data: { ...node.data, config: { ...node.data.config, model: 'reasoning', effort: 'low' } } } + render() + expect(screen.queryByRole('button', { name: 'Temperature' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Effort' })).toBeInTheDocument() + expect(screen.getByRole('combobox', { name: 'Level 1 effort' })).toHaveTextContent('low') + fireEvent.click(screen.getByRole('combobox', { name: 'Level 2 effort' })) + expect(await screen.findByRole('option', { name: 'low' })).toHaveAttribute('aria-disabled', 'true') + expect(screen.getByRole('option', { name: 'high' })).not.toHaveAttribute('aria-disabled', 'true') +}) + +it('updates whole-model level controls when its selected model changes', () => { + const { rerender } = render() + expect(screen.getByText('Temperature')).toBeInTheDocument() + expect(screen.queryByText('Effort')).not.toBeInTheDocument() + rerender() + expect(screen.queryByText('Temperature')).not.toBeInTheDocument() + expect(screen.getByText('Effort')).toBeInTheDocument() + rerender() + expect(screen.getByText('Temperature')).toBeInTheDocument() + expect(screen.queryByText('Effort')).not.toBeInTheDocument() +}) + +it('warns about duplicate model configurations regardless of property order and clears when changed', () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + render() + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + const temperatures = screen.getAllByRole('spinbutton') + fireEvent.change(temperatures[2], { target: { value: '1' } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() +}) + +it.each([ + ['config.temperature', '0.70', '1'], + ['config.max_tokens', '1000.0', '2000'], +])('warns about duplicate numeric values for %s and clears after correction', (path, duplicate, distinct) => { + render() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: duplicate } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: distinct } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() +}) + +it('does not treat an empty numeric level as a duplicate of zero', () => { + render() + fireEvent.change(screen.getByLabelText('Level 1 value'), { target: { value: '0' } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '0.0' } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') +}) + +it('focuses the level selected from a canvas preview', async () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + render() + await waitFor(() => expect(screen.getByLabelText('Level 2 label')).toHaveFocus()) +}) diff --git a/frontend/src/components/protocol/ModelFactorDialog.tsx b/frontend/src/components/protocol/ModelFactorDialog.tsx new file mode 100644 index 00000000..1016208a --- /dev/null +++ b/frontend/src/components/protocol/ModelFactorDialog.tsx @@ -0,0 +1,126 @@ +import { useEffect, useRef, useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import type { DesignFactor } from '@/types/experiments' +import { factorValueKey } from '@/lib/experiment' +import type { ProtocolNode } from '@/types/protocols' +import { bindableFieldsForNode } from './bindableFields' +import { LlmConfigLevelRow } from './FactorEditorDialog' +import { ModelField } from './ModelField' +import { useProviderModels } from './useProviderModels' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' + +export function ModelFactorDialog({ node, nodeLabel, factors, initialFieldPath, initialLevelIndex, onClose, onSave, onRemove }: { + node: ProtocolNode + nodeLabel: string + factors: DesignFactor[] + initialFieldPath?: string + initialLevelIndex?: number + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string) => Promise +}) { + const config = { ...node.data.config } as Record + const { models, modelsQuery } = useProviderModels(config.provider as string | undefined) + const modelInfo = models.find((model) => model.id === config.model) + const showTemperature = modelInfo?.supports_temperature ?? true + const showEffort = modelInfo?.supports_effort ?? false + const effortLevels = modelInfo?.effort_levels.length ? modelInfo.effort_levels : ['low', 'medium', 'high', 'xhigh', 'max'] + const fields = bindableFieldsForNode(node) + const available = (fieldPath: string) => fieldPath === 'config.temperature' ? showTemperature : fieldPath === 'config.effort' ? showEffort : true + const visibleFields = fields.filter((field) => available(field.fieldPath) || node.data.factor_bindings?.[field.fieldPath] || initialFieldPath === field.fieldPath) + .sort((a, b) => Number(b.fieldPath === 'config') - Number(a.fieldPath === 'config') || a.label.localeCompare(b.label)) + const bindings = node.data.factor_bindings ?? {} + const [path, setPath] = useState(initialFieldPath ?? Object.keys(bindings)[0] ?? 'config') + const existingFor = (fieldPath: string) => factors.find((factor) => factor.name === bindings[fieldPath]) + function seed(fieldPath: string): DesignFactor { + const existing = existingFor(fieldPath) + if (existing) return { ...existing, level_labels: existing.level_labels ?? defaultFactorLevelLabels(existing.name, existing.levels.length, existing.level_type) } + const field = fields.find((field) => field.fieldPath === fieldPath)! + const value = fieldPath === 'config' ? config : config[fieldPath.slice(7)] + const levels = field.levelType === 'model_config' ? [value, { ...config, model: '' }] : seedLevels(value) + const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) + return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } + } + const [draft, setDraft] = useState(() => seed(path)) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const initialLevelRef = useRef(null) + useEffect(() => { + if (initialLevelIndex == null) return + initialLevelRef.current?.scrollIntoView?.({ block: 'nearest' }) + }, [initialLevelIndex]) + const current = existingFor(path) + const blocked = (fieldPath: string) => fieldPath === 'config' + ? Object.keys(bindings).some((key) => key.startsWith('config.')) + : !!bindings.config + const labels = draft.level_labels ?? [] + const numeric = draft.level_type === 'number' + const structured = draft.level_type === 'model_config' + const levelKeys = draft.levels.map((level) => numeric && !String(level).trim() ? undefined : factorValueKey(numeric ? Number(level) : level)) + const duplicateLevels = draft.levels.flatMap((level, index) => { + const configured = structured ? !!(level as Record).model : !!String(level).trim() + return configured && levelKeys.some((key, other) => other !== index && key === levelKeys[index]) ? [index + 1] : [] + }) + const valid = draft.name.trim() && draft.levels.length >= 2 && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length && draft.levels.every((level) => structured ? !!(level as Record).model : String(level).trim() && (!numeric || Number.isFinite(Number(level)))) && duplicateLevels.length === 0 + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path) + else { + await onSave({ ...draft, name: draft.name.trim(), levels: numeric ? draft.levels.map(Number) : draft.levels, level_labels: labels.map((label) => label.trim()) }, path, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the model factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + initialLevelRef.current?.querySelector('input') ?? true}> + Model factorChoose how models vary across experimental cells. +
+
{visibleFields.map((field) => )}
+ {blocked(path) ?

Model levels cannot be combined with individual configuration factors. Select the existing factor type and remove its binding first.

: <> +

{structured ? 'Compare complete model configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this configuration field while keeping the other model settings.'}

+ {!available(path) &&

The selected model does not support this field. Remove its factor or choose a model that supports it.

} +
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

Level {index + 1}{index === 0 ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {structured ? } onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> : path === 'config.model' ? <> + + i !== index).map(String)} onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> + : path === 'config.effort' ? <> + + + : setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? event.target.value : value) })} />} +
+ + + +
)}
+ {duplicateLevels.length > 0 &&

Duplicate levels: {duplicateLevels.join(', ')}. {structured ? 'Each level must use a different model configuration.' : 'Each level must have a different value.'} Change or remove a duplicate before saving.

} + +

Changes require design review and regeneration.

+ {current && } + } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ModelField.tsx b/frontend/src/components/protocol/ModelField.tsx index 715f81b2..d19fcb10 100644 --- a/frontend/src/components/protocol/ModelField.tsx +++ b/frontend/src/components/protocol/ModelField.tsx @@ -33,12 +33,14 @@ export function ModelField({ value, models, isLoading, + disabledModelIds = [], onChange, }: { id?: string value: string models: LLMModelInfo[] isLoading: boolean + disabledModelIds?: string[] onChange: (model: string) => void }) { const known = models.find((m) => m.id === value) @@ -84,7 +86,7 @@ export function ModelField({ patchConfig({ temperature: e.target.value === '' ? null : Number(e.target.value) })} - /> - - )} - +
+ + patchConfig({ temperature: e.target.value === '' ? null : Number(e.target.value) })} + /> +
)} {showEffort && ( - ({ value: level, label: level }))} - boundFactorName={bindings['config.effort']} - onBind={(name) => bindFactor('config.effort', name)} - onUnbind={() => unbindFactor('config.effort')} - > - {(trigger) => ( -
- - -
- )} - +
+ + +
)} - bindFactor('config.max_tokens', name)} - onUnbind={() => unbindFactor('config.max_tokens')} - > - {(trigger) => ( -
- - { - if (e.target.value === '') { - patchConfig({ max_tokens: null }) - return - } - patchConfig({ max_tokens: Math.min(200000, Math.max(1, Math.trunc(Number(e.target.value)))) }) - }} - /> -
- )} - +
+ + { + if (e.target.value === '') { + patchConfig({ max_tokens: null }) + return + } + patchConfig({ max_tokens: Math.min(200000, Math.max(1, Math.trunc(Number(e.target.value)))) }) + }} + /> +
{ + it('groups whole-pattern and parameter choices under the Pattern node in New Factor', () => { + const fields = factorCreationFields(graph.nodes, graph.edges) + const whole = fields.find((field) => field.fieldPath === 'pattern_override')! + expect(whole).toMatchObject({ nodeId: 'agent', fieldLabel: 'Pattern levels', connectorFactor: { kind: 'pattern', nodeId: 'pattern', agentId: 'agent' }, pickerGroup: { id: 'agent', category: 'Pattern', componentId: 'pattern', componentLabel: 'Reason + Act' } }) + expect(fields.filter((field) => field.nodeId === 'pattern')).toHaveLength(4) + expect(fields.filter((field) => field.nodeId === 'pattern').every((field) => field.connectorFactor?.kind === 'pattern')).toBe(true) + }) + + it.each(['pattern_reason_act', 'pattern_single_agent_baseline'] as const)('seeds %s and its alternate and saves on the Agent', async (type) => { + const save = vi.fn().mockResolvedValue(undefined) + const config = type === 'pattern_reason_act' ? defaultReasonActPatternNodeData() : defaultSingleAgentBaselinePatternNodeData() + const nextGraph = { ...graph, nodes: [{ ...pattern, type, data: config }, agent] } + render() + expect(screen.getByRole('dialog', { name: 'Pattern factor' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ level_type: 'pattern', levels: [expect.objectContaining({ pattern_params: expect.objectContaining({ [type.slice(8)]: config.config }) }), expect.any(Object)] }), 'pattern_override', 'agent', undefined)) + }) + + it('blocks whole-pattern factors while allowing existing parameter factors to be edited and removed', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const nextGraph = { ...graph, nodes: [{ ...pattern, data: { ...pattern.data, factor_bindings: { 'config.max_iterations': 'Budget' } } }, agent] } + render() + fireEvent.click(screen.getByRole('button', { name: 'Pattern levels' })) + expect(screen.getByRole('status')).toHaveTextContent('cannot be combined') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'Max iterations' })) + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Budget', 'config.max_iterations', 'pattern')) + expect(factorCreationFields(nextGraph.nodes, nextGraph.edges).some((field) => field.fieldPath === 'pattern_override')).toBe(false) + }) + + it('blocks parameters controlled by an Agent override, including shared Pattern connections', () => { + const otherAgent = { ...agent, id: 'other', data: { ...defaultAgentNodeData('Other'), factor_bindings: { pattern_override: 'Patterns' } } } + const nextGraph = { ...graph, nodes: [...graph.nodes, otherAgent], edges: [...graph.edges, { id: 'other-edge', source: 'pattern', target: 'other', targetHandle: 'architectural_pattern' }] } + expect(patternFactorConflict('pattern', 'config.max_iterations', nextGraph.nodes, nextGraph.edges)).toContain('cannot be combined') + render() + fireEvent.click(screen.getByRole('button', { name: 'Max iterations' })) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + }) + + it('requires an Agent for Pattern levels while permitting orphan parameter factors', () => { + render() + expect(screen.getByRole('status')).toHaveTextContent('Connect this Pattern to an Agent') + fireEvent.click(screen.getByRole('button', { name: 'Include scratchpad' })) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + }) + + it('preserves numeric labels when reordered and rejects duplicate and invalid budgets', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '30' } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels') + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '0' } }) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '10' } }) + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'Long' } }) + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'Short' } }) + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ levels: [10, 30], level_labels: ['Short', 'Long'] }), 'config.max_iterations', 'pattern', undefined)) + }) +}) diff --git a/frontend/src/components/protocol/PatternFactorDialog.tsx b/frontend/src/components/protocol/PatternFactorDialog.tsx new file mode 100644 index 00000000..05d1c088 --- /dev/null +++ b/frontend/src/components/protocol/PatternFactorDialog.tsx @@ -0,0 +1,107 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { factorValueKey } from '@/lib/experiment' +import { PatternLevelRow } from './FactorEditorDialog' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { PATTERN_FACTOR_PATH, patternFactorFields, patternFactorConflict } from './patternFactors' + +export function PatternFactorDialog({ patternNodeId, graph, nodeLabel, factors, initialFieldPath, initialAgentId, onClose, onSave, onRemove }: { + patternNodeId: string + graph: ProtocolGraph + nodeLabel: string + factors: DesignFactor[] + initialFieldPath?: string + initialAgentId?: string + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, ownerId: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string, ownerId: string) => Promise +}) { + const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === patternNodeId && edge.target === node.id && edge.targetHandle === 'architectural_pattern')) + const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((node) => node.data.factor_bindings?.[PATTERN_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const fields = patternFactorFields(patternNodeId, graph.nodes, graph.edges, agentId) + .sort((a, b) => Number(b.fieldPath === PATTERN_FACTOR_PATH) - Number(a.fieldPath === PATTERN_FACTOR_PATH) || a.label.localeCompare(b.label)) + const fieldFor = (path: string) => fields.find((field) => field.fieldPath === path)! + const existingFor = (path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === fieldFor(path).nodeId)?.data.factor_bindings?.[path]) + const [path, setPath] = useState(initialFieldPath ?? fields.find((field) => existingFor(field.fieldPath))?.fieldPath ?? PATTERN_FACTOR_PATH) + function seed(path: string, ownerId = agentId): DesignFactor { + const field = patternFactorFields(patternNodeId, graph.nodes, graph.edges, ownerId).find((field) => field.fieldPath === path)! + const existing = factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === field.nodeId)?.data.factor_bindings?.[path]) + if (existing) return { ...existing, level_labels: existing.level_labels ?? defaultFactorLevelLabels(existing.name, existing.levels.length, existing.level_type) } + const levels = field.levelType === 'pattern' ? [field.currentValue, field.alternateValue] : field.levelType === 'boolean' ? [false, true] : seedLevels(field.currentValue) + const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) + return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } + } + const [draft, setDraft] = useState(() => seed(path)) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const current = existingFor(path) + const blocked = (path: string) => patternFactorConflict(patternNodeId, path, graph.nodes, graph.edges, agentId) + const labels = draft.level_labels ?? [] + const numeric = draft.level_type === 'number' + const structured = draft.level_type === 'pattern' + const boolean = draft.level_type === 'boolean' + const keys = draft.levels.map((level) => numeric && !String(level).trim() ? undefined : factorValueKey(numeric ? Number(level) : level)) + const duplicates = keys.flatMap((key, index) => key !== undefined && keys.some((other, i) => i !== index && key === other) ? [index + 1] : []) + const validLevel = (level: unknown) => { + if (boolean) return typeof level === 'boolean' + if (!structured) return !!String(level).trim() && (!numeric || Number.isInteger(Number(level)) && Number(level) >= 1) && (path !== 'config.observation_format' || ['raw', 'summarized'].includes(String(level))) + const value = level as Record + const slug = String(value.execution_pattern) + const params = (value.pattern_params as Record>)?.[slug] + return ['reason_act', 'single_agent_baseline'].includes(slug) && params && Number.isInteger(params.max_iterations) && Number(params.max_iterations) >= 1 && (slug !== 'reason_act' || !params.include_scratchpad || Number.isInteger(params.scratchpad_window) && Number(params.scratchpad_window) >= 1) + } + const valid = draft.name.trim() && draft.levels.length >= 2 && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length && draft.levels.every(validLevel) && !duplicates.length + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && (!valid || blocked(path))) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path, fieldFor(path).nodeId) + else { + await onSave({ ...draft, name: draft.name.trim(), levels: numeric ? draft.levels.map(Number) : draft.levels, level_labels: labels.map((label) => label.trim()) }, path, fieldFor(path).nodeId, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the pattern factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + + Pattern factorChoose how patterns vary across experimental cells. +
+ {agents.length > 1 &&
} +
{fields.map((field) => )}
+ {blocked(path) ?

{blocked(path)}

: <> +

{structured ? 'Compare complete pattern configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this parameter while keeping the other pattern settings.'}

+
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

Level {index + 1}{index === 0 ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {structured ? } onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> : boolean ?

{level ? 'Enabled' : 'Disabled'}

: path === 'config.observation_format' ? : setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? event.target.value : value) })} />} +
+ {!boolean && <>} +
)}
+ {duplicates.length > 0 &&

Duplicate levels: {duplicates.join(', ')}. Change or remove a duplicate before saving.

} + {!boolean && } +

Changes require design review and regeneration.

+ } + {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index a8580567..a04754f6 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -177,6 +177,49 @@ describe('ProtocolCanvas connector adds', () => { }) }) + it('creates and removes a whole-pattern factor from the Pattern toolbar, clearing the Agent override', async () => { + let experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + const save = vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + experiment = { ...experiment, ...patch } + return experiment + }) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + { id: 'pattern', type: 'pattern_reason_act', position: { x: 100, y: 0 }, data: defaultReasonActPatternNodeData() }, + ], + edges: [{ id: 'pattern-edge', source: 'pattern', target: 'agent', targetHandle: 'architectural_pattern' }], + } + const { client } = renderCanvas(graph, 'experiment-1') + const toolbar = () => document.querySelector('[data-testid="rf__node-pattern"] button[aria-label="Make experimental factor"]')! + fireEvent.click(toolbar()) + await screen.findByRole('dialog', { name: 'Pattern factor' }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Writer:Reason + Act:Pattern levels', level_type: 'pattern' })] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ pattern_override: 'Writer:Reason + Act:Pattern levels' }) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toEqual({ execution_pattern: 'reason_act', pattern_params: { reason_act: graph.nodes[1].data.config } }) + expect(persisted.nodes.find((node) => node.id === 'pattern')?.data.factor_bindings).toBeUndefined() + expect(persisted.edges).toHaveLength(1) + }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + fireEvent.click(toolbar()) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toBeUndefined() + expect(experiment.design_spec?.factors).toEqual([]) + }) + }) + it.each(['dataset_selection', 'dataset_toggle'] as const)('creates a %s dataset factor from the node hover menu and preserves connected nodes', async (mode) => { const experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index a720c9ab..aa987a12 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -1,3 +1,6 @@ +import { ModelFactorDialog } from './ModelFactorDialog' +import { PatternFactorDialog } from './PatternFactorDialog' +import { PATTERN_FACTOR_PATH, isPatternNode, patternFactorConflict } from './patternFactors' import { reconcileKnowledgeFactor, knowledgeFactorConflict, knowledgeFactorOwner, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' import { KnowledgeFactorDialog } from './KnowledgeFactorDialog' import { KnowledgeTestSelectors } from './KnowledgeTestSelectors' @@ -437,6 +440,7 @@ export const ProtocolCanvas = forwardRef(null) const [editingFactorName, setEditingFactorName] = useState(null) const [knowledgeFactorNodeId, setKnowledgeFactorNodeId] = useState(null) + const [modelFactorNodeId, setModelFactorNodeId] = useState(null) + const [patternFactorNodeId, setPatternFactorNodeId] = useState(null) + const [patternFactorFieldPath, setPatternFactorFieldPath] = useState() + const [patternFactorAgentId, setPatternFactorAgentId] = useState() + const [modelFactorFieldPath, setModelFactorFieldPath] = useState() + const [modelFactorLevelIndex, setModelFactorLevelIndex] = useState() const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) const [datasetFactorNodeId, setDatasetFactorNodeId] = useState(null) const [connectorFactorChoice, setConnectorFactorChoice] = useState() @@ -1159,6 +1171,10 @@ export const ProtocolCanvas = forwardRef edge.source === n.id && edge.targetHandle === 'architectural_pattern').flatMap((edge) => { + const name = (nodes.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[PATTERN_FACTOR_PATH] + return name ? [name] : [] + })).size : 0, knowledgeFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH])?.levels.length, knowledgeFactorControlled: ['okf_bundle', 'okf_document'].includes(n.type ?? '') && !!knowledgeFactorOwner(toPersistedGraph(nodes, edges), n.id), skillFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH])?.levels.length, @@ -1369,6 +1385,24 @@ export const ProtocolCanvas = forwardRef { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setModelFactorFieldPath(levelIndex == null ? undefined : 'config') + setModelFactorLevelIndex(levelIndex) + setModelFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestPatternFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setPatternFactorFieldPath(undefined) + setPatternFactorAgentId(undefined) + setPatternFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const modelFactorForNode = useCallback((nodeId: string) => { + const name = (nodes.find((node) => node.id === nodeId)?.data.factor_bindings as Record | undefined)?.config + return factors.find((factor) => factor.name === name && factor.level_type === 'model_config') + }, [nodes, factors]) const requestSkillFactor = useCallback((nodeId: string) => { if (!experimentId || experimentLocked) return setSelectedNodeId(null) @@ -1470,6 +1504,9 @@ export const ProtocolCanvas = forwardRef { if (experimentLocked) return setAddPanelOpen(false) + if (modelFactorForNode(node.id)) { + setModelFactorFieldPath('config') + setModelFactorLevelIndex(undefined) + setModelFactorNodeId(node.id) + setSelectedNodeId(null) + return + } setSelectedNodeId(node.id) }} onPaneClick={() => setSelectedNodeId(null)} @@ -2364,8 +2411,6 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2439,8 +2484,6 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> @@ -2552,7 +2593,9 @@ export const ProtocolCanvas = forwardRef )} + {patternFactorNodeId && nodes.some((node) => node.id === patternFactorNodeId) && node.id === patternFactorNodeId)!, edges, nodes)} + initialFieldPath={patternFactorFieldPath} + initialAgentId={patternFactorAgentId} + factors={factors} + onClose={() => setPatternFactorNodeId(null)} + onSave={async (factor, fieldPath, ownerId, previousName) => { + if (experimentLocked) throw new Error('Unlock the experiment before editing factors.') + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + const conflict = patternFactorConflict(patternFactorNodeId, fieldPath, nodes, edges, fieldPath === PATTERN_FACTOR_PATH ? ownerId : undefined) + if (conflict) throw new Error(conflict) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + bindFactorOnNode(ownerId, fieldPath, factor.name) + setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath, ownerId) => { + if (experimentLocked) throw new Error('Unlock the experiment before editing factors.') + const fresh = await experimentsApi.get(experimentId!) + const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== ownerId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => { + if (node.id !== ownerId) return node + const data: Record = { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } + if (fieldPath === PATTERN_FACTOR_PATH) delete data.pattern_override + return { ...node, data } + })) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {modelFactorNodeId && nodes.some((node) => node.id === modelFactorNodeId) && node.id === modelFactorNodeId)!} + nodeLabel={agentTracedLabel(nodes.find((node) => node.id === modelFactorNodeId)!, edges, nodes)} + initialFieldPath={modelFactorFieldPath} + initialLevelIndex={modelFactorLevelIndex} + factors={factors} + onClose={() => setModelFactorNodeId(null)} + onSave={async (factor, fieldPath, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + const bindings = nodes.find((node) => node.id === modelFactorNodeId)?.data.factor_bindings as Record | undefined + if (fieldPath === 'config' ? Object.keys(bindings ?? {}).some((key) => key.startsWith('config.')) : !!bindings?.config) throw new Error('Remove the conflicting model factor binding first.') + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + bindFactorOnNode(modelFactorNodeId, fieldPath, factor.name) + // Apply the first level to every binding, including the newly added one. + setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath) => { + const fresh = await experimentsApi.get(experimentId!) + const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== modelFactorNodeId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => node.id === modelFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} {skillFactorNodeId && void requestKnowledgeFactor?: (nodeId: string) => void + requestModelFactor?: (nodeId: string, levelIndex?: number) => void + requestPatternFactor?: (nodeId: string) => void + modelFactorForNode?: (nodeId: string) => DesignFactor | undefined requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). diff --git a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx index d43b9666..a5406b56 100644 --- a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx +++ b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx @@ -1,4 +1,4 @@ -import { Repeat2 } from 'lucide-react' +import { Repeat2, Split, RefreshCw } from 'lucide-react' import { useState } from 'react' import { nodeAccent } from '@/lib/nodeAccent' import { isUnderIterated } from '@/lib/reasonActIterations' @@ -8,7 +8,6 @@ import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { Switch } from '@/components/ui/switch' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { ReasonActPatternConfig, ReasonActPatternNodeData, ProtocolNode } from '@/types/protocols' @@ -29,19 +28,12 @@ const ACCENT = nodeAccent('pattern_reason_act') // (the node's own canvas hover toolbar), never a bare delete. export function ReasonActPatternNodeInspector({ node, - experimentId, - factorNodeLabel, suggestedIterations, truncatedAt, onChange, onClose, }: { node: (ProtocolNode & { data: ReasonActPatternNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, which is this node's own plain label shown - // in the header title. - factorNodeLabel: string // What the driven agent's wiring implies (lib/reasonActIterations.ts), or // null when this pattern drives no agent yet. suggestedIterations: number | null @@ -60,7 +52,6 @@ export function ReasonActPatternNodeInspector({ if (!node) return null const data = node.data const config = data.config - const bindings = data.factor_bindings ?? {} // Only ever offered as a raise. Going below what the wiring needs truncates // the run into a payload of nulls that still reports as completed, while @@ -84,16 +75,6 @@ export function ReasonActPatternNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - - function unbindFactor(fieldPath: string) { - const next = { ...bindings } - delete next[fieldPath] - onChange(node!.id, { ...data, factor_bindings: next }) - } - return ( +

Make factor has moved to the node toolbar. Hover over the Pattern node and click the

- bindFactor('config.max_iterations', name)} - onUnbind={() => unbindFactor('config.max_iterations')} - > - {(trigger) => ( -
- - patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} - /> - {underIterated && ( -

- {truncatedAt != null - ? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — ` - : `This agent's wiring suggests at least ${suggestedIterations} — `} - each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '} - -

- )} -
+
+ + patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} + /> + {underIterated && ( +

+ {truncatedAt != null + ? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — ` + : `This agent's wiring suggests at least ${suggestedIterations} — `} + each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '} + +

)} - - ({ value: f, label: f }))} - boundFactorName={bindings['config.observation_format']} - onBind={(name) => bindFactor('config.observation_format', name)} - onUnbind={() => unbindFactor('config.observation_format')} - > - {(trigger) => ( -
- - -
- )} - +
+
+ + +
- bindFactor('config.include_scratchpad', name)} - onUnbind={() => unbindFactor('config.include_scratchpad')} - > - {(trigger) => ( -
-
- -

Carries a running record of prior reasoning/observations into each iteration.

-
- patchConfig({ include_scratchpad: checked })} - /> -
- )} - +
+
+ +

Carries a running record of prior reasoning/observations into each iteration.

+
+ patchConfig({ include_scratchpad: checked })} + /> +
{config.include_scratchpad && ( - bindFactor('config.scratchpad_window', name)} - onUnbind={() => unbindFactor('config.scratchpad_window')} - > - {(trigger) => ( -
- - patchConfig({ scratchpad_window: e.target.value === '' ? null : Number(e.target.value) })} - /> -
- )} - +
+ + patchConfig({ scratchpad_window: e.target.value === '' ? null : Number(e.target.value) })} + /> +
)} !open && setPendingCloseWarning(false)}> diff --git a/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx b/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx index 876d8a9a..65cb12cf 100644 --- a/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx +++ b/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx @@ -1,9 +1,8 @@ -import { ArrowRight } from 'lucide-react' +import { ArrowRight, Split, RefreshCw } from 'lucide-react' import { nodeAccent } from '@/lib/nodeAccent' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { Switch } from '@/components/ui/switch' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { SingleAgentBaselinePatternConfig, SingleAgentBaselinePatternNodeData, ProtocolNode } from '@/types/protocols' @@ -20,17 +19,10 @@ const ACCENT = nodeAccent('pattern_single_agent_baseline') // (the node's own canvas hover toolbar), never a bare delete. export function SingleAgentBaselinePatternNodeInspector({ node, - experimentId, - factorNodeLabel, onChange, onClose, }: { node: (ProtocolNode & { data: SingleAgentBaselinePatternNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, which is this node's own plain label shown - // in the header title. - factorNodeLabel: string onChange: (nodeId: string, data: SingleAgentBaselinePatternNodeData) => void onClose: () => void }) { @@ -38,22 +30,11 @@ export function SingleAgentBaselinePatternNodeInspector({ if (!node) return null const data = node.data const config = data.config - const bindings = data.factor_bindings ?? {} function patchConfig(patch: Partial) { onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - - function unbindFactor(fieldPath: string) { - const next = { ...bindings } - delete next[fieldPath] - onChange(node!.id, { ...data, factor_bindings: next }) - } - return ( - bindFactor('config.max_iterations', name)} - onUnbind={() => unbindFactor('config.max_iterations')} - > - {(trigger) => ( -
- - patchConfig({ max_iterations: Number(e.target.value) })} - /> -
- )} - +

Make factor has moved to the node toolbar. Hover over the Pattern node and click the

+
+ + patchConfig({ max_iterations: Number(e.target.value) })} + /> +
- bindFactor('config.stop_on_first_success', name)} - onUnbind={() => unbindFactor('config.stop_on_first_success')} - > - {(trigger) => ( -
-
- -

Off: keeps looping for the full iteration budget even after a successful pass.

-
- patchConfig({ stop_on_first_success: checked })} - /> -
- )} - +
+
+ +

Off: keeps looping for the full iteration budget even after a successful pass.

+
+ patchConfig({ stop_on_first_success: checked })} + /> +
) } diff --git a/frontend/src/components/protocol/SkillFactorDialog.test.tsx b/frontend/src/components/protocol/SkillFactorDialog.test.tsx index 0d9009bc..87a5b560 100644 --- a/frontend/src/components/protocol/SkillFactorDialog.test.tsx +++ b/frontend/src/components/protocol/SkillFactorDialog.test.tsx @@ -11,6 +11,22 @@ const graph = { edges: [{ source: 'skill', target: 'agent', targetHandle: 'skill' }], } as unknown as ProtocolGraph +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Skill enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Skill levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + it.each(['Skill levels', 'All skills on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { const save = vi.fn().mockResolvedValue(undefined) render() diff --git a/frontend/src/components/protocol/SkillFactorDialog.tsx b/frontend/src/components/protocol/SkillFactorDialog.tsx index dcbc614c..252e2143 100644 --- a/frontend/src/components/protocol/SkillFactorDialog.tsx +++ b/frontend/src/components/protocol/SkillFactorDialog.tsx @@ -77,8 +77,10 @@ export function SkillFactorDialog({ skillNodeId, graph, factors, initialMode, in setError('') try { if (remove && current) await onRemove(current.name) - else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) - onClose() + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the skill factor.') } finally { setSaving(false) } diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 3210116e..b399bf7a 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -1,6 +1,7 @@ import type { Edge, Node } from '@xyflow/react' import type { LevelType } from './factorLevels' import type { McpServer } from '@/types/mcpServers' +import { isPatternNode, patternFactorFields, PATTERN_FACTOR_PATH, patternFactorConflict } from './patternFactors' export interface BindableFieldSpec { fieldPath: string @@ -191,7 +192,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { // a connected Model node's whole config verbatim (never the node's // xyflow `type`), so replacing it wholesale per cell already works // with zero backend changes. - { fieldPath: 'config', label: 'Provider & model', levelType: 'model_config' }, + { fieldPath: 'config', label: 'Model levels', levelType: 'model_config' }, ] case 'mcp_tool': case 'mcp_scikit_learn': @@ -286,7 +287,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { export interface UnboundField { pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } - connectorFactor?: { kind: 'skill' | 'dataset' | 'knowledge'; nodeId: string; mode: LevelType; agentId?: string } + connectorFactor?: { kind: 'skill' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -467,7 +468,20 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const fields = unboundBindableFields(nodes, edges).filter((field) => { const kind = nodes.find((node) => node.id === field.nodeId)?.type return kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' + && !isPatternNode(kind) && field.fieldPath !== PATTERN_FACTOR_PATH }) + for (const node of nodes.filter((node) => isPatternNode(node.type))) { + const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && edge.targetHandle === 'architectural_pattern')) + for (const owner of owners.length ? owners : [undefined]) { + for (const field of patternFactorFields(node.id, nodes, edges, owner?.id)) { + if ((nodes.find((node) => node.id === field.nodeId)?.data.factor_bindings as Record | undefined)?.[field.fieldPath]) continue + if (patternFactorConflict(node.id, field.fieldPath, nodes, edges, owner?.id)) continue + if (field.fieldPath !== PATTERN_FACTOR_PATH && fields.some((candidate) => candidate.nodeId === node.id && candidate.fieldPath === field.fieldPath)) continue + fields.push({ nodeId: field.nodeId, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: field.fieldPath, fieldLabel: field.label, levelType: field.levelType, currentValue: field.currentValue, + connectorFactor: { kind: 'pattern', nodeId: node.id, mode: field.levelType, agentId: owner?.id } }) + } + } + } const connectorOwners = new Set() for (const node of nodes) { if (!['skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue @@ -494,6 +508,11 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField } } } + for (const field of fields) { + if (MODEL_NODE_TYPES.has(nodes.find((node) => node.id === field.nodeId)?.type ?? '')) { + field.connectorFactor = { kind: 'model', nodeId: field.nodeId, mode: field.levelType } + } + } return fields.map((field) => { const node = nodes.find((candidate) => candidate.id === field.nodeId)! const kind = field.connectorFactor?.kind ?? node.type ?? 'node' @@ -503,19 +522,21 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const category = field.connectorFactor?.kind === 'skill' ? 'Skills' : field.connectorFactor?.kind === 'dataset' ? 'Datasets' : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' + : field.connectorFactor?.kind === 'model' ? 'Model' + : field.connectorFactor?.kind === 'pattern' ? 'Pattern' : agentKinds.includes(kind) ? 'Agent fields' : MODEL_NODE_TYPES.has(kind) ? 'Model' : kind.startsWith('pattern_') ? 'Pattern' : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', mcp_scikit_learn: 'Tools', mcp_client_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? String(node.data.label || kind) - const connectorChoice = field.connectorFactor && field.connectorFactor.mode !== 'boolean' + const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' return { ...field, pickerGroup: { id: owner?.id ?? 'shared-unconnected', label: owner ? String(owner.data.label || 'Agent') : 'Shared or unconnected components', category, - componentId: connectorChoice ? `${field.nodeId}:${kind}` : node.id, - componentLabel: connectorChoice ? `${category} on this agent` : String(node.data.label || kind), + componentId: field.connectorFactor?.kind === 'pattern' ? field.connectorFactor.nodeId : connectorChoice ? `${field.nodeId}:${kind}` : node.id, + componentLabel: field.connectorFactor?.kind === 'pattern' ? String(nodes.find((node) => node.id === field.connectorFactor?.nodeId)?.data.label || 'Pattern') : connectorChoice ? `${category} on this agent` : String(node.data.label || kind), }, } }) diff --git a/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx b/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx new file mode 100644 index 00000000..31a7416b --- /dev/null +++ b/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx @@ -0,0 +1,50 @@ +import { fireEvent, render, screen } from '@testing-library/react' +import { beforeEach, expect, it, vi } from 'vitest' +import type { DesignFactor } from '@/types/experiments' +import { ModelFactorGroup } from './ModelFactorGroup' + +const { edit, updateInternals } = vi.hoisted(() => ({ edit: vi.fn(), updateInternals: vi.fn() })) +vi.mock('@xyflow/react', () => ({ + Handle: ({ id }: { id: string }) =>
{id}
, + Position: { Top: 'top' }, + useUpdateNodeInternals: () => updateInternals, +})) +vi.mock('../ProtocolCanvasContext', () => ({ useProtocolCanvasActions: () => ({ requestModelFactor: edit }) })) +vi.mock('./NodeHoverToolbar', () => ({ NodeHoverToolbar: () => null })) +vi.mock('../useProviderModels', () => ({ useProviderModels: () => ({ models: [], modelsQuery: {} }) })) + +const factor: DesignFactor = { + name: 'Writer:Models', level_type: 'model_config', level_labels: ['Baseline', 'Alternative'], + levels: [ + { provider: 'openai', model: 'gpt-5', temperature: 0.7, max_tokens: 1000 }, + { provider: 'anthropic', model: 'claude-sonnet', temperature: 1, max_tokens: 2000 }, + ], +} + +beforeEach(() => { localStorage.clear(); vi.clearAllMocks() }) + +it('expands derived configurations with one handle and opens the selected level', () => { + render() + expect(screen.getByText('Model factor · 2 levels')).toBeInTheDocument() + expect(screen.getByText('OpenAI')).toBeInTheDocument() + expect(screen.getByText('Anthropic')).toBeInTheDocument() + expect(screen.queryByText('model=gpt-5')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Expand model levels' })) + expect(screen.getByText('model=gpt-5')).toBeInTheDocument() + expect(screen.getByText('model=claude-sonnet')).toBeInTheDocument() + expect(screen.getByText(/default test selection/)).toBeInTheDocument() + expect(screen.getAllByTestId('handle')).toHaveLength(1) + fireEvent.click(screen.getByRole('button', { name: 'Edit model level 2: Alternative' })) + expect(edit).toHaveBeenCalledWith('model', 1) + expect(updateInternals).toHaveBeenCalledWith('model') +}) + +it('remembers expansion per node and keeps same-provider configurations separate', () => { + const { unmount } = render() + fireEvent.click(screen.getByRole('button', { name: 'Expand model levels' })) + unmount() + render() + expect(screen.getByRole('button', { name: 'Collapse model levels' })).toHaveAttribute('aria-expanded', 'true') + expect(screen.getAllByText('model=gpt-5')).toHaveLength(2) + expect(screen.getAllByTestId('handle')).toHaveLength(1) +}) diff --git a/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx new file mode 100644 index 00000000..e1600cf5 --- /dev/null +++ b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx @@ -0,0 +1,92 @@ +import { useEffect, useState } from 'react' +import { Handle, Position, useUpdateNodeInternals } from '@xyflow/react' +import { ChevronDown, ChevronUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Card } from '@/components/ui/card' +import { nodeAccent } from '@/lib/nodeAccent' +import { cardAccent } from '@/lib/utils' +import type { DesignFactor } from '@/types/experiments' +import { factorLevelLabels } from '../factorLevels' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' +import { useProviderModels } from '../useProviderModels' +import { PROVIDER_META } from './ModelNode' +import { NodeHoverToolbar } from './NodeHoverToolbar' +import { WarningBadge } from './WarningBadge' + +const expansionKey = (id: string) => `asaree:model-factor-expanded:${id}` + +function LevelPreview({ level, label, index, disabled, onEdit }: { + level: unknown + label: string + index: number + disabled: boolean + onEdit: () => void +}) { + const config = (level ?? {}) as Record + const provider = String(config.provider ?? '') + const meta = PROVIDER_META[provider] + const Icon = meta?.icon ?? Split + const { models } = useProviderModels(provider) + const model = models.find((model) => model.id === config.model) + const effort = model?.supports_effort ?? false + const temperature = model?.supports_temperature ?? true + return + + +} + +/** Derived previews only: this group remains one graph node and one source handle. */ +export function ModelFactorGroup({ id, factor, selected, warning }: { + id: string + factor: DesignFactor + selected?: boolean + warning?: string[] +}) { + const { requestModelFactor, experimentLocked } = useProtocolCanvasActions() + const [expanded, setExpanded] = useState(() => { + try { return localStorage.getItem(expansionKey(id)) === 'true' } catch { return false } + }) + const updateNodeInternals = useUpdateNodeInternals() + useEffect(() => { updateNodeInternals(id) }, [id, expanded, factor.levels.length, updateNodeInternals]) + const providers = [...new Set(factor.levels.map((level) => String((level as Record)?.provider ?? '')))] + const labels = factorLevelLabels(factor) + const accent = nodeAccent('model') + return
+ requestModelFactor(id) : undefined} /> + + +
+ +

{factor.name}

Model factor · {factor.levels.length} levels

+ {!!warning?.length && } + +
+
{providers.map((provider) => { + const meta = PROVIDER_META[provider] + const Icon = meta?.icon ?? Split + return {meta?.label ?? provider} + })}
+ {expanded &&
+

One configuration per cell

+ {factor.levels.map((level, index) =>
+ requestModelFactor?.(id, index)} /> +
)} +
} + +
+} diff --git a/frontend/src/components/protocol/nodes/ModelNode.tsx b/frontend/src/components/protocol/nodes/ModelNode.tsx index 71f21501..0825c71b 100644 --- a/frontend/src/components/protocol/nodes/ModelNode.tsx +++ b/frontend/src/components/protocol/nodes/ModelNode.tsx @@ -4,7 +4,9 @@ import { nodeAccent } from '@/lib/nodeAccent' import type { ModelNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' import { useProviderModels } from '../useProviderModels' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' +import { ModelFactorGroup } from './ModelFactorGroup' // One shared card renderer for all three LLM provider node types // (model_anthropic/model_openai/model_azure_foundry -- see ModelNodeData's own @@ -31,6 +33,8 @@ export const PROVIDER_META: Record + return ( 0 ? warnings : undefined} factorCount={boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestModelFactor ? () => requestModelFactor(id) : undefined} /> ) } diff --git a/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx b/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx index 1320c74f..42fb9347 100644 --- a/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx +++ b/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx @@ -38,7 +38,7 @@ export function ReasonActPatternNode({ // toolbar instead. An unconnected/orphaned pattern node (dragged onto the // canvas but never wired) has nothing to swap, so it keeps plain Delete. const connections = useNodeConnections({ id, handleType: 'source', handleId: 'architectural_pattern' }) - const { requestConnectorAdd } = useProtocolCanvasActions() + const { requestConnectorAdd, requestPatternFactor } = useProtocolCanvasActions() const targetAgentId = connections[0]?.target // scratchpad_window only matters -- and so is only required -- while @@ -91,7 +91,8 @@ export function ReasonActPatternNode({ placeholder="Reason + Act" handleId="architectural_pattern" handlePosition="bottom" - factorCount={boundFactorCount(data)} + factorCount={boundFactorCount(data) + Number(data.patternFactorCount ?? 0)} + onMakeFactor={() => requestPatternFactor?.(id)} warning={warnings.length > 0 ? warnings : undefined} swap={ targetAgentId diff --git a/frontend/src/components/protocol/nodes/SingleAgentBaselinePatternNode.tsx b/frontend/src/components/protocol/nodes/SingleAgentBaselinePatternNode.tsx index 9dabb163..cf7f693c 100644 --- a/frontend/src/components/protocol/nodes/SingleAgentBaselinePatternNode.tsx +++ b/frontend/src/components/protocol/nodes/SingleAgentBaselinePatternNode.tsx @@ -22,7 +22,7 @@ export function SingleAgentBaselinePatternNode({ // delete treatment, since an agent's execution pattern must never go to // zero. const connections = useNodeConnections({ id, handleType: 'source', handleId: 'architectural_pattern' }) - const { requestConnectorAdd } = useProtocolCanvasActions() + const { requestConnectorAdd, requestPatternFactor } = useProtocolCanvasActions() const targetAgentId = connections[0]?.target return ( @@ -35,7 +35,8 @@ export function SingleAgentBaselinePatternNode({ placeholder="Single-Agent Baseline" handleId="architectural_pattern" handlePosition="bottom" - factorCount={boundFactorCount(data)} + factorCount={boundFactorCount(data) + Number(data.patternFactorCount ?? 0)} + onMakeFactor={() => requestPatternFactor?.(id)} swap={ targetAgentId ? { label: 'Swap pattern', onSwap: () => requestConnectorAdd({ nodeId: targetAgentId, slot: 'architectural_pattern' }) } diff --git a/frontend/src/components/protocol/patternFactors.ts b/frontend/src/components/protocol/patternFactors.ts new file mode 100644 index 00000000..7abec1be --- /dev/null +++ b/frontend/src/components/protocol/patternFactors.ts @@ -0,0 +1,31 @@ +import type { Node, Edge } from '@xyflow/react' +import { defaultReasonActPatternNodeData, defaultSingleAgentBaselinePatternNodeData } from '@/types/protocols' +import { bindableFieldsForNode } from './bindableFields' + +export const PATTERN_FACTOR_PATH = 'pattern_override' +export const isPatternNode = (type?: string) => type === 'pattern_reason_act' || type === 'pattern_single_agent_baseline' + +export function patternFactorFields(patternNodeId: string, nodes: Node[], edges: Edge[], agentId?: string) { + const node = nodes.find((node) => node.id === patternNodeId)! + const owner = nodes.find((candidate) => ['agent', 'sub_agent'].includes(candidate.type ?? '') && (!agentId || candidate.id === agentId) && edges.some((edge) => edge.source === node.id && edge.target === candidate.id && edge.targetHandle === 'architectural_pattern')) + const reasonAct = node.type === 'pattern_reason_act' + const slug = reasonAct ? 'reason_act' : 'single_agent_baseline' + const alternateSlug = reasonAct ? 'single_agent_baseline' : 'reason_act' + return [ + { nodeId: owner?.id ?? '', fieldPath: PATTERN_FACTOR_PATH, label: 'Pattern levels', levelType: 'pattern' as const, + currentValue: { execution_pattern: slug, pattern_params: { [slug]: node.data.config } }, + alternateValue: { execution_pattern: alternateSlug, pattern_params: { [alternateSlug]: reasonAct ? defaultSingleAgentBaselinePatternNodeData().config : defaultReasonActPatternNodeData().config } } }, + ...bindableFieldsForNode(node).map((field) => ({ ...field, nodeId: node.id, currentValue: (node.data.config as Record)[field.fieldPath.slice(7)], alternateValue: undefined })), + ] +} + +export function patternFactorConflict(patternNodeId: string, path: string, nodes: Node[], edges: Edge[], agentId?: string): string | undefined { + const node = nodes.find((node) => node.id === patternNodeId) + const owners = nodes.filter((candidate) => ['agent', 'sub_agent'].includes(candidate.type ?? '') && edges.some((edge) => edge.source === patternNodeId && edge.target === candidate.id && edge.targetHandle === 'architectural_pattern')) + if (path === PATTERN_FACTOR_PATH && !owners.some((owner) => !agentId || owner.id === agentId)) return 'Connect this Pattern to an Agent before creating Pattern levels.' + const bindings = node?.data.factor_bindings as Record | undefined + const conflict = path === PATTERN_FACTOR_PATH + ? Object.keys(bindings ?? {}).some((key) => key.startsWith('config.')) + : owners.some((owner) => (owner.data.factor_bindings as Record | undefined)?.[PATTERN_FACTOR_PATH]) + return conflict ? 'Pattern levels cannot be combined with individual parameter factors. Select the existing factor type and remove its binding first.' : undefined +} From dbda07fb79ec31473dabfc2471cb72db898e8d36 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 15:52:38 -0700 Subject: [PATCH 13/63] feat: add Sub-Agent connector factors --- frontend/src/api/client.ts | 2 +- .../protocol/AgentNodeInspector.tsx | 51 +++++--- .../components/protocol/DesignTab.test.tsx | 6 +- .../src/components/protocol/DesignTab.tsx | 10 +- .../protocol/FactorEditorDialog.tsx | 4 + .../protocol/ProtocolCanvas.test.tsx | 33 +++++ .../components/protocol/ProtocolCanvas.tsx | 101 +++++++++++++-- .../protocol/ProtocolCanvasContext.tsx | 1 + .../protocol/SubAgentFactorDialog.test.tsx | 35 +++++ .../protocol/SubAgentFactorDialog.tsx | 121 ++++++++++++++++++ .../protocol/SubAgentFactorEditor.tsx | 74 +++++++++++ .../protocol/SubAgentTestSelectors.tsx | 17 +++ .../src/components/protocol/bindableFields.ts | 28 +++- .../src/components/protocol/factorLevels.ts | 6 + .../components/protocol/nodes/AgentNode.tsx | 13 +- .../useSubAgentTestSelection.test.tsx | 21 +++ .../protocol/useSubAgentTestSelection.ts | 25 ++++ frontend/src/lib/factorBindings.ts | 9 +- frontend/src/lib/subAgentFactors.test.ts | 29 +++++ frontend/src/lib/subAgentFactors.ts | 93 ++++++++++++++ frontend/src/types/experiments.ts | 2 + frontend/src/types/protocols.ts | 2 + src/asaree/api/protocols.py | 7 + src/asaree/services/agent_messenger.py | 18 ++- src/asaree/services/design_generation.py | 9 +- src/asaree/services/factor_bindings.py | 5 +- src/asaree/services/protocol_execution.py | 33 ++++- src/asaree/services/protocol_revisions.py | 6 +- src/asaree/services/sub_agent_factors.py | 108 ++++++++++++++++ tests/test_agent_messenger.py | 28 ++++ tests/test_sub_agent_factors.py | 104 +++++++++++++++ 31 files changed, 951 insertions(+), 50 deletions(-) create mode 100644 frontend/src/components/protocol/SubAgentFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/SubAgentFactorDialog.tsx create mode 100644 frontend/src/components/protocol/SubAgentFactorEditor.tsx create mode 100644 frontend/src/components/protocol/SubAgentTestSelectors.tsx create mode 100644 frontend/src/components/protocol/useSubAgentTestSelection.test.tsx create mode 100644 frontend/src/components/protocol/useSubAgentTestSelection.ts create mode 100644 frontend/src/lib/subAgentFactors.test.ts create mode 100644 frontend/src/lib/subAgentFactors.ts create mode 100644 src/asaree/services/sub_agent_factors.py create mode 100644 tests/test_sub_agent_factors.py diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 97a88575..883a81e7 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -187,7 +187,7 @@ export interface ResultsScope { protocol_revision_id?: string } -export interface RowSelection { row_index?: number | null; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } +export interface RowSelection { row_index?: number | null; sub_agent_selections?: Record; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } function scopeQuery(scope?: ResultsScope): string { const params = new URLSearchParams() diff --git a/frontend/src/components/protocol/AgentNodeInspector.tsx b/frontend/src/components/protocol/AgentNodeInspector.tsx index f1d5babe..74b8d0a2 100644 --- a/frontend/src/components/protocol/AgentNodeInspector.tsx +++ b/frontend/src/components/protocol/AgentNodeInspector.tsx @@ -1,5 +1,5 @@ import { useRef } from 'react' -import { Bot } from 'lucide-react' +import { Bot, Power, Split, Trash2 } from 'lucide-react' import { useQuery } from '@tanstack/react-query' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' @@ -62,6 +62,7 @@ const GUTTER_WIDTH = 96 export function AgentNodeInspector({ node, experimentId, + subAgentFactorControlled, markedLeadAgentId, referenceScope, handoffPeers, @@ -75,6 +76,7 @@ export function AgentNodeInspector({ }: { node: (ProtocolNode & { data: AgentNodeData }) | null experimentId: string | null + subAgentFactorControlled?: boolean // Which agent on the canvas already carries the lead marker, if any -- the // inspector can't see its siblings, so ProtocolCanvas resolves it. markedLeadAgentId: string | null @@ -197,20 +199,22 @@ export function AgentNodeInspector({ placeholder={node.type === 'sub_agent' ? 'Sub-Agent' : 'Agent'} onCommit={(label) => onChange(node.id, { ...data, label })} /> - bindFactor('active', name)} - onUnbind={() => unbindFactor('active')} - > - {(trigger) => trigger} - + {!isSubAgent && <> + bindFactor('active', name)} + onUnbind={() => unbindFactor('active')} + > + {(trigger) => trigger} + + } } onDelete={() => onDelete(node.id)} @@ -268,6 +272,23 @@ export function AgentNodeInspector({ + {isSubAgent && <> +
+

+ Make factor has moved to the node toolbar. Hover over the Sub-Agent node and click the

+
+ {subAgentFactorControlled ?

Availability is controlled by the parent Agent’s Sub-Agents factor. Remove it to restore individual controls.

: bindings.active &&

Factor: {bindings.active}

} + {subAgentFactorControlled && bindings.active && } + } {/* First, above Prompt: which agent leads decides whose prompt becomes the task and whose answer gets scored, so it frames everything below it rather than being one more setting. It diff --git a/frontend/src/components/protocol/DesignTab.test.tsx b/frontend/src/components/protocol/DesignTab.test.tsx index 36301867..2cd2926d 100644 --- a/frontend/src/components/protocol/DesignTab.test.tsx +++ b/frontend/src/components/protocol/DesignTab.test.tsx @@ -43,6 +43,9 @@ const experiment: Experiment = { describe('DesignTab design generation', () => { it.each([ + ['Sub-Agent levels', 'sub_agent', 'sub_agent_selection'], + ['All agent sub-agents on/off', 'sub_agent', 'sub_agent_toggle'], + ['This sub-agent on/off', 'sub_agent', 'boolean'], ['Skill levels', 'skill', 'skill_selection'], ['All skills on/off', 'skill', 'skill_toggle'], ['This skill on/off', 'skill', 'boolean'], @@ -56,9 +59,10 @@ describe('DesignTab design generation', () => { nodes: [ { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer', config: {} } }, { id: 'skill', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Summarize', config: { skill_id: 'a' } } }, + { id: 'sub_agent', type: 'sub_agent', position: { x: 0, y: 0 }, data: { label: 'Worker', config: {} } }, { id: 'dataset', type: 'dataset', position: { x: 0, y: 0 }, data: { label: 'Cohort', config: { dataset_id: 'd' } } }, ], - edges: ['skill', 'dataset'].map(source => ({ id: source, source, target: 'agent', targetHandle: source })), + edges: ['skill', 'dataset', 'sub_agent'].map(source => ({ id: source, source, target: 'agent', targetHandle: source === 'sub_agent' ? 'sub_agents' : source })), }) vi.spyOn(experimentsApi, 'getMeasurementCapabilities').mockResolvedValue({ outputs: {} }) vi.spyOn(experimentsApi, 'validateMeasurementPlan').mockResolvedValue({ valid: true, issues: [] }) diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index ccf1146f..14cdd6da 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -1,3 +1,4 @@ +import { subAgentFactorIssues } from '@/lib/subAgentFactors' import { knowledgeFactorIssues, isKnowledgeFactor } from '@/lib/knowledgeFactors' import { useKnowledgeLibrary } from './useKnowledgeLibrary' import { datasetsApi } from '@/api/client' @@ -186,6 +187,7 @@ function AddFactorButton({ {dialogOpen && ( deleteMutation.mutate(factor.name)} > @@ -319,6 +321,7 @@ function FactorsEditor({ {editingFactor && ( { if (!open) setEditingFactor(null) @@ -507,6 +510,7 @@ export function DesignTab({ const skillLibrary = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list(), enabled: factors.some((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle')) }) const datasetLibrary = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list(), enabled: factors.some((factor) => factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') }) const datasetIssues = datasetFactorIssues(draftGraph, factors, datasetLibrary.data ? new Set(datasetLibrary.data.map((dataset) => dataset.id)) : undefined) + const subAgentIssues = subAgentFactorIssues(draftGraph, factors) const skillIssues = skillFactorIssues(draftGraph, factors, skillLibrary.data ? new Set(skillLibrary.data.map((skill) => skill.id)) : undefined) const impact = impactQuery.data @@ -876,14 +880,14 @@ export function DesignTab({ Unbound factor{unboundFactors.length === 1 ? '' : 's'}: {unboundFactors.join(', ')}. Rebind on the canvas or remove from this design.

)} - {[...skillIssues, ...datasetIssues, ...knowledgeIssues].map((issue) =>

{issue}

)} + {[...subAgentIssues, ...skillIssues, ...datasetIssues, ...knowledgeIssues].map((issue) =>

{issue}

)} )} + {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Sub-Agent node.' : mode === 'sub_agent_selection' ? 'Each cell receives exactly one connected sub-agent.' : 'Each cell enables every connected sub-agent, including deactivated sub-agents, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {mode !== 'boolean' && affected.length < (mode === 'sub_agent_toggle' ? 1 : 2) &&

Connect at least {mode === 'sub_agent_toggle' ? 'one sub-agent' : 'two different sub-agents'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'sub_agent_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable sub-agent')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+

Agent-level factors follow current Sub-Agent connections. Changes require design review and regeneration.

+ {current && } + } + {error &&

{error}

} + + + + +} diff --git a/frontend/src/components/protocol/SubAgentFactorEditor.tsx b/frontend/src/components/protocol/SubAgentFactorEditor.tsx new file mode 100644 index 00000000..e49e4ea5 --- /dev/null +++ b/frontend/src/components/protocol/SubAgentFactorEditor.tsx @@ -0,0 +1,74 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function SubAgentFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the sub-agent factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'sub_agent_toggle' ? 'All sub-agents on/off' : 'Sub-Agents as levels'} + {factor.level_type === 'sub_agent_toggle' ? 'Each cell receives all connected sub-agents or none.' : 'Each cell receives exactly one connected sub-agent.'} Add or disconnect Sub-Agent nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different sub-agents before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const subAgent = graph?.nodes.find((node) => node.id === id) + return
+
+

{factor.level_type === 'sub_agent_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : subAgent?.data.label ?? labels[index] ?? 'Unavailable sub-agent'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'sub_agent_toggle' && !!graph && !subAgent &&

Unavailable: disconnect or restore this sub-agent.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+

Individual sub-agent switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected sub-agents and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/SubAgentTestSelectors.tsx b/frontend/src/components/protocol/SubAgentTestSelectors.tsx new file mode 100644 index 00000000..43722275 --- /dev/null +++ b/frontend/src/components/protocol/SubAgentTestSelectors.tsx @@ -0,0 +1,17 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedSubAgents } from '@/lib/subAgentFactors' +import type { useSubAgentTestSelection } from './useSubAgentTestSelection' + +export function SubAgentTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedSubAgents(selection.graph!, node.id) + const toggle = node.data.sub_agent_factor_mode === 'sub_agent_toggle' + return
+

{node.data.label}: {toggle ? 'sub-agents' : 'sub-agent'} for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index b399bf7a..1fa9202f 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -287,7 +287,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { export interface UnboundField { pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } - connectorFactor?: { kind: 'skill' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } + connectorFactor?: { kind: 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -439,6 +439,7 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel const wholeConfigBound = !!bindings.config const configSubFieldBound = Object.keys(bindings).some((path) => path !== 'config' && path.startsWith('config.')) for (const field of bindableFieldsForNode(node)) { + if (node.type === 'sub_agent' && field.fieldPath === 'active' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'sub_agents' && (nodes.find((owner) => owner.id === edge.target)?.data.factor_bindings as Record | undefined)?.sub_agent_selection)) continue if (bindings[field.fieldPath]) continue if (field.fieldPath === 'config' && configSubFieldBound) continue if (field.fieldPath.startsWith('config.') && wholeConfigBound) continue @@ -468,6 +469,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const fields = unboundBindableFields(nodes, edges).filter((field) => { const kind = nodes.find((node) => node.id === field.nodeId)?.type return kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' + && !(kind === 'sub_agent' && field.fieldPath === 'active') && !isPatternNode(kind) && field.fieldPath !== PATTERN_FACTOR_PATH }) for (const node of nodes.filter((node) => isPatternNode(node.type))) { @@ -483,6 +485,25 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField } } const connectorOwners = new Set() + for (const node of nodes.filter((node) => node.type === 'sub_agent')) { + const owners = nodes.filter((owner) => owner.type === 'agent' && edges.some((edge) => edge.source === node.id && edge.target === owner.id && edge.targetHandle === 'sub_agents')) + if (!(node.data.factor_bindings as Record | undefined)?.active) fields.push({ + nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'active', + fieldLabel: 'This sub-agent on/off', levelType: 'boolean', currentValue: node.data.active ?? true, + connectorFactor: { kind: 'sub_agent', nodeId: node.id, mode: 'boolean' }, + }) + for (const owner of owners) { + const key = `${owner.id}:sub_agent` + if (connectorOwners.has(key)) continue + connectorOwners.add(key) + for (const mode of ['selection', 'toggle'] as const) fields.push({ + nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: 'sub_agent_selection', + fieldLabel: mode === 'selection' ? 'Sub-Agent levels' : 'All agent sub-agents on/off', + levelType: `sub_agent_${mode}`, currentValue: undefined, + connectorFactor: { kind: 'sub_agent', nodeId: node.id, mode: `sub_agent_${mode}`, agentId: owner.id }, + }) + } + } for (const node of nodes) { if (!['skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue const kind = (node.type === 'okf_bundle' || node.type === 'okf_document' ? 'knowledge' : node.type) as 'skill' | 'dataset' | 'knowledge' @@ -518,8 +539,9 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const kind = field.connectorFactor?.kind ?? node.type ?? 'node' const agentKinds = ['agent', 'sub_agent'] const targets = nodes.filter((candidate) => agentKinds.includes(candidate.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === candidate.id)) - const owner = agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined - const category = field.connectorFactor?.kind === 'skill' ? 'Skills' + const owner = field.connectorFactor?.kind === 'sub_agent' && field.connectorFactor.mode === 'boolean' ? targets.length === 1 ? targets[0] : undefined : agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined + const category = field.connectorFactor?.kind === 'sub_agent' ? 'Sub-Agents' + : field.connectorFactor?.kind === 'skill' ? 'Skills' : field.connectorFactor?.kind === 'dataset' ? 'Datasets' : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' : field.connectorFactor?.kind === 'model' ? 'Model' diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index 08d3e418..f732a188 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -23,6 +23,8 @@ export type LevelType = | 'tool_names' | 'skill_selection' | 'skill_toggle' + | 'sub_agent_selection' + | 'sub_agent_toggle' | 'dataset_selection' | 'dataset_toggle' | 'knowledge_selection' @@ -41,6 +43,8 @@ export const LEVEL_TYPE_LABELS: Record = { tool_names: 'Tools allowed', skill_selection: 'Skills as levels', skill_toggle: 'All skills on/off', + sub_agent_selection: 'Sub-Agent levels', + sub_agent_toggle: 'All agent sub-agents on/off', dataset_selection: 'Dataset levels', dataset_toggle: 'All datasets on/off', knowledge_selection: 'Knowledge levels', @@ -62,6 +66,8 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'tool_names' || type === 'skill_selection' || type === 'skill_toggle' || + type === 'sub_agent_selection' || + type === 'sub_agent_toggle' || type === 'dataset_selection' || type === 'dataset_toggle' || type === 'knowledge_selection' || diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index 3af87b6a..0b6bf7a6 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -82,8 +82,10 @@ export function AgentNode({ ...(peerNeedsToolCalling ? ["This model can't call tools, so this agent can't consult or delegate to connected agents"] : []), ] const { updateNodeData } = useReactFlow() - const { requestRunNode } = useProtocolCanvasActions() - const isActive = data.active ?? true + const { requestRunNode, requestSubAgentFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = isSubAgent && !!data.subAgentFactorControlled + const isActive = controlled || (data.active ?? true) + const factorCount = boundFactorCount(data) + (controlled ? 1 : 0) // Ids are stored, labels are only ever displayed -- the same invariant the // inspector's editor upholds via `toDisplayPrompt`. Without this the summary @@ -127,7 +129,8 @@ export function AgentNode({ updateNodeData(id, { active: !isActive })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { active: !isActive })} + onMakeFactor={isSubAgent && !experimentLocked && requestSubAgentFactor ? () => requestSubAgentFactor(id) : undefined} runAlone={{ canRun: !!data.canRunAlone, onRun: () => requestRunNode(id) }} /> {/* Sits inside the top-right corner so it stays clear of the Knowledge @@ -145,7 +148,7 @@ export function AgentNode({ node" anyway. Top-center (on hover) is NodeHoverToolbar, and the Architectural Pattern connector's own label/stub live OUTSIDE the card on the left of this edge, so neither competes for this corner. */} - {hasBoundFactor(data) && } + {factorCount > 0 && } {/* Main flow is left-to-right -- the bottom sub-connectors stay on the bottom edge regardless, since a config source hangs below a node no matter which way the main flow runs. @@ -379,7 +382,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Sub-Agents - + )} { + const graph = { + nodes: [ + { id: 'parent', type: 'agent', data: { factor_bindings: { sub_agent_selection: 'Workers' }, sub_agent_selection: ['a'] } }, + ...['a', 'b'].map((id) => ({ id, type: 'sub_agent', data: { label: id, active: false } })), + ], + edges: ['a', 'b'].map((source) => ({ source, target: 'parent', targetHandle: 'sub_agents' })), + } as unknown as ProtocolGraph + const { result, rerender } = renderHook(({ graph }) => useSubAgentTestSelection(graph), { initialProps: { graph } }) + expect(result.current.options).toEqual({ sub_agent_selections: { parent: 'a' } }) + act(() => result.current.setChoice('parent', 'b')) + expect(result.current.options).toEqual({ sub_agent_selections: { parent: 'b' } }) + rerender({ graph: { ...graph, edges: graph.edges.slice(0, 1) } }) + expect(result.current.options).toEqual({ sub_agent_selections: { parent: 'a' } }) + expect(result.current.error).toBeNull() +}) diff --git a/frontend/src/components/protocol/useSubAgentTestSelection.ts b/frontend/src/components/protocol/useSubAgentTestSelection.ts new file mode 100644 index 00000000..71474c5a --- /dev/null +++ b/frontend/src/components/protocol/useSubAgentTestSelection.ts @@ -0,0 +1,25 @@ +import { useEffect, useState } from 'react' +import { connectedSubAgents, SUB_AGENT_FACTOR_PATH } from '@/lib/subAgentFactors' +import type { ProtocolGraph } from '@/types/protocols' + +export function useSubAgentTestSelection(graph: ProtocolGraph | undefined, nodeId?: string) { + const [choices, setChoices] = useState>({}) + const owners = graph?.nodes.filter((node) => node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] && (!nodeId || node.id === nodeId)) ?? [] + const signature = JSON.stringify(owners.map((node) => [node.id, node.data.sub_agent_factor_mode, node.data.sub_agent_selection, connectedSubAgents(graph!, node.id).map((child) => child.id)])) + useEffect(() => { setChoices({}) }, [signature]) + const selections = Object.fromEntries(owners.map((node) => { + const selected = node.data.sub_agent_selection as string[] | undefined + if (node.data.sub_agent_factor_mode === 'sub_agent_toggle') return [node.id, choices[node.id] ?? (selected?.length ? 'all' : 'none')] + const ids = connectedSubAgents(graph!, node.id).map((child) => child.id) + const baseline = selected?.[0] + return [node.id, choices[node.id] ?? (baseline && ids.includes(baseline) ? baseline : ids[0] ?? '')] + })) + const error = owners.some((node) => { + const children = connectedSubAgents(graph!, node.id) + return children.some((child) => child.data.factor_bindings?.active) || + (node.data.sub_agent_factor_mode === 'sub_agent_toggle' + ? selections[node.id] === 'all' && children.length === 0 + : !children.some((child) => child.id === selections[node.id])) + }) ? 'Select a connected Sub-Agent for each agent and remove conflicting individual on/off bindings.' : null + return { graph, owners, selections, options: owners.length ? { sub_agent_selections: selections } : {}, error, setChoice: (agentId: string, id: string) => setChoices((previous) => ({ ...previous, [agentId]: id })) } +} diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index 10fc3d01..51172f61 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -1,3 +1,4 @@ +import { reconcileSubAgentFactor, SUB_AGENT_FACTOR_PATH } from './subAgentFactors' import { reconcileKnowledgeFactor, KNOWLEDGE_FACTOR_PATH } from './knowledgeFactors' import { reconcileDatasetFactor, DATASET_FACTOR_PATH } from './datasetFactors' import type { DesignFactor, DesignSpec } from '@/types/experiments' @@ -62,7 +63,7 @@ function withPathValue(root: Record, dottedPath: string, nextVa * as an extra level would turn each pause mid-edit into a new treatment. */ export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { - if ((factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle')) return factor + if ((factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle')) return factor if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) @@ -96,6 +97,10 @@ export function reconcileFactorBaselines(designSpec: DesignSpec | null | undefin } return factors.map((factor) => { + if ((factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle') && graph) { + const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] === factor.name) + return owner ? reconcileSubAgentFactor(factor, graph, owner.id) : factor + } if ((factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') && graph) { const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] === factor.name) return owner ? reconcileSkillFactor(factor, graph, owner.id) : factor @@ -159,7 +164,7 @@ export function factorBindingDiscrepancies( ? 'factor is no longer declared' : value === MISSING ? 'field no longer exists on the canvas' - : (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) + : (factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) ? null : 'published value does not match the first (canvas baseline) level' if (reason) discrepancies.push({ nodeId: node.id, nodeLabel, fieldPath, factorName, reason }) diff --git a/frontend/src/lib/subAgentFactors.test.ts b/frontend/src/lib/subAgentFactors.test.ts new file mode 100644 index 00000000..af244adf --- /dev/null +++ b/frontend/src/lib/subAgentFactors.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { reconcileSubAgentFactor, subAgentFactorIssues } from './subAgentFactors' + +const graph = { + nodes: [ + { id: 'parent', type: 'agent', data: { factor_bindings: { sub_agent_selection: 'Workers' } } }, + ...['a', 'b', 'c'].map((id) => ({ id, type: 'sub_agent', data: { label: `Worker ${id}` } })), + ], + edges: ['a', 'c'].map((source) => ({ source, target: 'parent', targetHandle: 'sub_agents' })), +} as unknown as ProtocolGraph + +it('preserves level order and labels while dropping disconnected workers and adding new ones', () => { + expect(reconcileSubAgentFactor({ name: 'Workers', level_type: 'sub_agent_selection', levels: [['b'], ['a']], level_labels: ['Old', 'Custom A'] }, graph, 'parent')).toEqual({ + name: 'Workers', level_type: 'sub_agent_selection', levels: [['a'], ['c']], level_labels: ['Custom A', 'Worker c'], + }) +}) + +it('keeps all-disabled as the baseline while refreshing the all-enabled level', () => { + expect(reconcileSubAgentFactor({ name: 'Workers', level_type: 'sub_agent_toggle', levels: [[], ['b']], level_labels: ['Without', 'With'] }, graph, 'parent')).toEqual({ + name: 'Workers', level_type: 'sub_agent_toggle', levels: [[], ['a', 'c']], level_labels: ['Without', 'With'], + }) +}) + +it('reports incomplete selection and unbound factors', () => { + const factor = { name: 'Workers', level_type: 'sub_agent_selection' as const, levels: [['a']] } + expect(subAgentFactorIssues({ ...graph, edges: graph.edges.slice(0, 1) }, [factor])).toEqual(['Workers: connect at least two different sub-agents.']) + expect(subAgentFactorIssues(undefined, [factor])).toEqual(['Workers: rebind or remove this sub-agent factor.']) +}) diff --git a/frontend/src/lib/subAgentFactors.ts b/frontend/src/lib/subAgentFactors.ts new file mode 100644 index 00000000..7d1d0d76 --- /dev/null +++ b/frontend/src/lib/subAgentFactors.ts @@ -0,0 +1,93 @@ +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +export const SUB_AGENT_FACTOR_PATH = 'sub_agent_selection' + +export const subAgentFactorModes = { + boolean: 'This sub-agent on/off', + sub_agent_selection: 'Sub-Agent levels', + sub_agent_toggle: 'All agent sub-agents on/off', +} +export type SubAgentFactorMode = keyof typeof subAgentFactorModes + +export function subAgentFactorConflict(graph: ProtocolGraph, factors: DesignFactor[], subAgentNodeId: string, mode: SubAgentFactorMode, agentId: string): string | undefined { + const owner = graph.nodes.find((node) => node.id === (mode === 'boolean' ? subAgentNodeId : agentId)) + const currentName = owner?.data.factor_bindings?.[mode === 'boolean' ? 'active' : SUB_AGENT_FACTOR_PATH] + const current = factors.find((factor) => factor.name === currentName) + if (current && (current.level_type ?? 'boolean') !== mode) { + return `${subAgentFactorModes[mode]} is unavailable because this Agent already uses the factor ${current.name} (${subAgentFactorModes[(current.level_type ?? 'boolean') as SubAgentFactorMode]}). Keep the current factor type, or select its factor type and remove it first.` + } + const conflicts = [...new Set(mode === 'boolean' + ? graph.edges.filter((edge) => edge.source === subAgentNodeId && edge.targetHandle === 'sub_agents') + .flatMap((edge) => { + const name = graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] + return name ? [name] : [] + }) + : connectedSubAgents(graph, agentId).flatMap((node) => node.data.factor_bindings?.active ? [node.data.factor_bindings.active] : []))] + if (!conflicts.length) return undefined + return mode === 'boolean' + ? `${subAgentFactorModes[mode]} is unavailable because a connected Agent already controls this Sub-Agent with the factor ${conflicts.join(', ')}. Use the existing Agent-level factor, or select its factor type and remove it first.` + : `${subAgentFactorModes[mode]} is unavailable because connected Sub-Agents already have factor bindings: ${conflicts.join(', ')}. Agent-level sub-agent factors cannot be combined with individual Sub-Agent on/off factor bindings. Use This sub-agent on/off, or remove those bindings in each Sub-Agent’s factor dialog first.` +} + +export function isSubAgentFactor(factor: DesignFactor): boolean { + return factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' +} + +export function connectedSubAgents(graph: ProtocolGraph, agentId: string): ProtocolNode[] { + return graph.edges.filter((edge) => edge.target === agentId && edge.targetHandle === 'sub_agents') + .flatMap((edge) => { + const node = graph.nodes.find((node) => node.id === edge.source && node.type === 'sub_agent') + return node ? [node] : [] + }) +} + +export function subAgentId(node: ProtocolNode): string { + return node.id +} + +export function reconcileSubAgentFactor(factor: DesignFactor, graph: ProtocolGraph, agentId: string): DesignFactor { + const subAgents = connectedSubAgents(graph, agentId) + const ids = [...new Set(subAgents.map(subAgentId))] + if (factor.level_type === 'sub_agent_toggle') { + const noneFirst = factor.levels.length === 2 && Array.isArray(factor.levels[0]) && factor.levels[0].length === 0 + return { + ...factor, + levels: noneFirst ? [[], ids] : [ids, []], + level_labels: factor.level_labels?.length === 2 ? factor.level_labels : noneFirst ? ['All disabled', 'All enabled'] : ['All enabled', 'All disabled'], + } + } + const oldIds = factor.levels.map((level) => Array.isArray(level) ? String(level[0] ?? '') : '') + const ordered = [...new Set([...oldIds.filter((id) => ids.includes(id)), ...ids.filter((id) => !oldIds.includes(id))])] + return { + ...factor, + levels: ordered.map((id) => [id]), + level_labels: ordered.map((id) => { + const index = oldIds.indexOf(id) + if (index >= 0 && factor.level_labels?.[index]) return factor.level_labels[index] + const node = subAgents.find((node) => subAgentId(node) === id) + const label = String(node?.data.label ?? 'Unavailable sub-agent') + return label || 'Unavailable sub-agent' + }), + } +} + +export function subAgentFactorOwner(graph: ProtocolGraph | undefined, subAgentNodeId: string): string | undefined { + return graph?.edges.filter((edge) => edge.source === subAgentNodeId && edge.targetHandle === 'sub_agents') + .map((edge) => graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH]) + .find(Boolean) +} + +export function subAgentFactorIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[]): string[] { + return factors.filter(isSubAgentFactor).flatMap((factor) => { + const owner = graph?.nodes.find((node) => node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] === factor.name) + if (!owner || !graph) return [`${factor.name}: rebind or remove this sub-agent factor.`] + const subAgents = connectedSubAgents(graph, owner.id) + const ids = [...new Set(subAgents.map(subAgentId))] + const conflicts = subAgents.some((node) => !!node.data.factor_bindings?.active) + if (conflicts) return [`${factor.name}: remove individual sub-agent on/off factor bindings first.`] + if (ids.length < (factor.level_type === 'sub_agent_toggle' ? 1 : 2)) return [`${factor.name}: ${factor.level_type === 'sub_agent_toggle' ? 'connect at least one sub-agent' : 'connect at least two different sub-agents'}.`] + if (ids.some((id) => !id)) return [`${factor.name}: disconnect or restore unavailable sub-agents.`] + return [] + }) +} diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 42ac0d60..1a65acb9 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -24,6 +24,8 @@ export interface DesignFactor { | 'tool_names' | 'skill_selection' | 'skill_toggle' + | 'sub_agent_selection' + | 'sub_agent_toggle' | 'dataset_selection' | 'dataset_toggle' | 'knowledge_selection' diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index 822f8a13..9fd8eb6d 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -362,6 +362,8 @@ export interface AgentNodeData { dataset_factor_mode?: 'dataset_selection' | 'dataset_toggle' knowledge_selection?: string[] knowledge_factor_mode?: 'knowledge_selection' | 'knowledge_toggle' + sub_agent_selection?: string[] + sub_agent_factor_mode?: 'sub_agent_selection' | 'sub_agent_toggle' skill_selection?: string[] skill_factor_mode?: 'skill_selection' | 'skill_toggle' label: string diff --git a/src/asaree/api/protocols.py b/src/asaree/api/protocols.py index 26a76df1..f5d655bd 100644 --- a/src/asaree/api/protocols.py +++ b/src/asaree/api/protocols.py @@ -70,6 +70,7 @@ update_protocol, ) from asaree.services.skill_factors import test_skill_factor_values +from asaree.services.sub_agent_factors import test_sub_agent_factor_values from asaree.services.test_run_results import FreshnessReason, project_test_run_result from asaree.worker.enqueue import enqueue_protocol_run @@ -280,6 +281,7 @@ class TestRunRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) knowledge_selections: dict[str, str] | None = None + sub_agent_selections: dict[str, str] | None = None skill_selections: dict[str, str] | None = None dataset_selections: dict[str, str] | None = None @@ -289,6 +291,7 @@ class NodePlayRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) knowledge_selections: dict[str, str] | None = None + sub_agent_selections: dict[str, str] | None = None skill_selections: dict[str, str] | None = None dataset_selections: dict[str, str] | None = None @@ -298,6 +301,7 @@ async def _node_test_factor_values( ) -> dict[str, Any]: try: return { + **await test_sub_agent_factor_values(graph, body.sub_agent_selections if body else None, owner_id, node_id), **await test_skill_factor_values(graph, body.skill_selections if body else None, owner_id, node_id), **await test_knowledge_factor_values(graph, body.knowledge_selections if body else None, owner_id, node_id), **await test_dataset_factor_values(graph, body.dataset_selections if body else None, owner_id, db, node_id), @@ -690,6 +694,9 @@ async def create_test_run_endpoint( protocol_revision_id=revision.id, dataset_row=dataset_row, factor_values={ + **await test_sub_agent_factor_values( + revision.graph, body.sub_agent_selections if body else None, user.id + ), **await test_skill_factor_values(revision.graph, body.skill_selections if body else None, user.id), **await test_knowledge_factor_values( revision.graph, body.knowledge_selections if body else None, user.id diff --git a/src/asaree/services/agent_messenger.py b/src/asaree/services/agent_messenger.py index 11c68628..01d59e6e 100644 --- a/src/asaree/services/agent_messenger.py +++ b/src/asaree/services/agent_messenger.py @@ -491,7 +491,23 @@ async def _refusal(self, from_agent_id: str, to_agent_id: str) -> str | None: # The live draft graph, not the revision this run's agents came from. live_graph = protocol.graph if protocol is not None else self._graph - if not _can_deliver_communication(live_graph, from_agent_id, to_agent_id): + run_nodes = {node["id"]: node for node in self._graph.get("nodes") or []} + parent_data = (run_nodes.get(from_agent_id) or {}).get("data") or {} + selected_sub_agent = (run_nodes.get(to_agent_id) or {}).get("type") == "sub_agent" and ( + parent_data.get("factor_bindings") or {} + ).get("sub_agent_selection") + if selected_sub_agent: + # Published factor values stay authoritative. The existing live + # unplug behavior still applies to the parent connection. + reachable = _can_deliver_communication(self._graph, from_agent_id, to_agent_id) and any( + edge.get("source") == to_agent_id + and edge.get("target") == from_agent_id + and edge.get("targetHandle") == "sub_agents" + for edge in live_graph.get("edges") or [] + ) + else: + reachable = _can_deliver_communication(live_graph, from_agent_id, to_agent_id) + if not reachable: return ( "That agent is not connected to you on the canvas, so it cannot be consulted. " "Answer using the agents listed for you, or with what you already have." diff --git a/src/asaree/services/design_generation.py b/src/asaree/services/design_generation.py index 8966fdd0..cfa62d7e 100644 --- a/src/asaree/services/design_generation.py +++ b/src/asaree/services/design_generation.py @@ -38,6 +38,7 @@ from asaree.services.factorial_cells import list_replicates, upsert_replicate from asaree.services.knowledge_factors import validate_knowledge_factors from asaree.services.skill_factors import validate_skill_factors +from asaree.services.sub_agent_factors import validate_sub_agent_factor_structure class DesignValidationError(ValueError): @@ -288,9 +289,11 @@ async def get_design_impact( """Compare the declared factorial matrix to its materialized revision.""" material = material_design_spec(design_spec) incomplete_skills = any( - (factor.get("level_type") in {"skill_selection", "dataset_selection", "knowledge_selection"} + (factor.get("level_type") in { + "sub_agent_selection", "skill_selection", "dataset_selection", "knowledge_selection" + } and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") in {"skill_toggle", "dataset_toggle", "knowledge_toggle"} + or (factor.get("level_type") in {"sub_agent_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle"} and not any(factor.get("levels") or [])) for factor in material["factors"] ) @@ -420,6 +423,7 @@ async def generate_design_cells( raise DesignValidationError("Experiment not found") if any( factor.get("level_type") in { + "sub_agent_selection", "sub_agent_toggle", "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle", "knowledge_selection", "knowledge_toggle", } @@ -427,6 +431,7 @@ async def generate_design_cells( ): protocol = await db.scalar(select(Protocol).where(Protocol.experiment_id == experiment_id)) try: + validate_sub_agent_factor_structure({"factors": factors}, protocol.graph if protocol else {}) await validate_skill_factors({"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id) await validate_knowledge_factors( {"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index 8b04f605..9a9befbc 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -7,6 +7,7 @@ from asaree.services.dataset_factors import validate_dataset_factor_structure from asaree.services.knowledge_factors import validate_knowledge_factor_structure from asaree.services.skill_factors import validate_skill_factor_structure +from asaree.services.sub_agent_factors import validate_sub_agent_factor_structure _MISSING = object() @@ -33,6 +34,7 @@ def unbound_factor_names(design_spec: dict[str, Any] | None, graph: dict[str, An def validate_factor_bindings( design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete_skills: bool = True ) -> None: + validate_sub_agent_factor_structure(design_spec, graph, complete=complete_skills) validate_skill_factor_structure(design_spec, graph, complete=complete_skills) validate_knowledge_factor_structure(design_spec, graph, complete=complete_skills) validate_dataset_factor_structure(design_spec, graph, complete=complete_skills) @@ -72,7 +74,8 @@ def validate_factor_bindings( ) if ( level_type not in { - "boolean", "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle", + "boolean", "sub_agent_selection", "sub_agent_toggle", + "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle", "knowledge_selection", "knowledge_toggle", } and not legacy_boolean diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 3e9cb855..38829e46 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -1065,6 +1065,7 @@ def topological_order(graph: dict[str, Any], *, require_acyclic: bool = True) -> except DatasetRowInputError as exc: raise ProtocolValidationError(str(exc)) from exc _, downstream, _ = _adjacency(graph) + graph = graph_with_sub_agent_selection(graph) nodes, ordered, complete = _kahn_order(graph) if not nodes: raise ProtocolValidationError("This protocol has no nodes.") @@ -1373,7 +1374,7 @@ def apply_factor_bindings(graph: dict[str, Any], factor_values: dict[str, Any]) for field_path, factor_name in bindings.items(): if factor_name in factor_values: _set_path(data, field_path, factor_values[factor_name]) - return patched + return graph_with_sub_agent_selection(patched, copy_graph=False) def find_gated_pairs(graph: dict[str, Any]) -> dict[str, dict[str, Any]]: @@ -2283,11 +2284,36 @@ def _connected_agent_ids(graph: dict[str, Any], node_id: str) -> list[str]: return peers +def graph_with_sub_agent_selection(graph: dict[str, Any], *, copy_graph: bool = True) -> dict[str, Any]: + """Materialize parent-owned availability for all execution consumers.""" + patched = copy.deepcopy(graph) if copy_graph else graph + nodes = {node["id"]: node for node in patched.get("nodes") or []} + for parent in nodes.values(): + data = parent.get("data") or {} + if parent.get("type") != "agent" or not (data.get("factor_bindings") or {}).get("sub_agent_selection"): + continue + selected = data.get("sub_agent_selection") or [] + for edge in patched.get("edges") or []: + child = nodes.get(edge.get("source")) + if ( + edge.get("target") == parent["id"] + and edge.get("targetHandle") == "sub_agents" + and child + and child.get("type") == "sub_agent" + ): + child.setdefault("data", {})["active"] = child["id"] in selected + return patched + + def _sub_agent_ids(graph: dict[str, Any], parent_id: str) -> list[str]: """Active Sub-Agents owned by *parent_id*, in canvas wiring order.""" nodes = {str(n.get("id")): n for n in graph.get("nodes") or [] if n.get("id")} if (nodes.get(parent_id) or {}).get("type") != "agent": return [] + data = (nodes[parent_id].get("data") or {}) + selected = ( + data.get("sub_agent_selection") if (data.get("factor_bindings") or {}).get("sub_agent_selection") else None + ) children: list[str] = [] for edge in graph.get("edges") or []: if edge.get("target") != parent_id or edge.get("targetHandle") != "sub_agents": @@ -2297,7 +2323,7 @@ def _sub_agent_ids(graph: dict[str, Any], parent_id: str) -> list[str]: if ( child is not None and child.get("type") == "sub_agent" - and _is_node_active(child) + and (child_id in selected if selected is not None else _is_node_active(child)) and child_id not in children ): children.append(child_id) @@ -5579,8 +5605,7 @@ async def run_protocol(protocol_run_id: uuid.UUID) -> None: # values into whichever fields the canvas bound to a matching factor # name before doing anything else, so every node below (including # topological_order's own validation) sees the already-patched graph. - if factor_values: - graph = apply_factor_bindings(graph, factor_values) + graph = apply_factor_bindings(graph, factor_values or {}) try: validate_coordination_strategy(design_spec, graph=graph) diff --git a/src/asaree/services/protocol_revisions.py b/src/asaree/services/protocol_revisions.py index 5b6f3cee..4e705d8d 100644 --- a/src/asaree/services/protocol_revisions.py +++ b/src/asaree/services/protocol_revisions.py @@ -150,9 +150,11 @@ async def publish_protocol( if experiment is not None: from asaree.services.design_generation import get_design_impact incomplete_skills = any( - (factor.get("level_type") in {"skill_selection", "dataset_selection", "knowledge_selection"} + (factor.get("level_type") in { + "sub_agent_selection", "skill_selection", "dataset_selection", "knowledge_selection" + } and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") in {"skill_toggle", "dataset_toggle", "knowledge_toggle"} + or (factor.get("level_type") in {"sub_agent_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle"} and not any(factor.get("levels") or [])) for factor in (experiment.design_spec or {}).get("factors") or [] ) diff --git a/src/asaree/services/sub_agent_factors.py b/src/asaree/services/sub_agent_factors.py new file mode 100644 index 00000000..e87120d6 --- /dev/null +++ b/src/asaree/services/sub_agent_factors.py @@ -0,0 +1,108 @@ +"""Connector-owned Sub-Agent choices shared by design validation and test runs.""" + +from __future__ import annotations + +import uuid +from typing import Any + +SUB_AGENT_SELECTION = "sub_agent_selection" +SUB_AGENT_TOGGLE = "sub_agent_toggle" +SUB_AGENT_FACTOR_TYPES = {SUB_AGENT_SELECTION, SUB_AGENT_TOGGLE} + + +def connected_sub_agent_nodes(graph: dict[str, Any], agent_id: str) -> list[dict[str, Any]]: + nodes = {node["id"]: node for node in graph.get("nodes") or []} + return [ + nodes[edge["source"]] + for edge in graph.get("edges") or [] + if edge.get("target") == agent_id + and edge.get("targetHandle") == "sub_agents" + and edge.get("source") in nodes + and nodes[edge["source"]].get("type") == "sub_agent" + ] + + +def validate_sub_agent_factor_structure( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete: bool = True +) -> None: + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + bound = { + (node.get("data") or {}).get("factor_bindings", {}).get(SUB_AGENT_SELECTION) + for node in graph.get("nodes") or [] + } + for name, factor in factors.items(): + if factor.get("level_type") in SUB_AGENT_FACTOR_TYPES and name not in bound: + raise ValueError(f"Rebind or remove sub-agent factor {name!r}.") + for node in graph.get("nodes") or []: + data = node.get("data") or {} + name = (data.get("factor_bindings") or {}).get(SUB_AGENT_SELECTION) + if not name: + continue + factor = factors.get(name) + if node.get("type") != "agent": + raise ValueError("Only an Agent Sub-Agents connector can own a sub-agent factor.") + if not factor or factor.get("level_type") not in SUB_AGENT_FACTOR_TYPES: + raise ValueError(f"Sub-Agent connector factor {name!r} must declare a sub-agent factor type.") + toggle = factor.get("level_type") == SUB_AGENT_TOGGLE + if data.get("sub_agent_factor_mode", SUB_AGENT_SELECTION) != factor.get("level_type"): + raise ValueError(f"Sub-Agent factor {name!r}: canvas mode does not match the declared factor.") + sub_agents = connected_sub_agent_nodes(graph, node["id"]) + ids = {sub_agent["id"] for sub_agent in sub_agents} + if complete and (None in ids or "" in ids): + raise ValueError(f"Sub-Agent factor {name!r}: disconnect or restore unavailable sub-agents.") + if complete and len(ids) < (1 if toggle else 2): + minimum = "one sub-agent" if toggle else "two different sub-agents" + raise ValueError(f"Sub-Agent factor {name!r}: connect at least {minimum}.") + if any(((sub_agent.get("data") or {}).get("factor_bindings") or {}).get("active") for sub_agent in sub_agents): + raise ValueError(f"Sub-Agent factor {name!r}: remove individual sub-agent on/off factor bindings first.") + levels = factor.get("levels") or [] + if toggle: + if ( + len(levels) != 2 + or any(not isinstance(level, list) for level in levels) + or sorted(len(level) for level in levels) != [0, len(ids)] + or any(level and (set(level) != ids or len(level) != len(ids)) for level in levels) + ): + raise ValueError(f"Sub-Agent factor {name!r}: levels must enable every connected sub-agent or none.") + continue + if any(not isinstance(level, list) or len(level) != 1 or level[0] not in ids for level in levels): + raise ValueError(f"Sub-Agent factor {name!r}: each level must name one connected sub-agent.") + if len(levels) != len(ids) or {level[0] for level in levels} != ids: + raise ValueError(f"Sub-Agent factor {name!r}: levels must include every connected sub-agent exactly once.") + + +async def test_sub_agent_factor_values( + graph: dict[str, Any], selections: dict[str, str] | None, owner_id: uuid.UUID, node_id: str | None = None +) -> dict[str, Any]: + """Validate preview choices against the publication and retain run provenance.""" + selections = selections or {} + owners = { + node["id"]: node + for node in graph.get("nodes") or [] + if (node.get("data") or {}).get("factor_bindings", {}).get(SUB_AGENT_SELECTION) + and (node_id is None or node["id"] == node_id) + } + if set(selections) - owners.keys(): + raise ValueError("Sub-Agent selection must belong to a sub-agent factor in this run.") + values: dict[str, Any] = {} + for agent_id, node in owners.items(): + data = node.get("data") or {} + connected = [sub_agent["id"] for sub_agent in connected_sub_agent_nodes(graph, agent_id)] + if any( + ((sub_agent.get("data") or {}).get("factor_bindings") or {}).get("active") + for sub_agent in connected_sub_agent_nodes(graph, agent_id) + ): + raise ValueError("Remove individual sub-agent on/off factor bindings before testing this connector factor.") + baseline = data.get(SUB_AGENT_SELECTION) or [] + if data.get("sub_agent_factor_mode") == SUB_AGENT_TOGGLE: + selected = selections.get(agent_id, "all" if baseline else "none") + if selected not in {"all", "none"}: + raise ValueError("Select all enabled or all disabled for the test run.") + chosen = list(dict.fromkeys(connected)) if selected == "all" else [] + values[data["factor_bindings"][SUB_AGENT_SELECTION]] = chosen + continue + selected = selections.get(agent_id, baseline[0] if baseline else None) + if not selected or selected not in connected: + raise ValueError("Select an available connected sub-agent for the test run.") + values[data["factor_bindings"][SUB_AGENT_SELECTION]] = [selected] + return values diff --git a/tests/test_agent_messenger.py b/tests/test_agent_messenger.py index ae2f7144..e16be9a5 100644 --- a/tests/test_agent_messenger.py +++ b/tests/test_agent_messenger.py @@ -377,6 +377,34 @@ async def test_pulling_the_edge_mid_run_stops_the_next_consultation(stubs: dict[ assert len(stubs["peer_runs"]) == 1 +async def test_sub_agent_factor_selection_survives_draft_changes_but_not_unplugging(stubs: dict[str, Any]) -> None: + graph = _graph() + graph["nodes"][0]["data"].update( + factor_bindings={"sub_agent_selection": "Workers"}, sub_agent_selection=["critic"] + ) + graph["nodes"][1]["type"] = "sub_agent" + graph["nodes"][1]["data"]["active"] = False + graph["nodes"][2]["type"] = "sub_agent" + graph["edges"] = [ + {"source": id, "target": "planner", "targetHandle": "sub_agents"} for id in ["critic", "loner"] + ] + stubs["live_graph"] = graph + messenger = AgentMessenger( + protocol_id=PROTOCOL_ID, + protocol_run_id=RUN_ID, + owner_id=OWNER, + graph=pe.apply_factor_bindings(graph, {}), + entry_agent_id="planner", + ) + assert (await _ask(messenger)).state == "completed" + assert (await _ask(messenger, to="loner")).state == "rejected" + graph["nodes"][0]["data"]["sub_agent_selection"] = ["loner"] + assert (await _ask(messenger)).state == "completed" + assert (await _ask(messenger, to="loner")).state == "rejected" + graph["edges"] = [] + assert (await _ask(messenger)).state == "rejected" + + async def test_a_refused_consultation_is_still_in_the_transcript(stubs: dict[str, Any]) -> None: """Invariant 10 -- a silently dropped question is the failure this makes debuggable.""" diff --git a/tests/test_sub_agent_factors.py b/tests/test_sub_agent_factors.py new file mode 100644 index 00000000..f5b34169 --- /dev/null +++ b/tests/test_sub_agent_factors.py @@ -0,0 +1,104 @@ +"""Sub-Agent factors preserve worker configurations and scope delegation.""" + +import copy +import uuid + +import pytest + +from asaree.services.factor_bindings import validate_factor_bindings +from asaree.services.protocol_execution import ( + ProtocolValidationError, + _sub_agent_ids, + apply_factor_bindings, + topological_order, +) +from asaree.services.sub_agent_factors import test_sub_agent_factor_values as preview_values + + +def worker_graph(): + graph = { + "nodes": [ + { + "id": "parent", + "type": "agent", + "data": {"factor_bindings": {"sub_agent_selection": "Workers"}, "sub_agent_selection": ["a"]}, + }, + *[ + {"id": id, "type": "sub_agent", "data": {"active": False, "config": {"prompt": id}}} + for id in ["a", "b"] + ], + ], + "edges": [{"source": id, "target": "parent", "targetHandle": "sub_agents"} for id in ["a", "b"]], + } + spec = {"factors": [{"name": "Workers", "level_type": "sub_agent_selection", "levels": [["a"], ["b"]]}]} + return graph, spec + + +def test_selection_preserves_draft_and_overrides_disabled_workers(): + graph, spec = worker_graph() + original = copy.deepcopy(graph) + validate_factor_bindings(spec, graph) + patched = apply_factor_bindings(graph, {"Workers": ["b"]}) + assert _sub_agent_ids(patched, "parent") == ["b"] + assert patched["nodes"][1]["data"]["active"] is False + assert patched["nodes"][2]["data"]["active"] is True + assert patched["nodes"][2]["data"]["config"] == {"prompt": "b"} + assert graph == original + + +def test_individual_on_off_only_controls_its_worker(): + graph, _ = worker_graph() + graph["nodes"][0]["data"] = {} + graph["nodes"][1]["data"]["factor_bindings"] = {"active": "Enabled"} + assert _sub_agent_ids(apply_factor_bindings(graph, {"Enabled": True}), "parent") == ["a"] + assert _sub_agent_ids(apply_factor_bindings(graph, {"Enabled": False}), "parent") == [] + + +@pytest.mark.parametrize("selection", [["a", "b"], []]) +def test_toggle_includes_all_disabled_workers_or_none(selection): + graph, spec = worker_graph() + graph["nodes"][0]["data"].update(sub_agent_factor_mode="sub_agent_toggle", sub_agent_selection=[]) + spec["factors"][0].update(level_type="sub_agent_toggle", levels=[["a", "b"], []]) + validate_factor_bindings(spec, graph) + assert _sub_agent_ids(apply_factor_bindings(graph, {"Workers": selection}), "parent") == selection + + +def test_prompt_factor_coexists_but_individual_active_factor_conflicts(): + graph, spec = worker_graph() + graph["nodes"][1]["data"]["factor_bindings"] = {"config.prompt": "Prompt"} + spec["factors"].append({"name": "Prompt", "level_type": "text", "levels": ["a", "other"]}) + validate_factor_bindings(spec, graph) + graph["nodes"][1]["data"]["factor_bindings"]["active"] = "Enabled" + with pytest.raises(ValueError, match="individual sub-agent on/off"): + validate_factor_bindings(spec, graph) + + +@pytest.mark.parametrize("levels", [[["a"], ["a"]], [["foreign"], ["b"]], [["a"]]]) +def test_selection_rejects_duplicate_foreign_or_missing_workers(levels): + graph, spec = worker_graph() + spec["factors"][0]["levels"] = levels + with pytest.raises(ValueError): + validate_factor_bindings(spec, graph) + + +@pytest.mark.asyncio +async def test_preview_choices_and_node_scope(): + graph, _ = worker_graph() + owner = uuid.uuid4() + assert await preview_values(graph, None, owner) == {"Workers": ["a"]} + assert await preview_values(graph, {"parent": "b"}, owner) == {"Workers": ["b"]} + with pytest.raises(ValueError, match="belong"): + await preview_values(graph, {"parent": "b"}, owner, "a") + with pytest.raises(ValueError, match="connected"): + await preview_values(graph, {"parent": "foreign"}, owner) + graph["nodes"][0]["data"]["sub_agent_factor_mode"] = "sub_agent_toggle" + assert await preview_values(graph, {"parent": "all"}, owner) == {"Workers": ["a", "b"]} + assert await preview_values(graph, {"parent": "none"}, owner) == {"Workers": []} + + +def test_selected_disabled_worker_requires_a_model_at_validation(): + graph, _ = worker_graph() + graph["nodes"].append({"id": "model", "type": "model_openai", "data": {"config": {}}}) + graph["edges"].append({"source": "model", "target": "parent", "targetHandle": "model"}) + with pytest.raises(ProtocolValidationError, match="exactly one Model"): + topological_order(graph) From 62a7ae89cc283bf7ada636a7d07ee9712941c8b4 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 16:17:47 -0700 Subject: [PATCH 14/63] Add Tool and Script connector factor workflows --- frontend/src/api/client.ts | 2 +- .../components/protocol/DesignTab.test.tsx | 11 +- .../src/components/protocol/DesignTab.tsx | 10 +- .../protocol/FactorEditorDialog.tsx | 9 +- .../protocol/McpToolNodeInspector.test.tsx | 20 ++ .../protocol/McpToolNodeInspector.tsx | 110 +++----- .../protocol/ProtocolCanvas.test.tsx | 67 ++++- .../components/protocol/ProtocolCanvas.tsx | 176 ++++++++++++- .../protocol/ProtocolCanvasContext.tsx | 2 + .../protocol/ScriptFactorDialog.test.tsx | 84 +++++++ .../protocol/ScriptFactorDialog.tsx | 124 +++++++++ .../protocol/ScriptFactorEditor.tsx | 74 ++++++ .../protocol/ScriptNodeInspector.tsx | 70 +++--- .../protocol/ScriptTestSelectors.tsx | 26 ++ .../protocol/ToolFactorDialog.test.tsx | 114 +++++++++ .../components/protocol/ToolFactorDialog.tsx | 125 ++++++++++ .../components/protocol/ToolFactorEditor.tsx | 74 ++++++ .../components/protocol/ToolTestSelectors.tsx | 26 ++ .../src/components/protocol/bindableFields.ts | 25 +- .../src/components/protocol/factorLevels.ts | 14 +- .../components/protocol/nodes/AgentNode.tsx | 5 +- .../protocol/nodes/McpClientToolNode.tsx | 10 +- .../components/protocol/nodes/McpToolNode.tsx | 10 +- .../components/protocol/nodes/ScriptNode.tsx | 18 +- .../protocol/useScriptTestSelection.test.tsx | 21 ++ .../protocol/useScriptTestSelection.ts | 21 ++ .../protocol/useToolTestSelection.test.tsx | 51 ++++ .../protocol/useToolTestSelection.ts | 28 +++ frontend/src/lib/factorBindings.ts | 14 +- frontend/src/lib/scriptFactors.test.ts | 37 +++ frontend/src/lib/scriptFactors.ts | 120 +++++++++ frontend/src/lib/toolFactors.test.ts | 38 +++ frontend/src/lib/toolFactors.ts | 97 ++++++++ frontend/src/types/experiments.ts | 4 + frontend/src/types/protocols.ts | 17 +- src/asaree/api/protocols.py | 10 + src/asaree/services/design_generation.py | 23 +- src/asaree/services/factor_bindings.py | 9 +- src/asaree/services/protocol_execution.py | 15 +- src/asaree/services/protocol_revisions.py | 7 +- src/asaree/services/script_factors.py | 166 +++++++++++++ src/asaree/services/tool_factors.py | 138 ++++++++++ tests/test_script_factors.py | 235 ++++++++++++++++++ tests/test_tool_factors.py | 187 ++++++++++++++ 44 files changed, 2258 insertions(+), 186 deletions(-) create mode 100644 frontend/src/components/protocol/McpToolNodeInspector.test.tsx create mode 100644 frontend/src/components/protocol/ScriptFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/ScriptFactorDialog.tsx create mode 100644 frontend/src/components/protocol/ScriptFactorEditor.tsx create mode 100644 frontend/src/components/protocol/ScriptTestSelectors.tsx create mode 100644 frontend/src/components/protocol/ToolFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/ToolFactorDialog.tsx create mode 100644 frontend/src/components/protocol/ToolFactorEditor.tsx create mode 100644 frontend/src/components/protocol/ToolTestSelectors.tsx create mode 100644 frontend/src/components/protocol/useScriptTestSelection.test.tsx create mode 100644 frontend/src/components/protocol/useScriptTestSelection.ts create mode 100644 frontend/src/components/protocol/useToolTestSelection.test.tsx create mode 100644 frontend/src/components/protocol/useToolTestSelection.ts create mode 100644 frontend/src/lib/scriptFactors.test.ts create mode 100644 frontend/src/lib/scriptFactors.ts create mode 100644 frontend/src/lib/toolFactors.test.ts create mode 100644 frontend/src/lib/toolFactors.ts create mode 100644 src/asaree/services/script_factors.py create mode 100644 src/asaree/services/tool_factors.py create mode 100644 tests/test_script_factors.py create mode 100644 tests/test_tool_factors.py diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 883a81e7..9e67c4ed 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -187,7 +187,7 @@ export interface ResultsScope { protocol_revision_id?: string } -export interface RowSelection { row_index?: number | null; sub_agent_selections?: Record; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } +export interface RowSelection { row_index?: number | null; sub_agent_selections?: Record; script_selections?: Record; tool_selections?: Record; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } function scopeQuery(scope?: ResultsScope): string { const params = new URLSearchParams() diff --git a/frontend/src/components/protocol/DesignTab.test.tsx b/frontend/src/components/protocol/DesignTab.test.tsx index 2cd2926d..1508b09f 100644 --- a/frontend/src/components/protocol/DesignTab.test.tsx +++ b/frontend/src/components/protocol/DesignTab.test.tsx @@ -46,6 +46,13 @@ describe('DesignTab design generation', () => { ['Sub-Agent levels', 'sub_agent', 'sub_agent_selection'], ['All agent sub-agents on/off', 'sub_agent', 'sub_agent_toggle'], ['This sub-agent on/off', 'sub_agent', 'boolean'], + ['Script levels', 'script', 'script_selection'], + ['All agent scripts on/off', 'script', 'script_toggle'], + ['This script on/off', 'script', 'boolean'], + ['Tool levels', 'tool', 'tool_selection'], + ['All agent tools on/off', 'tool', 'tool_toggle'], + ['This tool on/off', 'tool', 'boolean'], + ['Tools allowed', 'tool', 'tool_names'], ['Skill levels', 'skill', 'skill_selection'], ['All skills on/off', 'skill', 'skill_toggle'], ['This skill on/off', 'skill', 'boolean'], @@ -58,11 +65,13 @@ describe('DesignTab design generation', () => { client.setQueryData(protocolGraphQueryKey('protocol-1'), { nodes: [ { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer', config: {} } }, + { id: 'script', type: 'script', position: { x: 0, y: 0 }, data: { label: 'Scoring', config: { name: 'score', language: 'python', code: 'print(1)' } } }, + { id: 'tool', type: 'mcp_tool', position: { x: 0, y: 0 }, data: { label: 'Search', config: { server_id: 's', tool_names: ['search'] } } }, { id: 'skill', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Summarize', config: { skill_id: 'a' } } }, { id: 'sub_agent', type: 'sub_agent', position: { x: 0, y: 0 }, data: { label: 'Worker', config: {} } }, { id: 'dataset', type: 'dataset', position: { x: 0, y: 0 }, data: { label: 'Cohort', config: { dataset_id: 'd' } } }, ], - edges: ['skill', 'dataset', 'sub_agent'].map(source => ({ id: source, source, target: 'agent', targetHandle: source === 'sub_agent' ? 'sub_agents' : source })), + edges: ['script', 'tool', 'skill', 'dataset', 'sub_agent'].map(source => ({ id: source, source, target: 'agent', targetHandle: source === 'script' ? 'tool' : source === 'sub_agent' ? 'sub_agents' : source })), }) vi.spyOn(experimentsApi, 'getMeasurementCapabilities').mockResolvedValue({ outputs: {} }) vi.spyOn(experimentsApi, 'validateMeasurementPlan').mockResolvedValue({ valid: true, issues: [] }) diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 14cdd6da..3fe1f2d5 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -17,6 +17,9 @@ import { Textarea } from '@/components/ui/textarea' import { ApiError, experimentsApi, protocolsApi, skillsApi } from '@/api/client' import { coordinationStrategyIssues } from '@/lib/coordinationStrategy' import { unboundFactorNames } from '@/lib/factorBindings' +import { scriptFactorIssues } from '@/lib/scriptFactors' +import { toolFactorIssues } from '@/lib/toolFactors' +import { mcpServersApi } from '@/api/client' import { skillFactorIssues } from '@/lib/skillFactors' import { promptReferenceScope } from '@/lib/promptReferences' import { protocolGraphQueryKey } from '@/lib/protocolGraph' @@ -511,6 +514,9 @@ export function DesignTab({ const datasetLibrary = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list(), enabled: factors.some((factor) => factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') }) const datasetIssues = datasetFactorIssues(draftGraph, factors, datasetLibrary.data ? new Set(datasetLibrary.data.map((dataset) => dataset.id)) : undefined) const subAgentIssues = subAgentFactorIssues(draftGraph, factors) + const toolLibrary = useQuery({ queryKey: ['mcp-servers'], queryFn: () => mcpServersApi.list(), enabled: factors.some((factor) => factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle') }) + const scriptIssues = scriptFactorIssues(draftGraph, factors) + const toolIssues = toolFactorIssues(draftGraph, factors, toolLibrary.data ? new Set(toolLibrary.data.map((server) => server.id)) : undefined) const skillIssues = skillFactorIssues(draftGraph, factors, skillLibrary.data ? new Set(skillLibrary.data.map((skill) => skill.id)) : undefined) const impact = impactQuery.data @@ -880,14 +886,14 @@ export function DesignTab({ Unbound factor{unboundFactors.length === 1 ? '' : 's'}: {unboundFactors.join(', ')}. Rebind on the canvas or remove from this design.

)} - {[...subAgentIssues, ...skillIssues, ...datasetIssues, ...knowledgeIssues].map((issue) =>

{issue}

)} + {[...scriptIssues, ...toolIssues, ...subAgentIssues, ...skillIssues, ...datasetIssues, ...knowledgeIssues].map((issue) =>

{issue}

)}

Its declaration remains in Design until you remove or rebind it.

)} :
+

Off: the wired agent never sees this server’s tools at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} {/* Which process this node actually talks to. Most valuable on a client tool -- its endpoint is something a user typed, so it's part of the @@ -226,44 +206,14 @@ export function McpToolNodeInspector({ )}
- {/* The one factor this node type offers beyond Enabled: which of THIS - server's tools the agent may call, varied per cell. The node's - server stays pinned across every level -- see bindableFields.ts's - mcp_tool case for why swapping it isn't offered. The toggle list - below stays live even once bound (it's still this node's base - config, and every other inspector leaves its own bound control - editable too); the levels just override it per cell. */} - bindFactor('config.tool_names', name)} - onUnbind={() => unbindFactor('config.tool_names')} - > - {(trigger) => ( -
- - {tools.length > 0 && ( -
- - -
- )} -
- )} - +
+
{bindings['config.tool_names'] &&

Factor: {bindings['config.tool_names']}

}
+ {tools.length > 0 &&
+ + +
} +
+ {connectorFactorName &&

This allow-list is preserved when the connector selects this node.

} {serversQuery.isLoading ? ( ) : serversQuery.isError ? ( diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 0c4ed985..4f8ef1db 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -4,7 +4,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { MemoryRouter } from 'react-router-dom' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { datasetsApi, experimentsApi, protocolsApi, skillsApi } from '@/api/client' +import { datasetsApi, experimentsApi, mcpServersApi, protocolsApi, skillsApi } from '@/api/client' import { protocolGraphQueryKey } from '@/lib/protocolGraph' import type { Experiment } from '@/types/experiments' import { @@ -210,6 +210,71 @@ describe('ProtocolCanvas connector adds', () => { }) }) + it.each(['tool_selection', 'tool_toggle'] as const)('creates a %s tool factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'mcp_tool', position: { x: 100, y: 0 }, data: { label: `Tool ${id}`, config: { server_id: id, server_name: `Server ${id}`, tool_names: ['run'] } } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'tool_toggle' ? 'All agent tools on/off' : 'Tool levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Tools', level_type: mode, levels: mode === 'tool_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'tool_toggle' ? ['All enabled', 'All disabled'] : ['Tool a', 'Tool b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ tool_selection: 'Writer:Tools' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.tool_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'mcp_tool')).toHaveLength(2) + }) + }) + + it.each(['script_selection', 'script_toggle'] as const)('creates a %s script factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'script', position: { x: 100, y: 0 }, data: { label: `Script ${id}`, config: { name: `Script ${id}`, language: 'python', code: 'print(1)' } } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'script_toggle' ? 'All agent scripts on/off' : 'Script levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Scripts', level_type: mode, levels: mode === 'script_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'script_toggle' ? ['All enabled', 'All disabled'] : ['Script a', 'Script b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ script_selection: 'Writer:Scripts' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.script_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'script')).toHaveLength(2) + }) + }) + it('creates and removes a whole-pattern factor from the Pattern toolbar, clearing the Agent override', async () => { let experiment: Experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index e6b1ee96..a3f4905d 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -17,6 +17,14 @@ import { DatasetRowSelector } from './DatasetRowSelector' import { useDatasetRowSelection } from './useDatasetRowSelection' import { rowBindingForNode } from '@/lib/datasetRows' import { reconcileSkillFactor, skillFactorConflict, skillFactorOwner, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import { reconcileToolFactor, toolFactorConflict, toolFactorOwner, toolFactorPath, isIndividualToolMode, TOOL_FACTOR_PATH, type ToolFactorMode } from '@/lib/toolFactors' +import { reconcileScriptFactor, scriptFactorConflict, scriptFactorOwner, scriptFactorPath, isIndividualScriptMode, SCRIPT_FACTOR_PATH, type ScriptFactorMode } from '@/lib/scriptFactors' +import { ToolFactorDialog } from './ToolFactorDialog' +import { ScriptFactorDialog } from './ScriptFactorDialog' +import { ToolTestSelectors } from './ToolTestSelectors' +import { ScriptTestSelectors } from './ScriptTestSelectors' +import { useToolTestSelection } from './useToolTestSelection' +import { useScriptTestSelection } from './useScriptTestSelection' import { SkillFactorDialog } from './SkillFactorDialog' import { SkillTestSelectors } from './SkillTestSelectors' import { useSkillTestSelection } from './useSkillTestSelection' @@ -425,11 +433,13 @@ export const ProtocolCanvas = forwardRef { const knowledgeIds = new Set(edges.filter((edge) => edge.targetHandle === 'knowledge' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH] === factorName).map((edge) => edge.source)) const subAgentIds = new Set(edges.filter((edge) => edge.targetHandle === 'sub_agents' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SUB_AGENT_FACTOR_PATH] === factorName).map((edge) => edge.source)) + const toolIds = new Set(edges.filter((edge) => edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => MCP_TOOL_NODE_TYPES.includes(nds.find((node) => node.id === id)?.type ?? ''))) + const scriptIds = new Set(edges.filter((edge) => edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => nds.find((node) => node.id === id)?.type === 'script')) const skillIds = new Set(edges.filter((edge) => edge.targetHandle === 'skill' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factorName).map((edge) => edge.source)) const datasetIds = new Set(edges.filter((edge) => ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '') && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factorName).map((edge) => edge.source)) return nds.map((n) => { if (subAgentIds.has(n.id)) return { ...n, data: { ...n.data, active: true } } - if (skillIds.has(n.id) || datasetIds.has(n.id) || knowledgeIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } + if (scriptIds.has(n.id) || toolIds.has(n.id) || skillIds.has(n.id) || datasetIds.has(n.id) || knowledgeIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } const bindings = n.data.factor_bindings as Record | undefined if (!bindings || !Object.values(bindings).includes(factorName)) return n const next = Object.fromEntries(Object.entries(bindings).filter(([, name]) => name !== factorName)) @@ -442,6 +452,14 @@ export const ProtocolCanvas = forwardRef() const [modelFactorLevelIndex, setModelFactorLevelIndex] = useState() const [subAgentFactorNodeId, setSubAgentFactorNodeId] = useState(null) + const [toolFactorNodeId, setToolFactorNodeId] = useState(null) + const [scriptFactorNodeId, setScriptFactorNodeId] = useState(null) const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) const [datasetFactorNodeId, setDatasetFactorNodeId] = useState(null) const [connectorFactorChoice, setConnectorFactorChoice] = useState() @@ -652,12 +674,14 @@ export const ProtocolCanvas = forwardRef { if (draftRow.error) throw new Error(draftRow.error) if (draftSubAgents.error) throw new Error(draftSubAgents.error) + if (draftScripts.error) throw new Error(draftScripts.error) + if (draftTools.error) throw new Error(draftTools.error) if (draftSkills.error) throw new Error(draftSkills.error) if (draftKnowledge.error) throw new Error(draftKnowledge.error) if (draftDatasets.error) throw new Error(draftDatasets.error) await protocolsApi.update(protocolId, { graph: toPersistedGraph(nodes, edges) }) const published = await protocolsApi.publish(protocolId) - return { published, scope, rowIndex, skillOptions: { ...draftSubAgents.options, ...draftSkills.options, ...draftKnowledge.options, ...draftDatasets.options } } + return { published, scope, rowIndex, skillOptions: { ...draftSubAgents.options, ...draftScripts.options, ...draftTools.options, ...draftSkills.options, ...draftKnowledge.options, ...draftDatasets.options } } }, onSuccess: async ({ published, scope, rowIndex, skillOptions }) => { queryClient.invalidateQueries({ queryKey: ['protocols', protocolId, 'revisions'] }) @@ -691,7 +715,11 @@ export const ProtocolCanvas = forwardRef { if (publishedSourceError || (publishedRow.binding && publishedRow.error)) throw new Error(publishedSourceError ?? publishedRow.error!) if (publishedSubAgents.error) throw new Error(publishedSubAgents.error) + if (publishedScripts.error) throw new Error(publishedScripts.error) + if (publishedTools.error) throw new Error(publishedTools.error) if (publishedSkills.error) throw new Error(publishedSkills.error) if (publishedKnowledge.error) throw new Error(publishedKnowledge.error) if (publishedDatasets.error) throw new Error(publishedDatasets.error) - return protocolsApi.testRun(protocolId, publishedRow.binding || publishedSubAgents.owners.length || publishedSkills.owners.length || publishedKnowledge.owners.length || publishedDatasets.owners.length ? { ...(publishedRow.binding ? { row_index: rowIndex } : {}), ...publishedSubAgents.options, ...publishedSkills.options, ...publishedKnowledge.options, ...publishedDatasets.options } : undefined) + return protocolsApi.testRun(protocolId, publishedRow.binding || publishedSubAgents.owners.length || publishedScripts.owners.length || publishedTools.owners.length || publishedSkills.owners.length || publishedKnowledge.owners.length || publishedDatasets.owners.length ? { ...(publishedRow.binding ? { row_index: rowIndex } : {}), ...publishedSubAgents.options, ...publishedScripts.options, ...publishedTools.options, ...publishedSkills.options, ...publishedKnowledge.options, ...publishedDatasets.options } : undefined) }, onSuccess: (run) => { setRunId(run.id) @@ -728,10 +758,12 @@ export const ProtocolCanvas = forwardRef { setRunId(run.id) @@ -1196,6 +1228,10 @@ export const ProtocolCanvas = forwardRef factor.name === (n.data.factor_bindings as Record | undefined)?.[SUB_AGENT_FACTOR_PATH])?.levels.length, subAgentFactorControlled: n.type === 'sub_agent' && !!subAgentFactorOwner(toPersistedGraph(nodes, edges), n.id), + toolFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH])?.levels.length, + scriptFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH])?.levels.length, + toolFactorControlled: MCP_TOOL_NODE_TYPES.includes(n.type ?? '') && !!toolFactorOwner(toPersistedGraph(nodes, edges), n.id), + scriptFactorControlled: n.type === 'script' && !!scriptFactorOwner(toPersistedGraph(nodes, edges), n.id), skillFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH])?.levels.length, skillFactorControlled: n.type === 'skill' && !!skillFactorOwner(toPersistedGraph(nodes, edges), n.id), datasetFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH])?.levels.length, @@ -1434,6 +1470,18 @@ export const ProtocolCanvas = forwardRef { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setToolFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestScriptFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setScriptFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) const requestKnowledgeFactor = useCallback((nodeId: string) => { if (!experimentId || experimentLocked) return setSelectedNodeId(null) @@ -1533,6 +1581,8 @@ export const ProtocolCanvas = forwardRef { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle') && (node.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.tool_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, tool_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'script_selection' || factor.level_type === 'script_toggle') && (node.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.script_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, script_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) useEffect(() => { if (experimentLocked) return setNodes((current) => current.map((node) => { @@ -2416,8 +2484,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2505,8 +2572,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2600,13 +2666,17 @@ export const ProtocolCanvas = forwardRef publishAndRunMutation.mutate({ scope: pendingRunConfirm, rowIndex: draftRow.rowIndex })} confirmLabel={pendingRunConfirm.type === 'graph' ? 'Test Run' : undefined} - confirmDisabled={!!publishedSourceError || !!publishedSubAgents.error || !!publishedSkills.error || !!publishedKnowledge.error || !!publishedDatasets.error || (pendingHasRow && !!publishedRow.error)} - publishDisabled={!!draftRow.error || !!draftSubAgents.error || !!draftSkills.error || !!draftKnowledge.error || !!draftDatasets.error} + confirmDisabled={!!publishedSourceError || !!publishedSubAgents.error || !!publishedScripts.error || !!publishedTools.error || !!publishedSkills.error || !!publishedKnowledge.error || !!publishedDatasets.error || (pendingHasRow && !!publishedRow.error)} + publishDisabled={!!draftRow.error || !!draftSubAgents.error || !!draftScripts.error || !!draftTools.error || !!draftSkills.error || !!draftKnowledge.error || !!draftDatasets.error} additionalContent={
{publishedSubAgents.owners.length > 0 &&

Published Sub-Agents · run published v{publishedRevision}

} {showDraftInputs && draftSubAgents.owners.length > 0 &&

Draft Sub-Agents · publish and run

} {publishedSkills.owners.length > 0 &&

Published skills · run published v{publishedRevision}

} + {publishedTools.owners.length > 0 &&

Published tools · run published v{publishedRevision}

} + {publishedScripts.owners.length > 0 &&

Published scripts · run published v{publishedRevision}

} {showDraftInputs && draftSkills.owners.length > 0 &&

Draft skills · publish and run

} + {showDraftInputs && draftTools.owners.length > 0 &&

Draft tools · publish and run

} + {showDraftInputs && draftScripts.owners.length > 0 &&

Draft scripts · publish and run

} {publishedKnowledge.owners.length > 0 &&

Published knowledge · run published v{publishedRevision}

} {showDraftInputs && draftKnowledge.owners.length > 0 &&

Draft knowledge · publish and run

} {publishedDatasets.owners.length > 0 &&

Published datasets · run published v{publishedRevision}

} @@ -2631,7 +2701,9 @@ export const ProtocolCanvas = forwardRef} + {toolFactorNodeId && setToolFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = toolFactorConflict(graph, fresh.design_spec?.factors ?? [], toolFactorNodeId, factor.level_type as ToolFactorMode, ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = isIndividualToolMode(factor.level_type as ToolFactorMode) ? factor : reconcileToolFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : next.level_type === 'tool_names' ? { config: { ...(node.data.config as object), tool_names: next.levels[0] ?? [] } } : { tool_selection: next.levels[0] ?? [], tool_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [toolFactorPath(next.level_type as ToolFactorMode)]: next.name }, + } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individualPath = Object.entries(graph.nodes.find((node) => node.id === toolFactorNodeId)?.data.factor_bindings ?? {}).find(([, value]) => value === name)?.[0] + const individual = !!individualPath + const shared = individual && graph.nodes.some((node) => node.id !== toolFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === toolFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== individualPath)) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {scriptFactorNodeId && setScriptFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = scriptFactorConflict(graph, fresh.design_spec?.factors ?? [], scriptFactorNodeId, factor.level_type as ScriptFactorMode, ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = isIndividualScriptMode(factor.level_type as ScriptFactorMode) ? factor : reconcileScriptFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { script_selection: next.levels[0] ?? [], script_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [scriptFactorPath(next.level_type as ScriptFactorMode)]: next.name }, + } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individualPath = Object.entries(graph.nodes.find((node) => node.id === scriptFactorNodeId)?.data.factor_bindings ?? {}).find(([, value]) => value === name)?.[0] + const individual = !!individualPath + const shared = individual && graph.nodes.some((node) => node.id !== scriptFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === scriptFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== individualPath)) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} {knowledgeFactorNodeId && void modelFactorForNode?: (nodeId: string) => DesignFactor | undefined requestSubAgentFactor?: (nodeId: string) => void + requestScriptFactor?: (nodeId: string) => void + requestToolFactor?: (nodeId: string) => void requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). diff --git a/frontend/src/components/protocol/ScriptFactorDialog.test.tsx b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx new file mode 100644 index 00000000..f63a7e82 --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx @@ -0,0 +1,84 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { ScriptFactorDialog } from './ScriptFactorDialog' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'script', type: 'script', data: { label: 'Summarize', config: { name: 'Scoring', language: 'python', code: 'print(1)' }, factor_bindings: { 'config.enabled': 'Script enabled' } } }, + ], + edges: [{ source: 'script', target: 'agent', targetHandle: 'tool' }], +} as unknown as ProtocolGraph + +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Script enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Script levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + +it.each(['Script levels', 'All agent scripts on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Script enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Script’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This script on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Script enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected scripts', () => { + render() + expect(screen.getByRole('button', { name: 'This script on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Script levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Script to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('blocks off treatments for a Tool Step that requires script_code', () => { + const required = { ...graph, nodes: [ + { id: 'step', type: 'tool_step', data: { label: 'Fit model', config: { arguments: { code: { source: 'script_code' } } } } }, + { ...graph.nodes[1], data: { ...graph.nodes[1].data, factor_bindings: {} } }, + ], edges: [{ source: 'script', target: 'step', targetHandle: 'tool' }] } as unknown as ProtocolGraph + render() + expect(screen.getByRole('status')).toHaveTextContent('cannot include off') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This script on/off', 'Script levels'])('blocks %s when an Agent already has an all-scripts factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { script_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { name: 'Scoring', language: 'python', code: 'print(1)' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) diff --git a/frontend/src/components/protocol/ScriptFactorDialog.tsx b/frontend/src/components/protocol/ScriptFactorDialog.tsx new file mode 100644 index 00000000..a76a5934 --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorDialog.tsx @@ -0,0 +1,124 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedScripts, reconcileScriptFactor, requiredScriptIssues, scriptFactorConflict, scriptFactorModes as modes, SCRIPT_FACTOR_PATH, isIndividualScriptMode, scriptFactorPath, type ScriptFactorMode as Mode } from '@/lib/scriptFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, initialAgentId, onClose, onSave, onRemove }: { + scriptNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + initialAgentId?: string + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const script = graph.nodes.find((node) => node.id === scriptNodeId)! + const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === scriptNodeId && edge.target === node.id && edge.targetHandle === 'tool')) + const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((agent) => agent.data.factor_bindings?.[SCRIPT_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, SCRIPT_FACTOR_PATH) ?? existingFor(scriptNodeId, 'config.enabled') + const initial = initialFactor && !isIndividualScriptMode(initialFactor.level_type as Mode) && initialFactor.level_type ? reconcileScriptFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = isIndividualScriptMode(nextMode) ? scriptNodeId : nextAgentId + const existing = existingFor(ownerId, scriptFactorPath(nextMode)) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level, index) => nextMode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Script'), nextMode === 'boolean' ? 'Enabled' : 'Scripts', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return isIndividualScriptMode(nextMode) ? factor : reconcileScriptFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`) } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = isIndividualScriptMode(mode) ? scriptNodeId : agentId + const current = existingFor(ownerId, scriptFactorPath(mode)) + const affected = isIndividualScriptMode(mode) ? [script] : connectedScripts(graph, agentId) + const unavailable = (value: Mode) => !isIndividualScriptMode(value) && !agentId + ? `${modes[value]} is unavailable because this Script is not connected to an Agent. Connect this Script to an Agent first.` + : scriptFactorConflict(graph, factors, scriptNodeId, value, agentId) + const previewGraph = { ...graph, nodes: graph.nodes.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, factor_bindings: { ...node.data.factor_bindings, [scriptFactorPath(mode)]: draft.name } } } : node) } + const requiredIssues = requiredScriptIssues(previewGraph, [...factors.filter((factor) => factor.name !== current?.name), draft]) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setAgentId(nextAgentId) + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the script factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Script factor + Choose how scripts vary across experimental cells. + +
+ {agents.length > 1 &&
} +
{(Object.keys(modes) as Mode[]).map((value) => )}
+ {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Script node.' : mode === 'script_selection' ? 'Each cell receives exactly one connected script.' : 'Each cell enables every connected script, including deactivated scripts, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {!isIndividualScriptMode(mode) && affected.length < (mode === 'script_toggle' ? 1 : 2) &&

Connect at least {mode === 'script_toggle' ? 'one script' : 'two different scripts'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'script_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable script')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+

Agent-level factors follow current Script connections. Changes require design review and regeneration.

+ {requiredIssues.map((issue) =>

{issue}

)} + {current && } + } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ScriptFactorEditor.tsx b/frontend/src/components/protocol/ScriptFactorEditor.tsx new file mode 100644 index 00000000..ad96c93d --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorEditor.tsx @@ -0,0 +1,74 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function ScriptFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the script factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'script_toggle' ? 'All scripts on/off' : 'Scripts as levels'} + {factor.level_type === 'script_toggle' ? 'Each cell receives all connected scripts or none.' : 'Each cell receives exactly one connected script.'} Add or disconnect Script nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different scripts before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const script = graph?.nodes.find((node) => node.id === id) + return
+
+

{factor.level_type === 'script_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : script?.data.label ?? labels[index] ?? 'Unavailable script'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'script_toggle' && !!graph && !script &&

Unavailable: disconnect or restore this script.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+

Individual script switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected scripts and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ScriptNodeInspector.tsx b/frontend/src/components/protocol/ScriptNodeInspector.tsx index 8f8594e2..990f4ffa 100644 --- a/frontend/src/components/protocol/ScriptNodeInspector.tsx +++ b/frontend/src/components/protocol/ScriptNodeInspector.tsx @@ -1,11 +1,13 @@ -import { Code2 } from 'lucide-react' +import { Code2, Power, Split, Trash2 } from 'lucide-react' import { nodeAccent } from '@/lib/nodeAccent' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' +import { Button } from '@/components/ui/button' +import { Switch } from '@/components/ui/switch' import { NodeInspectorDialog } from './NodeInspectorDialog' import { PythonCodeEditor } from './PythonCodeEditor' +import { useProtocolCanvasActions } from './ProtocolCanvasContext' import type { ScriptNodeConfig, ScriptNodeData, ProtocolNode } from '@/types/protocols' const ACCENT = nodeAccent('script') @@ -13,28 +15,22 @@ const ACCENT = nodeAccent('script') // Same floating-dialog shell as every other node inspector. Python-only for // v1 (see ScriptNodeData's own comment in types/protocols.ts) -- "Language" // is a fixed label, not a picker, so there's nothing to configure there yet. -// The whole node is also factor-bindable (bindableFields.ts's 'script_config' -// kind) -- comparing two hand-written scoring scripts as an experimental -// factor is a direct use of the same whole-node-config mechanism model_config/ -// tool_config/pattern already have. +// Script factor creation lives in the hover toolbar. The inspector edits the +// base code and exposes existing factor links and connector-controlled availability. export function ScriptNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: ScriptNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's - // agentTracedLabel) -- distinct from data.label, which is this node's own - // plain label shown in the header title. - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: ScriptNodeData) => void onDelete: (nodeId: string) => void onClose: () => void }) { + const { requestEditFactor } = useProtocolCanvasActions() if (!node) return null const data = node.data const config = data.config @@ -44,10 +40,6 @@ export function ScriptNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -70,6 +62,24 @@ export function ScriptNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > +
+

+ Make factor has moved to the node toolbar. Hover over the Script node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this script at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} +
@@ -91,31 +101,7 @@ export function ScriptNodeInspector({ />
- bindFactor('config', name)} - onUnbind={() => unbindFactor('config')} - > - {(trigger) => ( -
- - patchConfig({ code })} - height="max(12rem, calc(100vh - 22rem))" - /> -
- )} - +
{bindings.config && } patchConfig({ code })} height="max(12rem, calc(100vh - 22rem))" />
) diff --git a/frontend/src/components/protocol/ScriptTestSelectors.tsx b/frontend/src/components/protocol/ScriptTestSelectors.tsx new file mode 100644 index 00000000..f1f56d78 --- /dev/null +++ b/frontend/src/components/protocol/ScriptTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedScripts, scriptId } from '@/lib/scriptFactors' +import type { useScriptTestSelection } from './useScriptTestSelection' + +export function ScriptTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedScripts(selection.graph!, node.id) + if (node.data.script_factor_mode === 'script_toggle') return
+

{node.data.label}: scripts for this test

+ +
+ return
+

{node.data.label}: script for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/ToolFactorDialog.test.tsx b/frontend/src/components/protocol/ToolFactorDialog.test.tsx new file mode 100644 index 00000000..bd080ac1 --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorDialog.test.tsx @@ -0,0 +1,114 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { ToolFactorDialog } from './ToolFactorDialog' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { mcpServersApi } from '@/api/client' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'tool', type: 'mcp_tool', data: { label: 'Summarize', config: { server_id: 'a' }, factor_bindings: { 'config.enabled': 'Tool enabled' } } }, + ], + edges: [{ source: 'tool', target: 'agent', targetHandle: 'tool' }], +} as unknown as ProtocolGraph + +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Tool enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Tool levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + +it.each(['Tool levels', 'All agent tools on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Tool enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Tool’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This tool on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Tool enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected tools', () => { + render() + expect(screen.getByRole('button', { name: 'This tool on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Tool levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Tool to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This tool on/off', 'Tool levels', 'Tools allowed'])('blocks %s when an Agent already has an all-tools factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { tool_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { server_id: 'a' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('edits allow-list levels while keeping the server pinned', async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([{ id: 'a', capabilities: { tools: [{ name: 'search' }, { name: 'fetch' }] } }] as Awaited>) + const save = vi.fn().mockResolvedValue(undefined) + const unbound = { ...graph, nodes: graph.nodes.map((node) => node.id === 'tool' ? { ...node, data: { label: 'Search server', config: { server_id: 'a', tool_names: ['search'] } } } : node) } as ProtocolGraph + render() + await screen.findAllByText('fetch') + fireEvent.click(screen.getAllByRole('switch')[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ level_type: 'tool_names', levels: [['search', 'fetch'], []] }), 'tool', undefined)) + expect((unbound.nodes[1].data.config as { server_id: string }).server_id).toBe('a') + client.clear() +}) + +it('saves edits to an existing individual factor without removing other declarations', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Renamed' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Renamed', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'tool', 'Tool enabled')) +}) + +it('opens the existing group factor when a shared tool has multiple connected agents', () => { + const sharedGraph = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'First agent' } }, + { id: 'second', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Second agent', tool_selection: ['a'], tool_factor_mode: 'tool_toggle', factor_bindings: { tool_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { server_id: 'a' } } }, + ], + edges: [...graph.edges, { id: 'second-edge', source: 'tool', target: 'second', targetHandle: 'tool' }], + } as ProtocolGraph + render() + expect(screen.getByRole('combobox')).toHaveTextContent('Second agent') + expect(screen.getByRole('button', { name: 'All agent tools on/off' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByLabelText('Factor name')).toHaveValue('Group') +}) diff --git a/frontend/src/components/protocol/ToolFactorDialog.tsx b/frontend/src/components/protocol/ToolFactorDialog.tsx new file mode 100644 index 00000000..2916763e --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorDialog.tsx @@ -0,0 +1,125 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split, Trash2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedTools, reconcileToolFactor, toolFactorConflict, toolFactorModes as modes, TOOL_FACTOR_PATH, isIndividualToolMode, toolFactorPath, type ToolFactorMode as Mode } from '@/lib/toolFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { ToolNamesLevelRow } from './FactorEditorDialog' +import { computeFactorName } from './factorLevels' + +export function ToolFactorDialog({ toolNodeId, graph, factors, initialMode, initialAgentId, onClose, onSave, onRemove }: { + toolNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + initialAgentId?: string + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const tool = graph.nodes.find((node) => node.id === toolNodeId)! + const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === toolNodeId && edge.target === node.id && edge.targetHandle === 'tool')) + const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((agent) => agent.data.factor_bindings?.[TOOL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, TOOL_FACTOR_PATH) ?? existingFor(toolNodeId, 'config.enabled') ?? existingFor(toolNodeId, 'config.tool_names') + const initial = initialFactor && !isIndividualToolMode(initialFactor.level_type as Mode) && initialFactor.level_type ? reconcileToolFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = isIndividualToolMode(nextMode) ? toolNodeId : nextAgentId + const existing = existingFor(ownerId, toolFactorPath(nextMode)) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level, index) => nextMode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Tool'), nextMode === 'boolean' ? 'Enabled' : nextMode === 'tool_names' ? 'Tools allowed' : 'Tools', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : nextMode === 'tool_names' ? [(tool.data.config as { tool_names?: string[] }).tool_names ?? [], []] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : nextMode === 'tool_names' ? ['Allowed tools', 'No tools'] : undefined, + } + return isIndividualToolMode(nextMode) ? factor : reconcileToolFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`) } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = isIndividualToolMode(mode) ? toolNodeId : agentId + const current = existingFor(ownerId, toolFactorPath(mode)) + const affected = isIndividualToolMode(mode) ? [tool] : connectedTools(graph, agentId) + const unavailable = (value: Mode) => !isIndividualToolMode(value) && !agentId + ? `${modes[value]} is unavailable because this Tool is not connected to an Agent. Connect this Tool to an Agent first.` + : toolFactorConflict(graph, factors, toolNodeId, value, agentId) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setAgentId(nextAgentId) + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the tool factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Tool factor + Choose how tools vary across experimental cells. + +
+ {agents.length > 1 &&
} +
{(Object.keys(modes) as Mode[]).map((value) => )}
+ {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Tool node.' : mode === 'tool_names' ? 'Vary the allowed tools of this node’s pinned server.' : mode === 'tool_selection' ? 'Each cell receives exactly one connected tool.' : 'Each cell enables every connected tool, including deactivated tools, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {!isIndividualToolMode(mode) && affected.length < (mode === 'tool_toggle' ? 1 : 2) &&

Connect at least {mode === 'tool_toggle' ? 'one tool' : 'two different tools'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'tool_names' ? `${(level as string[]).length} allowed tools` : mode === 'tool_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable tool')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ {mode === 'tool_names' && setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} />} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} + {mode === 'tool_names' && } +
)}
+ {mode === 'tool_names' && } +

Agent-level factors follow current Tool connections. Changes require design review and regeneration.

+ {current && } + } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ToolFactorEditor.tsx b/frontend/src/components/protocol/ToolFactorEditor.tsx new file mode 100644 index 00000000..101cd070 --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorEditor.tsx @@ -0,0 +1,74 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function ToolFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const [draft, setDraft] = useState(factor) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the tool factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'tool_toggle' ? 'All tools on/off' : 'Tools as levels'} + {factor.level_type === 'tool_toggle' ? 'Each cell receives all connected tools or none.' : 'Each cell receives exactly one connected tool.'} Add or disconnect Tool nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different tools before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const tool = graph?.nodes.find((node) => node.id === id) + return
+
+

{factor.level_type === 'tool_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : tool?.data.label ?? labels[index] ?? 'Unavailable tool'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'tool_toggle' && !!graph && !tool &&

Unavailable: disconnect or restore this tool.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+

Individual tool switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected tools and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ToolTestSelectors.tsx b/frontend/src/components/protocol/ToolTestSelectors.tsx new file mode 100644 index 00000000..b901d70e --- /dev/null +++ b/frontend/src/components/protocol/ToolTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedTools, toolId } from '@/lib/toolFactors' +import type { useToolTestSelection } from './useToolTestSelection' + +export function ToolTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedTools(selection.graph!, node.id) + if (node.data.tool_factor_mode === 'tool_toggle') return
+

{node.data.label}: tools for this test

+ +
+ return
+

{node.data.label}: tool for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 1fa9202f..6022bfe4 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -275,11 +275,9 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { // listed by factorCreationFields and edited in the dedicated dialog. return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'script': - // The whole node as a factor -- levels are entirely different scripts. - // _resolve_script_configs reads each wired script node's whole config - // verbatim, so comparing two scoring scripts already works with zero - // backend changes, same reasoning as model_config/tool_config. - return [{ fieldPath: 'config', label: 'Script', levelType: 'script_config' }] + // Keep config discoverable for editing existing variant bindings. + // New factors compare separate Script nodes through Script levels. + return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }, { fieldPath: 'config', label: 'Script variants (existing factors)', levelType: 'script_config' }] default: return [] } @@ -287,7 +285,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { export interface UnboundField { pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } - connectorFactor?: { kind: 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } + connectorFactor?: { kind: 'script' | 'tool' | 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -468,7 +466,7 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField[] { const fields = unboundBindableFields(nodes, edges).filter((field) => { const kind = nodes.find((node) => node.id === field.nodeId)?.type - return kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' + return !['script', 'mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool'].includes(kind ?? '') && kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' && !(kind === 'sub_agent' && field.fieldPath === 'active') && !isPatternNode(kind) && field.fieldPath !== PATTERN_FACTOR_PATH }) @@ -505,15 +503,16 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField } } for (const node of nodes) { - if (!['skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue - const kind = (node.type === 'okf_bundle' || node.type === 'okf_document' ? 'knowledge' : node.type) as 'skill' | 'dataset' | 'knowledge' - const handles = kind === 'skill' ? ['skill'] : kind === 'knowledge' ? ['knowledge'] : ['dataset', 'resource', 'tool'] + if (!['script', 'mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool', 'skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue + const kind = (node.type === 'okf_bundle' || node.type === 'okf_document' ? 'knowledge' : ['mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool'].includes(node.type ?? '') ? 'tool' : node.type) as 'script' | 'tool' | 'skill' | 'dataset' | 'knowledge' + const handles = (kind === 'tool' || kind === 'script') ? ['tool'] : kind === 'skill' ? ['skill'] : kind === 'knowledge' ? ['knowledge'] : ['dataset', 'resource', 'tool'] const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && handles.includes(edge.targetHandle ?? ''))) if (!(node.data.factor_bindings as Record | undefined)?.['config.enabled']) fields.push({ nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.enabled', fieldLabel: `This ${kind} on/off`, levelType: 'boolean', currentValue: undefined, connectorFactor: { kind, nodeId: node.id, mode: 'boolean' }, }) + if (kind === 'tool' && !(node.data.factor_bindings as Record | undefined)?.['config.tool_names']) fields.push({ nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.tool_names', fieldLabel: 'Tools allowed', levelType: 'tool_names', currentValue: (node.data.config as { tool_names?: string[] })?.tool_names, connectorFactor: { kind, nodeId: node.id, mode: 'tool_names' } }) for (const owner of owners) { const key = `${owner.id}:${kind}` if (connectorOwners.has(key)) continue @@ -522,7 +521,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const levelType = `${kind}_${mode}` as LevelType fields.push({ nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: `${kind}_selection`, - fieldLabel: mode === 'selection' ? `${kind === 'skill' ? 'Skill' : kind === 'knowledge' ? 'Knowledge' : 'Dataset'} levels` : `All ${kind === 'knowledge' ? 'knowledge' : `${kind}s`} on/off`, + fieldLabel: mode === 'selection' ? `${kind === 'script' ? 'Script' : kind === 'tool' ? 'Tool' : kind === 'skill' ? 'Skill' : kind === 'knowledge' ? 'Knowledge' : 'Dataset'} levels` : `All ${kind === 'script' ? 'agent scripts' : kind === 'tool' ? 'agent tools' : kind === 'knowledge' ? 'knowledge' : `${kind}s`} on/off`, levelType, currentValue: undefined, connectorFactor: { kind, nodeId: node.id, mode: levelType, agentId: owner.id }, }) @@ -540,7 +539,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField const agentKinds = ['agent', 'sub_agent'] const targets = nodes.filter((candidate) => agentKinds.includes(candidate.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === candidate.id)) const owner = field.connectorFactor?.kind === 'sub_agent' && field.connectorFactor.mode === 'boolean' ? targets.length === 1 ? targets[0] : undefined : agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined - const category = field.connectorFactor?.kind === 'sub_agent' ? 'Sub-Agents' + const category = field.connectorFactor?.kind === 'script' ? 'Scripts' : field.connectorFactor?.kind === 'tool' ? 'Tools' : field.connectorFactor?.kind === 'sub_agent' ? 'Sub-Agents' : field.connectorFactor?.kind === 'skill' ? 'Skills' : field.connectorFactor?.kind === 'dataset' ? 'Datasets' : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' @@ -550,7 +549,7 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField : MODEL_NODE_TYPES.has(kind) ? 'Model' : kind.startsWith('pattern_') ? 'Pattern' : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', mcp_scikit_learn: 'Tools', mcp_client_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? String(node.data.label || kind) - const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' + const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' && field.connectorFactor.mode !== 'tool_names' && field.connectorFactor.mode !== 'script_config' return { ...field, pickerGroup: { diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index f732a188..f909f5c2 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -21,6 +21,10 @@ export type LevelType = | 'script_config' | 'dataset_config' | 'tool_names' + | 'script_selection' + | 'script_toggle' + | 'tool_selection' + | 'tool_toggle' | 'skill_selection' | 'skill_toggle' | 'sub_agent_selection' @@ -38,9 +42,13 @@ export const LEVEL_TYPE_LABELS: Record = { model_config: 'Provider & model', tool_config: 'Server & tools', pattern: 'Execution pattern', - script_config: 'Script', + script_config: 'Script variants (legacy)', dataset_config: 'Dataset', tool_names: 'Tools allowed', + script_selection: 'Script levels', + script_toggle: 'All agent scripts on/off', + tool_selection: 'Tool levels', + tool_toggle: 'All agent tools on/off', skill_selection: 'Skills as levels', skill_toggle: 'All skills on/off', sub_agent_selection: 'Sub-Agent levels', @@ -64,6 +72,10 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'script_config' || type === 'dataset_config' || type === 'tool_names' || + type === 'script_selection' || + type === 'script_toggle' || + type === 'tool_selection' || + type === 'tool_toggle' || type === 'skill_selection' || type === 'skill_toggle' || type === 'sub_agent_selection' || diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index 0b6bf7a6..a873dc1d 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -82,7 +82,7 @@ export function AgentNode({ ...(peerNeedsToolCalling ? ["This model can't call tools, so this agent can't consult or delegate to connected agents"] : []), ] const { updateNodeData } = useReactFlow() - const { requestRunNode, requestSubAgentFactor, experimentLocked } = useProtocolCanvasActions() + const { requestRunNode, requestSubAgentFactor, requestEditFactor, experimentLocked } = useProtocolCanvasActions() const controlled = isSubAgent && !!data.subAgentFactorControlled const isActive = controlled || (data.active ?? true) const factorCount = boundFactorCount(data) + (controlled ? 1 : 0) @@ -405,7 +405,8 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Tool - + + {data.factor_bindings?.script_selection && } {/* Output Parser -- the field spec the agent's answer is written to and read back out of. Last on the bottom edge, at 95%: it's the only connector here whose work outlives the agent's own turn, so it sits diff --git a/frontend/src/components/protocol/nodes/McpClientToolNode.tsx b/frontend/src/components/protocol/nodes/McpClientToolNode.tsx index 913e8a04..87d3b3dd 100644 --- a/frontend/src/components/protocol/nodes/McpClientToolNode.tsx +++ b/frontend/src/components/protocol/nodes/McpClientToolNode.tsx @@ -3,6 +3,7 @@ import { useReactFlow, type NodeProps } from '@xyflow/react' import { Plug } from 'lucide-react' import type { McpToolNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' // Its own hue, distinct from McpToolNode's: "a server the deployment @@ -21,7 +22,9 @@ export function McpClientToolNode({ id, data, selected }: NodeProps & { data: Mc const toolNames = data.config?.tool_names ?? [] const summary = toolNames.length > 0 ? `${data.config.server_name ?? '?'}: ${toolNames.join(', ')}` : null const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestToolFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.toolFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) const allowListIsFactor = !!data.factor_bindings?.['config.tool_names'] return ( @@ -37,10 +40,11 @@ export function McpClientToolNode({ id, data, selected }: NodeProps & { data: Mc // too, so the allow-list is the only thing that can be empty -- and // same suppression once that allow-list is itself a factor. warning={summary || allowListIsFactor ? undefined : 'Not configured -- allow at least one tool'} - factorCount={boundFactorCount(data)} + factorCount={controlled ? 1 : boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestToolFactor ? () => requestToolFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/nodes/McpToolNode.tsx b/frontend/src/components/protocol/nodes/McpToolNode.tsx index 12adf198..407a3111 100644 --- a/frontend/src/components/protocol/nodes/McpToolNode.tsx +++ b/frontend/src/components/protocol/nodes/McpToolNode.tsx @@ -3,6 +3,7 @@ import { useReactFlow, type NodeProps } from '@xyflow/react' import { Wrench } from 'lucide-react' import type { McpToolNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' // One hue for the kind, not per instance: all mcp_tool nodes share this, and @@ -23,7 +24,9 @@ export function McpToolNode({ id, data, selected }: NodeProps & { data: McpToolN const toolNames = data.config?.tool_names ?? [] const summary = toolNames.length > 0 ? `${data.config.server_name ?? '?'}: ${toolNames.join(', ')}` : null const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestToolFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.toolFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) // Every MCP node already HAS its server (picked in the browser at creation // time), so telling its user to go pick one would send them looking for a // dropdown that isn't there -- only the allow-list can be empty. Not shown @@ -43,10 +46,11 @@ export function McpToolNode({ id, data, selected }: NodeProps & { data: McpToolN placeholder="MCP Tool" handleId="tool" warning={summary ? undefined : warning} - factorCount={boundFactorCount(data)} + factorCount={controlled ? 1 : boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestToolFactor ? () => requestToolFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/nodes/ScriptNode.tsx b/frontend/src/components/protocol/nodes/ScriptNode.tsx index 888472b7..7d88610e 100644 --- a/frontend/src/components/protocol/nodes/ScriptNode.tsx +++ b/frontend/src/components/protocol/nodes/ScriptNode.tsx @@ -1,13 +1,13 @@ import { nodeAccent } from '@/lib/nodeAccent' -import type { NodeProps } from '@xyflow/react' +import { useReactFlow, type NodeProps } from '@xyflow/react' import { Code2 } from 'lucide-react' import type { ScriptNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' -// Carries a fixed piece of code an Agent passes verbatim as some tool's own -// code-shaped argument -- a real runtime effect once wired (see -// ScriptNodeData's own comment in types/protocols.ts), so NOT dashed. +// Carries Python source staged for execution through run_wired_script, or +// supplied as a Tool Step argument. Availability factors control its presence. // Rendered as a small circle-with-icon (see CircleNode), same as // every other connector source. Wires into the Agent's shared Tool // connector (handleId="tool", alongside mcp_tool -- see AgentNode.tsx's own @@ -15,6 +15,10 @@ import { CircleNode } from './CircleNode' const ACCENT = nodeAccent('script') export function ScriptNode({ id, data, selected }: NodeProps & { data: ScriptNodeData }) { + const { updateNodeData } = useReactFlow() + const { requestScriptFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.scriptFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) return ( requestScriptFactor(id) : undefined} + dimmed={!enabled} + isActive={enabled} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/useScriptTestSelection.test.tsx b/frontend/src/components/protocol/useScriptTestSelection.test.tsx new file mode 100644 index 00000000..be304f8c --- /dev/null +++ b/frontend/src/components/protocol/useScriptTestSelection.test.tsx @@ -0,0 +1,21 @@ +import { act, renderHook } from '@testing-library/react' +import { expect, it } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { useScriptTestSelection } from './useScriptTestSelection' + +it('uses per-agent defaults and resets choices when the scope changes', () => { + const graph = { nodes: [ + { id: 'one', type: 'agent', data: { script_selection: ['a'], factor_bindings: { script_selection: 'One' } } }, + { id: 'two', type: 'sub_agent', data: { script_factor_mode: 'script_toggle', script_selection: [], factor_bindings: { script_selection: 'Two' } } }, + ...['a', 'b'].map((id) => ({ id, type: 'script', data: { config: { code: 'print(1)', enabled: false } } })), + ], edges: ['one', 'two'].flatMap((target) => ['a', 'b'].map((source) => ({ source, target, targetHandle: 'tool' }))) } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const { result, rerender } = renderHook(({ nodeId }) => useScriptTestSelection(graph, nodeId), { initialProps: { nodeId: undefined as string | undefined } }) + expect(result.current.error).toBeNull() + expect(result.current.options).toEqual({ script_selections: { one: 'a', two: 'none' } }) + act(() => result.current.setChoice('one', 'b')) + expect(result.current.selections.one).toBe('b') + rerender({ nodeId: 'one' }) + expect(result.current.selections).toEqual({ one: 'a' }) + expect(JSON.stringify(graph)).toBe(original) +}) diff --git a/frontend/src/components/protocol/useScriptTestSelection.ts b/frontend/src/components/protocol/useScriptTestSelection.ts new file mode 100644 index 00000000..2fef5484 --- /dev/null +++ b/frontend/src/components/protocol/useScriptTestSelection.ts @@ -0,0 +1,21 @@ +import { useEffect, useState } from 'react' +import { connectedScripts, scriptId, SCRIPT_FACTOR_PATH } from '@/lib/scriptFactors' +import type { ProtocolGraph } from '@/types/protocols' + +export function useScriptTestSelection(graph: ProtocolGraph | undefined, nodeId?: string) { + const [choices, setChoices] = useState>({}) + const owners = graph?.nodes.filter((node) => node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] && (!nodeId || node.id === nodeId)) ?? [] + const signature = JSON.stringify(owners.map((node) => [node.id, node.data.script_factor_mode, node.data.script_selection, connectedScripts(graph!, node.id).map(scriptId)])) + useEffect(() => { setChoices({}) }, [signature]) + const selections = Object.fromEntries(owners.map((node) => { + if (node.data.script_factor_mode === 'script_toggle') return [node.id, choices[node.id] ?? ((node.data.script_selection as string[] | undefined)?.length ? 'all' : 'none')] + const ids = connectedScripts(graph!, node.id).map(scriptId) + const baseline = (node.data.script_selection as string[] | undefined)?.[0] + return [node.id, choices[node.id] ?? (baseline && ids.includes(baseline) ? baseline : ids[0] ?? '')] + })) + const error = owners.some((node) => { + const ids = node.data.script_factor_mode === 'script_toggle' ? selections[node.id] === 'all' ? connectedScripts(graph!, node.id).map(scriptId) : [] : [selections[node.id]] + return ids.some((id) => !(graph!.nodes.find((node) => node.id === id)?.data.config as { code?: string })?.code?.trim()) + }) ? 'Select a connected Script with code for each agent.' : null + return { graph, owners, selections, options: owners.length ? { script_selections: selections } : {}, error, setChoice: (agentId: string, id: string) => setChoices((previous) => ({ ...previous, [agentId]: id })) } +} diff --git a/frontend/src/components/protocol/useToolTestSelection.test.tsx b/frontend/src/components/protocol/useToolTestSelection.test.tsx new file mode 100644 index 00000000..81a7077a --- /dev/null +++ b/frontend/src/components/protocol/useToolTestSelection.test.tsx @@ -0,0 +1,51 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, expect, it, vi } from 'vitest' +import { mcpServersApi } from '@/api/client' +import type { ProtocolGraph } from '@/types/protocols' +import { useToolTestSelection } from './useToolTestSelection' + +afterEach(() => vi.restoreAllMocks()) +it('uses all-or-none defaults per agent without altering shared tool switches', async () => { + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([{ id: 'a', capabilities: { tools: [{ name: 'run' }] } }] as Awaited>) + const graph = { + nodes: [ + { id: 'one', type: 'agent', data: { tool_factor_mode: 'tool_toggle', tool_selection: ['a'], factor_bindings: { tool_selection: 'One' } } }, + { id: 'two', type: 'agent', data: { tool_factor_mode: 'tool_toggle', tool_selection: [], factor_bindings: { tool_selection: 'Two' } } }, + { id: 'a', type: 'mcp_tool', data: { config: { server_id: 'a', tool_names: ['run'], enabled: false } } }, + ], + edges: ['one', 'two'].map((target) => ({ source: 'a', target, targetHandle: 'tool' })), + } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result } = renderHook(() => useToolTestSelection(graph), { wrapper }) + await waitFor(() => expect(result.current.error).toBeNull()) + expect(result.current.selections).toEqual({ one: 'all', two: 'none' }) + act(() => result.current.setChoice('one', 'none')) + expect(result.current.options).toEqual({ tool_selections: { one: 'none', two: 'none' } }) + expect(JSON.stringify(graph)).toBe(original) +}) +it('keeps preview selections scoped to the agent and does not alter the first level', async () => { + vi.spyOn(mcpServersApi, 'list').mockResolvedValue(['a', 'b'].map((id) => ({ id, capabilities: { tools: [{ name: 'run' }] } })) as Awaited>) + const graph = { + nodes: [ + { id: 'one', type: 'agent', data: { tool_selection: ['a'], factor_bindings: { tool_selection: 'One' } } }, + { id: 'two', type: 'agent', data: { tool_selection: ['b'], factor_bindings: { tool_selection: 'Two' } } }, + ...['a', 'b'].map((id) => ({ id, type: 'mcp_tool', data: { config: { server_id: id, tool_names: ['run'] } } })), + ], + edges: ['one', 'two'].flatMap((target) => ['a', 'b'].map((source) => ({ source, target, targetHandle: 'tool' }))), + } as unknown as ProtocolGraph + const original = JSON.stringify(graph) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result, rerender } = renderHook(({ nodeId }) => useToolTestSelection(graph, nodeId), { wrapper, initialProps: { nodeId: undefined as string | undefined } }) + await waitFor(() => expect(result.current.error).toBeNull()) + expect(result.current.selections).toEqual({ one: 'a', two: 'b' }) + act(() => result.current.setChoice('one', 'b')) + expect(result.current.options).toEqual({ tool_selections: { one: 'b', two: 'b' } }) + rerender({ nodeId: 'one' }) + expect(result.current.selections).toEqual({ one: 'a' }) + expect(JSON.stringify(graph)).toBe(original) +}) diff --git a/frontend/src/components/protocol/useToolTestSelection.ts b/frontend/src/components/protocol/useToolTestSelection.ts new file mode 100644 index 00000000..8ae34659 --- /dev/null +++ b/frontend/src/components/protocol/useToolTestSelection.ts @@ -0,0 +1,28 @@ +import { useEffect, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { mcpServersApi } from '@/api/client' +import { connectedTools, toolId, TOOL_FACTOR_PATH } from '@/lib/toolFactors' +import type { ProtocolGraph } from '@/types/protocols' + +export function useToolTestSelection(graph: ProtocolGraph | undefined, nodeId?: string) { + const [choices, setChoices] = useState>({}) + const library = useQuery({ queryKey: ['mcp-servers'], queryFn: () => mcpServersApi.list(), enabled: !!graph?.nodes.some((node) => node.data.factor_bindings?.[TOOL_FACTOR_PATH]) }) + const owners = graph?.nodes.filter((node) => node.data.factor_bindings?.[TOOL_FACTOR_PATH] && (!nodeId || node.id === nodeId)) ?? [] + const signature = JSON.stringify(owners.map((node) => [node.id, node.data.tool_factor_mode, node.data.tool_selection, connectedTools(graph!, node.id).map(toolId)])) + useEffect(() => { setChoices({}) }, [signature]) + const selections = Object.fromEntries(owners.map((node) => { + if (node.data.tool_factor_mode === 'tool_toggle') return [node.id, choices[node.id] ?? ((node.data.tool_selection as string[] | undefined)?.length ? 'all' : 'none')] + const ids = connectedTools(graph!, node.id).map(toolId) + const baseline = (node.data.tool_selection as string[] | undefined)?.[0] + return [node.id, choices[node.id] ?? (baseline && ids.includes(baseline) ? baseline : ids[0] ?? '')] + })) + const error = owners.length === 0 ? null : library.isError ? 'Could not load available tools.' : !library.data ? 'Loading tools…' : owners.some((node) => { + const ids = node.data.tool_factor_mode === 'tool_toggle' ? (selections[node.id] === 'all' ? connectedTools(graph!, node.id).map(toolId) : []) : [selections[node.id]] + return ids.some((id) => { + const config = graph!.nodes.find((node) => node.id === id)?.data.config as { server_id?: string; tool_names?: string[] } | undefined + const server = library.data.find((server) => server.id === config?.server_id) + return !server || !config?.tool_names?.length || config.tool_names.some((name) => !server.capabilities?.tools?.some((tool) => tool.name === name)) + }) + }) ? 'Select an available tool for each agent.' : null + return { graph, owners, library, selections, options: owners.length ? { tool_selections: selections } : {}, error, setChoice: (agentId: string, id: string) => setChoices((previous) => ({ ...previous, [agentId]: id })) } +} diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index 51172f61..5256b053 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -3,6 +3,8 @@ import { reconcileKnowledgeFactor, KNOWLEDGE_FACTOR_PATH } from './knowledgeFact import { reconcileDatasetFactor, DATASET_FACTOR_PATH } from './datasetFactors' import type { DesignFactor, DesignSpec } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' +import { reconcileScriptFactor, SCRIPT_FACTOR_PATH } from './scriptFactors' +import { reconcileToolFactor, TOOL_FACTOR_PATH } from './toolFactors' import { reconcileSkillFactor, SKILL_FACTOR_PATH } from './skillFactors' export interface FactorBindingDiscrepancy { @@ -63,7 +65,7 @@ function withPathValue(root: Record, dottedPath: string, nextVa * as an extra level would turn each pause mid-edit into a new treatment. */ export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { - if ((factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle')) return factor + if ((factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'script_selection' || factor.level_type === 'script_toggle' || factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle')) return factor if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) @@ -101,6 +103,14 @@ export function reconcileFactorBaselines(designSpec: DesignSpec | null | undefin const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] === factor.name) return owner ? reconcileSubAgentFactor(factor, graph, owner.id) : factor } + if ((factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle') && graph) { + const owner = graph.nodes.find((node) => node.data.factor_bindings?.[TOOL_FACTOR_PATH] === factor.name) + return owner ? reconcileToolFactor(factor, graph, owner.id) : factor + } + if ((factor.level_type === 'script_selection' || factor.level_type === 'script_toggle') && graph) { + const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] === factor.name) + return owner ? reconcileScriptFactor(factor, graph, owner.id) : factor + } if ((factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') && graph) { const owner = graph.nodes.find((node) => node.data.factor_bindings?.[SKILL_FACTOR_PATH] === factor.name) return owner ? reconcileSkillFactor(factor, graph, owner.id) : factor @@ -164,7 +174,7 @@ export function factorBindingDiscrepancies( ? 'factor is no longer declared' : value === MISSING ? 'field no longer exists on the canvas' - : (factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) + : (factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle' || factor.level_type === 'script_selection' || factor.level_type === 'script_toggle' || factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle' || factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle' || factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle' || factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') || isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) ? null : 'published value does not match the first (canvas baseline) level' if (reason) discrepancies.push({ nodeId: node.id, nodeLabel, fieldPath, factorName, reason }) diff --git a/frontend/src/lib/scriptFactors.test.ts b/frontend/src/lib/scriptFactors.test.ts new file mode 100644 index 00000000..38a28ebd --- /dev/null +++ b/frontend/src/lib/scriptFactors.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import { reconcileFactorBaselines } from './factorBindings' +import { reconcileScriptFactor, scriptFactorIssues, scriptFactorOwner } from './scriptFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Agent', script_selection: ['a'], factor_bindings: { script_selection: 'Scripts' } } }, + ...['a', 'b', 'c'].map((id) => ({ id, type: 'script', data: { label: `Script ${id}`, config: { code: 'print(1)' } } })), + ], + edges: ['a', 'b', 'c'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), +} as unknown as ProtocolGraph +const factor: DesignFactor = { name: 'Scripts', level_type: 'script_selection', levels: [['b'], ['a']], level_labels: ['Custom B', 'Custom A'] } + +describe('script connector factors', () => { + it('keeps all-or-none levels in their chosen order as connections change', () => { + const toggle: DesignFactor = { name: 'Scripts', level_type: 'script_toggle', levels: [['a', 'b'], []], level_labels: ['With scripts', 'Without scripts'] } + expect(reconcileScriptFactor(toggle, graph, 'agent')).toEqual({ ...toggle, levels: [['a', 'b', 'c'], []] }) + const noneFirst = { ...toggle, levels: [[], ['a', 'b']], level_labels: ['Without scripts', 'With scripts'] } + expect(reconcileScriptFactor(noneFirst, { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') }, 'agent')).toEqual({ ...noneFirst, levels: [[], ['a', 'c']] }) + expect(scriptFactorIssues({ ...graph, edges: [] }, [toggle])[0]).toContain('at least one') + }) + it('preserves ordered labels and appends newly connected scripts', () => { + expect(reconcileScriptFactor(factor, graph, 'agent')).toEqual({ ...factor, levels: [['b'], ['a'], ['c']], level_labels: ['Custom B', 'Custom A', 'Script c'] }) + }) + it('removes disconnected levels without replacing the first level from canvas data', () => { + const changed = { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') } + expect(reconcileFactorBaselines({ factors: [factor] }, changed)[0]).toEqual({ ...factor, levels: [['a'], ['c']], level_labels: ['Custom A', 'Script c'] }) + }) + it('keeps an empty factor visible and reports incomplete and unavailable scripts', () => { + const empty = { ...graph, edges: [] } + expect(reconcileScriptFactor(factor, empty, 'agent').levels).toEqual([]) + expect(scriptFactorIssues(empty, [factor])[0]).toContain('at least two') + expect(scriptFactorOwner(graph, 'b')).toBe('Scripts') + }) +}) diff --git a/frontend/src/lib/scriptFactors.ts b/frontend/src/lib/scriptFactors.ts new file mode 100644 index 00000000..ef0db498 --- /dev/null +++ b/frontend/src/lib/scriptFactors.ts @@ -0,0 +1,120 @@ +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +export const SCRIPT_FACTOR_PATH = 'script_selection' + +export const scriptFactorModes = { + boolean: 'This script on/off', + script_selection: 'Script levels', + script_toggle: 'All agent scripts on/off', +} +export type ScriptFactorMode = keyof typeof scriptFactorModes + +export function scriptFactorConflict(graph: ProtocolGraph, factors: DesignFactor[], scriptNodeId: string, mode: ScriptFactorMode, agentId: string): string | undefined { + const owner = graph.nodes.find((node) => node.id === (isIndividualScriptMode(mode) ? scriptNodeId : agentId)) + const currentName = owner?.data.factor_bindings?.[isIndividualScriptMode(mode) ? scriptFactorPath(mode) : SCRIPT_FACTOR_PATH] + const current = factors.find((factor) => factor.name === currentName) + if (current && (current.level_type ?? 'boolean') !== mode) { + return `${scriptFactorModes[mode]} is unavailable because this Agent already uses the factor ${current.name} (${scriptFactorModes[(current.level_type ?? 'boolean') as ScriptFactorMode]}). Keep the current factor type, or select its factor type and remove it first.` + } + if (isIndividualScriptMode(mode)) { + const other = Object.entries(owner?.data.factor_bindings ?? {}).filter(([path]) => path === 'config').filter(([path]) => path !== scriptFactorPath(mode)) + if (other.length) return `${scriptFactorModes[mode]} is unavailable because this Script already has factor bindings: ${other.map(([, name]) => name).join(', ')}. Remove those bindings first.` + } + const conflicts = [...new Set(isIndividualScriptMode(mode) + ? graph.edges.filter((edge) => edge.source === scriptNodeId && edge.targetHandle === 'tool') + .flatMap((edge) => { + const name = graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[SCRIPT_FACTOR_PATH] + return name ? [name] : [] + }) + : connectedScripts(graph, agentId).flatMap((node) => Object.values(node.data.factor_bindings ?? {})))] + if (!conflicts.length) return undefined + return isIndividualScriptMode(mode) + ? `${scriptFactorModes[mode]} is unavailable because a connected Agent already controls this Script with the factor ${conflicts.join(', ')}. Use the existing Agent-level factor, or select its factor type and remove it first.` + : `${scriptFactorModes[mode]} is unavailable because connected Scripts already have factor bindings: ${conflicts.join(', ')}. Agent-level script factors cannot be combined with individual Script factor bindings. Use This script on/off, or remove those bindings in each Script’s factor dialog first.` +} + +export function isScriptFactor(factor: DesignFactor): boolean { + return factor.level_type === 'script_selection' || factor.level_type === 'script_toggle' +} + +export function connectedScripts(graph: ProtocolGraph, agentId: string): ProtocolNode[] { + return graph.edges.filter((edge) => edge.target === agentId && edge.targetHandle === 'tool') + .flatMap((edge) => { + const node = graph.nodes.find((node) => node.id === edge.source && node.type === 'script') + return node ? [node] : [] + }) +} + +export function scriptId(node: ProtocolNode): string { return node.id } + +export function isIndividualScriptMode(mode: ScriptFactorMode): boolean { return mode === 'boolean' } +export function scriptFactorPath(mode: ScriptFactorMode): string { return mode === 'boolean' ? 'config.enabled' : SCRIPT_FACTOR_PATH } + +export function reconcileScriptFactor(factor: DesignFactor, graph: ProtocolGraph, agentId: string): DesignFactor { + const scripts = connectedScripts(graph, agentId) + const ids = [...new Set(scripts.map(scriptId))] + if (factor.level_type === 'script_toggle') { + const noneFirst = factor.levels.length === 2 && Array.isArray(factor.levels[0]) && factor.levels[0].length === 0 + return { + ...factor, + levels: noneFirst ? [[], ids] : [ids, []], + level_labels: factor.level_labels?.length === 2 ? factor.level_labels : noneFirst ? ['All disabled', 'All enabled'] : ['All enabled', 'All disabled'], + } + } + const oldIds = factor.levels.map((level) => Array.isArray(level) ? String(level[0] ?? '') : '') + const ordered = [...new Set([...oldIds.filter((id) => ids.includes(id)), ...ids.filter((id) => !oldIds.includes(id))])] + return { + ...factor, + levels: ordered.map((id) => [id]), + level_labels: ordered.map((id) => { + const index = oldIds.indexOf(id) + if (index >= 0 && factor.level_labels?.[index]) return factor.level_labels[index] + const node = scripts.find((node) => scriptId(node) === id) + const label = String(node?.data.label ?? 'Unavailable script') + return label || 'Unavailable script' + }), + } +} + +export function scriptFactorOwner(graph: ProtocolGraph | undefined, scriptNodeId: string): string | undefined { + return graph?.edges.filter((edge) => edge.source === scriptNodeId && edge.targetHandle === 'tool') + .map((edge) => graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[SCRIPT_FACTOR_PATH]) + .find(Boolean) +} + +export function scriptFactorIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[]): string[] { + const connectorIssues = factors.filter(isScriptFactor).flatMap((factor) => { + const owner = graph?.nodes.find((node) => node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] === factor.name) + if (!owner || !graph) return [`${factor.name}: rebind or remove this script factor.`] + const scripts = connectedScripts(graph, owner.id) + const ids = [...new Set(scripts.map(scriptId))] + const conflicts = scripts.some((node) => Object.keys(node.data.factor_bindings ?? {}).length > 0) + if (conflicts) return [`${factor.name}: remove individual script factor bindings first.`] + if (ids.length < (factor.level_type === 'script_toggle' ? 1 : 2)) return [`${factor.name}: ${factor.level_type === 'script_toggle' ? 'connect at least one script' : 'connect at least two different scripts'}.`] + if (scripts.some((node) => !(node.data.config as { code?: string })?.code?.trim())) return [`${factor.name}: configure code on every connected Script node.`] + return [] + }) + return [...connectorIssues, ...requiredScriptIssues(graph, factors)] +} + +export function requiredScriptIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[]): string[] { + return (graph?.nodes ?? []).filter((node) => node.type === 'tool_step' && node.data.active !== false).flatMap((step) => { + const args = (step.data.config as { arguments?: Record })?.arguments ?? {} + if (!Object.values(args).some((arg) => arg.source === 'script_code')) return [] + const scripts = connectedScripts(graph!, step.id) + const prefix = `${step.data.label || step.id}: requires script_code` + if (!scripts.length) return [`${prefix}; connect a Script with code.`] + return scripts.flatMap((script) => { + const bindings = script.data.factor_bindings ?? {} + const enabled = factors.find((factor) => factor.name === bindings['config.enabled']) + if (enabled?.levels.some((level) => level !== true)) return [`${prefix}; its Script factor cannot include off.`] + const variants = factors.find((factor) => factor.name === bindings.config)?.levels ?? [script.data.config] + if (variants.some((level) => { + const config = level as { code?: string; enabled?: boolean } | null + return !config?.code?.trim() || (!enabled && config.enabled === false) + })) return [`${prefix}; every Script variant needs enabled code.`] + return [] + }) + }) +} diff --git a/frontend/src/lib/toolFactors.test.ts b/frontend/src/lib/toolFactors.test.ts new file mode 100644 index 00000000..2e28076d --- /dev/null +++ b/frontend/src/lib/toolFactors.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { reconcileFactorBaselines } from './factorBindings' +import { reconcileToolFactor, toolFactorIssues, toolFactorOwner } from './toolFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Agent', tool_selection: ['a'], factor_bindings: { tool_selection: 'Tools' } } }, + ...['a', 'b', 'c'].map((id) => ({ id, type: 'mcp_tool', data: { label: `Tool ${id}`, config: { server_id: id, tool_names: ['run'] } } })), + ], + edges: ['a', 'b', 'c'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), +} as unknown as ProtocolGraph +const factor: DesignFactor = { name: 'Tools', level_type: 'tool_selection', levels: [['b'], ['a']], level_labels: ['Custom B', 'Custom A'] } + +describe('tool connector factors', () => { + it('keeps all-or-none levels in their chosen order as connections change', () => { + const toggle: DesignFactor = { name: 'Tools', level_type: 'tool_toggle', levels: [['a', 'b'], []], level_labels: ['With tools', 'Without tools'] } + expect(reconcileToolFactor(toggle, graph, 'agent')).toEqual({ ...toggle, levels: [['a', 'b', 'c'], []] }) + const noneFirst = { ...toggle, levels: [[], ['a', 'b']], level_labels: ['Without tools', 'With tools'] } + expect(reconcileToolFactor(noneFirst, { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') }, 'agent')).toEqual({ ...noneFirst, levels: [[], ['a', 'c']] }) + expect(toolFactorIssues({ ...graph, edges: [] }, [toggle])[0]).toContain('at least one') + }) + it('preserves ordered labels and appends newly connected tools', () => { + expect(reconcileToolFactor(factor, graph, 'agent')).toEqual({ ...factor, levels: [['b'], ['a'], ['c']], level_labels: ['Custom B', 'Custom A', 'Tool c'] }) + }) + it('removes disconnected levels without replacing the first level from canvas data', () => { + const changed = { ...graph, edges: graph.edges.filter((edge) => edge.source !== 'b') } + expect(reconcileFactorBaselines({ factors: [factor] }, changed)[0]).toEqual({ ...factor, levels: [['a'], ['c']], level_labels: ['Custom A', 'Tool c'] }) + }) + it('keeps an empty factor visible and reports incomplete and unavailable tools', () => { + const empty = { ...graph, edges: [] } + expect(reconcileToolFactor(factor, empty, 'agent').levels).toEqual([]) + expect(toolFactorIssues(empty, [factor])[0]).toContain('at least two') + expect(toolFactorIssues(graph, [factor], new Set(['a', 'c']))[0]).toContain('unavailable') + expect(toolFactorOwner(graph, 'b')).toBe('Tools') + }) +}) diff --git a/frontend/src/lib/toolFactors.ts b/frontend/src/lib/toolFactors.ts new file mode 100644 index 00000000..f2ce1d44 --- /dev/null +++ b/frontend/src/lib/toolFactors.ts @@ -0,0 +1,97 @@ +import type { DesignFactor } from '@/types/experiments' +import { MCP_TOOL_NODE_TYPES } from '@/components/protocol/mcpServerCatalog' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +export const TOOL_FACTOR_PATH = 'tool_selection' + +export const toolFactorModes = { + boolean: 'This tool on/off', + tool_selection: 'Tool levels', + tool_toggle: 'All agent tools on/off', + tool_names: 'Tools allowed', +} +export type ToolFactorMode = keyof typeof toolFactorModes + +export function toolFactorConflict(graph: ProtocolGraph, factors: DesignFactor[], toolNodeId: string, mode: ToolFactorMode, agentId: string): string | undefined { + const owner = graph.nodes.find((node) => node.id === (isIndividualToolMode(mode) ? toolNodeId : agentId)) + const currentName = owner?.data.factor_bindings?.[isIndividualToolMode(mode) ? toolFactorPath(mode) : TOOL_FACTOR_PATH] + const current = factors.find((factor) => factor.name === currentName) + if (current && (current.level_type ?? 'boolean') !== mode) { + return `${toolFactorModes[mode]} is unavailable because this Agent already uses the factor ${current.name} (${toolFactorModes[(current.level_type ?? 'boolean') as ToolFactorMode]}). Keep the current factor type, or select its factor type and remove it first.` + } + const conflicts = [...new Set(isIndividualToolMode(mode) + ? graph.edges.filter((edge) => edge.source === toolNodeId && edge.targetHandle === 'tool') + .flatMap((edge) => { + const name = graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[TOOL_FACTOR_PATH] + return name ? [name] : [] + }) + : connectedTools(graph, agentId).flatMap((node) => Object.values(node.data.factor_bindings ?? {})))] + if (!conflicts.length) return undefined + return isIndividualToolMode(mode) + ? `${toolFactorModes[mode]} is unavailable because a connected Agent already controls this Tool with the factor ${conflicts.join(', ')}. Use the existing Agent-level factor, or select its factor type and remove it first.` + : `${toolFactorModes[mode]} is unavailable because connected Tools already have factor bindings: ${conflicts.join(', ')}. Agent-level tool factors cannot be combined with individual Tool factor bindings. Use This tool on/off, or remove those bindings in each Tool’s factor dialog first.` +} + +export function isToolFactor(factor: DesignFactor): boolean { + return factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle' +} + +export function connectedTools(graph: ProtocolGraph, agentId: string): ProtocolNode[] { + return graph.edges.filter((edge) => edge.target === agentId && edge.targetHandle === 'tool') + .flatMap((edge) => { + const node = graph.nodes.find((node) => node.id === edge.source && MCP_TOOL_NODE_TYPES.includes(node.type)) + return node ? [node] : [] + }) +} + +export function toolId(node: ProtocolNode): string { return node.id } + +export function isIndividualToolMode(mode: ToolFactorMode): boolean { return mode === 'boolean' || mode === 'tool_names' } +export function toolFactorPath(mode: ToolFactorMode): string { return mode === 'boolean' ? 'config.enabled' : mode === 'tool_names' ? 'config.tool_names' : TOOL_FACTOR_PATH } + +export function reconcileToolFactor(factor: DesignFactor, graph: ProtocolGraph, agentId: string): DesignFactor { + const tools = connectedTools(graph, agentId) + const ids = [...new Set(tools.map(toolId))] + if (factor.level_type === 'tool_toggle') { + const noneFirst = factor.levels.length === 2 && Array.isArray(factor.levels[0]) && factor.levels[0].length === 0 + return { + ...factor, + levels: noneFirst ? [[], ids] : [ids, []], + level_labels: factor.level_labels?.length === 2 ? factor.level_labels : noneFirst ? ['All disabled', 'All enabled'] : ['All enabled', 'All disabled'], + } + } + const oldIds = factor.levels.map((level) => Array.isArray(level) ? String(level[0] ?? '') : '') + const ordered = [...new Set([...oldIds.filter((id) => ids.includes(id)), ...ids.filter((id) => !oldIds.includes(id))])] + return { + ...factor, + levels: ordered.map((id) => [id]), + level_labels: ordered.map((id) => { + const index = oldIds.indexOf(id) + if (index >= 0 && factor.level_labels?.[index]) return factor.level_labels[index] + const node = tools.find((node) => toolId(node) === id) + const label = String(node?.data.label ?? 'Unavailable tool') + return label || 'Unavailable tool' + }), + } +} + +export function toolFactorOwner(graph: ProtocolGraph | undefined, toolNodeId: string): string | undefined { + return graph?.edges.filter((edge) => edge.source === toolNodeId && edge.targetHandle === 'tool') + .map((edge) => graph.nodes.find((node) => node.id === edge.target)?.data.factor_bindings?.[TOOL_FACTOR_PATH]) + .find(Boolean) +} + +export function toolFactorIssues(graph: ProtocolGraph | undefined, factors: DesignFactor[], availableIds?: Set): string[] { + return factors.filter(isToolFactor).flatMap((factor) => { + const owner = graph?.nodes.find((node) => node.data.factor_bindings?.[TOOL_FACTOR_PATH] === factor.name) + if (!owner || !graph) return [`${factor.name}: rebind or remove this tool factor.`] + const tools = connectedTools(graph, owner.id) + const ids = [...new Set(tools.map(toolId))] + const conflicts = tools.some((node) => Object.keys(node.data.factor_bindings ?? {}).length > 0) + if (conflicts) return [`${factor.name}: remove individual tool factor bindings first.`] + if (ids.length < (factor.level_type === 'tool_toggle' ? 1 : 2)) return [`${factor.name}: ${factor.level_type === 'tool_toggle' ? 'connect at least one tool' : 'connect at least two different tools'}.`] + if (tools.some((node) => !(node.data.config as { server_id?: string })?.server_id || (availableIds && !availableIds.has(String((node.data.config as { server_id?: string }).server_id))))) return [`${factor.name}: disconnect or restore unavailable tools.`] + if (tools.some((node) => !(node.data.config as { tool_names?: string[] })?.tool_names?.length)) return [`${factor.name}: configure allowed tools on every connected Tool node.`] + return [] + }) +} diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 1a65acb9..5e1be6cd 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -22,6 +22,10 @@ export interface DesignFactor { | 'script_config' | 'dataset_config' | 'tool_names' + | 'script_selection' + | 'script_toggle' + | 'tool_selection' + | 'tool_toggle' | 'skill_selection' | 'skill_toggle' | 'sub_agent_selection' diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index 9fd8eb6d..b811d3d3 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -364,6 +364,10 @@ export interface AgentNodeData { knowledge_factor_mode?: 'knowledge_selection' | 'knowledge_toggle' sub_agent_selection?: string[] sub_agent_factor_mode?: 'sub_agent_selection' | 'sub_agent_toggle' + script_selection?: string[] + script_factor_mode?: 'script_selection' | 'script_toggle' + tool_selection?: string[] + tool_factor_mode?: 'tool_selection' | 'tool_toggle' skill_selection?: string[] skill_factor_mode?: 'skill_selection' | 'skill_toggle' label: string @@ -685,16 +689,11 @@ export function defaultDatasetNodeData(label = 'Dataset'): DatasetNodeData { return { label, config: { dataset_id: null, dataset_name: null, enabled: true } } } -// A "Script" node -- carries a fixed piece of code an Agent passes verbatim -// as some tool's own code-shaped argument (e.g. a domain MCP server's -// run_model_script's `code`). Not executed by ASAREE itself -- same "pure -// config source" status as every other connector; _build_user_input folds -// the code, fenced, into the wired agent's own instruction. Python-only for -// now (language is fixed, not a picker) -- matches the one real use case in -// evidence (a fixed XGBoost+Optuna scoring script whose only per-cell -// variation is the hyperparameters an upstream agent proposes, not the code -// itself). +// Script source stays outside the prompt and is staged in the run workspace. +// Agents can execute available Scripts through run_wired_script; Tool Steps +// can send a wired Script's source through a script_code argument. Python-only. export interface ScriptNodeConfig { + enabled?: boolean name: string description?: string language: 'python' diff --git a/src/asaree/api/protocols.py b/src/asaree/api/protocols.py index f5d655bd..65c47644 100644 --- a/src/asaree/api/protocols.py +++ b/src/asaree/api/protocols.py @@ -69,9 +69,11 @@ list_protocols, update_protocol, ) +from asaree.services.script_factors import test_script_factor_values from asaree.services.skill_factors import test_skill_factor_values from asaree.services.sub_agent_factors import test_sub_agent_factor_values from asaree.services.test_run_results import FreshnessReason, project_test_run_result +from asaree.services.tool_factors import test_tool_factor_values from asaree.worker.enqueue import enqueue_protocol_run router = APIRouter(prefix="/protocols", tags=["protocols"]) @@ -282,6 +284,8 @@ class TestRunRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) knowledge_selections: dict[str, str] | None = None sub_agent_selections: dict[str, str] | None = None + script_selections: dict[str, str] | None = None + tool_selections: dict[str, str] | None = None skill_selections: dict[str, str] | None = None dataset_selections: dict[str, str] | None = None @@ -292,6 +296,8 @@ class NodePlayRequest(BaseModel): row_index: StrictInt | None = Field(default=None, ge=0) knowledge_selections: dict[str, str] | None = None sub_agent_selections: dict[str, str] | None = None + script_selections: dict[str, str] | None = None + tool_selections: dict[str, str] | None = None skill_selections: dict[str, str] | None = None dataset_selections: dict[str, str] | None = None @@ -302,6 +308,8 @@ async def _node_test_factor_values( try: return { **await test_sub_agent_factor_values(graph, body.sub_agent_selections if body else None, owner_id, node_id), + **await test_script_factor_values(graph, body.script_selections if body else None, node_id), + **await test_tool_factor_values(graph, body.tool_selections if body else None, owner_id, node_id), **await test_skill_factor_values(graph, body.skill_selections if body else None, owner_id, node_id), **await test_knowledge_factor_values(graph, body.knowledge_selections if body else None, owner_id, node_id), **await test_dataset_factor_values(graph, body.dataset_selections if body else None, owner_id, db, node_id), @@ -697,6 +705,8 @@ async def create_test_run_endpoint( **await test_sub_agent_factor_values( revision.graph, body.sub_agent_selections if body else None, user.id ), + **await test_script_factor_values(revision.graph, body.script_selections if body else None), + **await test_tool_factor_values(revision.graph, body.tool_selections if body else None, user.id), **await test_skill_factor_values(revision.graph, body.skill_selections if body else None, user.id), **await test_knowledge_factor_values( revision.graph, body.knowledge_selections if body else None, user.id diff --git a/src/asaree/services/design_generation.py b/src/asaree/services/design_generation.py index cfa62d7e..f0e473f7 100644 --- a/src/asaree/services/design_generation.py +++ b/src/asaree/services/design_generation.py @@ -37,8 +37,10 @@ from asaree.services.design_revisions import get_current_revision, supersede_and_create from asaree.services.factorial_cells import list_replicates, upsert_replicate from asaree.services.knowledge_factors import validate_knowledge_factors +from asaree.services.script_factors import validate_script_factors, validate_tool_step_script_factors from asaree.services.skill_factors import validate_skill_factors from asaree.services.sub_agent_factors import validate_sub_agent_factor_structure +from asaree.services.tool_factors import validate_tool_factors class DesignValidationError(ValueError): @@ -290,10 +292,13 @@ async def get_design_impact( material = material_design_spec(design_spec) incomplete_skills = any( (factor.get("level_type") in { - "sub_agent_selection", "skill_selection", "dataset_selection", "knowledge_selection" + "sub_agent_selection", "script_selection", "tool_selection", "skill_selection", + "dataset_selection", "knowledge_selection" } and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") in {"sub_agent_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle"} + or (factor.get("level_type") in { + "sub_agent_toggle", "script_toggle", "tool_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle" + } and not any(factor.get("levels") or [])) for factor in material["factors"] ) @@ -421,17 +426,27 @@ async def generate_design_cells( ).scalar_one_or_none() if experiment is None: raise DesignValidationError("Experiment not found") + protocol_for_scripts = await db.scalar(select(Protocol).where(Protocol.experiment_id == experiment_id)) + try: + validate_tool_step_script_factors( + {"factors": factors}, protocol_for_scripts.graph if protocol_for_scripts else {} + ) + except ValueError as exc: + raise DesignValidationError(str(exc)) from exc if any( factor.get("level_type") in { "sub_agent_selection", "sub_agent_toggle", - "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle", + "script_selection", "script_toggle", "tool_selection", "tool_toggle", "skill_selection", "skill_toggle", + "dataset_selection", "dataset_toggle", "knowledge_selection", "knowledge_toggle", } for factor in factors ): - protocol = await db.scalar(select(Protocol).where(Protocol.experiment_id == experiment_id)) + protocol = protocol_for_scripts try: validate_sub_agent_factor_structure({"factors": factors}, protocol.graph if protocol else {}) + validate_script_factors({"factors": factors}, protocol.graph if protocol else {}) + await validate_tool_factors({"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id) await validate_skill_factors({"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id) await validate_knowledge_factors( {"factors": factors}, protocol.graph if protocol else {}, experiment.owner_id diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index 9a9befbc..61bd658b 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -6,8 +6,10 @@ from asaree.services.dataset_factors import validate_dataset_factor_structure from asaree.services.knowledge_factors import validate_knowledge_factor_structure +from asaree.services.script_factors import validate_script_factor_structure, validate_tool_step_script_factors from asaree.services.skill_factors import validate_skill_factor_structure from asaree.services.sub_agent_factors import validate_sub_agent_factor_structure +from asaree.services.tool_factors import validate_tool_factor_structure _MISSING = object() @@ -35,6 +37,9 @@ def validate_factor_bindings( design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete_skills: bool = True ) -> None: validate_sub_agent_factor_structure(design_spec, graph, complete=complete_skills) + validate_script_factor_structure(design_spec, graph, complete=complete_skills) + validate_tool_step_script_factors(design_spec, graph) + validate_tool_factor_structure(design_spec, graph, complete=complete_skills) validate_skill_factor_structure(design_spec, graph, complete=complete_skills) validate_knowledge_factor_structure(design_spec, graph, complete=complete_skills) validate_dataset_factor_structure(design_spec, graph, complete=complete_skills) @@ -75,7 +80,9 @@ def validate_factor_bindings( if ( level_type not in { "boolean", "sub_agent_selection", "sub_agent_toggle", - "skill_selection", "skill_toggle", "dataset_selection", "dataset_toggle", + "script_selection", "script_toggle", "tool_selection", "tool_toggle", + "skill_selection", "skill_toggle", + "dataset_selection", "dataset_toggle", "knowledge_selection", "knowledge_toggle", } and not legacy_boolean diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 38829e46..5d0e0365 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -2015,11 +2015,17 @@ def _resolve_script_configs(graph: dict[str, Any], node_id: str) -> list[dict[st selectable script to the wired agent. """ nodes, _downstream, _upstream = _adjacency(graph) + data = (nodes.get(node_id) or {}).get("data") or {} + selection = data.get("script_selection") if (data.get("factor_bindings") or {}).get("script_selection") else None configs: list[dict[str, Any]] = [] for edge in _edges_with_handle(graph, node_id, "tool", direction="incoming"): source = nodes.get(edge["source"]) if source is not None and source.get("type") in _SCRIPT_NODE_TYPES: config = (source.get("data") or {}).get("config") or {} + if selection is not None and source["id"] not in selection: + continue + if selection is None and config.get("enabled") is False: + continue configs.append({**config, "node_id": str(source.get("id") or "")}) return configs @@ -2484,6 +2490,11 @@ def _resolve_tool_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: meaningful level: the server still connects (its ``server_name`` is still reported) but contributes no tools to that cell.""" nodes, _downstream, _upstream = _adjacency(graph) + agent_data = (nodes.get(node_id) or {}).get("data") or {} + selected = ( + agent_data.get("tool_selection") + if (agent_data.get("factor_bindings") or {}).get("tool_selection") else None + ) server_names: list[str] = [] tool_names: list[str] = [] for edge in _edges_with_handle(graph, node_id, "tool", direction="incoming"): @@ -2491,7 +2502,9 @@ def _resolve_tool_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: if source is None or source.get("type") not in _MCP_TOOL_NODE_TYPES: continue tool_node_config = (source.get("data") or {}).get("config") or {} - if not tool_node_config.get("enabled", True): + if selected is not None and source["id"] not in selected: + continue + if selected is None and not tool_node_config.get("enabled", True): continue server_name = tool_node_config.get("server_name") node_tool_names = tool_node_config.get("tool_names") or [] diff --git a/src/asaree/services/protocol_revisions.py b/src/asaree/services/protocol_revisions.py index 4e705d8d..cb378ec1 100644 --- a/src/asaree/services/protocol_revisions.py +++ b/src/asaree/services/protocol_revisions.py @@ -151,10 +151,13 @@ async def publish_protocol( from asaree.services.design_generation import get_design_impact incomplete_skills = any( (factor.get("level_type") in { - "sub_agent_selection", "skill_selection", "dataset_selection", "knowledge_selection" + "sub_agent_selection", "script_selection", "tool_selection", "skill_selection", + "dataset_selection", "knowledge_selection" } and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") in {"sub_agent_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle"} + or (factor.get("level_type") in { + "sub_agent_toggle", "script_toggle", "tool_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle" + } and not any(factor.get("levels") or [])) for factor in (experiment.design_spec or {}).get("factors") or [] ) diff --git a/src/asaree/services/script_factors.py b/src/asaree/services/script_factors.py new file mode 100644 index 00000000..e9b5af25 --- /dev/null +++ b/src/asaree/services/script_factors.py @@ -0,0 +1,166 @@ +"""Connector-owned script choices shared by design validation and test runs.""" + +from __future__ import annotations + +from typing import Any + +SCRIPT_SELECTION = "script_selection" +SCRIPT_TOGGLE = "script_toggle" +SCRIPT_FACTOR_TYPES = {SCRIPT_SELECTION, SCRIPT_TOGGLE} +SCRIPT_NODE_TYPES = {"script"} + + +def connected_script_nodes(graph: dict[str, Any], agent_id: str) -> list[dict[str, Any]]: + nodes = {node["id"]: node for node in graph.get("nodes") or []} + return [ + nodes[edge["source"]] + for edge in graph.get("edges") or [] + if edge.get("target") == agent_id + and edge.get("targetHandle") == "tool" + and edge.get("source") in nodes + and nodes[edge["source"]].get("type") in SCRIPT_NODE_TYPES + ] + + +def validate_script_factor_structure( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete: bool = True +) -> None: + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + for script in graph.get("nodes") or []: + if script.get("type") != "script": + continue + bindings = (script.get("data") or {}).get("factor_bindings") or {} + if "config" in bindings and any(path.startswith("config.") for path in bindings): + raise ValueError("Script variants cannot be combined with individual Script field factors.") + bound = { + (node.get("data") or {}).get("factor_bindings", {}).get(SCRIPT_SELECTION) for node in graph.get("nodes") or [] + } + for name, factor in factors.items(): + if factor.get("level_type") in SCRIPT_FACTOR_TYPES and name not in bound: + raise ValueError(f"Rebind or remove script factor {name!r}.") + for node in graph.get("nodes") or []: + data = node.get("data") or {} + name = (data.get("factor_bindings") or {}).get(SCRIPT_SELECTION) + if not name: + continue + factor = factors.get(name) + if node.get("type") not in {"agent", "sub_agent"}: + raise ValueError("Only an Agent script connector can own a script factor.") + if not factor or factor.get("level_type") not in SCRIPT_FACTOR_TYPES: + raise ValueError(f"Script connector factor {name!r} must declare a script factor type.") + toggle = factor.get("level_type") == SCRIPT_TOGGLE + if data.get("script_factor_mode", SCRIPT_SELECTION) != factor.get("level_type"): + raise ValueError(f"Script factor {name!r}: canvas mode does not match the declared factor.") + scripts = connected_script_nodes(graph, node["id"]) + ids = {script["id"] for script in scripts} + if complete and (None in ids or "" in ids): + raise ValueError(f"Script factor {name!r}: disconnect or restore unavailable scripts.") + if complete and len(ids) < (1 if toggle else 2): + minimum = "one script" if toggle else "two different scripts" + raise ValueError(f"Script factor {name!r}: connect at least {minimum}.") + if any((script.get("data") or {}).get("factor_bindings") for script in scripts): + raise ValueError(f"Script factor {name!r}: remove individual script factor bindings first.") + levels = factor.get("levels") or [] + if toggle: + if ( + len(levels) != 2 + or any(not isinstance(level, list) for level in levels) + or sorted(len(level) for level in levels) != [0, len(ids)] + or any(level and (set(level) != ids or len(level) != len(ids)) for level in levels) + ): + raise ValueError(f"Script factor {name!r}: levels must enable every connected script or none.") + continue + if any(not isinstance(level, list) or len(level) != 1 or level[0] not in ids for level in levels): + raise ValueError(f"Script factor {name!r}: each level must name one connected script.") + if len(levels) != len(ids) or {level[0] for level in levels} != ids: + raise ValueError(f"Script factor {name!r}: levels must include every connected script exactly once.") + + +def require_available_scripts(nodes: list[dict[str, Any]]) -> None: + if any(not str(((node.get("data") or {}).get("config") or {}).get("code") or "").strip() for node in nodes): + raise ValueError("A script factor references a Script without code. Configure or disconnect it.") + + +def validate_script_factors(design_spec: dict[str, Any] | None, graph: dict[str, Any]) -> None: + validate_script_factor_structure(design_spec, graph) + nodes = [ + script + for node in graph.get("nodes") or [] + if ((node.get("data") or {}).get("factor_bindings") or {}).get(SCRIPT_SELECTION) + for script in connected_script_nodes(graph, node["id"]) + ] + require_available_scripts(nodes) + + +async def test_script_factor_values( + graph: dict[str, Any], selections: dict[str, str] | None, node_id: str | None = None +) -> dict[str, Any]: + """Validate preview choices against the publication and retain run provenance.""" + selections = selections or {} + owners = { + node["id"]: node + for node in graph.get("nodes") or [] + if (node.get("data") or {}).get("factor_bindings", {}).get(SCRIPT_SELECTION) + and (node_id is None or node["id"] == node_id) + } + if set(selections) - owners.keys(): + raise ValueError("Script selection must belong to a script factor in this run.") + values: dict[str, Any] = {} + for agent_id, node in owners.items(): + data = node.get("data") or {} + connected = [script["id"] for script in connected_script_nodes(graph, agent_id)] + if any((script.get("data") or {}).get("factor_bindings") for script in connected_script_nodes(graph, agent_id)): + raise ValueError("Remove individual script factor bindings before testing this connector factor.") + baseline = data.get(SCRIPT_SELECTION) or [] + if data.get("script_factor_mode") == SCRIPT_TOGGLE: + selected = selections.get(agent_id, "all" if baseline else "none") + if selected not in {"all", "none"}: + raise ValueError("Select all enabled or all disabled for the test run.") + chosen = list(dict.fromkeys(connected)) if selected == "all" else [] + require_available_scripts( + [script for script in connected_script_nodes(graph, agent_id) if script["id"] in chosen] + ) + values[data["factor_bindings"][SCRIPT_SELECTION]] = chosen + continue + selected = selections.get(agent_id, baseline[0] if baseline else None) + if not selected or selected not in connected: + raise ValueError("Select an available connected script for the test run.") + require_available_scripts( + [script for script in connected_script_nodes(graph, agent_id) if script["id"] == selected] + ) + values[data["factor_bindings"][SCRIPT_SELECTION]] = [selected] + return values + + +def validate_tool_step_script_factors(design_spec: dict[str, Any] | None, graph: dict[str, Any]) -> None: + """Every treatment must supply code when an active Tool Step requires it.""" + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + for step in graph.get("nodes") or []: + data = step.get("data") or {} + if step.get("type") != "tool_step" or data.get("active") is False: + continue + arguments = (data.get("config") or {}).get("arguments") or {} + if not any(isinstance(arg, dict) and arg.get("source") == "script_code" for arg in arguments.values()): + continue + scripts = connected_script_nodes(graph, step["id"]) + label = data.get("label") or step["id"] + if not scripts: + raise ValueError(f"Tool Step {label!r} requires script_code: connect a Script with code.") + for script in scripts: + script_data = script.get("data") or {} + config = script_data.get("config") or {} + bindings = script_data.get("factor_bindings") or {} + config_factor = factors.get(bindings.get("config")) + variants = (config_factor.get("levels") or []) if config_factor else [config] + enabled_factor = factors.get(bindings.get("config.enabled")) + if enabled_factor and any(level is not True for level in enabled_factor.get("levels") or []): + raise ValueError(f"Tool Step {label!r} requires script_code: its Script factor cannot include off.") + for variant in variants: + if ( + not isinstance(variant, dict) + or (not enabled_factor and variant.get("enabled") is False) + or not str(variant.get("code") or "").strip() + ): + raise ValueError( + f"Tool Step {label!r} requires script_code: every Script variant needs enabled code." + ) diff --git a/src/asaree/services/tool_factors.py b/src/asaree/services/tool_factors.py new file mode 100644 index 00000000..6136627c --- /dev/null +++ b/src/asaree/services/tool_factors.py @@ -0,0 +1,138 @@ +"""Connector-owned tool choices shared by design validation and test runs.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from motoro.services import mcp_service + +TOOL_SELECTION = "tool_selection" +TOOL_TOGGLE = "tool_toggle" +TOOL_FACTOR_TYPES = {TOOL_SELECTION, TOOL_TOGGLE} +MCP_TOOL_NODE_TYPES = {"mcp_tool", "mcp_scikit_learn", "mcp_client_tool"} + + +def connected_tool_nodes(graph: dict[str, Any], agent_id: str) -> list[dict[str, Any]]: + nodes = {node["id"]: node for node in graph.get("nodes") or []} + return [ + nodes[edge["source"]] + for edge in graph.get("edges") or [] + if edge.get("target") == agent_id + and edge.get("targetHandle") == "tool" + and edge.get("source") in nodes + and nodes[edge["source"]].get("type") in MCP_TOOL_NODE_TYPES + ] + + +def validate_tool_factor_structure( + design_spec: dict[str, Any] | None, graph: dict[str, Any], *, complete: bool = True +) -> None: + factors = {factor.get("name"): factor for factor in (design_spec or {}).get("factors") or []} + bound = { + (node.get("data") or {}).get("factor_bindings", {}).get(TOOL_SELECTION) for node in graph.get("nodes") or [] + } + for name, factor in factors.items(): + if factor.get("level_type") in TOOL_FACTOR_TYPES and name not in bound: + raise ValueError(f"Rebind or remove tool factor {name!r}.") + for node in graph.get("nodes") or []: + data = node.get("data") or {} + name = (data.get("factor_bindings") or {}).get(TOOL_SELECTION) + if not name: + continue + factor = factors.get(name) + if node.get("type") not in {"agent", "sub_agent"}: + raise ValueError("Only an Agent tool connector can own a tool factor.") + if not factor or factor.get("level_type") not in TOOL_FACTOR_TYPES: + raise ValueError(f"Tool connector factor {name!r} must declare a tool factor type.") + toggle = factor.get("level_type") == TOOL_TOGGLE + if data.get("tool_factor_mode", TOOL_SELECTION) != factor.get("level_type"): + raise ValueError(f"Tool factor {name!r}: canvas mode does not match the declared factor.") + tools = connected_tool_nodes(graph, node["id"]) + ids = {tool["id"] for tool in tools} + if complete and (None in ids or "" in ids): + raise ValueError(f"Tool factor {name!r}: disconnect or restore unavailable tools.") + if complete and len(ids) < (1 if toggle else 2): + minimum = "one tool" if toggle else "two different tools" + raise ValueError(f"Tool factor {name!r}: connect at least {minimum}.") + if any((tool.get("data") or {}).get("factor_bindings") for tool in tools): + raise ValueError(f"Tool factor {name!r}: remove individual tool factor bindings first.") + levels = factor.get("levels") or [] + if toggle: + if ( + len(levels) != 2 + or any(not isinstance(level, list) for level in levels) + or sorted(len(level) for level in levels) != [0, len(ids)] + or any(level and (set(level) != ids or len(level) != len(ids)) for level in levels) + ): + raise ValueError(f"Tool factor {name!r}: levels must enable every connected tool or none.") + continue + if any(not isinstance(level, list) or len(level) != 1 or level[0] not in ids for level in levels): + raise ValueError(f"Tool factor {name!r}: each level must name one connected tool.") + if len(levels) != len(ids) or {level[0] for level in levels} != ids: + raise ValueError(f"Tool factor {name!r}: levels must include every connected tool exactly once.") + + +async def require_available_tools(nodes: list[dict[str, Any]], owner_id: uuid.UUID) -> None: + for node in nodes: + config = (node.get("data") or {}).get("config") or {} + try: + server = await mcp_service.get_server(uuid.UUID(config.get("server_id"))) + except (ValueError, TypeError, AttributeError): + server = None + if server is None or not (server.owner_id == owner_id or server.is_system): + raise ValueError("A tool factor references an unavailable server. Disconnect or restore it.") + available = {tool["name"] for tool in (server.capabilities or {}).get("tools", [])} + if not config.get("tool_names") or any(name not in available for name in config["tool_names"]): + raise ValueError("A tool factor has an empty or unavailable allow-list. Configure its allowed tools.") + + +async def validate_tool_factors(design_spec: dict[str, Any] | None, graph: dict[str, Any], owner_id: uuid.UUID) -> None: + validate_tool_factor_structure(design_spec, graph) + nodes = [ + tool + for node in graph.get("nodes") or [] + if ((node.get("data") or {}).get("factor_bindings") or {}).get(TOOL_SELECTION) + for tool in connected_tool_nodes(graph, node["id"]) + ] + await require_available_tools(nodes, owner_id) + + +async def test_tool_factor_values( + graph: dict[str, Any], selections: dict[str, str] | None, owner_id: uuid.UUID, node_id: str | None = None +) -> dict[str, Any]: + """Validate preview choices against the publication and retain run provenance.""" + selections = selections or {} + owners = { + node["id"]: node + for node in graph.get("nodes") or [] + if (node.get("data") or {}).get("factor_bindings", {}).get(TOOL_SELECTION) + and (node_id is None or node["id"] == node_id) + } + if set(selections) - owners.keys(): + raise ValueError("Tool selection must belong to a tool factor in this run.") + values: dict[str, Any] = {} + for agent_id, node in owners.items(): + data = node.get("data") or {} + connected = [tool["id"] for tool in connected_tool_nodes(graph, agent_id)] + if any((tool.get("data") or {}).get("factor_bindings") for tool in connected_tool_nodes(graph, agent_id)): + raise ValueError("Remove individual tool factor bindings before testing this connector factor.") + baseline = data.get(TOOL_SELECTION) or [] + if data.get("tool_factor_mode") == TOOL_TOGGLE: + selected = selections.get(agent_id, "all" if baseline else "none") + if selected not in {"all", "none"}: + raise ValueError("Select all enabled or all disabled for the test run.") + chosen = list(dict.fromkeys(connected)) if selected == "all" else [] + await require_available_tools( + [tool for tool in connected_tool_nodes(graph, agent_id) if tool["id"] in chosen], owner_id + ) + values[data["factor_bindings"][TOOL_SELECTION]] = chosen + continue + selected = selections.get(agent_id, baseline[0] if baseline else None) + if not selected or selected not in connected: + raise ValueError("Select an available connected tool for the test run.") + await require_available_tools( + [tool for tool in connected_tool_nodes(graph, agent_id) if tool["id"] == selected], owner_id + ) + values[data["factor_bindings"][TOOL_SELECTION]] = [selected] + return values diff --git a/tests/test_script_factors.py b/tests/test_script_factors.py new file mode 100644 index 00000000..52da8b38 --- /dev/null +++ b/tests/test_script_factors.py @@ -0,0 +1,235 @@ +"""Script treatments control availability without changing Tool treatments.""" + +import copy +import uuid +from unittest.mock import AsyncMock + +import pytest + +from asaree.services.design_generation import DesignValidationError, generate_design_cells, get_design_impact +from asaree.services.factor_bindings import validate_factor_bindings +from asaree.services.protocol_execution import ( + _ambient_meta_for, + _resolve_script_configs, + _resolve_script_tool_config, + _resolve_tool_config, + apply_factor_bindings, +) +from asaree.services.script_factors import test_script_factor_values as preview_values +from asaree.services.script_factors import validate_script_factors, validate_tool_step_script_factors + + +def script_graph(mode="script_selection"): + ids = ["a", "b"] + graph = { + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": { + "factor_bindings": {"script_selection": "Scripts"}, + "script_factor_mode": mode, + "script_selection": ids if mode == "script_toggle" else ["a"], + }, + }, + *[ + { + "id": id, + "type": "script", + "data": { + "config": { + "name": id, + "language": "python", + "code": f"print('{id}')", + "enabled": False, + } + }, + } + for id in ids + ], + { + "id": "mcp", + "type": "mcp_tool", + "data": { + "config": { + "server_name": "search", + "tool_names": ["fetch"], + } + }, + }, + ], + "edges": [{"source": id, "target": "agent", "targetHandle": "tool"} for id in [*ids, "mcp"]], + } + spec = { + "factors": [ + { + "name": "Scripts", + "level_type": mode, + "levels": [ids, []] if mode == "script_toggle" else [[id] for id in ids], + } + ] + } + return graph, spec + + +@pytest.mark.parametrize("mode", ["script_selection", "script_toggle"]) +def test_selection_overrides_disabled_nodes_and_leaves_mcp_tools_unchanged(mode): + graph, spec = script_graph(mode) + original = copy.deepcopy(graph) + validate_factor_bindings(spec, graph) + for chosen in (["b"], ["a", "b"], []): + patched = apply_factor_bindings(graph, {"Scripts": chosen}) + assert [config["node_id"] for config in _resolve_script_configs(patched, "agent")] == chosen + assert bool(_resolve_script_tool_config(patched, "agent")["tool_names"]) == bool(chosen) + assert _resolve_tool_config(patched, "agent") == {"server_names": ["search"], "tool_names": ["search.fetch"]} + assert graph == original + + +def test_individual_off_removes_script_from_resolution_and_shared_agent(): + graph, _ = script_graph() + graph["nodes"][0]["data"] = {} + graph["nodes"].append({"id": "other", "type": "agent", "data": {}}) + graph["edges"].append({"source": "a", "target": "other", "targetHandle": "tool"}) + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "On"} + assert _resolve_script_configs(apply_factor_bindings(graph, {"On": False}), "agent") == [] + assert _resolve_script_configs(apply_factor_bindings(graph, {"On": False}), "other") == [] + assert _resolve_script_configs(apply_factor_bindings(graph, {"On": True}), "other")[0]["node_id"] == "a" + + +def test_agent_selection_does_not_enable_shared_script_for_other_agent(): + graph, _ = script_graph() + graph["nodes"].append({"id": "other", "type": "agent", "data": {}}) + graph["edges"].append({"source": "a", "target": "other", "targetHandle": "tool"}) + patched = apply_factor_bindings(graph, {"Scripts": ["a"]}) + assert len(_resolve_script_configs(patched, "agent")) == 1 + assert _resolve_script_configs(patched, "other") == [] + + +def test_off_removes_materialized_source_from_execution_context(monkeypatch): + graph, _ = script_graph("script_toggle") + materialize = [] + monkeypatch.setattr( + "asaree.services.protocol_execution._materialize_script", + lambda workspace, id, code: materialize.append(id) or f"/tmp/{id}.py", + ) + off = apply_factor_bindings(graph, {"Scripts": []}) + assert "script_paths" not in _ambient_meta_for(off, "agent", script_workspace_id="workspace") + assert materialize == [] + on = apply_factor_bindings(graph, {"Scripts": ["b"]}) + assert _ambient_meta_for(on, "agent", script_workspace_id="workspace")["script_paths"][0]["id"] == "b" + assert materialize == ["b"] + + +def test_script_variants_replace_code_description_and_name(): + graph, _ = script_graph() + graph["nodes"][0]["data"] = {} + graph["nodes"][1]["data"]["factor_bindings"] = {"config": "Variants"} + variant = {"name": "new", "description": "new algorithm", "language": "python", "code": "print(2)"} + assert _resolve_script_configs(apply_factor_bindings(graph, {"Variants": variant}), "agent") == [ + {**variant, "node_id": "a"} + ] + + +@pytest.mark.parametrize("change", ["missing", "duplicate", "foreign", "conflict", "empty-code"]) +def test_invalid_or_incomplete_script_factors_block_generation(change): + graph, spec = script_graph() + if change == "missing": + spec["factors"][0]["levels"].pop() + elif change == "duplicate": + spec["factors"][0]["levels"] = [["a"], ["a"]] + elif change == "foreign": + spec["factors"][0]["levels"] = [["a"], ["mcp"]] + elif change == "conflict": + graph["nodes"][1]["data"]["factor_bindings"] = {"config": "Variants"} + else: + graph["nodes"][1]["data"]["config"]["code"] = "" + with pytest.raises(ValueError): + validate_script_factors(spec, graph) + + +@pytest.mark.asyncio +async def test_preview_is_scoped_and_all_none_follow_baseline(): + graph, _ = script_graph() + assert await preview_values(graph, None) == {"Scripts": ["a"]} + assert await preview_values(graph, {"agent": "b"}, "agent") == {"Scripts": ["b"]} + with pytest.raises(ValueError, match="belong"): + await preview_values(graph, {"other": "a"}, "agent") + graph["nodes"][0]["data"].update(script_factor_mode="script_toggle", script_selection=[]) + assert await preview_values(graph, None) == {"Scripts": []} + assert await preview_values(graph, {"agent": "all"}) == {"Scripts": ["a", "b"]} + + +@pytest.mark.parametrize("change", ["off", "empty-code", "disabled-variant", "empty-variant"]) +def test_required_tool_step_rejects_invalid_script_treatments(change): + config = {"name": "score", "language": "python", "code": "print(1)"} + script_data = {"config": config, "factor_bindings": {}} + graph = { + "nodes": [ + { + "id": "step", + "type": "tool_step", + "data": { + "config": { + "arguments": {"code": {"source": "script_code"}}, + } + }, + }, + {"id": "script", "type": "script", "data": script_data}, + ], + "edges": [{"source": "script", "target": "step", "targetHandle": "tool"}], + } + spec = {"factors": []} + if change == "off": + script_data["factor_bindings"] = {"config.enabled": "On"} + spec["factors"] = [{"name": "On", "level_type": "boolean", "levels": [False, True]}] + elif change == "empty-code": + config["code"] = "" + else: + script_data["factor_bindings"] = {"config": "Variants"} + variant = {**config, "enabled": False} if change == "disabled-variant" else {**config, "code": ""} + spec["factors"] = [{"name": "Variants", "level_type": "script_config", "levels": [config, variant]}] + with pytest.raises(ValueError, match="requires script_code"): + validate_tool_step_script_factors(spec, graph) + graph["nodes"][0]["data"]["config"]["arguments"] = {} + validate_tool_step_script_factors(spec, graph) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode,levels", [("script_selection", []), ("script_toggle", [[], []])]) +async def test_empty_design_impact_is_readable(monkeypatch, mode, levels): + monkeypatch.setattr("asaree.services.design_generation.get_current_revision", AsyncMock(return_value=None)) + impact = await get_design_impact( + AsyncMock(), + experiment_id=uuid.uuid4(), + design_spec={ + "factors": [{"name": "Scripts", "level_type": mode, "levels": levels}], + }, + ) + assert impact.proposed_cell_count == 0 + + +@pytest.mark.asyncio +async def test_generation_rejects_required_script_off_before_creating_cells(): + from types import SimpleNamespace + from unittest.mock import MagicMock + + graph = { + "nodes": [ + {"id": "step", "type": "tool_step", "data": {"config": {"arguments": {"code": {"source": "script_code"}}}}}, + { + "id": "script", + "type": "script", + "data": {"config": {"code": "print(1)"}, "factor_bindings": {"config.enabled": "On"}}, + }, + ], + "edges": [{"source": "script", "target": "step", "targetHandle": "tool"}], + } + db = AsyncMock() + result = MagicMock() + result.scalar_one_or_none.return_value = SimpleNamespace(owner_id=uuid.uuid4()) + db.execute.return_value = result + db.scalar.return_value = SimpleNamespace(graph=graph) + with pytest.raises(DesignValidationError, match="cannot include off"): + await generate_design_cells( + db, experiment_id=uuid.uuid4(), factors=[{"name": "On", "level_type": "boolean", "levels": [False, True]}] + ) diff --git a/tests/test_tool_factors.py b/tests/test_tool_factors.py new file mode 100644 index 00000000..4651baae --- /dev/null +++ b/tests/test_tool_factors.py @@ -0,0 +1,187 @@ +"""Tool connector selections preserve pinned servers and per-node allow-lists.""" + +import copy +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from asaree.services.design_generation import get_design_impact +from asaree.services.factor_bindings import validate_factor_bindings +from asaree.services.protocol_execution import _resolve_tool_config, apply_factor_bindings +from asaree.services.tool_factors import test_tool_factor_values as preview_values +from asaree.services.tool_factors import validate_tool_factors + + +def tool_graph(mode="tool_selection"): + ids = ["tool-a", "tool-b"] + graph = { + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": { + "factor_bindings": {"tool_selection": "Tools"}, + "tool_selection": ids if mode == "tool_toggle" else [ids[0]], + "tool_factor_mode": mode, + }, + }, + *[ + { + "id": id, + "type": kind, + "data": { + "config": { + "server_id": str(uuid.uuid4()), + "server_name": id, + "tool_names": ["search", "fetch"], + "enabled": False, + } + }, + } + for id, kind in zip(ids, ["mcp_tool", "mcp_client_tool"], strict=True) + ], + {"id": "script", "type": "script", "data": {"config": {"code": "print(1)"}}}, + ], + "edges": [{"source": id, "target": "agent", "targetHandle": "tool"} for id in [*ids, "script"]], + } + spec = { + "factors": [ + { + "name": "Tools", + "level_type": mode, + "levels": [ids, []] if mode == "tool_toggle" else [[id] for id in ids], + } + ] + } + return graph, spec, ids + + +@pytest.mark.parametrize("mode", ["tool_selection", "tool_toggle"]) +def test_cell_selection_overrides_disabled_nodes_and_preserves_allow_lists(mode): + graph, spec, ids = tool_graph(mode) + original = copy.deepcopy(graph) + validate_factor_bindings(spec, graph) + for chosen in ([ids[1]], ids, []): + resolved = _resolve_tool_config(apply_factor_bindings(graph, {"Tools": chosen}), "agent") + assert resolved == { + "server_names": chosen, + "tool_names": [f"{id}.{name}" for id in chosen for name in ["search", "fetch"]], + } + assert graph == original + + +def test_shared_node_selection_is_scoped_to_its_agent(): + graph, _, ids = tool_graph() + graph["nodes"].append({"id": "other", "type": "agent", "data": {}}) + graph["edges"].append({"source": ids[0], "target": "other", "targetHandle": "tool"}) + patched = apply_factor_bindings(graph, {"Tools": [ids[0]]}) + assert _resolve_tool_config(patched, "agent")["server_names"] == [ids[0]] + assert _resolve_tool_config(patched, "other")["server_names"] == [] + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_individual_enabled_and_allow_list_factors_still_work(enabled): + graph, _, ids = tool_graph() + graph["nodes"][0]["data"] = {} + graph["nodes"][1]["data"]["factor_bindings"] = {"config.enabled": "On", "config.tool_names": "Allowed"} + resolved = _resolve_tool_config(apply_factor_bindings(graph, {"On": enabled, "Allowed": ["fetch"]}), "agent") + assert resolved["tool_names"] == ([f"{ids[0]}.fetch"] if enabled else []) + + +@pytest.mark.parametrize("change", ["one", "missing", "duplicate", "foreign", "conflict", "mode"]) +def test_invalid_or_conflicting_design_is_rejected(change): + graph, spec, ids = tool_graph() + factor = spec["factors"][0] + if change == "one": + graph["edges"].pop(1) + factor["levels"] = [[ids[0]]] + elif change == "missing": + factor["levels"].pop() + elif change == "duplicate": + factor["levels"] = [[ids[0]], [ids[0]]] + elif change == "foreign": + factor["levels"] = [[ids[0]], ["script"]] + elif change == "mode": + graph["nodes"][0]["data"]["tool_factor_mode"] = "tool_toggle" + else: + graph["nodes"][1]["data"]["factor_bindings"] = {"config.tool_names": "Allowed"} + with pytest.raises(ValueError): + validate_factor_bindings(spec, graph) + + +@pytest.mark.parametrize( + "levels", [[["tool-a"], []], [["tool-a", "tool-b", "tool-a"], []], [["tool-a", "tool-b"], ["tool-a", "tool-b"]]] +) +def test_toggle_requires_all_connected_nodes_or_none(levels): + graph, spec, _ = tool_graph("tool_toggle") + spec["factors"][0]["levels"] = levels + with pytest.raises(ValueError, match="every connected tool or none"): + validate_factor_bindings(spec, graph) + + +@pytest.mark.asyncio +async def test_preview_defaults_choices_and_availability(monkeypatch): + graph, spec, ids = tool_graph() + owner = uuid.uuid4() + server = SimpleNamespace( + owner_id=owner, is_system=False, capabilities={"tools": [{"name": "search"}, {"name": "fetch"}]} + ) + get = AsyncMock(return_value=server) + monkeypatch.setattr("asaree.services.tool_factors.mcp_service.get_server", get) + assert await preview_values(graph, None, owner) == {"Tools": [ids[0]]} + assert await preview_values(graph, {"agent": ids[1]}, owner, "agent") == {"Tools": [ids[1]]} + await validate_tool_factors(spec, graph, owner) + with pytest.raises(ValueError, match="belong"): + await preview_values(graph, {"other": ids[0]}, owner) + with pytest.raises(ValueError, match="connected"): + await preview_values(graph, {"agent": "script"}, owner) + get.return_value = None + with pytest.raises(ValueError, match="unavailable"): + await preview_values(graph, None, owner) + + +@pytest.mark.asyncio +async def test_toggle_none_does_not_require_servers_and_all_checks_every_server(monkeypatch): + graph, spec, ids = tool_graph("tool_toggle") + owner = uuid.uuid4() + get = AsyncMock(return_value=None) + monkeypatch.setattr("asaree.services.tool_factors.mcp_service.get_server", get) + assert await preview_values(graph, {"agent": "none"}, owner) == {"Tools": []} + get.assert_not_awaited() + with pytest.raises(ValueError, match="unavailable"): + await preview_values(graph, {"agent": "all"}, owner) + with pytest.raises(ValueError, match="all enabled"): + await preview_values(graph, {"agent": ids[0]}, owner) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("invalid", ["other-owner", "empty", "unknown-tool"]) +async def test_unavailable_server_or_allow_list_is_rejected(monkeypatch, invalid): + graph, spec, _ = tool_graph() + owner = uuid.uuid4() + server = SimpleNamespace( + owner_id=uuid.uuid4() if invalid == "other-owner" else owner, + is_system=False, + capabilities={"tools": [{"name": "search"}, {"name": "fetch"}]}, + ) + if invalid == "empty": + graph["nodes"][1]["data"]["config"]["tool_names"] = [] + elif invalid == "unknown-tool": + graph["nodes"][1]["data"]["config"]["tool_names"] = ["deleted"] + monkeypatch.setattr("asaree.services.tool_factors.mcp_service.get_server", AsyncMock(return_value=server)) + with pytest.raises(ValueError): + await validate_tool_factors(spec, graph, owner) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode,levels", [("tool_selection", []), ("tool_toggle", [[], []])]) +async def test_incomplete_design_has_zero_impact(monkeypatch, mode, levels): + monkeypatch.setattr("asaree.services.design_generation.get_current_revision", AsyncMock(return_value=None)) + impact = await get_design_impact( + AsyncMock(), + experiment_id=uuid.uuid4(), + design_spec={"factors": [{"name": "Tools", "level_type": mode, "levels": levels}]}, + ) + assert impact.proposed_cell_count == 0 From c74f37aa4a5449e5d70cab7cbbeaf9fe9e111fe7 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 16:25:45 -0700 Subject: [PATCH 15/63] fix: validate factor treatments and preserve unbound declarations --- .../protocol/ProtocolCanvas.test.tsx | 19 +++----- .../components/protocol/ProtocolCanvas.tsx | 44 +------------------ frontend/src/lib/scriptFactors.test.ts | 15 ++++++- frontend/src/lib/scriptFactors.ts | 2 + src/asaree/services/protocol_execution.py | 21 +++++++-- src/asaree/services/script_factors.py | 3 ++ tests/test_script_factors.py | 20 +++++++++ tests/test_sub_agent_factors.py | 30 +++++++++++++ 8 files changed, 95 insertions(+), 59 deletions(-) diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 4f8ef1db..d4f9c1f6 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -584,7 +584,7 @@ describe('ProtocolCanvas connector adds', () => { expect(screen.queryByRole('button', { name: /Create (another )?metric/ })).not.toBeInTheDocument() }) - it.each([false, true])('removes a Skill factor declaration only after its last binding is removed (shared: %s)', async (shared) => { + it.each([false, true])('keeps factor declarations when a bound Skill node is deleted (shared: %s)', async (shared) => { const factorName = 'Skill enabled' const experiment: Experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, @@ -607,21 +607,16 @@ describe('ProtocolCanvas connector adds', () => { edges: [], } const { client } = renderCanvas(graph, experiment.id) - const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') - fireEvent.click(makeFactor!) - fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) + fireEvent.doubleClick(await screen.findByText('Skill a')) + fireEvent.click(await screen.findByRole('button', { name: 'Delete node' })) + fireEvent.click(await screen.findByRole('button', { name: 'Delete' })) await waitFor(() => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) - expect(persisted?.nodes.find((node) => node.id === 'a')?.data.factor_bindings).toEqual({}) + expect(persisted?.nodes.some((node) => node.id === 'a')).toBe(false) }) - if (shared) { - expect(save).not.toHaveBeenCalled() - } else { - await waitFor(() => expect(save).toHaveBeenCalledWith(experiment.id, { - design_spec: { factors: [], metrics: [] }, - })) - } + expect(save).not.toHaveBeenCalled() + expect(client.getQueryData(['experiments', experiment.id])?.design_spec?.factors).toEqual(experiment.design_spec?.factors) }) it('opens a bound factor from the Script inspector', async () => { diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index a3f4905d..9b905b15 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -2166,34 +2166,8 @@ export const ProtocolCanvas = forwardRef Object.values((node.data as { factor_bindings?: Record }).factor_bindings ?? {})), - ) - const orphaned = removedNames.filter((name) => !stillBound.has(name)) - if (orphaned.length === 0) return - const experiment = await experimentsApi.get(experimentId) - const factors = experiment.design_spec?.factors ?? [] - const nextFactors = factors.filter((factor) => !orphaned.includes(factor.name)) - if (nextFactors.length === factors.length) return - await experimentsApi.update(experimentId, { design_spec: { ...experiment.design_spec, factors: nextFactors } }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) - } - - function removeFactorsForDeletedNodes(deleting: Node[], nextNodes: Node[]) { - const removedNames = [...new Set( - deleting.flatMap((node) => Object.values((node.data as { factor_bindings?: Record }).factor_bindings ?? {})), - )] - void removeUnboundFactors(nextNodes, removedNames) - } - + // Removing a node or field leaves its factor declared and unbound for + // explicit design review. Factor dialogs handle intentional factor removal. function updateNodeData( nodeId: string, data: @@ -2213,15 +2187,8 @@ export const ProtocolCanvas = forwardRef node.id === nodeId) - const oldBindings = (previous?.data as { factor_bindings?: Record } | undefined)?.factor_bindings ?? {} - const nextBindings = (data as { factor_bindings?: Record }).factor_bindings ?? {} - const removedNames = Object.entries(oldBindings) - .filter(([path, name]) => nextBindings[path] !== name) - .map(([, name]) => name) const nextNodes = nodes.map((node) => (node.id === nodeId ? { ...node, data } : node)) setNodes(nextNodes) - void removeUnboundFactors(nextNodes, removedNames) } // Client-side guardrail mirroring the backend's own connector validation @@ -2235,12 +2202,10 @@ export const ProtocolCanvas = forwardRef node.id === nodeId) const nextNodes = nodes.filter((node) => node.id !== nodeId) setNodes(nextNodes) setEdges((eds) => eds.filter((e) => e.source !== nodeId && e.target !== nodeId)) setSelectedNodeId(null) - removeFactorsForDeletedNodes(deleting, nextNodes) } // The node inspector's own Delete button calls deleteNode directly -- @@ -2630,11 +2595,6 @@ export const ProtocolCanvas = forwardRef { if (pendingDelete.resolve) { - const deletedNodeIds = new Set(pendingDelete.nodes.map((node) => node.id)) - removeFactorsForDeletedNodes( - pendingDelete.nodes, - nodes.filter((node) => !deletedNodeIds.has(node.id)), - ) pendingDelete.resolve({ nodes: pendingDelete.nodes, edges: pendingDelete.edges }) } else { // requestDeleteNode's path -- no xyflow deletion pending, just diff --git a/frontend/src/lib/scriptFactors.test.ts b/frontend/src/lib/scriptFactors.test.ts index 38a28ebd..41195728 100644 --- a/frontend/src/lib/scriptFactors.test.ts +++ b/frontend/src/lib/scriptFactors.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { reconcileFactorBaselines } from './factorBindings' -import { reconcileScriptFactor, scriptFactorIssues, scriptFactorOwner } from './scriptFactors' +import { reconcileScriptFactor, requiredScriptIssues, scriptFactorIssues, scriptFactorOwner } from './scriptFactors' import type { DesignFactor } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' @@ -14,6 +14,19 @@ const graph = { const factor: DesignFactor = { name: 'Scripts', level_type: 'script_selection', levels: [['b'], ['a']], level_labels: ['Custom B', 'Custom A'] } describe('script connector factors', () => { + it.each(['', ' ', null])('reports an empty code treatment for a required Script (%s)', (code) => { + const requiredGraph: ProtocolGraph = { + nodes: [ + { id: 'step', type: 'tool_step', position: { x: 0, y: 0 }, data: { label: 'step', config: { tool_name: 'execute', arguments: { code: { source: 'script_code' } } } } }, + { id: 'script', type: 'script', position: { x: 0, y: 0 }, data: { label: 'Script', config: { name: 'Script', language: 'python', code: 'print(1)' }, factor_bindings: { 'config.code': 'Code' } } }, + ], + edges: [{ id: 'script-step', source: 'script', target: 'step', targetHandle: 'tool' }], + } + expect(requiredScriptIssues(requiredGraph, [{ name: 'Code', levels: ['print(1)', code] }])).toEqual([ + 'step: requires script_code; every Script code level needs code.', + ]) + expect(requiredScriptIssues(requiredGraph, [{ name: 'Code', levels: ['print(1)', 'print(2)'] }])).toEqual([]) + }) it('keeps all-or-none levels in their chosen order as connections change', () => { const toggle: DesignFactor = { name: 'Scripts', level_type: 'script_toggle', levels: [['a', 'b'], []], level_labels: ['With scripts', 'Without scripts'] } expect(reconcileScriptFactor(toggle, graph, 'agent')).toEqual({ ...toggle, levels: [['a', 'b', 'c'], []] }) diff --git a/frontend/src/lib/scriptFactors.ts b/frontend/src/lib/scriptFactors.ts index ef0db498..4f9eb315 100644 --- a/frontend/src/lib/scriptFactors.ts +++ b/frontend/src/lib/scriptFactors.ts @@ -109,6 +109,8 @@ export function requiredScriptIssues(graph: ProtocolGraph | undefined, factors: const bindings = script.data.factor_bindings ?? {} const enabled = factors.find((factor) => factor.name === bindings['config.enabled']) if (enabled?.levels.some((level) => level !== true)) return [`${prefix}; its Script factor cannot include off.`] + const code = factors.find((factor) => factor.name === bindings['config.code']) + if (code?.levels.some((level) => !String(level ?? '').trim())) return [`${prefix}; every Script code level needs code.`] const variants = factors.find((factor) => factor.name === bindings.config)?.levels ?? [script.data.config] if (variants.some((level) => { const config = level as { code?: string; enabled?: boolean } | null diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 5d0e0365..a57ef8ac 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -1042,7 +1042,9 @@ def _kahn_order(graph: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], list[ return nodes, ordered, complete -def topological_order(graph: dict[str, Any], *, require_acyclic: bool = True) -> list[dict[str, Any]]: +def topological_order( + graph: dict[str, Any], *, require_acyclic: bool = True, validate_all_sub_agents: bool = False +) -> list[dict[str, Any]]: """Kahn's algorithm. Raises :class:`ProtocolValidationError` on an empty graph, a cycle (any node Kahn's algorithm can't reach stays with a nonzero in-degree, which is exactly the cycle signature), or a malformed @@ -1065,6 +1067,17 @@ def topological_order(graph: dict[str, Any], *, require_acyclic: bool = True) -> except DatasetRowInputError as exc: raise ProtocolValidationError(str(exc)) from exc _, downstream, _ = _adjacency(graph) + # Production must validate workers from every connector level, including + # those disabled in the baseline. Previews only need their selected workers. + factor_parents = { + node["id"] for node in graph.get("nodes") or [] + if node.get("type") == "agent" + and ((node.get("data") or {}).get("factor_bindings") or {}).get("sub_agent_selection") + } if validate_all_sub_agents else set() + selectable_sub_agents = { + edge.get("source") for edge in graph.get("edges") or [] + if edge.get("target") in factor_parents and edge.get("targetHandle") == "sub_agents" + } graph = graph_with_sub_agent_selection(graph) nodes, ordered, complete = _kahn_order(graph) if not nodes: @@ -1107,7 +1120,7 @@ def topological_order(graph: dict[str, Any], *, require_acyclic: bool = True) -> sub_agent_is_callable = ( node_type == "sub_agent" - and _is_node_active(node) + and (_is_node_active(node) or nid in selectable_sub_agents) and bool(_edges_with_handle(graph, nid, "sub_agents", direction="outgoing")) ) if node_type in ("agent", "critic_gate") or sub_agent_is_callable: @@ -4786,7 +4799,7 @@ async def plan_cell_runs( validate_stage_plan(design_spec) validate_prompt_references(graph=graph) conversation = is_conversation_strategy(design_spec) - topological_order(graph, require_acyclic=not conversation) # also raises on an empty graph + topological_order(graph, require_acyclic=not conversation, validate_all_sub_agents=True) if not conversation: sinks = sink_node_ids(graph) if len(sinks) != 1: @@ -5176,7 +5189,7 @@ async def plan_single_replicate_run( validate_stage_plan(design_spec) validate_prompt_references(graph=graph) conversation = is_conversation_strategy(design_spec) - topological_order(graph, require_acyclic=not conversation) # also raises on an empty graph + topological_order(graph, require_acyclic=not conversation, validate_all_sub_agents=True) if not conversation: sinks = sink_node_ids(graph) if len(sinks) != 1: diff --git a/src/asaree/services/script_factors.py b/src/asaree/services/script_factors.py index e9b5af25..5014f129 100644 --- a/src/asaree/services/script_factors.py +++ b/src/asaree/services/script_factors.py @@ -155,6 +155,9 @@ def validate_tool_step_script_factors(design_spec: dict[str, Any] | None, graph: enabled_factor = factors.get(bindings.get("config.enabled")) if enabled_factor and any(level is not True for level in enabled_factor.get("levels") or []): raise ValueError(f"Tool Step {label!r} requires script_code: its Script factor cannot include off.") + code_factor = factors.get(bindings.get("config.code")) + if code_factor and any(not str(level or "").strip() for level in code_factor.get("levels") or []): + raise ValueError(f"Tool Step {label!r} requires script_code: every Script code level needs code.") for variant in variants: if ( not isinstance(variant, dict) diff --git a/tests/test_script_factors.py b/tests/test_script_factors.py index 52da8b38..0e4a7b47 100644 --- a/tests/test_script_factors.py +++ b/tests/test_script_factors.py @@ -19,6 +19,26 @@ from asaree.services.script_factors import validate_script_factors, validate_tool_step_script_factors +@pytest.mark.parametrize("code", ["", " ", None]) +def test_required_script_rejects_empty_individual_code_levels(code): + graph = { + "nodes": [ + {"id": "step", "type": "tool_step", "data": {"config": { + "arguments": {"code": {"source": "script_code"}}, + }}}, + {"id": "script", "type": "script", "data": { + "config": {"code": "print(1)"}, "factor_bindings": {"config.code": "Code"}, + }}, + ], + "edges": [{"source": "script", "target": "step", "targetHandle": "tool"}], + } + factor = {"name": "Code", "level_type": "text", "levels": ["print(1)", code]} + with pytest.raises(ValueError, match="requires script_code"): + validate_tool_step_script_factors({"factors": [factor]}, graph) + factor["levels"] = ["print(1)", "print(2)"] + validate_tool_step_script_factors({"factors": [factor]}, graph) + + def script_graph(mode="script_selection"): ids = ["a", "b"] graph = { diff --git a/tests/test_sub_agent_factors.py b/tests/test_sub_agent_factors.py index f5b34169..fc680461 100644 --- a/tests/test_sub_agent_factors.py +++ b/tests/test_sub_agent_factors.py @@ -2,6 +2,8 @@ import copy import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -10,6 +12,8 @@ ProtocolValidationError, _sub_agent_ids, apply_factor_bindings, + plan_cell_runs, + plan_single_replicate_run, topological_order, ) from asaree.services.sub_agent_factors import test_sub_agent_factor_values as preview_values @@ -102,3 +106,29 @@ def test_selected_disabled_worker_requires_a_model_at_validation(): graph["edges"].append({"source": "model", "target": "parent", "targetHandle": "model"}) with pytest.raises(ProtocolValidationError, match="exactly one Model"): topological_order(graph) + + +@pytest.mark.parametrize("mode", ["sub_agent_selection", "sub_agent_toggle"]) +@pytest.mark.parametrize("planner", [plan_cell_runs, plan_single_replicate_run]) +@pytest.mark.asyncio +async def test_production_validation_requires_models_for_every_selectable_worker(monkeypatch, mode, planner): + graph, spec = worker_graph() + graph["nodes"][0]["data"].update(sub_agent_factor_mode=mode) + graph["nodes"].extend([ + {"id": "parent-model", "type": "model_openai", "data": {"config": {}}}, + {"id": "a-model", "type": "model_openai", "data": {"config": {}}}, + ]) + graph["edges"].extend([ + {"source": "parent-model", "target": "parent", "targetHandle": "model"}, + {"source": "a-model", "target": "a", "targetHandle": "model"}, + ]) + # A preview of a configured worker remains runnable. + topological_order(graph) + experiment = SimpleNamespace(design_spec=spec, measurement_plan=None, locked_at=None) + monkeypatch.setattr("asaree.services.protocol_execution.get_experiment", AsyncMock(return_value=experiment)) + kwargs = {"replicate_label": "rep1"} if planner is plan_single_replicate_run else {} + with pytest.raises(ProtocolValidationError, match="exactly one Model"): + await planner( + AsyncMock(), protocol_id=uuid.uuid4(), experiment_id=uuid.uuid4(), + owner_id=uuid.uuid4(), graph=graph, **kwargs, + ) From d6027ac24916ee33c8ac4aafd86ec2e9537a904e Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 16:39:38 -0700 Subject: [PATCH 16/63] fix: honor connector factor selections throughout execution --- .../protocol/ProtocolCanvas.test.tsx | 44 +++++++++ .../components/protocol/ProtocolCanvas.tsx | 2 +- src/asaree/services/protocol_execution.py | 93 +++++++++++-------- tests/test_knowledge_factors.py | 24 ++++- tests/test_stage_plans.py | 28 ++++++ 5 files changed, 148 insertions(+), 43 deletions(-) diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index d4f9c1f6..0104b003 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -351,6 +351,50 @@ describe('ProtocolCanvas connector adds', () => { }) }) + it.each(['dataset', 'resource', 'tool'])('removes a Dataset factor on %s without enabling unrelated Tools or Scripts', async (handle) => { + let experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [{ name: 'Datasets', level_type: 'dataset_toggle', levels: [['dataset-id'], []], level_labels: ['All enabled', 'All disabled'] }], metrics: [] }, + measurement_plan: null, dataset_ids: ['dataset-id'], dataset_id: 'dataset-id', + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + experiment = { ...experiment, ...patch } + return experiment + }) + vi.spyOn(datasetsApi, 'list').mockResolvedValue([]) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: { ...defaultAgentNodeData('Writer'), dataset_selection: ['dataset-id'], dataset_factor_mode: 'dataset_toggle', factor_bindings: { dataset_selection: 'Datasets' } } }, + { id: 'dataset', type: 'dataset', position: { x: 100, y: 0 }, data: { ...defaultDatasetNodeData(), config: { dataset_id: 'dataset-id', dataset_name: 'Dataset', enabled: false } } }, + { id: 'tool', type: 'mcp_tool', position: { x: 300, y: 0 }, data: { label: 'Tool', config: { server_id: 'server', server_name: 'Server', tool_names: ['run'], enabled: false } } }, + { id: 'script', type: 'script', position: { x: 500, y: 0 }, data: { ...defaultScriptNodeData(), config: { name: 'Script', language: 'python', code: 'print(1)', enabled: false } } }, + ], + edges: [ + { id: 'dataset-edge', source: 'dataset', target: 'agent', targetHandle: handle }, + ...['tool', 'script'].map((id) => ({ id: `${id}-edge`, source: id, target: 'agent', targetHandle: 'tool' })), + ], + } + const { client } = renderCanvas(graph, 'experiment-1') + await waitFor(() => expect(client.getQueryData(['experiments', 'experiment-1'])?.design_spec?.factors?.[0]?.name).toBe('Datasets')) + fireEvent.click(document.querySelector('[data-testid="rf__node-dataset"] button[aria-label="Make experimental factor"]')!) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.dataset_selection).toBeUndefined() + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.dataset_factor_mode).toBeUndefined() + expect(persisted.nodes.find((node) => node.id === 'dataset')?.data.config).toMatchObject({ enabled: true }) + for (const id of ['tool', 'script']) { + expect(persisted.nodes.find((node) => node.id === id)?.data.config).toEqual(graph.nodes.find((node) => node.id === id)?.data.config) + } + expect(experiment.design_spec?.factors).toEqual([]) + }) + }) + it.each([null, 1])('selects a draft row for Publish & Test Run with published version %s', async publishedRevision => { const dataset = defaultDatasetNodeData() dataset.config.dataset_id = '11111111-1111-4111-8111-111111111111' diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 9b905b15..86fa9e8d 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -436,7 +436,7 @@ export const ProtocolCanvas = forwardRef edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => MCP_TOOL_NODE_TYPES.includes(nds.find((node) => node.id === id)?.type ?? ''))) const scriptIds = new Set(edges.filter((edge) => edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => nds.find((node) => node.id === id)?.type === 'script')) const skillIds = new Set(edges.filter((edge) => edge.targetHandle === 'skill' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factorName).map((edge) => edge.source)) - const datasetIds = new Set(edges.filter((edge) => ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '') && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factorName).map((edge) => edge.source)) + const datasetIds = new Set(edges.filter((edge) => ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '') && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => nds.find((node) => node.id === id)?.type === 'dataset')) return nds.map((n) => { if (subAgentIds.has(n.id)) return { ...n, data: { ...n.data, active: true } } if (scriptIds.has(n.id) || toolIds.has(n.id) || skillIds.has(n.id) || datasetIds.has(n.id) || knowledgeIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index a57ef8ac..c39440ae 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -608,13 +608,8 @@ def derive_stage_plan(graph: dict[str, Any]) -> Any: node = nodes[nid] if node.get("type") not in ("agent", "sub_agent"): continue - for edge in _edges_with_handle(graph, nid, "tool", direction="incoming"): - source = nodes.get(str(edge.get("source"))) - if source is None or source.get("type") not in _MCP_TOOL_NODE_TYPES: - continue + for source in _resolve_tool_nodes(graph, nid): config = (source.get("data") or {}).get("config") or {} - if not config.get("enabled", True): - continue stage_id = STAGE_WRITING_SERVERS.get(str(config.get("server_name") or "")) if stage_id is not None and stage_id not in stage_ids: stage_ids.append(stage_id) @@ -2468,6 +2463,32 @@ def _resolve_pattern_config(graph: dict[str, Any], node_id: str) -> dict[str, An return {"execution_pattern": slug, "pattern_params": {slug: (source.get("data") or {}).get("config") or {}}} +def _resolve_tool_nodes(graph: dict[str, Any], node_id: str) -> list[dict[str, Any]]: + """Tool sources selected for this agent, in connector wiring order. + + A bound selection overrides source enabled flags for this agent only. + Stage planning and tool access must use the same treatment. + """ + nodes, _downstream, _upstream = _adjacency(graph) + agent_data = (nodes.get(node_id) or {}).get("data") or {} + selected = ( + agent_data.get("tool_selection") + if (agent_data.get("factor_bindings") or {}).get("tool_selection") else None + ) + sources: list[dict[str, Any]] = [] + for edge in _edges_with_handle(graph, node_id, "tool", direction="incoming"): + source = nodes.get(edge.get("source")) + if source is None or source.get("type") not in _MCP_TOOL_NODE_TYPES: + continue + config = (source.get("data") or {}).get("config") or {} + if selected is not None and source["id"] not in selected: + continue + if selected is None and not config.get("enabled", True): + continue + sources.append(source) + return sources + + def _resolve_tool_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: """``{"server_names": [...], "tool_names": [...]}`` -- built from every ``mcp_tool`` node wired into this agent's Tool connector -- replaces the @@ -2502,23 +2523,10 @@ def _resolve_tool_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: server's tools are offered, never which server). An empty allow-list is a meaningful level: the server still connects (its ``server_name`` is still reported) but contributes no tools to that cell.""" - nodes, _downstream, _upstream = _adjacency(graph) - agent_data = (nodes.get(node_id) or {}).get("data") or {} - selected = ( - agent_data.get("tool_selection") - if (agent_data.get("factor_bindings") or {}).get("tool_selection") else None - ) server_names: list[str] = [] tool_names: list[str] = [] - for edge in _edges_with_handle(graph, node_id, "tool", direction="incoming"): - source = nodes.get(edge["source"]) - if source is None or source.get("type") not in _MCP_TOOL_NODE_TYPES: - continue + for source in _resolve_tool_nodes(graph, node_id): tool_node_config = (source.get("data") or {}).get("config") or {} - if selected is not None and source["id"] not in selected: - continue - if selected is None and not tool_node_config.get("enabled", True): - continue server_name = tool_node_config.get("server_name") node_tool_names = tool_node_config.get("tool_names") or [] if server_name: @@ -2527,6 +2535,28 @@ def _resolve_tool_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: return {"server_names": server_names, "tool_names": tool_names} +def _resolve_knowledge_nodes(graph: dict[str, Any], node_id: str) -> list[dict[str, Any]]: + """Knowledge sources selected for this agent's tools and prompt catalog.""" + nodes, _downstream, _upstream = _adjacency(graph) + agent_data = (nodes.get(node_id) or {}).get("data") or {} + selection = ( + agent_data.get("knowledge_selection") + if (agent_data.get("factor_bindings") or {}).get("knowledge_selection") else None + ) + sources: list[dict[str, Any]] = [] + for edge in _edges_with_handle(graph, node_id, "knowledge", direction="incoming"): + source = nodes.get(edge["source"]) + if source is None or source.get("type") not in _KNOWLEDGE_NODE_TYPES: + continue + config = (source.get("data") or {}).get("config") or {} + if selection is not None and knowledge_id(source) not in selection: + continue + if selection is None and not config.get("enabled", True): + continue + sources.append(source) + return sources + + def _resolve_knowledge_config(graph: dict[str, Any], node_id: str) -> dict[str, Any]: """The Knowledge connector's contribution to the agent's tool allow-list, shaped exactly like ``_resolve_tool_config``'s so the two can be merged. @@ -2552,24 +2582,11 @@ def _resolve_knowledge_config(graph: dict[str, Any], node_id: str) -> dict[str, De-duplicated by server: two nodes pointing at the same bundle are one server, and listing it twice would just double every tool name.""" - nodes, _downstream, _upstream = _adjacency(graph) - agent_data = (nodes.get(node_id) or {}).get("data") or {} - selection = ( - agent_data.get("knowledge_selection") - if (agent_data.get("factor_bindings") or {}).get("knowledge_selection") else None - ) server_names: list[str] = [] tool_names: list[str] = [] tool_descriptions: dict[str, str] = {} - for edge in _edges_with_handle(graph, node_id, "knowledge", direction="incoming"): - source = nodes.get(edge["source"]) - if source is None or source.get("type") not in _KNOWLEDGE_NODE_TYPES: - continue + for source in _resolve_knowledge_nodes(graph, node_id): bundle_config = (source.get("data") or {}).get("config") or {} - if selection is not None and knowledge_id(source) not in selection: - continue - if selection is None and not bundle_config.get("enabled", True): - continue server_name = bundle_config.get("server_name") if not server_name or server_name in server_names: continue @@ -3443,15 +3460,9 @@ def _resource_catalog( lines.append(f"- {name}: {detail}" if detail else f"- {name}") sections.append("Available datasets:\n" + "\n".join(lines)) - nodes, _downstream, _upstream = _adjacency(graph) knowledge_lines: list[str] = [] - for edge in _edges_with_handle(graph, node_id, "knowledge", direction="incoming"): - source = nodes.get(edge["source"]) - if source is None or source.get("type") not in _KNOWLEDGE_NODE_TYPES: - continue + for source in _resolve_knowledge_nodes(graph, node_id): config = (source.get("data") or {}).get("config") or {} - if not config.get("enabled", True): - continue label = str( config.get("document_title") or config.get("bundle_label") diff --git a/tests/test_knowledge_factors.py b/tests/test_knowledge_factors.py index 2c9a0a69..da1931b8 100644 --- a/tests/test_knowledge_factors.py +++ b/tests/test_knowledge_factors.py @@ -10,7 +10,7 @@ from asaree.services.factor_bindings import validate_factor_bindings from asaree.services.knowledge_factors import test_knowledge_factor_values as preview_values from asaree.services.knowledge_factors import validate_knowledge_factors -from asaree.services.protocol_execution import _resolve_knowledge_config, apply_factor_bindings +from asaree.services.protocol_execution import _build_user_input, _resolve_knowledge_config, apply_factor_bindings def knowledge_graph(mode="knowledge_selection"): @@ -67,6 +67,28 @@ def test_treatments_override_disabled_sources_and_preserve_graph(mode): assert graph == original +@pytest.mark.parametrize("mode", ["knowledge_selection", "knowledge_toggle"]) +@pytest.mark.parametrize("enabled", [False, True]) +def test_prompt_only_describes_selected_knowledge_sources(mode, enabled): + graph, _, ids = knowledge_graph(mode) + for index, source in enumerate(graph["nodes"][1:]): + source["data"]["config"].update( + enabled=enabled, + document_title=f"Knowledge source {index}", + document_description=f"Description {index}", + document_tags=[f"tag-{index}"], + ) + original = copy.deepcopy(graph) + choices = [[ids[0]], [ids[1]]] if mode == "knowledge_selection" else [ids, []] + for selected in choices: + patched = apply_factor_bindings(graph, {"Knowledge": selected}) + prompt = _build_user_input(patched["nodes"][0], patched, {}) + for index, source_id in enumerate(ids): + for metadata in [f"Knowledge source {index}", f"Description {index}", f"tag-{index}"]: + assert (metadata in prompt) is (source_id in selected) + assert graph == original + + @pytest.mark.parametrize("enabled", [False, True]) def test_individual_treatment_changes_only_one_source(enabled): graph, _, _ = knowledge_graph() diff --git a/tests/test_stage_plans.py b/tests/test_stage_plans.py index 5de04326..8e888480 100644 --- a/tests/test_stage_plans.py +++ b/tests/test_stage_plans.py @@ -445,6 +445,34 @@ def test_a_disabled_tool_node_is_not_a_stage() -> None: assert pe.derive_stage_plan(_staged_graph("asaree-sklearn-dc", enabled=False)) is None +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize("mode", ["tool_selection", "tool_toggle"]) +def test_tool_factor_stages_follow_the_selected_tools(enabled: bool, mode: str) -> None: + graph = _staged_graph("asaree-sklearn-dc", "asaree-sklearn-fte", enabled=enabled) + # Both alternatives belong to the same Agent connector. + graph["nodes"] = [node for node in graph["nodes"] if node["id"] != "a1"] + graph["edges"] = [edge for edge in graph["edges"] if edge.get("targetHandle") == "tool"] + for edge in graph["edges"]: + edge["target"] = "a0" + graph["nodes"][0]["data"].update( + factor_bindings={"tool_selection": "Tools"}, tool_factor_mode=mode, tool_selection=["t0"] + ) + for node in graph["nodes"][1:]: + node["data"]["config"]["tool_names"] = ["run"] + + choices = [["t0"], ["t1"]] if mode == "tool_selection" else [["t0", "t1"], []] + for chosen in choices: + patched = pe.apply_factor_bindings(graph, {"Tools": chosen}) + plan = pe.stage_plan_spec({}, graph=patched) + if not chosen: + assert plan is None + continue + stages = resolve_stage_plan(plan) + assert stages.ids == [stage for node_id, stage in [("t0", "dc"), ("t1", "fte")] if node_id in chosen] + assert stages.previous(stages.ids[0]) is None + assert stages.stages[0].version_id == f"v1_{stages.ids[0]}" + + def test_a_declared_plan_wins_over_the_canvas() -> None: """The SDK escape hatch. Deriving is how the GUI gets a plan, because there is no field for one; a notebook that names a plan outright is describing a From 43d042fb3cfb4e8b58ce7fc655d20a0077cd845a Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 16:54:12 -0700 Subject: [PATCH 17/63] Add Agent factor dialog and enforce activity across coordination strategies --- .../protocol/AgentFactorDialog.test.tsx | 71 +++++++++ .../components/protocol/AgentFactorDialog.tsx | 88 +++++++++++ .../protocol/AgentNodeInspector.tsx | 11 +- .../protocol/FactorBindableField.tsx | 5 + .../protocol/ProtocolCanvas.test.tsx | 46 +++++- .../components/protocol/ProtocolCanvas.tsx | 57 +++++++- .../protocol/ProtocolCanvasContext.tsx | 1 + .../src/components/protocol/bindableFields.ts | 15 +- .../components/protocol/nodes/AgentNode.tsx | 4 +- src/asaree/services/protocol_execution.py | 40 ++++- tests/test_agent_factors.py | 137 ++++++++++++++++++ tests/test_supervisor_architecture.py | 25 ++++ 12 files changed, 480 insertions(+), 20 deletions(-) create mode 100644 frontend/src/components/protocol/AgentFactorDialog.test.tsx create mode 100644 frontend/src/components/protocol/AgentFactorDialog.tsx create mode 100644 tests/test_agent_factors.py diff --git a/frontend/src/components/protocol/AgentFactorDialog.test.tsx b/frontend/src/components/protocol/AgentFactorDialog.test.tsx new file mode 100644 index 00000000..a925e26c --- /dev/null +++ b/frontend/src/components/protocol/AgentFactorDialog.test.tsx @@ -0,0 +1,71 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import { EMPTY_SCOPE } from '@/lib/promptReferences' +import { defaultAgentNodeData, type AgentNodeData, type ProtocolNode } from '@/types/protocols' +import { AgentFactorDialog } from './AgentFactorDialog' +import { factorCreationFields } from './bindableFields' + +const node: ProtocolNode & { data: AgentNodeData } = { + id: 'writer', type: 'agent', position: { x: 0, y: 0 }, + data: { ...defaultAgentNodeData('Writer'), config: { ...defaultAgentNodeData().config, prompt: 'Summarize {{node:source}}' } }, +} + +it('routes the three Agent choices from New Factor and keeps Sub-Agent choices separate', () => { + const fields = factorCreationFields([node], []) + expect(fields.map((field) => field.fieldLabel)).toEqual(['This agent on/off', 'Prompt levels', 'System prompt levels']) + expect(fields.every((field) => field.connectorFactor?.kind === 'agent' && field.pickerGroup?.category === 'Agent' && field.pickerGroup?.componentId === node.id)).toBe(true) + const subAgentFields = factorCreationFields([{ ...node, type: 'sub_agent' }], []) + expect(subAgentFields.some((field) => field.connectorFactor?.kind === 'agent')).toBe(false) +}) + +it('saves this agent on/off without changing its current activity', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + expect(screen.queryByRole('button', { name: 'Agent levels' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'All agents on/off' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Writer:Enabled', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'active', undefined)) + expect(node.data.active).not.toBe(false) +}) + +it('keeps prompt references in storage form while reordering levels and their labels', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const referenceScope = { targets: [{ id: 'source', name: 'Researcher' }], names: { source: 'Researcher' } } + render() + expect(screen.getByLabelText('Level 1 prompt')).toHaveValue('Summarize {{Researcher}}') + fireEvent.change(screen.getByLabelText('Level 2 prompt'), { target: { value: 'Analyze {{Researcher}}' } }) + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'Analysis' } }) + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ levels: ['Analyze {{node:source}}', 'Summarize {{node:source}}'], level_labels: ['Analysis', 'level1'] }), 'config.prompt', undefined)) +}) + +it('supports an empty system prompt level and blocks duplicate values and labels', () => { + render() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 system prompt'), { target: { value: 'Be concise.' } }) + expect(screen.getByLabelText('Level 1 system prompt')).toHaveValue('') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'level1' } }) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('edits legacy prompt bindings and passes the previous factor name on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.prompt': 'Prompts' } } } + render() + expect(screen.getByRole('button', { name: 'Prompt levels' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Tasks' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ name: 'Tasks', level_type: 'text', levels: ['Original', 'Alternate'] }), 'config.prompt', 'Prompts')) +}) + +it('removes a factor through its selected field without closing the dialog', async () => { + const remove = vi.fn().mockResolvedValue(undefined), close = vi.fn() + const bound = { ...node, data: { ...node.data, factor_bindings: { active: 'Available' } } } + render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Available', 'active')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() +}) diff --git a/frontend/src/components/protocol/AgentFactorDialog.tsx b/frontend/src/components/protocol/AgentFactorDialog.tsx new file mode 100644 index 00000000..317f3547 --- /dev/null +++ b/frontend/src/components/protocol/AgentFactorDialog.tsx @@ -0,0 +1,88 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { factorValueKey } from '@/lib/experiment' +import type { PromptReferenceScope } from '@/lib/promptReferences' +import type { DesignFactor } from '@/types/experiments' +import type { AgentNodeData, ProtocolNode } from '@/types/protocols' +import { agentFactorFields } from './bindableFields' +import { defaultSystemPrompt } from './defaultSystemPrompt' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { PromptReferenceField } from './PromptReferenceField' + +export function AgentFactorDialog({ node, factors, initialFieldPath, referenceScope, onClose, onSave, onRemove }: { + node: ProtocolNode & { data: AgentNodeData } + factors: DesignFactor[] + initialFieldPath?: string + referenceScope: PromptReferenceScope + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string) => Promise +}) { + const bindings = node.data.factor_bindings ?? {} + const existingFor = (path: string) => factors.find((factor) => factor.name === bindings[path]) + const [path, setPath] = useState(initialFieldPath ?? agentFactorFields.find((field) => existingFor(field.fieldPath))?.fieldPath ?? 'active') + function seed(fieldPath: string): DesignFactor { + const existing = existingFor(fieldPath) + const boolean = fieldPath === 'active' + const type = boolean ? 'boolean' : 'text' + if (existing) return { ...existing, level_type: type, level_labels: existing.level_labels ?? (boolean ? existing.levels.map((level) => level ? 'Enabled' : 'Disabled') : defaultFactorLevelLabels(existing.name, existing.levels.length, type)) } + const label = boolean ? 'Enabled' : fieldPath === 'config.prompt' ? 'Prompt' : 'System prompt' + const name = computeFactorName(node.data.label || 'Agent', label, factors.map((factor) => factor.name)) + const value = fieldPath === 'config.prompt' ? node.data.config.prompt : node.data.config.system_prompt + return { name, level_type: type, levels: boolean ? [false, true] : seedLevels(value), level_labels: boolean ? ['Disabled', 'Enabled'] : defaultFactorLevelLabels(name, 2, type) } + } + const [draft, setDraft] = useState(() => seed(path)) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const boolean = path === 'active' + const current = existingFor(path) + const labels = draft.level_labels ?? [] + const duplicateLevels = new Set(draft.levels.map(factorValueKey)).size !== draft.levels.length + const valid = draft.name.trim() && draft.levels.length >= 2 && !duplicateLevels && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, path, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the agent factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + + Agent factorChoose how this agent varies across experimental cells. +
+
{agentFactorFields.map((field) => )}
+

{boolean ? 'Enable or disable this Agent node. Disabled sequential agents pass upstream output through. Cells must keep an executable agent and every required lead, supervisor, or gated worker active.' : 'Replace this field for each cell while keeping this agent’s other settings and connections. An empty level uses the normal default for this field.'}

+
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

{boolean ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}${index === 0 ? ' · default test selection' : ''}`}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {!boolean && setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} />} +
+ {!boolean && <>} +
)}
+ {duplicateLevels &&

Each level must have a different value. Change or remove a duplicate before saving.

} + {!boolean && } +

Changes require design review and regeneration.

+ {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/AgentNodeInspector.tsx b/frontend/src/components/protocol/AgentNodeInspector.tsx index 74b8d0a2..1747b14d 100644 --- a/frontend/src/components/protocol/AgentNodeInspector.tsx +++ b/frontend/src/components/protocol/AgentNodeInspector.tsx @@ -204,6 +204,7 @@ export function AgentNodeInspector({ experimentId={experimentId} nodeId={node.id} fieldPath="active" + creationInToolbar defaultLabel="Active" nodeLabel={data.label || (isSubAgent ? 'Sub-Agent' : 'Agent')} levelType="boolean" @@ -272,10 +273,10 @@ export function AgentNodeInspector({ - {isSubAgent && <> + <>

- Make factor has moved to the node toolbar. Hover over the Sub-Agent node and click the

- {subAgentFactorControlled ?

Availability is controlled by the parent Agent’s Sub-Agents factor. Remove it to restore individual controls.

: bindings.active &&

Factor: {bindings.active}

} + {isSubAgent && (subAgentFactorControlled ?

Availability is controlled by the parent Agent’s Sub-Agents factor. Remove it to restore individual controls.

: bindings.active &&

Factor: {bindings.active}

)} {subAgentFactorControlled && bindings.active && } - } + {/* First, above Prompt: which agent leads decides whose prompt becomes the task and whose answer gets scored, so it frames everything below it rather than being one more setting. It @@ -338,6 +339,7 @@ export function AgentNodeInspector({ experimentId={experimentId} nodeId={node.id} fieldPath="config.prompt" + creationInToolbar={!isSubAgent} defaultLabel="Prompt" nodeLabel={data.label || 'Agent'} levelType="text" @@ -481,6 +483,7 @@ export function AgentNodeInspector({ void onUnbind: () => void // Receives the trigger/badge element to place -- e.g. @@ -169,6 +172,8 @@ export function FactorBindableField({ ) } + if (creationInToolbar) return children(null) + if (!experimentId) { return children( diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 0104b003..4e2bd8d5 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -1,6 +1,6 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { ReactFlowProvider } from '@xyflow/react' -import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { MemoryRouter } from 'react-router-dom' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -16,6 +16,7 @@ import { defaultOutputParserNodeData, defaultReasonActPatternNodeData, defaultScriptNodeData, + type AgentNodeData, type ProtocolGraph, type ProtocolRun, type TestRun, @@ -691,25 +692,54 @@ describe('ProtocolCanvas connector adds', () => { expect(screen.getByText('Factor name')).toBeInTheDocument() }) - it('binds the Agent inspector header action directly to Active', async () => { + it('moves Agent factor creation to the hover toolbar and saves a prompt baseline', async () => { const user = userEvent.setup() - vi.spyOn(experimentsApi, 'get').mockResolvedValue({ + const experiment: Experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + const save = vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + Object.assign(experiment, patch) + return experiment }) - renderCanvas({ + const { client } = renderCanvas({ nodes: [{ id: 'agent-1', type: 'agent', position: { x: 100, y: 100 }, data: defaultAgentNodeData('Writer') }], edges: [], }, 'experiment-1') fireEvent.doubleClick(await screen.findByText('Writer')) - await user.click(screen.getAllByRole('button', { name: 'Make experimental factor' })[0]) - - expect(await screen.findByText('Writer:Active')).toBeInTheDocument() - expect(screen.getByText('Levels: false, true')).toBeInTheDocument() + expect(screen.getByText(/has moved to the node toolbar/)).toHaveTextContent('Hover over the Agent node') + expect(screen.getByRole('img', { name: /Node toolbar preview/ })).toBeInTheDocument() + // The inspector has no inline creation buttons. + expect(within(screen.getByRole('dialog')).queryByRole('button', { name: 'Make experimental factor' })).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Close' })) + fireEvent.click(screen.getByTitle('Make experimental factor')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() expect(screen.queryByText('Bind to a field on the canvas')).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Prompt levels' })) + fireEvent.change(screen.getByLabelText('Level 1 prompt'), { target: { value: 'Baseline task' } }) + fireEvent.change(screen.getByLabelText('Level 2 prompt'), { target: { value: 'Alternate task' } }) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Writer:Prompt', levels: ['Baseline task', 'Alternate task'] })] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ 'config.prompt': 'Writer:Prompt' }) + expect((persisted?.nodes[0]?.data as AgentNodeData | undefined)?.config.prompt).toBe('Baseline task') + }) + fireEvent.doubleClick(await screen.findByText('Writer')) + await user.click(await screen.findByText('Factor: Writer:Prompt')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() + expect(screen.getByLabelText('Factor name')).toHaveValue('Writer:Prompt') + await user.click(screen.getByRole('button', { name: 'This agent on/off' })) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.active).toBe(true) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ 'config.prompt': 'Writer:Prompt', active: 'Writer:Enabled' }) + }) }) it('keeps the Test Run label when a production replicate is running', async () => { diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 86fa9e8d..238c11a8 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -3,6 +3,7 @@ import { SubAgentFactorDialog } from './SubAgentFactorDialog' import { SubAgentTestSelectors } from './SubAgentTestSelectors' import { useSubAgentTestSelection } from './useSubAgentTestSelection' import { ModelFactorDialog } from './ModelFactorDialog' +import { AgentFactorDialog } from './AgentFactorDialog' import { PatternFactorDialog } from './PatternFactorDialog' import { PATTERN_FACTOR_PATH, isPatternNode, patternFactorConflict } from './patternFactors' import { reconcileKnowledgeFactor, knowledgeFactorConflict, knowledgeFactorOwner, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' @@ -510,7 +511,8 @@ export const ProtocolCanvas = forwardRef(null) const [knowledgeFactorNodeId, setKnowledgeFactorNodeId] = useState(null) const [modelFactorNodeId, setModelFactorNodeId] = useState(null) + const [agentFactorNodeId, setAgentFactorNodeId] = useState(null) + const [agentFactorFieldPath, setAgentFactorFieldPath] = useState() const [patternFactorNodeId, setPatternFactorNodeId] = useState(null) const [patternFactorFieldPath, setPatternFactorFieldPath] = useState() const [patternFactorAgentId, setPatternFactorAgentId] = useState() @@ -1436,10 +1440,23 @@ export const ProtocolCanvas = forwardRef { if (!experimentId || experimentLocked) return + const owner = nodes.find((node) => node.type === 'agent' && Object.entries(node.data.factor_bindings as Record ?? {}).some(([path, name]) => name === factorName && ['active', 'config.prompt', 'config.system_prompt'].includes(path))) + if (owner) { + setSelectedNodeId(null) + setAgentFactorFieldPath(Object.entries(owner.data.factor_bindings as Record).find(([path, name]) => name === factorName && ['active', 'config.prompt', 'config.system_prompt'].includes(path))![0]) + setAgentFactorNodeId(owner.id) + return + } setEditingFactorName(factorName) }, - [experimentId, experimentLocked], + [experimentId, experimentLocked, nodes], ) + const requestAgentFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setAgentFactorFieldPath(undefined) + setAgentFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) const requestModelFactor = useCallback((nodeId: string, levelIndex?: number) => { if (!experimentId || experimentLocked) return setSelectedNodeId(null) @@ -1578,6 +1595,7 @@ export const ProtocolCanvas = forwardRef} + {agentFactorNodeId && nodes.some((node) => node.id === agentFactorNodeId && node.type === 'agent') && node.id === agentFactorNodeId)! as ProtocolNode & { data: AgentNodeData }} + factors={factors} + initialFieldPath={agentFactorFieldPath} + referenceScope={promptScopeFor(agentFactorNodeId)} + onClose={() => setAgentFactorNodeId(null)} + onSave={async (factor, fieldPath, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + bindFactorOnNode(agentFactorNodeId, fieldPath, factor.name) + if (fieldPath === 'active') { + // Legacy nodes omit active; make the bound field explicit without + // changing its current state to the factor's Disabled level. + setNodes((current) => current.map((node) => (node.data.factor_bindings as Record | undefined)?.active === factor.name ? { ...node, data: { ...node.data, active: node.data.active ?? true } } : node)) + } else setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath) => { + const fresh = await experimentsApi.get(experimentId!) + const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record ?? {}).some(([path, factorName]) => factorName === name && (node.id !== agentFactorNodeId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => node.id === agentFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} {modelFactorNodeId && nodes.some((node) => node.id === modelFactorNodeId) && node.id === modelFactorNodeId)!} diff --git a/frontend/src/components/protocol/ProtocolCanvasContext.tsx b/frontend/src/components/protocol/ProtocolCanvasContext.tsx index 0446a8ee..feaf8534 100644 --- a/frontend/src/components/protocol/ProtocolCanvasContext.tsx +++ b/frontend/src/components/protocol/ProtocolCanvasContext.tsx @@ -65,6 +65,7 @@ interface ProtocolCanvasActions { requestConnectorAdd: (request: ConnectorAddRequest) => void // Opens the consolidated factor dialog from a Skill node's hover toolbar. requestDatasetFactor?: (nodeId: string) => void + requestAgentFactor?: (nodeId: string) => void requestKnowledgeFactor?: (nodeId: string) => void requestModelFactor?: (nodeId: string, levelIndex?: number) => void requestPatternFactor?: (nodeId: string) => void diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 6022bfe4..c38913d8 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -9,6 +9,12 @@ export interface BindableFieldSpec { levelType: LevelType } +export const agentFactorFields: BindableFieldSpec[] = [ + { fieldPath: 'active', label: 'This agent on/off', levelType: 'boolean' }, + { fieldPath: 'config.prompt', label: 'Prompt levels', levelType: 'text' }, + { fieldPath: 'config.system_prompt', label: 'System prompt levels', levelType: 'text' }, +] + // The bundled system servers (see the backend's // services/system_mcp_servers.py) that are hidden from every MCP Tool server // picker -- they're still registered, still connected, and still usable by @@ -156,6 +162,7 @@ const MODEL_NODE_TYPES = new Set([ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { switch (node.type) { case 'agent': + return [...agentFactorFields, { fieldPath: 'pattern_override', label: 'Execution pattern', levelType: 'pattern' }] case 'sub_agent': return [ // The run's own ask, and the field `{{...}}` references resolve in -- @@ -285,7 +292,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { export interface UnboundField { pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } - connectorFactor?: { kind: 'script' | 'tool' | 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } + connectorFactor?: { kind: 'agent' | 'script' | 'tool' | 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -529,6 +536,9 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField } } for (const field of fields) { + if (nodes.find((node) => node.id === field.nodeId)?.type === 'agent' && agentFactorFields.some((candidate) => candidate.fieldPath === field.fieldPath)) { + field.connectorFactor = { kind: 'agent', nodeId: field.nodeId, mode: field.levelType } + } if (MODEL_NODE_TYPES.has(nodes.find((node) => node.id === field.nodeId)?.type ?? '')) { field.connectorFactor = { kind: 'model', nodeId: field.nodeId, mode: field.levelType } } @@ -545,11 +555,12 @@ export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' : field.connectorFactor?.kind === 'model' ? 'Model' : field.connectorFactor?.kind === 'pattern' ? 'Pattern' + : kind === 'agent' ? 'Agent' : agentKinds.includes(kind) ? 'Agent fields' : MODEL_NODE_TYPES.has(kind) ? 'Model' : kind.startsWith('pattern_') ? 'Pattern' : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', mcp_scikit_learn: 'Tools', mcp_client_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? String(node.data.label || kind) - const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' && field.connectorFactor.mode !== 'tool_names' && field.connectorFactor.mode !== 'script_config' + const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'agent' && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' && field.connectorFactor.mode !== 'tool_names' && field.connectorFactor.mode !== 'script_config' return { ...field, pickerGroup: { diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index a873dc1d..770b998b 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -82,7 +82,7 @@ export function AgentNode({ ...(peerNeedsToolCalling ? ["This model can't call tools, so this agent can't consult or delegate to connected agents"] : []), ] const { updateNodeData } = useReactFlow() - const { requestRunNode, requestSubAgentFactor, requestEditFactor, experimentLocked } = useProtocolCanvasActions() + const { requestRunNode, requestAgentFactor, requestSubAgentFactor, requestEditFactor, experimentLocked } = useProtocolCanvasActions() const controlled = isSubAgent && !!data.subAgentFactorControlled const isActive = controlled || (data.active ?? true) const factorCount = boundFactorCount(data) + (controlled ? 1 : 0) @@ -130,7 +130,7 @@ export function AgentNode({ nodeId={id} isActive={isActive} onToggleActive={controlled ? undefined : () => updateNodeData(id, { active: !isActive })} - onMakeFactor={isSubAgent && !experimentLocked && requestSubAgentFactor ? () => requestSubAgentFactor(id) : undefined} + onMakeFactor={experimentLocked ? undefined : isSubAgent ? requestSubAgentFactor && (() => requestSubAgentFactor(id)) : requestAgentFactor && (() => requestAgentFactor(id))} runAlone={{ canRun: !!data.canRunAlone, onRun: () => requestRunNode(id) }} /> {/* Sits inside the top-right corner so it stays clear of the Knowledge diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index c39440ae..ae339972 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -666,10 +666,25 @@ def validate_stage_plan(design_spec: dict[str, Any] | None) -> None: raise ProtocolValidationError(f"This experiment's stage plan is not usable: {e}") from e +def validate_agent_activity(graph: dict[str, Any]) -> None: + """Availability must leave executable work and preserve required gated roles.""" + agents = {node["id"]: node for node in graph.get("nodes") or [] if node.get("type") == "agent"} + if agents and not any(_is_node_active(node) for node in agents.values()): + raise ProtocolValidationError("This cell disables every Agent. Keep at least one executable Agent active.") + for worker_id in find_gated_pairs(graph): + worker = agents.get(worker_id) + if worker is not None and not _is_node_active(worker): + raise ProtocolValidationError( + f"Agent node {_node_display_name(worker)!r} is gated by a Critic Gate and can't be " + "deactivated -- deactivate the Critic Gate instead." + ) + + def validate_coordination_strategy(design_spec: dict[str, Any] | None, *, graph: dict[str, Any]) -> None: """Checks the experiment's declared strategy against the protocol it will run. Takes the whole graph rather than a pre-computed fact about it because each strategy asks a different question of the canvas.""" + validate_agent_activity(graph) slug = coordination_strategy_slug(design_spec) if slug == "sequential": validate_sequential_chain(graph) @@ -2271,7 +2286,13 @@ def _names(ids: Iterable[str]) -> str: "report to the supervisor, not to each other. Remove that connection, or switch the strategy to " "'Peer Collaboration' on the Design tab." ) - return SupervisorRoles(supervisor=supervisor, workers=workers, reviewer=reviewer) + if not _is_node_active(nodes[supervisor]): + raise ProtocolValidationError(f"The supervisor {_name(supervisor)!r} must remain active in every cell.") + active_workers = tuple(nid for nid in workers if _is_node_active(nodes[nid])) + if not active_workers: + raise ProtocolValidationError("A supervisor cell must keep at least one worker Agent active.") + active_reviewer = reviewer if reviewer is not None and _is_node_active(nodes[reviewer]) else None + return SupervisorRoles(supervisor=supervisor, workers=active_workers, reviewer=active_reviewer) def _connected_agent_ids(graph: dict[str, Any], node_id: str) -> list[str]: @@ -4955,6 +4976,7 @@ async def plan_cell_runs( expected_sha256=slot.raw_sha256, ) patched_graph = apply_factor_bindings(revision.graph, parent.factor_values or {}) + validate_coordination_strategy(design_spec, graph=patched_graph) patched_plan = resolve_dataset_row_plan(patched_graph, design_spec) if patched_plan is None or patched_plan["driver_dataset_id"] != str(slot.dataset_id): raise DatasetRowInputError("driver_mismatch", "factor substitution changed the row driver") @@ -5031,6 +5053,7 @@ async def plan_cell_runs( planned = [] for candidate in candidates: patched_graph = apply_factor_bindings(revision.graph, candidate["factor_values"]) + validate_coordination_strategy(design_spec, graph=patched_graph) patched_plan = resolve_dataset_row_plan(patched_graph, design_spec) if patched_plan is None or patched_plan["driver_dataset_id"] != str(registration.id): raise DatasetRowInputError("driver_mismatch", "factor substitution changed the row driver") @@ -5137,6 +5160,14 @@ def is_obsolete(run: ProtocolRun) -> bool: for replicate in replicates if replicate.replicate_label not in completed_labels or replicate.replicate_label in requested_reruns ] + # Validate every requested cell before creating any run in this batch. + for replicate in pending: + try: + validate_coordination_strategy( + design_spec, graph=apply_factor_bindings(graph, replicate.factor_values or {}) + ) + except ProtocolValidationError as exc: + raise ProtocolValidationError(f"Replicate {replicate.replicate_label!r}: {exc}") from exc runs = [ await create_protocol_run( db, @@ -5240,6 +5271,7 @@ async def plan_single_replicate_run( if replicate_label is not None: raise ProtocolValidationError(f"No such replicate: {replicate_label!r}") raise ProtocolValidationError("No current design replicate is available for row execution.") + validate_coordination_strategy(design_spec, graph=apply_factor_bindings(graph, replicate.factor_values or {})) if row_plan is not None: if protocol_revision_id is None: raise ProtocolValidationError("A published protocol revision is required for row execution.") @@ -5352,6 +5384,10 @@ def validate_conversation_entry(graph: dict[str, Any], node_id: str) -> dict[str raise ProtocolValidationError(f"No such node: {node_id!r}") if node.get("type") != "agent": raise ProtocolValidationError("Only Agent nodes can start a conversation.") + if not _is_node_active(node): + raise ProtocolValidationError( + f"The conversation lead {_node_display_name(node)!r} must remain active in every cell." + ) peers = _connected_agent_ids(graph, node_id) if not peers: @@ -5363,6 +5399,8 @@ def validate_conversation_entry(graph: dict[str, Any], node_id: str) -> dict[str # or the consultation fails partway through a run the user already paid for. for participant_id in [node_id, *peers]: participant = nodes[participant_id] + if not _is_node_active(participant): + continue model_edges = _edges_with_handle(graph, participant_id, "model", direction="incoming") if len(model_edges) != 1: raise ProtocolValidationError( diff --git a/tests/test_agent_factors.py b/tests/test_agent_factors.py new file mode 100644 index 00000000..2b263d86 --- /dev/null +++ b/tests/test_agent_factors.py @@ -0,0 +1,137 @@ +"""Agent factors preserve required roles before execution or batch creation.""" + +from __future__ import annotations + +import uuid +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +from asaree.services import protocol_execution as pe + + +def _graph(*ids: str) -> dict[str, Any]: + return { + "nodes": [ + {"id": nid, "type": "agent", "data": {"label": nid, "active": True, "config": {"prompt": "Task"}}} + for nid in ids + ] + [{"id": "model", "type": "model_openai", "data": {"config": {"provider": "openai", "model": "gpt-5"}}}], + "edges": [ + {"id": f"model-{nid}", "source": "model", "target": nid, "targetHandle": "model"} for nid in ids + ] + [{"id": f"{a}-{b}", "source": a, "target": b} for a, b in zip(ids, ids[1:], strict=False)], + } + + +def _disable(graph: dict[str, Any], *ids: str) -> None: + for node in graph["nodes"]: + if node["id"] in ids: + node["data"]["active"] = False + + +def test_on_off_factor_cannot_disable_the_last_executable_agent() -> None: + graph = _graph("writer") + graph["nodes"][0]["data"]["factor_bindings"] = {"active": "Writer enabled"} + patched = pe.apply_factor_bindings(graph, {"Writer enabled": False}) + with pytest.raises(pe.ProtocolValidationError, match="at least one executable Agent"): + pe.validate_coordination_strategy(None, graph=patched) + assert graph["nodes"][0]["data"]["active"] is True + + +def test_sequential_cells_can_disable_a_stage_when_another_agent_remains() -> None: + graph = _graph("first", "middle", "last") + _disable(graph, "middle") + pe.validate_coordination_strategy(None, graph=graph) + + +@pytest.mark.parametrize("marked", [False, True]) +def test_peer_lead_cannot_be_disabled(marked: bool) -> None: + graph = _graph("lead", "peer") + graph["nodes"][0]["data"]["conversation_lead"] = marked + _disable(graph, "lead") + with pytest.raises(pe.ProtocolValidationError, match="conversation lead.*must remain active"): + pe.validate_coordination_strategy({"coordination_strategy": {"slug": "peer_collaboration"}}, graph=graph) + + +async def test_peer_factor_can_disable_a_collaborator() -> None: + graph = _graph("lead", "peer") + _disable(graph, "peer") + pe.validate_coordination_strategy({"coordination_strategy": {"slug": "peer_collaboration"}}, graph=graph) + assert await pe.resolve_agent_card(graph, "peer", owner_id=uuid.uuid4()) is None + + +def test_gated_worker_cannot_be_disabled_by_a_cell_factor() -> None: + graph = _graph("worker", "next") + graph["nodes"].append({"id": "gate", "type": "critic_gate", "data": {"config": {"enabled": True}}}) + graph["edges"] = [edge for edge in graph["edges"] if edge["id"] != "worker-next"] + [ + {"id": "worker-gate", "source": "worker", "target": "gate"}, + {"id": "gate-next", "source": "gate", "target": "next"}, + ] + _disable(graph, "worker") + with pytest.raises(pe.ProtocolValidationError, match="gated by a Critic Gate.*deactivated"): + pe.validate_coordination_strategy(None, graph=graph) + + +@pytest.fixture +def planning(monkeypatch: pytest.MonkeyPatch) -> tuple[dict[str, Any], AsyncMock]: + graph = _graph("first", "last") + for node in graph["nodes"]: + if node["type"] == "agent": + node["data"]["factor_bindings"] = {"active": "Enabled"} + experiment = SimpleNamespace( + design_spec={"factors": [{"name": "Enabled", "level_type": "boolean", "levels": [False, True]}]}, + locked_at=None, measurement_plan=None, + ) + replicates = [SimpleNamespace( + id=uuid.uuid4(), design_revision_id=uuid.uuid4(), replicate_label=f"replicate-{enabled}", + factor_values={"Enabled": enabled}, metric_values={}, run_id=None, + ) for enabled in (True, False)] + monkeypatch.setattr(pe, "get_experiment", AsyncMock(return_value=experiment)) + monkeypatch.setattr(pe, "get_design_impact", AsyncMock(return_value=SimpleNamespace(regeneration_required=False))) + monkeypatch.setattr(pe, "validate_experiment_measurement_plan", AsyncMock(return_value=[])) + monkeypatch.setattr(pe, "blocking_measurement_plan_issues", lambda _report: []) + for name in ("validate_skill_factors", "validate_knowledge_factors", "validate_dataset_factors"): + monkeypatch.setattr(pe, name, AsyncMock()) + monkeypatch.setattr(pe, "list_replicates", AsyncMock(return_value=replicates)) + monkeypatch.setattr(pe, "get_replicate", AsyncMock(return_value=replicates[1])) + create = AsyncMock() + monkeypatch.setattr(pe, "create_protocol_run", create) + return graph, create + + +async def test_batch_rejects_an_invalid_cell_before_creating_any_runs( + planning: tuple[dict[str, Any], AsyncMock], +) -> None: + graph, create = planning + with pytest.raises(pe.ProtocolValidationError, match="replicate-False.*disables every Agent"): + await pe.plan_cell_runs( + AsyncMock(), protocol_id=uuid.uuid4(), experiment_id=uuid.uuid4(), owner_id=uuid.uuid4(), graph=graph + ) + create.assert_not_awaited() + + +async def test_single_replicate_rejects_an_invalid_cell_before_creating_a_run( + planning: tuple[dict[str, Any], AsyncMock], +) -> None: + graph, create = planning + with pytest.raises(pe.ProtocolValidationError, match="disables every Agent"): + await pe.plan_single_replicate_run( + AsyncMock(), protocol_id=uuid.uuid4(), experiment_id=uuid.uuid4(), owner_id=uuid.uuid4(), graph=graph, + replicate_label="replicate-False", + ) + create.assert_not_awaited() + + +async def test_batch_can_run_a_valid_selection_without_rejecting_unselected_cells( + planning: tuple[dict[str, Any], AsyncMock], +) -> None: + graph, create = planning + runs, skipped = await pe.plan_cell_runs( + AsyncMock(), protocol_id=uuid.uuid4(), experiment_id=uuid.uuid4(), owner_id=uuid.uuid4(), graph=graph, + replicate_labels={"replicate-True"}, + ) + assert len(runs) == 1 + assert skipped == 0 + create.assert_awaited_once() + assert create.await_args.kwargs["factor_values"] == {"Enabled": True} diff --git a/tests/test_supervisor_architecture.py b/tests/test_supervisor_architecture.py index 81a40198..92cd30cd 100644 --- a/tests/test_supervisor_architecture.py +++ b/tests/test_supervisor_architecture.py @@ -247,6 +247,31 @@ async def test_a_run_with_no_reviewer_skips_only_the_review(stubs: dict[str, Any assert sorted(_ran(stubs)) == ["sup", "sup", "w1", "w2", "w3"] +@pytest.mark.parametrize("parallel", [False, True]) +async def test_agent_factors_skip_disabled_workers_and_the_optional_reviewer( + stubs: dict[str, Any], parallel: bool +) -> None: + graph = _graph() + for node in graph["nodes"]: + if node["id"] in {"w2", "qc"}: + node["data"]["factor_bindings"] = {"active": "Enabled"} + graph = pe.apply_factor_bindings(graph, {"Enabled": False}) + final, status = await _run(stubs, graph, parallel_workers=parallel) + assert status == "completed" + assert sorted(_ran(stubs)) == ["sup", "sup", "w1", "w3"] + assert final["output_text"] == "sup did its part." + + +@pytest.mark.parametrize("disabled", [{"sup"}, {"w1", "w2", "w3"}]) +def test_agent_factors_cannot_disable_required_supervisor_roles(disabled: set[str]) -> None: + graph = _graph() + for node in graph["nodes"]: + if node["id"] in disabled: + node["data"]["active"] = False + with pytest.raises(pe.ProtocolValidationError, match="must remain active|at least one worker"): + pe.resolve_supervisor_roles(graph) + + async def test_nobody_is_offered_a_peer_schema(stubs: dict[str, Any]) -> None: """`resolve_available_agents` raises in the fixture. Passing peers here would give the supervisor an `ask_` function *alongside* the forced From 11bf3b7f9f58b395a82280a878c018340eddec7b Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Wed, 7 Oct 2026 17:32:33 -0700 Subject: [PATCH 18/63] Apply shared resource factors to every connected agent --- .../src/components/protocol/AddNodePanel.tsx | 2 +- .../protocol/DatasetFactorDialog.test.tsx | 3 +- .../protocol/DatasetFactorDialog.tsx | 20 ++-- .../protocol/DatasetTestSelectors.tsx | 6 +- .../components/protocol/DesignTab.test.tsx | 6 +- .../src/components/protocol/DesignTab.tsx | 6 +- .../protocol/KnowledgeFactorDialog.test.tsx | 3 +- .../protocol/KnowledgeFactorDialog.tsx | 20 ++-- .../protocol/KnowledgeTestSelectors.tsx | 6 +- .../protocol/PatternFactorDialog.test.tsx | 2 +- .../protocol/PatternFactorDialog.tsx | 10 +- .../protocol/ProtocolCanvas.test.tsx | 29 +++-- .../components/protocol/ProtocolCanvas.tsx | 63 +++++------ .../protocol/ScriptFactorDialog.test.tsx | 2 +- .../protocol/ScriptFactorDialog.tsx | 20 ++-- .../protocol/ScriptTestSelectors.tsx | 6 +- .../protocol/SkillFactorDialog.test.tsx | 19 +++- .../components/protocol/SkillFactorDialog.tsx | 20 ++-- .../protocol/SkillTestSelectors.tsx | 6 +- .../protocol/SubAgentFactorDialog.test.tsx | 2 +- .../protocol/SubAgentFactorDialog.tsx | 20 ++-- .../protocol/SubAgentTestSelectors.tsx | 4 +- .../protocol/ToolFactorDialog.test.tsx | 7 +- .../components/protocol/ToolFactorDialog.tsx | 20 ++-- .../components/protocol/ToolTestSelectors.tsx | 6 +- .../src/components/protocol/bindableFields.ts | 29 +++-- .../protocol/connectionValidation.test.ts | 5 + .../protocol/connectionValidation.ts | 3 +- .../src/components/protocol/patternFactors.ts | 6 ++ .../protocol/useDatasetTestSelection.test.tsx | 16 +-- .../protocol/useDatasetTestSelection.ts | 11 +- .../protocol/useKnowledgeTestSelection.ts | 11 +- .../protocol/useScriptTestSelection.test.tsx | 6 +- .../protocol/useScriptTestSelection.ts | 11 +- .../protocol/useSkillTestSelection.test.tsx | 16 +-- .../protocol/useSkillTestSelection.ts | 11 +- .../protocol/useSubAgentTestSelection.ts | 11 +- .../protocol/useToolTestSelection.test.tsx | 16 +-- .../protocol/useToolTestSelection.ts | 11 +- frontend/src/lib/datasetFactors.test.ts | 6 +- frontend/src/lib/datasetFactors.ts | 16 ++- frontend/src/lib/knowledgeFactors.test.ts | 6 +- frontend/src/lib/knowledgeFactors.ts | 16 ++- frontend/src/lib/scriptFactors.test.ts | 6 +- frontend/src/lib/scriptFactors.ts | 18 +++- frontend/src/lib/sharedFactors.test.ts | 76 +++++++++++++ frontend/src/lib/sharedFactors.ts | 86 +++++++++++++++ frontend/src/lib/skillFactors.test.ts | 6 +- frontend/src/lib/skillFactors.ts | 16 ++- frontend/src/lib/subAgentFactors.ts | 18 +++- frontend/src/lib/toolFactors.test.ts | 6 +- frontend/src/lib/toolFactors.ts | 18 +++- src/asaree/services/dataset_factors.py | 4 +- src/asaree/services/factor_bindings.py | 2 + src/asaree/services/knowledge_factors.py | 4 +- src/asaree/services/protocol_execution.py | 9 +- src/asaree/services/script_factors.py | 5 +- src/asaree/services/shared_factors.py | 101 ++++++++++++++++++ src/asaree/services/skill_factors.py | 5 +- src/asaree/services/sub_agent_factors.py | 5 +- src/asaree/services/tool_factors.py | 5 +- tests/test_dataset_factors.py | 8 +- tests/test_protocol_execution.py | 7 +- tests/test_shared_factors.py | 98 +++++++++++++++++ tests/test_skill_factors.py | 26 ++++- 65 files changed, 766 insertions(+), 278 deletions(-) create mode 100644 frontend/src/lib/sharedFactors.test.ts create mode 100644 frontend/src/lib/sharedFactors.ts create mode 100644 src/asaree/services/shared_factors.py create mode 100644 tests/test_shared_factors.py diff --git a/frontend/src/components/protocol/AddNodePanel.tsx b/frontend/src/components/protocol/AddNodePanel.tsx index fbaa9f33..4a95bc49 100644 --- a/frontend/src/components/protocol/AddNodePanel.tsx +++ b/frontend/src/components/protocol/AddNodePanel.tsx @@ -33,7 +33,7 @@ type NodeCategory = (typeof NODE_CATEGORIES)[number]['id'] // allowedTypes, IS that picker. const NODE_CATALOG = [ { type: 'agent', category: 'agents', label: 'Agent', description: 'An LLM agent stage in the pipeline', icon: Bot }, - { type: 'sub_agent', category: 'agents', label: 'Sub-Agent', description: 'A delegated worker callable by one parent Agent', icon: Bot }, + { type: 'sub_agent', category: 'agents', label: 'Sub-Agent', description: 'A delegated worker callable by connected Agents', icon: Bot }, // Not a node type -- picking this opens the server browser // (McpServerBrowserPanel), and the node gets created from whichever // server is chosen there. It replaced a plain "MCP Tool" entry that made diff --git a/frontend/src/components/protocol/DatasetFactorDialog.test.tsx b/frontend/src/components/protocol/DatasetFactorDialog.test.tsx index 1e87886b..487a1aad 100644 --- a/frontend/src/components/protocol/DatasetFactorDialog.test.tsx +++ b/frontend/src/components/protocol/DatasetFactorDialog.test.tsx @@ -92,7 +92,8 @@ it('opens the existing group factor when a shared dataset has multiple connected edges: [...graph.edges, { id: 'second-edge', source: 'dataset', target: 'second', targetHandle: 'dataset' }], } as ProtocolGraph render() - expect(screen.getByRole('combobox')).toHaveTextContent('Second agent') + expect(screen.queryByRole('combobox')).not.toBeInTheDocument() + expect(screen.getByText('Applies to: First agent, Second agent')).toBeInTheDocument() expect(screen.getByRole('button', { name: 'All datasets on/off' })).toHaveAttribute('aria-pressed', 'true') expect(screen.getByLabelText('Factor name')).toHaveValue('Group') }) diff --git a/frontend/src/components/protocol/DatasetFactorDialog.tsx b/frontend/src/components/protocol/DatasetFactorDialog.tsx index bdd17860..5cc38e96 100644 --- a/frontend/src/components/protocol/DatasetFactorDialog.tsx +++ b/frontend/src/components/protocol/DatasetFactorDialog.tsx @@ -1,28 +1,27 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' import { useState } from 'react' import { ArrowDown, ArrowUp, Split } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { connectedDatasets, reconcileDatasetFactor, datasetFactorConflict, datasetFactorModes as modes, DATASET_FACTOR_PATH, type DatasetFactorMode as Mode } from '@/lib/datasetFactors' import type { DesignFactor } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' import { computeFactorName } from './factorLevels' -export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode, initialAgentId, onClose, onSave, onRemove }: { +export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { datasetNodeId: string graph: ProtocolGraph factors: DesignFactor[] initialMode?: Mode - initialAgentId?: string onClose: () => void onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise onRemove: (name: string) => Promise }) { const dataset = graph.nodes.find((node) => node.id === datasetNodeId)! - const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === datasetNodeId && edge.target === node.id && ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? ''))) - const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((agent) => agent.data.factor_bindings?.[DATASET_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const agents = factorRecipients(graph, datasetNodeId, 'dataset') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[DATASET_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) const legacy = existingFor(datasetNodeId, 'config') const initialFactor = existingFor(agentId, DATASET_FACTOR_PATH) ?? existingFor(datasetNodeId, 'config.enabled') @@ -60,7 +59,6 @@ export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode const blockedReason = unavailable(mode) const labels = draft.level_labels ?? [] function change(nextMode: Mode, nextAgentId = agentId) { - setAgentId(nextAgentId) setMode(nextMode) setDraft(seed(nextMode, nextAgentId)) setError('') @@ -94,11 +92,7 @@ export function DatasetFactorDialog({ datasetNodeId, graph, factors, initialMode Choose how datasets vary across experimental cells.
- {agents.length > 1 &&
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, datasetNodeId, 'dataset')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{legacy &&

This Dataset has an existing configuration factor: {legacy.name}. Remove it before creating a Dataset factor.

{mode !== 'boolean' && <>}
)}
-

Agent-level factors follow current Dataset connections. Changes require design review and regeneration.

- {current && } +

Shared factors require matching Dataset connections. Changes require design review and regeneration.

} + {current && } {error &&

{error}

}
diff --git a/frontend/src/components/protocol/DatasetTestSelectors.tsx b/frontend/src/components/protocol/DatasetTestSelectors.tsx index 03d43c31..e092e172 100644 --- a/frontend/src/components/protocol/DatasetTestSelectors.tsx +++ b/frontend/src/components/protocol/DatasetTestSelectors.tsx @@ -1,19 +1,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedDatasets, datasetId } from '@/lib/datasetFactors' +import { connectedDatasets, datasetId, DATASET_FACTOR_PATH } from '@/lib/datasetFactors' import type { useDatasetTestSelection } from './useDatasetTestSelection' export function DatasetTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedDatasets(selection.graph!, node.id) if (node.data.dataset_factor_mode === 'dataset_toggle') return
-

{node.data.label}: datasets for this test

+

{node.data.factor_bindings?.[DATASET_FACTOR_PATH] || node.data.label}: datasets for this test

return
-

{node.data.label}: dataset for this test

+

{node.data.factor_bindings?.[DATASET_FACTOR_PATH] || node.data.label}: dataset for this test

{ - if (!id) return - const existing = existingFor(id, KNOWLEDGE_FACTOR_PATH) - change((existing?.level_type as Mode) ?? mode, id) - }}>{String(agents.find((agent) => agent.id === agentId)?.data.label)}{agents.map((agent) => {String(agent.data.label)})}
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, knowledgeNodeId, 'knowledge')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Knowledge node.' : mode === 'knowledge_selection' ? 'Each cell receives exactly one connected knowledge source.' : 'Each cell enables every connected knowledge source, including deactivated sources, or disables them all.'}

@@ -108,9 +102,9 @@ export function KnowledgeFactorDialog({ knowledgeNodeId, graph, factors, initial
{mode !== 'boolean' && <>}
)} -

Agent-level factors follow current Knowledge connections. Changes require design review and regeneration.

- {current && } +

Shared factors require matching Knowledge connections. Changes require design review and regeneration.

} + {current && } {error &&

{error}

} diff --git a/frontend/src/components/protocol/KnowledgeTestSelectors.tsx b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx index 9b736a24..5bd8be47 100644 --- a/frontend/src/components/protocol/KnowledgeTestSelectors.tsx +++ b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx @@ -1,19 +1,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedKnowledge, knowledgeId } from '@/lib/knowledgeFactors' +import { connectedKnowledge, knowledgeId, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' import type { useKnowledgeTestSelection } from './useKnowledgeTestSelection' export function KnowledgeTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedKnowledge(selection.graph!, node.id) if (node.data.knowledge_factor_mode === 'knowledge_toggle') return
-

{node.data.label}: knowledge for this test

+

{node.data.factor_bindings?.[KNOWLEDGE_FACTOR_PATH] || node.data.label}: knowledge for this test

return
-

{node.data.label}: knowledge for this test

+

{node.data.factor_bindings?.[KNOWLEDGE_FACTOR_PATH] || node.data.label}: knowledge for this test

{ if (id) { setAgentId(id); setDraft(seed(path, id)); setError('') } }}>{String(agents.find((node) => node.id === agentId)?.data.label)}{agents.map((node) => {String(node.data.label)})}
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, patternNodeId, 'pattern')}

}
{fields.map((field) => )}
{blocked(path) ?

{blocked(path)}

: <>

{structured ? 'Compare complete pattern configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this parameter while keeping the other pattern settings.'}

diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 4e2bd8d5..d40ed18e 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -167,7 +167,7 @@ describe('ProtocolCanvas connector adds', () => { expect(makeFactor).not.toBeNull() fireEvent.click(makeFactor!) await screen.findByRole('dialog') - fireEvent.click(screen.getByRole('button', { name: mode === 'sub_agent_toggle' ? 'All agent sub-agents on/off' : 'Sub-Agent levels' })) + fireEvent.click(screen.getByRole('button', { name: mode === 'sub_agent_toggle' ? 'All sub-agents on/off' : 'Sub-Agent levels' })) fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Sub-Agents', level_type: mode, levels: mode === 'sub_agent_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'sub_agent_toggle' ? ['All enabled', 'All disabled'] : ['Worker a', 'Worker b'] }] }) }))) await waitFor(() => { @@ -178,7 +178,7 @@ describe('ProtocolCanvas connector adds', () => { }) }) - it.each(['skill_selection', 'skill_toggle'] as const)('creates a %s skill factor from the node hover menu and preserves connected nodes', async (mode) => { + it.each(['skill_selection', 'skill_toggle'] as const)('creates a %s skill factor from a shared node and binds every connected Agent', async (mode) => { const experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, @@ -191,9 +191,10 @@ describe('ProtocolCanvas connector adds', () => { const graph: ProtocolGraph = { nodes: [ { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + { id: 'other', type: 'agent', position: { x: 300, y: 200 }, data: defaultAgentNodeData('Other') }, ...['a', 'b'].map((id) => ({ id, type: 'skill', position: { x: 100, y: 0 }, data: { label: `Skill ${id}`, config: { skill_id: id, skill_name: `Skill ${id}` } } })), ], - edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'skill' })), + edges: ['a', 'b'].flatMap((id) => ['agent', 'other'].map((target) => ({ id: `${id}-${target}`, source: id, target, targetHandle: 'skill' }))), } const { client } = renderCanvas(graph, 'experiment-1') const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') @@ -207,6 +208,8 @@ describe('ProtocolCanvas connector adds', () => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ skill_selection: 'Writer:Skills' }) expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.skill_factor_mode).toBe(mode) + expect(persisted?.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({ skill_selection: 'Writer:Skills' }) + expect(persisted?.nodes.find((node) => node.id === 'other')?.data.skill_factor_mode).toBe(mode) expect(persisted?.nodes.filter((node) => node.type === 'skill')).toHaveLength(2) }) }) @@ -233,7 +236,7 @@ describe('ProtocolCanvas connector adds', () => { expect(makeFactor).not.toBeNull() fireEvent.click(makeFactor!) await screen.findByRole('dialog') - fireEvent.click(screen.getByRole('button', { name: mode === 'tool_toggle' ? 'All agent tools on/off' : 'Tool levels' })) + fireEvent.click(screen.getByRole('button', { name: mode === 'tool_toggle' ? 'All tools on/off' : 'Tool levels' })) fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Tools', level_type: mode, levels: mode === 'tool_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'tool_toggle' ? ['All enabled', 'All disabled'] : ['Tool a', 'Tool b'] }] }) }))) await waitFor(() => { @@ -265,7 +268,7 @@ describe('ProtocolCanvas connector adds', () => { expect(makeFactor).not.toBeNull() fireEvent.click(makeFactor!) await screen.findByRole('dialog') - fireEvent.click(screen.getByRole('button', { name: mode === 'script_toggle' ? 'All agent scripts on/off' : 'Script levels' })) + fireEvent.click(screen.getByRole('button', { name: mode === 'script_toggle' ? 'All scripts on/off' : 'Script levels' })) fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Scripts', level_type: mode, levels: mode === 'script_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'script_toggle' ? ['All enabled', 'All disabled'] : ['Script a', 'Script b'] }] }) }))) await waitFor(() => { @@ -276,7 +279,7 @@ describe('ProtocolCanvas connector adds', () => { }) }) - it('creates and removes a whole-pattern factor from the Pattern toolbar, clearing the Agent override', async () => { + it('creates and removes a whole-pattern factor from the Pattern toolbar, clearing every shared Agent override', async () => { let experiment: Experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, @@ -291,22 +294,24 @@ describe('ProtocolCanvas connector adds', () => { const graph: ProtocolGraph = { nodes: [ { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + { id: 'other', type: 'agent', position: { x: 300, y: 200 }, data: defaultAgentNodeData('Other') }, { id: 'pattern', type: 'pattern_reason_act', position: { x: 100, y: 0 }, data: defaultReasonActPatternNodeData() }, ], - edges: [{ id: 'pattern-edge', source: 'pattern', target: 'agent', targetHandle: 'architectural_pattern' }], + edges: ['agent', 'other'].map((target) => ({ id: `pattern-${target}`, source: 'pattern', target, targetHandle: 'architectural_pattern' })), } const { client } = renderCanvas(graph, 'experiment-1') const toolbar = () => document.querySelector('[data-testid="rf__node-pattern"] button[aria-label="Make experimental factor"]')! fireEvent.click(toolbar()) await screen.findByRole('dialog', { name: 'Pattern factor' }) fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) - await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Writer:Reason + Act:Pattern levels', level_type: 'pattern' })] }) }))) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Reason + Act:Pattern levels', level_type: 'pattern' })] }) }))) await waitFor(() => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! - expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ pattern_override: 'Writer:Reason + Act:Pattern levels' }) - expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toEqual({ execution_pattern: 'reason_act', pattern_params: { reason_act: graph.nodes[1].data.config } }) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ pattern_override: 'Reason + Act:Pattern levels' }) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({ pattern_override: 'Reason + Act:Pattern levels' }) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toEqual({ execution_pattern: 'reason_act', pattern_params: { reason_act: graph.nodes.find((node) => node.id === 'pattern')!.data.config } }) expect(persisted.nodes.find((node) => node.id === 'pattern')?.data.factor_bindings).toBeUndefined() - expect(persisted.edges).toHaveLength(1) + expect(persisted.edges).toHaveLength(2) }) await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) fireEvent.click(toolbar()) @@ -314,6 +319,8 @@ describe('ProtocolCanvas connector adds', () => { await waitFor(() => { const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.pattern_override).toBeUndefined() expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toBeUndefined() expect(experiment.design_spec?.factors).toEqual([]) }) diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 238c11a8..3f25dd04 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -1,3 +1,4 @@ +import { graphWithSharedFactor } from '@/lib/sharedFactors' import { reconcileSubAgentFactor, subAgentFactorConflict, subAgentFactorOwner, SUB_AGENT_FACTOR_PATH } from '@/lib/subAgentFactors' import { SubAgentFactorDialog } from './SubAgentFactorDialog' import { SubAgentTestSelectors } from './SubAgentTestSelectors' @@ -518,7 +519,7 @@ export const ProtocolCanvas = forwardRef() const [patternFactorNodeId, setPatternFactorNodeId] = useState(null) const [patternFactorFieldPath, setPatternFactorFieldPath] = useState() - const [patternFactorAgentId, setPatternFactorAgentId] = useState() const [modelFactorFieldPath, setModelFactorFieldPath] = useState() const [modelFactorLevelIndex, setModelFactorLevelIndex] = useState() const [subAgentFactorNodeId, setSubAgentFactorNodeId] = useState(null) @@ -1468,7 +1468,6 @@ export const ProtocolCanvas = forwardRef { @@ -2686,7 +2685,7 @@ export const ProtocolCanvas = forwardRef )} {patternFactorNodeId && nodes.some((node) => node.id === patternFactorNodeId) && node.id === patternFactorNodeId)!, edges, nodes)} initialFieldPath={patternFactorFieldPath} - initialAgentId={patternFactorAgentId} factors={factors} onClose={() => setPatternFactorNodeId(null)} onSave={async (factor, fieldPath, ownerId, previousName) => { @@ -2716,7 +2714,8 @@ export const ProtocolCanvas = forwardRef candidate.name !== previousName), factor] } }) if (previousName) renameFactorBindings(previousName, factor.name) - bindFactorOnNode(ownerId, fieldPath, factor.name) + if (fieldPath === PATTERN_FACTOR_PATH) setNodes((current) => graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, patternFactorNodeId, 'pattern', factor).nodes as unknown as Node[]) + else bindFactorOnNode(ownerId, fieldPath, factor.name) setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) @@ -2724,10 +2723,10 @@ export const ProtocolCanvas = forwardRef { if (experimentLocked) throw new Error('Unlock the experiment before editing factors.') const fresh = await experimentsApi.get(experimentId!) - const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== ownerId || path !== fieldPath))) + const shared = fieldPath !== PATTERN_FACTOR_PATH && nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== ownerId || path !== fieldPath))) if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) setNodes((current) => current.map((node) => { - if (node.id !== ownerId) return node + if (fieldPath === PATTERN_FACTOR_PATH ? (node.data.factor_bindings as Record | undefined)?.[PATTERN_FACTOR_PATH] !== name : node.id !== ownerId) return node const data: Record = { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } if (fieldPath === PATTERN_FACTOR_PATH) delete data.pattern_override return { ...node, data } @@ -2802,7 +2801,6 @@ export const ProtocolCanvas = forwardRef setSubAgentFactorNodeId(null)} @@ -2818,11 +2816,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, - ...(next.level_type === 'boolean' ? { active: next.levels[0] } : { sub_agent_selection: next.levels[0] ?? [], sub_agent_factor_mode: next.level_type }), - factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'active' : SUB_AGENT_FACTOR_PATH]: next.name }, - } } : node)) + active: next.levels[0], + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), active: next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, subAgentFactorNodeId, 'sub_agent', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} @@ -2842,7 +2840,6 @@ export const ProtocolCanvas = forwardRef setSkillFactorNodeId(null)} @@ -2858,11 +2855,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, - ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { skill_selection: next.levels[0] ?? [], skill_factor_mode: next.level_type }), - factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH]: next.name }, - } } : node)) + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, skillFactorNodeId, 'skill', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} @@ -2882,7 +2879,6 @@ export const ProtocolCanvas = forwardRef setToolFactorNodeId(null)} @@ -2898,11 +2894,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => isIndividualToolMode(next.level_type as ToolFactorMode) ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : next.level_type === 'tool_names' ? { config: { ...(node.data.config as object), tool_names: next.levels[0] ?? [] } } : { tool_selection: next.levels[0] ?? [], tool_factor_mode: next.level_type }), factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [toolFactorPath(next.level_type as ToolFactorMode)]: next.name }, - } } : node)) + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, toolFactorNodeId, 'tool', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} @@ -2923,7 +2919,6 @@ export const ProtocolCanvas = forwardRef setScriptFactorNodeId(null)} @@ -2939,11 +2934,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => isIndividualScriptMode(next.level_type as ScriptFactorMode) ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { script_selection: next.levels[0] ?? [], script_factor_mode: next.level_type }), factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [scriptFactorPath(next.level_type as ScriptFactorMode)]: next.name }, - } } : node)) + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, scriptFactorNodeId, 'script', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} @@ -2964,7 +2959,6 @@ export const ProtocolCanvas = forwardRef setKnowledgeFactorNodeId(null)} @@ -2980,11 +2974,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, - ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { knowledge_selection: next.levels[0] ?? [], knowledge_factor_mode: next.level_type }), - factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'config.enabled' : KNOWLEDGE_FACTOR_PATH]: next.name }, - } } : node)) + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, knowledgeFactorNodeId, 'knowledge', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} @@ -3004,7 +2998,6 @@ export const ProtocolCanvas = forwardRef setDatasetFactorNodeId(null)} @@ -3020,11 +3013,11 @@ export const ProtocolCanvas = forwardRef current.map((node) => node.id === ownerId ? { ...node, data: { + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, - ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { dataset_selection: next.levels[0] ?? [], dataset_factor_mode: next.level_type }), - factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [next.level_type === 'boolean' ? 'config.enabled' : DATASET_FACTOR_PATH]: next.name }, - } } : node)) + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, datasetFactorNodeId, 'dataset', next).nodes as unknown as Node[]) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) }} diff --git a/frontend/src/components/protocol/ScriptFactorDialog.test.tsx b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx index f63a7e82..287ac70a 100644 --- a/frontend/src/components/protocol/ScriptFactorDialog.test.tsx +++ b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx @@ -27,7 +27,7 @@ it('keeps the dialog open after removal and permits choosing another factor type expect(screen.queryByRole('status')).not.toBeInTheDocument() }) -it.each(['Script levels', 'All agent scripts on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { +it.each(['Script levels', 'All scripts on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { const save = vi.fn().mockResolvedValue(undefined) render() const option = screen.getByRole('button', { name: mode }) diff --git a/frontend/src/components/protocol/ScriptFactorDialog.tsx b/frontend/src/components/protocol/ScriptFactorDialog.tsx index a76a5934..4e3ed05f 100644 --- a/frontend/src/components/protocol/ScriptFactorDialog.tsx +++ b/frontend/src/components/protocol/ScriptFactorDialog.tsx @@ -1,28 +1,27 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' import { useState } from 'react' import { ArrowDown, ArrowUp, Split } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { connectedScripts, reconcileScriptFactor, requiredScriptIssues, scriptFactorConflict, scriptFactorModes as modes, SCRIPT_FACTOR_PATH, isIndividualScriptMode, scriptFactorPath, type ScriptFactorMode as Mode } from '@/lib/scriptFactors' import type { DesignFactor } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' import { computeFactorName } from './factorLevels' -export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, initialAgentId, onClose, onSave, onRemove }: { +export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { scriptNodeId: string graph: ProtocolGraph factors: DesignFactor[] initialMode?: Mode - initialAgentId?: string onClose: () => void onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise onRemove: (name: string) => Promise }) { const script = graph.nodes.find((node) => node.id === scriptNodeId)! - const agents = graph.nodes.filter((node) => ['agent', 'sub_agent'].includes(node.type) && graph.edges.some((edge) => edge.source === scriptNodeId && edge.target === node.id && edge.targetHandle === 'tool')) - const [agentId, setAgentId] = useState(initialAgentId ?? agents.find((agent) => agent.data.factor_bindings?.[SCRIPT_FACTOR_PATH])?.id ?? agents[0]?.id ?? '') + const agents = factorRecipients(graph, scriptNodeId, 'script') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[SCRIPT_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) const initialFactor = existingFor(agentId, SCRIPT_FACTOR_PATH) ?? existingFor(scriptNodeId, 'config.enabled') const initial = initialFactor && !isIndividualScriptMode(initialFactor.level_type as Mode) && initialFactor.level_type ? reconcileScriptFactor(initialFactor, graph, agentId) : initialFactor @@ -61,7 +60,6 @@ export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, const blockedReason = unavailable(mode) const labels = draft.level_labels ?? [] function change(nextMode: Mode, nextAgentId = agentId) { - setAgentId(nextAgentId) setMode(nextMode) setDraft(seed(nextMode, nextAgentId)) setError('') @@ -95,11 +93,7 @@ export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, Choose how scripts vary across experimental cells.
- {agents.length > 1 &&
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, scriptNodeId, 'script')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Script node.' : mode === 'script_selection' ? 'Each cell receives exactly one connected script.' : 'Each cell enables every connected script, including deactivated scripts, or disables them all.'}

@@ -112,10 +106,10 @@ export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode,
{mode !== 'boolean' && <>}
)} -

Agent-level factors follow current Script connections. Changes require design review and regeneration.

+

Shared factors require matching Script connections. Changes require design review and regeneration.

{requiredIssues.map((issue) =>

{issue}

)} - {current && } } + {current && } {error &&

{error}

} diff --git a/frontend/src/components/protocol/ScriptTestSelectors.tsx b/frontend/src/components/protocol/ScriptTestSelectors.tsx index f1f56d78..6b3d776e 100644 --- a/frontend/src/components/protocol/ScriptTestSelectors.tsx +++ b/frontend/src/components/protocol/ScriptTestSelectors.tsx @@ -1,19 +1,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedScripts, scriptId } from '@/lib/scriptFactors' +import { connectedScripts, scriptId, SCRIPT_FACTOR_PATH } from '@/lib/scriptFactors' import type { useScriptTestSelection } from './useScriptTestSelection' export function ScriptTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedScripts(selection.graph!, node.id) if (node.data.script_factor_mode === 'script_toggle') return
-

{node.data.label}: scripts for this test

+

{node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] || node.data.label}: scripts for this test

return
-

{node.data.label}: script for this test

+

{node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] || node.data.label}: script for this test

{ - if (!id) return - const existing = existingFor(id, SKILL_FACTOR_PATH) - change((existing?.level_type as Mode) ?? mode, id) - }}>{String(agents.find((agent) => agent.id === agentId)?.data.label)}{agents.map((agent) => {String(agent.data.label)})}
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, skillNodeId, 'skill')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Skill node.' : mode === 'skill_selection' ? 'Each cell receives exactly one connected skill.' : 'Each cell enables every connected skill, including deactivated skills, or disables them all.'}

@@ -110,9 +104,9 @@ export function SkillFactorDialog({ skillNodeId, graph, factors, initialMode, in
{mode !== 'boolean' && <>} )} -

Agent-level factors follow current Skill connections. Changes require design review and regeneration.

- {current && } +

Shared factors require matching Skill connections. Changes require design review and regeneration.

} + {current && } {error &&

{error}

} diff --git a/frontend/src/components/protocol/SkillTestSelectors.tsx b/frontend/src/components/protocol/SkillTestSelectors.tsx index ea2b239e..c7680894 100644 --- a/frontend/src/components/protocol/SkillTestSelectors.tsx +++ b/frontend/src/components/protocol/SkillTestSelectors.tsx @@ -1,19 +1,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedSkills, skillId } from '@/lib/skillFactors' +import { connectedSkills, skillId, SKILL_FACTOR_PATH } from '@/lib/skillFactors' import type { useSkillTestSelection } from './useSkillTestSelection' export function SkillTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedSkills(selection.graph!, node.id) if (node.data.skill_factor_mode === 'skill_toggle') return
-

{node.data.label}: skills for this test

+

{node.data.factor_bindings?.[SKILL_FACTOR_PATH] || node.data.label}: skills for this test

return
-

{node.data.label}: skill for this test

+

{node.data.factor_bindings?.[SKILL_FACTOR_PATH] || node.data.label}: skill for this test

{ - if (!id) return - const existing = existingFor(id, SUB_AGENT_FACTOR_PATH) - change((existing?.level_type as Mode) ?? mode, id) - }}>{String(agents.find((agent) => agent.id === agentId)?.data.label)}{agents.map((agent) => {String(agent.data.label)})}
} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, subAgentNodeId, 'sub_agent')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Sub-Agent node.' : mode === 'sub_agent_selection' ? 'Each cell receives exactly one connected sub-agent.' : 'Each cell enables every connected sub-agent, including deactivated sub-agents, or disables them all.'}

@@ -110,9 +104,9 @@ export function SubAgentFactorDialog({ subAgentNodeId, graph, factors, initialMo
{mode !== 'boolean' && <>} )} -

Agent-level factors follow current Sub-Agent connections. Changes require design review and regeneration.

- {current && } +

Shared factors require matching Sub-Agent connections. Changes require design review and regeneration.

} + {current && } {error &&

{error}

} diff --git a/frontend/src/components/protocol/SubAgentTestSelectors.tsx b/frontend/src/components/protocol/SubAgentTestSelectors.tsx index 43722275..49961e73 100644 --- a/frontend/src/components/protocol/SubAgentTestSelectors.tsx +++ b/frontend/src/components/protocol/SubAgentTestSelectors.tsx @@ -1,5 +1,5 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedSubAgents } from '@/lib/subAgentFactors' +import { connectedSubAgents, SUB_AGENT_FACTOR_PATH } from '@/lib/subAgentFactors' import type { useSubAgentTestSelection } from './useSubAgentTestSelection' export function SubAgentTestSelectors({ selection }: { selection: ReturnType }) { @@ -7,7 +7,7 @@ export function SubAgentTestSelectors({ selection }: { selection: ReturnType -

{node.data.label}: {toggle ? 'sub-agents' : 'sub-agent'} for this test

+

{node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] || node.data.label}: {toggle ? 'sub-agents' : 'sub-agent'} for this test

{ - if (!id) return - const existing = existingFor(id, TOOL_FACTOR_PATH) - change((existing?.level_type as Mode) ?? mode, id) - }}>{String(agents.find((agent) => agent.id === agentId)?.data.label)}{agents.map((agent) => {String(agent.data.label)})}} + {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, toolNodeId, 'tool')}

}
{(Object.keys(modes) as Mode[]).map((value) => )}
{blockedReason ?

{blockedReason}

: <>

{mode === 'boolean' ? 'Enable or disable this Tool node.' : mode === 'tool_names' ? 'Vary the allowed tools of this node’s pinned server.' : mode === 'tool_selection' ? 'Each cell receives exactly one connected tool.' : 'Each cell enables every connected tool, including deactivated tools, or disables them all.'}

@@ -114,9 +108,9 @@ export function ToolFactorDialog({ toolNodeId, graph, factors, initialMode, init {mode === 'tool_names' && } )} {mode === 'tool_names' && } -

Agent-level factors follow current Tool connections. Changes require design review and regeneration.

- {current && } +

Shared factors require matching Tool connections. Changes require design review and regeneration.

} + {current && } {error &&

{error}

} diff --git a/frontend/src/components/protocol/ToolTestSelectors.tsx b/frontend/src/components/protocol/ToolTestSelectors.tsx index b901d70e..f0929b98 100644 --- a/frontend/src/components/protocol/ToolTestSelectors.tsx +++ b/frontend/src/components/protocol/ToolTestSelectors.tsx @@ -1,19 +1,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { connectedTools, toolId } from '@/lib/toolFactors' +import { connectedTools, toolId, TOOL_FACTOR_PATH } from '@/lib/toolFactors' import type { useToolTestSelection } from './useToolTestSelection' export function ToolTestSelectors({ selection }: { selection: ReturnType }) { return
{selection.owners.map((node) => { const options = connectedTools(selection.graph!, node.id) if (node.data.tool_factor_mode === 'tool_toggle') return
-

{node.data.label}: tools for this test

+

{node.data.factor_bindings?.[TOOL_FACTOR_PATH] || node.data.label}: tools for this test

return
-

{node.data.label}: tool for this test

+

{node.data.factor_bindings?.[TOOL_FACTOR_PATH] || node.data.label}: tool for this test

v && patch({ provider: v, model: '' })}> + Effort value && patchConfig({ + resolved_capabilities: value === 'temperature' + ? { supports_temperature: true, supports_effort: false, effort_levels: [], default_effort: null } + : null, + })} + > + + {(value: string) => value === 'temperature' ? 'Temperature only' : 'Automatic'} + + + Automatic + Temperature only + + +

Choose Temperature only if your custom model does not support effort.

+
+ )} + + {capabilitiesPending &&

{capabilitiesError ?? 'Resolving model capabilities…'}

} + {capabilitiesError && } +
{showTemperature && (
@@ -279,10 +282,10 @@ export function ModelNodeInspector({ }} > - {(value: string) => (value === '__none__' ? '(none)' : value)} + {(value: string) => (value === '__none__' ? `Default (${defaultEffort})` : value)} - (none) + Default ({defaultEffort}) {effortLevels.map((level) => ( {level} diff --git a/frontend/src/components/protocol/nodeConfigIssues.ts b/frontend/src/components/protocol/nodeConfigIssues.ts index e53c0f91..fcf44240 100644 --- a/frontend/src/components/protocol/nodeConfigIssues.ts +++ b/frontend/src/components/protocol/nodeConfigIssues.ts @@ -1,8 +1,9 @@ +import { modelCapabilities } from '@/lib/modelCapabilities' import type { QueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' import { toPersistedGraph } from '@/lib/protocolGraph' import { isUnderIterated, raiseForTruncation, suggestedMaxIterations } from '@/lib/reasonActIterations' -import type { LLMSettingModelsResponse } from '@/types/llmSettings' +import type { LLMSettingModelsResponse, ModelCapabilities } from '@/types/llmSettings' import type { OkfBundle, OkfDocument } from '@/types/okf' import type { Skill } from '@/types/skills' import type { @@ -17,7 +18,7 @@ import type { SkillNodeData, } from '@/types/protocols' import { PROVIDER_META } from './nodes/ModelNode' -import { providerModelsKey } from './useProviderModels' +import { modelCapabilitiesKey, providerModelsKey } from './useProviderModels' export interface NodeConfigIssue { nodeId: string @@ -41,9 +42,8 @@ export interface NodeConfigIssue { // among the provider's own discovered list (ModelNode.tsx's own richer // check) -- read from cache only, via the exact same queryKey ModelNode.tsx // already populates by rendering on this same canvas, so this never fires -// its own network request or makes clicking Run wait on one. A cache miss -// (that query never ran, or hasn't resolved yet) just means "can't tell," -// same as ModelNode.tsx's own empty-list case -- not treated as an issue. +// its own network request or makes clicking Run wait on one. An unresolved +// custom model is flagged until its sampling capabilities load. // `truncatedCaps` maps a Reason + Act node id to the `max_iterations` the last // run of its agent actually hit (ProtocolCanvas.tsx's truncationByPattern). // Optional because the callers that run from a stored `Protocol.graph` alone @@ -106,11 +106,14 @@ export function findNodeConfigIssues( } } if (config?.max_tokens == null) issues.push('Max tokens is required') - // Same "unrecognized model defaults to temperature-only" fallback as - // ModelNodeInspector.tsx's own showTemperature -- required whenever - // it's the field actually offered for this model, so it's never - // Motoro's own silent ModelConfig default (0.7) filling the gap. - if ((selectedModelInfo?.supports_temperature ?? true) && config?.temperature == null) { + const customKey = modelCapabilitiesKey(config?.provider, config?.model) + const resolved = config?.resolved_capabilities ?? selectedModelInfo ?? queryClient.getQueryData(customKey) + if (config?.model && !resolved) { + issues.push(queryClient.getQueryState(customKey)?.status === 'error' + ? 'Could not resolve model capabilities. Try again.' + : 'Model capabilities are still loading') + } + if (resolved && modelCapabilities(resolved).supports_temperature && config?.temperature == null) { issues.push('Temperature is required') } break diff --git a/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx index e1600cf5..78ec343a 100644 --- a/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx +++ b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx @@ -1,3 +1,5 @@ +import type { ModelCapabilities } from '@/types/llmSettings' +import { modelCapabilities } from '@/lib/modelCapabilities' import { useEffect, useState } from 'react' import { Handle, Position, useUpdateNodeInternals } from '@xyflow/react' import { ChevronDown, ChevronUp, Split } from 'lucide-react' @@ -26,10 +28,11 @@ function LevelPreview({ level, label, index, disabled, onEdit }: { const provider = String(config.provider ?? '') const meta = PROVIDER_META[provider] const Icon = meta?.icon ?? Split - const { models } = useProviderModels(provider) + const { models, capabilities: resolvedCapabilities, capabilitiesPending, capabilitiesError } = useProviderModels(provider, config.model as string | undefined, config.resolved_capabilities as ModelCapabilities | undefined) const model = models.find((model) => model.id === config.model) - const effort = model?.supports_effort ?? false - const temperature = model?.supports_temperature ?? true + const capabilities = modelCapabilities(resolvedCapabilities ?? model, config.resolved_capabilities as ModelCapabilities | undefined) + const effort = !capabilitiesPending && capabilities.supports_effort + const temperature = !capabilitiesPending && capabilities.supports_temperature return {open && (
diff --git a/frontend/src/components/protocol/TestRunResults.test.tsx b/frontend/src/components/protocol/TestRunResults.test.tsx index 63299072..30fb1d26 100644 --- a/frontend/src/components/protocol/TestRunResults.test.tsx +++ b/frontend/src/components/protocol/TestRunResults.test.tsx @@ -37,6 +37,23 @@ function testRun(overrides: Partial = {}): TestRun { } describe('TestRunResults', () => { + it('distinguishes bypassed nodes from completed execution', () => { + render() + + const progress = screen.getByRole('region', { name: 'Node progress' }) + expect(within(progress).getByText('Bypassed')).toBeInTheDocument() + expect(within(progress).getByText('Done')).toBeInTheDocument() + }) + it.each([ ['completed', 'Completed'], ['failed', 'Failed'], diff --git a/frontend/src/components/protocol/TestRunResults.tsx b/frontend/src/components/protocol/TestRunResults.tsx index 0d899c76..657cc139 100644 --- a/frontend/src/components/protocol/TestRunResults.tsx +++ b/frontend/src/components/protocol/TestRunResults.tsx @@ -188,7 +188,7 @@ export function TestRunResults({

{group.label}

{group.entries.map(([nodeId, nodeRun]) => { - const badge = nodeRunBadge(nodeRun.status, Boolean(nodeRun.truncation)) + const badge = nodeRunBadge(nodeRun.status, Boolean(nodeRun.truncation), nodeRun.bypassed) const label = nodeTypes.get(nodeId) === 'sub_agent' && nodeRun.status === 'skipped' ? (running ? 'Available' : 'Not invoked') : badge?.label diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index 81e2e94d..6713f7aa 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -38,6 +38,7 @@ export function AgentNode({ // Paired with runStatus, never derivable from it: a truncated run is still // `completed` (see NodeRunState.truncation). runTruncated?: boolean + runBypassed?: boolean missingModel?: boolean missingOutputParser?: boolean canRunAlone?: boolean @@ -62,7 +63,7 @@ export function AgentNode({ }) { const isSubAgent = data.isSubAgent === true const accent = nodeAccent(isSubAgent ? 'sub_agent' : 'agent') - const badge = nodeRunBadge(data.runStatus, data.runTruncated) + const badge = nodeRunBadge(data.runStatus, data.runTruncated, data.runBypassed) // Peers are offered to the model as function schemas -- that is the only // channel a consultation can be *chosen* through -- so an agent on a model // that can't accept them would know its peers exist and never be able to diff --git a/frontend/src/components/protocol/nodes/CriticGateNode.tsx b/frontend/src/components/protocol/nodes/CriticGateNode.tsx index 634d2cb9..995079b3 100644 --- a/frontend/src/components/protocol/nodes/CriticGateNode.tsx +++ b/frontend/src/components/protocol/nodes/CriticGateNode.tsx @@ -20,11 +20,11 @@ export function CriticGateNode({ id, data, selected, -}: NodeProps & { data: CriticGateNodeData & { runStatus?: NodeRunStatus; runTruncated?: boolean } }) { +}: NodeProps & { data: CriticGateNodeData & { runStatus?: NodeRunStatus; runTruncated?: boolean; runBypassed?: boolean } }) { const enabled = data.config?.enabled ?? true const Icon = enabled ? ShieldCheck : ShieldOff const summary = enabled ? `Up to ${data.config?.max_revisions ?? 1} revision(s)` : 'Gate disabled' - const badge = nodeRunBadge(data.runStatus, data.runTruncated) + const badge = nodeRunBadge(data.runStatus, data.runTruncated, data.runBypassed) const modelConnections = useNodeConnections({ id, handleType: 'target', handleId: 'model' }) const { updateNodeData } = useReactFlow() const { requestMakeFactor } = useProtocolCanvasActions() diff --git a/frontend/src/components/protocol/nodes/ToolStepNode.tsx b/frontend/src/components/protocol/nodes/ToolStepNode.tsx index 771a260e..7e780cad 100644 --- a/frontend/src/components/protocol/nodes/ToolStepNode.tsx +++ b/frontend/src/components/protocol/nodes/ToolStepNode.tsx @@ -28,13 +28,14 @@ export function ToolStepNode({ }: NodeProps & { data: ToolStepNodeData & { runStatus?: NodeRunStatus runTruncated?: boolean + runBypassed?: boolean mainInFull?: boolean mainOutFull?: boolean } }) { const isActive = data.active ?? true const toolName = data.config?.tool_name const summary = !isActive ? 'Step disabled' : toolName ? `${toolName}()` : 'No tool selected' - const badge = nodeRunBadge(data.runStatus, data.runTruncated) + const badge = nodeRunBadge(data.runStatus, data.runTruncated, data.runBypassed) const { updateNodeData } = useReactFlow() const { requestMakeFactor } = useProtocolCanvasActions() diff --git a/frontend/src/lib/protocolRun.ts b/frontend/src/lib/protocolRun.ts index 026db434..bf684ea3 100644 --- a/frontend/src/lib/protocolRun.ts +++ b/frontend/src/lib/protocolRun.ts @@ -17,7 +17,8 @@ export const TERMINAL_RUN_STATUSES = new Set([ // failed. A different status domain, but the same color meanings everywhere // reads as one consistent system rather than two competing ones. // -// *truncated* is the one thing that overrides the status it is paired with: a +// Bypassed nodes completed by passing input through, without executing. +// *truncated* also overrides the status it is paired with: a // run cut off by its iteration ceiling is recorded as `completed` on purpose // (its work is real and downstream nodes consumed it), but a green "Done" is // the wrong thing to tell someone whose agent never got to write its answer. @@ -25,7 +26,11 @@ export const TERMINAL_RUN_STATUSES = new Set([ export function nodeRunBadge( status: NodeRunStatus | undefined, truncated = false, + bypassed = false, ): { label: string; className: string } | null { + if (status === 'completed' && bypassed) { + return { label: 'Bypassed', className: 'border-transparent bg-muted text-muted-foreground' } + } if (status === 'completed' && truncated) { return { label: 'Hit iteration limit', diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 5e1be6cd..8cf221de 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -311,6 +311,7 @@ export interface ResultNodeRun { node_id: string node_label: string status: string + bypassed?: boolean output_text: string | null error: string | null // Senders this node's prompt referenced that produced nothing, by label diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index 3f260c2b..b958c916 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -77,6 +77,8 @@ export type NodeRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'sk export interface NodeRunState { status: NodeRunStatus + // Completed by passing upstream output through; this node's logic did not run. + bypassed?: boolean run_id?: string | null output_text?: string | null error?: string | null diff --git a/src/asaree/services/experiment_run_results.py b/src/asaree/services/experiment_run_results.py index 798ed947..236f3603 100644 --- a/src/asaree/services/experiment_run_results.py +++ b/src/asaree/services/experiment_run_results.py @@ -246,11 +246,12 @@ def _has_execution_evidence(node_run: dict[str, Any]) -> bool: Connector/config nodes are persisted as ``completed`` so the executor has a total graph record, but they don't independently run, spend, emit, or - fail. An agent run ID, output, error, or non-terminal execution state is - the evidence that makes a row useful to an end user. + fail. An agent run ID, output, error, explicit bypass outcome, or + non-terminal execution state makes a row useful to an end user. """ return bool( node_run.get("run_id") + or node_run.get("bypassed") or node_run.get("output_text") or node_run.get("error") or node_run.get("status") in {"running", "failed", "cancelled"} @@ -856,6 +857,7 @@ def execution_detail(protocol_run: ProtocolRun) -> dict[str, Any]: "node_id": node_id, "node_label": node_labels.get(node_id, node_id), "status": node_run.get("status", "unknown"), + "bypassed": bool(node_run.get("bypassed")), "output_text": node_run.get("output_text"), "error": node_run.get("error"), # Labels, not ids -- the walk records ids (see diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index d0d0733f..0085ebf2 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -4654,6 +4654,7 @@ async def _run_gated_worker( _completed_worker_record(output_text, attempt, run_id_str, extraction), { "status": "completed", + "bypassed": True, "output_text": output_text, "approved": None, "revisions_used": 0, @@ -6102,6 +6103,7 @@ async def record_step_provenance(step: dict[str, Any], *, step_node_id: str = no else: node_runs[node_id] = { "status": "failed" if error else "completed", + **({"bypassed": True} if not _is_node_active(node) else {}), "output_text": output_text, "error": error, "run_id": str(run_id) if run_id else None, diff --git a/tests/test_experiment_run_results.py b/tests/test_experiment_run_results.py index 199a0e92..623fa065 100644 --- a/tests/test_experiment_run_results.py +++ b/tests/test_experiment_run_results.py @@ -678,3 +678,4 @@ def test_execution_evidence_excludes_completed_configuration_nodes() -> None: assert _has_execution_evidence({"status": "completed", "output_text": None, "error": None}) is False assert _has_execution_evidence({"status": "completed", "run_id": "agent-run"}) is True assert _has_execution_evidence({"status": "failed", "error": "boom"}) is True + assert _has_execution_evidence({"status": "completed", "bypassed": True, "output_text": ""}) is True diff --git a/tests/test_protocol_bypass.py b/tests/test_protocol_bypass.py new file mode 100644 index 00000000..2be93abb --- /dev/null +++ b/tests/test_protocol_bypass.py @@ -0,0 +1,95 @@ +"""Pipeline bypass outcomes without a database or provider connection.""" + +import uuid +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +from asaree.services import protocol_execution as pe + + +async def test_inactive_agent_passes_output_through_and_records_bypass(monkeypatch): + graph = { + "nodes": [ + {"id": name, "type": "agent", "data": {"active": name != "b", "config": {}}} for name in ("a", "b", "c") + ] + + [{"id": "model", "type": "model_openai", "data": {"config": {}}}], + "edges": [ + {"id": "a-b", "source": "a", "target": "b"}, + {"id": "b-c", "source": "b", "target": "c"}, + ] + + [ + {"id": f"model-{name}", "source": "model", "target": name, "targetHandle": "model"} + for name in ("a", "b", "c") + ], + } + protocol = SimpleNamespace(id=uuid.uuid4(), experiment_id=None, graph=graph) + run = SimpleNamespace( + id=uuid.uuid4(), + protocol_id=protocol.id, + owner_id=uuid.uuid4(), + status="pending", + node_runs={}, + cancel_requested_at=None, + row_result_id=None, + replicate_result_id=None, + dataset_row=None, + attempt_result=None, + protocol_revision_id=None, + design_revision_id=None, + target_node_id=None, + factor_values=None, + replicate_label=None, + ) + db = MagicMock() + selected = MagicMock() + selected.scalar_one_or_none.return_value = run + db.execute = AsyncMock(return_value=selected) + + @asynccontextmanager + async def session(): + yield db + + async def status(_db, _run_id, *, status, **kwargs): + run.status = status + + async def update(_db, _run_id, node_id, fields): + run.node_runs[node_id] = {**run.node_runs.get(node_id, {}), **fields} + + calls = [] + + async def execute(node, **kwargs): + calls.append(node["id"]) + return f"Output from {node['id']}", None, None, None + + monkeypatch.setattr(pe, "get_session", session) + monkeypatch.setattr(pe, "get_protocol_run", AsyncMock(return_value=run)) + monkeypatch.setattr(pe, "get_protocol", AsyncMock(return_value=protocol)) + monkeypatch.setattr(pe, "hydrate_registry", AsyncMock()) + monkeypatch.setattr(pe, "set_status", status) + monkeypatch.setattr(pe, "update_node_run", update) + monkeypatch.setattr(pe, "_run_agent_node", execute) + monkeypatch.setattr( + pe, + "_node_run_context", + AsyncMock( + return_value=( + {}, + SimpleNamespace(seeded=(), unsplit_name=""), + ) + ), + ) + monkeypatch.setattr("asaree.services.agent_messenger.record_sequential_transcript", AsyncMock()) + + await pe.run_protocol(run.id) + assert run.status == "completed" + assert calls == ["a", "c"] + assert run.node_runs["b"] == { + "status": "completed", + "bypassed": True, + "output_text": "Output from a", + "error": None, + "run_id": None, + } + assert "bypassed" not in run.node_runs["a"] + assert "bypassed" not in run.node_runs["c"] diff --git a/tests/test_protocol_execution.py b/tests/test_protocol_execution.py index edb98029..895c09d5 100644 --- a/tests/test_protocol_execution.py +++ b/tests/test_protocol_execution.py @@ -1255,6 +1255,7 @@ async def fake_run_critic(gate, **kwargs): assert worker_run["payload"] == {"score": 0.91} assert gate_run["output_text"] == "" assert gate_run["payload"] == {"score": 0.91} + assert "bypassed" not in gate_run @pytest.mark.parametrize(("max_revisions", "enabled"), [(0, True), (1, False)]) @@ -1273,6 +1274,7 @@ async def unexpected_critic_call(gate, **kwargs): _worker_run, gate_run = await _run(*_worker_gate(max_revisions=max_revisions, enabled=enabled)) assert gate_run["payload"] == {"score": 0.91} + assert gate_run.get("bypassed", False) is (not enabled) async def test_gated_worker_rejected_then_approved_on_revision(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1391,6 +1393,9 @@ async def fake_run_critic(gate, **kwargs): assert worker_run["attempts"] == 1 assert critic_calls == [] assert gate_run["approved"] is None + assert gate_run["status"] == "completed" + assert gate_run["bypassed"] is True + assert "bypassed" not in worker_run async def test_gated_worker_worker_failure_stops_immediately(monkeypatch: pytest.MonkeyPatch) -> None: @@ -2171,6 +2176,9 @@ async def fake_run_agent_node(node, **kwargs): assert fetched is not None assert fetched.status == "completed" assert fetched.node_runs["b"]["status"] == "completed" + assert fetched.node_runs["b"]["bypassed"] is True + assert "bypassed" not in fetched.node_runs["a"] + assert "bypassed" not in fetched.node_runs["c"] assert fetched.node_runs["b"]["output_text"] == "real output from a" assert fetched.node_runs["c"]["output_text"] == "real output from c" finally: From 3f24fcc42d47136bf49cb1a2f67f3bfe5bc7a136 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 12 Oct 2026 14:56:37 -0700 Subject: [PATCH 31/63] fix: prevent untouched factor dialogs from saving stale drafts --- .../protocol/DatasetFactorEditor.tsx | 3 +- .../protocol/FactorEditorDialog.test.tsx | 117 ++++++++++++++++++ .../protocol/FactorEditorDialog.tsx | 61 +++++---- .../protocol/KnowledgeFactorEditor.tsx | 3 +- .../protocol/ModelFactorDialog.test.tsx | 16 +++ .../components/protocol/ModelFactorDialog.tsx | 5 +- .../protocol/PatternFactorDialog.test.tsx | 16 +++ .../protocol/PatternFactorDialog.tsx | 5 +- .../protocol/ScriptFactorEditor.tsx | 3 +- .../components/protocol/SkillFactorEditor.tsx | 3 +- .../protocol/SubAgentFactorEditor.tsx | 3 +- .../protocol/ToolFactorDialog.test.tsx | 15 +++ .../components/protocol/ToolFactorDialog.tsx | 13 +- .../components/protocol/ToolFactorEditor.tsx | 3 +- .../protocol/useDialogAutosave.test.tsx | 13 ++ .../components/protocol/useDialogAutosave.ts | 8 +- .../src/components/protocol/useDialogDraft.ts | 37 ++++++ 17 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 frontend/src/components/protocol/FactorEditorDialog.test.tsx create mode 100644 frontend/src/components/protocol/useDialogDraft.ts diff --git a/frontend/src/components/protocol/DatasetFactorEditor.tsx b/frontend/src/components/protocol/DatasetFactorEditor.tsx index 8bf38aa9..dc0c8e21 100644 --- a/frontend/src/components/protocol/DatasetFactorEditor.tsx +++ b/frontend/src/components/protocol/DatasetFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { useQuery } from '@tanstack/react-query' import { ArrowDown, ArrowUp } from 'lucide-react' import { datasetsApi } from '@/api/client' @@ -15,7 +16,7 @@ export function DatasetFactorEditor({ factor, open, onOpenChange, onSave, onRemo onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const datasets = useQuery({ queryKey: ['datasets'], queryFn: () => datasetsApi.list() }) diff --git a/frontend/src/components/protocol/FactorEditorDialog.test.tsx b/frontend/src/components/protocol/FactorEditorDialog.test.tsx new file mode 100644 index 00000000..dc9bd379 --- /dev/null +++ b/frontend/src/components/protocol/FactorEditorDialog.test.tsx @@ -0,0 +1,117 @@ +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { expect, it, vi } from 'vitest' +import type { DesignFactor } from '@/types/experiments' +import { FactorEditorDialog } from './FactorEditorDialog' + +const factorA: DesignFactor = { name: 'Model parameter', level_type: 'number', levels: [2048, 1024], level_labels: ['high', 'low'] } +const factorB: DesignFactor = { ...factorA, levels: [1024, 2048], level_labels: ['low', 'high'] } + +it.each(['tool_selection', 'dataset_selection', 'skill_selection', 'knowledge_selection', 'script_selection', 'sub_agent_selection'] as const)( + 'refreshes untouched %s editors and preserves level-label pairs', async (level_type) => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { onSave: save, onOpenChange: vi.fn() } + const factor = { name: 'Selection', level_type, levels: [['a'], ['b']], level_labels: ['first', 'second'] } + const refreshed = { ...factor, levels: [['b'], ['a']], level_labels: ['second', 'first'] } + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false } } }) + for (const key of ['datasets', 'skills', 'knowledge-factor-library']) client.setQueryData([key], []) + const { rerender } = render(, { + wrapper: ({ children }) => {children}, + }) + rerender() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('second') + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'renamed' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['renamed', 'first'] })) + }, +) + +it('does not save after a refetch followed by opening and closing without edits', () => { + const save = vi.fn() + const props = { onSave: save, onOpenChange: vi.fn() } + const { rerender } = render() + rerender() + rerender() + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + expect(save).not.toHaveBeenCalled() +}) + +it('refreshes an untouched open dialog without writing stale levels on close', () => { + const save = vi.fn() + const props = { onSave: save, onOpenChange: vi.fn(), open: true } + const { rerender } = render() + rerender() + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + expect(save).not.toHaveBeenCalled() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('low') +}) + +it('saves one label edit with the refreshed levels still paired correctly', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { onSave: save, onOpenChange: vi.fn(), open: true } + const { rerender } = render() + rerender() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'small' } }) + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...factorB, level_labels: ['small', 'high'] }, undefined)) +}) + +it('preserves a pending user edit when a refetch arrives', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { onSave: save, onOpenChange: vi.fn(), open: true } + const { rerender } = render() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'large' } }) + rerender() + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...factorA, level_labels: ['large', 'low'] }, undefined)) +}) + +it('reseeds on reopening and never autosaves an untouched refetch', async () => { + vi.useFakeTimers() + try { + const save = vi.fn() + const props = { onSave: save, onOpenChange: vi.fn() } + const { rerender } = render() + rerender() + rerender() + await act(() => vi.advanceTimersByTimeAsync(1000)) + expect(screen.getByLabelText('Level 1 label')).toHaveValue('low') + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + expect(save).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } +}) + +it('can save the same edit again after an acknowledged save and external reset', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { onSave: save, onOpenChange: vi.fn(), open: true } + const edited = { ...factorA, level_labels: ['large', 'low'] } + const { rerender } = render() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'large' } }) + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + await waitFor(() => expect(save).toHaveBeenCalledTimes(1)) + rerender() + rerender() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('high') + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'large' } }) + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + await waitFor(() => expect(save).toHaveBeenCalledTimes(2)) +}) + +it('recognizes a normalized save as clean before applying a later external change', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { onSave: save, onOpenChange: vi.fn(), open: true } + const { rerender } = render() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: ' large ' } }) + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + await waitFor(() => expect(save).toHaveBeenCalledTimes(1)) + rerender() + rerender() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('low') + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + expect(save).toHaveBeenCalledTimes(1) +}) diff --git a/frontend/src/components/protocol/FactorEditorDialog.tsx b/frontend/src/components/protocol/FactorEditorDialog.tsx index 47118e93..35ae3613 100644 --- a/frontend/src/components/protocol/FactorEditorDialog.tsx +++ b/frontend/src/components/protocol/FactorEditorDialog.tsx @@ -2,7 +2,7 @@ import type { ModelCapabilities } from '@/types/llmSettings' import { modelCapabilities } from '@/lib/modelCapabilities' import { SubAgentFactorEditor } from './SubAgentFactorEditor' import type { ProtocolGraph } from '@/types/protocols' -import { useEffect, useMemo, useState } from 'react' +import { useEffect, useMemo, useState, type SetStateAction } from 'react' import { useQuery } from '@tanstack/react-query' import { Plus, Split, X } from 'lucide-react' import { Badge } from '@/components/ui/badge' @@ -37,6 +37,7 @@ import { PROVIDER_META } from './nodes/ModelNode' import { PromptReferenceField } from './PromptReferenceField' import { PythonCodeEditor } from './PythonCodeEditor' import { useDialogAutosave } from './useDialogAutosave' +import { useDialogDraft } from './useDialogDraft' import type { DesignFactor } from '@/types/experiments' import { DatasetFactorEditor } from './DatasetFactorEditor' import { KnowledgeFactorEditor } from './KnowledgeFactorEditor' @@ -593,46 +594,45 @@ function StandardFactorEditorDialog({ // actual canvas node (this dialog has no way to do that itself). onSave: (factor: DesignFactor, field?: UnboundField) => void | Promise }) { - const [selectedField, setSelectedField] = useState(null) - const [selectedName, setSelectedName] = useState('') const [search, setSearch] = useState('') - const [levelType, setLevelType] = useState(levelTypeOf(factor)) - const [levels, setLevels] = useState(() => (isStructuredLevelType(levelTypeOf(factor)) ? factor.levels : factor.levels.map((l) => String(l)))) - const [levelLabels, setLevelLabels] = useState(() => factorLevelLabels(factor)) - - // Re-seed the local draft every time this dialog (re)opens -- same - // "re-seed on open" convention DesignTab.tsx and FactorBindableField.tsx's - // popover already use. In field-picker mode there's nothing to seed yet - // until a field is actually picked (see pickField below). useEffect(() => { - if (open) { - setSelectedField(null) - setSelectedName('') - setSearch('') - if (!pickableFields) { - const t = levelTypeOf(factor) - setLevelType(t) - setLevels(isStructuredLevelType(t) ? factor.levels : factor.levels.map((l) => String(l))) - setLevelLabels(factorLevelLabels(factor)) - } - } - // eslint-disable-next-line react-hooks/exhaustive-deps + if (open) setSearch('') }, [open]) + const initialLevelType = levelTypeOf(factor) + const { draft: localDraft, setDraft: setLocalDraft, dirty } = useDialogDraft({ + selectedField: null as UnboundField | null, + selectedName: '', + levelType: initialLevelType, + levels: isStructuredLevelType(initialLevelType) ? factor.levels : factor.levels.map(String), + levelLabels: factorLevelLabels(factor), + }, { + open, sessionKey: pickableFields ? 'new' : factor.name, + signatureOf: (value) => JSON.stringify(makeDraft( + pickableFields ? value.selectedName : factor.name, + value.levelType, value.levels, value.levelLabels, value.selectedField ?? undefined, + )), + }) + const { selectedField, selectedName, levelType, levels, levelLabels } = localDraft + function setLevels(value: SetStateAction) { + setLocalDraft((previous) => ({ ...previous, levels: typeof value === 'function' ? value(previous.levels) : value })) + } + function setLevelLabels(value: SetStateAction) { + setLocalDraft((previous) => ({ ...previous, levelLabels: typeof value === 'function' ? value(previous.levelLabels) : value })) + } function pickField(field: UnboundField) { if (field.connectorFactor && onPickConnectorFactor) { onPickConnectorFactor(field) return } - setSelectedField(field) const nextName = computeFactorName(field.nodeLabel, field.fieldLabel, existingNames ?? []) - setSelectedName(nextName) - setLevelType(field.levelType) const nextLevels = isStructuredLevelType(field.levelType) ? seedStructuredLevels(field.currentValue, field.levelType) : seedLevels(field.currentValue) - setLevels(nextLevels) - setLevelLabels(defaultFactorLevelLabels(nextName, nextLevels.length, field.levelType)) + setLocalDraft({ + selectedField: field, selectedName: nextName, levelType: field.levelType, + levels: nextLevels, levelLabels: defaultFactorLevelLabels(nextName, nextLevels.length, field.levelType), + }) } // Computed, not user-typed (see factorLevels.ts's computeFactorName) -- @@ -671,7 +671,6 @@ function StandardFactorEditorDialog({ const draft = makeDraft(name, levelType, levels, levelLabels, selectedField ?? undefined) const draftSignature = JSON.stringify(draft) - const initialLevelType = levelTypeOf(factor) const initialSignature = JSON.stringify(makeDraft( factor.name, initialLevelType, @@ -679,7 +678,7 @@ function StandardFactorEditorDialog({ factorLevelLabels(factor), )) const factorAutosave = useDialogAutosave({ - draft: open && !needsFieldPick && draftSignature !== initialSignature ? draft : null, + draft: open && dirty && !needsFieldPick && draftSignature !== initialSignature ? draft : null, signature: draftSignature, onSave: ({ factor: next, field }) => onSave(next, field), }) @@ -777,7 +776,7 @@ function StandardFactorEditorDialog({ diff --git a/frontend/src/components/protocol/KnowledgeFactorEditor.tsx b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx index bcd27137..a5350830 100644 --- a/frontend/src/components/protocol/KnowledgeFactorEditor.tsx +++ b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { ArrowDown, ArrowUp } from 'lucide-react' import { useKnowledgeLibrary } from './useKnowledgeLibrary' import { Button } from '@/components/ui/button' @@ -14,7 +15,7 @@ export function KnowledgeFactorEditor({ factor, open, onOpenChange, onSave, onRe onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const knowledge = useKnowledgeLibrary() diff --git a/frontend/src/components/protocol/ModelFactorDialog.test.tsx b/frontend/src/components/protocol/ModelFactorDialog.test.tsx index d7c59183..7f7e1975 100644 --- a/frontend/src/components/protocol/ModelFactorDialog.test.tsx +++ b/frontend/src/components/protocol/ModelFactorDialog.test.tsx @@ -18,6 +18,22 @@ const node: ProtocolNode & { data: ModelNodeData } = { data: { label: 'OpenAI', config: { provider: 'openai', model: 'gpt-5', temperature: 0.7, max_tokens: 1000 } }, } +it('refreshes untouched levels after a refetch and saves an edit with paired labels', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.max_tokens': 'Tokens' } } } + const props = { node: bound, nodeLabel: 'OpenAI', initialFieldPath: 'config.max_tokens', onClose: vi.fn(), onSave: save, onRemove: vi.fn() } + const factor = { name: 'Tokens', level_type: 'number' as const, levels: [2048, 1024], level_labels: ['high', 'low'] } + const refreshed = { ...factor, levels: [1024, 2048], level_labels: ['low', 'high'] } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 value')).toHaveValue(1024) + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'small' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['small', 'high'] }, 'config.max_tokens', 'Tokens')) +}) + it('routes New Factor model choices to the dedicated dialog while retaining their node grouping', () => { const fields = factorCreationFields([node], []) expect(fields).toHaveLength(5) diff --git a/frontend/src/components/protocol/ModelFactorDialog.tsx b/frontend/src/components/protocol/ModelFactorDialog.tsx index ad688f37..aff2c7aa 100644 --- a/frontend/src/components/protocol/ModelFactorDialog.tsx +++ b/frontend/src/components/protocol/ModelFactorDialog.tsx @@ -15,6 +15,7 @@ import { LlmConfigLevelRow } from './FactorEditorDialog' import { ModelField } from './ModelField' import { useProviderModels } from './useProviderModels' import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { useDialogDraft } from './useDialogDraft' export function ModelFactorDialog({ node, nodeLabel, factors, initialFieldPath, initialLevelIndex, onClose, onSave, onRemove }: { node: ProtocolNode @@ -49,7 +50,7 @@ export function ModelFactorDialog({ node, nodeLabel, factors, initialFieldPath, const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } } - const [draft, setDraft] = useState(() => seed(path)) + const { draft, setDraft, reset: resetDraft } = useDialogDraft(seed(path), { sessionKey: path }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const initialLevelRef = useRef(null) @@ -92,7 +93,7 @@ export function ModelFactorDialog({ node, nodeLabel, factors, initialFieldPath, initialLevelRef.current?.querySelector('input') ?? true}> Model factorChoose how models vary across experimental cells.
-
{visibleFields.map((field) => )}
+
{visibleFields.map((field) => )}
{blocked(path) ?

Model levels cannot be combined with individual configuration factors. Select the existing factor type and remove its binding first.

: <>

{structured ? 'Compare complete model configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this configuration field while keeping the other model settings.'}

{!available(path) &&

The selected model does not support this field. Remove its factor or choose a model that supports it.

} diff --git a/frontend/src/components/protocol/PatternFactorDialog.test.tsx b/frontend/src/components/protocol/PatternFactorDialog.test.tsx index be694abb..a26c8fca 100644 --- a/frontend/src/components/protocol/PatternFactorDialog.test.tsx +++ b/frontend/src/components/protocol/PatternFactorDialog.test.tsx @@ -12,6 +12,22 @@ const graph: ProtocolGraph = { nodes: [pattern, agent], edges: [{ id: 'edge', so const props = { patternNodeId: 'pattern', graph, nodeLabel: 'Writer:Reason + Act', factors: [], onClose: vi.fn(), onSave: vi.fn(), onRemove: vi.fn() } describe('Pattern factors', () => { + it('refreshes untouched levels after a refetch and preserves label pairing on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const boundGraph = { ...graph, nodes: [{ ...pattern, data: { ...pattern.data, factor_bindings: { 'config.max_iterations': 'Iterations' } } }, agent] } + const factor = { name: 'Iterations', level_type: 'number' as const, levels: [6, 3], level_labels: ['six', 'three'] } + const refreshed = { ...factor, levels: [3, 6], level_labels: ['three', 'six'] } + const next = { ...props, graph: boundGraph, initialFieldPath: 'config.max_iterations', onSave: save } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 value')).toHaveValue(3) + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'short' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['short', 'six'] }, 'config.max_iterations', 'pattern', 'Iterations')) + }) + it('groups whole-pattern and parameter choices under the Pattern node in New Factor', () => { const fields = factorCreationFields(graph.nodes, graph.edges) const whole = fields.find((field) => field.fieldPath === 'pattern_override')! diff --git a/frontend/src/components/protocol/PatternFactorDialog.tsx b/frontend/src/components/protocol/PatternFactorDialog.tsx index 6ab20589..647fd812 100644 --- a/frontend/src/components/protocol/PatternFactorDialog.tsx +++ b/frontend/src/components/protocol/PatternFactorDialog.tsx @@ -12,6 +12,7 @@ import { factorValueKey } from '@/lib/experiment' import { PatternLevelRow } from './FactorEditorDialog' import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' import { PATTERN_FACTOR_PATH, patternFactorFields, patternFactorConflict } from './patternFactors' +import { useDialogDraft } from './useDialogDraft' export function PatternFactorDialog({ patternNodeId, graph, nodeLabel, factors, initialFieldPath, onClose, onSave, onRemove }: { patternNodeId: string @@ -38,7 +39,7 @@ export function PatternFactorDialog({ patternNodeId, graph, nodeLabel, factors, const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } } - const [draft, setDraft] = useState(() => seed(path)) + const { draft, setDraft, reset: resetDraft } = useDialogDraft(seed(path), { sessionKey: path }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const current = existingFor(path) @@ -82,7 +83,7 @@ export function PatternFactorDialog({ patternNodeId, graph, nodeLabel, factors, Pattern factorChoose how patterns vary across experimental cells.
{agents.length > 0 &&

Applies to: {factorScopeLabel(graph, patternNodeId, 'pattern')}

} -
{fields.map((field) => )}
+
{fields.map((field) => )}
{blocked(path) ?

{blocked(path)}

: <>

{structured ? 'Compare complete pattern configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this parameter while keeping the other pattern settings.'}

setDraft({ ...draft, name: event.target.value })} />
diff --git a/frontend/src/components/protocol/ScriptFactorEditor.tsx b/frontend/src/components/protocol/ScriptFactorEditor.tsx index ad96c93d..b16a46b9 100644 --- a/frontend/src/components/protocol/ScriptFactorEditor.tsx +++ b/frontend/src/components/protocol/ScriptFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { ArrowDown, ArrowUp } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -16,7 +17,7 @@ export function ScriptFactorEditor({ graph, factor, open, onOpenChange, onSave, onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const labels = draft.level_labels ?? [] diff --git a/frontend/src/components/protocol/SkillFactorEditor.tsx b/frontend/src/components/protocol/SkillFactorEditor.tsx index afb725f7..b5f0fea1 100644 --- a/frontend/src/components/protocol/SkillFactorEditor.tsx +++ b/frontend/src/components/protocol/SkillFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { useQuery } from '@tanstack/react-query' import { ArrowDown, ArrowUp } from 'lucide-react' import { skillsApi } from '@/api/client' @@ -15,7 +16,7 @@ export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const skills = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list() }) diff --git a/frontend/src/components/protocol/SubAgentFactorEditor.tsx b/frontend/src/components/protocol/SubAgentFactorEditor.tsx index e49e4ea5..a8fb29bc 100644 --- a/frontend/src/components/protocol/SubAgentFactorEditor.tsx +++ b/frontend/src/components/protocol/SubAgentFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { ArrowDown, ArrowUp } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -16,7 +17,7 @@ export function SubAgentFactorEditor({ graph, factor, open, onOpenChange, onSave onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const labels = draft.level_labels ?? [] diff --git a/frontend/src/components/protocol/ToolFactorDialog.test.tsx b/frontend/src/components/protocol/ToolFactorDialog.test.tsx index 4867e558..6c262c98 100644 --- a/frontend/src/components/protocol/ToolFactorDialog.test.tsx +++ b/frontend/src/components/protocol/ToolFactorDialog.test.tsx @@ -13,6 +13,21 @@ const graph = { edges: [{ source: 'tool', target: 'agent', targetHandle: 'tool' }], } as unknown as ProtocolGraph +it('refreshes untouched levels after a refetch and preserves label pairing on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { toolNodeId: 'tool', graph, onClose: vi.fn(), onSave: save, onRemove: vi.fn() } + const factor = { name: 'Tool enabled', level_type: 'boolean' as const, levels: [true, false], level_labels: ['on', 'off'] } + const refreshed = { ...factor, levels: [false, true], level_labels: ['off', 'on'] } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('off') + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'disabled' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['disabled', 'on'] }, 'tool', 'Tool enabled')) +}) + it('keeps the dialog open after removal and permits choosing another factor type', async () => { const remove = vi.fn().mockResolvedValue(undefined) const close = vi.fn() diff --git a/frontend/src/components/protocol/ToolFactorDialog.tsx b/frontend/src/components/protocol/ToolFactorDialog.tsx index 08a05f0f..1949f6f7 100644 --- a/frontend/src/components/protocol/ToolFactorDialog.tsx +++ b/frontend/src/components/protocol/ToolFactorDialog.tsx @@ -1,5 +1,6 @@ import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { ArrowDown, ArrowUp, Split, Trash2 } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -44,10 +45,12 @@ export function ToolFactorDialog({ toolNodeId, graph, factors, initialMode, onCl } return isIndividualToolMode(nextMode) ? factor : reconcileToolFactor(factor, graph, ownerId) } - const [draft, setDraft] = useState(() => { - const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) - return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`) } - }) + const initialDraft = !initialMode && initial && (initial.level_type ?? 'boolean') === mode ? initial : seed(mode, agentId) + const { draft, setDraft, reset: resetDraft } = useDialogDraft({ + ...initialDraft, + level_type: initialDraft.level_type ?? 'boolean', + level_labels: initialDraft.level_labels ?? initialDraft.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + }, { sessionKey: mode }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const ownerId = isIndividualToolMode(mode) ? toolNodeId : agentId @@ -60,7 +63,7 @@ export function ToolFactorDialog({ toolNodeId, graph, factors, initialMode, onCl const labels = draft.level_labels ?? [] function change(nextMode: Mode, nextAgentId = agentId) { setMode(nextMode) - setDraft(seed(nextMode, nextAgentId)) + resetDraft(seed(nextMode, nextAgentId)) setError('') } function move(index: number, offset: number) { diff --git a/frontend/src/components/protocol/ToolFactorEditor.tsx b/frontend/src/components/protocol/ToolFactorEditor.tsx index 101cd070..8fed779e 100644 --- a/frontend/src/components/protocol/ToolFactorEditor.tsx +++ b/frontend/src/components/protocol/ToolFactorEditor.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' import { ArrowDown, ArrowUp } from 'lucide-react' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' @@ -16,7 +17,7 @@ export function ToolFactorEditor({ graph, factor, open, onOpenChange, onSave, on onSave: (factor: DesignFactor) => void | Promise onRemove?: () => Promise }) { - const [draft, setDraft] = useState(factor) + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') const labels = draft.level_labels ?? [] diff --git a/frontend/src/components/protocol/useDialogAutosave.test.tsx b/frontend/src/components/protocol/useDialogAutosave.test.tsx index cd3539e1..a588e6bd 100644 --- a/frontend/src/components/protocol/useDialogAutosave.test.tsx +++ b/frontend/src/components/protocol/useDialogAutosave.test.tsx @@ -67,4 +67,17 @@ describe('useDialogAutosave', () => { expect(result.current.status).toBe('idle') expect(save).not.toHaveBeenCalled() }) + + it('permits a fresh edit with a previously acknowledged signature', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const { result, rerender } = renderHook( + ({ draft }) => useDialogAutosave({ draft, signature: 'same', onSave: save }), + { initialProps: { draft: { name: 'Edited' } as { name: string } | null } }, + ) + await act(async () => result.current.flush()) + rerender({ draft: null }) + rerender({ draft: { name: 'Edited' } }) + await act(async () => result.current.flush()) + expect(save).toHaveBeenCalledTimes(2) + }) }) diff --git a/frontend/src/components/protocol/useDialogAutosave.ts b/frontend/src/components/protocol/useDialogAutosave.ts index 742eb942..5f23736c 100644 --- a/frontend/src/components/protocol/useDialogAutosave.ts +++ b/frontend/src/components/protocol/useDialogAutosave.ts @@ -79,6 +79,12 @@ export function useDialogAutosave({ useEffect(() => { if (!candidate) { + if (!draft) { + // Acknowledged, cancelled, or closed drafts end this edit. A later + // user edit may legitimately return to a previously saved value. + submittedSignatureRef.current = null + failedSignatureRef.current = null + } // A pending draft can become invalid or be moved into a nested editor // before its debounce fires. Do not leave the parent dialog claiming a // save is still queued after that timer has been cancelled. @@ -91,7 +97,7 @@ export function useDialogAutosave({ // The signature is the canonical identity of the draft. Depending on the // object itself would restart the timer on unrelated parent renders. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [delay, signature, submit, Boolean(candidate)]) + }, [delay, signature, submit, Boolean(candidate), Boolean(draft)]) useEffect(() => { return () => { diff --git a/frontend/src/components/protocol/useDialogDraft.ts b/frontend/src/components/protocol/useDialogDraft.ts new file mode 100644 index 00000000..bb13412f --- /dev/null +++ b/frontend/src/components/protocol/useDialogDraft.ts @@ -0,0 +1,37 @@ +import { useCallback, useState, type SetStateAction } from 'react' + +// Prop refreshes are not edits. Keep one atomic draft so levels and labels +// refresh together, and preserve user changes until the parent acknowledges +// that draft or a new editing session opens. +export function useDialogDraft(initial: T, { open = true, sessionKey = '', signatureOf = JSON.stringify }: { + open?: boolean + sessionKey?: string + signatureOf?: (draft: T) => string +} = {}) { + const signature = signatureOf(initial) + const [state, setState] = useState(() => ({ draft: initial, signature, open, sessionKey, dirty: false })) + let current = state + const newSession = (open && !state.open) || sessionKey !== state.sessionKey + const acknowledged = state.dirty && signatureOf(state.draft) === signature + // Reconcile before autosave sees the draft; an effect would expose stale + // state for one render after opening or receiving new props. + if (newSession || acknowledged || (!state.dirty && signature !== state.signature)) { + current = { draft: initial, signature, open, sessionKey, dirty: false } + setState(current) + } else if (open !== state.open || signature !== state.signature) { + current = { ...state, open, signature } + setState(current) + } + + const setDraft = useCallback((value: SetStateAction) => { + setState((previous) => ({ + ...previous, + draft: typeof value === 'function' ? (value as (previous: T) => T)(previous.draft) : value, + dirty: true, + })) + }, []) + const reset = useCallback((draft: T) => { + setState((previous) => ({ ...previous, draft, dirty: false })) + }, []) + return { draft: current.draft, setDraft, dirty: current.dirty, reset } +} From 3b0dd6815bea697e96b58f5c7c79dd652e3a1bc3 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 12 Oct 2026 16:01:22 -0700 Subject: [PATCH 32/63] test: cover Critic Gate bypass badge and inactive Agent on/off factor creation --- .../protocol/ProtocolCanvas.test.tsx | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index b4a71ded..90784a26 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -163,6 +163,24 @@ describe('ProtocolCanvas connector adds', () => { }) }) + it('shows the bypass marker on a disabled Critic Gate badge', async () => { + const run = protocolRun({ + status: 'completed', + node_runs: { gate: { status: 'completed', bypassed: true, approved: null, output_text: 'READY' } }, + }) + vi.spyOn(protocolsApi, 'listRuns').mockResolvedValue([run]) + vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) + renderCanvas({ + nodes: [{ id: 'gate', type: 'critic_gate', position: { x: 100, y: 100 }, data: defaultCriticGateNodeData('Marker reviewer') }], + edges: [], + }) + await waitFor(() => { + const node = document.querySelector('[data-testid="rf__node-gate"]') + expect(node).not.toBeNull() + expect(within(node!).getByText('Bypassed')).toBeInTheDocument() + }) + }) + it.each(['sub_agent_selection', 'sub_agent_toggle'] as const)('creates a %s Sub-Agent factor from the node hover menu and preserves connected nodes', async (mode) => { const experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, @@ -767,6 +785,36 @@ describe('ProtocolCanvas connector adds', () => { }) }) + it('keeps an inactive Agent inactive when an on/off factor is created for it', async () => { + const user = userEvent.setup() + const experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + Object.assign(experiment, patch) + return experiment + }) + const { client } = renderCanvas({ + nodes: [{ id: 'agent-1', type: 'agent', position: { x: 100, y: 100 }, data: { ...defaultAgentNodeData('Writer'), active: false } }], + edges: [], + }, 'experiment-1') + + await screen.findByText('Writer') + fireEvent.click(screen.getByTitle('Make experimental factor')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'This agent on/off' })) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ active: 'Writer:Enabled' }) + expect(persisted?.nodes[0].data.active).toBe(false) + }) + }) + it('keeps the Test Run label when a production replicate is running', async () => { const run = protocolRun() vi.mocked(protocolsApi.listRuns).mockResolvedValue([run]) From 63c98aef6165b7302873913a2131583ade6c5f36 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 12 Oct 2026 16:49:11 -0700 Subject: [PATCH 33/63] Paginate experiment results and load attempt evidence on demand --- frontend/src/api/client.ts | 30 +- .../protocol/DatasetExecutionSummary.tsx | 18 +- .../components/protocol/ExecutionBrowser.tsx | 130 +++ .../protocol/ExecutionInspector.tsx | 99 ++ .../protocol/ExperimentDefinitionDownload.tsx | 112 +++ .../protocol/ExperimentSidePanel.tsx | 1 + .../protocol/ExperimentVersionView.tsx | 15 +- .../components/protocol/ResultsTab.test.tsx | 445 ++------- .../src/components/protocol/ResultsTab.tsx | 515 +--------- .../src/components/protocol/RunsTab.test.tsx | 2 + frontend/src/components/protocol/RunsTab.tsx | 877 ++---------------- frontend/src/lib/executionQueries.ts | 23 + frontend/src/types/executionResults.ts | 64 ++ src/asaree/api/result_reads.py | 148 +++ src/asaree/app.py | 2 + src/asaree/migrations/env.py | 1 + .../3d4e5f607182_bounded_result_reads.py | 147 +++ src/asaree/models/protocol_run.py | 4 + src/asaree/models/result_read_model.py | 75 ++ src/asaree/services/result_reads.py | 624 +++++++++++++ src/asaree/services/result_rollups.py | 280 ++++++ src/asaree/worker/settings.py | 3 +- tests/test_result_reads.py | 254 +++++ 23 files changed, 2211 insertions(+), 1658 deletions(-) create mode 100644 frontend/src/components/protocol/ExecutionBrowser.tsx create mode 100644 frontend/src/components/protocol/ExecutionInspector.tsx create mode 100644 frontend/src/components/protocol/ExperimentDefinitionDownload.tsx create mode 100644 frontend/src/lib/executionQueries.ts create mode 100644 frontend/src/types/executionResults.ts create mode 100644 src/asaree/api/result_reads.py create mode 100644 src/asaree/migrations/versions/3d4e5f607182_bounded_result_reads.py create mode 100644 src/asaree/models/result_read_model.py create mode 100644 src/asaree/services/result_reads.py create mode 100644 src/asaree/services/result_rollups.py create mode 100644 tests/test_result_reads.py diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 9e08d0b1..3b7732e0 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -32,6 +32,7 @@ import type { OkfBundle, OkfDocument } from '@/types/okf' import type { CellRunBatch, DatasetRowSchema, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun, TestRun } from '@/types/protocols' import type { Run, RunStep } from '@/types/runs' import type { Skill, SkillListResponse, SkillUrlPreview } from '@/types/skills' +import type { AttemptDetail, AttemptPage, CursorPage, ExecutionCell, ExecutionSlot, ExecutionSummary } from '@/types/executionResults' const ACCESS_TOKEN_KEY = 'asaree_access_token' @@ -138,7 +139,7 @@ async function authedFetch(path: string, options: RequestOptions = {}): Promise< } } - if (!res.ok) { + if (!res.ok && res.status !== 304) { let detail: string | ApiErrorDetail = res.statusText try { const data = await res.json() @@ -195,6 +196,12 @@ function scopeQuery(scope?: ResultsScope): string { return params.size ? `?${params}` : '' } +function resultPageQuery(scope: ResultsScope, params: Record): string { + const query = new URLSearchParams(scopeQuery(scope).replace(/^\?/, '')) + for (const [key, value] of Object.entries(params)) if (value !== undefined) query.set(key, String(value)) + return query.size ? `?${query}` : '' +} + function rowSelection(options?: RowSelection): RowSelection | undefined { const index = options?.row_index if (index !== undefined && index !== null && (!Number.isInteger(index) || index < 0)) { @@ -204,6 +211,27 @@ function rowSelection(options?: RowSelection): RowSelection | undefined { } export const experimentsApi = { + getExecutionSummary: async (id: string, scope: ResultsScope, signal?: AbortSignal, cached?: ExecutionSummary): Promise => { + const res = await authedFetch(`/experiments/${id}/execution-results/summary${scopeQuery(scope)}`, { + signal, headers: cached?.etag ? { 'If-None-Match': cached.etag } : undefined, + }) + if (res.status === 304 && cached) return cached + return { ...await res.json() as ExecutionSummary, etag: res.headers.get('ETag') ?? undefined } + }, + listExecutionCells: (id: string, scope: ResultsScope, params: Record = {}, signal?: AbortSignal) => + request>(`/experiments/${id}/execution-results/cells${resultPageQuery(scope, params)}`, { signal }), + listExecutions: (id: string, scope: ResultsScope, params: Record = {}, signal?: AbortSignal) => + request>(`/experiments/${id}/execution-results/executions${resultPageQuery(scope, params)}`, { signal }), + listExecutionAttempts: (id: string, slotId: string, scope: ResultsScope, cursor?: string, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/slots/${slotId}/attempts${resultPageQuery(scope, { cursor })}`, { signal }), + lookupExecution: (id: string, replicateLabel: string, scope: ResultsScope, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/lookup${resultPageQuery(scope, { replicate_label: replicateLabel })}`, { signal }), + getExecutionCell: (id: string, scope: ResultsScope, selection: { cell_id?: string; cell_label?: string }, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/cell${resultPageQuery(scope, selection)}`, { signal }), + getExecutionAttempt: (id: string, runId: string, scope: ResultsScope, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/attempts/${runId}${scopeQuery(scope)}`, { signal }), + getExecutionConversation: (id: string, runId: string, scope: ResultsScope, offset = 0, signal?: AbortSignal) => + request<{ items: unknown[]; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/conversation${resultPageQuery(scope, { offset })}`, { signal }), list: (opts?: { includeArchived?: boolean }) => request(opts?.includeArchived ? '/experiments?include_archived=true' : '/experiments'), get: (id: string) => request(`/experiments/${id}`), diff --git a/frontend/src/components/protocol/DatasetExecutionSummary.tsx b/frontend/src/components/protocol/DatasetExecutionSummary.tsx index e91b7827..f60ec1d8 100644 --- a/frontend/src/components/protocol/DatasetExecutionSummary.tsx +++ b/frontend/src/components/protocol/DatasetExecutionSummary.tsx @@ -1,5 +1,6 @@ import { useQuery } from '@tanstack/react-query' -import { datasetsApi, experimentsApi, protocolsApi } from '@/api/client' +import { datasetsApi, protocolsApi } from '@/api/client' +import { useExecutionSummary } from '@/lib/executionQueries' import { Card } from '@/components/ui/card' import { datasetRowTopologyError, rowBindingForNode } from '@/lib/datasetRows' import type { Protocol, ProtocolGraph } from '@/types/protocols' @@ -24,15 +25,10 @@ export function DatasetExecutionSummary({ graph, protocol, experimentId, cellCou queryFn: () => protocolsApi.getRevision(protocol!.id, protocol!.published_revision_id!), enabled: !!protocol?.published_revision_id, }) - const results = useQuery({ - queryKey: ['experiments', experimentId, 'run-results', protocol?.id, protocol?.published_revision_id], - queryFn: () => experimentsApi.getRunResults(experimentId, { protocol_id: protocol!.id }), - enabled: !!protocol?.id && !!protocol.published_revision_id, - refetchInterval: 5000, - }) + const results = useExecutionSummary(experimentId, { protocol_id: protocol?.id }, !!protocol?.published_revision_id) const error = graph && datasetRowTopologyError(graph) const publishedBinding = publication.data && rowBindingForNode(publication.data.graph) - const summary = results.data?.row_summary + const summary = results.data const rows = source.data?.row_count const columnsMissing = binding?.input.mode === 'per_row' && source.data && binding.input.columns.some(column => !source.data.columns.includes(column)) @@ -49,9 +45,9 @@ export function DatasetExecutionSummary({ graph, protocol, experimentId, cellCou

Production runs · {protocol?.published_revision_id ? `experiment version ${protocol.published_revision}` : 'not published'}

- {!protocol?.published_revision_id ?

Publish the experiment before running.

: publication.isError || results.isError ?

Published execution forecast unavailable.

: !publication.data || !results.data ?

Loading published inputs…

: results.data.consumption_mode === 'per_row' ? <> -

Per row · {publishedBinding?.config?.dataset_name ?? publishedBinding?.config?.dataset_id ?? 'Dataset'} · {summary?.row_count ?? '?'} original rows

- {summary?.parent_replicate_count ?

{summary.cell_count} cells / {summary.parent_replicate_count} generated replicates × {summary.row_count ?? '?'} rows = {summary.expected ?? '?'} executions

:

No generated replicates. Generate cells before running a batch.

} + {!protocol?.published_revision_id ?

Publish the experiment before running.

: publication.isError || results.isError ?

Published execution forecast unavailable.

: !publication.data || !results.data ?

Loading published inputs…

: results.data.unit === 'row' ? <> +

Per row · {publishedBinding?.config?.dataset_name ?? publishedBinding?.config?.dataset_id ?? 'Dataset'} · {summary?.source_row_count ?? '?'} original rows

+ {summary?.replicate_count ?

{summary.cell_count} cells / {summary.replicate_count} generated replicates × {summary.source_row_count ?? '?'} rows = {summary.expected ?? '?'} executions

:

No generated replicates. Generate cells before running a batch.

}

Uses the design saved with this experiment version. Review the forecast in Runs before launching.

:

Whole dataset · one protocol execution per replicate.

}

Draft edits take effect after publishing; design changes require generation before running.

diff --git a/frontend/src/components/protocol/ExecutionBrowser.tsx b/frontend/src/components/protocol/ExecutionBrowser.tsx new file mode 100644 index 00000000..047ce402 --- /dev/null +++ b/frontend/src/components/protocol/ExecutionBrowser.tsx @@ -0,0 +1,130 @@ +import { useEffect, useRef, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { experimentsApi, type ResultsScope } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Badge } from '@/components/ui/badge' +import { Skeleton } from '@/components/ui/skeleton' +import { Card } from '@/components/ui/card' +import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' +import { displayFactorCondition, formatMetricLabel, formatMetricValue } from '@/lib/experiment' +import type { ExecutionSlot, ExecutionSummary } from '@/types/executionResults' +import type { Experiment } from '@/types/experiments' +import type { ResultsSelection } from './ResultsTab' +import { ResultsPanel } from './ResultsPanel' +import { CellCard, ReplicateRow } from './CellPresentation' + +export function CursorControls({ next, back, onNext, onBack, busy }: { + next: string | null | undefined; back: boolean; onNext: (cursor: string) => void; onBack: () => void; busy: boolean +}) { + return
+ + +
+} + +export function useCursor() { + const [cursors, setCursors] = useState<(string | undefined)[]>([undefined]) + return { cursor: cursors.at(-1), back: cursors.length > 1, + next: (cursor: string) => setCursors(current => [...current, cursor]), + previous: () => setCursors(current => current.slice(0, -1)), + reset: () => setCursors([undefined]) } +} + +function Usage({ usage }: { usage: ExecutionSummary['usage'] }) { + if (!usage) return null + return + {usage.cost_usd != null && `${new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD' }).format(usage.cost_usd)} · `} + {usage.total_tokens != null && `${usage.total_tokens.toLocaleString()} tokens`} + +} + +export function ExecutionRows({ experimentId, scope, cellId, generation, summary, onSelect, actions }: { + experimentId: string; scope: ResultsScope; cellId?: string; generation?: number; summary?: ExecutionSummary + onSelect: (selection: ResultsSelection) => void; actions?: (row: ExecutionSlot) => React.ReactNode +}) { + const pager = useCursor() + const query = useQuery({ + queryKey: [...executionKeys.root(experimentId), 'executions', scope, cellId, pager.cursor], + queryFn: ({ signal }) => experimentsApi.listExecutions(experimentId, scope, { cell_id: cellId, cursor: pager.cursor }, signal), + // Refresh only visible pages when the summary publishes a newer generation. + staleTime: 5000, + }) + useGenerationRefresh(query.refetch, generation) + return
+ {query.isPending ? : query.error && !query.data ?

{query.error.message}

: <> + {!query.data?.items.length &&

No executions planned in this scope.

} +
    {query.data?.items.map(row => onSelect({ type: 'execution', slotId: row.slot_id, scope }) : undefined} + detail={<>{row.row_index !== null && Source row {row.row_index + 1} · } + {row.result.observations?.map(observation => {observation.metric_name}: {observation.status === 'measured' ? observation.preview : observation.status}{observation.preview_truncated && '…'})}} + status={<>{row.status.replaceAll('_', ' ')}{row.result.truncated && Hit iteration limit}} + error={row.error} actions={actions?.(row)} />)}
+ {query.error && query.data &&

Refresh failed; showing the last loaded page.

} + + {summary?.unit === 'row' && !query.data?.items.length &&

The published design forecasts {summary.expected ?? '?'} row executions. Rows appear when the batch is planned.

} + } +
+} + +function useGenerationRefresh(refetch: () => unknown, generation?: number) { + const previous = useRef(generation) + useEffect(() => { + if (previous.current !== generation && previous.current !== undefined) void refetch() + previous.current = generation + }, [generation, refetch]) +} + +export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSelect, controls, actions }: { + experimentId: string; scope: ResultsScope; experiment?: Experiment; mode: 'runs' | 'results' + onSelect: (selection: ResultsSelection) => void; controls?: React.ReactNode + actions?: (row: ExecutionSlot) => React.ReactNode +}) { + const summary = useExecutionSummary(experimentId, scope) + const [expanded, setExpanded] = useState>(() => new Set()) + const [metricPreference, setMetricPreference] = useState() + const metric = (metricPreference ?? summary.data?.primary_metric) || undefined + const [directionPreference, setDirectionPreference] = useState() + const direction = directionPreference ?? summary.data?.metric_directions[metric ?? ''] ?? 'maximize' + const sortDirection = direction === 'minimize' ? 'minimize' : 'maximize' + const pager = useCursor() + const cells = useQuery({ + queryKey: [...executionKeys.root(experimentId), 'cells', scope, pager.cursor, metric, sortDirection], + queryFn: ({ signal }) => experimentsApi.listExecutionCells(experimentId, scope, { cursor: pager.cursor, metric, direction: sortDirection }, signal), + }) + useGenerationRefresh(cells.refetch, summary.data?.generation) + const metricKeys = summary.data?.unit === 'replicate' ? Object.keys(summary.data.metric_types) : [] + const refresh = () => { void summary.refetch(); void cells.refetch() } + const cellBody =
+ {summary.data?.unit === 'row' &&

Per-row outcomes are reported individually. Aggregation, ranking and factorial analysis are unavailable.

} + {mode === 'results' && metricKeys.length > 0 &&
+ + {metric && } +
} + {cells.isPending ? : cells.error && !cells.data ?

{cells.error.message}

: <> + {!cells.data?.items.length &&

No generated cells in this version.

} + {mode === 'results' && metric &&
{cells.data?.items.map((cell, index) => )}
{metric === summary.data?.primary_metric ? 'Rank' : '—'}Condition{formatMetricLabel(metric)}
{metric === summary.data?.primary_metric && cell.metric_values?.[metric] !== undefined && !pager.back ? index+1 : '—'}{cell.cell_label}{cell.metric_values?.[metric] !== undefined ? formatMetricValue(metric, cell.metric_values[metric]) : 'Unavailable'}
} + {cells.data?.items.map(cell => displayFactorCondition(name, value, summary.data?.design_spec ?? experiment?.design_spec, cell.cell_label))} + replicateCount={cell.replicate_count ?? 0} expanded={expanded.has(cell.cell_id)} listId={`executions-${cell.cell_id}`} + onToggle={() => setExpanded(current => { const next = new Set(current); if (next.has(cell.cell_id)) next.delete(cell.cell_id); else next.add(cell.cell_id); return next })} + progress={<>{cell.counts?.completed ?? 0}/{cell.total ?? '?'} complete{metric && cell.metric_values?.[metric] !== undefined && {formatMetricValue(metric, cell.metric_values[metric])}}} + badges={cell.truncated ? {cell.truncated} hit iteration limit : undefined} + actions={}> + {expanded.has(cell.cell_id) && } + )} + {cells.error && cells.data &&

Refresh failed; showing the last loaded cells.

} + + } +
+ const overview =

{mode === 'runs' ? 'Production runs' : 'Current results'}

{controls}
+ {summary.isPending ?

Loading progress…

: summary.error && !summary.data ?

Progress unavailable. Execution records remain available below.

: !summary.data?.ready ?

Preparing the version summary. Execution records remain available below.

: <> +

{summary.data.counts?.completed ?? 0}/{summary.data.expected ?? summary.data.total ?? 0} {summary.data.unit === 'row' ? 'row executions' : 'replicates'} complete · {summary.data.counts?.running ?? 0} running · {summary.data.counts?.pending ?? 0} queued · {summary.data.counts?.failed ?? 0} failed

+ +

Usage reported for {summary.data.usage_complete_count ?? 0}/{summary.data.total ?? 0} current executions.{summary.data.refreshing && ' Updating summary…'}

+ } + {summary.error && summary.data &&

Progress refresh failed; showing the last summary.

} +
+ return mode === 'results' && experiment + ? + :
{overview}{cellBody}
+} diff --git a/frontend/src/components/protocol/ExecutionInspector.tsx b/frontend/src/components/protocol/ExecutionInspector.tsx new file mode 100644 index 00000000..a24f25d1 --- /dev/null +++ b/frontend/src/components/protocol/ExecutionInspector.tsx @@ -0,0 +1,99 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { experimentsApi, type ResultsScope } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Skeleton } from '@/components/ui/skeleton' +import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' +import type { AttemptDetail, ExecutionSlot } from '@/types/executionResults' +import type { Experiment, ResultReplicate, ResultNodeRun } from '@/types/experiments' +import { ReplicateResultDetail, type ResultsSelection } from './ResultsTab' +import { CursorControls, useCursor } from './ExecutionBrowser' +import { ExecutionRows } from './ExecutionBrowser' + +export function attemptAsReplicate(slot: ExecutionSlot, detail: AttemptDetail): ResultReplicate { + const usage = detail.usage + const observations = detail.attempt_result?.measurement?.observations ?? [] + const metrics = { ...detail.attempt_result?.metric_values } + for (const observation of observations) if (observation.status === 'measured') metrics[observation.metric_name] = observation.value + const nodes: ResultNodeRun[] = Object.entries(detail.node_runs).filter(([, node]) => + !!node.run_id || !!node.output_text || !!node.error || ['running', 'failed', 'cancelled'].includes(node.status), + ).map(([id, node]) => ({ node_id: id, node_label: detail.node_labels[id] ?? id, status: node.status, + bypassed: node.bypassed, output_text: node.output_text ?? null, error: node.error ?? null, + agent_run_id: node.run_id ?? null, input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, + unresolved_reference_labels: (node.unresolved_references ?? []).map(ref => typeof ref === 'string' ? detail.node_labels[ref] ?? ref : String(ref)), + })) + return { replicate_label: slot.replicate_label, replicate_number: slot.replicate_number, cell_label: slot.cell_label, + factor_values: slot.factor_values, metric_values: metrics, status: detail.status === 'pending' ? 'queued' : detail.status as ResultReplicate['status'], + obsolete: false, error: detail.error, run_id: detail.run_id, protocol_revision_id: detail.protocol_revision_id, + updated_at: detail.completed_at ?? detail.started_at ?? '', node_runs: nodes, + duration_seconds: usage?.duration_seconds ?? null, cost_usd: usage?.cost_usd ?? null, + input_tokens: usage?.input_tokens ?? null, output_tokens: usage?.output_tokens ?? null, total_tokens: usage?.total_tokens ?? null, + agent_run_count: usage?.agent_run_count ?? 0, reported_usage_count: usage?.complete ? 1 : 0, reported_cost_count: usage?.cost_usd != null ? 1 : 0, + metric_evaluation: detail.attempt_result?.metric_evaluation, + metric_observations: observations, evaluation_artifacts: detail.attempt_result?.measurement?.artifacts ?? [], + obsolete_runs: [], superseded_runs: [] } +} + +function ConversationPage({ experimentId, runId, scope }: { experimentId: string; runId: string; scope: ResultsScope }) { + const [offsets, setOffsets] = useState([0]) + const offset = offsets.at(-1) ?? 0 + const query = useQuery({ queryKey: [...executionKeys.root(experimentId), 'conversation', scope, runId, offset], + queryFn: ({ signal }) => experimentsApi.getExecutionConversation(experimentId, runId, scope, offset, signal) }) + return

Conversation

+ {query.isPending ? : query.error ?

{query.error.message}

: query.data?.items.map((item, index) => 
{JSON.stringify(item, null, 2)}
)} +
+
+} + +function SlotInspector({ experimentId, slotId, scope, onClose }: { experimentId: string; slotId: string; scope: ResultsScope; onClose: () => void }) { + const [selected, setSelected] = useState() + const [conversationOpen, setConversationOpen] = useState(false) + const pager = useCursor() + const summary = useExecutionSummary(experimentId, scope) + const history = useQuery({ queryKey: [...executionKeys.root(experimentId), 'attempts', scope, slotId, pager.cursor], + queryFn: ({ signal }) => experimentsApi.listExecutionAttempts(experimentId, slotId, scope, pager.cursor, signal) }) + const runId = selected ?? history.data?.slot.run_id ?? history.data?.items[0]?.run_id + const detail = useQuery({ queryKey: [...executionKeys.root(experimentId), 'attempt', scope, runId], + queryFn: ({ signal }) => experimentsApi.getExecutionAttempt(experimentId, runId!, scope, signal), enabled: !!runId, + refetchInterval: query => query.state.data && ['pending','running','finalizing'].includes(query.state.data.status) ? 5000 : false }) + const slot = history.data?.slot + return +} + +export function ExecutionInspector({ experimentId, selection, onClose }: { experimentId: string; experiment: Experiment; selection: ResultsSelection | null; onClose: () => void }) { + const scope = selection?.scope ?? {} + const legacyLabel = selection?.type === 'replicate' ? selection.replicateLabel : undefined + const lookup = useQuery({ queryKey: [...executionKeys.root(experimentId), 'lookup', scope, legacyLabel], + queryFn: ({ signal }) => experimentsApi.lookupExecution(experimentId, legacyLabel!, scope, signal), enabled: !!legacyLabel }) + if (!selection) return null + if (selection.type === 'cell') return + const slotId = selection.type === 'execution' ? selection.slotId : selection.type === 'row' ? selection.rowResultId : lookup.data?.slot_id + if (slotId) return + return +} + +function CellInspector({ experimentId, selection, scope, onClose }: { experimentId: string; selection: Extract; scope: ResultsScope; onClose: () => void }) { + const [slot, setSlot] = useState() + const cell = useQuery({ queryKey: [...executionKeys.root(experimentId), 'cell', scope, selection.cellId ?? selection.cellLabel], + queryFn: ({ signal }) => experimentsApi.getExecutionCell(experimentId, scope, { cell_id: selection.cellId, cell_label: selection.cellLabel }, signal) }) + const summary = useExecutionSummary(experimentId, scope) + if (slot) return setSlot(undefined)} /> + return +} diff --git a/frontend/src/components/protocol/ExperimentDefinitionDownload.tsx b/frontend/src/components/protocol/ExperimentDefinitionDownload.tsx new file mode 100644 index 00000000..03f496c7 --- /dev/null +++ b/frontend/src/components/protocol/ExperimentDefinitionDownload.tsx @@ -0,0 +1,112 @@ +import { useState } from 'react' +import { Download } from 'lucide-react' +import { experimentsApi, protocolsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import type { Protocol } from '@/types/protocols' + +export function ExperimentDefinitionDownload({ experimentId, protocol }: { experimentId: string; protocol: Protocol }) { + const [downloadingDefinition, setDownloadingDefinition] = useState(false) + const [definitionDownloadError, setDefinitionDownloadError] = useState(null) + async function downloadDefinition() { + if (!protocol) return + setDownloadingDefinition(true) + setDefinitionDownloadError(null) + try { + // Read fresh rather than exporting whichever React Query snapshot + // happens to be on screen. A portable definition should describe one + // coherent point in time, including the currently published canvas. + const [experiment, canvas, designRevisions] = await Promise.all([ + experimentsApi.get(experimentId), + protocolsApi.get(protocol.id), + experimentsApi.listDesignRevisions(experimentId), + ]) + const revisionIds = [...new Set([canvas.published_revision_id, experiment.locked_protocol_revision_id].filter((id): id is string => !!id))] + const revisions = await Promise.all(revisionIds.map((revisionId) => protocolsApi.getRevision(canvas.id, revisionId))) + const revisionById = new Map(revisions.map((revision) => [revision.id, revision])) + const publishedCanvas = canvas.published_revision_id ? revisionById.get(canvas.published_revision_id) ?? null : null + const lockedCanvas = experiment.locked_protocol_revision_id ? revisionById.get(experiment.locked_protocol_revision_id) ?? null : null + const payload = { + format: 'asaree.experiment-definition', + schema_version: 1, + exported_at: new Date().toISOString(), + // The existing canvas importer consumes these two top-level fields. + // Keep them as the current draft while the richer envelope below + // carries immutable published/locked snapshots and provenance. + name: experiment.name, + description: canvas.description, + graph: canvas.graph, + design_spec: experiment.design_spec, + measurement_plan: experiment.measurement_plan, + experiment: { + source_id: experiment.id, + name: experiment.name, + description: experiment.description, + hypothesis: experiment.hypothesis, + design_type: experiment.design_type, + task_brief: experiment.task_brief, + design_spec: experiment.design_spec, + measurement_plan: experiment.measurement_plan, + dataset_ids: experiment.dataset_ids, + archived_at: experiment.archived_at, + created_at: experiment.created_at, + updated_at: experiment.updated_at, + lock: experiment.locked_at ? { + locked_at: experiment.locked_at, + source_protocol_revision_id: experiment.locked_protocol_revision_id, + design_spec: experiment.locked_design_spec, + measurement_plan: experiment.locked_measurement_plan, + canvas_revision: lockedCanvas ? { + source_id: lockedCanvas.id, + revision: lockedCanvas.revision, + published_at: lockedCanvas.published_at, + graph: lockedCanvas.graph, + } : null, + } : null, + }, + canvas: { + source_protocol_id: canvas.id, + name: canvas.name, + description: canvas.description, + draft: { graph: canvas.graph, updated_at: canvas.updated_at }, + published: publishedCanvas ? { + source_id: publishedCanvas.id, + revision: publishedCanvas.revision, + published_at: publishedCanvas.published_at, + graph: publishedCanvas.graph, + } : null, + has_unpublished_changes: canvas.has_unpublished_changes, + }, + design_revisions: designRevisions.map((revision) => ({ + source_id: revision.id, + revision: revision.revision, + superseded_at: revision.superseded_at, + design_spec: revision.design_spec, + cell_count: revision.cell_count, + replicate_count: revision.replicate_count, + scored_replicate_count: revision.scored_replicate_count, + created_at: revision.created_at, + })), + portability: { + excluded: ['provider credential secrets', 'replicate results and run history', 'dataset file contents', 'knowledge bundle/document contents', 'skill file contents'], + note: 'All canvas nodes and their configuration are included. Dataset, Knowledge, and Skill nodes retain their labels and resource IDs as references; recreate or map their external artifacts before importing into another workspace.', + }, + } + const name = experiment.name || canvas.name + const blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }) + const url = URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = `${name.trim().replace(/[^a-z0-9]+/gi, '-').replace(/(^-|-$)/g, '') || 'experiment'}.json` + link.click() + URL.revokeObjectURL(url) + } catch { + setDefinitionDownloadError('Could not prepare the experiment definition. Please try again.') + } finally { + setDownloadingDefinition(false) + } + } + + return <>{definitionDownloadError &&

{definitionDownloadError}

} +} diff --git a/frontend/src/components/protocol/ExperimentSidePanel.tsx b/frontend/src/components/protocol/ExperimentSidePanel.tsx index bc26c2e7..8b041e05 100644 --- a/frontend/src/components/protocol/ExperimentSidePanel.tsx +++ b/frontend/src/components/protocol/ExperimentSidePanel.tsx @@ -271,6 +271,7 @@ export function ExperimentSidePanel({ regenerationRequired={regenerationRequired} unboundFactors={unboundFactors} onViewRowResult={rowResultId => { onResultSelection({ type: 'row', rowResultId, scope: { protocol_id: protocol?.id } }); openPanel('results') }} + onSelectResult={selection => { onResultSelection(selection); openPanel('results') }} onViewResult={viewReplicateResult} /> ) : activeTab === 'results' ? ( diff --git a/frontend/src/components/protocol/ExperimentVersionView.tsx b/frontend/src/components/protocol/ExperimentVersionView.tsx index 22fc65c1..6a078895 100644 --- a/frontend/src/components/protocol/ExperimentVersionView.tsx +++ b/frontend/src/components/protocol/ExperimentVersionView.tsx @@ -1,16 +1,13 @@ import { useState } from 'react' -import { useQuery } from '@tanstack/react-query' import { Background, ReactFlow, type NodeTypes, type NodeProps } from '@xyflow/react' import { Card } from '@/components/ui/card' import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { NODE_TYPES } from './protocolNodeTypes' import { ProtocolCanvasActionsProvider } from './ProtocolCanvasContext' import type { ProtocolRevision } from '@/types/protocols' -import { experimentsApi } from '@/api/client' import { datasetRowBindings } from '@/lib/datasetRows' -import { Button } from '@/components/ui/button' import type { ResultsSelection } from './ResultsTab' -import { DatasetRowCells } from './DatasetRowCells' +import { ExecutionBrowser } from './ExecutionBrowser' // Historical graphs have no editing callbacks or autosave effects. Keep them // outside ProtocolCanvas so browsing can never write into the draft cache. @@ -50,14 +47,6 @@ export function ExperimentVersionDesign({ version }: { version: ProtocolRevision } export function ExperimentVersionRuns({ experimentId, protocolId, version, onSelectResult }: { experimentId: string; protocolId?: string; version: ProtocolRevision; onSelectResult: (selection: ResultsSelection) => void }) { - const [selectedCellId, setSelectedCellId] = useState('') - const [selectedReplicateId, setSelectedReplicateId] = useState('') const scope = { protocol_id: protocolId, protocol_revision_id: version.id } - const query = useQuery({ queryKey: ['experiments', experimentId, 'run-results', protocolId, version.id], queryFn: () => experimentsApi.getRunResults(experimentId, scope), refetchInterval: 5000 }) - if (query.isLoading) return

Loading version runs…

- if (!query.data) return

Could not load version runs.

- const rows = query.data.consumption_mode === 'per_row' - ? (query.data.row_results ?? []).filter(row => (!selectedCellId || row.cell_id === selectedCellId) && (!selectedReplicateId || row.replicate_result_id === selectedReplicateId)).map(row => ({ key: row.row_result_id, label: `${row.replicate_label} · row ${row.dataset_row.row_index + 1}`, status: row.status, runId: row.latest_attempt?.run_id, selection: { type: 'row' as const, rowResultId: row.row_result_id, scope } })) - : query.data.replicates.map(row => ({ key: row.replicate_label, label: row.replicate_label, status: row.status, runId: row.run_id, selection: { type: 'replicate' as const, replicateLabel: row.replicate_label, scope } })) - return

Runs for experiment version {version.revision}. Select the current experiment to launch new runs.

{query.data.consumption_mode === 'per_row' && { setSelectedCellId(cellId); setSelectedReplicateId('') }} onSelectReplicate={(cellId, replicateId) => { setSelectedCellId(cellId); setSelectedReplicateId(replicateId) }} />}{!rows.length &&

No runs planned for this version.

}
{rows.map(row => )}
ReplicateStatusDetails
{row.label}{row.status.replaceAll('_', ' ')}
+ return } diff --git a/frontend/src/components/protocol/ResultsTab.test.tsx b/frontend/src/components/protocol/ResultsTab.test.tsx index 1680284d..4d20a2c5 100644 --- a/frontend/src/components/protocol/ResultsTab.test.tsx +++ b/frontend/src/components/protocol/ResultsTab.test.tsx @@ -1,374 +1,127 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { fireEvent, render, screen, within } from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import { describe, expect, it, vi } from 'vitest' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { experimentsApi } from '@/api/client' -import type { Experiment, ExperimentRunResults } from '@/types/experiments' +import type { AttemptDetail, ExecutionCell, ExecutionSlot, ExecutionSummary } from '@/types/executionResults' +import type { Experiment } from '@/types/experiments' import { ResultsInspectorPanel, ResultsTab } from './ResultsTab' -vi.mock('@/api/client', async (importOriginal) => { +vi.mock('@/api/client', async importOriginal => { const actual = await importOriginal() - return { ...actual, experimentsApi: { ...actual.experimentsApi, getRunResults: vi.fn() } } + return { ...actual, experimentsApi: { ...actual.experimentsApi, + getExecutionSummary: vi.fn(), listExecutionCells: vi.fn(), listExecutions: vi.fn(), + listExecutionAttempts: vi.fn(), getExecutionAttempt: vi.fn(), getRunResults: vi.fn(), + } } }) - -function renderWithQuery(ui: React.ReactNode) { - const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) - return render({ui}) -} - -const overview = { - total_replicates: 1, - completed_replicates: 1, - running_replicates: 0, - queued_replicates: 0, - failed_replicates: 0, - not_started_replicates: 0, - obsolete_replicates: 0, - total_cost_usd: null, - total_input_tokens: null, - total_output_tokens: null, - total_tokens: null, - total_duration_seconds: null, - agent_run_count: 0, - reported_usage_count: 0, - reported_cost_count: 0, +vi.mock('./NodeRunOutputPanel', () => ({ ReceivedPromptPanel: () => null, RunStepTrace: () => null, UnresolvedReferencesNote: () => null })) + +const experiment = { id: 'experiment', name: 'Measurements', design_spec: { metrics: [] } } as unknown as Experiment +const summary: ExecutionSummary = { + scope_key: 'scope', unit: 'replicate', ready: true, protocol_id: 'protocol', protocol_revision_id: 'v1', design_revision_id: 'design', + generation: 1, change_id: 1, refreshing: false, updated_at: '', counts: { completed: 15786 }, total: 15786, + has_active: false, design_spec: null, metric_types: { duration_seconds: 'number' }, metric_aggregations: { duration_seconds: 'mean' }, + metric_directions: { duration_seconds: 'minimize' }, primary_metric: 'duration_seconds', primary_metric_direction: 'minimize', } - -const experiment = { - id: 'experiment-1', - name: 'Measurement UX', - design_spec: { metrics: [] }, -} as unknown as Experiment - -function orderedMetricsFixture() { - const orderedExperiment = { - ...experiment, - design_spec: { - metrics: [ - { id: 'clarity', name: 'Clarity', kind: 'custom', valueType: 'number', direction: 'maximize', primary: false }, - { id: 'safety', name: 'Safety', kind: 'custom', valueType: 'number', direction: 'maximize', primary: false }, - ], - }, - measurement_plan: { - metrics: [ - { id: 'safety', name: 'Safety', value_type: 'number', direction: 'maximize', aggregation: 'mean', primary: false }, - { id: 'clarity', name: 'Clarity', value_type: 'number', direction: 'maximize', aggregation: 'mean', primary: false }, - ], - producers: [], - inputs: [], - }, - } as unknown as Experiment - const results = { - overview, - metric_keys: ['Clarity', 'Safety'], - metric_types: { Clarity: 'number', Safety: 'number' }, - metric_aggregations: { Clarity: 'mean', Safety: 'mean' }, - metric_directions: { Clarity: 'maximize', Safety: 'maximize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 1, - completed_count: 1, current_completed_count: 1, obsolete_count: 0, - metric_means: { Clarity: 0.7, Safety: 0.9 }, metric_counts: { Clarity: 1, Safety: 1 }, - cost_usd: null, total_tokens: null, duration_seconds: null, - }], - replicates: [], - } satisfies ExperimentRunResults - return { orderedExperiment, results } +const cell: ExecutionCell = { cell_id: 'cell-1', cell_label: 'model=a', factor_values: { model: 'a' }, + replicate_count: 15786, total: 15786, counts: { completed: 15786 }, metric_values: { duration_seconds: 65 } } +const slot: ExecutionSlot = { slot_id: 'slot-1', cell_id: 'cell-1', cell_label: 'model=a', factor_values: { model: 'a' }, + replicate_id: 'parent-1', replicate_label: 'replicate-1', replicate_number: 1, row_result_id: null, row_index: null, + dataset_id: null, raw_sha256: null, run_id: 'run-1', status: 'completed', updated_at: '', error: null, result: {}, usage: null } +const detail: AttemptDetail = { run_id: 'run-1', status: 'completed', error: null, started_at: '', completed_at: '', + protocol_revision_id: 'v1', dataset_row: null, node_runs: {}, node_labels: {}, + usage: { cost_usd: 0.0123, total_tokens: 200, input_tokens: 100, output_tokens: 100, duration_seconds: 65, agent_run_count: 1 }, + attempt_result: { measurement: { observations: [], artifacts: [] } } } + +function mount(ui: React.ReactNode) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + return { ...render({ui}), client } } +beforeEach(() => { + vi.mocked(experimentsApi.getExecutionSummary).mockResolvedValue(summary) + vi.mocked(experimentsApi.listExecutionCells).mockResolvedValue({ items: [cell], next_cursor: null }) + vi.mocked(experimentsApi.listExecutions).mockResolvedValue({ items: [slot], next_cursor: null }) + vi.mocked(experimentsApi.listExecutionAttempts).mockResolvedValue({ slot, items: [{ run_id: 'run-1', status: 'completed', + created_at: '', completed_at: '', error: null, current: true, result: {}, usage: null }], next_cursor: null }) + vi.mocked(experimentsApi.getExecutionAttempt).mockResolvedValue(detail) +}) +afterEach(() => { vi.clearAllMocks(); vi.useRealTimers() }) + +describe('bounded Results reads', () => { + it('renders cells independently of summary preparation and fetches records only on expansion', async () => { + vi.mocked(experimentsApi.getExecutionSummary).mockResolvedValue({ ...summary, ready: false }) + mount() + expect(await screen.findByText(/Preparing the version summary/)).toBeInTheDocument() + expect(await screen.findByRole('button', { name: 'View results' })).toBeInTheDocument() + expect(experimentsApi.listExecutions).not.toHaveBeenCalled() + expect(experimentsApi.getRunResults).not.toHaveBeenCalled() + expect(experimentsApi.getExecutionAttempt).not.toHaveBeenCalled() + }) -describe('results measurement states', () => { - it('uses the shared maximize controls for whole-dataset results', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - renderWithQuery() + it('uses server metric ordering and preserves the shared maximize controls', async () => { + mount() fireEvent.click(await screen.findByRole('button', { name: 'Maximize Results' })) expect(document.body.style.overflow).toBe('hidden') - expect(screen.getByRole('button', { name: 'Restore Results' })).toBeInTheDocument() fireEvent.keyDown(document, { key: 'Escape' }) expect(document.body.style.overflow).not.toBe('hidden') expect(screen.getByRole('button', { name: 'Download CSV' })).toBeInTheDocument() + await screen.findByRole('combobox', { name: 'Choose comparison metric' }) + await waitFor(() => expect(experimentsApi.listExecutionCells).toHaveBeenCalledWith('experiment', expect.anything(), + expect.objectContaining({ metric: 'duration_seconds', direction: 'minimize' }), expect.any(AbortSignal))) }) - it('shows explicit units for cost and duration in an individual replicate result', async () => { - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: { cost_usd: 0.0123, duration_seconds: 65 }, status: 'completed' as const, obsolete: false, - error: null, run_id: 'run-1', protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', - duration_seconds: 65, node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: 0.0123, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [], evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - } - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['cost_usd', 'duration_seconds'], - metric_types: { cost_usd: 'number', duration_seconds: 'number' }, - metric_aggregations: { cost_usd: 'sum', duration_seconds: 'sum' }, - metric_directions: { cost_usd: 'minimize', duration_seconds: 'minimize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [], - replicates: [replicate], - } satisfies ExperimentRunResults) - - renderWithQuery() - - expect(await screen.findByText('$0.01')).toBeInTheDocument() - expect(screen.getByText('1.1 min')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Outcome' })).not.toBeInTheDocument() - const usage = screen.getByRole('heading', { name: 'Usage' }).parentElement! - expect(within(usage).getAllByText(/^(Estimated cost|Duration|Total tokens|Agent calls)$/).map((label) => label.textContent)).toEqual([ - 'Estimated cost', - 'Duration', - 'Total tokens', - 'Agent calls', - ]) - }) - - it('shows a measured observation once in Outcome and keeps its provenance there', async () => { - const producer = { binding_id: 'judge', producer_id: 'asaree.judge', kind: 'reported' as const, version: '1' } - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: { Quality: 0.9 }, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', - protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: null, - node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [{ metric_id: 'quality', metric_name: 'Quality', value_type: 'number' as const, status: 'measured' as const, value: 0.9, error: null, attempt_id: 'run-1', producer, input_provenance: {} }], - evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - } - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['Quality'], metric_types: { Quality: 'number' }, metric_aggregations: { Quality: 'mean' }, - metric_directions: { Quality: 'maximize' }, primary_metric: null, primary_metric_direction: null, - cells: [], replicates: [replicate], - } satisfies ExperimentRunResults) - - renderWithQuery() - - expect(await screen.findByRole('heading', { name: 'Outcome' })).toBeInTheDocument() - expect(screen.getAllByText('0.9')).toHaveLength(1) - expect(screen.getByText('asaree.judge · v1')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Measurement observations' })).not.toBeInTheDocument() + it('paginates execution records on the server', async () => { + vi.mocked(experimentsApi.listExecutions).mockImplementation(async (_id, _scope, params) => + params?.cursor ? { items: [{ ...slot, slot_id: 'slot-2', replicate_number: 2 }], next_cursor: null } + : { items: [slot], next_cursor: 'page-2' }) + mount() + fireEvent.click(await screen.findByRole('button', { name: /model: a/ })) + await screen.findByText('Replicate 1') + const next = await screen.findAllByRole('button', { name: 'Next' }) + fireEvent.click(next.find(button => !button.hasAttribute('disabled'))!) + expect(await screen.findByText('Replicate 2')).toBeInTheDocument() + expect(experimentsApi.listExecutions).toHaveBeenCalledWith('experiment', expect.anything(), + { cell_id: 'cell-1', cursor: 'page-2' }, expect.any(AbortSignal)) }) - it('presents metrics in the measurement-plan order declared by the designer', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByRole('combobox', { name: 'Choose comparison metric' })).toHaveTextContent('Safety') - expect(screen.getAllByText(/Safety/).length).toBeGreaterThan(0) - }) - - it('uses measurement-plan order in the separate cell result details', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - const summary = await screen.findByRole('region', { name: 'Cell result summary' }) - expect(within(summary).getAllByText(/^(Safety|Clarity) · average$/).map((label) => label.textContent)).toEqual([ - 'Safety · average', - 'Clarity · average', - ]) - }) - - it('shows condition-level cost and duration once under Usage', async () => { - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['cost_usd', 'duration_seconds'], - metric_types: { cost_usd: 'number', duration_seconds: 'number' }, - metric_aggregations: { cost_usd: 'sum', duration_seconds: 'sum' }, - metric_directions: { cost_usd: 'minimize', duration_seconds: 'minimize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 2, completed_count: 2, - current_completed_count: 2, obsolete_count: 0, metric_means: { cost_usd: 0.0123, duration_seconds: 65 }, - metric_counts: { cost_usd: 2, duration_seconds: 2 }, cost_usd: 0.0123, total_tokens: null, duration_seconds: 65, - }], - replicates: [], - } satisfies ExperimentRunResults) - - renderWithQuery() - + it('fetches the selected attempt independently of the experiment list', async () => { + mount() expect(await screen.findByText('$0.01')).toBeInTheDocument() expect(screen.getByText('1.1 min')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Outcome metrics' })).not.toBeInTheDocument() - }) - - it('ranks measured minimize values before missing values without skipping rank one', async () => { - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['loss'], - metric_types: { loss: 'number' }, - metric_aggregations: { loss: 'mean' }, - metric_directions: { loss: 'minimize' }, - primary_metric: 'loss', - primary_metric_direction: 'minimize', - cells: [ - { cell_label: 'missing', factor_values: { model: 'missing' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: {} as Record, metric_counts: {} as Record, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'measured', factor_values: { model: 'measured' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.2 }, metric_counts: { loss: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - const rows = await screen.findAllByRole('row') - expect(rows[1]).toHaveTextContent('model: measured') - expect(rows[1].firstElementChild).toHaveTextContent('1') - expect(rows[2]).toHaveTextContent('No measured observations') - }) - - it('summarizes unavailable condition observations instead of collapsing them into a blank', async () => { - const producer = { binding_id: 'python', producer_id: 'asaree.python_script', kind: 'reported' as const, version: '1' } - const results = { - overview, - metric_keys: ['ROC-AUC'], - metric_types: { 'ROC-AUC': 'number' }, - metric_aggregations: { 'ROC-AUC': 'mean' }, - metric_directions: { 'ROC-AUC': 'maximize' }, - primary_metric: 'ROC-AUC', - primary_metric_direction: 'maximize', - cells: [{ cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 2, completed_count: 2, current_completed_count: 2, obsolete_count: 0, metric_means: { 'ROC-AUC': 0.8 }, metric_counts: { 'ROC-AUC': 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }], - replicates: [{ - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, metric_values: {}, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: null, node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [{ metric_id: 'auc', metric_name: 'ROC-AUC', value_type: 'number' as const, status: 'unavailable' as const, value: null, error: 'Probability output is missing.', attempt_id: 'run-1', producer, input_provenance: {} }], - evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - }], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Unavailable (1)')).toBeInTheDocument() + expect(experimentsApi.getExecutionAttempt).toHaveBeenCalledWith('experiment', 'run-1', + { protocol_revision_id: 'v1' }, expect.any(AbortSignal)) + expect(experimentsApi.getRunResults).not.toHaveBeenCalled() + expect(experimentsApi.listExecutions).not.toHaveBeenCalled() }) - it('keeps a neutral scalar comparable without declaring a winner', async () => { - const results = { - overview, - metric_keys: ['variance'], - metric_types: { variance: 'number' }, - metric_aggregations: { variance: 'mean' }, - metric_directions: { variance: 'neutral' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 1, - completed_count: 1, current_completed_count: 1, obsolete_count: 0, - metric_means: { variance: 0.2 }, metric_counts: { variance: 1 }, - cost_usd: null, total_tokens: null, duration_seconds: null, - }], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/No primary metric is declared/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getAllByText('0.2').length).toBeGreaterThan(0) + it('preserves measured-null versus unavailable reported values and provenance', async () => { + vi.mocked(experimentsApi.getExecutionAttempt).mockResolvedValue({ ...detail, attempt_result: { measurement: { observations: [ + { metric_id: 'report', metric_name: 'Report', status: 'measured', value: null, error: null, attempt_id: 'run-1', + producer: { binding_id: 'reported', producer_id: 'asaree.agent_output', kind: 'reported', version: '1' }, input_provenance: {} }, + { metric_id: 'absent', metric_name: 'Absent', status: 'unavailable', value: null, error: 'Agent did not complete', attempt_id: 'run-1', + producer: { binding_id: 'reported', producer_id: 'asaree.agent_output', kind: 'reported', version: '1' }, input_provenance: {} }, + ], artifacts: [] } } }) + mount() + expect(await screen.findByText('Report')).toBeInTheDocument() + expect(screen.getByText('null')).toBeInTheDocument() + expect(screen.getByText('Agent did not complete')).toBeInTheDocument() + expect(screen.getAllByText('asaree.agent_output · v1')).toHaveLength(2) }) - it('compares directional metrics without ranking or naming a winner when no primary is declared', async () => { - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['quality'], - metric_types: { quality: 'number' }, - metric_aggregations: { quality: 'mean' }, - metric_directions: { quality: 'maximize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [ - { cell_label: 'b', factor_values: { model: 'b' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { quality: 0.9 }, metric_counts: { quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'a', factor_values: { model: 'a' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { quality: 0.2 }, metric_counts: { quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/No primary metric is declared/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getByRole('heading', { name: /Cell comparison/ })).toBeInTheDocument() - const rows = screen.getAllByRole('row') - expect(rows[1]).toHaveTextContent('model: a') - expect(rows[1].firstElementChild).toHaveTextContent('—') + it('stops polling settled experiments', async () => { + vi.useFakeTimers() + mount() + await act(async () => { await vi.advanceTimersByTimeAsync(100) }) + const calls = vi.mocked(experimentsApi.getExecutionSummary).mock.calls.length + await act(async () => { await vi.advanceTimersByTimeAsync(15000) }) + expect(experimentsApi.getExecutionSummary).toHaveBeenCalledTimes(calls) }) - it('does not rank an inspected metric that is not the declared primary', async () => { - const user = userEvent.setup() - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['loss', 'quality'], - metric_types: { loss: 'number', quality: 'number' }, - metric_aggregations: { loss: 'mean', quality: 'mean' }, - metric_directions: { loss: 'minimize', quality: 'maximize' }, - primary_metric: 'loss', - primary_metric_direction: 'minimize', - cells: [ - { cell_label: 'a', factor_values: { model: 'a' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.8, quality: 0.9 }, metric_counts: { loss: 1, quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'b', factor_values: { model: 'b' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.2, quality: 0.2 }, metric_counts: { loss: 1, quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - await screen.findByText('Best current result') - await user.click(screen.getByRole('combobox', { name: 'Choose comparison metric' })) - await user.click(screen.getByRole('option', { name: 'quality' })) - - expect(screen.getByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/quality is not the declared primary metric/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getByRole('heading', { name: /Cell comparison/ })).toBeInTheDocument() - }) - - it('renders unavailable, failed, and not-applicable observations plus diagnostic artifacts', async () => { - const producer = { binding_id: 'python', producer_id: 'asaree.python_script', kind: 'reported' as const, version: '1' } - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: {}, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', - protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: 1, - node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [ - { metric_id: 'auc', metric_name: 'ROC-AUC', value_type: 'number' as const, status: 'unavailable' as const, value: null, error: 'Probability output is missing.', attempt_id: 'run-1', producer, input_provenance: {} }, - { metric_id: 'loss', metric_name: 'Log loss', value_type: 'number' as const, status: 'failed' as const, value: null, error: 'Evaluator crashed.', attempt_id: 'run-1', producer, input_provenance: {} }, - { metric_id: 'mcc', metric_name: 'MCC', value_type: 'number' as const, status: 'not_applicable' as const, value: null, error: 'Regression task.', attempt_id: 'run-1', producer, input_provenance: {} }, - ], - evaluation_artifacts: [ - { artifact_key: 'confusion_matrix', kind: 'confusion_matrix', payload: { labels: ['no', 'yes'], matrix: [[8, 1], [2, 9]] }, attempt_id: 'run-1', producer, input_provenance: {} }, - { artifact_key: 'calibration', kind: 'calibration', payload: { probability: [0.2], observed: [0.1] }, attempt_id: 'run-1', producer, input_provenance: {} }, - { artifact_key: 'per_class', kind: 'per_class', payload: [{ label: 'yes', precision: 0.9, recall: 0.82, f1: 0.86, support: 11 }], attempt_id: 'run-1', producer, input_provenance: {} }, - ], - obsolete_runs: [], superseded_runs: [], - } - const results = { - overview, - metric_keys: [], metric_types: {}, metric_aggregations: {}, metric_directions: {}, - primary_metric: null, primary_metric_direction: null, - cells: [], replicates: [replicate], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Unavailable')).toBeInTheDocument() - expect(screen.getByText('Probability output is missing.')).toBeInTheDocument() - expect(screen.getByText('Failed')).toBeInTheDocument() - expect(screen.getByText('Evaluator crashed.')).toBeInTheDocument() - expect(screen.getByText('Not applicable')).toBeInTheDocument() - expect(screen.getByText('Regression task.')).toBeInTheDocument() - expect(screen.getByText('Confusion matrix')).toBeInTheDocument() - expect(screen.getByText('Calibration')).toBeInTheDocument() - expect(screen.getByText('Per-class report')).toBeInTheDocument() + it('aborts abandoned list requests', async () => { + vi.mocked(experimentsApi.listExecutionCells).mockImplementation(() => new Promise(() => {})) + const view = mount() + const signal = vi.mocked(experimentsApi.listExecutionCells).mock.calls[0][3]! + view.unmount() + expect(signal.aborted).toBe(true) }) }) diff --git a/frontend/src/components/protocol/ResultsTab.tsx b/frontend/src/components/protocol/ResultsTab.tsx index ad073497..2a1ce196 100644 --- a/frontend/src/components/protocol/ResultsTab.tsx +++ b/frontend/src/components/protocol/ResultsTab.tsx @@ -1,21 +1,14 @@ -import { useState } from 'react' -import { useQuery } from '@tanstack/react-query' -import { AlertTriangle, ChevronDown, ChevronRight, CircleDollarSign, Clock3, Coins, Cpu, Trophy, X } from 'lucide-react' -import { experimentsApi, type ResultsScope } from '@/api/client' +import { CircleDollarSign, Clock3, Coins, Cpu } from 'lucide-react' +import type { ResultsScope } from '@/api/client' import { Badge } from '@/components/ui/badge' -import { Button } from '@/components/ui/button' import { Card } from '@/components/ui/card' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { Skeleton } from '@/components/ui/skeleton' -import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' -import { displayFactorLevel, formatMetricLabel, formatMetricValue } from '@/lib/experiment' +import { formatMetricLabel, formatMetricValue } from '@/lib/experiment' import { OBSERVATION_LABELS } from '@/lib/measurementPlan' -import type { DesignMetric, EvaluationArtifact, Experiment, HistoricalRun, MetricObservation, ObsoleteRun, ResultCell, ResultReplicate, SupersededRun } from '@/types/experiments' -import { DatasetRowDetail, DatasetRowResults } from './DatasetRowResults' -import { ResultsPanel, ResultsScorecard as Scorecard } from './ResultsPanel' -import { InfoTooltip } from './InfoTooltip' -import { CellCard, ReplicateRow } from './CellPresentation' +import type { EvaluationArtifact, Experiment, MetricObservation, ResultReplicate } from '@/types/experiments' +import { ResultsScorecard as Scorecard } from './ResultsPanel' import { ResultTimelineNode } from './ResultTimelineNode' +import { ExecutionBrowser } from './ExecutionBrowser' +import { ExecutionInspector } from './ExecutionInspector' function formatNumber(value: number | null, maximumFractionDigits = 0): string { if (value === null || !Number.isFinite(value)) return 'Not reported' @@ -34,28 +27,9 @@ function formatDuration(value: number | null): string { return `${(value / 3600).toFixed(1)} hr` } -function factorSummary( - values: Record, - designSpec: Experiment['design_spec'], - cellLabel?: string | null, -): string { - const entries = Object.entries(values) - if (entries.length === 0) return 'No varying factors' - return entries.map(([name, value]) => `${name.split(':').join(' · ')}: ${displayFactorLevel(designSpec, name, value, cellLabel)}`).join(' · ') -} -function numericMetricValue(replicate: ResultReplicate, metricKey: string | null): number | null { - if (!metricKey) return null - const value = replicate.metric_values[metricKey] - return typeof value === 'number' && Number.isFinite(value) ? value : null -} type ResultMetricTypes = Record -type ResultMetricAggregations = Record - -function isBinaryMetric(metricKey: string | null, metricTypes: ResultMetricTypes): boolean { - return metricKey !== null && metricTypes[metricKey] === 'boolean' -} function formatResultMetricValue(metricKey: string, value: unknown, metricTypes: ResultMetricTypes, individual = false): string { if (typeof value !== 'number') return '—' @@ -66,19 +40,8 @@ function formatResultMetricValue(metricKey: string, value: unknown, metricTypes: return formatMetricValue(metricKey, value) } -function binaryMetricNote(metricKey: string, value: number, count: number | undefined, metricTypes: ResultMetricTypes): string | null { - if (metricTypes[metricKey] !== 'boolean' || !count) return null - return `${Math.round(value * count)}/${count} passed` -} -function metricAggregationLabel(metricKey: string, metricTypes: ResultMetricTypes, metricAggregations: ResultMetricAggregations): string { - if (metricTypes[metricKey] === 'boolean') return 'pass rate' - return metricAggregations[metricKey] === 'sum' ? 'total' : 'average' -} -function metricDisplayLabel(metricKey: string, metricTypes: ResultMetricTypes, metricAggregations: ResultMetricAggregations): string { - return `${formatMetricLabel(metricKey)} · ${metricAggregationLabel(metricKey, metricTypes, metricAggregations)}` -} function statusLabel(status: ResultReplicate['status'], obsolete: boolean): string { if (obsolete) return 'Obsolete' @@ -122,23 +85,6 @@ function observationForMetric(replicate: ResultReplicate, metricId: string | und ) } -function missingObservationSummary( - replicates: ResultReplicate[], - metricId: string | undefined, - metricKey: string | null, -): string | null { - const counts = new Map() - for (const replicate of replicates) { - if (replicate.obsolete) continue - const status = observationForMetric(replicate, metricId, metricKey)?.status - if (status && status !== 'measured') counts.set(status, (counts.get(status) ?? 0) + 1) - } - if (counts.size === 0) return null - return (['unavailable', 'failed', 'timed_out', 'cancelled', 'not_applicable'] as const) - .filter((status) => counts.has(status)) - .map((status) => `${OBSERVATION_LABELS[status]} (${counts.get(status)})`) - .join(' · ') -} function ObservationCard({ observation }: { observation: MetricObservation }) { const problem = observation.status === 'failed' || observation.status === 'timed_out' @@ -191,57 +137,15 @@ function ArtifactCard({ artifact }: { artifact: EvaluationArtifact }) { function usageSummarizesMetric( key: string, - result: Pick, + result: Pick, ): boolean { return (key === 'cost_usd' && result.cost_usd !== null) || (key === 'total_tokens' && result.total_tokens !== null) || (key === 'duration_seconds' && result.duration_seconds !== null) } -function CellResultSummary({ cell, metricKeys, metricTypes, metricAggregations }: { cell: ResultCell; metricKeys: string[]; metricTypes: ResultMetricTypes; metricAggregations: ResultMetricAggregations }) { - const metrics = metricKeys.filter((key) => typeof cell.metric_means[key] === 'number' && !usageSummarizesMetric(key, cell)) - const hasUsage = cell.cost_usd !== null || cell.total_tokens !== null || cell.duration_seconds !== null - return ( -
-
-
-

Condition summaryMetrics are averages over current, non-obsolete replicates in this condition.

-

Current replicates only

-
- - {cell.current_completed_count}/{cell.replicate_count} complete - -
- {metrics.length > 0 && ( -
-

Outcome metricsEach metric is labeled with its cell aggregation: average, total, or pass rate for Boolean outcomes.

-
- {metrics.map((key) => ( -
-

{metricDisplayLabel(key, metricTypes, metricAggregations)}

-

{formatResultMetricValue(key, cell.metric_means[key], metricTypes)}

- {binaryMetricNote(key, cell.metric_means[key], cell.metric_counts[key], metricTypes) &&

{binaryMetricNote(key, cell.metric_means[key], cell.metric_counts[key], metricTypes)}

} -
- ))} -
-
- )} - {hasUsage && ( -
0 ? 'mt-4 border-t pt-3' : 'mt-4'}`}> -

UsageTotals across current replicates in this condition. Provider telemetry can be unavailable for some calls.

-
- {cell.cost_usd !== null &&

Cost

{formatCurrency(cell.cost_usd)}

} - {cell.duration_seconds !== null &&

Duration

{formatDuration(cell.duration_seconds)}

} - {cell.total_tokens !== null &&

Tokens

{formatNumber(cell.total_tokens)}

} -
-
- )} - {metrics.length === 0 && !hasUsage &&

No current numeric results or provider usage have been reported yet.

} -
- ) -} -function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { +export function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { replicate: ResultReplicate metricKeys: string[] metricTypes: ResultMetricTypes @@ -311,401 +215,22 @@ function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { ) } -function historicalRunAsReplicate(replicate: ResultReplicate, historicalRun: HistoricalRun): ResultReplicate { - // The shared detail component needs the stable replicate identity/factors, - // while the historical run contributes the immutable execution facts. - return { - ...replicate, - ...historicalRun, - metric_values: historicalRun.metric_values ?? {}, - metric_evaluation: historicalRun.metric_evaluation, - obsolete_runs: [], - superseded_runs: [], - } -} - -function latestObsoleteRun(replicate: ResultReplicate | null): ObsoleteRun | null { - if (!replicate?.obsolete || !replicate.run_id) return null - return { - run_id: replicate.run_id, - metric_values: replicate.metric_values, - metric_evaluation: replicate.metric_evaluation, - metric_observations: replicate.metric_observations, - evaluation_artifacts: replicate.evaluation_artifacts, - legacy_values: replicate.legacy_values, - status: replicate.status, - obsolete: true, - error: replicate.error, - protocol_revision_id: replicate.protocol_revision_id, - updated_at: replicate.updated_at, - duration_seconds: replicate.duration_seconds, - node_runs: replicate.node_runs, - input_tokens: replicate.input_tokens, - output_tokens: replicate.output_tokens, - total_tokens: replicate.total_tokens, - cost_usd: replicate.cost_usd, - agent_run_count: replicate.agent_run_count, - reported_usage_count: replicate.reported_usage_count, - reported_cost_count: replicate.reported_cost_count, - } -} export type ResultsSelection = - | { type: 'cell'; cellLabel: string; scope?: ResultsScope } + | { type: 'cell'; cellLabel: string; cellId?: string; scope?: ResultsScope } | { type: 'replicate'; replicateLabel: string; scope?: ResultsScope } | { type: 'row'; rowResultId: string; scope: ResultsScope } + | { type: 'execution'; slotId: string; scope: ResultsScope } -function resultsMetricKey(metric: DesignMetric): string { - return metric.kind === 'runtime' && metric.catalogKey ? metric.catalogKey : metric.name -} - -function orderedResultsMetricKeys(metricKeys: string[], experiment: Experiment): string[] { - const available = new Set(metricKeys) - const designMetricsById = new Map((experiment.design_spec?.metrics ?? []).map((metric) => [metric.id, metric])) - const declared = (experiment.measurement_plan?.metrics ?? []).flatMap((definition) => { - const designMetric = designMetricsById.get(definition.id) - const key = designMetric ? resultsMetricKey(designMetric) : definition.name - return available.has(key) ? [key] : [] - }) - const declaredKeys = new Set(declared) - return [...declared, ...metricKeys.filter((key) => !declaredKeys.has(key))] -} +export function ResultsInspectorPanel(props: { + experimentId: string; experiment: Experiment; selection: ResultsSelection | null; onClose: () => void +}) { return } -// This lives alongside the canvas rather than inside the left Results panel. -// Keeping selection in the page lets the left panel remain usable while this -// inspector updates in place for every cell or replicate the user chooses. -export function ResultsInspectorPanel({ - experimentId, - experiment, - selection, - onClose, -}: { - experimentId: string - experiment: Experiment - selection: ResultsSelection | null - onClose: () => void +export function ResultsTab({ protocolId, experimentId, experiment, onSelectResult, versionId }: { + protocolId?: string; experimentId: string; experimentName: string; experiment: Experiment + onSelectResult: (selection: ResultsSelection) => void; versionId?: string }) { - const [expandedObsoleteRuns, setExpandedObsoleteRuns] = useState>(() => new Set()) - const resultsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'run-results', selection?.scope], - queryFn: () => experimentsApi.getRunResults(experimentId, selection?.scope), - enabled: selection !== null, - refetchInterval: 5000, - }) - if (!selection) return null - if (selection.type === 'row') { - const row = resultsQuery.data?.row_results?.find(item => item.row_result_id === selection.rowResultId) - if (!row) return - return - } - - const replicate = selection.type === 'replicate' - ? resultsQuery.data?.replicates.find((candidate) => candidate.replicate_label === selection.replicateLabel) ?? null - : null - const cell = selection.type === 'cell' - ? resultsQuery.data?.cells.find((candidate) => candidate.cell_label === selection.cellLabel) ?? null - : null - const title = replicate ? `Replicate ${replicate.replicate_number}` : 'Cell results' - const description = replicate - ? factorSummary(replicate.factor_values, experiment.design_spec, replicate.cell_label) - : cell - ? factorSummary(cell.factor_values, experiment.design_spec, cell.cell_label) - : undefined - // During a rolling frontend/backend restart, accept the prior response name - // too. The earlier API omitted the latest stale run from its history, so add - // it locally when necessary rather than hiding a real obsolete result. - const legacyObsoleteRuns = (replicate as (ResultReplicate & { previous_obsolete_runs?: ObsoleteRun[] }) | null) - ?.previous_obsolete_runs - const reportedObsoleteRuns = Array.isArray(replicate?.obsolete_runs) - ? replicate.obsolete_runs - : Array.isArray(legacyObsoleteRuns) ? legacyObsoleteRuns : [] - const latestStaleRun = latestObsoleteRun(replicate) - const obsoleteRuns = latestStaleRun && !reportedObsoleteRuns.some((run) => run.run_id === latestStaleRun.run_id) - ? [latestStaleRun, ...reportedObsoleteRuns] - : reportedObsoleteRuns - const supersededRuns: SupersededRun[] = Array.isArray(replicate?.superseded_runs) ? replicate.superseded_runs : [] - const metricKeys = resultsQuery.data ? orderedResultsMetricKeys(resultsQuery.data.metric_keys, experiment) : [] - - return ( - - ) -} - -export function ResultsTab({ - protocolId, - experimentId, - experimentName, - experiment, - onSelectResult, - versionId, -}: { - protocolId?: string - experimentId: string - experimentName: string - experiment: Experiment - onSelectResult: (selection: ResultsSelection) => void - versionId?: string -}) { - const [metricPreference, setMetricPreference] = useState(null) - const [expandedResultCells, setExpandedResultCells] = useState>(() => new Set()) - const scope: ResultsScope = { protocol_id: protocolId, protocol_revision_id: versionId || undefined } - const resultsQuery = useQuery({ queryKey: ['experiments', experimentId, 'run-results', scope], queryFn: () => experimentsApi.getRunResults(experimentId, scope), refetchInterval: 5000 }) - if (resultsQuery.isLoading) return
- if (resultsQuery.isError || !resultsQuery.data) return

{resultsQuery.error?.message ?? 'Could not load this experiment’s results.'}

- - if (resultsQuery.data.consumption_mode === 'per_row') return onSelectResult({ type: 'row', rowResultId, scope })} /> - - const { overview, cells, replicates, metric_types: metricTypes, metric_aggregations: metricAggregations, metric_directions: metricDirections, primary_metric: primaryMetric, primary_metric_direction: primaryMetricDirection } = resultsQuery.data - const metricKeys = orderedResultsMetricKeys(resultsQuery.data.metric_keys, experiment) - const metricKey = metricPreference && metricKeys.includes(metricPreference) - ? metricPreference - : primaryMetric && metricKeys.includes(primaryMetric) - ? primaryMetric - : metricKeys[0] ?? null - const selectedDirection = metricKey ? (metricDirections?.[metricKey] ?? 'neutral') : null - const selectedMetricId = metricKey ? experiment.design_spec?.metrics?.find((metric) => { - return resultsMetricKey(metric) === metricKey - })?.id : undefined - const rankable = metricKey === primaryMetric && (selectedDirection === 'maximize' || selectedDirection === 'minimize') - const sortDirection = selectedDirection === 'minimize' ? -1 : 1 - const sortedCells = [...cells].sort((left, right) => { - if (!metricKey || !rankable) return left.cell_label.localeCompare(right.cell_label) - const leftValue = left.metric_means[metricKey] - const rightValue = right.metric_means[metricKey] - const leftMeasured = typeof leftValue === 'number' - const rightMeasured = typeof rightValue === 'number' - if (leftMeasured && rightMeasured) return sortDirection * (rightValue - leftValue) - if (leftMeasured) return -1 - if (rightMeasured) return 1 - return left.cell_label.localeCompare(right.cell_label) - }) - const replicatesByCell = new Map() - for (const replicate of replicates) { - const cellReplicates = replicatesByCell.get(replicate.cell_label) ?? [] - cellReplicates.push(replicate) - replicatesByCell.set(replicate.cell_label, cellReplicates) - } - for (const cellReplicates of replicatesByCell.values()) cellReplicates.sort((left, right) => left.replicate_number - right.replicate_number) - if (overview.total_replicates === 0) return

Generate a design and run a replicate to see results here.

- const bestCell = metricKey && rankable ? sortedCells.find((cell) => typeof cell.metric_means[metricKey] === 'number') ?? null : null - const directionLabel = selectedDirection === 'minimize' ? 'Lowest' : 'Highest' - - return ( - 0 && ( -
- Inspect metric - -
- )} - summary={bestCell && metricKey ? ( -
-
Best current result
-

{formatResultMetricValue(metricKey, bestCell.metric_means[metricKey], metricTypes)}

- {binaryMetricNote(metricKey, bestCell.metric_means[metricKey], bestCell.metric_counts[metricKey], metricTypes) &&

{binaryMetricNote(metricKey, bestCell.metric_means[metricKey], bestCell.metric_counts[metricKey], metricTypes)}

} -

{directionLabel} {metricDisplayLabel(metricKey, metricTypes, metricAggregations)} · {factorSummary(bestCell.factor_values, experiment.design_spec)}

-
- ) : metricKey ? ( -
Comparison only

{formatMetricLabel(metricKey)}

{primaryMetric === null ? 'No primary metric is declared, so Results compares values without ranking cells or declaring a winner.' : metricKey !== primaryMetric ? `${formatMetricLabel(metricKey)} is not the declared primary metric, so Results compares its values without ranking cells or declaring a winner.` : `${formatMetricLabel(metricKey)} has no maximize/minimize direction, so Results compares values without declaring a winner.`}

- ) : ( -

Results are arriving

Complete a run with reported metrics to rank conditions here.

- )} - scorecards={<> - - - - - } - notice={overview.obsolete_replicates > 0 &&
{overview.obsolete_replicates} run{overview.obsolete_replicates === 1 ? '' : 's'} used an older canvas version and are excluded from current totals.
} - comparison={ -
-

{rankable ? 'Cell ranking' : 'Cell comparison'}{rankable ? 'Cells are ranked by the selected metric using current, non-obsolete replicates only.' : 'Metrics are shown for comparison without ordering cells or naming a winner.'}

Current replicates only{rankable && isBinaryMetric(metricKey, metricTypes) ? ' · ranked by pass rate' : rankable && metricKey === primaryMetric ? ` · ${primaryMetricDirection === 'minimize' ? 'lower is better' : 'higher is better'}` : ''}

- {metricKeys.length === 0 ?

No scalar metrics are declared or reported yet. Run details and provider usage are still available below.

:
{sortedCells.map((cell, index) => { const value = metricKey ? cell.metric_means[metricKey] : undefined; const measured = typeof value === 'number'; const ranked = rankable && measured; const note = measured && metricKey ? binaryMetricNote(metricKey, value, cell.metric_counts[metricKey], metricTypes) : null; const missingSummary = missingObservationSummary(replicatesByCell.get(cell.cell_label) ?? [], selectedMetricId, metricKey); return })}
{rankable ? 'Rank' : '—'}{rankable && Rank among conditions with a measured value for the selected metric.}ConditionThe factor levels used for this group of replicates.{metricKey ? metricDisplayLabel(metricKey, metricTypes, metricAggregations) : ''}CostDurationRunsCompleted current replicates out of all generated replicates for this condition.
{ranked ? index + 1 : '—'}

{factorSummary(cell.factor_values, experiment.design_spec)}

{cell.obsolete_count > 0 && {cell.obsolete_count} obsolete}		{measured ? <>{formatResultMetricValue(metricKey!, value, metricTypes)}{note && {note}}{missingSummary && {missingSummary}} : {missingSummary ?? 'No measured observations'}}{formatCurrency(cell.cost_usd)}{formatDuration(cell.duration_seconds)}{cell.current_completed_count}/{cell.replicate_count}
} -
- } - cells={ -
- {sortedCells.map((cell) => { - const expanded = expandedResultCells.has(cell.cell_label) - const cellReplicates = replicatesByCell.get(cell.cell_label) ?? [] - const replicateListId = `result-cell-${cell.cell_label}-replicates` - const metric = metricKey ? cell.metric_means[metricKey] : undefined - const currentMetricValues = cellReplicates - .filter((replicate) => !replicate.obsolete) - .map((replicate) => numericMetricValue(replicate, metricKey)) - .filter((value): value is number => value !== null) - const replicateMetricTotal = currentMetricValues.length > 0 - ? currentMetricValues.reduce((sum, value) => sum + value, 0) - : null - const replicateMetricAverage = replicateMetricTotal !== null ? replicateMetricTotal / currentMetricValues.length : null - const replicateMetricDeviation = replicateMetricAverage !== null && currentMetricValues.length > 1 - ? Math.sqrt(currentMetricValues.reduce((sum, value) => sum + (value - replicateMetricAverage) ** 2, 0) / currentMetricValues.length) - : null - const orderedReplicates = [...cellReplicates].sort((left, right) => { - const leftValue = left.obsolete ? null : numericMetricValue(left, metricKey) - const rightValue = right.obsolete ? null : numericMetricValue(right, metricKey) - if (leftValue !== null && rightValue !== null) return sortDirection * (rightValue - leftValue) - if (leftValue !== null) return -1 - if (rightValue !== null) return 1 - return left.replicate_number - right.replicate_number - }) - return ( - setExpandedResultCells(current => { - const next = new Set(current) - if (next.has(cell.cell_label)) next.delete(cell.cell_label) - else next.add(cell.cell_label) - return next - })} - progress={<>{cell.current_completed_count}/{cell.replicate_count} current runs complete} - badges={cell.obsolete_count > 0 && {cell.obsolete_count} obsolete {cell.obsolete_count === 1 ? 'run' : 'runs'}} - metric={metric !== undefined && {metricDisplayLabel(metricKey!, metricTypes, metricAggregations)}{formatResultMetricValue(metricKey!, metric, metricTypes)}{binaryMetricNote(metricKey!, metric, cell.metric_counts[metricKey!], metricTypes) && {binaryMetricNote(metricKey!, metric, cell.metric_counts[metricKey!], metricTypes)}}} - actions={}> - {cellReplicates.length === 0 ?

No replicate results are available for this cell.

: ( - <> - {metricKey && replicateMetricTotal !== null && ( -
-
-

Replicate comparison · {metricDisplayLabel(metricKey, metricTypes, metricAggregations)}

-

Current results only

-
- {isBinaryMetric(metricKey, metricTypes) ?

{Math.round((replicateMetricAverage ?? 0) * currentMetricValues.length)}/{currentMetricValues.length} passed · {formatResultMetricValue(metricKey, replicateMetricAverage, metricTypes)}

:

{metricAggregations[metricKey] === 'sum' ? <>Total {formatMetricValue(metricKey, replicateMetricTotal)} : <>Average {formatMetricValue(metricKey, replicateMetricAverage)}} · Range {formatMetricValue(metricKey, Math.min(...currentMetricValues))}–{formatMetricValue(metricKey, Math.max(...currentMetricValues))}{replicateMetricDeviation !== null && <> · SD {formatMetricValue(metricKey, replicateMetricDeviation)}}

} -
- )} -

Individual replicates

Select one for full output

-
    - {orderedReplicates.map((replicate) => { - const observation = observationForMetric(replicate, selectedMetricId, metricKey) - const value = numericMetricValue(replicate, metricKey) - return ( - onSelectResult({ type: 'replicate', replicateLabel: replicate.replicate_label })} - detail={<>{replicate.cost_usd !== null ? formatCurrency(replicate.cost_usd) : 'Cost not reported'}{replicate.duration_seconds !== null ? ` · ${formatDuration(replicate.duration_seconds)}` : ''}} - status={<>{value !== null && {formatResultMetricValue(metricKey!, value, metricTypes, true)}}{observation ? {OBSERVATION_LABELS[observation.status]} : evaluationStatusLabel(replicate) && {evaluationStatusLabel(replicate)}}{statusLabel(replicate.status, replicate.obsolete)}} - /> - )})} -
- - )} - - ) - })} -
- } - /> - ) + const scope = { protocol_id: protocolId, protocol_revision_id: versionId } + return } diff --git a/frontend/src/components/protocol/RunsTab.test.tsx b/frontend/src/components/protocol/RunsTab.test.tsx index cbd19ee5..f752a30f 100644 --- a/frontend/src/components/protocol/RunsTab.test.tsx +++ b/frontend/src/components/protocol/RunsTab.test.tsx @@ -10,6 +10,8 @@ describe('RunsTab factor discrepancy warning', () => { afterEach(() => vi.restoreAllMocks()) it('warns immediately and blocks runs when the canvas disagrees with generated factor levels', async () => { + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue({ ready: true, metric_types: {}, metric_directions: {} } as never) + vi.spyOn(experimentsApi, 'listExecutionCells').mockResolvedValue({ items: [], next_cursor: null }) vi.spyOn(experimentsApi, 'listReplicates').mockResolvedValue([{ id: 'replicate-1', cell_id: 'cell-1', cell_label: 'Prompt:original', replicate_label: 'Prompt:original__rep1', replicate_number: 1, design_revision_id: 'design-1', run_id: null, workspace_id: null, diff --git a/frontend/src/components/protocol/RunsTab.tsx b/frontend/src/components/protocol/RunsTab.tsx index ab81dfb2..dc70170d 100644 --- a/frontend/src/components/protocol/RunsTab.tsx +++ b/frontend/src/components/protocol/RunsTab.tsx @@ -1,531 +1,75 @@ import { useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' -import { AlertTriangle, ChevronDown, Download, Lock, Square } from 'lucide-react' -import { Badge } from '@/components/ui/badge' +import { AlertTriangle, Square } from 'lucide-react' +import { ApiError, experimentsApi, protocolsApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' -import { Skeleton } from '@/components/ui/skeleton' -import { ApiError, experimentsApi, protocolsApi } from '@/api/client' -import { displayFactorCondition, factorValueKey, groupReplicatesIntoCells, type ExperimentalCell } from '@/lib/experiment' import { factorBindingDiscrepancies, type FactorBindingDiscrepancy } from '@/lib/factorBindings' +import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' import { protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' -import type { Experiment, ResultCell, ResultReplicate, Trial } from '@/types/experiments' +import type { Experiment } from '@/types/experiments' import type { Protocol } from '@/types/protocols' -import { DatasetRowRuns } from './DatasetRowRuns' +import type { ExecutionSlot } from '@/types/executionResults' +import { ExecutionBrowser, ExecutionRows } from './ExecutionBrowser' +import { ExperimentDefinitionDownload } from './ExperimentDefinitionDownload' import { RunConfirmDialog } from './RunConfirmDialog' -import { CellCard, ReplicateRow, ReplicateRunAction, CellRunAction, ReplicateStatus } from './CellPresentation' -import { WarningBadge } from './nodes/WarningBadge' - -function factorEntries(cell: ExperimentalCell): [string, unknown][] { - // Factor ordering is meaningful when a user declared it, so preserve the - // object order here for the condition summary. Sorting the cells themselves - // below still gives a stable list no matter which replicate arrived first. - return Object.entries(cell.factorValues) -} - -function cellSortKey(cell: ExperimentalCell): string { - return factorEntries(cell) - .map(([name, value]) => `${name}:${factorValueKey(value)}`) - .join('|') -} - -function isActiveTrialStatus(status: Trial['status']): boolean { - return status === 'queued' || status === 'running' || status === 'finalizing' -} - -const OBSOLETE_TRIAL_BADGE = { - label: 'Obsolete', - className: 'border-transparent bg-[color:var(--chart-2)]/10 text-[color:var(--chart-2)]', -} - -// Ranked above the plain status badge for the same reason Obsolete is: the -// row's status really is "completed", and that is exactly the misreading -// worth preventing. Amber, the app's "finished, with a caveat" color. -const TRUNCATED_TRIAL_BADGE = { - label: 'Hit iteration limit', - className: 'border-transparent bg-[color:var(--chart-4)]/10 text-[color:var(--chart-4)]', -} +import { ReplicateRunAction } from './CellPresentation' +import type { ResultsSelection } from './ResultsTab' -const TRUNCATED_REPLICATE_HELP = - 'An agent in this replicate was stopped by its iteration limit, so it finished without finishing its work. It is not counted as scored. Raise Max iterations on the Reason + Act node and run it again.' - -function formatCurrency(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - return new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) -} - -function formatNumber(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - return new Intl.NumberFormat(undefined, { maximumFractionDigits: 0 }).format(value) -} - -function formatDuration(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - if (value < 60) return `${Math.round(value)} sec` - if (value < 3600) return `${(value / 60).toFixed(1)} min` - return `${(value / 3600).toFixed(1)} hr` -} - -function usageSummary(result: Pick | Pick): string[] { - const tokens = formatNumber(result.total_tokens) - return [ - formatCurrency(result.cost_usd), - tokens ? `${tokens} tokens` : null, - formatDuration(result.duration_seconds), - ].filter((value): value is string => !!value) -} - -// The panel's run control resumes only replicates without a current completed -// result. A completed run is meaningful even when it deliberately produced no -// score metrics, so it stays out of a later batch unless the user explicitly -// elects to re-run it (and therefore re-bill it). Obsolete results are current -// work again because they belong to an older canvas version. -export function RunAllCellsButton({ - protocol, - experimentId, - regenerationRequired, - unboundFactors, - bindingDiscrepancies, - replicateLabels, - label = 'Run all cells', - dialogTitle, - compact = false, - hasCompletedRun = false, -}: { - protocol: Protocol | undefined - experimentId: string - regenerationRequired: boolean - unboundFactors: string[] - bindingDiscrepancies: FactorBindingDiscrepancy[] - replicateLabels?: string[] - label?: string - dialogTitle?: string - compact?: boolean - hasCompletedRun?: boolean +export function RunAllCellsButton({ protocol, experimentId, regenerationRequired, unboundFactors, bindingDiscrepancies, label = 'Run all cells', dialogTitle, compact = false, replicateLabels }: { + protocol: Protocol | undefined; experimentId: string; regenerationRequired: boolean; unboundFactors: string[] + bindingDiscrepancies: FactorBindingDiscrepancy[]; label?: string; dialogTitle?: string; compact?: boolean + replicateLabels?: string[]; hasCompletedRun?: boolean }) { - const queryClient = useQueryClient() - const [dialogOpen, setDialogOpen] = useState(false) - const [selectedReruns, setSelectedReruns] = useState>(() => new Set()) - const [expandedCells, setExpandedCells] = useState>(() => new Set()) - const [lockBeforeRun, setLockBeforeRun] = useState(false) - const [downloadingDefinition, setDownloadingDefinition] = useState(false) - const [definitionDownloadError, setDefinitionDownloadError] = useState(null) - const experimentQuery = useQuery({ - queryKey: ['experiments', experimentId], - queryFn: () => experimentsApi.get(experimentId), - }) - const trialsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'runs'], - queryFn: () => experimentsApi.listTrials(experimentId), - refetchInterval: dialogOpen ? 3000 : false, - }) - const replicatesQuery = useQuery({ - queryKey: ['experiments', experimentId, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experimentId), - }) - const allReplicates = replicatesQuery.data ?? [] - const requestedLabels = replicateLabels ? new Set(replicateLabels) : null - const replicates = requestedLabels - ? allReplicates.filter((replicate) => requestedLabels.has(replicate.replicate_label)) - : allReplicates - const activeReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => isActiveTrialStatus(trial.status)) - .map((trial) => trial.replicate_label), - ) - const completedReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => trial.status === 'completed' && !trial.obsolete) - .map((trial) => trial.replicate_label), - ) - const obsoleteReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => trial.obsolete) - .map((trial) => trial.replicate_label), - ) - const eligibleReplicateLabels = replicates - .filter((replicate) => !activeReplicateLabels.has(replicate.replicate_label)) - .map((replicate) => replicate.replicate_label) - // Replicates in THIS button's scope whose agent ran out of iterations. The - // dialog says so before a re-run because nothing about the cap has changed - // in between: the same run at the same cap truncates again, and comes back - // unscored again, having cost the same money. Named by count rather than by - // node -- this button only ever has the stored graph and the trial list, so - // it can't tell which node's cap did it (the canvas can; see - // RunConfirmDialog's `truncatedCaps`). - const truncatedLabels = new Set( - (trialsQuery.data ?? []).filter((trial) => trial.truncated && !trial.obsolete).map((trial) => trial.replicate_label), - ) - const truncatedInScope = replicates.filter((replicate) => truncatedLabels.has(replicate.replicate_label)).length - const cellCount = groupReplicatesIntoCells(replicates).length - const previouslyRunReplicateLabels = new Set( - replicates - .filter( - (replicate) => - !obsoleteReplicateLabels.has(replicate.replicate_label) && - (!!replicate.metric_values || completedReplicateLabels.has(replicate.replicate_label)), - ) - .map((replicate) => replicate.replicate_label), - ) - const previouslyRunReplicates = replicates.filter((replicate) => previouslyRunReplicateLabels.has(replicate.replicate_label)) - const pendingReplicateCount = replicates.filter( - (replicate) => - !previouslyRunReplicateLabels.has(replicate.replicate_label) && !activeReplicateLabels.has(replicate.replicate_label), - ).length - const previouslyRunCells = groupReplicatesIntoCells( - previouslyRunReplicates.filter((replicate) => !activeReplicateLabels.has(replicate.replicate_label)), - ).sort((a, b) => - cellSortKey(a).localeCompare(cellSortKey(b)), - ) - - const runMutation = useMutation({ - mutationFn: () => - protocolsApi.runCells(protocol!.id, { - // Do not launch a second concurrent attempt for an in-flight - // replicate. The Stop controls let users finish that attempt first. - replicateLabels: eligibleReplicateLabels, - rerunReplicateLabels: [...selectedReruns], - }), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'replicates'] }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) - queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'runs'] }) - setDialogOpen(false) - }, - }) - const lockMutation = useMutation({ - mutationFn: () => experimentsApi.lock(experimentId), - onSuccess: (locked) => { - queryClient.setQueryData(['experiments', experimentId], locked) - queryClient.invalidateQueries({ queryKey: ['experiments'] }) - runMutation.mutate() - }, - }) - - function beginRun() { - if (lockBeforeRun && !experimentQuery.data?.locked_at) { - lockMutation.mutate() - return - } - runMutation.mutate() - } - - const publishAndRunMutation = useMutation({ + const client = useQueryClient() + const [open, setOpen] = useState(false) + const [lock, setLock] = useState(false) + const [reruns, setReruns] = useState>(() => new Set()) + const scope = { protocol_id: protocol?.id } + const summary = useExecutionSummary(experimentId, scope, open && !!protocol) + const experiment = useQuery({ queryKey: ['experiments', experimentId], queryFn: () => experimentsApi.get(experimentId), enabled: open }) + const run = useMutation({ mutationFn: async () => { - const live = queryClient.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol!.id)) - if (live) await protocolsApi.update(protocol!.id, { graph: toPersistedGraph(live.nodes, live.edges) }) - return protocolsApi.publish(protocol!.id) - }, - onSuccess: (published) => { - queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'revisions'] }) - queryClient.setQueryData(protocolForExperimentQueryKey(experimentId), published) - beginRun() + if (lock && !experiment.data?.locked_at) await experimentsApi.lock(experimentId) + return protocolsApi.runCells(protocol!.id, { replicateLabels, rerunReplicateLabels: [...reruns] }) }, + onSuccess: () => { setOpen(false); void client.invalidateQueries({ queryKey: ['experiments', experimentId] }) }, }) - - function setReplicatesSelected(replicateLabels: string[], selected: boolean) { - setSelectedReruns((current) => { - const next = new Set(current) - for (const label of replicateLabels) { - if (selected) next.add(label) - else next.delete(label) - } - return next - }) - } - - function toggleExpanded(cellLabel: string) { - setExpandedCells((current) => { - const next = new Set(current) - if (next.has(cellLabel)) next.delete(cellLabel) - else next.add(cellLabel) - return next - }) - } - - function openDialog() { - // Defaulting all previous work to unchecked makes the safe (existing) - // choice explicit: previous results are skipped unless chosen again. - setSelectedReruns(new Set()) - setExpandedCells(new Set()) - setLockBeforeRun(false) - setDialogOpen(true) - } - - async function downloadDefinition() { - if (!protocol) return - setDownloadingDefinition(true) - setDefinitionDownloadError(null) - try { - // Read fresh rather than exporting whichever React Query snapshot - // happens to be on screen. A portable definition should describe one - // coherent point in time, including the currently published canvas. - const [experiment, canvas, designRevisions] = await Promise.all([ - experimentsApi.get(experimentId), - protocolsApi.get(protocol.id), - experimentsApi.listDesignRevisions(experimentId), - ]) - const revisionIds = [...new Set([canvas.published_revision_id, experiment.locked_protocol_revision_id].filter((id): id is string => !!id))] - const revisions = await Promise.all(revisionIds.map((revisionId) => protocolsApi.getRevision(canvas.id, revisionId))) - const revisionById = new Map(revisions.map((revision) => [revision.id, revision])) - const publishedCanvas = canvas.published_revision_id ? revisionById.get(canvas.published_revision_id) ?? null : null - const lockedCanvas = experiment.locked_protocol_revision_id ? revisionById.get(experiment.locked_protocol_revision_id) ?? null : null - const payload = { - format: 'asaree.experiment-definition', - schema_version: 1, - exported_at: new Date().toISOString(), - // The existing canvas importer consumes these two top-level fields. - // Keep them as the current draft while the richer envelope below - // carries immutable published/locked snapshots and provenance. - name: experiment.name, - description: canvas.description, - graph: canvas.graph, - design_spec: experiment.design_spec, - measurement_plan: experiment.measurement_plan, - experiment: { - source_id: experiment.id, - name: experiment.name, - description: experiment.description, - hypothesis: experiment.hypothesis, - design_type: experiment.design_type, - task_brief: experiment.task_brief, - design_spec: experiment.design_spec, - measurement_plan: experiment.measurement_plan, - dataset_ids: experiment.dataset_ids, - archived_at: experiment.archived_at, - created_at: experiment.created_at, - updated_at: experiment.updated_at, - lock: experiment.locked_at ? { - locked_at: experiment.locked_at, - source_protocol_revision_id: experiment.locked_protocol_revision_id, - design_spec: experiment.locked_design_spec, - measurement_plan: experiment.locked_measurement_plan, - canvas_revision: lockedCanvas ? { - source_id: lockedCanvas.id, - revision: lockedCanvas.revision, - published_at: lockedCanvas.published_at, - graph: lockedCanvas.graph, - } : null, - } : null, - }, - canvas: { - source_protocol_id: canvas.id, - name: canvas.name, - description: canvas.description, - draft: { graph: canvas.graph, updated_at: canvas.updated_at }, - published: publishedCanvas ? { - source_id: publishedCanvas.id, - revision: publishedCanvas.revision, - published_at: publishedCanvas.published_at, - graph: publishedCanvas.graph, - } : null, - has_unpublished_changes: canvas.has_unpublished_changes, - }, - design_revisions: designRevisions.map((revision) => ({ - source_id: revision.id, - revision: revision.revision, - superseded_at: revision.superseded_at, - design_spec: revision.design_spec, - cell_count: revision.cell_count, - replicate_count: revision.replicate_count, - scored_replicate_count: revision.scored_replicate_count, - created_at: revision.created_at, - })), - portability: { - excluded: ['provider credential secrets', 'replicate results and run history', 'dataset file contents', 'knowledge bundle/document contents', 'skill file contents'], - note: 'All canvas nodes and their configuration are included. Dataset, Knowledge, and Skill nodes retain their labels and resource IDs as references; recreate or map their external artifacts before importing into another workspace.', - }, - } - const name = experiment.name || canvas.name - const blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }) - const url = URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `${name.trim().replace(/[^a-z0-9]+/gi, '-').replace(/(^-|-$)/g, '') || 'experiment'}.json` - link.click() - URL.revokeObjectURL(url) - } catch { - setDefinitionDownloadError('Could not prepare the experiment definition. Please try again.') - } finally { - setDownloadingDefinition(false) - } - } - - const runBlocked = - !protocol || - replicatesQuery.isLoading || - replicates.length === 0 || - (pendingReplicateCount === 0 && previouslyRunCells.length === 0) || - regenerationRequired || - unboundFactors.length > 0 || - bindingDiscrepancies.length > 0 - const blockedTitle = regenerationRequired - ? 'Design changed — review and regenerate before running the experiment.' - : unboundFactors.length > 0 - ? `Rebind or remove: ${unboundFactors.join(', ')}.` - : bindingDiscrepancies.length > 0 - ? 'Canvas values and generated factor levels disagree. Resolve the warning in Runs before starting.' - : replicates.length === 0 - ? 'Generate design first — there are no cells to run yet.' - : pendingReplicateCount === 0 && previouslyRunCells.length === 0 - ? 'All selected replicates are already running.' - : undefined - const runnableReplicateCount = pendingReplicateCount + selectedReruns.size - const actionLabel = label ?? (hasCompletedRun ? 'Re-run all cells' : 'Run all cells') - const errorMessage = lockMutation.error instanceof ApiError && typeof lockMutation.error.detail === 'string' - ? lockMutation.error.detail - : lockMutation.isError - ? 'Could not lock the experiment before starting it.' - : runMutation.error instanceof ApiError && typeof runMutation.error.detail === 'string' - ? runMutation.error.detail - : runMutation.isError - ? 'Could not start the experiment.' - : null - - return ( - <> - - {compact ? {actionLabel} : } - - {dialogOpen && protocol && ( - 0 - ? `${truncatedInScope} replicate${truncatedInScope === 1 ? '' : 's'} here stopped at the iteration limit and came back unscored. Unless you raise Max iterations on the Reason + Act node first, this run will hit the same limit.` - : null - } - onCancel={() => setDialogOpen(false)} - onConfirm={beginRun} - hasUnpublishedChanges={protocol.has_unpublished_changes} - publishedRevision={protocol.published_revision} - isPublishing={publishAndRunMutation.isPending} - publishError={ - publishAndRunMutation.error instanceof ApiError && typeof publishAndRunMutation.error.detail === 'string' - ? publishAndRunMutation.error.detail - : publishAndRunMutation.isError - ? 'Could not publish the latest canvas.' - : null - } - onPublishAndRun={() => publishAndRunMutation.mutate()} - confirmLabel={actionLabel} - confirmDisabled={runnableReplicateCount === 0} - isConfirming={runMutation.isPending || lockMutation.isPending} - confirmError={errorMessage} - additionalContent={ -
-
- {!experimentQuery.data?.locked_at && ( -
- setLockBeforeRun(checked === true)} - className="mt-0.5" - /> - -
- )} - - {definitionDownloadError &&

{definitionDownloadError}

} -
-
-
-

Previously run cells

-

- Completed replicates are skipped by default. Check a cell or replicate to run it again. -

-
- {previouslyRunCells.length === 0 ? ( -

No completed replicates to review.

- ) : ( -
- {previouslyRunCells.map((cell) => { - const labels = cell.replicates.map((replicate) => replicate.replicate_label) - const selectedCount = labels.filter((label) => selectedReruns.has(label)).length - const expanded = expandedCells.has(cell.label) - const summary = factorEntries(cell).map(([name, value]) => displayFactorCondition(name, value, experimentQuery.data?.design_spec, cell.label)).join(' · ') || 'Cell' - const listId = `rerun-cell-${cell.label}` - return ( -
-
- setReplicatesSelected(labels, checked === true)} - aria-label={`Run all replicates in ${summary} again`} - className="mt-0.5" - /> - -
- {expanded && ( -
    - {cell.replicates.map((replicate) => { - const scored = !!replicate.metric_values - return ( -
  • - setReplicatesSelected([replicate.replicate_label], checked === true)} - aria-label={`Run replicate ${replicate.replicate_number} again`} - /> - Replicate {replicate.replicate_number} - - {scored ? 'Scored' : 'Completed without metrics'} - -
    • - ) - })} -
- )} -
- ) - })} -
- )} - {runnableReplicateCount === 0 && ( -

Select at least one previously run replicate to run again.

- )} -
-
- } - /> - )} - - ) + const publish = useMutation({ mutationFn: async () => { + const live = client.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol!.id)) + if (live) await protocolsApi.update(protocol!.id, { graph: toPersistedGraph(live.nodes, live.edges) }) + return protocolsApi.publish(protocol!.id) + }, onSuccess: published => { + client.setQueryData(protocolForExperimentQueryKey(experimentId), published) + void client.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) + // A new publication changes the execution scope. Refresh its counts before confirming. + setReruns(new Set()) + } }) + const blocked = !protocol || regenerationRequired || unboundFactors.length > 0 || bindingDiscrepancies.length > 0 + const pending = (summary.data?.counts?.not_started ?? 0) + (summary.data?.counts?.failed ?? 0) + (summary.data?.counts?.cancelled ?? 0) + return <> + {open && protocol && setOpen(false)} onConfirm={() => run.mutate()} + hasUnpublishedChanges={protocol.has_unpublished_changes} publishedRevision={protocol.published_revision} + onPublishAndRun={() => publish.mutate()} isPublishing={publish.isPending} publishError={publish.error?.message} + confirmLabel={label} confirmDisabled={!summary.data?.ready || run.isPending || publish.isPending} + isConfirming={run.isPending} confirmError={run.error?.message} + additionalContent={
+ {!summary.data?.ready &&

Preparing execution counts…

} + {summary.error &&

{summary.error.message}

} + {summary.data?.unit === 'row' &&

{summary.data.expected ?? '?'} expected row executions across {summary.data.replicate_count ?? 0} replicates.

} + {!experiment.data?.locked_at && } + +

Completed replicates are skipped unless selected to run again.

+ {}} actions={row => row.status === 'completed' && setReruns(current => { const next = new Set(current); if (checked) next.add(row.replicate_label); else next.delete(row.replicate_label); return next })} />} /> +
} />} + } -// A protocol run is the actual cancellable unit. A cell/experiment stop is -// simply this same operation applied to each of its active replicate runs. -// Queued runs are cancelled by the API immediately; active runs are flagged -// for a safe interruption point. Polling keeps the visible status in sync. function StopRunsButton({ protocol, experimentId, @@ -577,6 +121,7 @@ function RunReplicateButton({ regenerationRequired, unboundFactors, bindingDiscrepancies, + row, }: { protocol: Protocol | undefined experimentId: string @@ -590,12 +135,19 @@ function RunReplicateButton({ regenerationRequired: boolean unboundFactors: string[] bindingDiscrepancies: FactorBindingDiscrepancy[] + row?: ExecutionSlot }) { const queryClient = useQueryClient() const [dialogOpen, setDialogOpen] = useState(false) const runMutation = useMutation({ - mutationFn: () => protocolsApi.run(protocol!.id, replicateLabel), + mutationFn: async () => { + if (row?.row_index != null) await protocolsApi.runCells(protocol!.id, row.row_result_id && ['failed', 'cancelled'].includes(row.status) + ? { retry_row_result_ids: [row.row_result_id] } + : { replicateLabels: [replicateLabel], row_indices: [row.row_index], rerunReplicateLabels: hasCompletedRun ? [replicateLabel] : [] }) + else await protocolsApi.run(protocol!.id, replicateLabel) + }, onSuccess: () => { + queryClient.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'replicates'] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'runs'] }) @@ -675,283 +227,26 @@ function RunReplicateButton({ ) } -// This view deliberately begins at the design's natural unit: a cell is one -// unique factor combination, regardless of how many independently-run -// replicates it contains. A cell expands in place to reveal those replicates, -// preserving the high-level overview instead of replacing it with a dialog. -export function RunsTab({ - experimentId, - designSpec, - protocol, - regenerationRequired, - unboundFactors, - onViewResult, - onViewRowResult, -}: { - experimentId: string - designSpec: Experiment['design_spec'] - protocol: Protocol | undefined - regenerationRequired: boolean - unboundFactors: string[] - onViewRowResult?: (rowId: string) => void - onViewResult: (replicateLabel: string) => void +export function RunsTab({ experimentId, designSpec, protocol, regenerationRequired, unboundFactors, onViewResult, onViewRowResult, onSelectResult }: { + experimentId: string; designSpec: Experiment['design_spec']; protocol: Protocol | undefined + regenerationRequired: boolean; unboundFactors: string[]; onViewResult: (label: string) => void + onViewRowResult?: (id: string) => void; onSelectResult?: (selection: ResultsSelection) => void }) { - const [expandedCells, setExpandedCells] = useState>(() => new Set()) - const replicatesQuery = useQuery({ - queryKey: ['experiments', experimentId, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experimentId), - }) - const trialsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'runs'], - queryFn: () => experimentsApi.listTrials(experimentId), - // Obsolescence is surfaced at the Cells and cell-header levels, not only - // inside an expanded replicate list, so load this while Runs is visible. - enabled: true, - refetchInterval: 3000, - }) - const resultsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'run-results', protocol?.id, protocol?.published_revision_id], - queryFn: () => experimentsApi.getRunResults(experimentId, { protocol_id: protocol?.id }), - refetchInterval: 5000, - }) - - if (protocol && resultsQuery.data?.consumption_mode === 'per_row') { - return 0 || factorBindingDiscrepancies(designSpec, protocol.graph).length > 0} onInspect={onViewRowResult} /> - } - - if (protocol && resultsQuery.isLoading) return - if (protocol && resultsQuery.isError) return

Could not load the published execution scope.

- - if (replicatesQuery.isLoading) { - return ( -
- - - -
- ) - } - - if (replicatesQuery.isError || !replicatesQuery.data) { - return

Could not load this experiment’s cells.

- } - - const cells = groupReplicatesIntoCells(replicatesQuery.data).sort((a, b) => cellSortKey(a).localeCompare(cellSortKey(b))) - const bindingDiscrepancies = factorBindingDiscrepancies(designSpec, protocol?.graph) - const trialsByLabel = new Map((trialsQuery.data ?? []).map((trial) => [trial.replicate_label, trial])) - const cellResultsByLabel = new Map((resultsQuery.data?.cells ?? []).map((cell) => [cell.cell_label, cell])) - const replicateResultsByLabel = new Map((resultsQuery.data?.replicates ?? []).map((replicate) => [replicate.replicate_label, replicate])) - const obsoleteCells = cells.filter((cell) => cell.replicates.some((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete)) - const obsoleteReplicateCount = cells.reduce( - (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete).length, - 0, - ) - const truncatedCells = cells.filter((cell) => cell.replicates.some((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated)) - const truncatedReplicateCount = cells.reduce( - (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length, - 0, - ) - // Runs stays operational rather than becoming a second Results dashboard: - // this one compact line answers whether there is work in flight, while - // comparison metrics, spend, and outputs stay in the Results rail item. - const trials = [...trialsByLabel.values()] - const currentTrialCount = trials.filter((trial) => !trial.obsolete).length - const completedCount = trials.filter((trial) => trial.status === 'completed' && !trial.obsolete).length - const runningCount = trials.filter((trial) => trial.status === 'running' || trial.status === 'finalizing').length - const queuedCount = trials.filter((trial) => trial.status === 'queued').length - const failedCount = trials.filter((trial) => trial.status === 'failed' || trial.status === 'cancelled').length - const overviewUsage = resultsQuery.data ? usageSummary({ - cost_usd: resultsQuery.data.overview.total_cost_usd, - total_tokens: resultsQuery.data.overview.total_tokens, - duration_seconds: resultsQuery.data.overview.total_duration_seconds, - }) : [] - const isActiveTrial = (trial: Trial | undefined): trial is Trial => - !!trial?.run_id && isActiveTrialStatus(trial.status) - const activeExperimentRunIds = trials.filter(isActiveTrial).map((trial) => trial.run_id) - // Completion is the user-visible truth here. A result can be completed - // from persisted metrics even when it has no ProtocolRun provenance (for - // example, imported or notebook-scored results), so run_id is not a valid - // test for whether this scope has already been run. - const hasFinishedRun = (trial: Trial | undefined) => - !!trial && trial.status !== 'not_started' && !isActiveTrial(trial) - const experimentHasCompletedRun = cells.length > 0 && cells.every((cell) => - cell.replicates.every((replicate) => hasFinishedRun(trialsByLabel.get(replicate.replicate_label))), - ) - - function toggleCell(cellLabel: string) { - setExpandedCells((current) => { - const next = new Set(current) - if (next.has(cellLabel)) next.delete(cellLabel) - else next.add(cellLabel) - return next - }) - } - - if (cells.length === 0) { - return

No cells yet — generate this experiment’s design first.

- } - - return ( -
- {bindingDiscrepancies.length > 0 && ( -
- -
-

Canvas and generated runs disagree

-

- Cell runs would replace these canvas values with saved factor levels. Update the factor levels and regenerate the design, or remove the bindings before running. -

-
    - {bindingDiscrepancies.map((issue) => ( -
  • - {issue.nodeLabel} · {issue.factorName}: {issue.reason} -
    • - ))} -
-
-
- )} -
-
-

Cells

- {obsoleteReplicateCount > 0 && ( - - )} - {truncatedReplicateCount > 0 && ( - - )} - - - {cells.length} -
-

- {completedCount}/{currentTrialCount || (trials.length === 0 ? replicatesQuery.data.length : 0)} current complete - {runningCount > 0 && ` · ${runningCount} running`} - {queuedCount > 0 && ` · ${queuedCount} queued`} - {failedCount > 0 && ` · ${failedCount} failed`} -

- {overviewUsage.length > 0 && ( -

- Current usage - {overviewUsage.map((value) => {value})} -

- )} -
-
- {cells.map((cell) => { - const entries = factorEntries(cell) - const expanded = expandedCells.has(cell.label) - const replicateListId = `cell-${cell.label}-replicates` - const obsoleteCount = cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete).length - const truncatedCount = cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length - const cellResult = cellResultsByLabel.get(cell.label) - const cellUsage = cellResult ? usageSummary(cellResult) : [] - const activeCellRunIds = cell.replicates - .map((replicate) => trialsByLabel.get(replicate.replicate_label)) - .filter(isActiveTrial) - .map((trial) => trial.run_id) - const cellHasCompletedRun = cell.replicates.every((replicate) => - hasFinishedRun(trialsByLabel.get(replicate.replicate_label)), - ) - return ( - displayFactorCondition(name, value, designSpec, cell.label))} - replicateCount={cell.replicates.length} expanded={expanded} listId={replicateListId} - onToggle={() => toggleCell(cell.label)} - progress={cellResult && <>{cellResult.current_completed_count}/{cellResult.replicate_count} current complete{cellUsage.map(value => {value})}} - badges={<> - {obsoleteCount > 0 && } - {truncatedCount > 0 && } - } - actions={<> - replicate.replicate_label)} - label={cellHasCompletedRun ? 'Re-run all replicates' : 'Run all replicates'} - dialogTitle={cellHasCompletedRun ? 'Re-run all replicates?' : 'Run all replicates?'} - compact - /> - - }> - {trialsQuery.isLoading ? ( -
- - -
- ) : ( -
    - {cell.replicates.map((replicate) => { - const trial = trialsByLabel.get(replicate.replicate_label) - const replicateResult = replicateResultsByLabel.get(replicate.replicate_label) - const replicateUsage = replicateResult ? usageSummary(replicateResult) : [] - const badge = trial - ? trial.obsolete - ? OBSOLETE_TRIAL_BADGE - : trial.truncated - ? TRUNCATED_TRIAL_BADGE - : null - : null - return ( - - {trial?.obsolete && ( - - )} - {trial?.truncated && !trial.obsolete && ( - - )} - } - detail={replicateUsage.length > 0 && replicateUsage.map(value => {value})} - status={badge ? {badge.label} : trial ? : Status unavailable} - onView={trial?.run_id ? () => onViewResult(replicate.replicate_label) : undefined} - actions={ - - } - error={trial?.error} - /> - ) - })} -
- )} - - ) - })} -
-
- ) + const discrepancies = factorBindingDiscrepancies(designSpec, protocol?.graph) + const scope = { protocol_id: protocol?.id } + return <> + {discrepancies.length > 0 &&

Canvas and generated runs disagree

{discrepancies.map(issue =>

{issue.nodeLabel} · {issue.factorName}: {issue.reason}

)}
} + { + if (onSelectResult) onSelectResult(selection) + else if (selection.type === 'row') onViewRowResult?.(selection.rowResultId) + else if (selection.type === 'replicate') onViewResult(selection.replicateLabel) + }} + controls={} + actions={row => } + /> + } diff --git a/frontend/src/lib/executionQueries.ts b/frontend/src/lib/executionQueries.ts new file mode 100644 index 00000000..c230c852 --- /dev/null +++ b/frontend/src/lib/executionQueries.ts @@ -0,0 +1,23 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { experimentsApi, type ResultsScope } from '@/api/client' +import type { ExecutionSummary } from '@/types/executionResults' + +export const executionKeys = { + root: (id: string) => ['experiments', id, 'execution-results'] as const, + summary: (id: string, scope: ResultsScope) => [...executionKeys.root(id), 'summary', scope] as const, +} + +export function useExecutionSummary(experimentId: string, scope: ResultsScope, enabled = true) { + const client = useQueryClient() + const queryKey = executionKeys.summary(experimentId, scope) + return useQuery({ + queryKey, + queryFn: ({ signal }) => experimentsApi.getExecutionSummary(experimentId, scope, signal, client.getQueryData(queryKey)), + enabled, + refetchInterval: query => { + const data = query.state.data + return !data || !data.ready || data.refreshing || data.has_active ? 5000 : false + }, + retry: 1, + }) +} diff --git a/frontend/src/types/executionResults.ts b/frontend/src/types/executionResults.ts new file mode 100644 index 00000000..1b611bbf --- /dev/null +++ b/frontend/src/types/executionResults.ts @@ -0,0 +1,64 @@ +import type { DesignSpec, EvaluationArtifact, MetricObservation, MeasurementDirection } from './experiments' +import type { DatasetRowSnapshot, NodeRunState } from './protocols' + +export interface CursorPage { items: T[]; next_cursor: string | null } +export interface UsageSummary { + cost_usd: number | null + input_tokens: number | null + output_tokens: number | null + total_tokens: number | null + duration_seconds: number | null + agent_run_count: number | null + complete?: boolean +} +export interface ObservationPreview { + metric_id: string; metric_name: string; status: string + preview: string; preview_truncated: boolean +} +export interface ExecutionPreview { + metrics?: Record + observations?: ObservationPreview[] + truncated?: boolean + evaluation_state?: string | null +} +export interface ExecutionSummary { + scope_key: string; unit: 'row' | 'replicate'; ready: boolean + protocol_id: string; protocol_revision_id: string | null; design_revision_id: string | null + generation: number; change_id: number; refreshing: boolean; updated_at: string | null; as_of?: string + counts?: Record; total?: number; cell_count?: number; replicate_count?: number + source_row_count?: number | null; expected?: number | null; has_active?: boolean + usage?: UsageSummary; usage_complete_count?: number; truncated_count?: number; evaluating_count?: number + design_spec: DesignSpec | null + metric_types: Record + metric_aggregations: Record + metric_directions: Record + primary_metric: string | null; primary_metric_direction: string | null + etag?: string +} +export interface ExecutionCell { + cell_id: string; cell_label: string; factor_values: Record + replicate_count?: number; total?: number; counts?: Record; truncated?: number + usage?: UsageSummary; metric_values?: Record; metric_counts?: Record +} +export interface ExecutionSlot { + slot_id: string; cell_id: string; cell_label: string; factor_values: Record + replicate_id: string; replicate_label: string; replicate_number: number; row_result_id: string | null + row_index: number | null; dataset_id: string | null; raw_sha256: string | null + run_id: string | null; status: string; updated_at: string | null; error: string | null + result: ExecutionPreview; usage: UsageSummary | null +} +export interface AttemptPreview { + run_id: string; status: string; created_at: string; completed_at: string | null + error: string | null; current: boolean; result: ExecutionPreview; usage: UsageSummary | null +} +export interface AttemptPage extends CursorPage { slot: ExecutionSlot } +export interface AttemptDetail { + run_id: string; status: string; error: string | null; started_at: string | null; completed_at: string | null + protocol_revision_id: string | null; dataset_row: DatasetRowSnapshot | null + node_runs: Record; node_labels: Record; usage: UsageSummary | null + attempt_result: { + metric_values?: Record + metric_evaluation?: { status: 'queued' | 'running' | 'completed' | 'failed' | 'skipped'; error?: string | null } + measurement?: { observations?: MetricObservation[]; artifacts?: EvaluationArtifact[] } + } | null +} diff --git a/src/asaree/api/result_reads.py b/src/asaree/api/result_reads.py new file mode 100644 index 00000000..4331d4fc --- /dev/null +++ b/src/asaree/api/result_reads.py @@ -0,0 +1,148 @@ +"""Version-scoped, bounded operational result reads.""" + +from __future__ import annotations + +import uuid +from collections.abc import Awaitable, Callable +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response +from sqlalchemy import text + +from asaree.api.experiments import _get_owned_experiment +from asaree.deps import CurrentUser, DbSession +from asaree.services import result_reads +from asaree.services.experiment_run_results import RunResultsProjectionError + +router = APIRouter(prefix="/experiments/{experiment_id}/execution-results", tags=["execution-results"]) +Limit = Annotated[int, Query(ge=1, le=result_reads.MAX_PAGE_SIZE)] + + +async def selected_scope( + experiment_id: uuid.UUID, + user: CurrentUser, + db: DbSession, + protocol_id: uuid.UUID | None = None, + protocol_revision_id: uuid.UUID | None = None, + design_revision_id: uuid.UUID | None = None, +) -> result_reads.ResultScope: + await _get_owned_experiment(db, experiment_id, user) + # Bound database work and release connections before an upstream gateway deadline. + await db.execute(text("SET LOCAL statement_timeout = '5s'")) + try: + return await result_reads.resolve_result_scope( + db, + experiment_id=experiment_id, + protocol_id=protocol_id, + protocol_revision_id=protocol_revision_id, + design_revision_id=design_revision_id, + ) + except RunResultsProjectionError as exc: + raise HTTPException(status_code=422 if str(exc) == "ambiguous_protocol" else 404, detail=str(exc)) from exc + + +Scope = Annotated[result_reads.ResultScope, Depends(selected_scope)] + + +async def _read(operation: Callable[..., Awaitable[dict[str, Any]]], *args: Any, **kwargs: Any) -> dict[str, Any]: + try: + return await operation(*args, **kwargs) + except RunResultsProjectionError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + +@router.get("/summary", response_model=None) +async def summary( + db: DbSession, + scope: Scope, + response: Response, + if_none_match: Annotated[str | None, Header()] = None, +) -> Any: + result = await result_reads.read_summary(db, scope) + etag = f'"{scope.key}:{result["change_id"]}:{result["generation"]}"' + response.headers["ETag"] = etag + response.headers["Cache-Control"] = "private, no-cache" + if if_none_match == etag: + return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "private, no-cache"}) + return result + + +@router.get("/cells") +async def cells( + db: DbSession, + scope: Scope, + limit: Limit = result_reads.PAGE_SIZE, + cursor: str | None = None, + metric: str | None = None, + direction: str = "maximize", +) -> dict[str, Any]: + return await _read( + result_reads.read_cells, db, scope, limit=limit, cursor=cursor, metric=metric, direction=direction + ) + + +@router.get("/executions") +async def executions( + db: DbSession, + scope: Scope, + limit: Limit = result_reads.PAGE_SIZE, + cursor: str | None = None, + cell_id: uuid.UUID | None = None, + replicate_id: uuid.UUID | None = None, + status: Annotated[ + str | None, Query(pattern="^(not_started|pending|running|finalizing|completed|failed|cancelled|limit_reached)$") + ] = None, +) -> dict[str, Any]: + return await _read( + result_reads.read_executions, + db, + scope, + limit=limit, + cursor=cursor, + cell_id=cell_id, + replicate_id=replicate_id, + status=status, + ) + + +@router.get("/slots/{slot_id}/attempts") +async def attempts( + slot_id: uuid.UUID, + db: DbSession, + scope: Scope, + limit: Limit = result_reads.PAGE_SIZE, + cursor: str | None = None, +) -> dict[str, Any]: + return await _read(result_reads.read_attempts, db, scope, slot_id=slot_id, limit=limit, cursor=cursor) + + +@router.get("/lookup") +async def lookup(db: DbSession, scope: Scope, replicate_label: str) -> dict[str, Any]: + return await _read(result_reads.lookup_slot, db, scope, replicate_label) + + +@router.get("/cell") +async def cell( + db: DbSession, scope: Scope, cell_id: uuid.UUID | None = None, cell_label: str | None = None +) -> dict[str, Any]: + if cell_id is None and cell_label is None: + raise HTTPException(status_code=422, detail="cell_selection_required") + return await _read(result_reads.read_cell, db, scope, cell_id, cell_label) + + +@router.get("/attempts/{run_id}") +async def attempt(run_id: uuid.UUID, db: DbSession, scope: Scope) -> dict[str, Any]: + return await _read(result_reads.read_attempt_detail, db, scope, run_id) + + +@router.get("/attempts/{run_id}/conversation") +async def conversation( + run_id: uuid.UUID, + db: DbSession, + scope: Scope, + limit: Limit = result_reads.PAGE_SIZE, + offset: Annotated[int, Query(ge=0)] = 0, +) -> dict[str, Any]: + return await _read(result_reads.read_conversation, db, scope, run_id, limit=limit, offset=offset) diff --git a/src/asaree/app.py b/src/asaree/app.py index 954a7b52..3b48bafd 100644 --- a/src/asaree/app.py +++ b/src/asaree/app.py @@ -26,6 +26,7 @@ from asaree.api.mcp_servers import router as mcp_servers_router from asaree.api.okf import router as okf_router from asaree.api.protocols import router as protocols_router +from asaree.api.result_reads import router as result_reads_router from asaree.api.runs import router as runs_router from asaree.api.skills import router as skills_router from asaree.api.users import router as users_router @@ -137,6 +138,7 @@ async def version() -> dict[str, str]: app.include_router(agents_router, prefix="/api") app.include_router(runs_router, prefix="/api") app.include_router(experiments_router, prefix="/api") + app.include_router(result_reads_router, prefix="/api") app.include_router(protocols_router, prefix="/api") return app diff --git a/src/asaree/migrations/env.py b/src/asaree/migrations/env.py index 7be61c2a..e176ecb8 100644 --- a/src/asaree/migrations/env.py +++ b/src/asaree/migrations/env.py @@ -31,6 +31,7 @@ import asaree.models.protocol # noqa: F401 import asaree.models.protocol_revision # noqa: F401 import asaree.models.protocol_run # noqa: F401 +import asaree.models.result_read_model # noqa: F401 import asaree.models.user # noqa: F401 import asaree.models.user_api_token # noqa: F401 import asaree.models.user_llm_setting # noqa: F401 diff --git a/src/asaree/migrations/versions/3d4e5f607182_bounded_result_reads.py b/src/asaree/migrations/versions/3d4e5f607182_bounded_result_reads.py new file mode 100644 index 00000000..98870d3b --- /dev/null +++ b/src/asaree/migrations/versions/3d4e5f607182_bounded_result_reads.py @@ -0,0 +1,147 @@ +"""Compact result previews, durable refresh events and paginated read indexes.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "3d4e5f607182" +down_revision = "2c3d4e5f6071" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("protocol_runs", sa.Column("result_summary", postgresql.JSONB(), nullable=False, server_default="{}")) + op.add_column("protocol_runs", sa.Column("usage_summary", postgresql.JSONB(), nullable=True)) + op.create_table( + "result_refreshes", + sa.Column("scope_key", sa.String(120), primary_key=True), + sa.Column("protocol_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("protocols.id", ondelete="CASCADE"), nullable=False), + sa.Column("protocol_revision_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("protocol_revisions.id", ondelete="CASCADE")), + sa.Column("design_revision_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("experiment_design_revisions.id", ondelete="CASCADE")), + sa.Column("generation", sa.BigInteger(), nullable=False, server_default="1"), + sa.Column("processed_generation", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("summary", postgresql.JSONB()), sa.Column("error", sa.Text()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_result_refreshes_protocol_id", "result_refreshes", ["protocol_id"]) + op.create_table( + "result_cell_rollups", + sa.Column("scope_key", sa.String(120), sa.ForeignKey("result_refreshes.scope_key", ondelete="CASCADE"), primary_key=True), + sa.Column("cell_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("factorial_cells.id", ondelete="CASCADE"), primary_key=True), + sa.Column("generation", sa.BigInteger(), nullable=False), sa.Column("data", postgresql.JSONB(), nullable=False), + ) + op.create_table("result_cell_metrics", + sa.Column("scope_key", sa.String(120), sa.ForeignKey("result_refreshes.scope_key", ondelete="CASCADE"), primary_key=True), + sa.Column("metric_key", sa.String(255), primary_key=True), + sa.Column("cell_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("factorial_cells.id", ondelete="CASCADE"), primary_key=True), + sa.Column("value", sa.Float()), sa.Column("generation", sa.BigInteger(), nullable=False)) + op.create_index("ix_result_cell_metric_rank", "result_cell_metrics", ["scope_key", "metric_key", "value", "cell_id"]) + op.create_index("ix_protocol_runs_slot_version_order", "protocol_runs", + ["protocol_id", "protocol_revision_id", "design_revision_id", "replicate_result_id", "created_at", "id"], + postgresql_where=sa.text("row_result_id IS NULL")) + op.create_index("ix_protocol_runs_row_version_order", "protocol_runs", + ["row_result_id", "protocol_revision_id", "created_at", "id"], + postgresql_where=sa.text("row_result_id IS NOT NULL")) + op.create_index("ix_factorial_cells_revision_page", "factorial_cells", ["experiment_id", "design_revision_id", "id"]) + op.create_index("ix_factorial_rows_version_page", "factorial_row_results", ["protocol_revision_id", "replicate_result_id", "id"]) + # Previews are derived at the persistence seam, including notebook/legacy writes. + # Reported values stay opaque; only bounded textual previews leave list reads. + op.execute(""" + CREATE FUNCTION asaree_result_preview(doc jsonb, nodes jsonb) RETURNS jsonb + LANGUAGE sql IMMUTABLE AS $$ + SELECT jsonb_build_object( + 'observations', COALESCE((SELECT jsonb_agg(jsonb_build_object( + 'metric_id', o->'metric_id', 'metric_name', o->'metric_name', + 'status', o->'status', 'value_type', o->'value_type', + 'preview', left(COALESCE(o->'value', 'null'::jsonb)::text, 512), + 'preview_truncated', length(COALESCE(o->'value', 'null'::jsonb)::text) > 512, + 'producer', o->'producer' + )) FROM jsonb_array_elements(CASE WHEN jsonb_typeof(doc#>'{measurement,observations}') = 'array' + THEN doc#>'{measurement,observations}' ELSE '[]'::jsonb END) o), '[]'::jsonb), + 'metrics', COALESCE((SELECT jsonb_object_agg(o->>'metric_name', CASE + WHEN jsonb_typeof(o->'value') = 'boolean' THEN to_jsonb(CASE WHEN (o->>'value')::boolean THEN 1 ELSE 0 END) + ELSE o->'value' END) + FROM jsonb_array_elements(CASE WHEN jsonb_typeof(doc#>'{measurement,observations}') = 'array' + THEN doc#>'{measurement,observations}' ELSE '[]'::jsonb END) o + WHERE o->>'status' = 'measured' AND o#>>'{producer,kind}' NOT IN ('reported', 'legacy') + AND o->>'metric_name' IS NOT NULL AND jsonb_typeof(o->'value') IN ('number', 'boolean')), '{}'::jsonb), + 'truncated', EXISTS (SELECT 1 FROM jsonb_each(COALESCE(nodes, '{}'::jsonb)) n + WHERE jsonb_typeof(n.value->'truncation') = 'object'), + 'evaluation_state', doc->'evaluation_state' + ); + $$; + CREATE FUNCTION asaree_preview_run() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN NEW.result_summary := asaree_result_preview(NEW.attempt_result, NEW.node_runs); RETURN NEW; END $$; + CREATE TRIGGER result_preview BEFORE INSERT OR UPDATE OF attempt_result, node_runs ON protocol_runs + FOR EACH ROW EXECUTE FUNCTION asaree_preview_run(); + + CREATE FUNCTION asaree_dirty_results(p uuid, v uuid, d uuid) RETURNS void LANGUAGE plpgsql AS $$ + DECLARE k text; actual_design uuid; + BEGIN + IF NOT EXISTS (SELECT 1 FROM protocols WHERE id=p AND experiment_id IS NOT NULL) THEN RETURN; END IF; + SELECT COALESCE(d, (SELECT design_revision_id FROM protocol_revisions WHERE id=v), + (SELECT id FROM experiment_design_revisions WHERE experiment_id=(SELECT experiment_id FROM protocols WHERE id=p) + AND superseded_at IS NULL)) INTO actual_design; + k := p::text || ':' || COALESCE(v::text, 'legacy') || ':' || COALESCE(actual_design::text, 'none'); + INSERT INTO result_refreshes(scope_key, protocol_id, protocol_revision_id, design_revision_id) + VALUES(k,p,v,actual_design) + ON CONFLICT(scope_key) DO UPDATE SET generation=result_refreshes.generation+1, updated_at=now(); + END $$; + CREATE FUNCTION asaree_run_results_changed() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN PERFORM asaree_dirty_results(OLD.protocol_id,OLD.protocol_revision_id,OLD.design_revision_id); RETURN OLD; END IF; + PERFORM asaree_dirty_results(NEW.protocol_id,NEW.protocol_revision_id,NEW.design_revision_id); RETURN NEW; + END $$; + CREATE TRIGGER results_changed AFTER INSERT OR DELETE OR UPDATE OF status, attempt_result, usage_summary ON protocol_runs + FOR EACH ROW EXECUTE FUNCTION asaree_run_results_changed(); + CREATE FUNCTION asaree_design_results_changed() RETURNS trigger LANGUAGE plpgsql AS $$ + DECLARE e uuid; p record; d uuid; + BEGIN + IF TG_TABLE_NAME='protocols' THEN + PERFORM asaree_dirty_results(NEW.id, NEW.published_revision_id, NULL); RETURN NEW; + END IF; + IF TG_TABLE_NAME='factorial_cells' THEN e:=COALESCE(NEW.experiment_id, OLD.experiment_id); + ELSIF TG_TABLE_NAME='factorial_replicate_results' THEN + SELECT experiment_id INTO e FROM factorial_cells WHERE id=COALESCE(NEW.cell_id, OLD.cell_id); + ELSE + SELECT c.experiment_id INTO e FROM factorial_cells c JOIN factorial_replicate_results r ON r.cell_id=c.id + WHERE r.id=COALESCE(NEW.replicate_result_id, OLD.replicate_result_id); + END IF; + FOR p IN SELECT id,published_revision_id FROM protocols WHERE experiment_id=e LOOP + PERFORM asaree_dirty_results(p.id,p.published_revision_id,NULL); + END LOOP; + RETURN COALESCE(NEW,OLD); + END $$; + CREATE TRIGGER protocol_results_changed AFTER INSERT OR UPDATE OF published_revision_id ON protocols + FOR EACH ROW EXECUTE FUNCTION asaree_design_results_changed(); + CREATE TRIGGER cell_results_changed AFTER INSERT OR DELETE OR UPDATE ON factorial_cells + FOR EACH ROW EXECUTE FUNCTION asaree_design_results_changed(); + CREATE TRIGGER replicate_results_changed AFTER INSERT OR DELETE OR UPDATE ON factorial_replicate_results + FOR EACH ROW EXECUTE FUNCTION asaree_design_results_changed(); + CREATE TRIGGER row_results_changed AFTER INSERT OR DELETE OR UPDATE ON factorial_row_results + FOR EACH ROW EXECUTE FUNCTION asaree_design_results_changed(); + """) + # Seed only refresh work. Historical evidence/usage is backfilled in bounded worker batches. + op.execute("SELECT asaree_dirty_results(id, published_revision_id, NULL) FROM protocols") + + +def downgrade() -> None: + for table, trigger in [("factorial_row_results", "row_results_changed"), + ("factorial_replicate_results", "replicate_results_changed"), + ("factorial_cells", "cell_results_changed"), ("protocols", "protocol_results_changed"), + ("protocol_runs", "results_changed"), ("protocol_runs", "result_preview")]: + op.execute(f"DROP TRIGGER {trigger} ON {table}") + for function in ["asaree_design_results_changed()", "asaree_run_results_changed()", + "asaree_preview_run()", "asaree_dirty_results(uuid, uuid, uuid)", "asaree_result_preview(jsonb, jsonb)"]: + op.execute(f"DROP FUNCTION {function}") + for table, index in [("protocol_runs", "ix_protocol_runs_slot_version_order"), + ("protocol_runs", "ix_protocol_runs_row_version_order"), + ("factorial_cells", "ix_factorial_cells_revision_page"), + ("factorial_row_results", "ix_factorial_rows_version_page")]: + op.drop_index(index, table_name=table) + op.drop_table("result_cell_metrics") + op.drop_table("result_cell_rollups") + op.drop_table("result_refreshes") + op.drop_column("protocol_runs", "usage_summary") + op.drop_column("protocol_runs", "result_summary") diff --git a/src/asaree/models/protocol_run.py b/src/asaree/models/protocol_run.py index e48c008b..e76713eb 100644 --- a/src/asaree/models/protocol_run.py +++ b/src/asaree/models/protocol_run.py @@ -56,6 +56,10 @@ class ProtocolRun(Base, TimestampMixin): # "task_completed_at": iso, "evaluation_started_at": iso, # "evaluation_completed_at": iso}. attempt_result: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True) + # Small database-maintained preview, separate from potentially enormous evidence. + result_summary: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False, default=dict) + # Operational usage is captured independently of declared measurement bindings. + usage_summary: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True) # The agent-to-agent transcript, present only once a run's agents actually # communicate -- including a pipeline Agent delegating to a Sub-Agent. # Shape: {"state": , "messages": [{"message_id", "sequence", diff --git a/src/asaree/models/result_read_model.py b/src/asaree/models/result_read_model.py new file mode 100644 index 00000000..37100e40 --- /dev/null +++ b/src/asaree/models/result_read_model.py @@ -0,0 +1,75 @@ +"""Rebuildable, version-scoped summaries; attempt documents remain authoritative.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import BigInteger, DateTime, Float, ForeignKey, Index, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from asaree.models.base import Base + + +class ResultRefresh(Base): + __tablename__ = "result_refreshes" + + scope_key: Mapped[str] = mapped_column(String(120), primary_key=True) + protocol_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("protocols.id", ondelete="CASCADE"), + index=True, + ) + protocol_revision_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("protocol_revisions.id", ondelete="CASCADE"), + nullable=True, + ) + design_revision_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("experiment_design_revisions.id", ondelete="CASCADE"), + nullable=True, + ) + generation: Mapped[int] = mapped_column(BigInteger, default=1) + processed_generation: Mapped[int] = mapped_column(BigInteger, default=0) + summary: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class ResultCellRollup(Base): + __tablename__ = "result_cell_rollups" + + scope_key: Mapped[str] = mapped_column( + String(120), + ForeignKey("result_refreshes.scope_key", ondelete="CASCADE"), + primary_key=True, + ) + cell_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("factorial_cells.id", ondelete="CASCADE"), + primary_key=True, + ) + generation: Mapped[int] = mapped_column(BigInteger) + data: Mapped[dict[str, Any]] = mapped_column(JSONB) + + +class ResultCellMetric(Base): + __tablename__ = "result_cell_metrics" + __table_args__ = (Index("ix_result_cell_metric_rank", "scope_key", "metric_key", "value", "cell_id"),) + + scope_key: Mapped[str] = mapped_column( + String(120), + ForeignKey("result_refreshes.scope_key", ondelete="CASCADE"), + primary_key=True, + ) + metric_key: Mapped[str] = mapped_column(String(255), primary_key=True) + cell_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("factorial_cells.id", ondelete="CASCADE"), + primary_key=True, + ) + value: Mapped[float | None] = mapped_column(Float, nullable=True) + generation: Mapped[int] = mapped_column(BigInteger) diff --git a/src/asaree/services/result_reads.py b/src/asaree/services/result_reads.py new file mode 100644 index 00000000..99c934fe --- /dev/null +++ b/src/asaree/services/result_reads.py @@ -0,0 +1,624 @@ +"""Bounded reads over stable execution slots and rebuildable version summaries. + +No normal read consults Motoro or selects an attempt's evidence documents. +The same scope and slot query is used by reads, rollups, batch controls and exports. +""" + +from __future__ import annotations + +import base64 +import json +import uuid +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import and_, case, cast, func, literal, or_, select +from sqlalchemy.dialects.postgresql import JSONB, insert +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import defer + +from asaree.models.experiment_design_revision import ExperimentDesignRevision +from asaree.models.factorial_cell import FactorialCell +from asaree.models.factorial_replicate_result import FactorialReplicateResult +from asaree.models.factorial_row_result import FactorialRowResult +from asaree.models.protocol import Protocol +from asaree.models.protocol_revision import ProtocolRevision +from asaree.models.protocol_run import ProtocolRun +from asaree.models.result_read_model import ResultCellMetric, ResultCellRollup, ResultRefresh +from asaree.services.dataset_row_inputs import resolve_dataset_row_plan +from asaree.services.experiment_run_results import ( + RunResultsProjectionError, + _declared_metric_aggregations, + _declared_metric_directions, + _declared_metric_types, + _node_labels, + _primary_metric, +) + +PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 + + +@dataclass(frozen=True) +class ResultScope: + experiment_id: uuid.UUID + protocol_id: uuid.UUID + protocol_revision_id: uuid.UUID | None + design_revision_id: uuid.UUID | None + design_spec: dict[str, Any] | None + graph: dict[str, Any] + versioned: bool + row_plan: dict[str, Any] | None + + @property + def key(self) -> str: + return f"{self.protocol_id}:{self.protocol_revision_id or 'legacy'}:{self.design_revision_id or 'none'}" + + @property + def unit(self) -> str: + return "row" if self.row_plan is not None else "replicate" + + +async def resolve_result_scope( + db: AsyncSession, + *, + experiment_id: uuid.UUID, + protocol_id: uuid.UUID | None = None, + protocol_revision_id: uuid.UUID | None = None, + design_revision_id: uuid.UUID | None = None, + use_current_publication: bool = True, +) -> ResultScope: + statement = select(Protocol.id, Protocol.published_revision_id).where(Protocol.experiment_id == experiment_id) + if protocol_id is not None: + statement = statement.where(Protocol.id == protocol_id) + protocols = (await db.execute(statement.limit(2))).all() + if not protocols: + raise RunResultsProjectionError("protocol_not_found") + if len(protocols) > 1: + raise RunResultsProjectionError("ambiguous_protocol") + selected_protocol_id, current_publication = protocols[0] + publication_id = (protocol_revision_id or current_publication) if use_current_publication else protocol_revision_id + publication = None + if publication_id is not None: + publication = await db.scalar( + select(ProtocolRevision).where( + ProtocolRevision.id == publication_id, + ProtocolRevision.protocol_id == selected_protocol_id, + ) + ) + if publication is None: + raise RunResultsProjectionError("protocol_revision_not_found") + versioned = publication is not None and publication.experiment_snapshot is not None + if versioned: + assert publication is not None and publication.experiment_snapshot is not None + if design_revision_id is not None and design_revision_id != publication.design_revision_id: + raise RunResultsProjectionError("design_does_not_belong_to_experiment_version") + design_revision_id = publication.design_revision_id + revision_query = select(ExperimentDesignRevision).where( + ExperimentDesignRevision.experiment_id == experiment_id, + ) + if design_revision_id is not None: + revision_query = revision_query.where(ExperimentDesignRevision.id == design_revision_id) + else: + revision_query = revision_query.where(ExperimentDesignRevision.superseded_at.is_(None)) + revision = None if versioned and design_revision_id is None else await db.scalar(revision_query) + if design_revision_id is not None and revision is None: + raise RunResultsProjectionError("design_revision_not_found") + design = revision.design_spec if revision else None + if versioned: + assert publication is not None and publication.experiment_snapshot is not None + design = publication.experiment_snapshot.get("design_spec") + graph = publication.graph if publication is not None and isinstance(publication.graph, dict) else {} + return ResultScope( + experiment_id, + selected_protocol_id, + publication_id, + revision.id if revision else None, + design, + graph, + versioned, + resolve_dataset_row_plan(graph) if publication is not None else None, + ) + + +def slot_query(scope: ResultScope) -> Any: + """One current execution per stable slot, including slots never attempted. + + Lateral lookup uses the slot/version indexes; no client-sized IN lists or full + JSON documents participate. Legacy label fallback also matches factor values. + """ + cell, parent, run, row = FactorialCell, FactorialReplicateResult, ProtocolRun, FactorialRowResult + base = ( + select( + cell.id.label("cell_id"), + cell.cell_label, + cell.factor_values, + parent.id.label("replicate_id"), + parent.replicate_label, + parent.replicate_number, + ) + .select_from(cell) + .join(parent, parent.cell_id == cell.id) + .where( + cell.experiment_id == scope.experiment_id, + cell.design_revision_id == scope.design_revision_id, + ) + ) + if scope.unit == "row": + base = ( + base.join(row, row.replicate_result_id == parent.id) + .where( + row.protocol_revision_id == scope.protocol_revision_id, + ) + .add_columns( + row.id.label("slot_id"), + row.id.label("row_result_id"), + row.row_index, + row.dataset_id, + row.raw_sha256, + ) + ) + match = and_(run.row_result_id == row.id, run.id == row.run_id) + else: + base = base.add_columns( + parent.id.label("slot_id"), + literal(None).label("row_result_id"), + literal(None).label("row_index"), + literal(None).label("dataset_id"), + literal(None).label("raw_sha256"), + ) + match = and_( + run.row_result_id.is_(None), + or_( + run.replicate_result_id == parent.id, + and_( + run.replicate_result_id.is_(None), + run.replicate_label == parent.replicate_label, + run.factor_values == cell.factor_values, + ), + ), + ) + latest = ( + select( + run.id.label("run_id"), + run.status, + run.created_at, + run.updated_at, + run.started_at, + run.completed_at, + func.left(run.error, 512).label("error"), + run.result_summary, + run.usage_summary, + ) + .where( + run.protocol_id == scope.protocol_id, + run.design_revision_id == scope.design_revision_id, + run.protocol_revision_id == scope.protocol_revision_id, + match, + run.is_test_run.is_(False), + run.target_node_id.is_(None), + ) + .order_by(run.created_at.desc(), run.id.desc()) + .limit(1) + .correlate(cell, parent, row) + .lateral("latest") + ) + imported = and_( + literal(not scope.versioned and scope.unit == "replicate"), + parent.metric_values.is_not(None), + parent.metric_values != cast(literal("{}"), JSONB), + ) + status = func.coalesce(latest.c.status, case((imported, "completed"), else_="not_started")) + return base.outerjoin(latest, literal(True)).add_columns( + latest.c.run_id, + status.label("status"), + latest.c.created_at, + latest.c.updated_at, + latest.c.started_at, + latest.c.completed_at, + latest.c.error, + func.coalesce(latest.c.result_summary, cast(literal("{}"), JSONB)).label("result"), + latest.c.usage_summary.label("usage"), + ) + + +def encode_cursor(scope: ResultScope, kind: str, position: list[Any], filters: dict[str, Any]) -> str: + document = {"scope": scope.key, "kind": kind, "position": position, "filters": filters} + return base64.urlsafe_b64encode(json.dumps(document, separators=(",", ":")).encode()).decode() + + +def decode_cursor(cursor: str | None, scope: ResultScope, kind: str, filters: dict[str, Any]) -> list[Any] | None: + if cursor is None: + return None + if len(cursor) > 2048: + raise ValueError("invalid_cursor") + try: + document = json.loads(base64.b64decode(cursor, altchars=b"-_", validate=True)) + if document["scope"] != scope.key or document["kind"] != kind or document["filters"] != filters: + raise ValueError("cursor_scope_mismatch") + position = document["position"] + if not isinstance(position, list): + raise ValueError("invalid_cursor") + return position + except (KeyError, TypeError, json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ValueError("invalid_cursor") from exc + + +def page_size(size: int) -> int: + if not 1 <= size <= MAX_PAGE_SIZE: + raise ValueError("invalid_page_size") + return size + + +async def request_refresh(db: AsyncSession, scope: ResultScope) -> None: + await db.execute( + insert(ResultRefresh) + .values( + scope_key=scope.key, + protocol_id=scope.protocol_id, + protocol_revision_id=scope.protocol_revision_id, + design_revision_id=scope.design_revision_id, + generation=1, + processed_generation=0, + ) + .on_conflict_do_nothing(index_elements=[ResultRefresh.scope_key]) + ) + + +async def read_summary(db: AsyncSession, scope: ResultScope) -> dict[str, Any]: + refresh = await db.get(ResultRefresh, scope.key) + if refresh is None: + await request_refresh(db, scope) + return { + "scope_key": scope.key, + "unit": scope.unit, + "protocol_id": str(scope.protocol_id), + "protocol_revision_id": str(scope.protocol_revision_id) if scope.protocol_revision_id else None, + "design_revision_id": str(scope.design_revision_id) if scope.design_revision_id else None, + "ready": bool(refresh and refresh.summary is not None), + "generation": refresh.processed_generation if refresh else 0, + "change_id": refresh.generation if refresh else 1, + "refreshing": refresh is None or refresh.generation != refresh.processed_generation, + "updated_at": refresh.updated_at if refresh else None, + "design_spec": scope.design_spec, + "metric_types": _declared_metric_types(scope.design_spec), + "metric_aggregations": _declared_metric_aggregations(scope.design_spec), + "metric_directions": _declared_metric_directions(scope.design_spec), + "primary_metric": _primary_metric(scope.design_spec)[0] if scope.unit == "replicate" else None, + "primary_metric_direction": _primary_metric(scope.design_spec)[1], + **(refresh.summary if refresh and refresh.summary is not None else {}), + } + + +async def read_executions( + db: AsyncSession, + scope: ResultScope, + *, + cursor: str | None = None, + limit: int = PAGE_SIZE, + cell_id: uuid.UUID | None = None, + replicate_id: uuid.UUID | None = None, + status: str | None = None, +) -> dict[str, Any]: + filters = { + "cell_id": str(cell_id) if cell_id else None, + "replicate_id": str(replicate_id) if replicate_id else None, + "status": status, + } + position = decode_cursor(cursor, scope, "executions", filters) + query = slot_query(scope) + if cell_id is not None: + query = query.where(FactorialCell.id == cell_id) + if replicate_id is not None: + query = query.where(FactorialReplicateResult.id == replicate_id) + slots = query.subquery("slots") + statement = select(slots) + if status is not None: + statement = statement.where(slots.c.status == status) + if position is not None: + if len(position) != 1: + raise ValueError("invalid_cursor") + statement = statement.where(slots.c.slot_id > uuid.UUID(position[0])) + rows = (await db.execute(statement.order_by(slots.c.slot_id).limit(page_size(limit) + 1))).mappings().all() + items = [execution_payload(dict(row)) for row in rows[:limit]] + return { + "items": items, + "next_cursor": encode_cursor(scope, "executions", [str(rows[limit - 1]["slot_id"])], filters) + if len(rows) > limit + else None, + } + + +def execution_payload(row: dict[str, Any]) -> dict[str, Any]: + return { + **{ + key: str(row[key]) if row.get(key) is not None else None + for key in ("slot_id", "cell_id", "replicate_id", "row_result_id", "run_id", "dataset_id") + }, + **{ + key: row.get(key) + for key in ( + "cell_label", + "factor_values", + "replicate_label", + "replicate_number", + "row_index", + "raw_sha256", + "status", + "error", + "updated_at", + ) + }, + "result": row.get("result") or {}, + "usage": row.get("usage"), + } + + +async def read_cells( + db: AsyncSession, + scope: ResultScope, + *, + cursor: str | None = None, + limit: int = PAGE_SIZE, + metric: str | None = None, + direction: str = "maximize", +) -> dict[str, Any]: + if metric is not None and (scope.unit == "row" or metric not in _declared_metric_types(scope.design_spec)): + raise ValueError("invalid_sort_metric") + if direction not in {"minimize", "maximize"}: + raise ValueError("invalid_sort_direction") + filters = {"metric": metric, "direction": direction} + position = decode_cursor(cursor, scope, "cells", filters) + cached = ResultCellRollup + statement = ( + select(FactorialCell.id, FactorialCell.cell_label, FactorialCell.factor_values, cached.data) + .outerjoin( + cached, + and_(cached.cell_id == FactorialCell.id, cached.scope_key == scope.key), + ) + .where( + FactorialCell.experiment_id == scope.experiment_id, + FactorialCell.design_revision_id == scope.design_revision_id, + ) + ) + if metric: + statement = statement.outerjoin( + ResultCellMetric, + and_( + ResultCellMetric.scope_key == scope.key, + ResultCellMetric.cell_id == FactorialCell.id, + ResultCellMetric.metric_key == metric, + ), + ) + value = ResultCellMetric.value + # A measured value ranks before an unavailable value. Negation makes + # the cursor order ascending for both directions, with UUID tie-break. + order_value = -value if direction == "maximize" else value + missing = case((value.is_(None), 1), else_=0) + if position is not None: + if len(position) != 3: + raise ValueError("invalid_cursor") + m, v, identifier = position + statement = statement.where( + or_( + missing > m, + and_( + missing == m, + or_(order_value > v, and_(order_value == v, FactorialCell.id > uuid.UUID(identifier))) + if v is not None + else FactorialCell.id > uuid.UUID(identifier), + ), + ) + ) + statement = statement.add_columns(missing.label("missing"), order_value.label("order_value")) + statement = statement.order_by(missing, order_value.asc().nulls_last(), FactorialCell.id) + else: + if position is not None: + if len(position) != 1: + raise ValueError("invalid_cursor") + statement = statement.where(FactorialCell.id > uuid.UUID(position[0])) + statement = statement.order_by(FactorialCell.id) + rows = (await db.execute(statement.limit(page_size(limit) + 1))).mappings().all() + items = [ + { + "cell_id": str(row["id"]), + "cell_label": row["cell_label"], + "factor_values": row["factor_values"] or {}, + **(row["data"] or {}), + } + for row in rows[:limit] + ] + next_position = None + if len(rows) > limit: + last = rows[limit - 1] + next_position = [last["missing"], last["order_value"], str(last["id"])] if metric else [str(last["id"])] + return { + "items": items, + "next_cursor": encode_cursor(scope, "cells", next_position, filters) if next_position else None, + } + + +async def read_attempts( + db: AsyncSession, + scope: ResultScope, + *, + slot_id: uuid.UUID, + cursor: str | None = None, + limit: int = PAGE_SIZE, +) -> dict[str, Any]: + filters = {"slot_id": str(slot_id)} + position = decode_cursor(cursor, scope, "attempts", filters) + slots = slot_query(scope).subquery("slots") + slot = (await db.execute(select(slots).where(slots.c.slot_id == slot_id))).mappings().one_or_none() + if slot is None: + raise RunResultsProjectionError("slot_not_found") + run = ProtocolRun + match = ( + run.row_result_id == slot_id + if scope.unit == "row" + else and_( + run.row_result_id.is_(None), + or_( + run.replicate_result_id == slot_id, + and_( + run.replicate_result_id.is_(None), + run.replicate_label == slot["replicate_label"], + run.factor_values == slot["factor_values"], + ), + ), + ) + ) + query = select( + run.id, + run.status, + run.created_at, + run.completed_at, + func.left(run.error, 512).label("error"), + run.result_summary, + run.usage_summary, + ).where( + run.protocol_id == scope.protocol_id, + run.protocol_revision_id == scope.protocol_revision_id, + run.design_revision_id == scope.design_revision_id, + run.is_test_run.is_(False), + run.target_node_id.is_(None), + match, + ) + if position is not None: + from datetime import datetime + + if len(position) != 2: + raise ValueError("invalid_cursor") + time, identifier = datetime.fromisoformat(position[0]), uuid.UUID(position[1]) + query = query.where(or_(run.created_at < time, and_(run.created_at == time, run.id < identifier))) + rows = ( + (await db.execute(query.order_by(run.created_at.desc(), run.id.desc()).limit(page_size(limit) + 1))) + .mappings() + .all() + ) + return { + "slot": execution_payload(dict(slot)), + "items": [ + { + "run_id": str(row["id"]), + "status": row["status"], + "created_at": row["created_at"], + "completed_at": row["completed_at"], + "error": row["error"], + "current": row["id"] == slot["run_id"], + "result": row["result_summary"], + "usage": row["usage_summary"], + } + for row in rows[:limit] + ], + "next_cursor": encode_cursor( + scope, "attempts", [rows[limit - 1]["created_at"].isoformat(), str(rows[limit - 1]["id"])], filters + ) + if len(rows) > limit + else None, + } + + +async def lookup_slot(db: AsyncSession, scope: ResultScope, replicate_label: str) -> dict[str, Any]: + slots = slot_query(scope).subquery("slots") + rows = (await db.execute(select(slots).where(slots.c.replicate_label == replicate_label).limit(2))).mappings().all() + if not rows: + raise RunResultsProjectionError("slot_not_found") + if len(rows) != 1: + raise ValueError("ambiguous_replicate_label_use_slot_id") + return execution_payload(dict(rows[0])) + + +async def read_cell( + db: AsyncSession, scope: ResultScope, cell_id: uuid.UUID | None, cell_label: str | None +) -> dict[str, Any]: + query = ( + select(FactorialCell.id, FactorialCell.cell_label, FactorialCell.factor_values, ResultCellRollup.data) + .outerjoin( + ResultCellRollup, + and_(ResultCellRollup.cell_id == FactorialCell.id, ResultCellRollup.scope_key == scope.key), + ) + .where( + FactorialCell.experiment_id == scope.experiment_id, + FactorialCell.design_revision_id == scope.design_revision_id, + ) + ) + query = query.where(FactorialCell.id == cell_id) if cell_id else query.where(FactorialCell.cell_label == cell_label) + row = (await db.execute(query)).mappings().one_or_none() + if row is None: + raise RunResultsProjectionError("cell_not_found") + return { + "cell_id": str(row["id"]), + "cell_label": row["cell_label"], + "factor_values": row["factor_values"] or {}, + **(row["data"] or {}), + } + + +async def read_attempt_detail(db: AsyncSession, scope: ResultScope, run_id: uuid.UUID) -> dict[str, Any]: + # This is the only interactive operation that selects evidence for a run. + # Ownership and publication/design scope are checked before it can be returned. + run = await db.scalar( + select(ProtocolRun) + .options(defer(ProtocolRun.conversation)) + .where( + ProtocolRun.id == run_id, + ProtocolRun.protocol_id == scope.protocol_id, + ProtocolRun.protocol_revision_id == scope.protocol_revision_id, + ProtocolRun.design_revision_id == scope.design_revision_id, + ProtocolRun.is_test_run.is_(False), + ProtocolRun.target_node_id.is_(None), + ) + ) + if run is None: + raise RunResultsProjectionError("attempt_not_found") + return { + "run_id": str(run.id), + "status": run.status, + "error": run.error, + "attempt_result": run.attempt_result, + "node_runs": run.node_runs, + "node_labels": _node_labels(scope.graph), + "usage": run.usage_summary, + "started_at": run.started_at, + "completed_at": run.completed_at, + "protocol_revision_id": str(run.protocol_revision_id) if run.protocol_revision_id else None, + "dataset_row": run.dataset_row, + } + + +async def read_conversation( + db: AsyncSession, + scope: ResultScope, + run_id: uuid.UUID, + *, + offset: int = 0, + limit: int = PAGE_SIZE, +) -> dict[str, Any]: + if offset < 0: + raise ValueError("invalid_offset") + page_size(limit) + # Slice JSON in PostgreSQL. A long transcript never enters API memory whole. + elements = func.jsonb_array_elements(ProtocolRun.conversation["messages"]).table_valued( + "value", with_ordinality="ordinal" + ) + items = ( + select(elements.c.value) + .where(elements.c.ordinal > offset) + .order_by(elements.c.ordinal) + .limit(limit + 1) + .lateral() + ) + query = ( + select(items.c.value) + .select_from(ProtocolRun) + .join(items, literal(True)) + .where( + ProtocolRun.id == run_id, + ProtocolRun.protocol_id == scope.protocol_id, + ProtocolRun.protocol_revision_id == scope.protocol_revision_id, + ProtocolRun.design_revision_id == scope.design_revision_id, + ) + ) + rows = list((await db.execute(query)).scalars().all()) + return {"items": rows[:limit], "next_offset": offset + limit if len(rows) > limit else None} diff --git a/src/asaree/services/result_rollups.py b/src/asaree/services/result_rollups.py new file mode 100644 index 00000000..83829fee --- /dev/null +++ b/src/asaree/services/result_rollups.py @@ -0,0 +1,280 @@ +"""Coalesced SQL rollups over compact slots, driven by the transactional outbox.""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy import Float, and_, cast, delete, func, literal, select, update +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from asaree.models.database import get_session +from asaree.models.factorial_cell import FactorialCell +from asaree.models.factorial_replicate_result import FactorialReplicateResult +from asaree.models.protocol import Protocol +from asaree.models.result_read_model import ResultCellMetric, ResultCellRollup, ResultRefresh +from asaree.services.experiment_run_results import _declared_metric_aggregations, _source_row_count +from asaree.services.result_reads import ResultScope, resolve_result_scope, slot_query + +logger = logging.getLogger(__name__) +STATUSES = ("not_started", "pending", "running", "finalizing", "completed", "failed", "cancelled", "limit_reached") + + +def rollup_query(scope: ResultScope) -> Any: + slots = slot_query(scope).cte("slots") + parent_query = select( + FactorialCell.id.label("cell_id"), func.count(FactorialReplicateResult.id).label("replicate_count") + ) + parents = ( + parent_query.outerjoin(FactorialReplicateResult, FactorialReplicateResult.cell_id == FactorialCell.id) + .where( + FactorialCell.experiment_id == scope.experiment_id, + FactorialCell.design_revision_id == scope.design_revision_id, + ) + .group_by(FactorialCell.id) + .cte("parents") + ) + stats_columns = [slots.c.cell_id, func.count().label("total")] + stats_columns += [func.count().filter(slots.c.status == status).label(status) for status in STATUSES] + stats_columns += [ + func.count().filter(slots.c.result["truncated"].as_boolean().is_(True)).label("truncated"), + func.count().filter(slots.c.result["evaluation_state"].as_string() == "running").label("evaluating"), + func.count().filter(slots.c.usage["complete"].as_boolean().is_(True)).label("usage_complete"), + func.count() + .filter( + and_( + slots.c.status.in_(("completed", "failed", "cancelled", "limit_reached")), + slots.c.usage["settled"].as_boolean().is_not(True), + ) + ) + .label("usage_pending"), + ] + for key in ("cost_usd", "input_tokens", "output_tokens", "total_tokens", "duration_seconds", "agent_run_count"): + stats_columns.append(func.sum(slots.c.usage[key].as_float()).label(key)) + stats = select(*stats_columns).group_by(slots.c.cell_id).cte("stats") + metrics = func.jsonb_each(slots.c.result["metrics"]).table_valued("key", "value").lateral("metrics") + metric_query = select( + slots.c.cell_id, + metrics.c.key, + func.avg(cast(metrics.c.value, Float)).label("mean"), + func.sum(cast(metrics.c.value, Float)).label("sum"), + func.count().label("count"), + ) + values = ( + metric_query.select_from(slots) + .join(metrics, literal(True)) + .where( + slots.c.status == "completed", + slots.c.result["truncated"].as_boolean().is_(False), + ) + .group_by(slots.c.cell_id, metrics.c.key) + .cte("metric_values") + ) + metric_stats = ( + select( + values.c.cell_id, + func.jsonb_object_agg( + values.c.key, + func.jsonb_build_object("mean", values.c.mean, "sum", values.c.sum, "count", values.c.count), + ).label("metrics"), + ) + .group_by(values.c.cell_id) + .cte("metric_stats") + ) + return ( + select(parents, *[column for column in stats.c if column.key != "cell_id"], metric_stats.c.metrics) + .select_from( + parents.outerjoin(stats, stats.c.cell_id == parents.c.cell_id).outerjoin( + metric_stats, metric_stats.c.cell_id == parents.c.cell_id + ) + ) + .order_by(parents.c.cell_id) + ) + + +async def rebuild_result_summary(db: AsyncSession, scope: ResultScope, generation: int) -> None: + """One SQL snapshot supplies both cell and overall counts, published atomically. + + Rebuilding from current contributions makes repeated deliveries and reruns + idempotent. No decrement/increment arithmetic can drift after a worker crash. + Cell rows are streamed and upserted in batches to bound worker memory. + """ + existing = await db.get(ResultRefresh, scope.key) + cached = existing.summary if existing is not None else None + totals: dict[str, int] = {status: 0 for status in STATUSES} + totals.update(total=0, cells=0, replicates=0, truncated=0, evaluating=0, usage_complete=0, usage_pending=0) + usage: dict[str, float | None] = { + key: None + for key in ("cost_usd", "input_tokens", "output_tokens", "total_tokens", "duration_seconds", "agent_run_count") + } + aggregations = _declared_metric_aggregations(scope.design_spec) + batch = [] + result = await db.stream(rollup_query(scope)) + async for row in result.mappings(): + counts = {status: int(row[status] or 0) for status in STATUSES} + metrics = row["metrics"] or {} + data = { + "replicate_count": int(row["replicate_count"]), + "total": int(row["total"] or 0), + "counts": counts, + "truncated": int(row["truncated"] or 0), + "usage": {key: row[key] for key in usage}, + "metric_values": { + key: value["sum" if aggregations.get(key) == "sum" else "mean"] for key, value in metrics.items() + } + if scope.unit == "replicate" + else {}, + "metric_counts": {key: value["count"] for key, value in metrics.items()} + if scope.unit == "replicate" + else {}, + } + totals["cells"] += 1 + totals["replicates"] += int(row["replicate_count"]) + for key in ("total", "truncated", "evaluating", "usage_complete", "usage_pending", *STATUSES): + totals[key] += int(row[key] or 0) + for key in usage: + if row[key] is not None: + usage[key] = (usage[key] or 0) + float(row[key]) + batch.append({"scope_key": scope.key, "cell_id": row["cell_id"], "generation": generation, "data": data}) + if len(batch) == 100: + await _write_cells(db, batch) + batch = [] + if batch: + await _write_cells(db, batch) + await db.execute( + delete(ResultCellRollup).where( + ResultCellRollup.scope_key == scope.key, + ResultCellRollup.generation != generation, + ) + ) + await db.execute( + delete(ResultCellMetric).where( + ResultCellMetric.scope_key == scope.key, + ResultCellMetric.generation != generation, + ) + ) + row_count = cached.get("source_row_count") if cached else None + if scope.row_plan is not None and row_count is None: + dataset_id = scope.row_plan.get("driver_dataset_id") + import uuid + + from asaree.models.dataset import RegisteredDataset + + dataset = await db.get(RegisteredDataset, uuid.UUID(dataset_id)) if dataset_id else None + row_count = await _source_row_count( + db, dataset_id=dataset.id if dataset else None, raw_sha256=dataset.raw_sha256 if dataset else None + ) + summary = { + "counts": {status: totals[status] for status in STATUSES}, + "total": totals["total"], + "cell_count": totals["cells"], + "replicate_count": totals["replicates"], + "usage": usage, + "usage_complete_count": totals["usage_complete"], + "truncated_count": totals["truncated"], + "evaluating_count": totals["evaluating"], + "source_row_count": row_count, + "expected": row_count * totals["replicates"] + if row_count is not None + else None + if scope.unit == "row" + else totals["total"], + "has_active": any(totals[key] for key in ("pending", "running", "finalizing", "evaluating", "usage_pending")), + "usage_pending_count": totals["usage_pending"], + "as_of": datetime.now(UTC).isoformat(), + } + # generation may have advanced during this rebuild. Preserve the new dirty + # generation; the next delivery rebuilds it. Cell/overview publication is one transaction. + await db.execute( + update(ResultRefresh) + .where(ResultRefresh.scope_key == scope.key) + .values( + summary=summary, + processed_generation=generation, + error=None, + ) + ) + + +async def _write_cells(db: AsyncSession, rows: list[dict[str, Any]]) -> None: + statement = insert(ResultCellRollup).values(rows) + await db.execute( + statement.on_conflict_do_update( + index_elements=[ResultCellRollup.scope_key, ResultCellRollup.cell_id], + set_={"generation": statement.excluded.generation, "data": statement.excluded.data}, + ) + ) + metric_rows = [ + { + "scope_key": row["scope_key"], + "cell_id": row["cell_id"], + "generation": row["generation"], + "metric_key": key, + "value": value, + } + for row in rows + for key, value in row["data"]["metric_values"].items() + ] + if metric_rows: + statement = insert(ResultCellMetric).values(metric_rows) + await db.execute( + statement.on_conflict_do_update( + index_elements=[ResultCellMetric.scope_key, ResultCellMetric.metric_key, ResultCellMetric.cell_id], + set_={"value": statement.excluded.value, "generation": statement.excluded.generation}, + ) + ) + + +async def refresh_result_summaries(_ctx: dict[str, Any]) -> None: + """Durable work survives Redis loss; the existing worker sweeps the SQL outbox.""" + async with get_session() as db: + keys = list( + ( + await db.execute( + select(ResultRefresh.scope_key) + .where( + ResultRefresh.generation != ResultRefresh.processed_generation, + ) + .order_by(ResultRefresh.updated_at) + .limit(8) + ) + ).scalars() + ) + for key in keys: + try: + async with get_session() as db: + # At most one projector publishes this scope, without holding an + # outbox row lock while execution workers deliver new changes. + locked = await db.scalar(select(func.pg_try_advisory_xact_lock(func.hashtextextended(key, 0)))) + if not locked: + continue + refresh = await db.get(ResultRefresh, key) + if refresh is None or refresh.generation == refresh.processed_generation: + continue + experiment_id = await db.scalar( + select(Protocol.experiment_id).where(Protocol.id == refresh.protocol_id) + ) + if experiment_id is None: + continue + scope = await resolve_result_scope( + db, + experiment_id=experiment_id, + protocol_id=refresh.protocol_id, + protocol_revision_id=refresh.protocol_revision_id, + design_revision_id=refresh.design_revision_id, + use_current_publication=False, + ) + if scope.key != key: + await db.execute( + update(ResultRefresh) + .where(ResultRefresh.scope_key == key) + .values( + processed_generation=refresh.generation, + ) + ) + continue + await rebuild_result_summary(db, scope, refresh.generation) + except Exception: + logger.exception("result_summary_refresh_failed", extra={"scope_key": key}) diff --git a/src/asaree/worker/settings.py b/src/asaree/worker/settings.py index 91375bb1..dba7bd7d 100644 --- a/src/asaree/worker/settings.py +++ b/src/asaree/worker/settings.py @@ -30,6 +30,7 @@ from asaree.models.database import dispose_engine from asaree.redis_client import dispose_redis from asaree.services.credential_resolver import resolve as resolve_credentials +from asaree.services.result_rollups import refresh_result_summaries from asaree.services.system_mcp_servers import ensure_system_servers, refresh_system_server_capabilities from asaree.worker.tasks import ( check_stale_runs, @@ -65,7 +66,7 @@ class WorkerSettings: execute_run_task, execute_protocol_run_task, ] - cron_jobs = [cron(check_stale_runs, second={0, 30})] + cron_jobs = [cron(check_stale_runs, second={0, 30}), cron(refresh_result_summaries, second=set(range(0, 60, 5)))] redis_settings = RedisSettings.from_dsn(get_settings().redis_url) on_startup = on_startup on_shutdown = on_shutdown diff --git a/tests/test_result_reads.py b/tests/test_result_reads.py new file mode 100644 index 00000000..fdb73b9a --- /dev/null +++ b/tests/test_result_reads.py @@ -0,0 +1,254 @@ +"""Read contracts, SQL scoping and optional isolated PostgreSQL integration checks. + +Set ASAREE_RESULTS_TEST_DATABASE_URL to a disposable PostgreSQL database. All +migrations and fixtures live in a unique schema inside a rolled-back transaction. +This suite never reads application credential files or touches application tables. +""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path + +import pytest +import pytest_asyncio +from alembic.migration import MigrationContext +from alembic.operations import Operations +from alembic.script import ScriptDirectory +from sqlalchemy import select, text +from sqlalchemy.dialects import postgresql +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine + +from asaree.models.experiment import ResearchExperiment +from asaree.models.experiment_design_revision import ExperimentDesignRevision +from asaree.models.factorial_cell import FactorialCell +from asaree.models.factorial_replicate_result import FactorialReplicateResult +from asaree.models.protocol import Protocol +from asaree.models.protocol_revision import ProtocolRevision +from asaree.models.protocol_run import ProtocolRun +from asaree.models.result_read_model import ResultRefresh +from asaree.models.user import User +from asaree.services.result_reads import ( + ResultScope, + decode_cursor, + encode_cursor, + page_size, + read_attempt_detail, + read_attempts, + read_cells, + read_executions, + read_summary, + resolve_result_scope, + slot_query, +) +from asaree.services.result_rollups import rebuild_result_summary, rollup_query + + +def scope(**changes): + values = dict( + experiment_id=uuid.UUID(int=1), + protocol_id=uuid.UUID(int=2), + protocol_revision_id=uuid.UUID(int=3), + design_revision_id=uuid.UUID(int=4), + design_spec=None, + graph={}, + versioned=True, + row_plan=None, + ) + return ResultScope(**(values | changes)) + + +def test_slot_reads_do_not_select_evidence_or_agent_runs(): + query = select(slot_query(scope()).subquery()).limit(21) + sql = str(query.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) + assert "LIMIT 21" in sql + assert "protocol_runs.node_runs" not in sql + assert "protocol_runs.conversation" not in sql + assert "protocol_runs.attempt_result" not in sql + assert "agent_runs" not in sql + assert "factorial_cells.experiment_id" in sql + assert "protocol_runs.protocol_revision_id" in sql + assert "protocol_runs.design_revision_id" in sql + assert "LATERAL" in sql + + +def test_rollup_query_uses_compact_data_and_database_aggregates(): + sql = str(rollup_query(scope()).compile(dialect=postgresql.dialect())) + assert "GROUP BY" in sql + assert "protocol_runs.result_summary" in sql + assert "protocol_runs.attempt_result" not in sql + assert "protocol_runs.conversation" not in sql + + +def test_cursor_binds_scope_kind_and_filters(): + selected = scope() + filters = {"cell_id": "first"} + cursor = encode_cursor(selected, "executions", [str(uuid.UUID(int=8))], filters) + assert decode_cursor(cursor, selected, "executions", filters) == [str(uuid.UUID(int=8))] + for other, kind, other_filters in [ + (scope(protocol_revision_id=uuid.UUID(int=9)), "executions", filters), + (selected, "cells", filters), + (selected, "executions", {}), + ]: + with pytest.raises(ValueError): + decode_cursor(cursor, other, kind, other_filters) + + +@pytest.mark.parametrize("cursor", ["!invalid!", "", "x" * 2049, "e30=", "bnVsbA=="]) +def test_invalid_cursors_are_rejected(cursor): + with pytest.raises(ValueError): + decode_cursor(cursor, scope(), "executions", {}) + + +@pytest.mark.parametrize("size", [-1, 0, 101, 100000]) +def test_page_size_is_bounded(size): + with pytest.raises(ValueError): + page_size(size) + + +@pytest_asyncio.fixture +async def results_db(): + url = os.environ.get("ASAREE_RESULTS_TEST_DATABASE_URL") + if not url: + pytest.skip("Set ASAREE_RESULTS_TEST_DATABASE_URL for isolated PostgreSQL checks") + engine = create_async_engine(url) + async with engine.connect() as connection: + transaction = await connection.begin() + schema = "results_test_" + uuid.uuid4().hex + await connection.execute(text(f'CREATE SCHEMA "{schema}"')) + await connection.execute(text(f'SET LOCAL search_path TO "{schema}"')) + + def migrate(sync_connection): + scripts = ScriptDirectory(str(Path(__file__).parents[1] / "src/asaree/migrations")) + with Operations.context(MigrationContext.configure(sync_connection)): + for revision in reversed(list(scripts.walk_revisions())): + revision.module.upgrade() + + await connection.run_sync(migrate) + async with AsyncSession(bind=connection, expire_on_commit=False) as db: + user = User(email="results@example.test", hashed_password="test", display_name="Results test") + db.add(user) + await db.flush() + experiment = ResearchExperiment(name="Results test", owner_id=user.id, design_type="factorial") + db.add(experiment) + await db.flush() + design = ExperimentDesignRevision(experiment_id=experiment.id, revision=1, design_spec={"factors": []}) + db.add(design) + await db.flush() + protocol = Protocol(name="Results protocol", owner_id=user.id, experiment_id=experiment.id, graph={}) + db.add(protocol) + await db.flush() + from datetime import UTC, datetime + + publication = ProtocolRevision( + protocol_id=protocol.id, + revision=1, + graph={}, + experiment_snapshot={"design_spec": {"factors": []}}, + design_revision_id=design.id, + published_at=datetime.now(UTC), + ) + db.add(publication) + await db.flush() + protocol.published_revision_id = publication.id + cell = FactorialCell( + experiment_id=experiment.id, design_revision_id=design.id, cell_label="condition", factor_values={} + ) + db.add(cell) + await db.flush() + selected = await resolve_result_scope(db, experiment_id=experiment.id, protocol_id=protocol.id) + yield db, selected, cell, user + await transaction.rollback() + await engine.dispose() + + +async def test_current_attempt_history_and_rerun_rollups(results_db): + db, selected, cell, user = results_db + parent = FactorialReplicateResult(cell_id=cell.id, replicate_number=1, replicate_label="same-label") + db.add(parent) + await db.flush() + ids = [] + for status in ("completed", "failed"): + run = ProtocolRun( + protocol_id=selected.protocol_id, + owner_id=user.id, + status=status, + protocol_revision_id=selected.protocol_revision_id, + design_revision_id=selected.design_revision_id, + replicate_result_id=parent.id, + replicate_label=parent.replicate_label, + factor_values={}, + node_runs={"agent": {"status": status, "output_text": "x" * 100000}}, + conversation={"messages": [{"text": "y" * 100000}]}, + ) + db.add(run) + await db.flush() + ids.append(run.id) + page = await read_executions(db, selected) + assert len(page["items"]) == 1 + assert page["items"][0]["run_id"] == str(ids[-1]) + assert "node_runs" not in page["items"][0] + history = await read_attempts(db, selected, slot_id=parent.id, limit=1) + assert len(history["items"]) == 1 + assert history["next_cursor"] + assert "conversation" not in history["items"][0] + next_page = await read_attempts(db, selected, slot_id=parent.id, limit=1, cursor=history["next_cursor"]) + assert next_page["items"][0]["run_id"] == str(ids[0]) + detail = await read_attempt_detail(db, selected, ids[0]) + assert len(detail["node_runs"]["agent"]["output_text"]) == 100000 + refresh = await db.get(ResultRefresh, selected.key) + await rebuild_result_summary(db, selected, refresh.generation) + await db.refresh(refresh) + summary = await read_summary(db, selected) + assert summary["total"] == 1 + assert summary["counts"]["failed"] == 1 + assert summary["counts"]["completed"] == 0 + assert len((await read_cells(db, selected))["items"]) == 1 + + +async def test_opaque_previews_and_eligibility(results_db): + db, selected, cell, user = results_db + parent = FactorialReplicateResult(cell_id=cell.id, replicate_number=1, replicate_label="replicate") + db.add(parent) + await db.flush() + run = ProtocolRun( + protocol_id=selected.protocol_id, + owner_id=user.id, + status="completed", + protocol_revision_id=selected.protocol_revision_id, + design_revision_id=selected.design_revision_id, + replicate_result_id=parent.id, + replicate_label=parent.replicate_label, + factor_values={}, + node_runs={}, + attempt_result={ + "measurement": { + "observations": [ + { + "metric_id": "opaque", + "metric_name": "Report", + "status": "measured", + "value": "x" * 100000, + "producer": {"kind": "reported"}, + }, + { + "metric_id": "runtime", + "metric_name": "tokens", + "status": "measured", + "value": 100, + "producer": {"kind": "runtime"}, + }, + ] + } + }, + ) + db.add(run) + await db.flush() + preview = (await read_executions(db, selected))["items"][0]["result"] + assert preview["metrics"] == {"tokens": 100} + assert len(preview["observations"][0]["preview"]) == 512 + assert preview["observations"][0]["preview_truncated"] + assert (await read_attempt_detail(db, selected, run.id))["attempt_result"]["measurement"]["observations"][0][ + "value" + ] == "x" * 100000 From 3b388ee4f5b4f85996a1774132a629413bb08779 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 12 Oct 2026 17:22:43 -0700 Subject: [PATCH 34/63] Persist operational accounting and rebuild result summaries asynchronously --- .../protocol/DatasetExecutionSummary.tsx | 2 +- .../components/protocol/ExecutionBrowser.tsx | 13 +- .../protocol/ExperimentSidePanel.tsx | 2 +- frontend/src/components/protocol/RunsTab.tsx | 4 +- frontend/src/types/executionResults.ts | 1 + scripts/create_results_telemetry_index.py | 32 +++ scripts/rebuild_result_summaries.py | 21 ++ src/asaree/api/result_reads.py | 4 + src/asaree/config.py | 2 + .../4e5f60718293_result_usage_indexes.py | 57 +++++ src/asaree/services/dataset_row_csv.py | 2 +- src/asaree/services/design_generation.py | 78 ++++-- src/asaree/services/factorial_cells.py | 69 ++--- src/asaree/services/indexed_row_source.py | 136 ++++++++++ src/asaree/services/motoro_telemetry.py | 63 +++++ src/asaree/services/result_reads.py | 61 +++-- src/asaree/services/result_rollups.py | 61 ++++- src/asaree/services/result_usage.py | 238 ++++++++++++++++++ src/asaree/worker/settings.py | 8 +- src/asaree/worker/tasks.py | 7 + tests/test_indexed_row_source.py | 49 ++++ tests/test_result_usage.py | 61 +++++ 22 files changed, 879 insertions(+), 92 deletions(-) create mode 100644 scripts/create_results_telemetry_index.py create mode 100644 scripts/rebuild_result_summaries.py create mode 100644 src/asaree/migrations/versions/4e5f60718293_result_usage_indexes.py create mode 100644 src/asaree/services/indexed_row_source.py create mode 100644 src/asaree/services/motoro_telemetry.py create mode 100644 src/asaree/services/result_usage.py create mode 100644 tests/test_indexed_row_source.py create mode 100644 tests/test_result_usage.py diff --git a/frontend/src/components/protocol/DatasetExecutionSummary.tsx b/frontend/src/components/protocol/DatasetExecutionSummary.tsx index f60ec1d8..a50cc399 100644 --- a/frontend/src/components/protocol/DatasetExecutionSummary.tsx +++ b/frontend/src/components/protocol/DatasetExecutionSummary.tsx @@ -25,7 +25,7 @@ export function DatasetExecutionSummary({ graph, protocol, experimentId, cellCou queryFn: () => protocolsApi.getRevision(protocol!.id, protocol!.published_revision_id!), enabled: !!protocol?.published_revision_id, }) - const results = useExecutionSummary(experimentId, { protocol_id: protocol?.id }, !!protocol?.published_revision_id) + const results = useExecutionSummary(experimentId, { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined }, !!protocol?.published_revision_id) const error = graph && datasetRowTopologyError(graph) const publishedBinding = publication.data && rowBindingForNode(publication.data.graph) const summary = results.data diff --git a/frontend/src/components/protocol/ExecutionBrowser.tsx b/frontend/src/components/protocol/ExecutionBrowser.tsx index 047ce402..2496f729 100644 --- a/frontend/src/components/protocol/ExecutionBrowser.tsx +++ b/frontend/src/components/protocol/ExecutionBrowser.tsx @@ -82,7 +82,7 @@ export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSele const summary = useExecutionSummary(experimentId, scope) const [expanded, setExpanded] = useState>(() => new Set()) const [metricPreference, setMetricPreference] = useState() - const metric = (metricPreference ?? summary.data?.primary_metric) || undefined + const metric = mode === 'results' ? (metricPreference ?? summary.data?.primary_metric) || undefined : undefined const [directionPreference, setDirectionPreference] = useState() const direction = directionPreference ?? summary.data?.metric_directions[metric ?? ''] ?? 'maximize' const sortDirection = direction === 'minimize' ? 'minimize' : 'maximize' @@ -91,7 +91,14 @@ export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSele queryKey: [...executionKeys.root(experimentId), 'cells', scope, pager.cursor, metric, sortDirection], queryFn: ({ signal }) => experimentsApi.listExecutionCells(experimentId, scope, { cursor: pager.cursor, metric, direction: sortDirection }, signal), }) - useGenerationRefresh(cells.refetch, summary.data?.generation) + const previousGeneration = useRef(summary.data?.generation) + useEffect(() => { + if (previousGeneration.current !== summary.data?.generation) { + pager.reset() + void cells.refetch() + } + previousGeneration.current = summary.data?.generation + }, [summary.data?.generation]) const metricKeys = summary.data?.unit === 'replicate' ? Object.keys(summary.data.metric_types) : [] const refresh = () => { void summary.refetch(); void cells.refetch() } const cellBody =
@@ -120,7 +127,9 @@ export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSele {summary.isPending ?

Loading progress…

: summary.error && !summary.data ?

Progress unavailable. Execution records remain available below.

: !summary.data?.ready ?

Preparing the version summary. Execution records remain available below.

: <>

{summary.data.counts?.completed ?? 0}/{summary.data.expected ?? summary.data.total ?? 0} {summary.data.unit === 'row' ? 'row executions' : 'replicates'} complete · {summary.data.counts?.running ?? 0} running · {summary.data.counts?.pending ?? 0} queued · {summary.data.counts?.failed ?? 0} failed

+ {summary.data.all_attempt_usage?.cost_usd != null &&

All-attempt spend: {new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD' }).format(summary.data.all_attempt_usage.cost_usd)}

}

Usage reported for {summary.data.usage_complete_count ?? 0}/{summary.data.total ?? 0} current executions.{summary.data.refreshing && ' Updating summary…'}

+ {!!summary.data.usage_pending_count &&

Preparing usage for {summary.data.usage_pending_count} executions. Execution records and evidence remain available.

} } {summary.error && summary.data &&

Progress refresh failed; showing the last summary.

}
diff --git a/frontend/src/components/protocol/ExperimentSidePanel.tsx b/frontend/src/components/protocol/ExperimentSidePanel.tsx index 8b041e05..9318640f 100644 --- a/frontend/src/components/protocol/ExperimentSidePanel.tsx +++ b/frontend/src/components/protocol/ExperimentSidePanel.tsx @@ -276,7 +276,7 @@ export function ExperimentSidePanel({ /> ) : activeTab === 'results' ? ( >(() => new Set()) - const scope = { protocol_id: protocol?.id } + const scope = { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined } const summary = useExecutionSummary(experimentId, scope, open && !!protocol) const experiment = useQuery({ queryKey: ['experiments', experimentId], queryFn: () => experimentsApi.get(experimentId), enabled: open }) const run = useMutation({ @@ -233,7 +233,7 @@ export function RunsTab({ experimentId, designSpec, protocol, regenerationRequir onViewRowResult?: (id: string) => void; onSelectResult?: (selection: ResultsSelection) => void }) { const discrepancies = factorBindingDiscrepancies(designSpec, protocol?.graph) - const scope = { protocol_id: protocol?.id } + const scope = { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined } return <> {discrepancies.length > 0 &&

Canvas and generated runs disagree

{discrepancies.map(issue =>

{issue.nodeLabel} · {issue.factorName}: {issue.reason}

)}
} ; total?: number; cell_count?: number; replicate_count?: number source_row_count?: number | null; expected?: number | null; has_active?: boolean usage?: UsageSummary; usage_complete_count?: number; truncated_count?: number; evaluating_count?: number + all_attempt_usage?: UsageSummary; usage_pending_count?: number design_spec: DesignSpec | null metric_types: Record metric_aggregations: Record diff --git a/scripts/create_results_telemetry_index.py b/scripts/create_results_telemetry_index.py new file mode 100644 index 00000000..85d4a9c6 --- /dev/null +++ b/scripts/create_results_telemetry_index.py @@ -0,0 +1,32 @@ +"""Create the read-only telemetry adapter's core index without blocking writers. + +Run once during deployment after core migrations. Uses configured application +settings; never prints credentials. This belongs to core's separate database, +so it deliberately is not an ASAREE Alembic migration. +""" + +import asyncio + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from asaree.config import get_settings + + +async def main() -> None: + engine = create_async_engine(get_settings().database_url, isolation_level="AUTOCOMMIT") + try: + async with engine.connect() as connection: + await connection.execute( + text(""" + CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_asaree_agent_run_attribution + ON agent_runs(owner_id, (run_metadata ->> 'protocol_run_id'), id) + """) + ) + print("Operational telemetry index ready.") + finally: + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/rebuild_result_summaries.py b/scripts/rebuild_result_summaries.py new file mode 100644 index 00000000..404bd4d8 --- /dev/null +++ b/scripts/rebuild_result_summaries.py @@ -0,0 +1,21 @@ +"""Requeue rebuildable result summaries without rewriting measurement evidence. + +Run in the deployed application environment: python scripts/rebuild_result_summaries.py +The results worker performs the bounded backfill and rebuild asynchronously. +""" + +import asyncio + +from sqlalchemy import update + +from asaree.models.database import get_session +from asaree.models.result_read_model import ResultRefresh + + +async def main() -> None: + async with get_session() as db: + await db.execute(update(ResultRefresh).values(generation=ResultRefresh.generation + 1)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/asaree/api/result_reads.py b/src/asaree/api/result_reads.py index 4331d4fc..e5e98456 100644 --- a/src/asaree/api/result_reads.py +++ b/src/asaree/api/result_reads.py @@ -8,6 +8,8 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response from sqlalchemy import text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import TimeoutError as DatabaseTimeout from asaree.api.experiments import _get_owned_experiment from asaree.deps import CurrentUser, DbSession @@ -51,6 +53,8 @@ async def _read(operation: Callable[..., Awaitable[dict[str, Any]]], *args: Any, raise HTTPException(status_code=404, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc + except (DBAPIError, DatabaseTimeout) as exc: + raise HTTPException(status_code=503, detail="Results temporarily unavailable; retry shortly.") from exc @router.get("/summary", response_model=None) diff --git a/src/asaree/config.py b/src/asaree/config.py index 5363e4e5..c20d4d7b 100644 --- a/src/asaree/config.py +++ b/src/asaree/config.py @@ -61,6 +61,8 @@ class AsareeSettings(CoreSettings): # hashes are what matter; where the bytes physically live is a separate, # deliberately deferred decision. dataset_storage_dir: str = "./data/datasets" + result_export_dir: str = "./data/result-exports" + result_row_index_dir: str = "./data/result-row-indexes" # The one directory a user may browse and register OKF bundles under # (GET /okf/browse, POST /okf/bundles -- services/okf_bundles.py). Every diff --git a/src/asaree/migrations/versions/4e5f60718293_result_usage_indexes.py b/src/asaree/migrations/versions/4e5f60718293_result_usage_indexes.py new file mode 100644 index 00000000..392fb33a --- /dev/null +++ b/src/asaree/migrations/versions/4e5f60718293_result_usage_indexes.py @@ -0,0 +1,57 @@ +"""Index deferred accounting work and protect compact previews from large configs.""" + +from alembic import op +import sqlalchemy as sa + +revision = "4e5f60718293" +down_revision = "3d4e5f607182" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("CREATE INDEX ix_result_cell_metric_rank_desc ON result_cell_metrics " + "(scope_key, metric_key, value DESC, cell_id)") + op.create_index( + "ix_protocol_runs_usage_pending", + "protocol_runs", + ["created_at", "id"], + postgresql_where=sa.text( + "status IN ('completed','failed','cancelled','limit_reached') AND NOT is_test_run " + "AND target_node_id IS NULL AND (usage_summary IS NULL OR usage_summary->>'settled'='false')" + ), + ) + # A producer binding may contain a long rubric/config. List reads need only + # producer identity, never its full configuration document. + op.execute(""" + CREATE OR REPLACE FUNCTION asaree_result_preview(doc jsonb, nodes jsonb) RETURNS jsonb + LANGUAGE sql IMMUTABLE AS $$ + SELECT jsonb_build_object( + 'observations', COALESCE((SELECT jsonb_agg(jsonb_build_object( + 'metric_id', o->'metric_id', 'metric_name', o->'metric_name', + 'status', o->'status', 'value_type', o->'value_type', + 'preview', left(COALESCE(o->'value', 'null'::jsonb)::text, 512), + 'preview_truncated', length(COALESCE(o->'value', 'null'::jsonb)::text) > 512, + 'producer', jsonb_build_object('binding_id', o#>'{producer,binding_id}', + 'producer_id', o#>'{producer,producer_id}', 'kind', o#>'{producer,kind}', 'version', o#>'{producer,version}') + )) FROM jsonb_array_elements(CASE WHEN jsonb_typeof(doc#>'{measurement,observations}') = 'array' + THEN doc#>'{measurement,observations}' ELSE '[]'::jsonb END) o), '[]'::jsonb), + 'metrics', COALESCE((SELECT jsonb_object_agg(o->>'metric_id', CASE + WHEN jsonb_typeof(o->'value') = 'boolean' THEN to_jsonb(CASE WHEN (o->>'value')::boolean THEN 1 ELSE 0 END) + ELSE o->'value' END) + FROM jsonb_array_elements(CASE WHEN jsonb_typeof(doc#>'{measurement,observations}') = 'array' + THEN doc#>'{measurement,observations}' ELSE '[]'::jsonb END) o + WHERE o->>'status' = 'measured' AND o#>>'{producer,producer_id}' IN ('asaree.runtime','asaree.node_runtime') + AND o->>'metric_id' IS NOT NULL AND jsonb_typeof(o->'value') IN ('number', 'boolean')), '{}'::jsonb), + 'truncated', EXISTS (SELECT 1 FROM jsonb_each(CASE WHEN jsonb_typeof(nodes)='object' + THEN nodes ELSE '{}'::jsonb END) n WHERE jsonb_typeof(n.value->'truncation')='object'), + 'evaluation_state', doc->'evaluation_state' + ); + $$; + """) + + +def downgrade() -> None: + op.drop_index("ix_result_cell_metric_rank_desc", table_name="result_cell_metrics") + op.drop_index("ix_protocol_runs_usage_pending", table_name="protocol_runs") + # The bounded producer identity is backward compatible with the original preview contract. diff --git a/src/asaree/services/dataset_row_csv.py b/src/asaree/services/dataset_row_csv.py index 5b37377b..4677e91c 100644 --- a/src/asaree/services/dataset_row_csv.py +++ b/src/asaree/services/dataset_row_csv.py @@ -25,7 +25,7 @@ class RowSource: dataset_id: str raw_sha256: str columns: tuple[str, ...] - rows: tuple[tuple[str, ...], ...] + rows: Sequence[tuple[str, ...]] _SHA256_RE = re.compile(r"[0-9a-f]{64}\Z") diff --git a/src/asaree/services/design_generation.py b/src/asaree/services/design_generation.py index f0e473f7..9a3d8ebe 100644 --- a/src/asaree/services/design_generation.py +++ b/src/asaree/services/design_generation.py @@ -68,9 +68,7 @@ def _validate_factors(factors: list[dict[str, Any]]) -> None: or len(level_labels) != len(levels) or any(not isinstance(label, str) or not label.strip() for label in level_labels) ): - raise DesignValidationError( - f"factor {name!r} level_labels must contain one non-empty string per level" - ) + raise DesignValidationError(f"factor {name!r} level_labels must contain one non-empty string per level") normalized_labels = [label.strip() for label in level_labels] if len(set(normalized_labels)) != len(normalized_labels): raise DesignValidationError(f"factor {name!r} level_labels must be unique") @@ -166,9 +164,7 @@ def _factor_level_label(factor: dict[str, Any], value: Any) -> str | None: ( label.strip() for level, label in zip(levels, labels, strict=True) - if isinstance(label, str) - and label.strip() - and json.dumps(level, sort_keys=True, default=str) == value_key + if isinstance(label, str) and label.strip() and json.dumps(level, sort_keys=True, default=str) == value_key ), None, ) @@ -291,20 +287,36 @@ async def get_design_impact( """Compare the declared factorial matrix to its materialized revision.""" material = material_design_spec(design_spec) incomplete_skills = any( - (factor.get("level_type") in { - "sub_agent_selection", "script_selection", "tool_selection", "skill_selection", - "dataset_selection", "knowledge_selection" + ( + factor.get("level_type") + in { + "sub_agent_selection", + "script_selection", + "tool_selection", + "skill_selection", + "dataset_selection", + "knowledge_selection", } - and len(factor.get("levels") or []) < 2) - or (factor.get("level_type") in { - "sub_agent_toggle", "script_toggle", "tool_toggle", "skill_toggle", "dataset_toggle", "knowledge_toggle" - } - and not any(factor.get("levels") or [])) + and len(factor.get("levels") or []) < 2 + ) + or ( + factor.get("level_type") + in { + "sub_agent_toggle", + "script_toggle", + "tool_toggle", + "skill_toggle", + "dataset_toggle", + "knowledge_toggle", + } + and not any(factor.get("levels") or []) + ) for factor in material["factors"] ) planned = ( _planned_replicates(material["factors"], material["replicates"]) - if material["factors"] and not incomplete_skills else [] + if material["factors"] and not incomplete_skills + else [] ) planned_labels = {label for label, _, _ in planned} planned_cell_keys = {_cell_key(combo, label) for label, combo, _ in planned} @@ -325,7 +337,9 @@ async def get_design_impact( retained_replicate_count=0, removed_replicate_count=0, ) - current_replicates = await list_replicates(db, experiment_id=experiment_id, revision_id=current.id) + current_replicates = await list_replicates( + db, experiment_id=experiment_id, revision_id=current.id, include_evidence=False + ) current_labels = {replicate.replicate_label for replicate in current_replicates} current_cell_keys = { _cell_key(replicate.factor_values, replicate.replicate_label) for replicate in current_replicates @@ -434,11 +448,20 @@ async def generate_design_cells( except ValueError as exc: raise DesignValidationError(str(exc)) from exc if any( - factor.get("level_type") in { - "sub_agent_selection", "sub_agent_toggle", - "script_selection", "script_toggle", "tool_selection", "tool_toggle", "skill_selection", "skill_toggle", - "dataset_selection", "dataset_toggle", - "knowledge_selection", "knowledge_toggle", + factor.get("level_type") + in { + "sub_agent_selection", + "sub_agent_toggle", + "script_selection", + "script_toggle", + "tool_selection", + "tool_toggle", + "skill_selection", + "skill_toggle", + "dataset_selection", + "dataset_toggle", + "knowledge_selection", + "knowledge_toggle", } for factor in factors ): @@ -508,11 +531,14 @@ async def generate_design_cells( and coordination_strategy_slug(current.design_spec) != coordination_strategy_slug(design_spec) ) - published_design_changes = current is not None and ( - current.design_spec != design_spec or set(existing) != planned_labels or strategy_changed - ) and await db.scalar( - select(ProtocolRevision.id).where(ProtocolRevision.design_revision_id == current.id).limit(1) - ) is not None + published_design_changes = ( + current is not None + and (current.design_spec != design_spec or set(existing) != planned_labels or strategy_changed) + and await db.scalar( + select(ProtocolRevision.id).where(ProtocolRevision.design_revision_id == current.id).limit(1) + ) + is not None + ) if current is not None and not dropped and not strategy_changed and not published_design_changes: # Keep the revision and merge into its rows -- factor_values is # re-merged because a level's *value* can change without changing its diff --git a/src/asaree/services/factorial_cells.py b/src/asaree/services/factorial_cells.py index 261eba9a..5cadcc23 100644 --- a/src/asaree/services/factorial_cells.py +++ b/src/asaree/services/factorial_cells.py @@ -15,7 +15,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import selectinload +from sqlalchemy.orm import load_only, selectinload from asaree.models.factorial_cell import FactorialCell from asaree.models.factorial_replicate_result import FactorialReplicateResult @@ -108,24 +108,38 @@ async def get_replicate( async def list_replicates( - db: AsyncSession, *, experiment_id: uuid.UUID, revision_id: uuid.UUID | None = None + db: AsyncSession, + *, + experiment_id: uuid.UUID, + revision_id: uuid.UUID | None = None, + replicate_ids: set[uuid.UUID] | None = None, + include_evidence: bool = True, ) -> Sequence[FactorialReplicateResult]: resolved = await _resolve_revision_id(db, experiment_id, revision_id) if resolved is None: return [] - return ( - ( - await db.execute( - select(FactorialReplicateResult) - .join(FactorialReplicateResult.cell) - .options(selectinload(FactorialReplicateResult.cell)) - .where( - FactorialCell.experiment_id == experiment_id, - FactorialCell.design_revision_id == resolved, - ) - .order_by(FactorialCell.cell_label, FactorialReplicateResult.replicate_number) + query = ( + select(FactorialReplicateResult) + .join(FactorialReplicateResult.cell) + .options( + selectinload(FactorialReplicateResult.cell), + ) + .where(FactorialCell.experiment_id == experiment_id, FactorialCell.design_revision_id == resolved) + ) + if replicate_ids is not None: + query = query.where(FactorialReplicateResult.id.in_(replicate_ids)) + if not include_evidence: + query = query.options( + load_only( + FactorialReplicateResult.id, + FactorialReplicateResult.cell_id, + FactorialReplicateResult.replicate_label, + FactorialReplicateResult.replicate_number, + FactorialReplicateResult.run_id, ) ) + return ( + (await db.execute(query.order_by(FactorialCell.cell_label, FactorialReplicateResult.replicate_number))) .scalars() .all() ) @@ -147,9 +161,7 @@ async def upsert_replicate( cell_label, replicate_number = split_replicate_label(replicate_label) supplied_factors = { - key: value - for key, value in (fields.get("factor_values") or {}).items() - if key.lower() not in _NON_FACTOR_KEYS + key: value for key, value in (fields.get("factor_values") or {}).items() if key.lower() not in _NON_FACTOR_KEYS } cell = None if supplied_factors: @@ -165,14 +177,13 @@ async def upsert_replicate( ) ).scalar_one_or_none() if cell is None: - cell = await get_factorial_cell( - db, experiment_id=experiment_id, cell_label=cell_label, revision_id=revision_id - ) + cell = await get_factorial_cell(db, experiment_id=experiment_id, cell_label=cell_label, revision_id=revision_id) if cell is None: from asaree.models.protocol_revision import ProtocolRevision - if await db.scalar(select(ProtocolRevision.id).where( - ProtocolRevision.design_revision_id == revision_id - ).limit(1)): + + if await db.scalar( + select(ProtocolRevision.id).where(ProtocolRevision.design_revision_id == revision_id).limit(1) + ): raise ValueError("Generate a draft design before adding cells to a published experiment version.") cell = FactorialCell( experiment_id=experiment_id, @@ -186,9 +197,10 @@ async def upsert_replicate( merged_factors = dict(cell.factor_values or {}) merged_factors.update(supplied_factors) from asaree.models.protocol_revision import ProtocolRevision - if merged_factors != (cell.factor_values or {}) and await db.scalar(select(ProtocolRevision.id).where( - ProtocolRevision.design_revision_id == revision_id - ).limit(1)): + + if merged_factors != (cell.factor_values or {}) and await db.scalar( + select(ProtocolRevision.id).where(ProtocolRevision.design_revision_id == revision_id).limit(1) + ): raise ValueError("Published experiment cell factors are immutable. Generate a draft design first.") cell.factor_values = merged_factors @@ -197,9 +209,10 @@ async def upsert_replicate( ) if replicate is None: from asaree.models.protocol_revision import ProtocolRevision - if await db.scalar(select(ProtocolRevision.id).where( - ProtocolRevision.design_revision_id == revision_id - ).limit(1)): + + if await db.scalar( + select(ProtocolRevision.id).where(ProtocolRevision.design_revision_id == revision_id).limit(1) + ): raise ValueError("Generate a draft design before adding replicates to a published experiment version.") replicate = FactorialReplicateResult( cell=cell, diff --git a/src/asaree/services/indexed_row_source.py b/src/asaree/services/indexed_row_source.py new file mode 100644 index 00000000..73221818 --- /dev/null +++ b/src/asaree/services/indexed_row_source.py @@ -0,0 +1,136 @@ +"""Verified original rows with bounded memory and random access for batch planning.""" + +from __future__ import annotations + +import csv +import fcntl +import hashlib +import json +import re +import sqlite3 +import tempfile +from collections.abc import Sequence +from contextlib import closing +from pathlib import Path +from typing import BinaryIO, overload +from urllib.parse import quote + +from asaree.services.dataset_row_csv import DatasetRowCsvError, RowSource + + +class IndexedRows(Sequence[tuple[str, ...]]): + def __init__(self, path: Path, count: int): + self.path = path + self._length = count + + def __len__(self) -> int: + return self._length + + @overload + def __getitem__(self, index: int) -> tuple[str, ...]: ... + + @overload + def __getitem__(self, index: slice) -> Sequence[tuple[str, ...]]: ... + + def __getitem__(self, index: int | slice) -> tuple[str, ...] | Sequence[tuple[str, ...]]: + if isinstance(index, slice): + return tuple(self[item] for item in range(*index.indices(self._length))) + if index < 0: + index += self._length + if not 0 <= index < self._length: + raise IndexError(index) + with closing(sqlite3.connect(f"file:{quote(str(self.path))}?mode=ro", uri=True)) as connection: + record = connection.execute("SELECT data FROM rows WHERE row_index=?", (index,)).fetchone() + if record is None: + raise DatasetRowCsvError("source_unavailable", "verified row index is incomplete") + return tuple(json.loads(record[0])) + + +def read_indexed_row_source( + *, + dataset_id: str, + raw_path: str | None, + raw_sha256: str | None, + cache_root: Path, +) -> RowSource: + if raw_path is None or not isinstance(raw_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", raw_sha256): + raise DatasetRowCsvError("source_unavailable", "registered original CSV or SHA-256 is unavailable") + cache_root.mkdir(parents=True, exist_ok=True, mode=0o700) + cache_path = cache_root / f"{raw_sha256}.sqlite" + # Serialize cache creation across workers; cached rows are immutable after + # the atomic rename. Every job still verifies the current registered bytes. + with (cache_root / f"{raw_sha256}.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + try: + digest = hashlib.sha256() + with Path(raw_path).open("rb") as source, tempfile.TemporaryFile() as verified: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + if not cache_path.exists(): + verified.write(chunk) + if digest.hexdigest() != raw_sha256: + raise DatasetRowCsvError( + "source_hash_mismatch", "original CSV does not match its registered SHA-256" + ) + if not cache_path.exists(): + verified.seek(0) + _build_index(verified, cache_path, raw_sha256) + with closing(sqlite3.connect(f"file:{quote(str(cache_path))}?mode=ro", uri=True)) as connection: + metadata = dict(connection.execute("SELECT key,value FROM metadata")) + if metadata["sha256"] != raw_sha256: + raise DatasetRowCsvError("source_hash_mismatch", "verified index identity does not match original CSV") + return RowSource( + dataset_id, + raw_sha256, + tuple(json.loads(metadata["columns"])), + IndexedRows(cache_path, int(metadata["count"])), + ) + except (OSError, sqlite3.Error, KeyError) as exc: + raise DatasetRowCsvError("source_unavailable", "verified original row index cannot be read") from exc + + +def _build_index(verified: BinaryIO, destination: Path, sha256: str) -> None: + import io + + # The verified temporary copy makes CSV parsing and hashing refer to exactly + # the same bytes even if the external original changes during preparation. + wrapper = io.TextIOWrapper(verified, encoding="utf-8-sig", newline="") + temporary = destination.with_suffix(".building") + try: + temporary.unlink(missing_ok=True) + with closing(sqlite3.connect(temporary)) as connection, connection: + connection.execute("CREATE TABLE rows(row_index INTEGER PRIMARY KEY, data TEXT NOT NULL)") + connection.execute("CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL)") + records = (record for record in csv.reader(wrapper, strict=True) if record) + header = next(records, None) + if header is None or any(not column.strip() for column in header) or len(set(header)) != len(header): + raise DatasetRowCsvError("invalid_csv", "CSV headers must be distinct and nonempty") + batch = [] + count = 0 + for record in records: + if len(record) != len(header): + raise DatasetRowCsvError( + "invalid_csv", "CSV record has a different number of fields than its header" + ) + batch.append((count, json.dumps(record, ensure_ascii=False))) + count += 1 + if len(batch) == 1000: + connection.executemany("INSERT INTO rows VALUES (?,?)", batch) + batch = [] + if batch: + connection.executemany("INSERT INTO rows VALUES (?,?)", batch) + connection.executemany( + "INSERT INTO metadata VALUES (?,?)", + [ + ("sha256", sha256), + ("columns", json.dumps(header)), + ("count", str(count)), + ], + ) + temporary.chmod(0o600) + temporary.replace(destination) + except (UnicodeDecodeError, csv.Error) as exc: + raise DatasetRowCsvError("invalid_csv", "CSV syntax or encoding is malformed") from exc + finally: + wrapper.detach() + temporary.unlink(missing_ok=True) diff --git a/src/asaree/services/motoro_telemetry.py b/src/asaree/services/motoro_telemetry.py new file mode 100644 index 00000000..2470b861 --- /dev/null +++ b/src/asaree/services/motoro_telemetry.py @@ -0,0 +1,63 @@ +"""Read-only telemetry adapter at the Motoro storage seam. + +Motoro's public runner reads eagerly hydrate Agent and RunStep relationships. +Operational accounting needs only scalar columns. Keep that storage knowledge +here, use its public audited system_session, and never join product/core tables. +No transcript, input, output, or step documents cross this seam. +""" + +from __future__ import annotations + +import uuid +from typing import Any + +from motoro.models.database import system_session +from motoro.models.run import AgentRun +from sqlalchemy import false, or_, select + +PAGE_SIZE = 200 + + +def usage_query( + owner_id: uuid.UUID, + protocol_run_ids: list[uuid.UUID], + referenced_ids: list[uuid.UUID], + after_id: uuid.UUID | None = None, +) -> Any: + if len(protocol_run_ids) > PAGE_SIZE or len(referenced_ids) > PAGE_SIZE: + raise ValueError("telemetry_batch_too_large") + matches = [] + if protocol_run_ids: + matches.append( + AgentRun.run_metadata["protocol_run_id"].astext.in_([str(identifier) for identifier in protocol_run_ids]) + ) + if referenced_ids: + matches.append(AgentRun.id.in_(referenced_ids)) + statement = select( + AgentRun.id, + AgentRun.token_usage, + AgentRun.cost_estimate, + AgentRun.run_metadata["protocol_run_id"].astext.label("protocol_run_id"), + AgentRun.run_metadata["node_id"].astext.label("node_id"), + ) + statement = statement.where(AgentRun.owner_id == owner_id, or_(*matches) if matches else false()) + if after_id: + statement = statement.where(AgentRun.id > after_id) + return statement.order_by(AgentRun.id).limit(PAGE_SIZE) + + +async def read_usage_page( + owner_id: uuid.UUID, + protocol_run_ids: list[uuid.UUID], + referenced_ids: list[uuid.UUID], + after_id: uuid.UUID | None = None, +) -> list[dict[str, Any]]: + async with system_session(reason="asaree.results: capture bounded operational telemetry") as db: + return [ + dict(row) + for row in ( + await db.execute( + usage_query(owner_id, protocol_run_ids, referenced_ids, after_id), + ) + ).mappings() + ] diff --git a/src/asaree/services/result_reads.py b/src/asaree/services/result_reads.py index 99c934fe..ce8a4b66 100644 --- a/src/asaree/services/result_reads.py +++ b/src/asaree/services/result_reads.py @@ -381,44 +381,57 @@ async def read_cells( FactorialCell.design_revision_id == scope.design_revision_id, ) ) + rows = [] if metric: - statement = statement.outerjoin( + refresh = await db.get(ResultRefresh, scope.key) + generation = refresh.processed_generation if refresh else 0 + if position is not None and (len(position) != 4 or position[3] != generation): + raise ValueError("stale_sort_cursor") + ranked = statement.join( ResultCellMetric, and_( ResultCellMetric.scope_key == scope.key, ResultCellMetric.cell_id == FactorialCell.id, ResultCellMetric.metric_key == metric, + ResultCellMetric.generation == generation, + ResultCellMetric.value.is_not(None), ), ) value = ResultCellMetric.value - # A measured value ranks before an unavailable value. Negation makes - # the cursor order ascending for both directions, with UUID tie-break. - order_value = -value if direction == "maximize" else value - missing = case((value.is_(None), 1), else_=0) - if position is not None: - if len(position) != 3: - raise ValueError("invalid_cursor") - m, v, identifier = position - statement = statement.where( - or_( - missing > m, - and_( - missing == m, - or_(order_value > v, and_(order_value == v, FactorialCell.id > uuid.UUID(identifier))) - if v is not None - else FactorialCell.id > uuid.UUID(identifier), - ), + if position is None or position[0] == 0: + if position is not None: + v, identifier = position[1], uuid.UUID(position[2]) + ranked = ranked.where( + or_( + value < v if direction == "maximize" else value > v, + and_(value == v, FactorialCell.id > identifier), + ) ) + ranked = ranked.add_columns(literal(0).label("missing"), value.label("order_value")) + ranked = ranked.order_by(value.desc() if direction == "maximize" else value.asc(), FactorialCell.id) + rows = list((await db.execute(ranked.limit(page_size(limit) + 1))).mappings()) + if len(rows) <= limit: + missing = statement.where( + ~select(ResultCellMetric.cell_id) + .where( + ResultCellMetric.scope_key == scope.key, + ResultCellMetric.metric_key == metric, + ResultCellMetric.generation == generation, + ResultCellMetric.cell_id == FactorialCell.id, + ResultCellMetric.value.is_not(None), + ) + .exists() ) - statement = statement.add_columns(missing.label("missing"), order_value.label("order_value")) - statement = statement.order_by(missing, order_value.asc().nulls_last(), FactorialCell.id) + if position is not None and position[0] == 1: + missing = missing.where(FactorialCell.id > uuid.UUID(position[2])) + missing = missing.add_columns(literal(1).label("missing"), literal(None).label("order_value")) + rows += list((await db.execute(missing.order_by(FactorialCell.id).limit(limit + 1 - len(rows)))).mappings()) else: if position is not None: if len(position) != 1: raise ValueError("invalid_cursor") statement = statement.where(FactorialCell.id > uuid.UUID(position[0])) - statement = statement.order_by(FactorialCell.id) - rows = (await db.execute(statement.limit(page_size(limit) + 1))).mappings().all() + rows = list((await db.execute(statement.order_by(FactorialCell.id).limit(page_size(limit) + 1))).mappings()) items = [ { "cell_id": str(row["id"]), @@ -431,7 +444,9 @@ async def read_cells( next_position = None if len(rows) > limit: last = rows[limit - 1] - next_position = [last["missing"], last["order_value"], str(last["id"])] if metric else [str(last["id"])] + next_position = ( + [last["missing"], last["order_value"], str(last["id"]), generation] if metric else [str(last["id"])] + ) return { "items": items, "next_cursor": encode_cursor(scope, "cells", next_position, filters) if next_position else None, diff --git a/src/asaree/services/result_rollups.py b/src/asaree/services/result_rollups.py index 83829fee..373b3819 100644 --- a/src/asaree/services/result_rollups.py +++ b/src/asaree/services/result_rollups.py @@ -2,11 +2,12 @@ from __future__ import annotations +import asyncio import logging from datetime import UTC, datetime from typing import Any -from sqlalchemy import Float, and_, cast, delete, func, literal, select, update +from sqlalchemy import Float, and_, case, cast, delete, func, literal, select, update from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession @@ -14,8 +15,11 @@ from asaree.models.factorial_cell import FactorialCell from asaree.models.factorial_replicate_result import FactorialReplicateResult from asaree.models.protocol import Protocol +from asaree.models.protocol_run import ProtocolRun from asaree.models.result_read_model import ResultCellMetric, ResultCellRollup, ResultRefresh -from asaree.services.experiment_run_results import _declared_metric_aggregations, _source_row_count +from asaree.services.experiment_run_results import _declared_metric_aggregations +from asaree.services.indexed_row_source import read_indexed_row_source +from asaree.services.metrics import normalize_metrics from asaree.services.result_reads import ResultScope, resolve_result_scope, slot_query logger = logging.getLogger(__name__) @@ -55,9 +59,15 @@ def rollup_query(scope: ResultScope) -> Any: stats_columns.append(func.sum(slots.c.usage[key].as_float()).label(key)) stats = select(*stats_columns).group_by(slots.c.cell_id).cte("stats") metrics = func.jsonb_each(slots.c.result["metrics"]).table_valued("key", "value").lateral("metrics") + identities = { + metric["id"]: metric.get("catalogKey", metric["name"]) + for metric in normalize_metrics((scope.design_spec or {}).get("metrics")) + if metric.get("catalogKey") and metric.get("kind") != "custom" + } + metric_key = case(identities, value=metrics.c.key, else_=None) if identities else literal(None) metric_query = select( slots.c.cell_id, - metrics.c.key, + metric_key.label("key"), func.avg(cast(metrics.c.value, Float)).label("mean"), func.sum(cast(metrics.c.value, Float)).label("sum"), func.count().label("count"), @@ -66,10 +76,11 @@ def rollup_query(scope: ResultScope) -> Any: metric_query.select_from(slots) .join(metrics, literal(True)) .where( + metric_key.is_not(None), slots.c.status == "completed", slots.c.result["truncated"].as_boolean().is_(False), ) - .group_by(slots.c.cell_id, metrics.c.key) + .group_by(slots.c.cell_id, metric_key) .cte("metric_values") ) metric_stats = ( @@ -163,9 +174,23 @@ async def rebuild_result_summary(db: AsyncSession, scope: ResultScope, generatio from asaree.models.dataset import RegisteredDataset dataset = await db.get(RegisteredDataset, uuid.UUID(dataset_id)) if dataset_id else None - row_count = await _source_row_count( - db, dataset_id=dataset.id if dataset else None, raw_sha256=dataset.raw_sha256 if dataset else None - ) + if dataset is not None: + from pathlib import Path + + from asaree.config import get_settings + from asaree.services.dataset_row_csv import DatasetRowCsvError + + try: + indexed = await asyncio.to_thread( + read_indexed_row_source, + dataset_id=str(dataset.id), + raw_path=dataset.raw_path, + raw_sha256=dataset.raw_sha256, + cache_root=Path(get_settings().result_row_index_dir), + ) + row_count = len(indexed.rows) + except DatasetRowCsvError: + row_count = None summary = { "counts": {status: totals[status] for status in STATUSES}, "total": totals["total"], @@ -185,6 +210,22 @@ async def rebuild_result_summary(db: AsyncSession, scope: ResultScope, generatio "usage_pending_count": totals["usage_pending"], "as_of": datetime.now(UTC).isoformat(), } + all_usage = ( + ( + await db.execute( + select(*[func.sum(ProtocolRun.usage_summary[key].as_float()).label(key) for key in usage]).where( + ProtocolRun.protocol_id == scope.protocol_id, + ProtocolRun.protocol_revision_id == scope.protocol_revision_id, + ProtocolRun.design_revision_id == scope.design_revision_id, + ProtocolRun.is_test_run.is_(False), + ProtocolRun.target_node_id.is_(None), + ) + ) + ) + .mappings() + .one() + ) + summary["all_attempt_usage"] = dict(all_usage) # generation may have advanced during this rebuild. Preserve the new dirty # generation; the next delivery rebuilds it. Cell/overview publication is one transaction. await db.execute( @@ -278,3 +319,9 @@ async def refresh_result_summaries(_ctx: dict[str, Any]) -> None: await rebuild_result_summary(db, scope, refresh.generation) except Exception: logger.exception("result_summary_refresh_failed", extra={"scope_key": key}) + async with get_session() as failed_db: + await failed_db.execute( + update(ResultRefresh) + .where(ResultRefresh.scope_key == key) + .values(error="Summary refresh failed; retrying in the background.", updated_at=func.now()) + ) diff --git a/src/asaree/services/result_usage.py b/src/asaree/services/result_usage.py new file mode 100644 index 00000000..ddfcfab4 --- /dev/null +++ b/src/asaree/services/result_usage.py @@ -0,0 +1,238 @@ +"""Resumable operational accounting and preview backfill, outside HTTP reads.""" + +from __future__ import annotations + +import logging +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from math import isfinite +from typing import Any + +from sqlalchemy import and_, cast, func, literal, or_, select, update +from sqlalchemy.dialects.postgresql import JSONB + +from asaree.models.database import get_session +from asaree.models.protocol import Protocol +from asaree.models.protocol_run import ProtocolRun +from asaree.services.motoro_telemetry import PAGE_SIZE, read_usage_page + +logger = logging.getLogger(__name__) +BATCH_SIZE = 50 +TERMINAL = ("completed", "failed", "cancelled", "limit_reached") + + +@dataclass +class UsageAccumulator: + totals: dict[str, float | int | None] = field( + default_factory=lambda: { + "input_tokens": None, + "output_tokens": None, + "total_tokens": None, + "cost_usd": None, + } + ) + calls: int = 0 + reported_tokens: int = 0 + reported_cost: int = 0 + + def add(self, row: dict[str, Any]) -> None: + usage = provider_usage(row.get("token_usage"), row.get("cost_estimate")) + self.calls += 1 + self.reported_tokens += usage["total_tokens"] is not None + self.reported_cost += usage["cost_usd"] is not None + for key, value in usage.items(): + if value is not None: + self.totals[key] = (self.totals[key] or 0) + value + + def finish(self, started_at: datetime | None, completed_at: datetime | None) -> dict[str, Any]: + return { + **self.totals, + "agent_run_count": self.calls, + "reported_usage_count": self.reported_tokens, + "reported_cost_count": self.reported_cost, + "complete": self.reported_tokens == self.calls and self.reported_cost == self.calls, + "settled": True, + "captured_at": datetime.now(UTC).isoformat(), + "duration_seconds": max(0, (completed_at - started_at).total_seconds()) + if started_at and completed_at + else None, + } + + +def provider_usage(raw: Any, cost: Any) -> dict[str, float | int | None]: + """Preserve unreported provider fields, including Motoro's ambiguous zero defaults.""" + source = raw if isinstance(raw, dict) else {} + result: dict[str, float | int | None] = {} + for key, aliases in { + "input_tokens": ("input_tokens", "prompt_tokens", "input_token_count"), + "output_tokens": ("output_tokens", "completion_tokens", "output_token_count"), + "total_tokens": ("total_tokens", "total_token_count"), + }.items(): + values = [source.get(alias) for alias in aliases] + result[key] = next( + ( + int(value) + for value in values + if isinstance(value, int | float | Decimal) + and not isinstance(value, bool) + and isfinite(float(value)) + and value > 0 + ), + None, + ) + if result["total_tokens"] is None and result["input_tokens"] is not None and result["output_tokens"] is not None: + result["total_tokens"] = result["input_tokens"] + result["output_tokens"] + result["cost_usd"] = ( + float(cost) if isinstance(cost, int | float | Decimal) and isfinite(float(cost)) and cost > 0 else None + ) + return result + + +def summarize_usage( + rows: list[dict[str, Any]], + *, + started_at: datetime | None, + completed_at: datetime | None, +) -> dict[str, Any]: + unique = {row["id"]: row for row in rows} + accumulator = UsageAccumulator() + for row in unique.values(): + accumulator.add(row) + return accumulator.finish(started_at, completed_at) + + +async def capture_result_usage(_ctx: dict[str, Any], *, only_run_id: uuid.UUID | None = None) -> None: + """Select bounded durable work, capture all causally attributed runs, then publish. + + A null/unfinished usage document is the durable work item. Retrying replaces + the projection; it never edits immutable measurement/evidence documents. + Advisory locking prevents duplicate collectors across worker processes. + """ + async with get_session() as db: + locked = await db.scalar(select(func.pg_try_advisory_xact_lock(73912841))) + if not locked: + return + now = datetime.now(UTC) + node = func.jsonb_each(ProtocolRun.node_runs).table_valued("key", "value").lateral() + runs = ( + ( + await db.execute( + select( + ProtocolRun.id, + ProtocolRun.owner_id, + ProtocolRun.started_at, + ProtocolRun.completed_at, + ProtocolRun.attempt_result["task_completed_at"].astext.label("task_completed_at"), + ProtocolRun.usage_summary, + # Extract only final pointers, not complete output/trace documents. + select(func.jsonb_agg(func.jsonb_build_object("run_id", cast(node.c.value, JSONB)["run_id"]))) + .select_from(node) + .correlate(ProtocolRun) + .scalar_subquery() + .label("pointers"), + ) + .join(Protocol, Protocol.id == ProtocolRun.protocol_id) + .where( + Protocol.experiment_id.is_not(None), + ProtocolRun.is_test_run.is_(False), + ProtocolRun.target_node_id.is_(None), + ProtocolRun.status.in_(TERMINAL), + ProtocolRun.id == only_run_id if only_run_id else literal(True), + or_( + ProtocolRun.usage_summary.is_(None), + and_( + ProtocolRun.usage_summary["settled"].as_boolean().is_not(True), + ProtocolRun.usage_summary["retry_at"].astext <= now.isoformat(), + ), + ), + ) + .order_by(ProtocolRun.created_at, ProtocolRun.id) + .limit(BATCH_SIZE) + ) + ) + .mappings() + .all() + ) + if not runs: + return + owners: defaultdict[uuid.UUID, list[Any]] = defaultdict(list) + for run in runs: + owners[run["owner_id"]].append(run) + for owner_id, owned_runs in owners.items(): + try: + async with db.begin_nested(): + identifiers = [run["id"] for run in owned_runs] + references: defaultdict[uuid.UUID, set[uuid.UUID]] = defaultdict(set) + for run in owned_runs: + for pointer in run["pointers"] or []: + try: + references[uuid.UUID(pointer["run_id"])].add(run["id"]) + except (ValueError, TypeError, KeyError): + continue + # Pointers are compatibility fallback for runs predating causal metadata. + # Attribution queries also include rejected critics/delegated work. + accumulators: defaultdict[uuid.UUID, UsageAccumulator] = defaultdict(UsageAccumulator) + identifier_set = set(identifiers) + seen_references: set[uuid.UUID] = set() + reference_ids = list(references) + reference_batches = [ + reference_ids[index : index + PAGE_SIZE] for index in range(0, len(reference_ids), PAGE_SIZE) + ] + for attribution_ids, fallback in [(identifiers, []), *[([], batch) for batch in reference_batches]]: + after = None + while True: + page = await read_usage_page(owner_id, attribution_ids, fallback, after) + for row in page: + if fallback and row["id"] in seen_references: + continue + if row["id"] in references: + seen_references.add(row["id"]) + attributed = row.get("protocol_run_id") + try: + subjects = ( + {uuid.UUID(attributed)} if attributed else references.get(row["id"], set()) + ) + except ValueError: + subjects = references.get(row["id"], set()) + for subject in subjects & identifier_set: + accumulators[subject].add(row) + if len(page) < PAGE_SIZE: + break + after = page[-1]["id"] + for run in owned_runs: + task_end = run["task_completed_at"] + completed = datetime.fromisoformat(task_end) if task_end else run["completed_at"] + summary = accumulators[run["id"]].finish(run["started_at"], completed) + await db.execute( + update(ProtocolRun) + .where(ProtocolRun.id == run["id"]) + .values( + usage_summary=summary, + updated_at=ProtocolRun.updated_at, + result_summary=func.asaree_result_preview( + ProtocolRun.attempt_result, ProtocolRun.node_runs + ), + ) + ) + except Exception: + logger.exception("result_usage_capture_failed", extra={"batch_size": len(owned_runs)}) + for run in owned_runs: + tries = min(10, int((run["usage_summary"] or {}).get("tries", 0)) + 1) + await db.execute( + update(ProtocolRun) + .where(ProtocolRun.id == run["id"]) + .values( + usage_summary={ + "settled": False, + "tries": tries, + "retry_at": (now + timedelta(seconds=min(300, 5 * 2**tries))).isoformat(), + }, + updated_at=ProtocolRun.updated_at, + result_summary=func.asaree_result_preview( + ProtocolRun.attempt_result, ProtocolRun.node_runs + ), + ) + ) diff --git a/src/asaree/worker/settings.py b/src/asaree/worker/settings.py index dba7bd7d..b1dc378a 100644 --- a/src/asaree/worker/settings.py +++ b/src/asaree/worker/settings.py @@ -31,6 +31,7 @@ from asaree.redis_client import dispose_redis from asaree.services.credential_resolver import resolve as resolve_credentials from asaree.services.result_rollups import refresh_result_summaries +from asaree.services.result_usage import capture_result_usage from asaree.services.system_mcp_servers import ensure_system_servers, refresh_system_server_capabilities from asaree.worker.tasks import ( check_stale_runs, @@ -61,12 +62,17 @@ async def on_shutdown(ctx: dict[str, Any]) -> None: await dispose_redis() +async def maintain_results(ctx: dict[str, Any]) -> None: + await capture_result_usage(ctx) + await refresh_result_summaries(ctx) + + class WorkerSettings: functions = [ execute_run_task, execute_protocol_run_task, ] - cron_jobs = [cron(check_stale_runs, second={0, 30}), cron(refresh_result_summaries, second=set(range(0, 60, 5)))] + cron_jobs = [cron(check_stale_runs, second={0, 30}), cron(maintain_results, second=set(range(0, 60, 5)))] redis_settings = RedisSettings.from_dsn(get_settings().redis_url) on_startup = on_startup on_shutdown = on_shutdown diff --git a/src/asaree/worker/tasks.py b/src/asaree/worker/tasks.py index 6d4a59c4..58a27196 100644 --- a/src/asaree/worker/tasks.py +++ b/src/asaree/worker/tasks.py @@ -155,6 +155,13 @@ async def _guarded_protocol_run( logger.exception("execute_protocol_run_task_failed", extra={"protocol_run_id": protocol_run_id_str}) async with get_session() as db: await fail_protocol_run(db, protocol_run_id, error=f"{type(e).__name__}: {e}") + finally: + # Operational accounting is independent of optional declared metrics. + # A bounded best-effort capture never turns a successful execution into + # a failure. The durable maintenance sweep retries any unfinished capture. + from asaree.services.result_usage import capture_result_usage + + await _best_effort(capture_result_usage({}, only_run_id=protocol_run_id)) async def _fail_protocol_run_now(protocol_run_id: uuid.UUID, error: str) -> None: diff --git a/tests/test_indexed_row_source.py b/tests/test_indexed_row_source.py new file mode 100644 index 00000000..4d7378cc --- /dev/null +++ b/tests/test_indexed_row_source.py @@ -0,0 +1,49 @@ +import hashlib + +import pytest + +from asaree.services.dataset_row_csv import DatasetRowCsvError, project_row, read_row_source +from asaree.services.indexed_row_source import read_indexed_row_source + + +def test_indexed_rows_preserve_original_indices_strings_and_multiline_csv(tmp_path): + source = tmp_path / "original.csv" + raw = b'\xef\xbb\xbfquestion,answer\n"two\nlines",001\n,0\nlast,yes\n' + source.write_bytes(raw) + arguments = dict(dataset_id="dataset", raw_path=str(source), raw_sha256=hashlib.sha256(raw).hexdigest()) + indexed = read_indexed_row_source(**arguments, cache_root=tmp_path / "index") + original = read_row_source(**arguments) + assert indexed.columns == original.columns + assert len(indexed.rows) == len(original.rows) + for index in range(len(original.rows)): + assert project_row(indexed, row_index=index, columns=["answer", "question"]) == project_row( + original, row_index=index, columns=["answer", "question"] + ) + reused = read_indexed_row_source(**arguments, cache_root=tmp_path / "index") + assert reused.rows[2] == ("last", "yes") + + +def test_cached_index_does_not_bypass_original_hash_verification(tmp_path): + source = tmp_path / "original.csv" + raw = b"question,answer\nfirst,1\n" + source.write_bytes(raw) + arguments = dict(dataset_id="dataset", raw_path=str(source), raw_sha256=hashlib.sha256(raw).hexdigest()) + read_indexed_row_source(**arguments, cache_root=tmp_path / "index") + source.write_bytes(b"question,answer\nchanged,2\n") + with pytest.raises(DatasetRowCsvError, match="source_hash_mismatch"): + read_indexed_row_source(**arguments, cache_root=tmp_path / "index") + + +def test_large_source_uses_disk_index_and_preserves_random_access(tmp_path, monkeypatch): + source = tmp_path / "large.csv" + raw = b"question,answer\n" + b"prompt,001\n" * 15786 + source.write_bytes(raw) + monkeypatch.setattr(type(source), "read_bytes", lambda _: pytest.fail("must not materialize the original file")) + indexed = read_indexed_row_source( + dataset_id="dataset", + raw_path=str(source), + raw_sha256=hashlib.sha256(raw).hexdigest(), + cache_root=tmp_path / "index", + ) + assert len(indexed.rows) == 15786 + assert indexed.rows[15785] == ("prompt", "001") diff --git a/tests/test_result_usage.py b/tests/test_result_usage.py new file mode 100644 index 00000000..7caea363 --- /dev/null +++ b/tests/test_result_usage.py @@ -0,0 +1,61 @@ +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from uuid import UUID + +from sqlalchemy.dialects import postgresql + +from asaree.services.motoro_telemetry import usage_query +from asaree.services.result_usage import provider_usage, summarize_usage + + +def test_usage_read_is_owner_scoped_batched_and_never_loads_traces(): + sql = str( + usage_query(UUID(int=1), [UUID(int=2)], [UUID(int=3)]).compile( + dialect=postgresql.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + assert "agent_runs.owner_id" in sql + assert "protocol_run_id" in sql + assert "LIMIT 200" in sql + assert "run_steps" not in sql + assert "agent_runs.input" not in sql + assert "agent_runs.output" not in sql + assert "config_snapshot" not in sql + + +def test_usage_sums_distinct_causal_runs_and_excludes_queue_time(): + started = datetime(2026, 10, 12, tzinfo=UTC) + rows = [ + { + "id": UUID(int=1), + "token_usage": {"prompt_tokens": 100, "completion_tokens": 10}, + "cost_estimate": Decimal("0.01"), + }, + { + "id": UUID(int=2), + "token_usage": {"input_tokens": 200, "output_tokens": 20}, + "cost_estimate": Decimal("0.02"), + }, + ] + result = summarize_usage([*rows, rows[0]], started_at=started, completed_at=started + timedelta(seconds=15)) + assert result["agent_run_count"] == 2 + assert result["total_tokens"] == 330 + assert result["cost_usd"] == 0.03 + assert result["duration_seconds"] == 15 + assert result["complete"] + + +def test_missing_provider_fields_and_legacy_timestamps_stay_unknown(): + assert provider_usage({"input_tokens": 0}, 0) == { + "input_tokens": None, + "output_tokens": None, + "total_tokens": None, + "cost_usd": None, + } + result = summarize_usage([{"id": UUID(int=1)}], started_at=None, completed_at=None) + assert result["duration_seconds"] is None + assert result["cost_usd"] is None + assert result["total_tokens"] is None + assert result["settled"] + assert not result["complete"] From 5aa94c0d3019d946e7a8a5e2772dec7e10fea395 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 12 Oct 2026 20:39:40 -0700 Subject: [PATCH 35/63] Move large execution batches and exports to durable result jobs --- README.md | 6 + RESULTS_OPERATIONS.md | 168 ++++++ compose.dev.yml | 7 + compose.yml | 42 ++ frontend/src/api/client.ts | 19 +- .../protocol/DatasetExecutionSummary.tsx | 7 +- .../components/protocol/ExecutionBrowser.tsx | 34 +- .../protocol/ExecutionInspector.tsx | 37 +- .../protocol/ExecutionNodeEvidence.tsx | 35 ++ .../protocol/ProtocolCanvas.test.tsx | 10 +- .../components/protocol/ProtocolCanvas.tsx | 2 +- .../protocol/ResultJobFlows.test.tsx | 79 +++ .../components/protocol/ResultJobProgress.tsx | 23 + .../protocol/ResultTimelineNode.tsx | 9 +- .../src/components/protocol/ResultsPanel.tsx | 40 +- .../src/components/protocol/ResultsTab.tsx | 15 +- .../components/protocol/RunConfirmDialog.tsx | 4 +- frontend/src/components/protocol/RunsTab.tsx | 62 ++- .../protocol/edges/DatasetInputPanel.tsx | 4 +- .../protocol/useDatasetRowSelection.ts | 4 +- frontend/src/lib/executionPagination.ts | 9 + frontend/src/lib/experiment.ts | 5 +- frontend/src/lib/resultJobs.ts | 28 + frontend/src/pages/ExperimentsPage.tsx | 20 +- .../src/pages/ProtocolCanvasPage.test.tsx | 3 + frontend/src/pages/ProtocolCanvasPage.tsx | 84 +-- frontend/src/types/executionResults.ts | 13 +- frontend/src/types/experiments.ts | 4 + frontend/src/types/protocols.ts | 3 +- scripts/create_results_telemetry_index.py | 3 + sdk/src/asaree_client/client.py | 8 + .../asaree_client/resources/experiments.py | 95 ++++ sdk/tests/test_result_jobs.py | 43 ++ src/asaree/api/datasets.py | 38 +- src/asaree/api/experiments.py | 8 + src/asaree/api/protocol_run_heads.py | 68 +++ src/asaree/api/protocols.py | 79 ++- src/asaree/api/result_evidence.py | 105 ++++ src/asaree/api/result_jobs.py | 148 +++++ src/asaree/api/result_reads.py | 16 +- src/asaree/app.py | 6 + src/asaree/migrations/env.py | 1 + .../5f60718293a4_durable_result_jobs.py | 75 +++ .../60718293a4b5_bounded_attempt_evidence.py | 69 +++ src/asaree/models/dataset.py | 4 +- src/asaree/models/protocol_run.py | 2 +- src/asaree/models/result_job.py | 54 ++ src/asaree/services/experiment_run_results.py | 231 +++++--- src/asaree/services/factorial_row_results.py | 2 +- src/asaree/services/motoro_telemetry.py | 96 +++- src/asaree/services/protocol_execution.py | 131 ++--- src/asaree/services/result_exports.py | 217 ++++++++ src/asaree/services/result_jobs.py | 525 ++++++++++++++++++ src/asaree/services/result_reads.py | 348 ++++++++++-- src/asaree/services/result_rollups.py | 218 ++++++-- src/asaree/services/result_usage.py | 56 +- src/asaree/services/row_schema_reads.py | 55 ++ src/asaree/worker/result_settings.py | 41 ++ src/asaree/worker/settings.py | 9 +- tests/test_dataset_row_schema.py | 35 ++ tests/test_result_commands.py | 45 ++ tests/test_result_reads.py | 296 +++++++++- tests/test_result_usage.py | 20 + tests/test_result_worker_accounting.py | 72 +++ 64 files changed, 3567 insertions(+), 428 deletions(-) create mode 100644 RESULTS_OPERATIONS.md create mode 100644 frontend/src/components/protocol/ExecutionNodeEvidence.tsx create mode 100644 frontend/src/components/protocol/ResultJobFlows.test.tsx create mode 100644 frontend/src/components/protocol/ResultJobProgress.tsx create mode 100644 frontend/src/lib/executionPagination.ts create mode 100644 frontend/src/lib/resultJobs.ts create mode 100644 sdk/tests/test_result_jobs.py create mode 100644 src/asaree/api/protocol_run_heads.py create mode 100644 src/asaree/api/result_evidence.py create mode 100644 src/asaree/api/result_jobs.py create mode 100644 src/asaree/migrations/versions/5f60718293a4_durable_result_jobs.py create mode 100644 src/asaree/migrations/versions/60718293a4b5_bounded_attempt_evidence.py create mode 100644 src/asaree/models/result_job.py create mode 100644 src/asaree/services/result_exports.py create mode 100644 src/asaree/services/result_jobs.py create mode 100644 src/asaree/services/row_schema_reads.py create mode 100644 src/asaree/worker/result_settings.py create mode 100644 tests/test_result_commands.py create mode 100644 tests/test_result_worker_accounting.py diff --git a/README.md b/README.md index 731bfc3a..d3c5eb1f 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,12 @@ servers (`asaree-sklearn-dc`, `-eda`, `-fs`, `-fte`, `-model`, `-stats`, from `mcp-servers/`) — ships as a dependency of the app and registers itself each time the app or worker starts (see `asaree.services.system_mcp_servers`). +## Large experiment results + +For large experiment results, the dedicated result worker handles summaries, +batch commands and exports. Deployment, recovery, API/SDK usage and pending +PostgreSQL acceptance checks are documented in [RESULTS_OPERATIONS.md](RESULTS_OPERATIONS.md). + ## Resetting your dev environment One Postgres server hosts two databases in a single volume — `motoro` (core's diff --git a/RESULTS_OPERATIONS.md b/RESULTS_OPERATIONS.md new file mode 100644 index 00000000..931e36b4 --- /dev/null +++ b/RESULTS_OPERATIONS.md @@ -0,0 +1,168 @@ +# Scalable execution results + +Runs and Results use `/api/experiments/{id}/execution-results`. The canvas uses +`/api/protocols/{id}/run-heads?limit=1` to recover its latest execution. None of +these list reads hydrates Motoro agents, traces, complete outputs, or histories. + +Execution and cell pages default to 20 records and cap at 100. Cursors bind the +protocol publication, design revision and filters. Metric cursors also bind the +published summary generation; refresh from the first page after it changes. +Per-row pages generate only the requested window, including unstarted source +rows once their count has been prepared. Stable slot IDs identify completed +records. `unplanned::` identifies a row without a +persisted slot and cannot be used to inspect an attempt. + +Summaries are rebuildable projections. Database triggers record dirty scopes in +`result_refreshes` in the same transaction as result changes. The result worker +coalesces and rebuilds each scope from current facts, streams cell writes, and +publishes cell/overview data atomically. Repeated deliveries do not double-count +attempts. The UI shows records while a summary or usage backfill is preparing. +It refreshes visible pages when a summary generation advances, retains loaded +data on transient failures, and stops polling settled executions. Refresh or +window focus discovers work started by another client after a scope settles. + +Operational usage belongs to each production `ProtocolRun`, independently of +declared measurements. Accounting reads scalar, owner-scoped Motoro rows in +batches of 200, including all causally attributed calls (not only final node +pointers). A pointer fallback supports older records. Legacy calls without +either attribution or a retained pointer cannot be recovered. Unreported or +ambiguous default-zero provider fields remain unknown. Execution duration uses +actual start/task completion, excluding queue time. Current execution usage +and all-attempt spend are separate. Immutable measurement/evidence documents +are never rewritten by accounting backfill. + +Only explicitly declared built-in runtime measurements enter scalar cell +aggregates and ranking. Global runtime metrics use catalog keys; node runtime +metrics use stable metric IDs with display labels. Display names do not identify +measurements. Reported and legacy values remain opaque, retain +their recorded provenance, and are not coerced or ranked. Per-row outcomes are +never aggregated or ranked. Truncated attempts are excluded from scalar +comparison. Primary ranks are maintained for the whole scope, including pages +after the first. + +Selected attempts return bounded previews. Oversized observation values and +artifact payloads carry explicit preview markers; a measured JSON null remains +different from a deferred value. Exact JSON documents are available through +`attempts/{run}/evidence` in chunks of at most 16,384 characters. Node output and +received prompt text use the same chunk bound. Trace pages contain 20 metadata +records; full step evidence loads only after selecting a step. Conversation +messages are paged and large messages are marked previews; complete transcripts +remain available through the recorded JSON reader. + +## Durable batch commands + +`POST execution-results/jobs` accepts `kind=plan|cancel|export`, a stable +`request_key`, and the selected publication/design query parameters. Creation +returns 202 immediately. Repeating the same owned request key and selection +returns the same job; conflicting reuse is rejected. + +A plan first prepares a preview in bounded windows and stores exact parent, +row, slot, and expected-attempt identities. `awaiting_confirmation` includes +the exact planned execution count. Opening a dialog never launches an LLM. +`POST jobs/{id}/confirm` queues that sealed selection. Each small transaction +creates runs and advances their intent checkpoints together. Slots changed +since preview are skipped, so a retry cannot silently re-bill a newer attempt. +Publication/design changes require preparing a new batch. Runs already created +remain pinned to their immutable publications. + +Completed executions are skipped unless explicitly selected by slot ID or +`rerun_completed=true`. Explicit selections cap at 100 IDs; an all-scope command +does not send a browser-sized ID list. Failed/cancelled/limit-reached work can be +retried. A plan's cancel action stops remaining planning; already-created runs +continue. The Runs tab's Stop all creates a scoped cancellation command, stops +matching planners and cancels all active attempts, including earlier attempts +that are no longer the latest slot result. + +Run cell and Stop cell use the same durable API with `cell_id`. Stopping one +cell excludes it from existing broad planners while allowing their other cells +to continue. Publication predicates also constrain the factor levels and +replicate count: adding cells to a mutable design revision cannot expand the +cohort belonging to an earlier immutable publication. + +SQL intent survives worker/Redis loss. Created pending runs are repeatedly +offered to ARQ with the same run job ID; executor guards prevent completed or +cancelled runs from executing again. Delivery continues for committed runs even +if later planning fails or is cancelled. Preview-only cancellation cannot +cancel a confirmed job. Unconfirmed previews expire after 24 hours. + +## Exports and source metadata + +An export is a background job, limited to one expensive COPY operation globally. +PostgreSQL streams CSV directly to an owned temporary file, followed by atomic +rename. One COPY statement supplies a coherent database snapshot; attempt +selection excludes attempts created after the request cutoff. Active attempts +may acquire evidence before COPY starts, so this is not a historical snapshot +of every field at the request timestamp. A restart repeats the export from its +cutoff. Progress reports bytes, and cancellation is checked during streaming. + +CSV v2 uses canonical execution columns and JSON columns for factor values, +usage, original attempt results, node records and imported legacy results. +`legacy_result` preserves imported metric values and artifacts. `attempt_result` contains +exact opaque values, JSON nulls, artifacts and producer provenance. The schema +sidecar lists all columns and metric declarations. This version intentionally +does not flatten arbitrary reported structures into scalar metric columns. +Legacy synchronous CSV retains its old format for small requests; large legacy +full-result requests return 413 with directions to the bounded/job API. + +Completed exports are downloaded natively in the browser or streamed to disk +by the SDK. Download capabilities bind job/owner, expire after five minutes, +and cannot supply filesystem paths. CSV/schema files expire after 24 hours. + +Original CSV rows are verified by streaming SHA-256 and indexed into an immutable +SQLite cache. Construction uses batches of 1,000 rows and atomic rename. +Subsequent planning windows use random indexed row access instead of reparsing +the original CSV for every retry. Dataset row-schema HTTP reads return 202 with +`ready=false`/`row_count=null` while the worker prepares metadata; they never +read or parse source files in the request. The UI polls that small metadata +record while preparing. Source preparation records hash/CSV validation failures +without exposing paths. Count metadata describes the last verified registered +hash; execution still verifies the pinned source identity before running. + +## Deployment and recovery + +1. Apply product Alembic migrations through `60718293a4b5` using the usual + migration deployment. Historical JSON evidence is not rewritten in migration. +2. Create the two core indexes with `python scripts/create_results_telemetry_index.py` + in the deployed application environment. It uses `CREATE INDEX CONCURRENTLY` + for owner/attribution lookup and `(run_id, sequence)` trace paging. +3. Run `arq asaree.worker.result_settings.ResultWorkerSettings` alongside the + production execution worker. Compose defines `asaree-results-index` and + `asaree-results-worker`; the latter waits for the core indexes. Both workers + and the API must share the data volume and configured export/row-index paths. +4. Keep `ASAREE_RESULT_EXPORT_DIR` and `ASAREE_RESULT_ROW_INDEX_DIR` on persistent + shared storage (defaults are `./data/result-exports` and `./data/result-row-indexes`). +5. To requeue all existing overview projections, run + `python scripts/rebuild_result_summaries.py`. The worker rebuilds from facts + and retries unfinished accounting; no evidence or execution timestamps change. + +Watch `result_refreshes.generation - processed_generation`, refresh `error`, +unfinished `protocol_runs.usage_summary`, pending `result_jobs`, and worker logs +for `result_summary_refresh_failed`, `result_usage_capture_failed`, +`result_command_database_retry`, and `result_command_delivery_failed`. +Size the dedicated result worker separately from LLM concurrency. Commands use +bounded work windows; exports, usage capture and source preparation each have +an independent global advisory lock. Read requests have a five-second SQL +statement deadline and bounded page sizes. Read replicas are not required. + +SDK entry points: `execution_summary`, `iter_executions`, `create_result_job`, +`result_job`, `confirm_result_job`, `cancel_result_job`, `iter_evidence_text`, +and `download_result_export`. Supply a pinned publication in `scope` while +iterating and keep the same `request_key` when retrying job creation. + +## Validation status + +Focused Python/SDK tests, frontend tests, TypeScript, Ruff and the new service +type checks run locally without infrastructure. PostgreSQL validation is +**pending**, as requested; no live migration or production-latency result is +claimed. `tests/test_result_reads.py` applies the complete migration chain in +a unique schema inside a rolled-back transaction when +`ASAREE_RESULTS_TEST_DATABASE_URL` points to a disposable PostgreSQL database. + +The opt-in acceptance tests (`ASAREE_RESULTS_RUN_SCALE=1`) exercise 15,786, +100,000 and 1,000,000 runs with 32 KiB node outputs and 8 KiB opaque reports. +They require 20-record pages below 64 KiB, cached summaries below 16 KiB, and +cached summary plus first-page latency below two seconds. Run these tests only +on the disposable database; the million-run case intentionally creates a large +fixture. Before rollout, execute these checks and inspect PostgreSQL plans for +slot lookup and metric ranking, including concurrent execution/outbox updates, +worker restart, Redis interruption, export COPY/download, and downgrade/upgrade. diff --git a/compose.dev.yml b/compose.dev.yml index c52b0b3f..c0f474ef 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -52,3 +52,10 @@ services: - ./src:/app/src - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro + + asaree-results-worker: + volumes: + - ./data:/app/data + - ./src:/app/src + - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro + - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro diff --git a/compose.yml b/compose.yml index ec087de5..6e4a0033 100644 --- a/compose.yml +++ b/compose.yml @@ -218,6 +218,48 @@ services: - ./data:/app/data restart: unless-stopped + asaree-results-index: + build: + context: . + secrets: + - gh_token + additional_contexts: + gitdir: ./.git + depends_on: + motoro-migrate: + condition: service_completed_successfully + env_file: .env + environment: + ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + command: ["python", "scripts/create_results_telemetry_index.py"] + restart: "no" + + # Result maintenance and bulk commands never compete for LLM worker slots. + asaree-results-worker: + build: + context: . + secrets: + - gh_token + additional_contexts: + gitdir: ./.git + depends_on: + asaree-results-index: + condition: service_completed_successfully + asaree-migrate: + condition: service_completed_successfully + motoro-redis: + condition: service_healthy + env_file: .env + environment: + ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree + ASAREE_REDIS_URL: redis://motoro-redis:6379/0 + ASAREE_OKF_BUNDLE_ROOT: /app/data + command: ["arq", "asaree.worker.result_settings.ResultWorkerSettings"] + volumes: + - ./data:/app/data + restart: unless-stopped + # Integration-test runner on the Compose network. The tests intentionally # use real Postgres (JSONB and core's vector schema rule out SQLite), so run # them beside the database instead of relying on a host-side localhost port. diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 3b7732e0..c37d81e9 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -32,7 +32,7 @@ import type { OkfBundle, OkfDocument } from '@/types/okf' import type { CellRunBatch, DatasetRowSchema, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun, TestRun } from '@/types/protocols' import type { Run, RunStep } from '@/types/runs' import type { Skill, SkillListResponse, SkillUrlPreview } from '@/types/skills' -import type { AttemptDetail, AttemptPage, CursorPage, ExecutionCell, ExecutionSlot, ExecutionSummary } from '@/types/executionResults' +import type { AttemptDetail, AttemptPage, CursorPage, ExecutionCell, ExecutionSlot, ExecutionSummary, ResultJob } from '@/types/executionResults' const ACCESS_TOKEN_KEY = 'asaree_access_token' @@ -211,6 +211,22 @@ function rowSelection(options?: RowSelection): RowSelection | undefined { } export const experimentsApi = { + getExecutionEvidence: (id: string, runId: string, scope: ResultsScope, document: string, offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/evidence${resultPageQuery(scope, { document, offset })}`, { signal }), + getExecutionNodeText: (id: string, runId: string, nodeId: string, scope: ResultsScope, field: 'prompt' | 'output', offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/text${resultPageQuery(scope, { field, offset })}`, { signal }), + getExecutionTrace: (id: string, runId: string, nodeId: string, scope: ResultsScope, after: number, signal?: AbortSignal) => + request<{ items: { id: string; sequence: number; iteration: number | null; phase: string; preview: string | null }[]; next_after: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/trace${resultPageQuery(scope, { after })}`, { signal }), + getExecutionStep: (id: string, runId: string, nodeId: string, stepId: string, scope: ResultsScope, offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/trace/${stepId}${resultPageQuery(scope, { offset })}`, { signal }), + createResultJob: (id: string, scope: ResultsScope, command: { kind: ResultJob['kind']; request_key: string; cell_id?: string; replicate_ids?: string[]; row_indices?: number[]; rerun_slot_ids?: string[]; rerun_completed?: boolean }, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/jobs${scopeQuery(scope)}`, { method: 'POST', body: command, signal }), + getResultJob: (id: string, jobId: string, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/jobs/${jobId}`, { signal }), + confirmResultJob: (id: string, jobId: string) => + request(`/experiments/${id}/execution-results/jobs/${jobId}/confirm`, { method: 'POST' }), + cancelResultJob: (id: string, jobId: string, previewOnly = false) => + request(`/experiments/${id}/execution-results/jobs/${jobId}/cancel${previewOnly ? '?preview_only=true' : ''}`, { method: 'POST' }), getExecutionSummary: async (id: string, scope: ResultsScope, signal?: AbortSignal, cached?: ExecutionSummary): Promise => { const res = await authedFetch(`/experiments/${id}/execution-results/summary${scopeQuery(scope)}`, { signal, headers: cached?.etag ? { 'If-None-Match': cached.etag } : undefined, @@ -366,6 +382,7 @@ export const protocolsApi = { // point after retaining work that already completed. cancelRun: (id: string, runId: string) => request(`/protocols/${id}/runs/${runId}/cancel`, { method: 'POST' }), listRuns: (id: string) => request(`/protocols/${id}/runs`), + listRunHeads: async (id: string) => (await request>(`/protocols/${id}/run-heads?limit=1`)).items, // "Run all cells" -- 422 if there's no linked experiment or the graph // doesn't have exactly one final node; fans out one ProtocolRun per pending // replicate. The optional list explicitly re-runs selected completed rows. diff --git a/frontend/src/components/protocol/DatasetExecutionSummary.tsx b/frontend/src/components/protocol/DatasetExecutionSummary.tsx index a50cc399..bdfd873b 100644 --- a/frontend/src/components/protocol/DatasetExecutionSummary.tsx +++ b/frontend/src/components/protocol/DatasetExecutionSummary.tsx @@ -19,6 +19,7 @@ export function DatasetExecutionSummary({ graph, protocol, experimentId, cellCou queryFn: () => datasetsApi.getRowSchema(datasetId!), enabled: !!datasetId, retry: false, + refetchInterval: query => query.state.data?.ready === false ? 1000 : false, }) const publication = useQuery({ queryKey: ['protocols', protocol?.id, 'row-published-graph', protocol?.published_revision_id], @@ -30,15 +31,15 @@ export function DatasetExecutionSummary({ graph, protocol, experimentId, cellCou const publishedBinding = publication.data && rowBindingForNode(publication.data.graph) const summary = results.data const rows = source.data?.row_count - const columnsMissing = binding?.input.mode === 'per_row' && source.data && binding.input.columns.some(column => !source.data.columns.includes(column)) + const columnsMissing = binding?.input.mode === 'per_row' && source.data?.ready !== false && source.data && binding.input.columns.some(column => !source.data.columns.includes(column)) return

Dataset execution

Experiment draft · proposed design

{!graph ?

Loading canvas inputs…

: binding ? <> -

Per row · {binding.config?.dataset_name ?? datasetId ?? 'Unregistered Dataset'} · {rows === undefined ? 'row count unavailable' : `${rows} original rows`}

-

{cellCount} cells × {replicateCount} replicates per cell × {rows ?? '?'} rows = {rows === undefined ? '?' : cellCount * replicateCount * rows} executions

+

Per row · {binding.config?.dataset_name ?? datasetId ?? 'Unregistered Dataset'} · {rows == null ? 'preparing row count' : `${rows} original rows`}

+

{cellCount} cells × {replicateCount} replicates per cell × {rows ?? '?'} rows = {rows == null ? '?' : cellCount * replicateCount * rows} executions

Each row runs the complete protocol. Agents receive their selected columns.

{(error || source.error || columnsMissing || rows === 0) &&

{error || source.error?.message || (columnsMissing ? 'Selected columns are absent from the original CSV.' : 'The original Dataset has no rows.')}

} :

Whole dataset · one protocol execution per replicate.

} diff --git a/frontend/src/components/protocol/ExecutionBrowser.tsx b/frontend/src/components/protocol/ExecutionBrowser.tsx index 2496f729..4c96b6a9 100644 --- a/frontend/src/components/protocol/ExecutionBrowser.tsx +++ b/frontend/src/components/protocol/ExecutionBrowser.tsx @@ -6,8 +6,9 @@ import { Badge } from '@/components/ui/badge' import { Skeleton } from '@/components/ui/skeleton' import { Card } from '@/components/ui/card' import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' +import { useCursor } from '@/lib/executionPagination' import { displayFactorCondition, formatMetricLabel, formatMetricValue } from '@/lib/experiment' -import type { ExecutionSlot, ExecutionSummary } from '@/types/executionResults' +import type { ExecutionCell, ExecutionSlot, ExecutionSummary } from '@/types/executionResults' import type { Experiment } from '@/types/experiments' import type { ResultsSelection } from './ResultsTab' import { ResultsPanel } from './ResultsPanel' @@ -22,14 +23,6 @@ export function CursorControls({ next, back, onNext, onBack, busy }: {
} -export function useCursor() { - const [cursors, setCursors] = useState<(string | undefined)[]>([undefined]) - return { cursor: cursors.at(-1), back: cursors.length > 1, - next: (cursor: string) => setCursors(current => [...current, cursor]), - previous: () => setCursors(current => current.slice(0, -1)), - reset: () => setCursors([undefined]) } -} - function Usage({ usage }: { usage: ExecutionSummary['usage'] }) { if (!usage) return null return @@ -54,7 +47,7 @@ export function ExecutionRows({ experimentId, scope, cellId, generation, summary {query.isPending ? : query.error && !query.data ?

{query.error.message}

: <> {!query.data?.items.length &&

No executions planned in this scope.

}
    {query.data?.items.map(row => onSelect({ type: 'execution', slotId: row.slot_id, scope }) : undefined} + onView={row.run_id || row.status === 'completed' ? () => onSelect({ type: 'execution', slotId: row.slot_id, scope }) : undefined} detail={<>{row.row_index !== null && Source row {row.row_index + 1} · } {row.result.observations?.map(observation => {observation.metric_name}: {observation.status === 'measured' ? observation.preview : observation.status}{observation.preview_truncated && '…'})}} status={<>{row.status.replaceAll('_', ' ')}{row.result.truncated && Hit iteration limit}} @@ -74,10 +67,11 @@ function useGenerationRefresh(refetch: () => unknown, generation?: number) { }, [generation, refetch]) } -export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSelect, controls, actions }: { +export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSelect, controls, actions, cellActions }: { experimentId: string; scope: ResultsScope; experiment?: Experiment; mode: 'runs' | 'results' onSelect: (selection: ResultsSelection) => void; controls?: React.ReactNode actions?: (row: ExecutionSlot) => React.ReactNode + cellActions?: (cell: ExecutionCell) => React.ReactNode }) { const summary = useExecutionSummary(experimentId, scope) const [expanded, setExpanded] = useState>(() => new Set()) @@ -86,38 +80,42 @@ export function ExecutionBrowser({ experimentId, scope, experiment, mode, onSele const [directionPreference, setDirectionPreference] = useState() const direction = directionPreference ?? summary.data?.metric_directions[metric ?? ''] ?? 'maximize' const sortDirection = direction === 'minimize' ? 'minimize' : 'maximize' + const ranking = metric === summary.data?.primary_metric && sortDirection === summary.data?.primary_metric_direction const pager = useCursor() const cells = useQuery({ queryKey: [...executionKeys.root(experimentId), 'cells', scope, pager.cursor, metric, sortDirection], queryFn: ({ signal }) => experimentsApi.listExecutionCells(experimentId, scope, { cursor: pager.cursor, metric, direction: sortDirection }, signal), }) const previousGeneration = useRef(summary.data?.generation) + const resetCells = pager.reset + const refetchCells = cells.refetch useEffect(() => { if (previousGeneration.current !== summary.data?.generation) { - pager.reset() - void cells.refetch() + resetCells() + void refetchCells() } previousGeneration.current = summary.data?.generation - }, [summary.data?.generation]) + }, [summary.data?.generation, resetCells, refetchCells]) const metricKeys = summary.data?.unit === 'replicate' ? Object.keys(summary.data.metric_types) : [] const refresh = () => { void summary.refetch(); void cells.refetch() } const cellBody =
    {summary.data?.unit === 'row' &&

    Per-row outcomes are reported individually. Aggregation, ranking and factorial analysis are unavailable.

    } + {mode === 'results' && metric &&

    Comparisons use completed, untruncated measurements. Rankings are provisional while measured counts differ.

    } {mode === 'results' && metricKeys.length > 0 &&
    - + {metric && }
    } {cells.isPending ? : cells.error && !cells.data ?

    {cells.error.message}

    : <> {!cells.data?.items.length &&

    No generated cells in this version.

    } - {mode === 'results' && metric &&
    {cells.data?.items.map((cell, index) => )}
    {metric === summary.data?.primary_metric ? 'Rank' : '—'}Condition{formatMetricLabel(metric)}
    {metric === summary.data?.primary_metric && cell.metric_values?.[metric] !== undefined && !pager.back ? index+1 : '—'}{cell.cell_label}{cell.metric_values?.[metric] !== undefined ? formatMetricValue(metric, cell.metric_values[metric]) : 'Unavailable'}
    } + {mode === 'results' && metric &&
    {cells.data?.items.map(cell => )}
    {ranking ? 'Rank' : '—'}Condition{summary.data?.metric_labels?.[metric] ?? formatMetricLabel(metric)}Measured
    {ranking ? cell.primary_rank ?? '—' : '—'}{cell.cell_label}{cell.metric_values?.[metric] !== undefined ? formatMetricValue(metric, cell.metric_values[metric]) : 'Unavailable'}{cell.metric_counts?.[metric] ?? 0}/{cell.replicate_count ?? '?'}
    } {cells.data?.items.map(cell => displayFactorCondition(name, value, summary.data?.design_spec ?? experiment?.design_spec, cell.cell_label))} replicateCount={cell.replicate_count ?? 0} expanded={expanded.has(cell.cell_id)} listId={`executions-${cell.cell_id}`} onToggle={() => setExpanded(current => { const next = new Set(current); if (next.has(cell.cell_id)) next.delete(cell.cell_id); else next.add(cell.cell_id); return next })} progress={<>{cell.counts?.completed ?? 0}/{cell.total ?? '?'} complete{metric && cell.metric_values?.[metric] !== undefined && {formatMetricValue(metric, cell.metric_values[metric])}}} badges={cell.truncated ? {cell.truncated} hit iteration limit : undefined} - actions={}> - {expanded.has(cell.cell_id) && } + actions={<>{cellActions?.(cell)}}> + {expanded.has(cell.cell_id) && } )} {cells.error && cells.data &&

    Refresh failed; showing the last loaded cells.

    } diff --git a/frontend/src/components/protocol/ExecutionInspector.tsx b/frontend/src/components/protocol/ExecutionInspector.tsx index a24f25d1..e5aa3a73 100644 --- a/frontend/src/components/protocol/ExecutionInspector.tsx +++ b/frontend/src/components/protocol/ExecutionInspector.tsx @@ -7,19 +7,23 @@ import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' import type { AttemptDetail, ExecutionSlot } from '@/types/executionResults' import type { Experiment, ResultReplicate, ResultNodeRun } from '@/types/experiments' import { ReplicateResultDetail, type ResultsSelection } from './ResultsTab' -import { CursorControls, useCursor } from './ExecutionBrowser' +import { CursorControls } from './ExecutionBrowser' +import { useCursor } from '@/lib/executionPagination' import { ExecutionRows } from './ExecutionBrowser' -export function attemptAsReplicate(slot: ExecutionSlot, detail: AttemptDetail): ResultReplicate { +function attemptAsReplicate(slot: ExecutionSlot, detail: AttemptDetail): ResultReplicate { const usage = detail.usage const observations = detail.attempt_result?.measurement?.observations ?? [] const metrics = { ...detail.attempt_result?.metric_values } - for (const observation of observations) if (observation.status === 'measured') metrics[observation.metric_name] = observation.value + for (const observation of observations) if (observation.status === 'measured') { + metrics[observation.metric_name] = observation.value + metrics[observation.metric_id] = observation.value + } const nodes: ResultNodeRun[] = Object.entries(detail.node_runs).filter(([, node]) => !!node.run_id || !!node.output_text || !!node.error || ['running', 'failed', 'cancelled'].includes(node.status), ).map(([id, node]) => ({ node_id: id, node_label: detail.node_labels[id] ?? id, status: node.status, bypassed: node.bypassed, output_text: node.output_text ?? null, error: node.error ?? null, - agent_run_id: node.run_id ?? null, input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, + agent_run_id: node.run_id ?? null, input_tokens: usage?.by_node?.[id]?.input_tokens ?? null, output_tokens: usage?.by_node?.[id]?.output_tokens ?? null, total_tokens: usage?.by_node?.[id]?.total_tokens ?? null, cost_usd: usage?.by_node?.[id]?.cost_usd ?? null, unresolved_reference_labels: (node.unresolved_references ?? []).map(ref => typeof ref === 'string' ? detail.node_labels[ref] ?? ref : String(ref)), })) return { replicate_label: slot.replicate_label, replicate_number: slot.replicate_number, cell_label: slot.cell_label, @@ -28,9 +32,10 @@ export function attemptAsReplicate(slot: ExecutionSlot, detail: AttemptDetail): updated_at: detail.completed_at ?? detail.started_at ?? '', node_runs: nodes, duration_seconds: usage?.duration_seconds ?? null, cost_usd: usage?.cost_usd ?? null, input_tokens: usage?.input_tokens ?? null, output_tokens: usage?.output_tokens ?? null, total_tokens: usage?.total_tokens ?? null, - agent_run_count: usage?.agent_run_count ?? 0, reported_usage_count: usage?.complete ? 1 : 0, reported_cost_count: usage?.cost_usd != null ? 1 : 0, + agent_run_count: usage?.agent_run_count ?? 0, reported_usage_count: usage?.reported_usage_count ?? 0, reported_cost_count: usage?.reported_cost_count ?? 0, metric_evaluation: detail.attempt_result?.metric_evaluation, metric_observations: observations, evaluation_artifacts: detail.attempt_result?.measurement?.artifacts ?? [], + legacy_values: Object.entries(detail.attempt_result?.metric_values ?? {}).filter(([name]) => !observations.some(item => item.metric_name === name)).map(([name, value]) => ({ metric_id: `legacy:${name}`, metric_name: name, value, attempt_id: detail.run_id, producer: { binding_id: 'legacy-unknown' as const, producer_id: 'legacy.unknown' as const, kind: 'legacy' as const, version: 'unknown' as const } })), obsolete_runs: [], superseded_runs: [] } } @@ -48,6 +53,7 @@ function ConversationPage({ experimentId, runId, scope }: { experimentId: string function SlotInspector({ experimentId, slotId, scope, onClose }: { experimentId: string; slotId: string; scope: ResultsScope; onClose: () => void }) { const [selected, setSelected] = useState() const [conversationOpen, setConversationOpen] = useState(false) + const [evidenceOpen, setEvidenceOpen] = useState(false) const pager = useCursor() const summary = useExecutionSummary(experimentId, scope) const history = useQuery({ queryKey: [...executionKeys.root(experimentId), 'attempts', scope, slotId, pager.cursor], @@ -65,14 +71,33 @@ function SlotInspector({ experimentId, slotId, scope, onClose }: { experimentId:
    {(history.isPending || detail.isPending && !!runId) && } {(history.error || detail.error) &&

    {history.error?.message ?? detail.error?.message}

    } - {slot && detail.data &&
    + {slot && detail.data &&
    + + {evidenceOpen && runId && } {conversationOpen && runId && }
    } {history.data && !runId &&

    This execution has no attempts.

    } + {history.data?.legacy_result &&

    Legacy result

    Original values; producer unknown. These values are not ranked.

    {JSON.stringify(history.data.legacy_result, null, 2)}
    } } +function RecordedEvidence({ experimentId, runId, scope }: { experimentId: string; runId: string; scope: ResultsScope }) { + const [document, setDocument] = useState('attempt_result') + return
    +} + +function EvidencePage({ experimentId, runId, scope, document }: { experimentId: string; runId: string; scope: ResultsScope; document: string }) { + const [offsets, setOffsets] = useState([0]) + const offset = offsets.at(-1) ?? 0 + const query = useQuery({ queryKey: [...executionKeys.root(experimentId), 'evidence', scope, runId, document, offset], + queryFn: ({ signal }) => experimentsApi.getExecutionEvidence(experimentId, runId, scope, document, offset, signal) }) + return

    Recorded JSON · characters {offset.toLocaleString()}–{(offset + (query.data?.text.length ?? 0)).toLocaleString()}

    {query.data?.text ?? 'Loading…'}
    + {query.error &&

    {query.error.message}

    } +
    +
    +} + export function ExecutionInspector({ experimentId, selection, onClose }: { experimentId: string; experiment: Experiment; selection: ResultsSelection | null; onClose: () => void }) { const scope = selection?.scope ?? {} const legacyLabel = selection?.type === 'replicate' ? selection.replicateLabel : undefined diff --git a/frontend/src/components/protocol/ExecutionNodeEvidence.tsx b/frontend/src/components/protocol/ExecutionNodeEvidence.tsx new file mode 100644 index 00000000..27c38481 --- /dev/null +++ b/frontend/src/components/protocol/ExecutionNodeEvidence.tsx @@ -0,0 +1,35 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { experimentsApi, type ResultsScope } from '@/api/client' +import { Button } from '@/components/ui/button' +import { executionKeys } from '@/lib/executionQueries' + +export interface ExecutionContext { experimentId: string; runId: string; scope: ResultsScope } + +export function ExecutionNodeText({ context, nodeId, field, stepId }: { context: ExecutionContext; nodeId: string; field: 'prompt' | 'output'; stepId?: string }) { + const [offsets, setOffsets] = useState([0]) + const offset = offsets.at(-1) ?? 0 + const query = useQuery({ queryKey: [...executionKeys.root(context.experimentId), 'node-text', context.scope, context.runId, nodeId, field, stepId, offset], + queryFn: ({ signal }) => stepId + ? experimentsApi.getExecutionStep(context.experimentId, context.runId, nodeId, stepId, context.scope, offset, signal) + : experimentsApi.getExecutionNodeText(context.experimentId, context.runId, nodeId, context.scope, field, offset, signal) }) + return

    {stepId ? 'Step evidence' : field === 'prompt' ? 'Received prompt' : 'Output'}

    + {query.error ?

    {query.error.message}

    : 
    {query.data?.text ?? 'Loading…'}
    } +
    +
    +} + +export function ExecutionNodeTrace({ context, nodeId }: { context: ExecutionContext; nodeId: string }) { + const [open, setOpen] = useState(false) + const [afters, setAfters] = useState([-1]) + const [selected, setSelected] = useState() + const after = afters.at(-1) ?? -1 + const query = useQuery({ queryKey: [...executionKeys.root(context.experimentId), 'trace', context.scope, context.runId, nodeId, after], + queryFn: ({ signal }) => experimentsApi.getExecutionTrace(context.experimentId, context.runId, nodeId, context.scope, after, signal), enabled: open }) + return
    + {open && <>{query.error &&

    {query.error.message}

    } + {query.data?.items.map(step =>

    {step.preview}

    {selected === step.id && }
    )} +
    + } +
    +} diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 90784a26..d6b7eedd 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -130,7 +130,7 @@ describe('ProtocolCanvas connector adds', () => { unobserve() {} disconnect() {} }) - vi.spyOn(protocolsApi, 'listRuns').mockResolvedValue([]) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([]) vi.spyOn(protocolsApi, 'update').mockImplementation(async (_id, { graph }) => ({ id: 'protocol-1', name: 'Protocol', @@ -150,7 +150,7 @@ describe('ProtocolCanvas connector adds', () => { status: 'completed', node_runs: { observer: { status: 'completed', bypassed: true, output_text: 'Upstream answer' } }, }) - vi.spyOn(protocolsApi, 'listRuns').mockResolvedValue([run]) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) renderCanvas({ nodes: [{ id: 'observer', type: 'agent', position: { x: 100, y: 100 }, data: defaultAgentNodeData('Observer') }], @@ -168,7 +168,7 @@ describe('ProtocolCanvas connector adds', () => { status: 'completed', node_runs: { gate: { status: 'completed', bypassed: true, approved: null, output_text: 'READY' } }, }) - vi.spyOn(protocolsApi, 'listRuns').mockResolvedValue([run]) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) renderCanvas({ nodes: [{ id: 'gate', type: 'critic_gate', position: { x: 100, y: 100 }, data: defaultCriticGateNodeData('Marker reviewer') }], @@ -817,7 +817,7 @@ describe('ProtocolCanvas connector adds', () => { it('keeps the Test Run label when a production replicate is running', async () => { const run = protocolRun() - vi.mocked(protocolsApi.listRuns).mockResolvedValue([run]) + vi.mocked(protocolsApi.listRunHeads).mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) renderCanvas({ nodes: [], edges: [] }) @@ -830,7 +830,7 @@ describe('ProtocolCanvas connector adds', () => { it('stops the active production replicate instead of a completed Test Run', async () => { const user = userEvent.setup() const run = protocolRun() - vi.mocked(protocolsApi.listRuns).mockResolvedValue([run]) + vi.mocked(protocolsApi.listRunHeads).mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) vi.spyOn(protocolsApi, 'getLatestTestRun').mockResolvedValue(testRun()) const cancelRun = vi.spyOn(protocolsApi, 'cancelRun').mockResolvedValue({ diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index b07d8ff5..f2665d1b 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -840,7 +840,7 @@ export const ProtocolCanvas = forwardRef protocolsApi.listRuns(protocolId), + queryFn: () => protocolsApi.listRunHeads(protocolId), }) // Seeded once and only into an empty slot: a run launched here must win over diff --git a/frontend/src/components/protocol/ResultJobFlows.test.tsx b/frontend/src/components/protocol/ResultJobFlows.test.tsx new file mode 100644 index 00000000..7e900210 --- /dev/null +++ b/frontend/src/components/protocol/ResultJobFlows.test.tsx @@ -0,0 +1,79 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { experimentsApi, protocolsApi } from '@/api/client' +import type { Protocol } from '@/types/protocols' +import type { ExecutionSlot, ResultJob } from '@/types/executionResults' +import { RunAllCellsButton } from './RunsTab' +import { ResultsPanel } from './ResultsPanel' + +vi.mock('./RunConfirmDialog', () => ({ RunConfirmDialog: ({ additionalContent, onConfirm, onPublishAndRun, confirmDisabled, publishDisabled }: { + additionalContent: React.ReactNode; onConfirm: () => void; onPublishAndRun: () => void; confirmDisabled: boolean; publishDisabled: boolean +}) =>
    {additionalContent}
    })) +vi.mock('./ExperimentDefinitionDownload', () => ({ ExperimentDefinitionDownload: () => null })) +const row: ExecutionSlot = { slot_id: 'stable-slot', cell_id: 'cell', cell_label: 'model=a', factor_values: {}, + replicate_id: 'parent', replicate_label: 'ambiguous-label', replicate_number: 1, row_result_id: null, row_index: null, + dataset_id: null, raw_sha256: null, run_id: 'attempt', status: 'completed', updated_at: null, error: null, result: {}, usage: null } +vi.mock('./ExecutionBrowser', () => ({ ExecutionBrowser: () => null, ExecutionRows: ({ actions }: { actions: (row: ExecutionSlot) => React.ReactNode }) =>
    {actions(row)}
    })) +const protocol = { id: 'protocol', published_revision_id: 'publication', published_revision: 1, graph: { nodes: [], edges: [] } } as unknown as Protocol +const pending: ResultJob = { id: 'job', kind: 'plan', status: 'pending', progress: {}, error: null, protocol_revision_id: 'publication' } + +function mount(children: React.ReactNode) { + return render({children}) +} +beforeEach(() => { + vi.spyOn(experimentsApi, 'createResultJob').mockResolvedValue(pending) + vi.spyOn(experimentsApi, 'getResultJob').mockResolvedValue({ ...pending, status: 'awaiting_confirmation', progress: { planned: 15786 } }) + vi.spyOn(experimentsApi, 'confirmResultJob').mockResolvedValue({ ...pending, status: 'queued' }) + vi.spyOn(experimentsApi, 'cancelResultJob').mockResolvedValue({ ...pending, status: 'cancelled' }) + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue({ ready: false, metric_types: {}, metric_directions: {} } as never) + vi.spyOn(experimentsApi, 'get').mockResolvedValue({ locked_at: 'already locked' } as never) + vi.spyOn(protocolsApi, 'runCells') + vi.spyOn(experimentsApi, 'downloadRunResultsCsv') +}) +afterEach(() => vi.restoreAllMocks()) + +describe('asynchronous result commands', () => { + it('prepares without executing, then confirms the sealed job independently of summary preparation', async () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + await screen.findByText('15786 executions will be queued.') + expect(protocolsApi.runCells).not.toHaveBeenCalled() + expect(experimentsApi.confirmResultJob).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Confirm batch' })) + await waitFor(() => expect(experimentsApi.confirmResultJob).toHaveBeenCalledWith('experiment', 'job')) + expect(experimentsApi.createResultJob).toHaveBeenCalledWith('experiment', + { protocol_id: 'protocol', protocol_revision_id: 'publication' }, + expect.objectContaining({ kind: 'plan' }), expect.any(AbortSignal)) + }) + + it('selects completed work by slot ID and abandons the previous preview safely', async () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + await screen.findByText('15786 executions will be queued.') + fireEvent.click(screen.getByRole('checkbox', { name: 'Run replicate 1 again' })) + await waitFor(() => expect(experimentsApi.createResultJob).toHaveBeenLastCalledWith('experiment', expect.anything(), + expect.objectContaining({ rerun_slot_ids: ['stable-slot'] }), expect.any(AbortSignal))) + expect(experimentsApi.cancelResultJob).toHaveBeenCalledWith('experiment', 'job', true) + expect(experimentsApi.confirmResultJob).not.toHaveBeenCalled() + }) + + it('allows first publication while the batch preview is unavailable', () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + expect(screen.getByRole('button', { name: 'Publish preview' })).toBeEnabled() + expect(screen.getByRole('button', { name: 'Confirm batch' })).toBeDisabled() + expect(experimentsApi.createResultJob).not.toHaveBeenCalled() + }) + + it('downloads a completed background export natively without constructing a browser CSV blob', async () => { + vi.mocked(experimentsApi.getResultJob).mockResolvedValue({ ...pending, kind: 'export', status: 'completed', + download_url: '/api/owned-export?token=signed', schema_url: '/api/owned-schema?token=signed' }) + const click = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {}) + mount(Summary

    } cells={

    Cells

    } />) + fireEvent.click(screen.getByRole('button', { name: 'Download CSV' })) + await waitFor(() => expect(click).toHaveBeenCalledOnce()) + expect(experimentsApi.downloadRunResultsCsv).not.toHaveBeenCalled() + expect(screen.getByRole('link', { name: 'Download prepared CSV' })).toHaveAttribute('href', '/api/owned-export?token=signed') + }) +}) diff --git a/frontend/src/components/protocol/ResultJobProgress.tsx b/frontend/src/components/protocol/ResultJobProgress.tsx new file mode 100644 index 00000000..5792615f --- /dev/null +++ b/frontend/src/components/protocol/ResultJobProgress.tsx @@ -0,0 +1,23 @@ +import { useMutation } from '@tanstack/react-query' +import { useQueryClient } from '@tanstack/react-query' +import { useEffect } from 'react' +import { experimentsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { jobActive, useResultJob } from '@/lib/resultJobs' +import { executionKeys } from '@/lib/executionQueries' + +export function ResultJobProgress({ experimentId, jobId }: { experimentId: string; jobId?: string }) { + const job = useResultJob(experimentId, jobId) + const client = useQueryClient() + const checkpoint = job.data ? JSON.stringify([job.data.status, job.data.progress.created, job.data.progress.cancelled]) : undefined + useEffect(() => { + if (checkpoint) void client.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) + }, [checkpoint, client, experimentId]) + const cancel = useMutation({ mutationFn: () => experimentsApi.cancelResultJob(experimentId, jobId!), onSuccess: () => { void job.refetch() } }) + if (!jobId) return null + return
    +

    {job.data?.kind === 'cancel' ? 'Stopping executions' : 'Batch'} · {job.data?.status.replaceAll('_', ' ') ?? 'loading'} · {job.data?.progress.created ?? 0} queued · {job.data?.progress.skipped ?? 0} skipped{job.data?.progress.cancelled != null && ` · ${job.data.progress.cancelled} stopped`}

    + {job.data?.kind === 'plan' && jobActive(job.data) && } + {(job.data?.error || job.error || cancel.error) &&

    {job.data?.error ?? job.error?.message ?? cancel.error?.message}

    } +
    +} diff --git a/frontend/src/components/protocol/ResultTimelineNode.tsx b/frontend/src/components/protocol/ResultTimelineNode.tsx index 0893560e..d792dbdb 100644 --- a/frontend/src/components/protocol/ResultTimelineNode.tsx +++ b/frontend/src/components/protocol/ResultTimelineNode.tsx @@ -4,6 +4,7 @@ import { Badge } from '@/components/ui/badge' import { nodeRunBadge } from '@/lib/protocolRun' import type { ResultNodeRun } from '@/types/experiments' import { ReceivedPromptPanel, RunStepTrace, UnresolvedReferencesNote } from './NodeRunOutputPanel' +import { ExecutionNodeText, ExecutionNodeTrace, type ExecutionContext } from './ExecutionNodeEvidence' function formatCurrency(value: number | null): string { return value === null || !Number.isFinite(value) ? 'Not reported' : new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) @@ -20,7 +21,7 @@ function nodeStatusClass(status: string): string { return 'border-transparent bg-muted text-muted-foreground' } -export function ResultTimelineNode({ node, defaultOpen = false }: { node: ResultNodeRun; defaultOpen?: boolean }) { +export function ResultTimelineNode({ node, defaultOpen = false, context }: { node: ResultNodeRun; defaultOpen?: boolean; context?: ExecutionContext }) { const [open, setOpen] = useState(defaultOpen) const hasDetails = Boolean(node.error || node.output_text || node.agent_run_id) const bypassBadge = node.status === 'completed' && node.bypassed ? nodeRunBadge('completed', false, true) : null @@ -52,15 +53,15 @@ export function ResultTimelineNode({ node, defaultOpen = false }: { node: Result {node.error &&

    Error

    {node.error}

    } {/* Received before produced, so one node reads as the handoff it was: what it was given, then what it made of it. */} - {node.agent_run_id && } + {node.agent_run_id && (context ? : )} - {node.output_text ? ( + {context && node.output_text ? : node.output_text ? (

    Output

    {node.output_text}

    ) : !node.error &&

    No output was recorded for this node.

    } - {node.agent_run_id && } + {node.agent_run_id && (context ? : )}
    )} diff --git a/frontend/src/components/protocol/ResultsPanel.tsx b/frontend/src/components/protocol/ResultsPanel.tsx index 364747bf..25ac9ae6 100644 --- a/frontend/src/components/protocol/ResultsPanel.tsx +++ b/frontend/src/components/protocol/ResultsPanel.tsx @@ -1,9 +1,11 @@ -import { useEffect, useState, type ReactNode } from 'react' +import { useEffect, useRef, useState, type ReactNode } from 'react' +import { useMutation } from '@tanstack/react-query' import { Download, Maximize2, Minimize2, type LucideIcon } from 'lucide-react' import { experimentsApi, type ResultsScope } from '@/api/client' import { Button } from '@/components/ui/button' import { Card } from '@/components/ui/card' import { sanitizeFilename } from '@/lib/utils' +import { jobActive, useResultJob } from '@/lib/resultJobs' import { InfoTooltip } from './InfoTooltip' export function ResultsScorecard({ label, help, value, note, icon: Icon }: { label: string; help: string; value: string; note?: string; icon: LucideIcon }) { @@ -21,8 +23,21 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su comparison?: ReactNode; cells: ReactNode; details?: ReactNode }) { const [maximized, setMaximized] = useState(false) - const [downloading, setDownloading] = useState(false) - const [error, setError] = useState(null) + const [jobId, setJobId] = useState() + const downloaded = useRef(undefined) + const job = useResultJob(experimentId, jobId) + const createExport = useMutation({ mutationFn: () => experimentsApi.createResultJob(experimentId, scope, + { kind: 'export', request_key: crypto.randomUUID() }), onSuccess: created => setJobId(created.id) }) + const downloading = createExport.isPending || (!!jobId && !job.data && !job.error) || jobActive(job.data) + const error = createExport.error?.message ?? job.error?.message ?? job.data?.error + useEffect(() => { + if (job.data?.status !== 'completed' || !job.data.download_url || downloaded.current === job.data.id) return + downloaded.current = job.data.id + const link = document.createElement('a') + link.href = job.data.download_url + link.download = `${sanitizeFilename(experimentName, 'experiment')}-results.csv` + link.click() + }, [job.data, experimentName]) useEffect(() => { if (!maximized) return const previous = document.body.style.overflow @@ -31,21 +46,6 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su document.addEventListener('keydown', close) return () => { document.body.style.overflow = previous; document.removeEventListener('keydown', close) } }, [maximized]) - async function download() { - setDownloading(true) - setError(null) - try { - const blob = await experimentsApi.downloadRunResultsCsv(experimentId, scope) - const url = URL.createObjectURL(blob) - try { - const link = document.createElement('a') - link.href = url - link.download = `${sanitizeFilename(experimentName, 'experiment')}-results.csv` - link.click() - } finally { URL.revokeObjectURL(url) } - } catch (error) { setError(error instanceof Error ? error.message : 'CSV unavailable') } - finally { setDownloading(false) } - } return
    {controls} @@ -53,12 +53,14 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su {summary}
    - +
    {scorecards &&
    {scorecards}
    } {notice} {error &&

    {error}

    } + {downloading &&

    Preparing CSV · {job.data?.progress.bytes?.toLocaleString() ?? 0} bytes

    } + {job.data?.status === 'completed' && }
    {comparison}
    diff --git a/frontend/src/components/protocol/ResultsTab.tsx b/frontend/src/components/protocol/ResultsTab.tsx index 2a1ce196..ba694bfe 100644 --- a/frontend/src/components/protocol/ResultsTab.tsx +++ b/frontend/src/components/protocol/ResultsTab.tsx @@ -7,6 +7,7 @@ import { OBSERVATION_LABELS } from '@/lib/measurementPlan' import type { EvaluationArtifact, Experiment, MetricObservation, ResultReplicate } from '@/types/experiments' import { ResultsScorecard as Scorecard } from './ResultsPanel' import { ResultTimelineNode } from './ResultTimelineNode' +import type { ExecutionContext } from './ExecutionNodeEvidence' import { ExecutionBrowser } from './ExecutionBrowser' import { ExecutionInspector } from './ExecutionInspector' @@ -97,7 +98,8 @@ function ObservationCard({ observation }: { observation: MetricObservation }) { : JSON.stringify(observation.value, null, 2) return

    {observation.metric_name}

    {OBSERVATION_LABELS[observation.status]}
    - {observation.status === 'measured' && 
    {measuredValue}
    } + {observation.status === 'measured' && 
    {observation.value_truncated ? observation.value_preview : measuredValue}
    } + {observation.value_truncated &&

    Preview · open complete recorded JSON for the original value.

    } {observation.error &&

    {observation.error}

    }

    {observation.producer.producer_id} · v{observation.producer.version}

     @@ -123,6 +125,7 @@ function CalibrationArtifact({ artifact }: { artifact: EvaluationArtifact }) { } function artifactPresentation(artifact: EvaluationArtifact) { + if (artifact.payload_truncated) return { title: formatMetricLabel(artifact.artifact_key), content: <>
    {artifact.payload_preview}

    Preview · open complete recorded JSON for the original artifact.

    } const kind = artifact.kind || artifact.artifact_key if (kind === 'confusion_matrix') return { title: 'Confusion matrix', content: } if (kind === 'binary_calibration_curve' || kind === 'calibration') return { title: 'Calibration', content: } @@ -145,10 +148,12 @@ function usageSummarizesMetric( } -export function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { +export function ReplicateResultDetail({ replicate, metricKeys, metricTypes, executionContext, metricLabels }: { replicate: ResultReplicate metricKeys: string[] metricTypes: ResultMetricTypes + executionContext?: ExecutionContext + metricLabels?: Record }) { const metrics = metricKeys.filter((key) => { if (typeof replicate.metric_values[key] !== 'number') return false @@ -177,9 +182,9 @@ export function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: {

    Outcome

    {metrics.map((key) => { - const observation = observationForMetric(replicate, undefined, key) + const observation = observationForMetric(replicate, key, metricLabels?.[key] ?? key) return
    -

    {formatMetricLabel(key)}

    +

    {metricLabels?.[key] ?? formatMetricLabel(key)}

    {formatResultMetricValue(key, replicate.metric_values[key], metricTypes, true)}

    {observation?.status === 'measured' &&

    {observation.producer.producer_id} · v{observation.producer.version}

    }
    @@ -207,7 +212,7 @@ export function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: {

    Run timeline

    {replicate.node_runs.length === 0 ?

    No node-level run details are available.

    : (
      - {replicate.node_runs.map((node) => )} + {replicate.node_runs.map((node) => )}
    )}
    diff --git a/frontend/src/components/protocol/RunConfirmDialog.tsx b/frontend/src/components/protocol/RunConfirmDialog.tsx index 1b862347..85c58b9e 100644 --- a/frontend/src/components/protocol/RunConfirmDialog.tsx +++ b/frontend/src/components/protocol/RunConfirmDialog.tsx @@ -50,6 +50,7 @@ export function RunConfirmDialog({ confirmLabel, confirmDisabled = false, publishDisabled = confirmDisabled, + publishLabel, isConfirming = false, confirmError = null, }: { @@ -75,6 +76,7 @@ export function RunConfirmDialog({ confirmLabel?: string confirmDisabled?: boolean publishDisabled?: boolean + publishLabel?: string isConfirming?: boolean confirmError?: string | null }) { @@ -222,7 +224,7 @@ export function RunConfirmDialog({ )} ) : ( diff --git a/frontend/src/components/protocol/RunsTab.tsx b/frontend/src/components/protocol/RunsTab.tsx index 30b78234..e551bddc 100644 --- a/frontend/src/components/protocol/RunsTab.tsx +++ b/frontend/src/components/protocol/RunsTab.tsx @@ -6,6 +6,8 @@ import { ApiError, experimentsApi, protocolsApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' import { factorBindingDiscrepancies, type FactorBindingDiscrepancy } from '@/lib/factorBindings' +import { useBatchPreview } from '@/lib/resultJobs' +import { ResultJobProgress } from './ResultJobProgress' import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' import { protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' import type { Experiment } from '@/types/experiments' @@ -17,24 +19,27 @@ import { RunConfirmDialog } from './RunConfirmDialog' import { ReplicateRunAction } from './CellPresentation' import type { ResultsSelection } from './ResultsTab' -export function RunAllCellsButton({ protocol, experimentId, regenerationRequired, unboundFactors, bindingDiscrepancies, label = 'Run all cells', dialogTitle, compact = false, replicateLabels }: { +export function RunAllCellsButton({ protocol, experimentId, regenerationRequired, unboundFactors, bindingDiscrepancies, label = 'Run all cells', dialogTitle, compact = false, cellId }: { protocol: Protocol | undefined; experimentId: string; regenerationRequired: boolean; unboundFactors: string[] bindingDiscrepancies: FactorBindingDiscrepancy[]; label?: string; dialogTitle?: string; compact?: boolean - replicateLabels?: string[]; hasCompletedRun?: boolean + cellId?: string }) { const client = useQueryClient() const [open, setOpen] = useState(false) const [lock, setLock] = useState(false) const [reruns, setReruns] = useState>(() => new Set()) + const [rerunAll, setRerunAll] = useState(false) const scope = { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined } + const [confirmedId, setConfirmedId] = useState() + const preview = useBatchPreview(experimentId, scope, { cell_id: cellId, rerun_slot_ids: [...reruns], rerun_completed: rerunAll }, open) const summary = useExecutionSummary(experimentId, scope, open && !!protocol) const experiment = useQuery({ queryKey: ['experiments', experimentId], queryFn: () => experimentsApi.get(experimentId), enabled: open }) const run = useMutation({ mutationFn: async () => { if (lock && !experiment.data?.locked_at) await experimentsApi.lock(experimentId) - return protocolsApi.runCells(protocol!.id, { replicateLabels, rerunReplicateLabels: [...reruns] }) + return experimentsApi.confirmResultJob(experimentId, preview.data!.id) }, - onSuccess: () => { setOpen(false); void client.invalidateQueries({ queryKey: ['experiments', experimentId] }) }, + onSuccess: job => { setConfirmedId(job.id); setOpen(false); void client.invalidateQueries({ queryKey: ['experiments', experimentId] }) }, }) const publish = useMutation({ mutationFn: async () => { const live = client.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol!.id)) @@ -47,25 +52,30 @@ export function RunAllCellsButton({ protocol, experimentId, regenerationRequired setReruns(new Set()) } }) const blocked = !protocol || regenerationRequired || unboundFactors.length > 0 || bindingDiscrepancies.length > 0 - const pending = (summary.data?.counts?.not_started ?? 0) + (summary.data?.counts?.failed ?? 0) + (summary.data?.counts?.cancelled ?? 0) + const pending = preview.data?.progress.planned ?? 0 return <> + {open && protocol && setOpen(false)} onConfirm={() => run.mutate()} hasUnpublishedChanges={protocol.has_unpublished_changes} publishedRevision={protocol.published_revision} onPublishAndRun={() => publish.mutate()} isPublishing={publish.isPending} publishError={publish.error?.message} - confirmLabel={label} confirmDisabled={!summary.data?.ready || run.isPending || publish.isPending} + publishDisabled={publish.isPending || run.isPending} publishLabel="Publish & prepare preview" + confirmLabel={label} confirmDisabled={preview.data?.status !== 'awaiting_confirmation' || !pending || run.isPending || publish.isPending} isConfirming={run.isPending} confirmError={run.error?.message} additionalContent={
    - {!summary.data?.ready &&

    Preparing execution counts…

    } + {preview.data?.status !== 'awaiting_confirmation' &&

    Preparing batch preview… {preview.data?.progress.scanned ?? 0} executions checked.

    } + {(preview.error || preview.data?.error) &&

    {preview.error?.message ?? preview.data?.error}

    } + {preview.data?.status === 'awaiting_confirmation' &&

    {pending} executions will be queued.

    } {summary.error &&

    {summary.error.message}

    } {summary.data?.unit === 'row' &&

    {summary.data.expected ?? '?'} expected row executions across {summary.data.replicate_count ?? 0} replicates.

    } {!experiment.data?.locked_at && }

    Completed replicates are skipped unless selected to run again.

    - {}} actions={row => row.status === 'completed' && setReruns(current => { const next = new Set(current); if (checked) next.add(row.replicate_label); else next.delete(row.replicate_label); return next })} />} /> + + {}} actions={row => row.status === 'completed' && = 100} onCheckedChange={checked => setReruns(current => { const next = new Set(current); if (checked) next.add(row.slot_id); else next.delete(row.slot_id); return next })} />} />
    } />} } @@ -139,14 +149,19 @@ function RunReplicateButton({ }) { const queryClient = useQueryClient() const [dialogOpen, setDialogOpen] = useState(false) + const [confirmedId, setConfirmedId] = useState() + const preview = useBatchPreview(experimentId, + { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined }, + { replicate_ids: row ? [row.replicate_id] : undefined, + row_indices: row?.row_index != null ? [row.row_index] : undefined, + rerun_slot_ids: hasCompletedRun && row ? [row.slot_id] : [] }, dialogOpen && !!row) const runMutation = useMutation({ mutationFn: async () => { - if (row?.row_index != null) await protocolsApi.runCells(protocol!.id, row.row_result_id && ['failed', 'cancelled'].includes(row.status) - ? { retry_row_result_ids: [row.row_result_id] } - : { replicateLabels: [replicateLabel], row_indices: [row.row_index], rerunReplicateLabels: hasCompletedRun ? [replicateLabel] : [] }) - else await protocolsApi.run(protocol!.id, replicateLabel) + if (row) return experimentsApi.confirmResultJob(experimentId, preview.data!.id) + await protocolsApi.run(protocol!.id, replicateLabel) }, - onSuccess: () => { + onSuccess: job => { + if (job) setConfirmedId(job.id) queryClient.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'replicates'] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) @@ -163,7 +178,6 @@ function RunReplicateButton({ onSuccess: (published) => { queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'revisions'] }) queryClient.setQueryData(protocolForExperimentQueryKey(experimentId), published) - runMutation.mutate() }, }) const runBlocked = !protocol || regenerationRequired || unboundFactors.length > 0 || bindingDiscrepancies.length > 0 @@ -193,6 +207,7 @@ function RunReplicateButton({ {actionLabel} + {dialogOpen && protocol && ( publishAndRunMutation.mutate()} + publishDisabled={publishAndRunMutation.isPending || runMutation.isPending} + publishLabel={row ? 'Publish & prepare preview' : undefined} confirmLabel={`${actionLabel} replicate`} + confirmDisabled={!!row && (preview.data?.status !== 'awaiting_confirmation' || !preview.data.progress.planned)} + additionalContent={row &&

    {preview.data?.status === 'awaiting_confirmation' ? `${preview.data.progress.planned ?? 0} executions will be queued.` : preview.data?.error ?? preview.error?.message ?? 'Preparing batch preview…'}

    } isConfirming={runMutation.isPending} confirmError={errorMessage} /> @@ -242,7 +261,8 @@ export function RunsTab({ experimentId, designSpec, protocol, regenerationRequir else if (selection.type === 'row') onViewRowResult?.(selection.rowResultId) else if (selection.type === 'replicate') onViewResult(selection.replicateLabel) }} - controls={} + controls={<>} + cellActions={cell => <>} actions={row => } + +function StopAllButton({ protocol, experimentId, cellId }: { protocol?: Protocol; experimentId: string; cellId?: string }) { + const [jobId, setJobId] = useState() + const stop = useMutation({ mutationFn: () => experimentsApi.createResultJob(experimentId, + { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined }, + { kind: 'cancel', cell_id: cellId, request_key: crypto.randomUUID() }), onSuccess: job => setJobId(job.id) }) + return <> + + {stop.error &&

    {stop.error.message}

    } +} diff --git a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx index 7df79e61..4963b31a 100644 --- a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx +++ b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx @@ -29,7 +29,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId binding.edge.id !== edgeId && binding.input.mode === 'per_row' && binding.config?.dataset_id?.toLowerCase() !== id?.toLowerCase(), ) - const schema = useQuery({ queryKey: ['datasets', id, 'row-schema'], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id }) + const schema = useQuery({ queryKey: ['datasets', id, 'row-schema'], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id, refetchInterval: query => query.state.data?.ready === false ? 1000 : false }) const availableColumns = schema.data?.columns ?? [] const allColumnsSelected = availableColumns.length > 0 && availableColumns.every(column => columns.includes(column)) const displayedColumns = [...new Set([...availableColumns, ...columns])] @@ -44,7 +44,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId
    const input: DatasetInput = mode === 'per_row' ? { mode, columns } : { mode } const graph = { nodes, edges: edges.map(item => item.id === edgeId ? { ...item, data: { ...item.data, dataset_input: input } } : item) } - const error = !id ? 'Select a registered Dataset.' : schema.isError ? schema.error.message : schema.isLoading ? 'Loading original columns…' : !schema.data ? 'Source metadata unavailable.' : mode === 'per_row' && !schema.data.row_count ? 'The original Dataset has no rows.' : mode === 'per_row' && columns.some(column => !schema.data.columns.includes(column)) ? 'Saved columns are missing from the original CSV.' : datasetRowTopologyError(graph) + const error = !id ? 'Select a registered Dataset.' : schema.isError ? schema.error.message : schema.isLoading || schema.data?.ready === false ? 'Preparing original columns…' : !schema.data ? 'Source metadata unavailable.' : mode === 'per_row' && !schema.data.row_count ? 'The original Dataset has no rows.' : mode === 'per_row' && columns.some(column => !schema.data.columns.includes(column)) ? 'Saved columns are missing from the original CSV.' : datasetRowTopologyError(graph) function apply() { if (error || disabled || !hasChanges) return setEdges(current => current.map(item => { diff --git a/frontend/src/components/protocol/useDatasetRowSelection.ts b/frontend/src/components/protocol/useDatasetRowSelection.ts index 4154d52c..bff8da94 100644 --- a/frontend/src/components/protocol/useDatasetRowSelection.ts +++ b/frontend/src/components/protocol/useDatasetRowSelection.ts @@ -7,10 +7,10 @@ import type { ProtocolGraph } from '@/types/protocols' export function useDatasetRowSelection(graph: ProtocolGraph | undefined, scope: string, nodeId?: string) { const binding = graph && rowBindingForNode(graph, nodeId) const id = binding?.config?.dataset_id - const schema = useQuery({ queryKey: ['datasets', id, 'row-schema', scope], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id, retry: false }) + const schema = useQuery({ queryKey: ['datasets', id, 'row-schema', scope], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id, retry: false, refetchInterval: query => query.state.data?.ready === false ? 1000 : false }) const key = `${scope}:${id}:${schema.data?.raw_sha256}:${nodeId ?? ''}` const [selection, setSelection] = useState({ key, index: 0 }) const rowIndex = selection.key === key ? selection.index : 0 - const error = !binding ? null : !id ? 'Registered row source unavailable.' : schema.isLoading ? 'Loading original rows…' : schema.isError ? schema.error.message : !schema.data ? 'Original row metadata unavailable.' : !schema.data.row_count ? 'Original Dataset has no rows.' : !Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= schema.data.row_count ? 'Source row is outside the original Dataset.' : binding.input.mode === 'per_row' && binding.input.columns.some(column => !schema.data.columns.includes(column)) ? 'Selected columns are absent from the original CSV.' : graph ? datasetRowTopologyError(graph) : null + const error = !binding ? null : !id ? 'Registered row source unavailable.' : schema.isLoading || schema.data?.ready === false ? 'Preparing original rows…' : schema.isError ? schema.error.message : !schema.data ? 'Original row metadata unavailable.' : !schema.data.row_count ? 'Original Dataset has no rows.' : !Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= schema.data.row_count ? 'Source row is outside the original Dataset.' : binding.input.mode === 'per_row' && binding.input.columns.some(column => !schema.data.columns.includes(column)) ? 'Selected columns are absent from the original CSV.' : graph ? datasetRowTopologyError(graph) : null return { binding, schema: schema.data, rowIndex, setRowIndex: (index: number) => setSelection({ key, index }), error } } diff --git a/frontend/src/lib/executionPagination.ts b/frontend/src/lib/executionPagination.ts new file mode 100644 index 00000000..7028babf --- /dev/null +++ b/frontend/src/lib/executionPagination.ts @@ -0,0 +1,9 @@ +import { useCallback, useState } from 'react' + +export function useCursor() { + const [cursors, setCursors] = useState<(string | undefined)[]>([undefined]) + const next = useCallback((cursor: string) => setCursors(current => [...current, cursor]), []) + const previous = useCallback(() => setCursors(current => current.slice(0, -1)), []) + const reset = useCallback(() => setCursors([undefined]), []) + return { cursor: cursors.at(-1), back: cursors.length > 1, next, previous, reset } +} diff --git a/frontend/src/lib/experiment.ts b/frontend/src/lib/experiment.ts index 579d8336..8f6f4737 100644 --- a/frontend/src/lib/experiment.ts +++ b/frontend/src/lib/experiment.ts @@ -375,8 +375,11 @@ export function heatColor(value: number, min: number, max: number): string { * "primary" (cyan, some replicates scored) / "muted-foreground" (dim, no cells yet). */ export function replicatesStatusAccent(replicates: Replicate[] | undefined): string { const total = replicates?.length ?? 0 + return cellsStatusAccent(total, replicates?.filter(replicate => replicate.metric_values).length ?? 0) +} + +export function cellsStatusAccent(total: number, scored: number): string { if (total === 0) return 'var(--muted-foreground)' - const scored = replicates!.filter((replicate) => replicate.metric_values).length if (scored === 0) return 'var(--chart-4)' if (scored === total) return 'var(--chart-3)' return 'var(--primary)' diff --git a/frontend/src/lib/resultJobs.ts b/frontend/src/lib/resultJobs.ts new file mode 100644 index 00000000..11f1639e --- /dev/null +++ b/frontend/src/lib/resultJobs.ts @@ -0,0 +1,28 @@ +import { useEffect, useRef } from 'react' +import { useQuery } from '@tanstack/react-query' +import { experimentsApi, type ResultsScope } from '@/api/client' +import type { ResultJob } from '@/types/executionResults' + +export const jobActive = (job?: ResultJob) => !!job && ['pending', 'queued', 'running'].includes(job.status) + +export function useResultJob(experimentId: string, id?: string) { + return useQuery({ queryKey: ['experiments', experimentId, 'result-job', id], + queryFn: ({ signal }) => experimentsApi.getResultJob(experimentId, id!, signal), enabled: !!id, + refetchInterval: query => jobActive(query.state.data) ? 1000 : false }) +} + +export function useBatchPreview(experimentId: string, scope: ResultsScope, selection: { cell_id?: string; replicate_ids?: string[]; row_indices?: number[]; rerun_slot_ids?: string[]; rerun_completed?: boolean }, enabled: boolean) { + const key = JSON.stringify({ scope, selection, enabled }) + const request = useRef({ key, id: crypto.randomUUID() }) + if (request.current.key !== key) request.current = { key, id: crypto.randomUUID() } + const requestKey = request.current.id + const prepare = useQuery({ queryKey: ['experiments', experimentId, 'batch-preview', requestKey], + queryFn: ({ signal }) => experimentsApi.createResultJob(experimentId, scope, { kind: 'plan', request_key: requestKey, ...selection }, signal), + enabled: enabled && !!scope.protocol_revision_id, staleTime: Infinity }) + const job = useResultJob(experimentId, enabled ? prepare.data?.id : undefined) + useEffect(() => { + const id = prepare.data?.id + return () => { if (id) void experimentsApi.cancelResultJob(experimentId, id, true).catch(() => {}) } + }, [experimentId, prepare.data?.id, enabled]) + return { data: job.data ?? prepare.data, error: job.error ?? prepare.error, isFetching: prepare.isFetching || job.isFetching } +} diff --git a/frontend/src/pages/ExperimentsPage.tsx b/frontend/src/pages/ExperimentsPage.tsx index 14cf9318..0dcf50bd 100644 --- a/frontend/src/pages/ExperimentsPage.tsx +++ b/frontend/src/pages/ExperimentsPage.tsx @@ -1,5 +1,5 @@ import { useMemo, useState } from 'react' -import { useQueries, useQuery } from '@tanstack/react-query' +import { useQueries, useQuery, type Query } from '@tanstack/react-query' import { Archive, ArrowDown, @@ -20,10 +20,12 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { formatDate, formatRelative } from '@/lib/format' -import { factorCount, replicatesStatusAccent } from '@/lib/experiment' +import { cellsStatusAccent, factorCount } from '@/lib/experiment' +import { executionKeys } from '@/lib/executionQueries' import { cardAccent } from '@/lib/utils' import { experimentsApi } from '@/api/client' import type { Experiment } from '@/types/experiments' +import type { ExecutionSummary } from '@/types/executionResults' type SortKey = 'name' | 'design_type' | 'created_at' type SortDir = 'asc' | 'desc' @@ -67,18 +69,16 @@ export function ExperimentsPage() { const currentPage = Math.min(page, totalPages) const paged = sorted?.slice((currentPage - 1) * PAGE_SIZE, currentPage * PAGE_SIZE) - // One replicate-list fetch per visible tile, to tint each by completion status -- - // the same N+1 tradeoff already accepted for the Agents section on the - // detail page. Fine at a page size of 9; revisit with a real aggregate - // endpoint if this page ever needs to show many more at once. - const replicateQueries = useQueries({ + // Only maintained summaries cross the list/detail navigation boundary. + const summaryQueries = useQueries({ queries: (paged ?? []).map((experiment) => ({ - queryKey: ['experiments', experiment.id, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experiment.id), + queryKey: executionKeys.summary(experiment.id, {}), + queryFn: ({ signal }) => experimentsApi.getExecutionSummary(experiment.id, {}, signal), + refetchInterval: (query: Query) => query.state.error && !query.state.data ? false : !query.state.data?.ready || query.state.data.refreshing || query.state.data.has_active ? 5000 : false, })), }) const accentByExperiment = new Map( - (paged ?? []).map((experiment, i) => [experiment.id, replicatesStatusAccent(replicateQueries[i]?.data)]), + (paged ?? []).map((experiment, i) => [experiment.id, cellsStatusAccent(summaryQueries[i]?.data?.total ?? 0, summaryQueries[i]?.data?.scored_count ?? 0)]), ) return ( diff --git a/frontend/src/pages/ProtocolCanvasPage.test.tsx b/frontend/src/pages/ProtocolCanvasPage.test.tsx index 22ffe28d..0c42cfed 100644 --- a/frontend/src/pages/ProtocolCanvasPage.test.tsx +++ b/frontend/src/pages/ProtocolCanvasPage.test.tsx @@ -26,6 +26,7 @@ it('switches canvas and design together and preserves local draft edits on retur vi.spyOn(experimentsApi, 'listReplicates').mockResolvedValue([]) vi.spyOn(experimentsApi, 'getDesignImpact').mockResolvedValue({} as Awaited>) vi.spyOn(experimentsApi, 'getRunResults').mockRejectedValue(new Error('No results')) + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue({ ready: false, metric_types: {}, metric_directions: {} } as never) vi.spyOn(experimentsApi, 'listTrials').mockResolvedValue([]) vi.spyOn(protocolsApi, 'list').mockResolvedValue([protocol]) vi.spyOn(protocolsApi, 'listRevisions').mockResolvedValue([...versions, { id: 'legacy', protocol_id: 'p', revision: 0, graph: { nodes: [], edges: [] }, published_at: '2025-12-01', experiment_snapshot: null }]) @@ -33,6 +34,8 @@ it('switches canvas and design together and preserves local draft edits on retur render(} />) const draft = await screen.findByRole('textbox', { name: 'Draft canvas' }) await screen.findByText('Results design: Published') + expect(experimentsApi.getRunResults).not.toHaveBeenCalled() + expect(experimentsApi.listTrials).not.toHaveBeenCalled() const historyButton = screen.getByRole('button', { name: 'Version history' }) expect(historyButton.closest('[aria-label="Experiment version controls"]')).toBeInTheDocument() expect(screen.getByLabelText('Experiment panels')).not.toContainElement(historyButton) diff --git a/frontend/src/pages/ProtocolCanvasPage.tsx b/frontend/src/pages/ProtocolCanvasPage.tsx index 147ed459..41f49f22 100644 --- a/frontend/src/pages/ProtocolCanvasPage.tsx +++ b/frontend/src/pages/ProtocolCanvasPage.tsx @@ -1,7 +1,7 @@ import { useEffect, useRef, useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { ReactFlowProvider } from '@xyflow/react' -import { AlertTriangle, Lock, Target, Trophy, type LucideIcon } from 'lucide-react' +import { Lock, Target, Trophy, type LucideIcon } from 'lucide-react' import { Link, useParams } from 'react-router-dom' import { AppHeader } from '@/components/AppHeader' import { ExperimentSidePanel } from '@/components/protocol/ExperimentSidePanel' @@ -16,7 +16,9 @@ import { Input } from '@/components/ui/input' import { Textarea } from '@/components/ui/textarea' import { Skeleton } from '@/components/ui/skeleton' import { ApiError, experimentsApi, protocolsApi } from '@/api/client' -import { bestMetric, formatMetricLabel, groupReplicatesIntoCells, metricValueSuffix, replicatesStatusAccent, scaledMetricValue } from '@/lib/experiment' +import { cellsStatusAccent, formatMetricLabel, formatMetricValue } from '@/lib/experiment' +import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' +import type { ExecutionSummary } from '@/types/executionResults' import { unboundFactorNames } from '@/lib/factorBindings' import { applyExperimentRenameToProtocolCache, @@ -25,7 +27,7 @@ import { protocolGraphQueryKey, toPersistedGraph, } from '@/lib/protocolGraph' -import type { Experiment, Replicate } from '@/types/experiments' +import type { Experiment } from '@/types/experiments' import type { Protocol } from '@/types/protocols' import type { Node, Edge } from '@xyflow/react' @@ -114,35 +116,16 @@ function TopBarStat({ icon: Icon, value, title, accent }: { icon: LucideIcon; va // Same design_type gate the Cells tab uses -- cells/metric_values are a // factorial concept, so a future non-factorial experiment gets no chips // rather than a nonsensical "0/0 scored". -function TopBarStats({ experiment, cells, obsoleteRunCount = 0, rowSummary }: { experiment: Experiment; cells: Replicate[] | undefined; obsoleteRunCount?: number; rowSummary?: import('@/types/experiments').RowSummary | null }) { - if (rowSummary) return - if (experiment.design_type !== 'factorial' || !cells) return null - const scored = cells.filter((c) => c.metric_values).length - const cellCount = groupReplicatesIntoCells(cells).length - const best = bestMetric(experiment, cells) - return ( - <> - - {obsoleteRunCount > 0 && ( - - {obsoleteRunCount} obsolete run{obsoleteRunCount === 1 ? '' : 's'} - - )} - {best && ( - - )} - - ) +function TopBarStats({ experiment, summary }: { experiment: Experiment; summary?: ExecutionSummary }) { + if (experiment.design_type !== 'factorial' || !summary?.ready) return null + const done = summary.counts?.completed ?? 0 + const total = summary.expected ?? summary.total ?? 0 + const accent = cellsStatusAccent(total, summary.scored_count ?? 0) + const best = summary.best_cell + return <> + {best && } } function ProtocolPublicationControl({ protocol, experimentId, draftBusy }: { protocol: Protocol; experimentId: string; draftBusy: boolean }) { @@ -150,15 +133,8 @@ function ProtocolPublicationControl({ protocol, experimentId, draftBusy }: { pro const [confirmOpen, setConfirmOpen] = useState(false) const [versionName, setVersionName] = useState('') const [versionNote, setVersionNote] = useState('') - const trialsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'runs'], - queryFn: () => experimentsApi.listTrials(experimentId), - enabled: protocol.has_unpublished_changes, - }) - // Obsolescence is based on immutable protocol-run provenance. Results that - // were scored directly without a ProtocolRun have no canvas version to - // become stale against, so do not overstate the impact in this warning. - const affectedReplicateCount = (trialsQuery.data ?? []).filter((trial) => !!trial.run_id && !trial.obsolete).length + const summary = useExecutionSummary(experimentId, { protocol_id: protocol.id, protocol_revision_id: protocol.published_revision_id ?? undefined }, confirmOpen) + const affectedReplicateCount = summary.data?.executed_count ?? 0 const publishMutation = useMutation({ mutationFn: async () => { const live = queryClient.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol.id)) @@ -168,7 +144,7 @@ function ProtocolPublicationControl({ protocol, experimentId, draftBusy }: { pro onSuccess: (published) => { queryClient.setQueryData(protocolForExperimentQueryKey(experimentId), published) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'run-results'] }) + queryClient.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) queryClient.invalidateQueries({ queryKey: ['protocols', protocol.id, 'revisions'] }) setConfirmOpen(false) setVersionName('') @@ -196,7 +172,7 @@ function ProtocolPublicationControl({ protocol, experimentId, draftBusy }: { pro