From b81a9264dc79e56d038542c2d7b81e2e4ad849fd Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 6 Oct 2026 20:14:31 -0700 Subject: [PATCH] Fix individual dataset row runs and result output presentation --- frontend/src/api/client.ts | 3 +- frontend/src/api/datasetRows.test.ts | 6 ++ .../components/protocol/DatasetRowCells.tsx | 4 +- .../protocol/DatasetRowHistory.test.tsx | 53 ++++++++++++++- .../components/protocol/DatasetRowResults.tsx | 39 +++++++++-- .../protocol/DatasetRowRetry.test.tsx | 4 +- .../protocol/DatasetRowRuns.test.tsx | 31 ++++++++- .../components/protocol/DatasetRowRuns.tsx | 43 ++++++++++-- .../protocol/ResultTimelineNode.tsx | 67 +++++++++++++++++++ .../src/components/protocol/ResultsTab.tsx | 61 ++--------------- frontend/src/types/experiments.ts | 1 + src/asaree/api/protocols.py | 3 + src/asaree/services/dataset_row_planning.py | 11 ++- src/asaree/services/experiment_run_results.py | 16 ++++- src/asaree/services/protocol_execution.py | 9 +++ tests/test_dataset_row_batch_planning.py | 59 ++++++++++++++++ tests/test_dataset_row_candidates.py | 19 ++++++ tests/test_experiment_run_results.py | 15 +++++ 18 files changed, 363 insertions(+), 81 deletions(-) create mode 100644 frontend/src/components/protocol/ResultTimelineNode.tsx diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index ea31d26c..8f008d70 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -341,7 +341,7 @@ export const protocolsApi = { // "Run all cells" -- 422 if there's no linked experiment or the graph // doesn't have exactly one final node; fans out one ProtocolRun per pending // replicate. The optional list explicitly re-runs selected completed rows. - runCells: (id: string, options?: { replicateLabels?: string[]; rerunReplicateLabels?: string[]; retry_row_result_ids?: string[] }) => + runCells: (id: string, options?: { replicateLabels?: string[]; rerunReplicateLabels?: string[]; retry_row_result_ids?: string[]; row_indices?: number[] }) => request(`/protocols/${id}/cell-runs`, { method: 'POST', body: options @@ -349,6 +349,7 @@ export const protocolsApi = { replicate_labels: options.replicateLabels, rerun_replicate_labels: options.retry_row_result_ids ? undefined : options.rerunReplicateLabels ?? [], retry_row_result_ids: options.retry_row_result_ids, + row_indices: options.row_indices, } : undefined, }), diff --git a/frontend/src/api/datasetRows.test.ts b/frontend/src/api/datasetRows.test.ts index d2271cda..0668790c 100644 --- a/frontend/src/api/datasetRows.test.ts +++ b/frontend/src/api/datasetRows.test.ts @@ -37,6 +37,12 @@ describe('row HTTP contracts', () => { await protocolsApi.runCells('p', { retry_row_result_ids: ['slot'] }) expect(lastBody()).toEqual({ retry_row_result_ids: ['slot'] }) }) + it('selects a source row within a replicate for a run or rerun', async () => { + await protocolsApi.runCells('p', { replicateLabels: ['rep'], row_indices: [0] }) + expect(lastBody()).toEqual({ replicate_labels: ['rep'], rerun_replicate_labels: [], row_indices: [0] }) + await protocolsApi.runCells('p', { replicateLabels: ['rep'], rerunReplicateLabels: ['rep'], row_indices: [2] }) + expect(lastBody()).toEqual({ replicate_labels: ['rep'], rerun_replicate_labels: ['rep'], row_indices: [2] }) + }) it('uses identical encoded scope on projection and exports', async () => { const scope = { protocol_id: 'p /', design_revision_id: 'd', protocol_revision_id: 'r' } await experimentsApi.getRunResults('e', scope) diff --git a/frontend/src/components/protocol/DatasetRowCells.tsx b/frontend/src/components/protocol/DatasetRowCells.tsx index b84fe552..23f1c9fc 100644 --- a/frontend/src/components/protocol/DatasetRowCells.tsx +++ b/frontend/src/components/protocol/DatasetRowCells.tsx @@ -71,9 +71,9 @@ export function DatasetRowCells({ results, selectedCellId, onSelect, designSpec, return {done}/{rowCount ?? '?'} row executions complete{active > 0 && ` · ${active} active`}{errors > 0 && ` · ${errors} failed`}} status={} - onView={hasAttempt && onSelectReplicate ? () => onSelectReplicate(cell.cell_id, replicate.replicate_result_id) : undefined} + onView={(hasAttempt || !!onRun) && onSelectReplicate ? () => onSelectReplicate(cell.cell_id, replicate.replicate_result_id) : undefined} viewAriaLabel={resultView ? `View ${cell.cell_label} replicate ${replicate.replicate_number}` : undefined} - viewLabel="View results" + viewLabel={resultView ? 'View results' : 'View rows'} actions={<>{onRun && 0 || !replicate.replicate_label} onClick={() => onRun([replicate.replicate_label], 1, done > 0 || errors > 0 ? [replicate.replicate_label] : [])} label={`Run ${cell.cell_label} replicate ${replicate.replicate_number}`}>{done > 0 || errors > 0 ? 'Re-run' : 'Run'}}{onStop?.(replicateRows.filter(row => ['pending', 'running', 'finalizing'].includes(row.status) && row.latest_attempt).map(row => row.latest_attempt!.run_id))}} /> })} diff --git a/frontend/src/components/protocol/DatasetRowHistory.test.tsx b/frontend/src/components/protocol/DatasetRowHistory.test.tsx index 86660bfb..924303d4 100644 --- a/frontend/src/components/protocol/DatasetRowHistory.test.tsx +++ b/frontend/src/components/protocol/DatasetRowHistory.test.tsx @@ -1,11 +1,62 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { fireEvent, render, screen, waitFor } from '@testing-library/react' import { afterEach, expect, it, vi } from 'vitest' -import { experimentsApi } from '@/api/client' +import { experimentsApi, protocolsApi } from '@/api/client' +import type { ProtocolRevision } from '@/types/protocols' import type { Experiment, ExperimentRunResults } from '@/types/experiments' import { ResultsInspectorPanel, ResultsTab } from './ResultsTab' +import { DatasetRowDetail } from './DatasetRowResults' +import type { RowResult } from '@/types/experiments' afterEach(() => vi.restoreAllMocks()) vi.mock('./NodeRunOutputPanel', () => ({ RunStepTrace: ({ runId }: { runId: string }) =>

Trace {runId}

, ReceivedPromptPanel: () => null, UnresolvedReferencesNote: () => null })) +it('hides empty configuration nodes but preserves output, traces and execution failures', () => { + const attempt = { run_id: 'run', status: 'completed', node_labels: { + model: 'Model config', dataset: 'Dataset config', skipped: 'Skipped node', answer: 'Answer agent', + trace: 'Trace agent', failed: 'Failed agent', active: 'Active agent', cancelled: 'Cancelled agent', + }, node_runs: { + model: { status: 'completed', output_text: null, error: null }, + dataset: { status: 'completed', output_text: '' }, + skipped: { status: 'skipped' }, + answer: { status: 'completed', output_text: 'Final answer' }, + trace: { status: 'completed', run_id: 'agent-run' }, + failed: { status: 'failed', error: 'Execution failed' }, + active: { status: 'running' }, + cancelled: { status: 'cancelled' }, + } } + const row = { row_result_id: 'slot', replicate_number: 1, dataset_row: { row_index: 0 }, latest_attempt: attempt, attempts: [attempt] } as unknown as RowResult + render() + for (const name of ['Model config', 'Dataset config', 'Skipped node']) expect(screen.queryByText(name)).not.toBeInTheDocument() + for (const name of ['Answer agent', 'Trace agent', 'Failed agent', 'Active agent', 'Cancelled agent']) expect(screen.getByText(name)).toBeInTheDocument() + expect(screen.getAllByRole('listitem')).toHaveLength(5) +}) +it('resolves friendly names from the pinned publication when row results omit labels', async () => { + const attempt = { run_id: 'run', status: 'completed', protocol_revision_id: 'publication-old', node_runs: { 'node-123': { status: 'completed', output_text: 'Answer' } } } + const row = { row_result_id: 'slot', replicate_number: 1, protocol_revision_id: 'publication-old', dataset_row: { row_index: 0 }, latest_attempt: attempt, attempts: [attempt] } + vi.spyOn(experimentsApi, 'getRunResults').mockResolvedValue({ row_results: [row] } as unknown as ExperimentRunResults) + const revision = vi.spyOn(protocolsApi, 'getRevision').mockResolvedValue({ graph: { nodes: [{ id: 'node-123', type: 'agent', data: { label: 'Published researcher' } }], edges: [] } } as unknown as ProtocolRevision) + render() + expect(await screen.findByRole('button', { name: /Published researcher/ })).toBeInTheDocument() + expect(screen.queryByText('Canvas node')).not.toBeInTheDocument() + expect(revision).toHaveBeenCalledWith('p', 'publication-old') +}) +it('shows published node names with expandable text outputs instead of raw attempt JSON', () => { + const attempt = { run_id: 'run', status: 'completed', node_labels: { 'node-123': 'Researcher', 'node-456': 'Reviewer' }, + node_runs: { 'node-123': { status: 'completed', output_text: 'Research findings' }, 'node-456': { status: 'completed', output_text: 'Review answer' } }, + attempt_result: { internal_marker: 'Raw attempt dump' }, + } + const row = { row_result_id: 'slot', replicate_number: 1, dataset_row: { row_index: 0 }, latest_attempt: attempt, attempts: [attempt] } as unknown as RowResult + render() + expect(screen.getByRole('button', { name: /Researcher/ })).toHaveAttribute('aria-expanded', 'false') + expect(screen.getByRole('button', { name: /Reviewer/ })).toHaveAttribute('aria-expanded', 'true') + expect(screen.queryByText('Research findings')).not.toBeInTheDocument() + expect(screen.getByText('Review answer')).toBeInTheDocument() + expect(screen.queryByText(/Raw attempt dump/)).not.toBeInTheDocument() + expect(screen.queryByText('node-123')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: /Researcher/ })) + expect(screen.getByText('Research findings')).toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: /Reviewer/ })) + expect(screen.queryByText('Review answer')).not.toBeInTheDocument() +}) it('preserves failed attempt output while inspecting the latest row', async () => { const row = { row_result_id: 'slot', replicate_number: 1, dataset_row: { dataset_id: 'dataset-uuid', raw_sha256: 'original-hash', row_index: 2 }, design_revision_id: 'design', protocol_revision_id: 'publication', latest_attempt: { run_id: 'B' }, attempts: [{ run_id: 'A', status: 'failed', current: false, error: 'old failure', node_runs: { agent: { run_id: 'trace-A', output_text: 'old output' } }, attempt_result: {} }, { run_id: 'B', status: 'completed', current: true, node_runs: { agent: { run_id: 'trace-B', output_text: 'new output' } }, attempt_result: {} }] } vi.spyOn(experimentsApi, 'getRunResults').mockResolvedValue({ row_results: [row] } as unknown as ExperimentRunResults) diff --git a/frontend/src/components/protocol/DatasetRowResults.tsx b/frontend/src/components/protocol/DatasetRowResults.tsx index 5878eb63..d0ce6d2c 100644 --- a/frontend/src/components/protocol/DatasetRowResults.tsx +++ b/frontend/src/components/protocol/DatasetRowResults.tsx @@ -1,10 +1,12 @@ import { useState } from 'react' -import type { ResultsScope } from '@/api/client' +import { useQuery } from '@tanstack/react-query' +import { protocolsApi, type ResultsScope } from '@/api/client' import { Button } from '@/components/ui/button' import { reportedMetricText } from '@/lib/reportedMetrics' import { displayFactorLevel } from '@/lib/experiment' import type { Experiment, ExperimentRunResults, RowResult } from '@/types/experiments' -import { RunStepTrace } from './NodeRunOutputPanel' +import { ResultTimelineNode } from './ResultTimelineNode' +import { referenceLabel } from '@/lib/promptReferences' import { DatasetRowCells } from './DatasetRowCells' import { ChevronRight, AlertTriangle, CircleCheck, CircleHelp } from 'lucide-react' import { ResultsPanel, ResultsScorecard } from './ResultsPanel' @@ -43,14 +45,39 @@ export function DatasetRowResults({ results, experiment, scope, onInspect }: { r /> } -export function DatasetRowDetail({ row, onClose }: { row: RowResult; onClose: () => void }) { +export function DatasetRowDetail({ row, protocolId, onClose }: { row: RowResult; protocolId?: string; onClose: () => void }) { const [attemptId, setAttemptId] = useState(row.latest_attempt?.run_id) const attempt = row.attempts.find(item => item.run_id === attemptId) ?? row.latest_attempt - const provenance = attempt?.attempt_result?.row_provenance + // Match the whole-dataset timeline: completed configuration nodes have + // no execution details, while agent traces and failures remain inspectable. + const visibleNodeRuns = Object.entries(attempt?.node_runs ?? {}).filter(([, node]) => + !!node.run_id || !!node.output_text || !!node.error || ['running', 'failed', 'cancelled'].includes(node.status), + ) + const suppliedLabels = { ...row.latest_attempt?.node_labels, ...attempt?.node_labels } + const revisionId = attempt?.protocol_revision_id ?? row.protocol_revision_id + const needsLabels = visibleNodeRuns.some(([id]) => !suppliedLabels[id]) + const revision = useQuery({ + queryKey: ['protocols', protocolId, 'revision', revisionId], + queryFn: () => protocolsApi.getRevision(protocolId!, revisionId), + enabled: !!protocolId && !!revisionId && needsLabels, + staleTime: Infinity, + }) + const publishedLabels = Object.fromEntries((revision.data?.graph.nodes ?? []).map(node => { + const name = node.data.label + return [node.id, typeof name === 'string' && name.trim() ? name.trim() : node.type === 'agent' ? 'Agent' : 'Canvas node'] + })) + const nodeLabels = { ...publishedLabels, ...suppliedLabels } + const nodes = visibleNodeRuns.map(([id, node]) => ({ + node_id: id, node_label: nodeLabels[id] ?? (revision.isFetching ? 'Loading node name…' : 'Canvas node'), status: node.status, + output_text: node.output_text ?? null, error: node.error ?? null, agent_run_id: node.run_id ?? null, + unresolved_reference_labels: (node.unresolved_references ?? []).map(ref => referenceLabel(ref, nodeLabels)), + input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, + })) + const defaultOpenNodeId = nodes.findLast(node => !!node.output_text || !!node.error)?.node_id return } diff --git a/frontend/src/components/protocol/DatasetRowRetry.test.tsx b/frontend/src/components/protocol/DatasetRowRetry.test.tsx index b47e6715..a313d1a6 100644 --- a/frontend/src/components/protocol/DatasetRowRetry.test.tsx +++ b/frontend/src/components/protocol/DatasetRowRetry.test.tsx @@ -7,13 +7,13 @@ import type { Protocol } from '@/types/protocols' import { DatasetRowRuns } from './DatasetRowRuns' afterEach(() => vi.restoreAllMocks()) -it('retries only the failed row and keeps completed unscored disabled', async () => { +it('retries only the failed row', async () => { const retry = vi.spyOn(protocolsApi, 'runCells').mockRejectedValue(new Error('invalid_retry_target')) const results = { row_summary: { cell_count: 1, parent_replicate_count: 1, row_count: 2, expected: 2 }, row_results: ['failed','completed'].map((status,index) => ({ row_result_id: `slot-${index}`, cell_label: 'same', dataset_row: { row_index: index }, replicate_number: 1, status, protocol_revision_id: 'r', latest_attempt: { run_id: `run-${index}` } })) } as unknown as ExperimentRunResults const protocol = { id: 'p', graph: { nodes: [], edges: [] }, published_revision_id: 'r' } as unknown as Protocol render() const buttons = screen.getAllByRole('button', { name: 'Retry' }) - expect(buttons[1]).toBeDisabled() + expect(buttons).toHaveLength(1) fireEvent.click(buttons[0]) await waitFor(() => expect(retry).toHaveBeenCalledWith('p', { retry_row_result_ids: ['slot-0'] })) expect(await screen.findByRole('alert')).toHaveTextContent('invalid_retry_target') diff --git a/frontend/src/components/protocol/DatasetRowRuns.test.tsx b/frontend/src/components/protocol/DatasetRowRuns.test.tsx index 309c8e23..9ca615ca 100644 --- a/frontend/src/components/protocol/DatasetRowRuns.test.tsx +++ b/frontend/src/components/protocol/DatasetRowRuns.test.tsx @@ -1,5 +1,5 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' import { afterEach, expect, it, vi } from 'vitest' import { experimentsApi, protocolsApi } from '@/api/client' import type { ExperimentRunResults } from '@/types/experiments' @@ -22,6 +22,32 @@ export function mountRows(inspect = vi.fn(), results = rowResults) { render() return client } +it('runs an unstarted source row in one selected replicate', async () => { + const run = vi.spyOn(protocolsApi, 'runCells').mockResolvedValue({} as never) + const results = { ...rowResults, row_results: [], row_cells: [{ cell_id: 'a', cell_label: 'short', factor_values: {}, replicate_count: 2, replicates: [ + { replicate_result_id: 'parent', replicate_label: 'short', replicate_number: 1 }, + { replicate_result_id: 'parent-2', replicate_label: 'short__rep2', replicate_number: 2 }, + ] }] } + render() + const sourceRows = screen.getAllByRole('row') + expect(sourceRows).toHaveLength(7) + fireEvent.click(within(sourceRows[2]).getByRole('button', { name: 'Run' })) + expect(run).not.toHaveBeenCalled() + expect(screen.getByText('Run source row 2?')).toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Run row' })) + await waitFor(() => expect(run).toHaveBeenCalledWith('p', { replicateLabels: ['short'], rerunReplicateLabels: [], row_indices: [1] })) +}) + +it('reruns only a finished source row and leaves active rows with stop controls', async () => { + const run = vi.spyOn(protocolsApi, 'runCells').mockResolvedValue({} as never) + render() + expect(screen.getAllByRole('button', { name: 'Re-run' })).toHaveLength(1) + expect(screen.getAllByRole('button', { name: 'Stop' })).toHaveLength(1) + fireEvent.click(screen.getByRole('button', { name: 'Re-run' })) + fireEvent.click(screen.getByRole('button', { name: 'Re-run row' })) + await waitFor(() => expect(run).toHaveBeenCalledWith('p', { replicateLabels: ['same'], rerunReplicateLabels: ['same'], row_indices: [1] })) +}) + it('renders authoritative forecast and distinct row identities', async () => { const inspect = vi.fn() mountRows(inspect) @@ -53,7 +79,8 @@ it('shows generated cells before row executions are planned', () => { fireEvent.click(screen.getByRole('button', { name: 'View cell Prompt:short' })) expect(screen.getByText('Replicate 1')).toBeInTheDocument() expect(screen.getByText('Replicate 2')).toBeInTheDocument() - expect(screen.queryByRole('button', { name: 'View results' })).not.toBeInTheDocument() + expect(screen.getAllByRole('button', { name: 'View results' }).every(button => button.hasAttribute('disabled'))).toBe(true) + expect(screen.getAllByRole('button', { name: 'View rows' })).toHaveLength(2) expect(screen.queryByRole('button', { name: 'View Prompt:short replicate 1' })).not.toBeInTheDocument() expect(screen.getByRole('button', { name: 'Run Prompt:short replicate 1' })).toBeEnabled() expect(screen.getByRole('button', { name: 'Run all replicates in Prompt:short' })).toBeEnabled() diff --git a/frontend/src/components/protocol/DatasetRowRuns.tsx b/frontend/src/components/protocol/DatasetRowRuns.tsx index 9d2c3d88..9b603b31 100644 --- a/frontend/src/components/protocol/DatasetRowRuns.tsx +++ b/frontend/src/components/protocol/DatasetRowRuns.tsx @@ -2,7 +2,7 @@ import { useState } from 'react' import { useMutation, useQueryClient } from '@tanstack/react-query' import { protocolsApi } from '@/api/client' import { Button } from '@/components/ui/button' -import type { DesignSpec, ExperimentRunResults, RowSummary } from '@/types/experiments' +import type { DesignSpec, ExperimentRunResults, RowResult, RowSummary } from '@/types/experiments' import type { Protocol } from '@/types/protocols' import { RunConfirmDialog } from './RunConfirmDialog' import { DatasetRowCells } from './DatasetRowCells' @@ -14,12 +14,34 @@ function rowExecutionForecast(summary: RowSummary): string { return `${summary.cell_count} cells / ${summary.parent_replicate_count} replicates / ${summary.row_count ?? 'unknown'} rows / ${summary.expected ?? 'unknown'} executions` } +type RunnableRow = Pick & { + row_result_id?: string + dataset_row: { row_index: number } + status: RowResult['status'] | 'not_started' +} + +function runnableRows(results: ExperimentRunResults, protocol: Protocol): RunnableRow[] { + if (!results.row_cells || results.row_summary?.row_count == null) return results.row_results ?? [] + const existing = new Map((results.row_results ?? []).map(row => [`${row.replicate_result_id}:${row.dataset_row.row_index}`, row])) + const unstarted = results.row_cells.flatMap(cell => (cell.replicates ?? []).flatMap(replicate => + Array.from({ length: results.row_summary!.row_count! }, (_, row_index) => existing.get(`${replicate.replicate_result_id}:${row_index}`) ?? { + ...replicate, cell_id: cell.cell_id, cell_label: cell.cell_label, factor_values: cell.factor_values, + dataset_row: { row_index }, status: 'not_started' as const, latest_attempt: null, + protocol_revision_id: protocol.published_revision_id ?? '', + }), + )) + const combined = new Map(unstarted.map(row => [`${row.replicate_result_id}:${row.dataset_row.row_index}`, row])) + for (const [key, row] of existing) combined.set(key, row) + return [...combined.values()] +} + export function DatasetRowRuns({ results, protocol, blocked, onInspect, designSpec }: { results: ExperimentRunResults; protocol: Protocol; blocked: boolean; onInspect?: (rowId: string) => void; designSpec?: DesignSpec | null }) { const queryClient = useQueryClient() const [confirm, setConfirm] = useState<{ labels?: string[]; cellCount: number; rerunLabels: string[] } | null>(null) const [retrying, setRetrying] = useState(null) + const [rowConfirm, setRowConfirm] = useState(null) const [page, setPage] = useState(0) const [selectedCellId, setSelectedCellId] = useState('') const [selectedReplicateId, setSelectedReplicateId] = useState('') @@ -28,7 +50,7 @@ export function DatasetRowRuns({ results, protocol, blocked, onInspect, designSp onSuccess: () => { setConfirm(null); queryClient.invalidateQueries({ queryKey: ['experiments', protocol.experiment_id] }) }, }) const summary = results.row_summary - const rows = (results.row_results ?? []).filter(row => (!selectedCellId || row.cell_id === selectedCellId) && (!selectedReplicateId || row.replicate_result_id === selectedReplicateId)) + const rows = runnableRows(results, protocol).filter(row => (!selectedCellId || row.cell_id === selectedCellId) && (!selectedReplicateId || row.replicate_result_id === selectedReplicateId)) const currentPage = Math.min(page, Math.max(0, Math.ceil(rows.length / 20) - 1)) const visibleRows = rows.slice(currentPage * 20, (currentPage + 1) * 20) const selectedReplicateCount = confirm?.labels?.length ?? summary?.parent_replicate_count ?? 0 @@ -41,6 +63,14 @@ export function DatasetRowRuns({ results, protocol, blocked, onInspect, designSp onSuccess: () => queryClient.invalidateQueries({ queryKey: ['experiments', protocol.experiment_id] }), onSettled: () => setRetrying(null), }) + const runRow = useMutation({ + mutationFn: (row: RunnableRow) => protocolsApi.runCells(protocol.id, { + replicateLabels: [row.replicate_label], + rerunReplicateLabels: row.latest_attempt ? [row.replicate_label] : [], + row_indices: [row.dataset_row.row_index], + }), + onSuccess: () => { setRowConfirm(null); queryClient.invalidateQueries({ queryKey: ['experiments', protocol.experiment_id] }) }, + }) return

{summary ? rowExecutionForecast(summary) : 'Execution forecast unavailable: original source metadata unavailable.'}

{summary &&

{summary.planned} planned · {summary.pending} pending · {summary.running} running · {summary.completed} completed · {summary.failed} failed · {summary.cancelled} cancelled · {summary.missing_reported} missing reported

} @@ -54,13 +84,14 @@ export function DatasetRowRuns({ results, protocol, blocked, onInspect, designSp }}>Run all cells ['pending', 'running', 'finalizing'].includes(row.status) && row.latest_attempt).map(row => row.latest_attempt!.run_id)} />} />

{selectedCellId ? 'Cell row executions' : 'Row executions'}

- {!rows.length &&

{selectedCellId ? 'No row executions planned for this cell.' : 'No row executions planned. Run all cells to create them.'}

} -
{['Cell', 'Source row', 'Replicate', 'Status', 'Actions'].map(label => )}{visibleRows.map(row =>
{label}
{Object.entries(row.factor_values ?? {}).map(([name, value]) => displayFactorCondition(name, value, designSpec, row.cell_label)).join(' · ') || 'No varying factors'}{row.dataset_row.row_index + 1}{row.replicate_number} - + {!rows.length &&

No source rows available. Generate the design and check the published dataset.

} +
{['Cell', 'Source row', 'Replicate', 'Status', 'Actions'].map(label => )}{visibleRows.map(row => )}
{label}
{Object.entries(row.factor_values ?? {}).map(([name, value]) => displayFactorCondition(name, value, designSpec, row.cell_label)).join(' · ') || 'No varying factors'}{row.dataset_row.row_index + 1}{row.replicate_number} + {row.latest_attempt && ['pending', 'running', 'finalizing'].includes(row.status) && } - {['failed', 'cancelled'].includes(row.status) && } + {['failed', 'cancelled'].includes(row.status) && row.row_result_id ? : !['pending', 'running', 'finalizing'].includes(row.status) && }
{rows.length > 20 &&
Page {currentPage + 1} of {Math.ceil(rows.length / 20)}
} + {rowConfirm && setRowConfirm(null)} onConfirm={() => runRow.mutate(rowConfirm)} hasUnpublishedChanges={protocol.has_unpublished_changes} publishedRevision={protocol.published_revision} confirmDisabled={blocked || runRow.isPending} isConfirming={runRow.isPending} confirmError={runRow.error?.message} additionalContent={

One execution in replicate {rowConfirm.replicate_number}. Other rows keep their results.{rowConfirm.latest_attempt && ' Previous attempts remain available.'}

} />} {confirm && setConfirm(null)} onConfirm={() => run.mutate()} hasUnpublishedChanges={protocol.has_unpublished_changes} publishedRevision={protocol.published_revision} confirmDisabled={blocked || run.isPending || !runnableExecutions} isConfirming={run.isPending} confirmError={run.error?.message} additionalContent={

{confirm.labels ? `${confirm.labels.length} replicates × ${summary?.row_count ?? '?'} rows = ${(summary?.row_count ?? 0) * confirm.labels.length} executions` : summary && rowExecutionForecast(summary)}

{runnableExecutions} executions will start. Active rows are skipped.{confirm.rerunLabels.length > 0 && ' Finished rows in the selected replicates will run again; previous attempts remain available.'}

} />} } diff --git a/frontend/src/components/protocol/ResultTimelineNode.tsx b/frontend/src/components/protocol/ResultTimelineNode.tsx new file mode 100644 index 00000000..53fd585e --- /dev/null +++ b/frontend/src/components/protocol/ResultTimelineNode.tsx @@ -0,0 +1,67 @@ +import { useState } from 'react' +import { ChevronDown, ChevronRight } from 'lucide-react' +import { Badge } from '@/components/ui/badge' +import type { ResultNodeRun } from '@/types/experiments' +import { ReceivedPromptPanel, RunStepTrace, UnresolvedReferencesNote } from './NodeRunOutputPanel' + +function formatCurrency(value: number | null): string { + return value === null || !Number.isFinite(value) ? 'Not reported' : new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) +} + +function formatNumber(value: number | null): string { + return value === null || !Number.isFinite(value) ? 'Not reported' : new Intl.NumberFormat(undefined, { maximumFractionDigits: 0 }).format(value) +} + +function nodeStatusClass(status: string): string { + if (status === 'completed') return 'border-transparent bg-[color:var(--chart-3)]/10 text-[color:var(--chart-3)]' + if (status === 'failed' || status === 'cancelled') return 'border-transparent bg-destructive/10 text-destructive' + if (status === 'running' || status === 'queued') return 'border-transparent bg-primary/10 text-primary' + return 'border-transparent bg-muted text-muted-foreground' +} + +export function ResultTimelineNode({ node, defaultOpen = false }: { node: ResultNodeRun; defaultOpen?: boolean }) { + const [open, setOpen] = useState(defaultOpen) + const hasDetails = Boolean(node.error || node.output_text || node.agent_run_id) + + return ( +
  • + + {open && ( +
    + {node.error &&

    Error

    {node.error}

    } + {/* Received before produced, so one node reads as the handoff it + was: what it was given, then what it made of it. */} + {node.agent_run_id && } + + {node.output_text ? ( +
    +

    Output

    +

    {node.output_text}

    +
    + ) : !node.error &&

    No output was recorded for this node.

    } + {node.agent_run_id && } +
    + )} +
    • + ) +} + diff --git a/frontend/src/components/protocol/ResultsTab.tsx b/frontend/src/components/protocol/ResultsTab.tsx index 31d0a8fa..ad073497 100644 --- a/frontend/src/components/protocol/ResultsTab.tsx +++ b/frontend/src/components/protocol/ResultsTab.tsx @@ -10,12 +10,12 @@ import { Skeleton } from '@/components/ui/skeleton' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' import { displayFactorLevel, formatMetricLabel, formatMetricValue } from '@/lib/experiment' import { OBSERVATION_LABELS } from '@/lib/measurementPlan' -import type { DesignMetric, EvaluationArtifact, Experiment, HistoricalRun, MetricObservation, ObsoleteRun, ResultCell, ResultNodeRun, ResultReplicate, SupersededRun } from '@/types/experiments' +import type { DesignMetric, EvaluationArtifact, Experiment, HistoricalRun, MetricObservation, ObsoleteRun, ResultCell, ResultReplicate, SupersededRun } from '@/types/experiments' import { DatasetRowDetail, DatasetRowResults } from './DatasetRowResults' import { ResultsPanel, ResultsScorecard as Scorecard } from './ResultsPanel' import { InfoTooltip } from './InfoTooltip' import { CellCard, ReplicateRow } from './CellPresentation' -import { ReceivedPromptPanel, RunStepTrace, UnresolvedReferencesNote } from './NodeRunOutputPanel' +import { ResultTimelineNode } from './ResultTimelineNode' function formatNumber(value: number | null, maximumFractionDigits = 0): string { if (value === null || !Number.isFinite(value)) return 'Not reported' @@ -189,13 +189,6 @@ function ArtifactCard({ artifact }: { artifact: EvaluationArtifact }) { return

    {presentation.title}

    {artifact.producer.producer_id} · v{artifact.producer.version}
    {presentation.content}     } -function nodeStatusClass(status: string): string { - if (status === 'completed') return 'border-transparent bg-[color:var(--chart-3)]/10 text-[color:var(--chart-3)]' - if (status === 'failed' || status === 'cancelled') return 'border-transparent bg-destructive/10 text-destructive' - if (status === 'running' || status === 'queued') return 'border-transparent bg-primary/10 text-primary' - return 'border-transparent bg-muted text-muted-foreground' -} - function usageSummarizesMetric( key: string, result: Pick, @@ -248,52 +241,6 @@ function CellResultSummary({ cell, metricKeys, metricTypes, metricAggregations } ) } -function ReplicateTimelineNode({ node, defaultOpen = false }: { node: ResultNodeRun; defaultOpen?: boolean }) { - const [open, setOpen] = useState(defaultOpen) - const hasDetails = Boolean(node.error || node.output_text || node.agent_run_id) - - return ( -
  • - - {open && ( -
    - {node.error &&

    Error

    {node.error}

    } - {/* Received before produced, so one node reads as the handoff it - was: what it was given, then what it made of it. */} - {node.agent_run_id && } - - {node.output_text ? ( -
    -

    Output

    -

    {node.output_text}

    -
    - ) : !node.error &&

    No output was recorded for this node.

    } - {node.agent_run_id && } -
    - )} -
    • - ) -} - function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { replicate: ResultReplicate metricKeys: string[] @@ -356,7 +303,7 @@ function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: {

    Run timeline

    {replicate.node_runs.length === 0 ?

    No node-level run details are available.

    : (
      - {replicate.node_runs.map((node) => )} + {replicate.node_runs.map((node) => )}
    )} @@ -449,7 +396,7 @@ export function ResultsInspectorPanel({ if (selection.type === 'row') { const row = resultsQuery.data?.row_results?.find(item => item.row_result_id === selection.rowResultId) if (!row) return - return + return } const replicate = selection.type === 'replicate' diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index 29365ed1..640ccb57 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -468,6 +468,7 @@ export interface RowAttempt { workspace_id: string | null attempt_result: Record | null node_runs: Record + node_labels?: Record conversation: Conversation | null dataset_row: DatasetRowSnapshot | null protocol_revision_id: string | null diff --git a/src/asaree/api/protocols.py b/src/asaree/api/protocols.py index 329a4ae5..7d1fa401 100644 --- a/src/asaree/api/protocols.py +++ b/src/asaree/api/protocols.py @@ -295,6 +295,8 @@ class CellRunBatchRequest(BaseModel): replicate_labels: list[str] | None = None rerun_replicate_labels: list[str] = [] retry_row_result_ids: list[uuid.UUID] | None = None + # Zero-based source rows, scoped to exactly one selected replicate. + row_indices: list[StrictInt] | None = None class CellRunBatchResponse(BaseModel): @@ -817,6 +819,7 @@ async def create_cell_runs_endpoint( replicate_labels=set(body.replicate_labels) if body and body.replicate_labels is not None else None, rerun_replicate_labels=set(body.rerun_replicate_labels) if body is not None else None, retry_row_result_ids=retry_row_result_ids, + row_indices=body.row_indices if body is not None else None, ) except ProtocolValidationError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc diff --git a/src/asaree/services/dataset_row_planning.py b/src/asaree/services/dataset_row_planning.py index e8e24430..b97baa81 100644 --- a/src/asaree/services/dataset_row_planning.py +++ b/src/asaree/services/dataset_row_planning.py @@ -22,6 +22,7 @@ def enumerate_row_candidates( graph: dict, design_spec: dict | None, protocol_revision_id: str, + row_indices: Sequence[int] | None = None, ) -> list[dict]: """Expand sorted replicate parents across original source rows. @@ -34,6 +35,14 @@ def enumerate_row_candidates( if not source.rows: raise DatasetRowInputError("empty_source", "row source must contain at least one data row") + selected_rows = range(len(source.rows)) if row_indices is None else row_indices + if row_indices is not None and ( + not row_indices + or any(isinstance(index, bool) or not isinstance(index, int) or index < 0 or index >= len(source.rows) + for index in row_indices) + or len(set(row_indices)) != len(row_indices) + ): + raise DatasetRowInputError("invalid_row_selection", "select unique source row indices within the dataset") original_plan = resolve_dataset_row_plan(graph, design_spec) if original_plan is None: @@ -90,7 +99,7 @@ def enumerate_row_candidates( "factor substitution must preserve the row driver and ordered Agent/column bindings", ) - for row_index in range(len(source.rows)): + for row_index in selected_rows: candidate = deepcopy(parent) candidate["protocol_revision_id"] = protocol_revision_id candidate["dataset_row"] = { diff --git a/src/asaree/services/experiment_run_results.py b/src/asaree/services/experiment_run_results.py index 79be7415..798ed947 100644 --- a/src/asaree/services/experiment_run_results.py +++ b/src/asaree/services/experiment_run_results.py @@ -398,6 +398,7 @@ async def _summarize_row_results( ) -> dict[str, Any]: revision_id = design_revision.id if design_revision is not None else None protocol_revision_id = protocol_revision.id + node_labels = _node_labels(protocol_revision.graph) cells = list( ( await db.execute( @@ -543,8 +544,14 @@ async def _summarize_row_results( "metric_values": slot.metric_values, "measurement": measurement, "artifacts": slot.artifacts, - "latest_attempt": _row_attempt_payload(latest, current=latest.id == slot.run_id) if latest else None, - "attempts": [_row_attempt_payload(run, current=run.id == slot.run_id) for run in history], + "latest_attempt": _row_attempt_payload( + latest, current=latest.id == slot.run_id, node_labels=node_labels + ) if latest else None, + "attempts": [ + _row_attempt_payload( + run, current=run.id == slot.run_id, node_labels=node_labels + ) for run in history + ], } ) @@ -613,7 +620,9 @@ async def _summarize_row_results( } -def _row_attempt_payload(run: ProtocolRun, *, current: bool) -> dict[str, Any]: +def _row_attempt_payload( + run: ProtocolRun, *, current: bool, node_labels: dict[str, str] | None = None +) -> dict[str, Any]: return { "run_id": str(run.id), "status": run.status, @@ -623,6 +632,7 @@ def _row_attempt_payload(run: ProtocolRun, *, current: bool) -> dict[str, Any]: "workspace_id": row_attempt_workspace_id(run.id), "attempt_result": run.attempt_result, "node_runs": run.node_runs, + "node_labels": node_labels or {}, "conversation": run.conversation, "dataset_row": run.dataset_row, "protocol_revision_id": str(run.protocol_revision_id) if run.protocol_revision_id else None, diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 82472965..328edc97 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -4626,6 +4626,7 @@ async def plan_cell_runs( replicate_labels: set[str] | None = None, rerun_replicate_labels: set[str] | None = None, retry_row_result_ids: Sequence[uuid.UUID] | None = None, + row_indices: Sequence[int] | None = None, ) -> tuple[list[ProtocolRun], int]: """ "Run all cells": creates one pending :class:`ProtocolRun` per not-yet-completed replicate row under *experiment_id*, each carrying its @@ -4652,6 +4653,13 @@ async def plan_cell_runs( # Serialize all production row planning against publication and design # regeneration. Keep this global lock order in retry paths too. row_mode_requested = resolve_dataset_row_plan(graph) is not None + if row_indices is not None and ( + not row_mode_requested + or retry_row_result_ids is not None + or replicate_labels is None + or len(replicate_labels) != 1 + ): + raise ProtocolValidationError("Row selection requires exactly one per-row replicate.") if retry_row_result_ids is not None and ( not retry_row_result_ids or len(set(retry_row_result_ids)) != len(retry_row_result_ids) @@ -4929,6 +4937,7 @@ async def plan_cell_runs( graph=revision.graph, design_spec=design_spec, protocol_revision_id=str(revision.id), + row_indices=row_indices, ) planned = [] for candidate in candidates: diff --git a/tests/test_dataset_row_batch_planning.py b/tests/test_dataset_row_batch_planning.py index f9bb6c9f..98491cb5 100644 --- a/tests/test_dataset_row_batch_planning.py +++ b/tests/test_dataset_row_batch_planning.py @@ -11,6 +11,7 @@ import pytest import pytest_asyncio +from fastapi import HTTPException from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine @@ -18,6 +19,7 @@ from asaree.api.protocols import CellRunBatchRequest, create_cell_runs_endpoint from asaree.models.dataset import RegisteredDataset from asaree.models.factorial_replicate_result import FactorialReplicateResult +from asaree.models.factorial_row_result import FactorialRowResult from asaree.models.protocol_run import ProtocolRun from asaree.services.experiments import create_experiment from asaree.services.factorial_cells import upsert_replicate @@ -234,3 +236,60 @@ async def enqueue(_run_id: uuid.UUID) -> None: assert first.protocol_revision_id != second.protocol_revision_id assert len(second.protocol_run_ids) == 30 assert len(set(first.row_result_ids).intersection(second.row_result_ids)) == 0 + + +@pytest.mark.asyncio +async def test_individual_row_run_and_rerun_preserve_other_rows(row_batch, monkeypatch) -> None: + db, ctx = row_batch + queued = [] + + async def enqueue(run_id): + queued.append(run_id) + + monkeypatch.setattr(protocol_api, "enqueue_protocol_run", enqueue) + protocol = ctx["protocol"] + selection = CellRunBatchRequest(replicate_labels=["batch-parent"], row_indices=[2]) + first = await create_cell_runs_endpoint(protocol.id, ctx["user"], db, selection) + assert len(first.protocol_run_ids) == len(first.row_result_ids) == 1 + original = await db.get(ProtocolRun, first.protocol_run_ids[0]) + assert original.dataset_row["row_index"] == 2 + assert original.protocol_revision_id == ctx["revision"].id + assert await db.scalar(select(func.count()).select_from(FactorialRowResult).where( + FactorialRowResult.protocol_revision_id == ctx["revision"].id, + )) == 1 + + active = await create_cell_runs_endpoint(protocol.id, ctx["user"], db, selection) + assert active.protocol_run_ids == [] and active.skipped == 1 + original.status = "completed" + await db.flush() + rerun = await create_cell_runs_endpoint(protocol.id, ctx["user"], db, CellRunBatchRequest( + replicate_labels=["batch-parent"], rerun_replicate_labels=["batch-parent"], row_indices=[2], + )) + assert len(rerun.protocol_run_ids) == 1 + assert rerun.row_result_ids == first.row_result_ids + assert rerun.protocol_run_ids != first.protocol_run_ids + await db.refresh(original) + assert original.status == "completed" + remaining = await create_cell_runs_endpoint(protocol.id, ctx["user"], db, CellRunBatchRequest()) + assert len(remaining.protocol_run_ids) == 29 + assert remaining.skipped == 1 + assert first.row_result_ids[0] not in remaining.row_result_ids + assert len(queued) == 31 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("selection", [ + {"row_indices": [0]}, + {"replicate_labels": [], "row_indices": [0]}, + {"replicate_labels": ["batch-parent"], "row_indices": []}, + {"replicate_labels": ["batch-parent"], "row_indices": [30]}, + {"replicate_labels": ["unknown"], "row_indices": [0]}, +]) +async def test_invalid_row_selection_creates_no_runs(row_batch, selection) -> None: + db, ctx = row_batch + with pytest.raises(HTTPException) as exc: + await create_cell_runs_endpoint(ctx["protocol"].id, ctx["user"], db, CellRunBatchRequest(**selection)) + assert exc.value.status_code == 422 + assert await db.scalar(select(func.count()).select_from(ProtocolRun).where( + ProtocolRun.protocol_id == ctx["protocol"].id, + )) == 0 diff --git a/tests/test_dataset_row_candidates.py b/tests/test_dataset_row_candidates.py index 608f416e..29b7296b 100644 --- a/tests/test_dataset_row_candidates.py +++ b/tests/test_dataset_row_candidates.py @@ -98,6 +98,25 @@ def test_duplicate_row_values_keep_distinct_original_positions(tmp_path): assert [candidate["dataset_row"]["row_index"] for candidate in candidates[:2]] == [0, 1] +def test_explicit_row_selection_keeps_original_index(tmp_path): + candidates = enumerate_row_candidates( + parents=make_parents()[:1], source=make_source(tmp_path), graph=make_graph(), + design_spec=None, protocol_revision_id="published-revision", row_indices=[2], + ) + assert len(candidates) == 1 + assert candidates[0]["dataset_row"]["row_index"] == 2 + assert candidates[0]["replicate_result_id"] == "result-0-1" + + +@pytest.mark.parametrize("indices", [[], [-1], [3], [1, 1], [True], [1.5]]) +def test_rejects_invalid_source_row_selections(tmp_path, indices): + with pytest.raises(DatasetRowInputError, match="invalid_row_selection"): + enumerate_row_candidates( + parents=make_parents()[:1], source=make_source(tmp_path), graph=make_graph(), + design_spec=None, protocol_revision_id="published-revision", row_indices=indices, + ) + + def test_candidate_order_does_not_depend_on_parent_input_order(tmp_path): arguments = { "source": make_source(tmp_path), diff --git a/tests/test_experiment_run_results.py b/tests/test_experiment_run_results.py index d6fd6f8e..199a0e92 100644 --- a/tests/test_experiment_run_results.py +++ b/tests/test_experiment_run_results.py @@ -3,6 +3,7 @@ import csv import io import json +import uuid from decimal import Decimal from types import SimpleNamespace @@ -18,6 +19,7 @@ _node_labels, _numeric_metrics, _primary_metric, + _row_attempt_payload, _sum, _usage, ) @@ -659,6 +661,19 @@ def test_node_labels_uses_the_canvas_placeholder_when_an_old_node_has_no_title() assert _node_labels({"nodes": [{"id": "node-old", "type": "agent", "data": {}}]}) == {"node-old": "Agent"} +def test_row_attempt_includes_published_names_without_modifying_output() -> None: + nodes = {"node-stable": {"status": "completed", "output_text": "Answer in plain text"}} + run = SimpleNamespace( + id=uuid.uuid4(), status="completed", error=None, started_at=None, completed_at=None, + attempt_result={}, node_runs=nodes, conversation=None, dataset_row=None, + protocol_revision_id=uuid.uuid4(), design_revision_id=uuid.uuid4(), + ) + payload = _row_attempt_payload(run, current=True, node_labels={"node-stable": "Published researcher"}) + assert payload["node_labels"] == {"node-stable": "Published researcher"} + assert payload["node_runs"] == nodes + assert payload["current"] is True + + def test_execution_evidence_excludes_completed_configuration_nodes() -> None: assert _has_execution_evidence({"status": "completed", "output_text": None, "error": None}) is False assert _has_execution_evidence({"status": "completed", "run_id": "agent-run"}) is True