diff --git a/.dockerignore b/.dockerignore index 79e3b3b9..905574ee 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,8 @@ __pycache__/ dist/ *.egg-info/ data/ +# Database dumps (up to hundreds of MB) -- mounted at runtime, never built in. +backups/ # .git/ stays excluded even though the build has to read it. hatch-vcs derives # the package version from the repo's tags, so the final `uv sync` can't build # this project without that metadata -- but it arrives as a *named build context* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b44e09de..3c8df93e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,11 @@ jobs: DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/asaree + # Opt-in suites that skip without these. Results checks run every revision + # in a throwaway schema inside a rolled-back transaction; the migration + # re-run checks create and drop their own scratch database on this server. + ASAREE_RESULTS_TEST_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/asaree + ASAREE_MIGRATIONS_TEST_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro steps: - uses: actions/checkout@v6 @@ -146,6 +151,11 @@ jobs: - name: Type check run: npx tsc -b + # Component and lib tests (vitest + jsdom). Without this step a stale + # test can sit red on a branch with CI still green. + - name: Unit tests + run: npx vitest run + # oxlint exits 0 on warnings, so this reports rather than gates. The # standing baseline is 9 warnings; it is here to make that number # visible and stop it climbing unnoticed. diff --git a/.gitignore b/.gitignore index 401dfe9f..078fd8fb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,10 @@ dist/ data/ local_setup/ local_files/ -backups/ +# Dumps stay local; the directory itself is tracked so a fresh clone has it, +# owned by you, before compose bind-mounts it into db-backup. +backups/* +!backups/.gitkeep # Generated by hatch-vcs at build time from the git tag (see pyproject.toml). src/asaree/_version.py diff --git a/Dockerfile b/Dockerfile index d72ede76..787772bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,10 @@ FROM python:3.13-slim AS application WORKDIR /app # git: needed by uv to resolve Motoro's pinned git dependency below. -RUN apt-get update && apt-get install -y --no-install-recommends git \ +# postgresql-client: pg_dump/pg_restore for the pre-migration backup step +# (asaree/migrations/backup.py). Debian's client is newer than the pg16 server, +# which pg_dump supports; restore with this same image's pg_restore. +RUN apt-get update && apt-get install -y --no-install-recommends git postgresql-client \ && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir uv diff --git a/README.md b/README.md index 731bfc3a..d3c5eb1f 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,12 @@ servers (`asaree-sklearn-dc`, `-eda`, `-fs`, `-fte`, `-model`, `-stats`, from `mcp-servers/`) — ships as a dependency of the app and registers itself each time the app or worker starts (see `asaree.services.system_mcp_servers`). +## Large experiment results + +For large experiment results, the dedicated result worker handles summaries, +batch commands and exports. Deployment, recovery, API/SDK usage and pending +PostgreSQL acceptance checks are documented in [RESULTS_OPERATIONS.md](RESULTS_OPERATIONS.md). + ## Resetting your dev environment One Postgres server hosts two databases in a single volume — `motoro` (core's diff --git a/RESULTS_OPERATIONS.md b/RESULTS_OPERATIONS.md new file mode 100644 index 00000000..d4b0f2d7 --- /dev/null +++ b/RESULTS_OPERATIONS.md @@ -0,0 +1,177 @@ +# Scalable execution results + +Runs and Results use `/api/experiments/{id}/execution-results`. The canvas uses +`/api/protocols/{id}/run-heads?limit=1` to recover its latest execution. None of +these list reads hydrates Motoro agents, traces, complete outputs, or histories. + +Execution and cell pages default to 20 records and cap at 100. Cursors bind the +protocol publication, design revision and filters. Metric cursors also bind the +published summary generation; refresh from the first page after it changes. +Per-row pages generate only the requested window, including unstarted source +rows once their count has been prepared. Stable slot IDs identify completed +records. `unplanned::` identifies a row without a +persisted slot and cannot be used to inspect an attempt. + +Summaries are rebuildable projections. Database triggers record dirty scopes in +`result_refreshes` in the same transaction as result changes. The result worker +coalesces and rebuilds each scope from current facts, streams cell writes, and +publishes cell/overview data atomically. Repeated deliveries do not double-count +attempts. The UI shows records while a summary or usage backfill is preparing. +It refreshes visible pages when a summary generation advances, retains loaded +data on transient failures, and stops polling settled executions. Refresh or +window focus discovers work started by another client after a scope settles. + +Operational usage belongs to each production `ProtocolRun`, independently of +declared measurements. Accounting reads scalar, owner-scoped Motoro rows in +batches of 200, including all causally attributed calls (not only final node +pointers). A pointer fallback supports older records. Legacy calls without +either attribution or a retained pointer cannot be recovered. Unreported or +ambiguous default-zero provider fields remain unknown. Execution duration uses +actual start/task completion, excluding queue time. Current execution usage +and all-attempt spend are separate. Immutable measurement/evidence documents +are never rewritten by accounting backfill. + +Only explicitly declared built-in runtime measurements enter scalar cell +aggregates and ranking. Global runtime metrics use catalog keys; node runtime +metrics use stable metric IDs with display labels. Display names do not identify +measurements. Reported and legacy values remain opaque, retain +their recorded provenance, and are not coerced or ranked. Per-row outcomes are +never aggregated or ranked. Truncated attempts are excluded from scalar +comparison. Primary ranks are maintained for the whole scope, including pages +after the first. + +Selected attempts return bounded previews. Oversized observation values and +artifact payloads carry explicit preview markers; a measured JSON null remains +different from a deferred value. Exact JSON documents are available through +`attempts/{run}/evidence` in chunks of at most 16,384 characters. Node output and +received prompt text use the same chunk bound. Trace pages contain 20 metadata +records; full step evidence loads only after selecting a step. Conversation +messages are paged and large messages are marked previews; complete transcripts +remain available through the recorded JSON reader. + +## Durable batch commands + +`POST execution-results/jobs` accepts `kind=plan|cancel|export`, a stable +`request_key`, and the selected publication/design query parameters. Creation +returns 202 immediately. Repeating the same owned request key and selection +returns the same job; conflicting reuse is rejected. + +A plan first prepares a preview in bounded windows and stores exact parent, +row, slot, and expected-attempt identities. `awaiting_confirmation` includes +the exact planned execution count. Opening a dialog never launches an LLM. +`POST jobs/{id}/confirm` queues that sealed selection. Each small transaction +creates runs and advances their intent checkpoints together. Slots changed +since preview are skipped, so a retry cannot silently re-bill a newer attempt. +Publication/design changes require preparing a new batch. Runs already created +remain pinned to their immutable publications. + +Completed and limit-reached executions (both finished with a result) are +skipped unless explicitly selected by slot ID or `rerun_completed=true`. +Explicit selections cap at 100 IDs; an all-scope command does not send a +browser-sized ID list. Failed/cancelled work is retried automatically. A plan's cancel action stops remaining planning; already-created runs +continue. The Runs tab's Stop all creates a scoped cancellation command, stops +matching planners and cancels all active attempts, including earlier attempts +that are no longer the latest slot result. + +Run cell and Stop cell use the same durable API with `cell_id`. Stopping one +cell excludes it from existing broad planners while allowing their other cells +to continue. Publication predicates also constrain the factor levels and +replicate count: adding cells to a mutable design revision cannot expand the +cohort belonging to an earlier immutable publication. + +SQL intent survives worker/Redis loss. Created pending runs are repeatedly +offered to ARQ with the same run job ID; executor guards prevent completed or +cancelled runs from executing again. Delivery continues for committed runs even +if later planning fails or is cancelled. Preview-only cancellation cannot +cancel a confirmed job. Unconfirmed previews expire after 24 hours. + +## Exports and source metadata + +An export is a background job, limited to one export at a time globally. It +writes the researcher-facing analysis CSV -- the same columns and values as the +synchronous `run-results.csv`: one column per factor (short level labels), +`duration_seconds`, `input_tokens`, `output_tokens`, `total_tokens`, +`cost_usd`, one column per metric, observation statuses and run metadata. Rows +are built by the Results projection a chunk of replicates at a time (two +passes: the first lays out columns from compact stubs, the second writes rows +in factor-level order), so memory stays bounded at any size; 15,786 runs take +about 20 seconds. Every chunk reads one REPEATABLE READ snapshot, so the file is +consistent even while runs finish. The file is written to an owned temporary +path and renamed into place; progress reports bytes, cancellation is checked +between chunks, and a restart repeats the export. The schema sidecar is the same +`result_rows_schema` document as `run-results.schema.json`. The synchronous +endpoint keeps a 500-run cap and returns 413 above it, pointing to this job. + +Completed exports are downloaded natively in the browser or streamed to disk +by the SDK. Download capabilities bind job/owner, expire after five minutes, +and cannot supply filesystem paths. CSV/schema files expire after 24 hours. + +Original CSV rows are verified by streaming SHA-256 and indexed into an immutable +SQLite cache. Construction uses batches of 1,000 rows and atomic rename. +Subsequent planning windows use random indexed row access instead of reparsing +the original CSV for every retry. Dataset row-schema HTTP reads return 202 with +`ready=false`/`row_count=null` while the worker prepares metadata; they never +read or parse source files in the request. The UI polls that small metadata +record while preparing. Source preparation records hash/CSV validation failures +without exposing paths. Count metadata describes the last verified registered +hash; execution still verifies the pinned source identity before running. + +## Deployment and recovery + +0. Compose's `db-backup` runs first: when either database has a pending + migration, it writes `backups/{db}-{revision}-before-v{release}-{UTC}.dump` + (the database at that schema revision, saved before that release migrated it) and keeps the newest + five per database; manual dumps in `backups/` are left alone. If it fails, no + migration runs. Outside compose, run `python -m asaree.migrations backup` + before migrating. To restore one (stop the app first): + `docker compose run --rm --no-deps -e PGPASSWORD=... --entrypoint pg_restore db-backup -h motoro-postgres -U agentic --clean --if-exists --no-owner -d asaree /app/backups/`. + One error about `transaction_timeout` is expected and harmless (the image's + client is newer than the server). These are troubleshooting snapshots taken + only at deploy time, not a backup strategy. +1. Apply Motoro's migrations, then ASAREE's (through `718293a4b5c6`), using the + usual migration deployment. Historical JSON evidence is not rewritten in migration. +2. The usage and trace reads go through Motoro's `list_run_summaries` and + `list_run_step_summaries`; their indexes (`ix_agent_runs_run_metadata`, + `ix_run_steps_run_sequence`) ship with Motoro's own migrations. ASAREE never + creates indexes on core tables. +3. Run `arq asaree.worker.result_settings.ResultWorkerSettings` alongside the + production execution worker. Compose defines `asaree-results-worker`, which + waits for both migration steps. Both workers and the API must share the data + volume and configured export/row-index paths. +4. Keep `ASAREE_RESULT_EXPORT_DIR` and `ASAREE_RESULT_ROW_INDEX_DIR` on persistent + shared storage (defaults are `./data/result-exports` and `./data/result-row-indexes`). +5. To requeue all existing overview projections, run + `python scripts/rebuild_result_summaries.py`. The worker rebuilds from facts + and retries unfinished accounting; no evidence or execution timestamps change. + +Watch `result_refreshes.generation - processed_generation`, refresh `error`, +unfinished `protocol_runs.usage_summary`, pending `result_jobs`, and worker logs +for `result_summary_refresh_failed`, `result_usage_capture_failed`, +`result_command_database_retry`, and `result_command_delivery_failed`. +Size the dedicated result worker separately from LLM concurrency. Commands use +bounded work windows; exports, usage capture and source preparation each have +an independent global advisory lock. Read requests have a five-second SQL +statement deadline and bounded page sizes. Read replicas are not required. + +SDK entry points: `execution_summary`, `iter_executions`, `create_result_job`, +`result_job`, `confirm_result_job`, `cancel_result_job`, `iter_evidence_text`, +and `download_result_export`. Supply a pinned publication in `scope` while +iterating and keep the same `request_key` when retrying job creation. + +## Validation status + +Focused Python/SDK tests, frontend tests, TypeScript, Ruff and the new service +type checks run locally without infrastructure. PostgreSQL validation is +**pending**, as requested; no live migration or production-latency result is +claimed. `tests/test_result_reads.py` applies the complete migration chain in +a unique schema inside a rolled-back transaction when +`ASAREE_RESULTS_TEST_DATABASE_URL` points to a disposable PostgreSQL database. + +The opt-in acceptance tests (`ASAREE_RESULTS_RUN_SCALE=1`) exercise 15,786, +100,000 and 1,000,000 runs with 32 KiB node outputs and 8 KiB opaque reports. +They require 20-record pages below 64 KiB, cached summaries below 16 KiB, and +cached summary plus first-page latency below two seconds. Run these tests only +on the disposable database; the million-run case intentionally creates a large +fixture. Before rollout, execute these checks and inspect PostgreSQL plans for +slot lookup and metric ranking, including concurrent execution/outbox updates, +worker restart, Redis interruption, export COPY/download, and downgrade/upgrade. diff --git a/backups/.gitkeep b/backups/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/compose.dev.yml b/compose.dev.yml index c52b0b3f..c0f474ef 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -52,3 +52,10 @@ services: - ./src:/app/src - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro + + asaree-results-worker: + volumes: + - ./data:/app/data + - ./src:/app/src + - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro + - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro diff --git a/compose.yml b/compose.yml index ec087de5..aad51cfd 100644 --- a/compose.yml +++ b/compose.yml @@ -75,6 +75,32 @@ services: # Runs to completion and exits 0 -- `Exited (0)` in `docker compose ps` is # success, hence restart: "no". `--url` is passed explicitly because core's CLI # otherwise reads a bare DATABASE_URL, which nothing here sets. + # Snapshots both databases before either migration chain runs, only when a + # migration is pending: backups/{db}-{revision}-before-v{release}-{UTC}.dump. + # Troubleshooting aid, not a backup strategy -- see asaree/migrations/backup.py. + # Runs as root so it can always write the mount, then hands each dump to the + # owner of ./backups (you), whatever your UID is. + db-backup: + build: + context: . + secrets: + - gh_token + additional_contexts: + gitdir: ./.git + container_name: db-backup + depends_on: + motoro-postgres: + condition: service_healthy + env_file: .env + environment: + ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree + entrypoint: ["python", "-m", "asaree.migrations"] + command: ["backup", "--dir", "/app/backups"] + volumes: + - ./backups:/app/backups + restart: "no" + motoro-migrate: build: context: . @@ -88,6 +114,9 @@ services: # running initdb, which races the creation of the database being migrated. motoro-postgres: condition: service_healthy + # No migration runs until both databases are snapshotted (or need none). + db-backup: + condition: service_completed_successfully entrypoint: ["python", "-m", "motoro.migrations"] command: - deploy @@ -218,6 +247,32 @@ services: - ./data:/app/data restart: unless-stopped + # Result maintenance and bulk commands never compete for LLM worker slots. + asaree-results-worker: + build: + context: . + secrets: + - gh_token + additional_contexts: + gitdir: ./.git + depends_on: + # asaree-migrate itself waits for motoro-migrate, which creates the core + # indexes the usage/trace reads rely on. + asaree-migrate: + condition: service_completed_successfully + motoro-redis: + condition: service_healthy + env_file: .env + environment: + ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro + ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree + ASAREE_REDIS_URL: redis://motoro-redis:6379/0 + ASAREE_OKF_BUNDLE_ROOT: /app/data + command: ["arq", "asaree.worker.result_settings.ResultWorkerSettings"] + volumes: + - ./data:/app/data + restart: unless-stopped + # Integration-test runner on the Compose network. The tests intentionally # use real Postgres (JSONB and core's vector schema rule out SQLite), so run # them beside the database instead of relying on a host-side localhost port. diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 8f008d70..a6fdb7d4 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -24,14 +24,16 @@ import type { MetricRecommendationMetadata, MeasurementPlanValidationReport, Replicate, + CellSummary, Trial, } from '@/types/experiments' -import type { LLMConnectionCheck, LLMProvider, LLMSetting, LLMSettingModelsResponse } from '@/types/llmSettings' +import type { LLMConnectionCheck, LLMProvider, LLMSetting, LLMSettingModelsResponse, ModelCapabilities } from '@/types/llmSettings' import type { McpOAuthAuthorization, McpServer } from '@/types/mcpServers' import type { OkfBundle, OkfDocument } from '@/types/okf' import type { CellRunBatch, DatasetRowSchema, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun, TestRun } from '@/types/protocols' import type { Run, RunStep } from '@/types/runs' import type { Skill, SkillListResponse, SkillUrlPreview } from '@/types/skills' +import type { AttemptDetail, AttemptPage, CursorPage, ExecutionCell, ExecutionSlot, ExecutionSummary, ResultJob } from '@/types/executionResults' const ACCESS_TOKEN_KEY = 'asaree_access_token' @@ -138,7 +140,7 @@ async function authedFetch(path: string, options: RequestOptions = {}): Promise< } } - if (!res.ok) { + if (!res.ok && res.status !== 304) { let detail: string | ApiErrorDetail = res.statusText try { const data = await res.json() @@ -187,7 +189,7 @@ export interface ResultsScope { protocol_revision_id?: string } -export interface RowSelection { row_index?: number | null } +export interface RowSelection { row_index?: number | null; sub_agent_selections?: Record; script_selections?: Record; tool_selections?: Record; skill_selections?: Record; dataset_selections?: Record; knowledge_selections?: Record } function scopeQuery(scope?: ResultsScope): string { const params = new URLSearchParams() @@ -195,6 +197,12 @@ function scopeQuery(scope?: ResultsScope): string { return params.size ? `?${params}` : '' } +function resultPageQuery(scope: ResultsScope, params: Record): string { + const query = new URLSearchParams(scopeQuery(scope).replace(/^\?/, '')) + for (const [key, value] of Object.entries(params)) if (value !== undefined) query.set(key, String(value)) + return query.size ? `?${query}` : '' +} + function rowSelection(options?: RowSelection): RowSelection | undefined { const index = options?.row_index if (index !== undefined && index !== null && (!Number.isInteger(index) || index < 0)) { @@ -204,6 +212,43 @@ function rowSelection(options?: RowSelection): RowSelection | undefined { } export const experimentsApi = { + getExecutionEvidence: (id: string, runId: string, scope: ResultsScope, document: string, offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/evidence${resultPageQuery(scope, { document, offset })}`, { signal }), + getExecutionNodeText: (id: string, runId: string, nodeId: string, scope: ResultsScope, field: 'prompt' | 'output', offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/text${resultPageQuery(scope, { field, offset })}`, { signal }), + getExecutionTrace: (id: string, runId: string, nodeId: string, scope: ResultsScope, after: number, signal?: AbortSignal) => + request<{ items: { id: string; sequence: number; iteration: number | null; phase: string; preview: string | null }[]; next_after: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/trace${resultPageQuery(scope, { after })}`, { signal }), + getExecutionStep: (id: string, runId: string, nodeId: string, stepId: string, scope: ResultsScope, offset: number, signal?: AbortSignal) => + request<{ text: string; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/nodes/${encodeURIComponent(nodeId)}/trace/${stepId}${resultPageQuery(scope, { offset })}`, { signal }), + createResultJob: (id: string, scope: ResultsScope, command: { kind: ResultJob['kind']; request_key: string; cell_id?: string; replicate_ids?: string[]; row_indices?: number[]; rerun_slot_ids?: string[]; rerun_completed?: boolean }, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/jobs${scopeQuery(scope)}`, { method: 'POST', body: command, signal }), + getResultJob: (id: string, jobId: string, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/jobs/${jobId}`, { signal }), + confirmResultJob: (id: string, jobId: string) => + request(`/experiments/${id}/execution-results/jobs/${jobId}/confirm`, { method: 'POST' }), + cancelResultJob: (id: string, jobId: string, previewOnly = false) => + request(`/experiments/${id}/execution-results/jobs/${jobId}/cancel${previewOnly ? '?preview_only=true' : ''}`, { method: 'POST' }), + getExecutionSummary: async (id: string, scope: ResultsScope, signal?: AbortSignal, cached?: ExecutionSummary): Promise => { + const res = await authedFetch(`/experiments/${id}/execution-results/summary${scopeQuery(scope)}`, { + signal, headers: cached?.etag ? { 'If-None-Match': cached.etag } : undefined, + }) + if (res.status === 304 && cached) return cached + return { ...await res.json() as ExecutionSummary, etag: res.headers.get('ETag') ?? undefined } + }, + listExecutionCells: (id: string, scope: ResultsScope, params: Record = {}, signal?: AbortSignal) => + request>(`/experiments/${id}/execution-results/cells${resultPageQuery(scope, params)}`, { signal }), + listExecutions: (id: string, scope: ResultsScope, params: Record = {}, signal?: AbortSignal) => + request>(`/experiments/${id}/execution-results/executions${resultPageQuery(scope, params)}`, { signal }), + listExecutionAttempts: (id: string, slotId: string, scope: ResultsScope, cursor?: string, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/slots/${slotId}/attempts${resultPageQuery(scope, { cursor })}`, { signal }), + lookupExecution: (id: string, replicateLabel: string, scope: ResultsScope, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/lookup${resultPageQuery(scope, { replicate_label: replicateLabel })}`, { signal }), + getExecutionCell: (id: string, scope: ResultsScope, selection: { cell_id?: string; cell_label?: string }, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/cell${resultPageQuery(scope, selection)}`, { signal }), + getExecutionAttempt: (id: string, runId: string, scope: ResultsScope, signal?: AbortSignal) => + request(`/experiments/${id}/execution-results/attempts/${runId}${scopeQuery(scope)}`, { signal }), + getExecutionConversation: (id: string, runId: string, scope: ResultsScope, offset = 0, signal?: AbortSignal) => + request<{ items: unknown[]; next_offset: number | null }>(`/experiments/${id}/execution-results/attempts/${runId}/conversation${resultPageQuery(scope, { offset })}`, { signal }), list: (opts?: { includeArchived?: boolean }) => request(opts?.includeArchived ? '/experiments?include_archived=true' : '/experiments'), get: (id: string) => request(`/experiments/${id}`), @@ -263,6 +308,10 @@ export const experimentsApi = { // superseded revision's instead (the design-history drill-down). listReplicates: (id: string, revisionId?: string) => request(revisionId ? `/experiments/${id}/replicates?revision_id=${revisionId}` : `/experiments/${id}/replicates`), + // Per-cell aggregates for the Cells tab -- never one row per replicate, so it + // stays small for designs with tens of thousands of replicates. + listCellSummaries: (id: string, revisionId?: string) => + request(revisionId ? `/experiments/${id}/cell-summaries?revision_id=${revisionId}` : `/experiments/${id}/cell-summaries`), // Every generation of this experiment's design, newest first -- the first // entry (superseded_at === null) is the current one. listDesignRevisions: (id: string) => request(`/experiments/${id}/design-revisions`), @@ -286,7 +335,8 @@ export const experimentsApi = { // labels carry forward, the rest stay in history (see // services.design_generation). Nothing is ever deleted here. generateDesign: (id: string, declaration?: { hypothesis?: string | null; design_spec?: DesignSpec | null; measurement_plan?: MeasurementPlan | null; measurement_validation_protocol_id?: string | null }) => - request(`/experiments/${id}/generate-design`, { method: 'POST', body: declaration }), + // The canvas re-reads cells/summaries afterwards, so skip the full replicate list (megabytes on large designs). + request(`/experiments/${id}/generate-design?include_replicates=false`, { method: 'POST', body: declaration }), // One row per replicate (a "trial"), not per ProtocolRun -- a replicate that's never // been run is still listed, with status "not_started" (see TrialResponse / // services.protocol_runs.list_experiment_trials). @@ -338,6 +388,7 @@ export const protocolsApi = { // point after retaining work that already completed. cancelRun: (id: string, runId: string) => request(`/protocols/${id}/runs/${runId}/cancel`, { method: 'POST' }), listRuns: (id: string) => request(`/protocols/${id}/runs`), + listRunHeads: async (id: string) => (await request>(`/protocols/${id}/run-heads?limit=1`)).items, // "Run all cells" -- 422 if there's no linked experiment or the graph // doesn't have exactly one final node; fans out one ProtocolRun per pending // replicate. The optional list explicitly re-runs selected completed rows. @@ -578,6 +629,8 @@ export const llmSettingsApi = { // works for anthropic/openai too, and answers even with no credential // saved (see llm_model_discovery.py's static catalog fallback). listModels: (provider: string) => request(`/llm-settings/${provider}/models`), + modelCapabilities: (provider: string, model: string) => + request(`/llm-settings/${provider}/model-capabilities?${new URLSearchParams({ model })}`), // Zero-token liveness check against the stored credential -- a free // authenticated GET per provider, never an inference call. On demand only // (a button, not an on-render fetch): it's free in tokens but it's still diff --git a/frontend/src/components/LlmConnectionCheck.test.tsx b/frontend/src/components/LlmConnectionCheck.test.tsx new file mode 100644 index 00000000..b0a71b58 --- /dev/null +++ b/frontend/src/components/LlmConnectionCheck.test.tsx @@ -0,0 +1,39 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, renderHook, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { expect, it, vi } from 'vitest' +import { llmSettingsApi } from '@/api/client' +import { providerModelsKey, useProviderModels } from './protocol/useProviderModels' +import { useConnectionCheck } from './LlmConnectionCheck' + +it.each([false, true])('uses all four checked deployments with an old request in flight: %s', async (inFlight) => { + vi.spyOn(llmSettingsApi, 'list').mockResolvedValue([ + { provider: 'azure_foundry', api_base: 'resource', azure_project_endpoint: 'project' }, + ]) + const deployments = ['FW-GLM-5.3', 'claude-sonnet-5', 'claude-sonnet-5-5', 'new-deployment'].map((id) => ({ + id, label: id, supports_temperature: true, supports_effort: false, effort_levels: [], supports_tool_calling: null, + })) + const models = vi.spyOn(llmSettingsApi, 'listModels').mockResolvedValue({ models: deployments.slice(0, 3), source: 'api', note: null }) + vi.spyOn(llmSettingsApi, 'testConnection').mockResolvedValue({ provider: 'azure_foundry', status: 'ok', detail: 'Listed 4 deployments', endpoint: 'project', models: deployments }) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const wrapper = ({ children }: { children: ReactNode }) => {children} + const { result, unmount } = renderHook(() => ({ + menu: useProviderModels('azure_foundry'), + check: useConnectionCheck('azure_foundry'), + }), { wrapper }) + await waitFor(() => expect(result.current.menu.modelsQuery.isSuccess).toBe(true)) + expect(result.current.menu.models).toHaveLength(3) + const oldResponse = { models: deployments.slice(0, 3), source: 'api' as const, note: null } + let finishOldRequest: (() => void) | undefined + let oldRequest: Promise | undefined + if (inFlight) { + models.mockImplementationOnce(() => new Promise((resolve) => { finishOldRequest = () => resolve(oldResponse) })) + act(() => { oldRequest = client.refetchQueries({ queryKey: providerModelsKey('azure_foundry') }) }) + } + await act(async () => { await result.current.check.mutateAsync() }) + await act(async () => { finishOldRequest?.(); await oldRequest }) + await waitFor(() => expect(result.current.menu.models.map((model) => model.id)).toEqual(deployments.map((model) => model.id))) + expect(models).toHaveBeenCalledTimes(inFlight ? 2 : 1) + unmount() + client.clear() +}) diff --git a/frontend/src/components/LlmConnectionCheck.tsx b/frontend/src/components/LlmConnectionCheck.tsx index fb60b363..dd094827 100644 --- a/frontend/src/components/LlmConnectionCheck.tsx +++ b/frontend/src/components/LlmConnectionCheck.tsx @@ -1,8 +1,9 @@ -import { useMutation } from '@tanstack/react-query' +import { useMutation, useQueryClient } from '@tanstack/react-query' import { CircleCheck, CircleHelp, CircleX } from 'lucide-react' import { llmSettingsApi } from '@/api/client' import { cn } from '@/lib/utils' import type { LLMConnectionStatus, LLMProvider } from '@/types/llmSettings' +import { providerModelsKey } from './protocol/useProviderModels' // Shared by all three places a credential's health is shown -- the Profile // page's credentials table, the save step in CreateCredentialDialog, and the @@ -38,7 +39,23 @@ export const CONNECTION_STATUS_META: Record< * one outbound request against a rate-limited endpoint (10/min per provider). */ export function useConnectionCheck(provider: LLMProvider | null | undefined) { - return useMutation({ mutationFn: () => llmSettingsApi.testConnection(provider!) }) + const queryClient = useQueryClient() + return useMutation({ + mutationFn: () => llmSettingsApi.testConnection(provider!), + onSuccess: async (result) => { + if (result.provider === 'azure_foundry' && result.status === 'ok') { + const queryKey = providerModelsKey(result.provider) + if (result.models != null) { + // A stale in-flight list must not overwrite the checked deployments. + await queryClient.cancelQueries({ queryKey }) + queryClient.setQueryData(queryKey, { models: result.models, source: 'api', note: null }) + } else { + // Compatibility with a backend that predates the checked-model list. + await queryClient.invalidateQueries({ queryKey }) + } + } + }, + }) } export function ConnectionStatusBadge({ diff --git a/frontend/src/components/protocol/AddNodePanel.tsx b/frontend/src/components/protocol/AddNodePanel.tsx index fbaa9f33..4a95bc49 100644 --- a/frontend/src/components/protocol/AddNodePanel.tsx +++ b/frontend/src/components/protocol/AddNodePanel.tsx @@ -33,7 +33,7 @@ type NodeCategory = (typeof NODE_CATEGORIES)[number]['id'] // allowedTypes, IS that picker. const NODE_CATALOG = [ { type: 'agent', category: 'agents', label: 'Agent', description: 'An LLM agent stage in the pipeline', icon: Bot }, - { type: 'sub_agent', category: 'agents', label: 'Sub-Agent', description: 'A delegated worker callable by one parent Agent', icon: Bot }, + { type: 'sub_agent', category: 'agents', label: 'Sub-Agent', description: 'A delegated worker callable by connected Agents', icon: Bot }, // Not a node type -- picking this opens the server browser // (McpServerBrowserPanel), and the node gets created from whichever // server is chosen there. It replaced a plain "MCP Tool" entry that made diff --git a/frontend/src/components/protocol/AgentFactorDialog.test.tsx b/frontend/src/components/protocol/AgentFactorDialog.test.tsx new file mode 100644 index 00000000..a925e26c --- /dev/null +++ b/frontend/src/components/protocol/AgentFactorDialog.test.tsx @@ -0,0 +1,71 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import { EMPTY_SCOPE } from '@/lib/promptReferences' +import { defaultAgentNodeData, type AgentNodeData, type ProtocolNode } from '@/types/protocols' +import { AgentFactorDialog } from './AgentFactorDialog' +import { factorCreationFields } from './bindableFields' + +const node: ProtocolNode & { data: AgentNodeData } = { + id: 'writer', type: 'agent', position: { x: 0, y: 0 }, + data: { ...defaultAgentNodeData('Writer'), config: { ...defaultAgentNodeData().config, prompt: 'Summarize {{node:source}}' } }, +} + +it('routes the three Agent choices from New Factor and keeps Sub-Agent choices separate', () => { + const fields = factorCreationFields([node], []) + expect(fields.map((field) => field.fieldLabel)).toEqual(['This agent on/off', 'Prompt levels', 'System prompt levels']) + expect(fields.every((field) => field.connectorFactor?.kind === 'agent' && field.pickerGroup?.category === 'Agent' && field.pickerGroup?.componentId === node.id)).toBe(true) + const subAgentFields = factorCreationFields([{ ...node, type: 'sub_agent' }], []) + expect(subAgentFields.some((field) => field.connectorFactor?.kind === 'agent')).toBe(false) +}) + +it('saves this agent on/off without changing its current activity', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + expect(screen.queryByRole('button', { name: 'Agent levels' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'All agents on/off' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Writer:Enabled', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'active', undefined)) + expect(node.data.active).not.toBe(false) +}) + +it('keeps prompt references in storage form while reordering levels and their labels', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const referenceScope = { targets: [{ id: 'source', name: 'Researcher' }], names: { source: 'Researcher' } } + render() + expect(screen.getByLabelText('Level 1 prompt')).toHaveValue('Summarize {{Researcher}}') + fireEvent.change(screen.getByLabelText('Level 2 prompt'), { target: { value: 'Analyze {{Researcher}}' } }) + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'Analysis' } }) + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ levels: ['Analyze {{node:source}}', 'Summarize {{node:source}}'], level_labels: ['Analysis', 'level1'] }), 'config.prompt', undefined)) +}) + +it('supports an empty system prompt level and blocks duplicate values and labels', () => { + render() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 system prompt'), { target: { value: 'Be concise.' } }) + expect(screen.getByLabelText('Level 1 system prompt')).toHaveValue('') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'level1' } }) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('edits legacy prompt bindings and passes the previous factor name on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.prompt': 'Prompts' } } } + render() + expect(screen.getByRole('button', { name: 'Prompt levels' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Tasks' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ name: 'Tasks', level_type: 'text', levels: ['Original', 'Alternate'] }), 'config.prompt', 'Prompts')) +}) + +it('removes a factor through its selected field without closing the dialog', async () => { + const remove = vi.fn().mockResolvedValue(undefined), close = vi.fn() + const bound = { ...node, data: { ...node.data, factor_bindings: { active: 'Available' } } } + render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Available', 'active')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() +}) diff --git a/frontend/src/components/protocol/AgentFactorDialog.tsx b/frontend/src/components/protocol/AgentFactorDialog.tsx new file mode 100644 index 00000000..317f3547 --- /dev/null +++ b/frontend/src/components/protocol/AgentFactorDialog.tsx @@ -0,0 +1,88 @@ +import { useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { factorValueKey } from '@/lib/experiment' +import type { PromptReferenceScope } from '@/lib/promptReferences' +import type { DesignFactor } from '@/types/experiments' +import type { AgentNodeData, ProtocolNode } from '@/types/protocols' +import { agentFactorFields } from './bindableFields' +import { defaultSystemPrompt } from './defaultSystemPrompt' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { PromptReferenceField } from './PromptReferenceField' + +export function AgentFactorDialog({ node, factors, initialFieldPath, referenceScope, onClose, onSave, onRemove }: { + node: ProtocolNode & { data: AgentNodeData } + factors: DesignFactor[] + initialFieldPath?: string + referenceScope: PromptReferenceScope + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string) => Promise +}) { + const bindings = node.data.factor_bindings ?? {} + const existingFor = (path: string) => factors.find((factor) => factor.name === bindings[path]) + const [path, setPath] = useState(initialFieldPath ?? agentFactorFields.find((field) => existingFor(field.fieldPath))?.fieldPath ?? 'active') + function seed(fieldPath: string): DesignFactor { + const existing = existingFor(fieldPath) + const boolean = fieldPath === 'active' + const type = boolean ? 'boolean' : 'text' + if (existing) return { ...existing, level_type: type, level_labels: existing.level_labels ?? (boolean ? existing.levels.map((level) => level ? 'Enabled' : 'Disabled') : defaultFactorLevelLabels(existing.name, existing.levels.length, type)) } + const label = boolean ? 'Enabled' : fieldPath === 'config.prompt' ? 'Prompt' : 'System prompt' + const name = computeFactorName(node.data.label || 'Agent', label, factors.map((factor) => factor.name)) + const value = fieldPath === 'config.prompt' ? node.data.config.prompt : node.data.config.system_prompt + return { name, level_type: type, levels: boolean ? [false, true] : seedLevels(value), level_labels: boolean ? ['Disabled', 'Enabled'] : defaultFactorLevelLabels(name, 2, type) } + } + const [draft, setDraft] = useState(() => seed(path)) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const boolean = path === 'active' + const current = existingFor(path) + const labels = draft.level_labels ?? [] + const duplicateLevels = new Set(draft.levels.map(factorValueKey)).size !== draft.levels.length + const valid = draft.name.trim() && draft.levels.length >= 2 && !duplicateLevels && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, path, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the agent factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + + Agent factorChoose how this agent varies across experimental cells. +
+
{agentFactorFields.map((field) => )}
+

{boolean ? 'Enable or disable this Agent node. Disabled sequential agents pass upstream output through. Cells must keep an executable agent and every required lead, supervisor, or gated worker active.' : 'Replace this field for each cell while keeping this agent’s other settings and connections. An empty level uses the normal default for this field.'}

+
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

{boolean ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}${index === 0 ? ' · default test selection' : ''}`}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {!boolean && setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} />} +
+ {!boolean && <>} +
)}
+ {duplicateLevels &&

Each level must have a different value. Change or remove a duplicate before saving.

} + {!boolean && } +

Changes require design review and regeneration.

+ {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/AgentNodeInspector.tsx b/frontend/src/components/protocol/AgentNodeInspector.tsx index f1d5babe..1747b14d 100644 --- a/frontend/src/components/protocol/AgentNodeInspector.tsx +++ b/frontend/src/components/protocol/AgentNodeInspector.tsx @@ -1,5 +1,5 @@ import { useRef } from 'react' -import { Bot } from 'lucide-react' +import { Bot, Power, Split, Trash2 } from 'lucide-react' import { useQuery } from '@tanstack/react-query' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' @@ -62,6 +62,7 @@ const GUTTER_WIDTH = 96 export function AgentNodeInspector({ node, experimentId, + subAgentFactorControlled, markedLeadAgentId, referenceScope, handoffPeers, @@ -75,6 +76,7 @@ export function AgentNodeInspector({ }: { node: (ProtocolNode & { data: AgentNodeData }) | null experimentId: string | null + subAgentFactorControlled?: boolean // Which agent on the canvas already carries the lead marker, if any -- the // inspector can't see its siblings, so ProtocolCanvas resolves it. markedLeadAgentId: string | null @@ -197,20 +199,23 @@ export function AgentNodeInspector({ placeholder={node.type === 'sub_agent' ? 'Sub-Agent' : 'Agent'} onCommit={(label) => onChange(node.id, { ...data, label })} /> - bindFactor('active', name)} - onUnbind={() => unbindFactor('active')} - > - {(trigger) => trigger} - + {!isSubAgent && <> + bindFactor('active', name)} + onUnbind={() => unbindFactor('active')} + > + {(trigger) => trigger} + + } } onDelete={() => onDelete(node.id)} @@ -268,6 +273,23 @@ export function AgentNodeInspector({ + <> +
+

+ Make factor has moved to the node toolbar. Hover over the {isSubAgent ? 'Sub-Agent' : 'Agent'} node and click the

+
+ {isSubAgent && (subAgentFactorControlled ?

Availability is controlled by the parent Agent’s Sub-Agents factor. Remove it to restore individual controls.

: bindings.active &&

Factor: {bindings.active}

)} + {subAgentFactorControlled && bindings.active && } + {/* First, above Prompt: which agent leads decides whose prompt becomes the task and whose answer gets scored, so it frames everything below it rather than being one more setting. It @@ -317,6 +339,7 @@ export function AgentNodeInspector({ experimentId={experimentId} nodeId={node.id} fieldPath="config.prompt" + creationInToolbar={!isSubAgent} defaultLabel="Prompt" nodeLabel={data.label || 'Agent'} levelType="text" @@ -460,6 +483,7 @@ export function AgentNodeInspector({ { + it('groups nodes alphabetically by agent and supports collapsing and search', async () => { + const user = userEvent.setup() + const graph = { + nodes: ['Zebra', 'Alpha'].map(label => ({ id: label, type: 'agent', position: { x: 0, y: 0 }, data: { label } })), + edges: [], + } as unknown as ProtocolGraph + renderFlow(graph) + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + const list = screen.getByRole('listbox', { name: 'Metric nodes' }) + const summaries = [...list.querySelectorAll('summary')] + expect(summaries.map(summary => summary.textContent)).toEqual(['Alpha', 'Runtime metrics for selected nodes', 'Zebra']) + expect(summaries.every(summary => !summary.parentElement?.hasAttribute('open'))).toBe(true) + await user.click(summaries[0]) + expect(summaries[0].parentElement).toHaveAttribute('open') + await user.click(summaries[0]) + expect(summaries[0].parentElement).not.toHaveAttribute('open') + await user.type(screen.getByRole('textbox', { name: 'Search metric nodes' }), 'Alpha') + expect(screen.queryByRole('option', { name: 'Zebra, Agent' })).not.toBeInTheDocument() + await user.click(screen.getByRole('option', { name: 'Alpha, Agent' })) + expect(screen.queryByRole('textbox', { name: 'Search metric nodes' })).not.toBeInTheDocument() + expect(screen.getByRole('combobox', { name: 'Metric node' })).toHaveTextContent('Alpha') + }) + + it('sorts categories and node names alphabetically within each agent', async () => { + const user = userEvent.setup() + const graph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + ...['Zulu', 'Alpha'].map(label => ({ id: label, type: 'script', position: { x: 0, y: 0 }, data: { label, config: { code: 'print(1)' } } })), + { id: 'tool', type: 'mcp_tool', position: { x: 0, y: 0 }, data: { label: 'Quality', config: { server_id: 'server-1', tool_names: ['score'] } } }, + ], + edges: ['Zulu', 'Alpha', 'tool'].map(id => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), + } as unknown as ProtocolGraph + renderFlow(graph) + await user.click(screen.getByRole('combobox', { name: 'Metric node' })) + const list = screen.getByRole('listbox', { name: 'Metric nodes' }) + const agentGroup = [...list.querySelectorAll('details')].find(group => group.querySelector('summary')?.textContent === 'Writer')! + expect(agentGroup).toHaveAttribute('open') + expect([...agentGroup.querySelectorAll('h3')].map(heading => heading.textContent)).toEqual(['Agent', 'Tools']) + const toolsGroup = within(agentGroup).getByRole('region', { name: 'Writer: Tools' }) + expect([...toolsGroup.querySelectorAll('h4')].map(heading => heading.textContent)).toEqual(['MCP Tools', 'Scripts']) + expect(toolsGroup.querySelector('h4')).toHaveClass('text-sm', 'text-chart-2') + expect(within(toolsGroup).getAllByRole('option').every(option => option.parentElement?.classList.contains('ml-3'))).toBe(true) + expect(within(agentGroup).getAllByRole('option').map(option => option.getAttribute('aria-label')?.split(', ').slice(0, 2).join(', '))).toEqual(['Writer, Agent', 'Writer:Quality, MCP Tool', 'Writer:Alpha, Script', 'Writer:Zulu, Script']) + }) + it('sums a runtime measure over several chosen nodes', async () => { const user = userEvent.setup() const graph = { @@ -45,7 +91,7 @@ describe('CustomMetricFlow', () => { const onSave = renderFlow(graph) await user.click(screen.getByRole('combobox', { name: 'Metric node' })) - await user.click(screen.getByRole('option', { name: 'Runs of chosen nodes, Node runtime' })) + await user.click(screen.getByRole('option', { name: 'Runtime metrics for selected nodes, Node runtime' })) expect(screen.getByRole('button', { name: 'Add custom metric' })).toBeDisabled() await user.click(screen.getByRole('checkbox', { name: /^Critic \(FS\)/ })) await user.click(screen.getByRole('checkbox', { name: /^Critic \(DC\)/ })) diff --git a/frontend/src/components/protocol/CustomMetricDialogs.tsx b/frontend/src/components/protocol/CustomMetricDialogs.tsx index 15884935..7d7c7fd7 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.tsx @@ -1,11 +1,14 @@ import { useEffect, useRef, useState } from 'react' import { useQuery } from '@tanstack/react-query' +import { ChevronsUpDown } from 'lucide-react' import { mcpServersApi, protocolsApi } from '@/api/client' import { Badge } from '@/components/ui/badge' import { Checkbox } from '@/components/ui/checkbox' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { HUD_ACCENT_RING_CLASSNAME, PICKER_GROUP_CLASSNAME, cn } from '@/lib/utils' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { AGENT_OUTPUT_PRODUCER_ID, agentOutputSourceOptions } from '@/lib/agentOutputMetrics' import { mcpToolSourceOptions, type McpToolSourceOption } from '@/lib/mcpToolMetrics' @@ -50,16 +53,55 @@ export function MetricNodeLabel({ display, reason }: { display: MetricNodeDispla ) } -function MetricNodeOption({ value, display, reason }: { value: string; display: MetricNodeDisplay; reason?: string }) { - return ( - - - - ) +type MetricNodeChoice = { value: string; display: MetricNodeDisplay; reason?: string; agentId?: string; category: string } + +function MetricNodePicker({ value, display, choices, graph, onChange }: { + value: string + display: MetricNodeDisplay | null | undefined + choices: MetricNodeChoice[] + graph?: ProtocolGraph + onChange: (value: string) => void +}) { + const [open, setOpen] = useState(false) + const [search, setSearch] = useState('') + const expandGroups = (graph?.nodes.filter(node => node.type === 'agent' || node.type === 'sub_agent').length ?? 0) < 2 + const groups = new Map> }>() + for (const choice of choices) { + const key = choice.agentId ?? (choice.value === 'node_runtime' ? 'runtime' : 'shared') + const label = choice.agentId ? String(graph?.nodes.find(node => node.id === choice.agentId)?.data.label || 'Agent') + : key === 'runtime' ? 'Runtime metrics for selected nodes' : 'Shared or unconnected components' + const connector = ['MCP Tools', 'Scripts', 'Tool steps'].includes(choice.category) ? 'Tools' : choice.category === 'Agent output' ? 'Agent' : choice.category + if (!`${label} ${connector} ${choice.category} ${choice.display.label} ${choice.display.type}`.toLowerCase().includes(search.trim().toLowerCase())) continue + if (!groups.has(key)) groups.set(key, { label, connectors: new Map() }) + const connectors = groups.get(key)!.connectors + if (!connectors.has(connector)) connectors.set(connector, new Map()) + const categories = connectors.get(connector)! + categories.set(choice.category, [...(categories.get(choice.category) ?? []), choice]) + } + return { setOpen(next); if (next) setSearch('') }}> + }> + {display ? : 'Select a node…'} + + + setSearch(event.target.value)} /> +
+ {groups.size === 0 &&

No matching nodes.

} + {[...groups.entries()].sort(([, a], [, b]) => a.label.localeCompare(b.label)).map(([id, group]) =>
+ {group.label} +
{[...group.connectors.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([connector, categories]) =>
+

{connector}

+
{[...categories.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([category, options]) =>
+ {connector === 'Tools' &&

{category}

} +
{options.sort((a, b) => a.display.label.localeCompare(b.display.label)).map(choice => )}
+
)}
+
)}
+
)} +
+ + } function duplicateName(metric: DesignMetric, name: string, metrics: DesignMetric[]) { @@ -206,7 +248,7 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM label: source.label, type: 'Tool Step', }) - const nodeRuntimeDisplay: MetricNodeDisplay = { label: 'Runs of chosen nodes', type: 'Node runtime' } + const nodeRuntimeDisplay: MetricNodeDisplay = { label: 'Runtime metrics for selected nodes', type: 'Node runtime' } const selectedNodeDisplay = producer === 'node_runtime' ? nodeRuntimeDisplay : selectedAgentSource ? agentNodeDisplay(selectedAgentSource) : selectedPythonSource @@ -380,17 +422,13 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM Metric node {!sourceContext ?
- + ({ value: `agent:${source.agentNodeId}`, display: agentNodeDisplay(source), reason: agentSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Agent output' })), + ...pythonSources.map(source => ({ value: `python:${source.key}`, display: pythonNodeDisplay(source), reason: pythonSourceDisabledReason(source), agentId: source.agentNodeId, category: 'Scripts' })), + ...mcpSources.map(source => ({ value: `mcp:${source.key}`, display: mcpNodeDisplay(source), reason: mcpSourceDisabledReason(source), agentId: source.agentNodeId, category: 'MCP Tools' })), + ...toolStepSources.map(source => ({ value: `tool_step:${source.nodeId}`, display: toolStepNodeDisplay(source), reason: source.disabledReason, category: 'Tool steps' })), + ...(offerNodeRuntime ? [{ value: 'node_runtime', display: nodeRuntimeDisplay, category: 'Runtime' }] : []), + ]} />
: <>

{producer === 'agent' ? 'Agent output' : producer === 'tool_step' ? 'Tool Step' : producer === 'node_runtime' ? 'Node runtime' : producer === 'python' ? 'Python Script' : 'MCP Tool'}

{producer === 'tool_step' &&

{selectedToolStepSource?.label}

{toolStepSourceError &&

{toolStepSourceError}

}
} @@ -402,7 +440,7 @@ export function CustomMetricFlow({ metric, binding, graph, protocolId, existingM {producer === 'node_runtime' &&
- Runtime total + Runtime metrics for selected nodes
{runtimeNodeSources.map((source) =>
+
patch({ description: event.target.value })} />
patch({ code })} rows={10} /> @@ -414,7 +431,7 @@ function DatasetConfigLevelRow({ // the factor arrived with an imported design_spec) the level falls back to // listing its stored names read-only, so an unresolvable server never looks // like "no tools selected". -function ToolNamesLevelRow({ +export function ToolNamesLevelRow({ value, serverId, onChange, @@ -498,11 +515,22 @@ function ToolNamesLevelRow({ // across the graph. Deleting a factor already has an obvious home (the // remove button on DesignTab's own summary row), so this dialog only ever // edits, never deletes. -export function FactorEditorDialog({ +export function FactorEditorDialog(props: Parameters[0]) { + if (props.factor.level_type === 'sub_agent_selection' || props.factor.level_type === 'sub_agent_toggle') return + if (props.factor.level_type === 'knowledge_selection' || props.factor.level_type === 'knowledge_toggle') return + if (props.factor.level_type === 'dataset_selection' || props.factor.level_type === 'dataset_toggle') return + if ((props.factor.level_type === 'skill_selection' || props.factor.level_type === 'skill_toggle')) return + if (props.factor.level_type === 'tool_selection' || props.factor.level_type === 'tool_toggle') return + if (props.factor.level_type === 'script_selection' || props.factor.level_type === 'script_toggle') return + return +} + +function StandardFactorEditorDialog({ open, onOpenChange, factor, pickableFields, + canvasAgentCount = 1, existingNames, emptyPickerMessage, toolServerId, @@ -510,7 +538,9 @@ export function FactorEditorDialog({ boundField, promptScopeFor, onSave, + onPickConnectorFactor, }: { + graph?: ProtocolGraph open: boolean onOpenChange: (open: boolean) => void factor: DesignFactor @@ -535,6 +565,8 @@ export function FactorEditorDialog({ // large number of fields to search through, and this dialog already has // the room a cramped popover wouldn't. pickableFields?: UnboundField[] + canvasAgentCount?: number + onPickConnectorFactor?: (field: UnboundField) => void // Needed to dedupe the computed name once a field is picked -- only // meaningful alongside pickableFields. existingNames?: string[] @@ -562,42 +594,45 @@ export function FactorEditorDialog({ // actual canvas node (this dialog has no way to do that itself). onSave: (factor: DesignFactor, field?: UnboundField) => void | Promise }) { - const [selectedField, setSelectedField] = useState(null) - const [selectedName, setSelectedName] = useState('') const [search, setSearch] = useState('') - const [levelType, setLevelType] = useState(levelTypeOf(factor)) - const [levels, setLevels] = useState(() => (isStructuredLevelType(levelTypeOf(factor)) ? factor.levels : factor.levels.map((l) => String(l)))) - const [levelLabels, setLevelLabels] = useState(() => factorLevelLabels(factor)) - - // Re-seed the local draft every time this dialog (re)opens -- same - // "re-seed on open" convention DesignTab.tsx and FactorBindableField.tsx's - // popover already use. In field-picker mode there's nothing to seed yet - // until a field is actually picked (see pickField below). useEffect(() => { - if (open) { - setSelectedField(null) - setSelectedName('') - setSearch('') - if (!pickableFields) { - const t = levelTypeOf(factor) - setLevelType(t) - setLevels(isStructuredLevelType(t) ? factor.levels : factor.levels.map((l) => String(l))) - setLevelLabels(factorLevelLabels(factor)) - } - } - // eslint-disable-next-line react-hooks/exhaustive-deps + if (open) setSearch('') }, [open]) + const initialLevelType = levelTypeOf(factor) + const { draft: localDraft, setDraft: setLocalDraft, dirty } = useDialogDraft({ + selectedField: null as UnboundField | null, + selectedName: '', + levelType: initialLevelType, + levels: isStructuredLevelType(initialLevelType) ? factor.levels : factor.levels.map(String), + levelLabels: factorLevelLabels(factor), + }, { + open, sessionKey: pickableFields ? 'new' : factor.name, + signatureOf: (value) => JSON.stringify(makeDraft( + pickableFields ? value.selectedName : factor.name, + value.levelType, value.levels, value.levelLabels, value.selectedField ?? undefined, + )), + }) + const { selectedField, selectedName, levelType, levels, levelLabels } = localDraft + function setLevels(value: SetStateAction) { + setLocalDraft((previous) => ({ ...previous, levels: typeof value === 'function' ? value(previous.levels) : value })) + } + function setLevelLabels(value: SetStateAction) { + setLocalDraft((previous) => ({ ...previous, levelLabels: typeof value === 'function' ? value(previous.levelLabels) : value })) + } function pickField(field: UnboundField) { - setSelectedField(field) + if (field.connectorFactor && onPickConnectorFactor) { + onPickConnectorFactor(field) + return + } const nextName = computeFactorName(field.nodeLabel, field.fieldLabel, existingNames ?? []) - setSelectedName(nextName) - setLevelType(field.levelType) const nextLevels = isStructuredLevelType(field.levelType) ? seedStructuredLevels(field.currentValue, field.levelType) : seedLevels(field.currentValue) - setLevels(nextLevels) - setLevelLabels(defaultFactorLevelLabels(nextName, nextLevels.length, field.levelType)) + setLocalDraft({ + selectedField: field, selectedName: nextName, levelType: field.levelType, + levels: nextLevels, levelLabels: defaultFactorLevelLabels(nextName, nextLevels.length, field.levelType), + }) } // Computed, not user-typed (see factorLevels.ts's computeFactorName) -- @@ -636,7 +671,6 @@ export function FactorEditorDialog({ const draft = makeDraft(name, levelType, levels, levelLabels, selectedField ?? undefined) const draftSignature = JSON.stringify(draft) - const initialLevelType = levelTypeOf(factor) const initialSignature = JSON.stringify(makeDraft( factor.name, initialLevelType, @@ -644,7 +678,7 @@ export function FactorEditorDialog({ factorLevelLabels(factor), )) const factorAutosave = useDialogAutosave({ - draft: open && !needsFieldPick && draftSignature !== initialSignature ? draft : null, + draft: open && dirty && !needsFieldPick && draftSignature !== initialSignature ? draft : null, signature: draftSignature, onSave: ({ factor: next, field }) => onSave(next, field), }) @@ -672,8 +706,9 @@ export function FactorEditorDialog({ const accent = hashToChartHue(name || 'factor') const filteredFields = (pickableFields ?? []).filter((f) => - `${f.nodeLabel}:${f.fieldLabel}`.toLowerCase().includes(search.trim().toLowerCase()), + `${f.nodeLabel}:${f.fieldLabel} ${f.pickerGroup?.label ?? ''} ${f.pickerGroup?.category ?? ''} ${f.pickerGroup?.componentLabel ?? ''}`.toLowerCase().includes(search.trim().toLowerCase()), ) + const fieldGroups = groupFactorCreationFields(filteredFields) return ( @@ -692,12 +727,12 @@ export function FactorEditorDialog({
-
+
{needsFieldPick ? ( -
+
setSearch(e.target.value)} /> -
+
{filteredFields.length === 0 && (

{pickableFields?.length === 0 @@ -705,21 +740,30 @@ export function FactorEditorDialog({ : 'No matching fields.'}

)} - {filteredFields.map((field) => ( + {fieldGroups.map((group) =>
+ {group.label}{group.connectors.reduce((count, connector) => count + connector.components.reduce((total, component) => total + component.fields.length, 0), 0)} options +
{group.connectors.map((connector) =>
+

{connector.label}

+
{connector.components.map((component) =>
+ {connector.label !== 'Agent fields' &&

{component.label}

} +
{component.fields.map((field) => ( - ))} + ))}
+
)}
+
)}
+
)}
) : ( @@ -732,7 +776,7 @@ export function FactorEditorDialog({ diff --git a/frontend/src/components/protocol/KnowledgeFactorDialog.test.tsx b/frontend/src/components/protocol/KnowledgeFactorDialog.test.tsx new file mode 100644 index 00000000..d887d613 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeFactorDialog.test.tsx @@ -0,0 +1,83 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { KnowledgeFactorDialog } from './KnowledgeFactorDialog' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'knowledge', type: 'okf_bundle', data: { label: 'Summarize', config: { bundle_id: 'a' }, factor_bindings: { 'config.enabled': 'Knowledge enabled' } } }, + ], + edges: [{ source: 'knowledge', target: 'agent', targetHandle: 'knowledge' }], +} as unknown as ProtocolGraph + +it.each(['Knowledge levels', 'All knowledge on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Knowledge enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Knowledge node’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This knowledge on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Knowledge enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected knowledge', () => { + render() + expect(screen.getByRole('button', { name: 'This knowledge on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Knowledge levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Knowledge to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This knowledge on/off', 'Knowledge levels'])('blocks %s when an Agent already has an all-knowledge factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { knowledge_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { bundle_id: 'a' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('saves edits to an existing individual factor without removing other declarations', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Renamed' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Renamed', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'knowledge', 'Knowledge enabled')) +}) + +it('opens the existing group factor when a shared knowledge has multiple connected agents', () => { + const sharedGraph = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'First agent' } }, + { id: 'second', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Second agent', knowledge_selection: ['a'], knowledge_factor_mode: 'knowledge_toggle', factor_bindings: { knowledge_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { bundle_id: 'a' } } }, + ], + edges: [...graph.edges, { id: 'second-edge', source: 'knowledge', target: 'second', targetHandle: 'knowledge' }], + } as ProtocolGraph + render() + expect(screen.queryByRole('combobox')).not.toBeInTheDocument() + expect(screen.getByText('Applies to: First agent, Second agent')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'All knowledge on/off' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByLabelText('Factor name')).toHaveValue('Group') +}) diff --git a/frontend/src/components/protocol/KnowledgeFactorDialog.tsx b/frontend/src/components/protocol/KnowledgeFactorDialog.tsx new file mode 100644 index 00000000..acbdb456 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeFactorDialog.tsx @@ -0,0 +1,113 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { connectedKnowledge, knowledgeId, reconcileKnowledgeFactor, knowledgeFactorConflict, knowledgeFactorModes as modes, KNOWLEDGE_FACTOR_PATH, type KnowledgeFactorMode as Mode } from '@/lib/knowledgeFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +export function KnowledgeFactorDialog({ knowledgeNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { + knowledgeNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const knowledge = graph.nodes.find((node) => node.id === knowledgeNodeId)! + const agents = factorRecipients(graph, knowledgeNodeId, 'knowledge') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[KNOWLEDGE_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, KNOWLEDGE_FACTOR_PATH) ?? existingFor(knowledgeNodeId, 'config.enabled') + const initial = initialFactor && initialFactor.level_type !== 'boolean' && initialFactor.level_type ? reconcileKnowledgeFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = nextMode === 'boolean' ? knowledgeNodeId : nextAgentId + const existing = existingFor(ownerId, nextMode === 'boolean' ? 'config.enabled' : KNOWLEDGE_FACTOR_PATH) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level) => level ? 'Enabled' : 'Disabled'), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Knowledge'), nextMode === 'boolean' ? 'Enabled' : 'Knowledge', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return nextMode === 'boolean' ? factor : reconcileKnowledgeFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level) => level ? 'Enabled' : 'Disabled') } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = mode === 'boolean' ? knowledgeNodeId : agentId + const current = existingFor(ownerId, mode === 'boolean' ? 'config.enabled' : KNOWLEDGE_FACTOR_PATH) + const affected = mode === 'boolean' ? [knowledge] : connectedKnowledge(graph, agentId) + const unavailable = (value: Mode) => value !== 'boolean' && !agentId + ? `${modes[value]} is unavailable because this Knowledge is not connected to an Agent. Connect this Knowledge to an Agent first.` + : knowledgeFactorConflict(graph, factors, knowledgeNodeId, value, agentId) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the knowledge factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Knowledge factor + Choose how knowledge varies across experimental cells. + +
+ {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, knowledgeNodeId, 'knowledge')}

} +
{(Object.keys(modes) as Mode[]).map((value) => )}
+ {blockedReason ?

{blockedReason}

: <> +

{mode === 'boolean' ? 'Enable or disable this Knowledge node.' : mode === 'knowledge_selection' ? 'Each cell receives exactly one connected knowledge source.' : 'Each cell enables every connected knowledge source, including deactivated sources, or disables them all.'}

+
setDraft({ ...draft, name: event.target.value })} />
+ {mode !== 'boolean' && affected.length < (mode === 'knowledge_toggle' ? 1 : 2) &&

Connect at least {mode === 'knowledge_toggle' ? 'one knowledge source' : 'two different knowledge sources'} before generating cells.

} +
{draft.levels.map((level, index) =>
+
+

{mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'knowledge_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => knowledgeId(node) === (level as string[])[0])?.data.label ?? 'Unavailable knowledge')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ {mode !== 'boolean' && <>} +
)}
+

Shared factors require matching Knowledge connections. Changes require design review and regeneration.

+ } + {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/KnowledgeFactorEditor.tsx b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx new file mode 100644 index 00000000..a5350830 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { useKnowledgeLibrary } from './useKnowledgeLibrary' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +export function KnowledgeFactorEditor({ factor, open, onOpenChange, onSave, onRemove }: { + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const knowledge = useKnowledgeLibrary() + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the knowledge factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'knowledge_toggle' ? 'All knowledge on/off' : 'Knowledge as levels'} + {factor.level_type === 'knowledge_toggle' ? 'Each cell receives all connected knowledge or none.' : 'Each cell receives exactly one connected knowledge source.'} Add or disconnect Knowledge nodes on the canvas to change the levels. + +
+ + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

Incomplete: connect at least two different knowledge sources before generating cells.

} +
+ {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const source = knowledge.data?.find((item) => item.id === id) + return
+
+

{factor.level_type === 'knowledge_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : source?.name ?? labels[index] ?? 'Unavailable knowledge'}{index === 0 ? ' · default test selection' : ''}

+ {factor.level_type !== 'knowledge_toggle' && knowledge.isSuccess && !source &&

Unavailable: disconnect or restore this knowledge.

} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
+ + +
+ })} +
+ {knowledge.isError &&

Could not load the knowledge library.

} +

Individual knowledge switches are controlled by this connector. Changes require design review and regeneration.

+ {onRemove &&

Removing this factor enables all connected knowledge and restores their individual controls.

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/KnowledgeTestSelectors.tsx b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx new file mode 100644 index 00000000..5bd8be47 --- /dev/null +++ b/frontend/src/components/protocol/KnowledgeTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedKnowledge, knowledgeId, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' +import type { useKnowledgeTestSelection } from './useKnowledgeTestSelection' + +export function KnowledgeTestSelectors({ selection }: { selection: ReturnType }) { + return
{selection.owners.map((node) => { + const options = connectedKnowledge(selection.graph!, node.id) + if (node.data.knowledge_factor_mode === 'knowledge_toggle') return
+

{node.data.factor_bindings?.[KNOWLEDGE_FACTOR_PATH] || node.data.label}: knowledge for this test

+ +
+ return
+

{node.data.factor_bindings?.[KNOWLEDGE_FACTOR_PATH] || node.data.label}: knowledge for this test

+ +
+ })}{selection.error &&

{selection.error}

}
+} diff --git a/frontend/src/components/protocol/McpToolNodeInspector.test.tsx b/frontend/src/components/protocol/McpToolNodeInspector.test.tsx new file mode 100644 index 00000000..311aa940 --- /dev/null +++ b/frontend/src/components/protocol/McpToolNodeInspector.test.tsx @@ -0,0 +1,20 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { McpToolNodeData, ProtocolNode } from '@/types/protocols' +import { McpToolNodeInspector } from './McpToolNodeInspector' + +it('shows the moved factor message and connector-controlled availability', () => { + const client = new QueryClient() + client.setQueryData(['mcp-servers'], []) + const node: ProtocolNode & { data: McpToolNodeData } = { + id: 'tool', type: 'mcp_tool', position: { x: 0, y: 0 }, + data: { label: 'Search', config: { server_id: 's', server_name: 'Search', tool_names: ['search'], enabled: false } }, + } + render() + expect(screen.getByText(/has moved to the node toolbar/)).toHaveTextContent('Hover over the Tool node') + expect(screen.getByText(/Availability is controlled/)).toHaveTextContent('Tools') + expect(screen.getByRole('switch')).toHaveAttribute('aria-disabled', 'true') + expect(screen.getByRole('switch')).toBeChecked() + client.clear() +}) diff --git a/frontend/src/components/protocol/McpToolNodeInspector.tsx b/frontend/src/components/protocol/McpToolNodeInspector.tsx index 1b0936f6..a200d26b 100644 --- a/frontend/src/components/protocol/McpToolNodeInspector.tsx +++ b/frontend/src/components/protocol/McpToolNodeInspector.tsx @@ -1,7 +1,7 @@ import { useState } from 'react' import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { KeyRound, Plug, RefreshCw, Wrench } from 'lucide-react' +import { KeyRound, Plug, RefreshCw, Wrench, Power, Split, Trash2 } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -9,7 +9,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { mcpServersApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { MCP_CLIENT_TOOL_NODE_TYPE } from './mcpServerCatalog' import { NodeInspectorDialog } from './NodeInspectorDialog' import { ManageMcpCredentialsDialog } from './ManageMcpCredentialsDialog' @@ -40,26 +39,17 @@ const TRANSPORT_LABELS: Record = { stdio: 'stdio', http: 'Stream // dropdown -- and the whole-config "Server & tools" factor binding that hung // off it -- are gone deliberately: reassigning a node's server after the // fact contradicts the one-node-per-server model. Use a second MCP Servers -// node instead. What DID survive that removal is varying the allow-list -// alone: "Tools allowed" is bindable as a `tool_names` factor whose levels -// are each a subset of this one server's tools. +// node instead. Factor creation lives in the node toolbar; this inspector +// preserves the pinned server and edits its base allow-list. export function McpToolNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: McpToolNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's - // agentTracedLabel) -- distinct from data.label, which is this node's own - // plain label shown in the header title. Only used to scope this node's - // "+ Make experimental factor" names, e.g. "Research Agent:Search API: - // Enabled" instead of an ambiguous plain "Search API:Enabled" that can't - // tell two agents' identically-labeled tool nodes apart. - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: McpToolNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -107,10 +97,6 @@ export function McpToolNodeInspector({ patchConfig({ tool_names: allowed ? [...selectedTools, name] : selectedTools.filter((t) => t !== name) }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -137,29 +123,23 @@ export function McpToolNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

Off: this server's tools aren't offered to the agent at all.

-
- patchConfig({ enabled: checked })} /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Tool node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this server’s tools at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} {/* Which process this node actually talks to. Most valuable on a client tool -- its endpoint is something a user typed, so it's part of the @@ -226,44 +206,14 @@ export function McpToolNodeInspector({ )}
- {/* The one factor this node type offers beyond Enabled: which of THIS - server's tools the agent may call, varied per cell. The node's - server stays pinned across every level -- see bindableFields.ts's - mcp_tool case for why swapping it isn't offered. The toggle list - below stays live even once bound (it's still this node's base - config, and every other inspector leaves its own bound control - editable too); the levels just override it per cell. */} - bindFactor('config.tool_names', name)} - onUnbind={() => unbindFactor('config.tool_names')} - > - {(trigger) => ( -
- - {tools.length > 0 && ( -
- - -
- )} -
- )} - +
+
{bindings['config.tool_names'] &&

Factor: {bindings['config.tool_names']}

}
+ {tools.length > 0 &&
+ + +
} +
+ {connectorFactorName &&

This allow-list is preserved when the connector selects this node.

} {serversQuery.isLoading ? ( ) : serversQuery.isError ? ( diff --git a/frontend/src/components/protocol/ModelFactorDialog.test.tsx b/frontend/src/components/protocol/ModelFactorDialog.test.tsx new file mode 100644 index 00000000..7f7e1975 --- /dev/null +++ b/frontend/src/components/protocol/ModelFactorDialog.test.tsx @@ -0,0 +1,161 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ModelNodeData, ProtocolNode } from '@/types/protocols' +import { factorCreationFields } from './bindableFields' +import { ModelFactorDialog } from './ModelFactorDialog' +import { LlmConfigLevelRow } from './FactorEditorDialog' + +vi.mock('./useProviderModels', () => ({ useProviderModels: (_provider: string, model?: string) => ({ models: [ + { id: 'gpt-5', label: 'GPT 5', supports_temperature: true, supports_effort: false, effort_levels: [] }, + { id: 'reasoning', label: 'Reasoning model', supports_temperature: false, supports_effort: true, effort_levels: ['low', 'high'] }, +], capabilities: model === 'custom-id' ? { + supports_temperature: false, supports_effort: true, + effort_levels: ['low', 'medium', 'high'], default_effort: 'medium', +} : undefined, modelsQuery: { isLoading: false } }) })) + +const node: ProtocolNode & { data: ModelNodeData } = { + id: 'model', type: 'model_openai', position: { x: 0, y: 0 }, + data: { label: 'OpenAI', config: { provider: 'openai', model: 'gpt-5', temperature: 0.7, max_tokens: 1000 } }, +} + +it('refreshes untouched levels after a refetch and saves an edit with paired labels', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.max_tokens': 'Tokens' } } } + const props = { node: bound, nodeLabel: 'OpenAI', initialFieldPath: 'config.max_tokens', onClose: vi.fn(), onSave: save, onRemove: vi.fn() } + const factor = { name: 'Tokens', level_type: 'number' as const, levels: [2048, 1024], level_labels: ['high', 'low'] } + const refreshed = { ...factor, levels: [1024, 2048], level_labels: ['low', 'high'] } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 value')).toHaveValue(1024) + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'small' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['small', 'high'] }, 'config.max_tokens', 'Tokens')) +}) + +it('routes New Factor model choices to the dedicated dialog while retaining their node grouping', () => { + const fields = factorCreationFields([node], []) + expect(fields).toHaveLength(5) + expect(fields.find((field) => field.fieldPath === 'config')).toMatchObject({ + fieldLabel: 'Model levels', connectorFactor: { kind: 'model', nodeId: 'model', mode: 'model_config' }, + pickerGroup: { category: 'Model', componentId: 'model', componentLabel: 'OpenAI' }, + }) + expect(fields.every((field) => field.connectorFactor?.kind === 'model')).toBe(true) +}) + +it('blocks whole-model factors when individual fields are already bound, and allows editing that field', () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.temperature': 'Heat' } } } + render() + fireEvent.click(screen.getByRole('button', { name: 'Model levels' })) + expect(screen.getByRole('status')).toHaveTextContent('cannot be combined') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'Temperature' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Heat') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('saves reordered numeric levels with their labels and previous factor name', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const bound = { ...node, data: { ...node.data, factor_bindings: { 'config.temperature': 'Heat' } } } + render() + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Temperature' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Temperature', level_type: 'number', levels: [1, 0.7], level_labels: ['High', 'Low'] }, 'config.temperature', 'Heat')) +}) + +it('removes an existing model factor through its bound path', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Models', 'config')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + rerender() + expect(screen.getByRole('dialog')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Temperature' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() +}) + +it('uses the shared model dropdown and disables models selected in other levels', async () => { + render() + expect(screen.getByRole('combobox', { name: 'Level 1 model' })).toHaveTextContent('GPT 5') + expect(screen.getByRole('combobox', { name: 'Level 2 model' })).toHaveTextContent('Select a model') + expect(screen.getByRole('button', { name: 'Temperature' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Effort' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('combobox', { name: 'Level 2 model' })) + expect(await screen.findByRole('option', { name: 'GPT 5' })).toHaveAttribute('aria-disabled', 'true') + expect(screen.getByRole('option', { name: 'Reasoning model' })).not.toHaveAttribute('aria-disabled', 'true') +}) + +it('offers only Effort for a reasoning model and disables efforts selected in other levels', async () => { + const reasoning = { ...node, data: { ...node.data, config: { ...node.data.config, model: 'reasoning', effort: 'low' } } } + render() + expect(screen.queryByRole('button', { name: 'Temperature' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Effort' })).toBeInTheDocument() + expect(screen.getByRole('combobox', { name: 'Level 1 effort' })).toHaveTextContent('low') + fireEvent.click(screen.getByRole('combobox', { name: 'Level 2 effort' })) + expect(await screen.findByRole('option', { name: 'low' })).toHaveAttribute('aria-disabled', 'true') + expect(screen.getByRole('option', { name: 'high' })).not.toHaveAttribute('aria-disabled', 'true') +}) + +it('updates whole-model level controls when its selected model changes', () => { + const { rerender } = render() + expect(screen.getByText('Temperature')).toBeInTheDocument() + expect(screen.queryByText('Effort')).not.toBeInTheDocument() + rerender() + expect(screen.queryByText('Temperature')).not.toBeInTheDocument() + expect(screen.getByText('Effort')).toBeInTheDocument() + rerender() + expect(screen.queryByText('Temperature')).not.toBeInTheDocument() + expect(screen.getByText('Effort')).toBeInTheDocument() + expect(screen.getByText('Default (medium)')).toBeInTheDocument() +}) + +it('warns about duplicate model configurations regardless of property order and clears when changed', () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + render() + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + const temperatures = screen.getAllByRole('spinbutton') + fireEvent.change(temperatures[2], { target: { value: '1' } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() +}) + +it.each([ + ['config.temperature', '0.70', '1'], + ['config.max_tokens', '1000.0', '2000'], +])('warns about duplicate numeric values for %s and clears after correction', (path, duplicate, distinct) => { + render() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: duplicate } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: distinct } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() +}) + +it('does not treat an empty numeric level as a duplicate of zero', () => { + render() + fireEvent.change(screen.getByLabelText('Level 1 value'), { target: { value: '0' } }) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '0.0' } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels: 1, 2') +}) + +it('focuses the level selected from a canvas preview', async () => { + const bound = { ...node, data: { ...node.data, factor_bindings: { config: 'Models' } } } + render() + await waitFor(() => expect(screen.getByLabelText('Level 2 label')).toHaveFocus()) +}) diff --git a/frontend/src/components/protocol/ModelFactorDialog.tsx b/frontend/src/components/protocol/ModelFactorDialog.tsx new file mode 100644 index 00000000..aff2c7aa --- /dev/null +++ b/frontend/src/components/protocol/ModelFactorDialog.tsx @@ -0,0 +1,131 @@ +import type { ModelCapabilities } from '@/types/llmSettings' +import { modelCapabilities } from '@/lib/modelCapabilities' +import { useEffect, useRef, useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import type { DesignFactor } from '@/types/experiments' +import { factorValueKey } from '@/lib/experiment' +import type { ProtocolNode } from '@/types/protocols' +import { bindableFieldsForNode } from './bindableFields' +import { LlmConfigLevelRow } from './FactorEditorDialog' +import { ModelField } from './ModelField' +import { useProviderModels } from './useProviderModels' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { useDialogDraft } from './useDialogDraft' + +export function ModelFactorDialog({ node, nodeLabel, factors, initialFieldPath, initialLevelIndex, onClose, onSave, onRemove }: { + node: ProtocolNode + nodeLabel: string + factors: DesignFactor[] + initialFieldPath?: string + initialLevelIndex?: number + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string) => Promise +}) { + const config = { ...node.data.config } as Record + const { models, modelsQuery, capabilities: resolvedCapabilities, capabilitiesPending, capabilitiesError } = useProviderModels(config.provider as string | undefined, config.model as string | undefined, config.resolved_capabilities as ModelCapabilities | undefined) + const modelInfo = models.find((model) => model.id === config.model) + const capabilities = modelCapabilities(resolvedCapabilities ?? modelInfo, config.resolved_capabilities as ModelCapabilities | undefined) + const showTemperature = !capabilitiesPending && capabilities.supports_temperature + const showEffort = !capabilitiesPending && capabilities.supports_effort + const effortLevels = capabilities.effort_levels + const fields = bindableFieldsForNode(node) + const available = (fieldPath: string) => fieldPath === 'config.temperature' ? showTemperature : fieldPath === 'config.effort' ? showEffort : true + const visibleFields = fields.filter((field) => available(field.fieldPath) || node.data.factor_bindings?.[field.fieldPath] || initialFieldPath === field.fieldPath) + .sort((a, b) => Number(b.fieldPath === 'config') - Number(a.fieldPath === 'config') || a.label.localeCompare(b.label)) + const bindings = node.data.factor_bindings ?? {} + const [path, setPath] = useState(initialFieldPath ?? Object.keys(bindings)[0] ?? 'config') + const existingFor = (fieldPath: string) => factors.find((factor) => factor.name === bindings[fieldPath]) + function seed(fieldPath: string): DesignFactor { + const existing = existingFor(fieldPath) + if (existing) return { ...existing, level_labels: existing.level_labels ?? defaultFactorLevelLabels(existing.name, existing.levels.length, existing.level_type) } + const field = fields.find((field) => field.fieldPath === fieldPath)! + const value = fieldPath === 'config' ? config : config[fieldPath.slice(7)] + const levels = field.levelType === 'model_config' ? [value, { ...config, model: '' }] : seedLevels(value) + const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) + return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } + } + const { draft, setDraft, reset: resetDraft } = useDialogDraft(seed(path), { sessionKey: path }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const initialLevelRef = useRef(null) + useEffect(() => { + if (initialLevelIndex == null) return + initialLevelRef.current?.scrollIntoView?.({ block: 'nearest' }) + }, [initialLevelIndex]) + const current = existingFor(path) + const blocked = (fieldPath: string) => fieldPath === 'config' + ? Object.keys(bindings).some((key) => key.startsWith('config.')) + : !!bindings.config + const labels = draft.level_labels ?? [] + const numeric = draft.level_type === 'number' + const structured = draft.level_type === 'model_config' + const levelKeys = draft.levels.map((level) => numeric && !String(level).trim() ? undefined : factorValueKey(numeric ? Number(level) : level)) + const duplicateLevels = draft.levels.flatMap((level, index) => { + const configured = structured ? !!(level as Record).model : !!String(level).trim() + return configured && levelKeys.some((key, other) => other !== index && key === levelKeys[index]) ? [index + 1] : [] + }) + const valid = draft.name.trim() && draft.levels.length >= 2 && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length && draft.levels.every((level) => structured ? !!(level as Record).model : String(level).trim() && (!numeric || Number.isFinite(Number(level)))) && duplicateLevels.length === 0 + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path) + else { + await onSave({ ...draft, name: draft.name.trim(), levels: numeric ? draft.levels.map(Number) : draft.levels, level_labels: labels.map((label) => label.trim()) }, path, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the model factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + initialLevelRef.current?.querySelector('input') ?? true}> + Model factorChoose how models vary across experimental cells. +
+
{visibleFields.map((field) => )}
+ {blocked(path) ?

Model levels cannot be combined with individual configuration factors. Select the existing factor type and remove its binding first.

: <> +

{structured ? 'Compare complete model configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this configuration field while keeping the other model settings.'}

+ {!available(path) &&

The selected model does not support this field. Remove its factor or choose a model that supports it.

} +
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

Level {index + 1}{index === 0 ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {structured ? } onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> : path === 'config.model' ? <> + + i !== index).map(String)} onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> + : path === 'config.effort' ? <> + + + : setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? event.target.value : value) })} />} +
+ + + +
)}
+ {duplicateLevels.length > 0 &&

Duplicate levels: {duplicateLevels.join(', ')}. {structured ? 'Each level must use a different model configuration.' : 'Each level must have a different value.'} Change or remove a duplicate before saving.

} + +

Changes require design review and regeneration.

+ {current && } + } + {capabilitiesPending &&

{capabilitiesError ?? 'Resolving model capabilities…'}

} + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ModelField.tsx b/frontend/src/components/protocol/ModelField.tsx index 715f81b2..d19fcb10 100644 --- a/frontend/src/components/protocol/ModelField.tsx +++ b/frontend/src/components/protocol/ModelField.tsx @@ -33,12 +33,14 @@ export function ModelField({ value, models, isLoading, + disabledModelIds = [], onChange, }: { id?: string value: string models: LLMModelInfo[] isLoading: boolean + disabledModelIds?: string[] onChange: (model: string) => void }) { const known = models.find((m) => m.id === value) @@ -84,7 +86,7 @@ export function ModelField({ value && patchConfig({ + resolved_capabilities: value === 'temperature' + ? { supports_temperature: true, supports_effort: false, effort_levels: [], default_effort: null } + : null, + })} + > + + {(value: string) => value === 'temperature' ? 'Temperature only' : 'Automatic'} + + + Automatic + Temperature only + + +

Choose Temperature only if your custom model does not support effort.

+
+ )} + + {capabilitiesPending &&

{capabilitiesError ?? 'Resolving model capabilities…'}

} + {capabilitiesError && } +
{showTemperature && ( - bindFactor('config.temperature', name)} - onUnbind={() => unbindFactor('config.temperature')} - > - {(trigger) => ( -
- - patchConfig({ temperature: e.target.value === '' ? null : Number(e.target.value) })} - /> -
- )} - +
+ + patchConfig({ temperature: e.target.value === '' ? null : Number(e.target.value) })} + /> +
)} {showEffort && ( - ({ value: level, label: level }))} - boundFactorName={bindings['config.effort']} - onBind={(name) => bindFactor('config.effort', name)} - onUnbind={() => unbindFactor('config.effort')} - > - {(trigger) => ( -
- - -
- )} - +
+ + +
)} - bindFactor('config.max_tokens', name)} - onUnbind={() => unbindFactor('config.max_tokens')} - > - {(trigger) => ( -
- - { - if (e.target.value === '') { - patchConfig({ max_tokens: null }) - return - } - patchConfig({ max_tokens: Math.min(200000, Math.max(1, Math.trunc(Number(e.target.value)))) }) - }} - /> -
- )} - +
+ + { + if (e.target.value === '') { + patchConfig({ max_tokens: null }) + return + } + patchConfig({ max_tokens: Math.min(200000, Math.max(1, Math.trunc(Number(e.target.value)))) }) + }} + /> +
-
{children}
+
{children}
) diff --git a/frontend/src/components/protocol/NodeRunOutputPanel.test.tsx b/frontend/src/components/protocol/NodeRunOutputPanel.test.tsx new file mode 100644 index 00000000..bcfcece9 --- /dev/null +++ b/frontend/src/components/protocol/NodeRunOutputPanel.test.tsx @@ -0,0 +1,12 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { NodeRunOutputPanel } from './NodeRunOutputPanel' + +describe('NodeRunOutputPanel', () => { + it('labels completed pass-through output as bypassed', () => { + render() + expect(screen.getByText('Bypassed')).toBeInTheDocument() + expect(screen.getByText('Upstream answer')).toBeInTheDocument() + expect(screen.queryByText('Done')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/protocol/NodeRunOutputPanel.tsx b/frontend/src/components/protocol/NodeRunOutputPanel.tsx index 251b61ae..eb70ef4f 100644 --- a/frontend/src/components/protocol/NodeRunOutputPanel.tsx +++ b/frontend/src/components/protocol/NodeRunOutputPanel.tsx @@ -282,7 +282,7 @@ export function NodeRunOutputPanel({ // viewport, matching the resize affordance on their prompt textareas. resizableOutput?: boolean }) { - const badge = nodeRunBadge(nodeRun?.status, Boolean(nodeRun?.truncation)) + const badge = nodeRunBadge(nodeRun?.status, Boolean(nodeRun?.truncation), nodeRun?.bypassed) const unresolved = nodeRun?.unresolved_references ?? [] if (!nodeRun) { diff --git a/frontend/src/components/protocol/OkfBundleNodeInspector.tsx b/frontend/src/components/protocol/OkfBundleNodeInspector.tsx index 0be12a05..dcdb1c7a 100644 --- a/frontend/src/components/protocol/OkfBundleNodeInspector.tsx +++ b/frontend/src/components/protocol/OkfBundleNodeInspector.tsx @@ -1,6 +1,6 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { BookMarked, RefreshCw } from 'lucide-react' +import { BookMarked, RefreshCw, Power, Trash2, Split } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -8,7 +8,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { okfApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import { OkfConceptList } from './OkfConceptList' import type { OkfBundleNodeData, ProtocolNode } from '@/types/protocols' @@ -28,17 +27,13 @@ const ACCENT = nodeAccent('okf_bundle') // bundle, not a useful configuration. export function OkfBundleNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: OkfBundleNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, this node's own plain header title. - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: OkfBundleNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -81,10 +76,6 @@ export function OkfBundleNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -111,35 +102,24 @@ export function OkfBundleNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

- Off: the wired agent gets none of this bundle’s tools, so it can neither read nor write it. -

-
- patchConfig({ enabled: checked })} - /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Knowledge node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this knowledge at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} +
diff --git a/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx b/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx index 73ca36a0..e7e17d5c 100644 --- a/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx +++ b/frontend/src/components/protocol/OkfDocumentNodeInspector.tsx @@ -1,6 +1,6 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { FileText, RefreshCw } from 'lucide-react' +import { FileText, RefreshCw, Power, Trash2, Split } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -8,7 +8,6 @@ import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { okfApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { OkfDocumentNodeData, ProtocolNode } from '@/types/protocols' @@ -26,15 +25,13 @@ const ACCENT = nodeAccent('okf_document') // document as it works, and this is where you see what it became. export function OkfDocumentNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: OkfDocumentNodeData }) | null - experimentId: string | null - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: OkfDocumentNodeData) => void onDelete: (nodeId: string) => void onClose: () => void @@ -72,10 +69,6 @@ export function OkfDocumentNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -102,35 +95,24 @@ export function OkfDocumentNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
-
- -

- Off: the wired agent gets none of this document’s tools, so it can neither read nor write it. -

-
- patchConfig({ enabled: checked })} - /> -
- )} - +
+

+ Make factor has moved to the node toolbar. Hover over the Knowledge node and click the

+
+ {connectorFactorName ?

Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

{Object.entries(bindings).map(([path, name]) =>

Individual factor {name} conflicts with this connector.

Its declaration remains in Design until you remove or rebind it.

)}
:
+

Off: the wired agent never sees this knowledge at all.

{bindings['config.enabled'] &&

Factor: {bindings['config.enabled']}

}
+ patchConfig({ enabled: checked })} /> +
} +
diff --git a/frontend/src/components/protocol/PatternFactorDialog.test.tsx b/frontend/src/components/protocol/PatternFactorDialog.test.tsx new file mode 100644 index 00000000..a26c8fca --- /dev/null +++ b/frontend/src/components/protocol/PatternFactorDialog.test.tsx @@ -0,0 +1,93 @@ +import { render, screen, fireEvent, waitFor } from '@testing-library/react' +import { describe, it, expect, vi } from 'vitest' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' +import { defaultAgentNodeData, defaultReasonActPatternNodeData, defaultSingleAgentBaselinePatternNodeData } from '@/types/protocols' +import { PatternFactorDialog } from './PatternFactorDialog' +import { factorCreationFields } from './bindableFields' +import { patternFactorConflict } from './patternFactors' + +const pattern: ProtocolNode = { id: 'pattern', type: 'pattern_reason_act', position: { x: 0, y: 0 }, data: defaultReasonActPatternNodeData() } +const agent: ProtocolNode = { id: 'agent', type: 'agent', position: { x: 0, y: 100 }, data: defaultAgentNodeData('Writer') } +const graph: ProtocolGraph = { nodes: [pattern, agent], edges: [{ id: 'edge', source: 'pattern', target: 'agent', targetHandle: 'architectural_pattern' }] } +const props = { patternNodeId: 'pattern', graph, nodeLabel: 'Writer:Reason + Act', factors: [], onClose: vi.fn(), onSave: vi.fn(), onRemove: vi.fn() } + +describe('Pattern factors', () => { + it('refreshes untouched levels after a refetch and preserves label pairing on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const boundGraph = { ...graph, nodes: [{ ...pattern, data: { ...pattern.data, factor_bindings: { 'config.max_iterations': 'Iterations' } } }, agent] } + const factor = { name: 'Iterations', level_type: 'number' as const, levels: [6, 3], level_labels: ['six', 'three'] } + const refreshed = { ...factor, levels: [3, 6], level_labels: ['three', 'six'] } + const next = { ...props, graph: boundGraph, initialFieldPath: 'config.max_iterations', onSave: save } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 value')).toHaveValue(3) + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'short' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['short', 'six'] }, 'config.max_iterations', 'pattern', 'Iterations')) + }) + + it('groups whole-pattern and parameter choices under the Pattern node in New Factor', () => { + const fields = factorCreationFields(graph.nodes, graph.edges) + const whole = fields.find((field) => field.fieldPath === 'pattern_override')! + expect(whole).toMatchObject({ nodeId: 'agent', fieldLabel: 'Pattern levels', connectorFactor: { kind: 'pattern', nodeId: 'pattern', agentId: 'agent' }, pickerGroup: { id: 'agent', category: 'Pattern', componentId: 'pattern', componentLabel: 'Reason + Act' } }) + expect(fields.filter((field) => field.nodeId === 'pattern')).toHaveLength(4) + expect(fields.filter((field) => field.nodeId === 'pattern').every((field) => field.connectorFactor?.kind === 'pattern')).toBe(true) + }) + + it.each(['pattern_reason_act', 'pattern_single_agent_baseline'] as const)('seeds %s and its alternate and saves on the Agent', async (type) => { + const save = vi.fn().mockResolvedValue(undefined) + const config = type === 'pattern_reason_act' ? defaultReasonActPatternNodeData() : defaultSingleAgentBaselinePatternNodeData() + const nextGraph = { ...graph, nodes: [{ ...pattern, type, data: config }, agent] } + render() + expect(screen.getByRole('dialog', { name: 'Pattern factor' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ level_type: 'pattern', levels: [expect.objectContaining({ pattern_params: expect.objectContaining({ [type.slice(8)]: config.config }) }), expect.any(Object)] }), 'pattern_override', 'agent', undefined)) + }) + + it('blocks whole-pattern factors while allowing existing parameter factors to be edited and removed', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const nextGraph = { ...graph, nodes: [{ ...pattern, data: { ...pattern.data, factor_bindings: { 'config.max_iterations': 'Budget' } } }, agent] } + render() + fireEvent.click(screen.getByRole('button', { name: 'Pattern levels' })) + expect(screen.getByRole('status')).toHaveTextContent('cannot be combined') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'Max iterations' })) + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Budget', 'config.max_iterations', 'pattern')) + expect(factorCreationFields(nextGraph.nodes, nextGraph.edges).some((field) => field.fieldPath === 'pattern_override')).toBe(false) + }) + + it('blocks parameters controlled by an Agent override, including shared Pattern connections', () => { + const otherAgent = { ...agent, id: 'other', data: { ...defaultAgentNodeData('Other'), factor_bindings: { pattern_override: 'Patterns' } } } + const nextGraph = { ...graph, nodes: [...graph.nodes, otherAgent], edges: [...graph.edges, { id: 'other-edge', source: 'pattern', target: 'other', targetHandle: 'architectural_pattern' }] } + expect(patternFactorConflict('pattern', 'config.max_iterations', nextGraph.nodes, nextGraph.edges)).toContain('cannot be combined') + render() + fireEvent.click(screen.getByRole('button', { name: 'Max iterations' })) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + }) + + it('requires an Agent for Pattern levels while permitting orphan parameter factors', () => { + render() + expect(screen.getByRole('status')).toHaveTextContent('Connect this node to an Agent') + fireEvent.click(screen.getByRole('button', { name: 'Include scratchpad' })) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeEnabled() + }) + + it('preserves numeric labels when reordered and rejects duplicate and invalid budgets', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '30' } }) + expect(screen.getByRole('alert')).toHaveTextContent('Duplicate levels') + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '0' } }) + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.change(screen.getByLabelText('Level 2 value'), { target: { value: '10' } }) + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'Long' } }) + fireEvent.change(screen.getByLabelText('Level 2 label'), { target: { value: 'Short' } }) + fireEvent.click(screen.getAllByRole('button', { name: 'Move level up' })[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ levels: [10, 30], level_labels: ['Short', 'Long'] }), 'config.max_iterations', 'pattern', undefined)) + }) +}) diff --git a/frontend/src/components/protocol/PatternFactorDialog.tsx b/frontend/src/components/protocol/PatternFactorDialog.tsx new file mode 100644 index 00000000..647fd812 --- /dev/null +++ b/frontend/src/components/protocol/PatternFactorDialog.tsx @@ -0,0 +1,108 @@ +import { factorRecipients, factorScopeLabel } from '@/lib/sharedFactors' +import { useState } from 'react' +import { ArrowDown, ArrowUp, Plus, Split, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { factorValueKey } from '@/lib/experiment' +import { PatternLevelRow } from './FactorEditorDialog' +import { computeFactorName, defaultFactorLevelLabels, seedLevels } from './factorLevels' +import { PATTERN_FACTOR_PATH, patternFactorFields, patternFactorConflict } from './patternFactors' +import { useDialogDraft } from './useDialogDraft' + +export function PatternFactorDialog({ patternNodeId, graph, nodeLabel, factors, initialFieldPath, onClose, onSave, onRemove }: { + patternNodeId: string + graph: ProtocolGraph + nodeLabel: string + factors: DesignFactor[] + initialFieldPath?: string + onClose: () => void + onSave: (factor: DesignFactor, fieldPath: string, ownerId: string, previousName?: string) => Promise + onRemove: (name: string, fieldPath: string, ownerId: string) => Promise +}) { + const agents = factorRecipients(graph, patternNodeId, 'pattern') + const agentId = agents.find((node) => node.data.factor_bindings?.[PATTERN_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const fields = patternFactorFields(patternNodeId, graph.nodes, graph.edges, agentId) + .sort((a, b) => Number(b.fieldPath === PATTERN_FACTOR_PATH) - Number(a.fieldPath === PATTERN_FACTOR_PATH) || a.label.localeCompare(b.label)) + const fieldFor = (path: string) => fields.find((field) => field.fieldPath === path)! + const existingFor = (path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === fieldFor(path).nodeId)?.data.factor_bindings?.[path]) + const [path, setPath] = useState(initialFieldPath ?? fields.find((field) => existingFor(field.fieldPath))?.fieldPath ?? PATTERN_FACTOR_PATH) + function seed(path: string, ownerId = agentId): DesignFactor { + const field = patternFactorFields(patternNodeId, graph.nodes, graph.edges, ownerId).find((field) => field.fieldPath === path)! + const existing = factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === field.nodeId)?.data.factor_bindings?.[path]) + if (existing) return { ...existing, level_labels: existing.level_labels ?? defaultFactorLevelLabels(existing.name, existing.levels.length, existing.level_type) } + const levels = field.levelType === 'pattern' ? [field.currentValue, field.alternateValue] : field.levelType === 'boolean' ? [false, true] : seedLevels(field.currentValue) + const name = computeFactorName(nodeLabel, field.label, factors.map((factor) => factor.name)) + return { name, level_type: field.levelType, levels, level_labels: defaultFactorLevelLabels(name, levels.length, field.levelType) } + } + const { draft, setDraft, reset: resetDraft } = useDialogDraft(seed(path), { sessionKey: path }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const current = existingFor(path) + const blocked = (path: string) => patternFactorConflict(patternNodeId, path, graph.nodes, graph.edges, agentId) + const labels = draft.level_labels ?? [] + const numeric = draft.level_type === 'number' + const structured = draft.level_type === 'pattern' + const boolean = draft.level_type === 'boolean' + const keys = draft.levels.map((level) => numeric && !String(level).trim() ? undefined : factorValueKey(numeric ? Number(level) : level)) + const duplicates = keys.flatMap((key, index) => key !== undefined && keys.some((other, i) => i !== index && key === other) ? [index + 1] : []) + const validLevel = (level: unknown) => { + if (boolean) return typeof level === 'boolean' + if (!structured) return !!String(level).trim() && (!numeric || Number.isInteger(Number(level)) && Number(level) >= 1) && (path !== 'config.observation_format' || ['raw', 'summarized'].includes(String(level))) + const value = level as Record + const slug = String(value.execution_pattern) + const params = (value.pattern_params as Record>)?.[slug] + return ['reason_act', 'single_agent_baseline'].includes(slug) && params && Number.isInteger(params.max_iterations) && Number(params.max_iterations) >= 1 && (slug !== 'reason_act' || !params.include_scratchpad || Number.isInteger(params.scratchpad_window) && Number(params.scratchpad_window) >= 1) + } + const valid = draft.name.trim() && draft.levels.length >= 2 && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length && draft.levels.every(validLevel) && !duplicates.length + function move(index: number, offset: number) { + const levels = [...draft.levels], nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && (!valid || blocked(path))) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name, path, fieldFor(path).nodeId) + else { + await onSave({ ...draft, name: draft.name.trim(), levels: numeric ? draft.levels.map(Number) : draft.levels, level_labels: labels.map((label) => label.trim()) }, path, fieldFor(path).nodeId, current?.name) + onClose() + } + } catch (error) { setError(error instanceof Error ? error.message : 'Could not save the pattern factor.') } + finally { setSaving(false) } + } + return !open && !saving && onClose()}> + + Pattern factorChoose how patterns vary across experimental cells. +
+ {agents.length > 0 &&

Applies to: {factorScopeLabel(graph, patternNodeId, 'pattern')}

} +
{fields.map((field) => )}
+ {blocked(path) ?

{blocked(path)}

: <> +

{structured ? 'Compare complete pattern configurations. Use individual parameter factors to measure their separate effects.' : 'Vary this parameter while keeping the other pattern settings.'}

+
setDraft({ ...draft, name: event.target.value })} />
+
{draft.levels.map((level, index) =>
+
+

Level {index + 1}{index === 0 ? ' · default test selection' : ''}

+ setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> + {structured ? } onChange={(next) => setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} /> : boolean ?

{level ? 'Enabled' : 'Disabled'}

: path === 'config.observation_format' ? : setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? event.target.value : value) })} />} +
+ {!boolean && <>} +
)}
+ {duplicates.length > 0 &&

Duplicate levels: {duplicates.join(', ')}. Change or remove a duplicate before saving.

} + {!boolean && } +

Changes require design review and regeneration.

+ } + {current && } + {error &&

{error}

} +
+ + + +} diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index 163db26c..d6b7eedd 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -1,11 +1,12 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { ReactFlowProvider } from '@xyflow/react' -import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { MemoryRouter } from 'react-router-dom' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { datasetsApi, experimentsApi, protocolsApi } from '@/api/client' +import { datasetsApi, experimentsApi, mcpServersApi, protocolsApi, skillsApi } from '@/api/client' import { protocolGraphQueryKey } from '@/lib/protocolGraph' +import type { Experiment } from '@/types/experiments' import { defaultAgentNodeData, defaultAnthropicModelNodeData, @@ -15,6 +16,7 @@ import { defaultOutputParserNodeData, defaultReasonActPatternNodeData, defaultScriptNodeData, + type AgentNodeData, type ProtocolGraph, type ProtocolRun, type TestRun, @@ -128,7 +130,7 @@ describe('ProtocolCanvas connector adds', () => { unobserve() {} disconnect() {} }) - vi.spyOn(protocolsApi, 'listRuns').mockResolvedValue([]) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([]) vi.spyOn(protocolsApi, 'update').mockImplementation(async (_id, { graph }) => ({ id: 'protocol-1', name: 'Protocol', @@ -143,6 +145,300 @@ describe('ProtocolCanvas connector adds', () => { })) }) + it('shows the bypass marker on the canvas Agent badge', async () => { + const run = protocolRun({ + status: 'completed', + node_runs: { observer: { status: 'completed', bypassed: true, output_text: 'Upstream answer' } }, + }) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([run]) + vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) + renderCanvas({ + nodes: [{ id: 'observer', type: 'agent', position: { x: 100, y: 100 }, data: defaultAgentNodeData('Observer') }], + edges: [], + }) + await waitFor(() => { + const node = document.querySelector('[data-testid="rf__node-observer"]') + expect(node).not.toBeNull() + expect(within(node!).getByText('Bypassed')).toBeInTheDocument() + }) + }) + + it('shows the bypass marker on a disabled Critic Gate badge', async () => { + const run = protocolRun({ + status: 'completed', + node_runs: { gate: { status: 'completed', bypassed: true, approved: null, output_text: 'READY' } }, + }) + vi.spyOn(protocolsApi, 'listRunHeads').mockResolvedValue([run]) + vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) + renderCanvas({ + nodes: [{ id: 'gate', type: 'critic_gate', position: { x: 100, y: 100 }, data: defaultCriticGateNodeData('Marker reviewer') }], + edges: [], + }) + await waitFor(() => { + const node = document.querySelector('[data-testid="rf__node-gate"]') + expect(node).not.toBeNull() + expect(within(node!).getByText('Bypassed')).toBeInTheDocument() + }) + }) + + it.each(['sub_agent_selection', 'sub_agent_toggle'] as const)('creates a %s Sub-Agent factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(skillsApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'sub_agent', position: { x: 100, y: 0 }, data: { ...defaultAgentNodeData(`Worker ${id}`), active: false } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'sub_agents' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'sub_agent_toggle' ? 'All sub-agents on/off' : 'Sub-Agent levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Sub-Agents', level_type: mode, levels: mode === 'sub_agent_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'sub_agent_toggle' ? ['All enabled', 'All disabled'] : ['Worker a', 'Worker b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ sub_agent_selection: 'Writer:Sub-Agents' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.sub_agent_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'sub_agent')).toHaveLength(2) + }) + }) + + it.each(['skill_selection', 'skill_toggle'] as const)('creates a %s skill factor from a shared node and binds every connected Agent', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(skillsApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + { id: 'other', type: 'agent', position: { x: 300, y: 200 }, data: defaultAgentNodeData('Other') }, + ...['a', 'b'].map((id) => ({ id, type: 'skill', position: { x: 100, y: 0 }, data: { label: `Skill ${id}`, config: { skill_id: id, skill_name: `Skill ${id}` } } })), + ], + edges: ['a', 'b'].flatMap((id) => ['agent', 'other'].map((target) => ({ id: `${id}-${target}`, source: id, target, targetHandle: 'skill' }))), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'skill_toggle' ? 'All skills on/off' : 'Skill levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Skills', level_type: mode, levels: mode === 'skill_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'skill_toggle' ? ['All enabled', 'All disabled'] : ['Skill a', 'Skill b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ skill_selection: 'Writer:Skills' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.skill_factor_mode).toBe(mode) + expect(persisted?.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({ skill_selection: 'Writer:Skills' }) + expect(persisted?.nodes.find((node) => node.id === 'other')?.data.skill_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'skill')).toHaveLength(2) + }) + }) + + it.each(['tool_selection', 'tool_toggle'] as const)('creates a %s tool factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'mcp_tool', position: { x: 100, y: 0 }, data: { label: `Tool ${id}`, config: { server_id: id, server_name: `Server ${id}`, tool_names: ['run'] } } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'tool_toggle' ? 'All tools on/off' : 'Tool levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Tools', level_type: mode, levels: mode === 'tool_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'tool_toggle' ? ['All enabled', 'All disabled'] : ['Tool a', 'Tool b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ tool_selection: 'Writer:Tools' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.tool_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'mcp_tool')).toHaveLength(2) + }) + }) + + it.each(['script_selection', 'script_toggle'] as const)('creates a %s script factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'script', position: { x: 100, y: 0 }, data: { label: `Script ${id}`, config: { name: `Script ${id}`, language: 'python', code: 'print(1)' } } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'tool' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'script_toggle' ? 'All scripts on/off' : 'Script levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Scripts', level_type: mode, levels: mode === 'script_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'script_toggle' ? ['All enabled', 'All disabled'] : ['Script a', 'Script b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ script_selection: 'Writer:Scripts' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.script_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'script')).toHaveLength(2) + }) + }) + + it('creates and removes a whole-pattern factor from the Pattern toolbar, clearing every shared Agent override', async () => { + let experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + const save = vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + experiment = { ...experiment, ...patch } + return experiment + }) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + { id: 'other', type: 'agent', position: { x: 300, y: 200 }, data: defaultAgentNodeData('Other') }, + { id: 'pattern', type: 'pattern_reason_act', position: { x: 100, y: 0 }, data: defaultReasonActPatternNodeData() }, + ], + edges: ['agent', 'other'].map((target) => ({ id: `pattern-${target}`, source: 'pattern', target, targetHandle: 'architectural_pattern' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const toolbar = () => document.querySelector('[data-testid="rf__node-pattern"] button[aria-label="Make experimental factor"]')! + fireEvent.click(toolbar()) + await screen.findByRole('dialog', { name: 'Pattern factor' }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Reason + Act:Pattern levels', level_type: 'pattern' })] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ pattern_override: 'Reason + Act:Pattern levels' }) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({ pattern_override: 'Reason + Act:Pattern levels' }) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toEqual({ execution_pattern: 'reason_act', pattern_params: { reason_act: graph.nodes.find((node) => node.id === 'pattern')!.data.config } }) + expect(persisted.nodes.find((node) => node.id === 'pattern')?.data.factor_bindings).toBeUndefined() + expect(persisted.edges).toHaveLength(2) + }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + fireEvent.click(toolbar()) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'other')?.data.pattern_override).toBeUndefined() + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.pattern_override).toBeUndefined() + expect(experiment.design_spec?.factors).toEqual([]) + }) + }) + + it.each(['dataset_selection', 'dataset_toggle'] as const)('creates a %s dataset factor from the node hover menu and preserves connected nodes', async (mode) => { + const experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(datasetsApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: defaultAgentNodeData('Writer') }, + ...['a', 'b'].map((id) => ({ id, type: 'dataset', position: { x: 100, y: 0 }, data: { label: `Dataset ${id}`, config: { dataset_id: id, dataset_name: `Dataset ${id}` } } })), + ], + edges: ['a', 'b'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'dataset' })), + } + const { client } = renderCanvas(graph, 'experiment-1') + const makeFactor = document.querySelector('[data-testid="rf__node-a"] button[aria-label="Make experimental factor"]') + expect(makeFactor).not.toBeNull() + fireEvent.click(makeFactor!) + await screen.findByRole('dialog') + fireEvent.click(screen.getByRole('button', { name: mode === 'dataset_toggle' ? 'All datasets on/off' : 'Dataset levels' })) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [{ name: 'Writer:Datasets', level_type: mode, levels: mode === 'dataset_toggle' ? [['a', 'b'], []] : [['a'], ['b']], level_labels: mode === 'dataset_toggle' ? ['All enabled', 'All disabled'] : ['Dataset a', 'Dataset b'] }] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({ dataset_selection: 'Writer:Datasets' }) + expect(persisted?.nodes.find((node) => node.id === 'agent')?.data.dataset_factor_mode).toBe(mode) + expect(persisted?.nodes.filter((node) => node.type === 'dataset')).toHaveLength(2) + }) + }) + + it.each(['dataset', 'resource', 'tool'])('removes a Dataset factor on %s without enabling unrelated Tools or Scripts', async (handle) => { + let experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [{ name: 'Datasets', level_type: 'dataset_toggle', levels: [['dataset-id'], []], level_labels: ['All enabled', 'All disabled'] }], metrics: [] }, + measurement_plan: null, dataset_ids: ['dataset-id'], dataset_id: 'dataset-id', + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + experiment = { ...experiment, ...patch } + return experiment + }) + vi.spyOn(datasetsApi, 'list').mockResolvedValue([]) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: [ + { id: 'agent', type: 'agent', position: { x: 100, y: 200 }, data: { ...defaultAgentNodeData('Writer'), dataset_selection: ['dataset-id'], dataset_factor_mode: 'dataset_toggle', factor_bindings: { dataset_selection: 'Datasets' } } }, + { id: 'dataset', type: 'dataset', position: { x: 100, y: 0 }, data: { ...defaultDatasetNodeData(), config: { dataset_id: 'dataset-id', dataset_name: 'Dataset', enabled: false } } }, + { id: 'tool', type: 'mcp_tool', position: { x: 300, y: 0 }, data: { label: 'Tool', config: { server_id: 'server', server_name: 'Server', tool_names: ['run'], enabled: false } } }, + { id: 'script', type: 'script', position: { x: 500, y: 0 }, data: { ...defaultScriptNodeData(), config: { name: 'Script', language: 'python', code: 'print(1)', enabled: false } } }, + ], + edges: [ + { id: 'dataset-edge', source: 'dataset', target: 'agent', targetHandle: handle }, + ...['tool', 'script'].map((id) => ({ id: `${id}-edge`, source: id, target: 'agent', targetHandle: 'tool' })), + ], + } + const { client } = renderCanvas(graph, 'experiment-1') + await waitFor(() => expect(client.getQueryData(['experiments', 'experiment-1'])?.design_spec?.factors?.[0]?.name).toBe('Datasets')) + fireEvent.click(document.querySelector('[data-testid="rf__node-dataset"] button[aria-label="Make experimental factor"]')!) + fireEvent.click(await screen.findByRole('button', { name: 'Remove factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1'))! + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.factor_bindings).toEqual({}) + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.dataset_selection).toBeUndefined() + expect(persisted.nodes.find((node) => node.id === 'agent')?.data.dataset_factor_mode).toBeUndefined() + expect(persisted.nodes.find((node) => node.id === 'dataset')?.data.config).toMatchObject({ enabled: true }) + for (const id of ['tool', 'script']) { + expect(persisted.nodes.find((node) => node.id === id)?.data.config).toEqual(graph.nodes.find((node) => node.id === id)?.data.config) + } + expect(experiment.design_spec?.factors).toEqual([]) + }) + }) + it.each([null, 1])('selects a draft row for Publish & Test Run with published version %s', async publishedRevision => { const dataset = defaultDatasetNodeData() dataset.config.dataset_id = '11111111-1111-4111-8111-111111111111' @@ -376,6 +672,41 @@ describe('ProtocolCanvas connector adds', () => { expect(screen.queryByRole('button', { name: /Create (another )?metric/ })).not.toBeInTheDocument() }) + it.each([false, true])('keeps factor declarations when a bound Skill node is deleted (shared: %s)', async (shared) => { + const factorName = 'Skill enabled' + const experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [{ name: factorName, levels: [true, false], level_type: 'boolean' }], metrics: [] }, + measurement_plan: null, dataset_ids: [], dataset_id: null, locked_at: null, + locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockResolvedValue(experiment) + const save = vi.spyOn(experimentsApi, 'update').mockResolvedValue(experiment) + vi.spyOn(skillsApi, 'list').mockResolvedValue([]) + const graph: ProtocolGraph = { + nodes: (shared ? ['a', 'b'] : ['a']).map((id) => ({ + id, type: 'skill', position: { x: id === 'a' ? 100 : 300, y: 100 }, + data: { + label: `Skill ${id}`, config: { skill_id: id, skill_name: `Skill ${id}`, enabled: true }, + factor_bindings: { 'config.enabled': factorName }, + }, + })), + edges: [], + } + const { client } = renderCanvas(graph, experiment.id) + fireEvent.doubleClick(await screen.findByText('Skill a')) + fireEvent.click(await screen.findByRole('button', { name: 'Delete node' })) + fireEvent.click(await screen.findByRole('button', { name: 'Delete' })) + + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes.some((node) => node.id === 'a')).toBe(false) + }) + expect(save).not.toHaveBeenCalled() + expect(client.getQueryData(['experiments', experiment.id])?.design_spec?.factors).toEqual(experiment.design_spec?.factors) + }) + it('opens a bound factor from the Script inspector', async () => { const user = userEvent.setup() const scriptData = defaultScriptNodeData('Score script') @@ -404,30 +735,89 @@ describe('ProtocolCanvas connector adds', () => { expect(screen.getByText('Factor name')).toBeInTheDocument() }) - it('binds the Agent inspector header action directly to Active', async () => { + it('moves Agent factor creation to the hover toolbar and saves a prompt baseline', async () => { const user = userEvent.setup() - vi.spyOn(experimentsApi, 'get').mockResolvedValue({ + const experiment: Experiment = { id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + const save = vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + Object.assign(experiment, patch) + return experiment }) - renderCanvas({ + const { client } = renderCanvas({ nodes: [{ id: 'agent-1', type: 'agent', position: { x: 100, y: 100 }, data: defaultAgentNodeData('Writer') }], edges: [], }, 'experiment-1') fireEvent.doubleClick(await screen.findByText('Writer')) - await user.click(screen.getAllByRole('button', { name: 'Make experimental factor' })[0]) - - expect(await screen.findByText('Writer:Active')).toBeInTheDocument() - expect(screen.getByText('Levels: false, true')).toBeInTheDocument() + expect(screen.getByText(/has moved to the node toolbar/)).toHaveTextContent('Hover over the Agent node') + expect(screen.getByRole('img', { name: /Node toolbar preview/ })).toBeInTheDocument() + // The inspector has no inline creation buttons. + expect(within(screen.getByRole('dialog')).queryByRole('button', { name: 'Make experimental factor' })).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Close' })) + fireEvent.click(screen.getByTitle('Make experimental factor')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() expect(screen.queryByText('Bind to a field on the canvas')).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Prompt levels' })) + fireEvent.change(screen.getByLabelText('Level 1 prompt'), { target: { value: 'Baseline task' } }) + fireEvent.change(screen.getByLabelText('Level 2 prompt'), { target: { value: 'Alternate task' } }) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith('experiment-1', expect.objectContaining({ design_spec: expect.objectContaining({ factors: [expect.objectContaining({ name: 'Writer:Prompt', levels: ['Baseline task', 'Alternate task'] })] }) }))) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ 'config.prompt': 'Writer:Prompt' }) + expect((persisted?.nodes[0]?.data as AgentNodeData | undefined)?.config.prompt).toBe('Baseline task') + }) + fireEvent.doubleClick(await screen.findByText('Writer')) + await user.click(await screen.findByText('Factor: Writer:Prompt')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() + expect(screen.getByLabelText('Factor name')).toHaveValue('Writer:Prompt') + await user.click(screen.getByRole('button', { name: 'This agent on/off' })) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.active).toBe(true) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ 'config.prompt': 'Writer:Prompt', active: 'Writer:Enabled' }) + }) + }) + + it('keeps an inactive Agent inactive when an on/off factor is created for it', async () => { + const user = userEvent.setup() + const experiment: Experiment = { + id: 'experiment-1', name: 'Experiment', description: null, hypothesis: null, design_type: 'factorial', task_brief: null, + design_spec: { factors: [], metrics: [] }, measurement_plan: null, dataset_ids: [], dataset_id: null, + locked_at: null, locked_protocol_revision_id: null, locked_design_spec: null, locked_measurement_plan: null, + created_at: '', updated_at: '', archived_at: null, + } + vi.spyOn(experimentsApi, 'get').mockImplementation(async () => experiment) + vi.spyOn(experimentsApi, 'update').mockImplementation(async (_id, patch) => { + Object.assign(experiment, patch) + return experiment + }) + const { client } = renderCanvas({ + nodes: [{ id: 'agent-1', type: 'agent', position: { x: 100, y: 100 }, data: { ...defaultAgentNodeData('Writer'), active: false } }], + edges: [], + }, 'experiment-1') + + await screen.findByText('Writer') + fireEvent.click(screen.getByTitle('Make experimental factor')) + expect(await screen.findByRole('heading', { name: 'Agent factor' })).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'This agent on/off' })) + await user.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => { + const persisted = client.getQueryData(protocolGraphQueryKey('protocol-1')) + expect(persisted?.nodes[0].data.factor_bindings).toEqual({ active: 'Writer:Enabled' }) + expect(persisted?.nodes[0].data.active).toBe(false) + }) }) it('keeps the Test Run label when a production replicate is running', async () => { const run = protocolRun() - vi.mocked(protocolsApi.listRuns).mockResolvedValue([run]) + vi.mocked(protocolsApi.listRunHeads).mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) renderCanvas({ nodes: [], edges: [] }) @@ -440,7 +830,7 @@ describe('ProtocolCanvas connector adds', () => { it('stops the active production replicate instead of a completed Test Run', async () => { const user = userEvent.setup() const run = protocolRun() - vi.mocked(protocolsApi.listRuns).mockResolvedValue([run]) + vi.mocked(protocolsApi.listRunHeads).mockResolvedValue([run]) vi.spyOn(protocolsApi, 'getRun').mockResolvedValue(run) vi.spyOn(protocolsApi, 'getLatestTestRun').mockResolvedValue(testRun()) const cancelRun = vi.spyOn(protocolsApi, 'cancelRun').mockResolvedValue({ diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index ff62ffac..f2665d1b 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -1,6 +1,35 @@ +import { graphWithSharedFactor } from '@/lib/sharedFactors' +import { reconcileSubAgentFactor, subAgentFactorConflict, subAgentFactorOwner, SUB_AGENT_FACTOR_PATH } from '@/lib/subAgentFactors' +import { SubAgentFactorDialog } from './SubAgentFactorDialog' +import { SubAgentTestSelectors } from './SubAgentTestSelectors' +import { useSubAgentTestSelection } from './useSubAgentTestSelection' +import { ModelFactorDialog } from './ModelFactorDialog' +import { AgentFactorDialog } from './AgentFactorDialog' +import { PatternFactorDialog } from './PatternFactorDialog' +import { PATTERN_FACTOR_PATH, isPatternNode, patternFactorConflict } from './patternFactors' +import { reconcileKnowledgeFactor, knowledgeFactorConflict, knowledgeFactorOwner, KNOWLEDGE_FACTOR_PATH } from '@/lib/knowledgeFactors' +import { KnowledgeFactorDialog } from './KnowledgeFactorDialog' +import { KnowledgeTestSelectors } from './KnowledgeTestSelectors' +import { useKnowledgeTestSelection } from './useKnowledgeTestSelection' +import { reconcileDatasetFactor, datasetFactorConflict, datasetFactorOwner, DATASET_FACTOR_PATH } from '@/lib/datasetFactors' +import { DatasetFactorDialog } from './DatasetFactorDialog' +import { DatasetTestSelectors } from './DatasetTestSelectors' +import { useDatasetTestSelection } from './useDatasetTestSelection' import { DatasetRowSelector } from './DatasetRowSelector' import { useDatasetRowSelection } from './useDatasetRowSelection' import { rowBindingForNode } from '@/lib/datasetRows' +import { reconcileSkillFactor, skillFactorConflict, skillFactorOwner, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import { reconcileToolFactor, toolFactorConflict, toolFactorOwner, toolFactorPath, isIndividualToolMode, TOOL_FACTOR_PATH, type ToolFactorMode } from '@/lib/toolFactors' +import { reconcileScriptFactor, scriptFactorConflict, scriptFactorOwner, scriptFactorPath, isIndividualScriptMode, SCRIPT_FACTOR_PATH, type ScriptFactorMode } from '@/lib/scriptFactors' +import { ToolFactorDialog } from './ToolFactorDialog' +import { ScriptFactorDialog } from './ScriptFactorDialog' +import { ToolTestSelectors } from './ToolTestSelectors' +import { ScriptTestSelectors } from './ScriptTestSelectors' +import { useToolTestSelection } from './useToolTestSelection' +import { useScriptTestSelection } from './useScriptTestSelection' +import { SkillFactorDialog } from './SkillFactorDialog' +import { SkillTestSelectors } from './SkillTestSelectors' +import { useSkillTestSelection } from './useSkillTestSelection' import { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { @@ -77,7 +106,7 @@ import type { OkfBundle, OkfDocument } from '@/types/okf' import type { Skill } from '@/types/skills' import { AddNodePanel } from './AddNodePanel' import { AgentNodeInspector } from './AgentNodeInspector' -import { agentTracedLabel, factorBoundField, revealsHiddenMcpServers, toolFactorServerId, unboundBindableFields, type UnboundField } from './bindableFields' +import { agentTracedLabel, factorBoundField, factorCreationFields, revealsHiddenMcpServers, toolFactorServerId, type UnboundField } from './bindableFields' import { CanvasControls } from './CanvasControls' import { CriticGateNodeInspector } from './CriticGateNodeInspector' import { ToolStepNodeInspector } from './ToolStepNodeInspector' @@ -356,6 +385,7 @@ export interface ProtocolCanvasHandle { // level from the sibling Design tab must update every field controlled by // the factor, not leave the graph and design as two sources of truth. setFactorBaseline: (factorName: string, baseline: unknown) => void + openConnectorFactor: (field: UnboundField) => void } export const ProtocolCanvas = forwardRef { - setNodes((nds) => - nds.map((n) => { + setNodes((nds) => { + const knowledgeIds = new Set(edges.filter((edge) => edge.targetHandle === 'knowledge' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH] === factorName).map((edge) => edge.source)) + const subAgentIds = new Set(edges.filter((edge) => edge.targetHandle === 'sub_agents' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SUB_AGENT_FACTOR_PATH] === factorName).map((edge) => edge.source)) + const toolIds = new Set(edges.filter((edge) => edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => MCP_TOOL_NODE_TYPES.includes(nds.find((node) => node.id === id)?.type ?? ''))) + const scriptIds = new Set(edges.filter((edge) => edge.targetHandle === 'tool' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => nds.find((node) => node.id === id)?.type === 'script')) + const skillIds = new Set(edges.filter((edge) => edge.targetHandle === 'skill' && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factorName).map((edge) => edge.source)) + const datasetIds = new Set(edges.filter((edge) => ['dataset', 'resource', 'tool'].includes(edge.targetHandle ?? '') && (nds.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factorName).map((edge) => edge.source).filter((id) => nds.find((node) => node.id === id)?.type === 'dataset')) + return nds.map((n) => { + if (subAgentIds.has(n.id)) return { ...n, data: { ...n.data, active: true } } + if (scriptIds.has(n.id) || toolIds.has(n.id) || skillIds.has(n.id) || datasetIds.has(n.id) || knowledgeIds.has(n.id)) return { ...n, data: { ...n.data, config: { ...(n.data.config as object), enabled: true } } } const bindings = n.data.factor_bindings as Record | undefined if (!bindings || !Object.values(bindings).includes(factorName)) return n const next = Object.fromEntries(Object.entries(bindings).filter(([, name]) => name !== factorName)) - return { ...n, data: { ...n.data, factor_bindings: next } } - }), - ) + const data: Record = { ...n.data, factor_bindings: next } + if (bindings[KNOWLEDGE_FACTOR_PATH] === factorName) { + delete data.knowledge_selection + delete data.knowledge_factor_mode + } + if (bindings[SUB_AGENT_FACTOR_PATH] === factorName) { + delete data.sub_agent_selection + delete data.sub_agent_factor_mode + } + if (bindings[TOOL_FACTOR_PATH] === factorName) { + delete data.tool_selection + delete data.tool_factor_mode + } + if (bindings[SCRIPT_FACTOR_PATH] === factorName) { + delete data.script_selection + delete data.script_factor_mode + } + if (bindings[SKILL_FACTOR_PATH] === factorName) { + delete data.skill_selection + delete data.skill_factor_mode + } + if (bindings[DATASET_FACTOR_PATH] === factorName) { + delete data.dataset_selection + delete data.dataset_factor_mode + } + if (bindings[PATTERN_FACTOR_PATH] === factorName) delete data.pattern_override + return { ...n, data } + }) + }) }, - [setNodes], + [edges, setNodes], ) const renameFactorBindings = useCallback( (oldName: string, newName: string) => { @@ -444,8 +508,22 @@ export const ProtocolCanvas = forwardRef { + if (!experimentId || experimentLocked || !field.connectorFactor) return + setSelectedNodeId(null) + setConnectorFactorChoice(field.connectorFactor) + if (field.connectorFactor.kind === 'agent') { setAgentFactorFieldPath(field.fieldPath); setAgentFactorNodeId(field.nodeId) } + else if (field.connectorFactor.kind === 'sub_agent') setSubAgentFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'tool') setToolFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'script') setScriptFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'skill') setSkillFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'dataset') setDatasetFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'model') { setModelFactorLevelIndex(undefined); setModelFactorFieldPath(field.fieldPath); setModelFactorNodeId(field.connectorFactor.nodeId) } + else if (field.connectorFactor.kind === 'pattern') { setPatternFactorFieldPath(field.fieldPath); setPatternFactorNodeId(field.connectorFactor.nodeId) } + else setKnowledgeFactorNodeId(field.connectorFactor.nodeId) + }, }), - [bindFactorOnNode, removeFactorBindings, renameFactorBindings, setFactorBaseline], + [bindFactorOnNode, removeFactorBindings, renameFactorBindings, setFactorBaseline, experimentId, experimentLocked], ) const [selectedNodeId, setSelectedNodeId] = useState(null) useEffect(() => { @@ -461,6 +539,20 @@ export const ProtocolCanvas = forwardRef(null) const [editingFactorName, setEditingFactorName] = useState(null) + const [knowledgeFactorNodeId, setKnowledgeFactorNodeId] = useState(null) + const [modelFactorNodeId, setModelFactorNodeId] = useState(null) + const [agentFactorNodeId, setAgentFactorNodeId] = useState(null) + const [agentFactorFieldPath, setAgentFactorFieldPath] = useState() + const [patternFactorNodeId, setPatternFactorNodeId] = useState(null) + const [patternFactorFieldPath, setPatternFactorFieldPath] = useState() + const [modelFactorFieldPath, setModelFactorFieldPath] = useState() + const [modelFactorLevelIndex, setModelFactorLevelIndex] = useState() + const [subAgentFactorNodeId, setSubAgentFactorNodeId] = useState(null) + const [toolFactorNodeId, setToolFactorNodeId] = useState(null) + const [scriptFactorNodeId, setScriptFactorNodeId] = useState(null) + const [skillFactorNodeId, setSkillFactorNodeId] = useState(null) + const [datasetFactorNodeId, setDatasetFactorNodeId] = useState(null) + const [connectorFactorChoice, setConnectorFactorChoice] = useState() const [addPanelOpen, setAddPanelOpen] = useState(false) // The second level of the add-node panel: AddNodePanel's "MCP Servers" // entry swaps the browser in over it, and its Back button returns. Only @@ -556,6 +648,7 @@ export const ProtocolCanvas = forwardRef experimentsApi.get(experimentId!), enabled: !!experimentId, }) + const factors = experimentQuery.data?.design_spec?.factors ?? EMPTY_FACTORS // Keeps the linked experiment's own dataset list (the experiment_datasets // join table) in step with the Dataset nodes on this canvas -- see the @@ -584,11 +677,17 @@ export const ProtocolCanvas = forwardRef { if (draftRow.error) throw new Error(draftRow.error) + if (draftSubAgents.error) throw new Error(draftSubAgents.error) + if (draftScripts.error) throw new Error(draftScripts.error) + if (draftTools.error) throw new Error(draftTools.error) + if (draftSkills.error) throw new Error(draftSkills.error) + if (draftKnowledge.error) throw new Error(draftKnowledge.error) + if (draftDatasets.error) throw new Error(draftDatasets.error) await protocolsApi.update(protocolId, { graph: toPersistedGraph(nodes, edges) }) const published = await protocolsApi.publish(protocolId) - return { published, scope, rowIndex } + return { published, scope, rowIndex, skillOptions: { ...draftSubAgents.options, ...draftScripts.options, ...draftTools.options, ...draftSkills.options, ...draftKnowledge.options, ...draftDatasets.options } } }, - onSuccess: async ({ published, scope, rowIndex }) => { + onSuccess: async ({ published, scope, rowIndex, skillOptions }) => { queryClient.invalidateQueries({ queryKey: ['protocols', protocolId, 'revisions'] }) if (published.experiment_id) { queryClient.setQueryData(protocolForExperimentQueryKey(published.experiment_id), published) @@ -598,9 +697,9 @@ export const ProtocolCanvas = forwardRef { setRunId(run.id); setRunResultsOpen(true); setTestResultsOpen(false) }) + await protocolsApi.runNode(protocolId, scope.nodeId, binding || Object.keys(skillOptions).length ? { ...(binding ? { row_index: rowIndex } : {}), ...skillOptions } : undefined).then(run => { setRunId(run.id); setRunResultsOpen(true); setTestResultsOpen(false) }) } else { - await protocolsApi.testRun(protocolId, binding ? { row_index: rowIndex } : undefined).then(run => { setRunId(run.id); setTestResultsOpen(true); setRunResultsOpen(false); queryClient.setQueryData(['protocols', protocolId, 'test-run'], run) }) + await protocolsApi.testRun(protocolId, binding || Object.keys(skillOptions).length ? { ...(binding ? { row_index: rowIndex } : {}), ...skillOptions } : undefined).then(run => { setRunId(run.id); setTestResultsOpen(true); setRunResultsOpen(false); queryClient.setQueryData(['protocols', protocolId, 'test-run'], run) }) } setPendingRunConfirm(null) }, @@ -617,6 +716,18 @@ export const ProtocolCanvas = forwardRef { if (publishedSourceError || (publishedRow.binding && publishedRow.error)) throw new Error(publishedSourceError ?? publishedRow.error!) - return protocolsApi.testRun(protocolId, publishedRow.binding ? { row_index: rowIndex } : undefined) + if (publishedSubAgents.error) throw new Error(publishedSubAgents.error) + if (publishedScripts.error) throw new Error(publishedScripts.error) + if (publishedTools.error) throw new Error(publishedTools.error) + if (publishedSkills.error) throw new Error(publishedSkills.error) + if (publishedKnowledge.error) throw new Error(publishedKnowledge.error) + if (publishedDatasets.error) throw new Error(publishedDatasets.error) + return protocolsApi.testRun(protocolId, publishedRow.binding || publishedSubAgents.owners.length || publishedScripts.owners.length || publishedTools.owners.length || publishedSkills.owners.length || publishedKnowledge.owners.length || publishedDatasets.owners.length ? { ...(publishedRow.binding ? { row_index: rowIndex } : {}), ...publishedSubAgents.options, ...publishedScripts.options, ...publishedTools.options, ...publishedSkills.options, ...publishedKnowledge.options, ...publishedDatasets.options } : undefined) }, onSuccess: (run) => { setRunId(run.id) @@ -644,7 +761,13 @@ export const ProtocolCanvas = forwardRef { const binding = publishedGraphQuery.data && rowBindingForNode(publishedGraphQuery.data.graph, nodeId) if (publishedSourceError || (binding && publishedRow.error)) throw new Error(publishedSourceError ?? publishedRow.error!) - return protocolsApi.runNode(protocolId, nodeId, binding ? { row_index: rowIndex } : undefined) + if (publishedSubAgents.error) throw new Error(publishedSubAgents.error) + if (publishedScripts.error) throw new Error(publishedScripts.error) + if (publishedTools.error) throw new Error(publishedTools.error) + if (publishedSkills.error) throw new Error(publishedSkills.error) + if (publishedKnowledge.error) throw new Error(publishedKnowledge.error) + if (publishedDatasets.error) throw new Error(publishedDatasets.error) + return protocolsApi.runNode(protocolId, nodeId, binding || publishedSubAgents.owners.length || publishedScripts.owners.length || publishedTools.owners.length || publishedSkills.owners.length || publishedKnowledge.owners.length || publishedDatasets.owners.length ? { ...(binding ? { row_index: rowIndex } : {}), ...publishedSubAgents.options, ...publishedScripts.options, ...publishedTools.options, ...publishedSkills.options, ...publishedKnowledge.options, ...publishedDatasets.options } : undefined) }, onSuccess: (run) => { setRunId(run.id) @@ -717,7 +840,7 @@ export const ProtocolCanvas = forwardRef protocolsApi.listRuns(protocolId), + queryFn: () => protocolsApi.listRunHeads(protocolId), }) // Seeded once and only into an empty slot: a run launched here must win over @@ -799,7 +922,7 @@ export const ProtocolCanvas = forwardRef { const activeSubAgents = new Set( - nodes.filter((node) => node.type === 'sub_agent' && node.data.active !== false).map((node) => node.id), + nodes.filter((node) => node.type === 'sub_agent' && (subAgentFactorOwner(toPersistedGraph(nodes, edges), node.id) || node.data.active !== false)).map((node) => node.id), ) return new Set( edges.filter((edge) => edge.targetHandle === 'sub_agents' && activeSubAgents.has(edge.source)).map((edge) => edge.target), @@ -809,7 +932,7 @@ export const ProtocolCanvas = forwardRef { const activeSubAgents = new Set( - nodes.filter((node) => node.type === 'sub_agent' && node.data.active !== false).map((node) => node.id), + nodes.filter((node) => node.type === 'sub_agent' && (subAgentFactorOwner(toPersistedGraph(nodes, edges), node.id) || node.data.active !== false)).map((node) => node.id), ) return new Set( edges.filter((edge) => edge.targetHandle === 'sub_agents' && activeSubAgents.has(edge.source)).map((edge) => edge.source), @@ -844,7 +967,7 @@ export const ProtocolCanvas = forwardRef { const nodeTypeById = new Map(nodes.map((n) => [n.id, n.type])) const activeSubAgents = new Set( - nodes.filter((node) => node.type === 'sub_agent' && node.data.active !== false).map((node) => node.id), + nodes.filter((node) => node.type === 'sub_agent' && (subAgentFactorOwner(toPersistedGraph(nodes, edges), node.id) || node.data.active !== false)).map((node) => node.id), ) return new Set( edges @@ -1101,9 +1224,26 @@ export const ProtocolCanvas = forwardRef edge.source === n.id && edge.targetHandle === 'architectural_pattern').flatMap((edge) => { + const name = (nodes.find((node) => node.id === edge.target)?.data.factor_bindings as Record | undefined)?.[PATTERN_FACTOR_PATH] + return name ? [name] : [] + })).size : 0, + knowledgeFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH])?.levels.length, + knowledgeFactorControlled: ['okf_bundle', 'okf_document'].includes(n.type ?? '') && !!knowledgeFactorOwner(toPersistedGraph(nodes, edges), n.id), + subAgentFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SUB_AGENT_FACTOR_PATH])?.levels.length, + subAgentFactorControlled: n.type === 'sub_agent' && !!subAgentFactorOwner(toPersistedGraph(nodes, edges), n.id), + toolFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH])?.levels.length, + scriptFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH])?.levels.length, + toolFactorControlled: MCP_TOOL_NODE_TYPES.includes(n.type ?? '') && !!toolFactorOwner(toPersistedGraph(nodes, edges), n.id), + scriptFactorControlled: n.type === 'script' && !!scriptFactorOwner(toPersistedGraph(nodes, edges), n.id), + skillFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH])?.levels.length, + skillFactorControlled: n.type === 'skill' && !!skillFactorOwner(toPersistedGraph(nodes, edges), n.id), + datasetFactorLevelCount: factors.find((factor) => factor.name === (n.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH])?.levels.length, + datasetFactorControlled: n.type === 'dataset' && !!datasetFactorOwner(toPersistedGraph(nodes, edges), n.id), isSubAgent: n.type === 'sub_agent', runStatus: latestNodeRuns?.[n.id]?.status, runTruncated: Boolean(latestNodeRuns?.[n.id]?.truncation), + runBypassed: Boolean(latestNodeRuns?.[n.id]?.bypassed), missingModel: (n.type === 'agent' || (n.type === 'sub_agent' && connectedActiveSubAgentIds.has(n.id))) && !agentIdsWithModel.has(n.id), @@ -1164,6 +1304,8 @@ export const ProtocolCanvas = forwardRef { if (!experimentId || experimentLocked) return + const owner = nodes.find((node) => node.type === 'agent' && Object.entries(node.data.factor_bindings as Record ?? {}).some(([path, name]) => name === factorName && ['active', 'config.prompt', 'config.system_prompt'].includes(path))) + if (owner) { + setSelectedNodeId(null) + setAgentFactorFieldPath(Object.entries(owner.data.factor_bindings as Record).find(([path, name]) => name === factorName && ['active', 'config.prompt', 'config.system_prompt'].includes(path))![0]) + setAgentFactorNodeId(owner.id) + return + } setEditingFactorName(factorName) }, - [experimentId, experimentLocked], + [experimentId, experimentLocked, nodes], ) + const requestAgentFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setAgentFactorFieldPath(undefined) + setAgentFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestModelFactor = useCallback((nodeId: string, levelIndex?: number) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setModelFactorFieldPath(levelIndex == null ? undefined : 'config') + setModelFactorLevelIndex(levelIndex) + setModelFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestPatternFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setPatternFactorFieldPath(undefined) + setPatternFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const modelFactorForNode = useCallback((nodeId: string) => { + const name = (nodes.find((node) => node.id === nodeId)?.data.factor_bindings as Record | undefined)?.config + return factors.find((factor) => factor.name === name && factor.level_type === 'model_config') + }, [nodes, factors]) + const requestSubAgentFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setSubAgentFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestSkillFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setSkillFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestToolFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setToolFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestScriptFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setScriptFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestKnowledgeFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setKnowledgeFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) + const requestDatasetFactor = useCallback((nodeId: string) => { + if (!experimentId || experimentLocked) return + setSelectedNodeId(null) + setConnectorFactorChoice(undefined) + setDatasetFactorNodeId(nodeId) + }, [experimentId, experimentLocked]) const metricsByNode = useMemo(() => { const definitions = new Map((experimentQuery.data?.measurement_plan?.metrics ?? []).map((metric) => [metric.id, metric])) const bindings = new Map>() @@ -1386,6 +1594,16 @@ export const ProtocolCanvas = forwardRef n.id === factorPickerNodeId) ?? null const factorPickerFields: UnboundField[] = factorPickerNodeId - ? unboundBindableFields(nodes, edges).filter((f) => f.nodeId === factorPickerNodeId) + ? factorCreationFields(nodes, edges).filter((f) => f.nodeId === factorPickerNodeId) : [] const factorPickerExistingNames = factorPickerExperimentQuery.data?.design_spec?.factors?.map((f) => f.name) ?? [] @@ -1433,6 +1661,7 @@ export const ProtocolCanvas = forwardRef { const fresh = await experimentsApi.get(experimentId!) + if (next.name !== oldName && (fresh.design_spec?.factors ?? []).some((factor) => factor.name === next.name)) throw new Error('A factor with this name already exists.') const nextFactors = (fresh.design_spec?.factors ?? []).map((factor) => factor.name === oldName ? next : factor) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: nextFactors } }) return { oldName, next } @@ -1835,12 +2064,59 @@ export const ProtocolCanvas = forwardRef { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'sub_agent_selection' || factor.level_type === 'sub_agent_toggle') && (node.data.factor_bindings as Record | undefined)?.[SUB_AGENT_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.sub_agent_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, sub_agent_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'skill_selection' || factor.level_type === 'skill_toggle') && (node.data.factor_bindings as Record | undefined)?.[SKILL_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.skill_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, skill_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'tool_selection' || factor.level_type === 'tool_toggle') && (node.data.factor_bindings as Record | undefined)?.[TOOL_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.tool_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, tool_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'script_selection' || factor.level_type === 'script_toggle') && (node.data.factor_bindings as Record | undefined)?.[SCRIPT_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.script_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, script_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'knowledge_selection' || factor.level_type === 'knowledge_toggle') && (node.data.factor_bindings as Record | undefined)?.[KNOWLEDGE_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.knowledge_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, knowledge_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) + useEffect(() => { + if (experimentLocked) return + setNodes((current) => current.map((node) => { + const factor = factors.find((factor) => (factor.level_type === 'dataset_selection' || factor.level_type === 'dataset_toggle') && (node.data.factor_bindings as Record | undefined)?.[DATASET_FACTOR_PATH] === factor.name) + if (!factor || JSON.stringify(node.data.dataset_selection) === JSON.stringify(factor.levels[0] ?? [])) return node + return { ...node, data: { ...node.data, dataset_selection: factor.levels[0] ?? [] } } + })) + }, [factors, experimentLocked, setNodes]) useEffect(() => { if (!experimentId || !experimentQuery.data || experimentLocked || experimentQuery.data.locked_at) return const graph = toPersistedGraph(nodes, edges) @@ -1909,34 +2185,8 @@ export const ProtocolCanvas = forwardRef Object.values((node.data as { factor_bindings?: Record }).factor_bindings ?? {})), - ) - const orphaned = removedNames.filter((name) => !stillBound.has(name)) - if (orphaned.length === 0) return - const experiment = await experimentsApi.get(experimentId) - const factors = experiment.design_spec?.factors ?? [] - const nextFactors = factors.filter((factor) => !orphaned.includes(factor.name)) - if (nextFactors.length === factors.length) return - await experimentsApi.update(experimentId, { design_spec: { ...experiment.design_spec, factors: nextFactors } }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) - } - - function removeFactorsForDeletedNodes(deleting: Node[], nextNodes: Node[]) { - const removedNames = [...new Set( - deleting.flatMap((node) => Object.values((node.data as { factor_bindings?: Record }).factor_bindings ?? {})), - )] - void removeUnboundFactors(nextNodes, removedNames) - } - + // Removing a node or field leaves its factor declared and unbound for + // explicit design review. Factor dialogs handle intentional factor removal. function updateNodeData( nodeId: string, data: @@ -1956,15 +2206,8 @@ export const ProtocolCanvas = forwardRef node.id === nodeId) - const oldBindings = (previous?.data as { factor_bindings?: Record } | undefined)?.factor_bindings ?? {} - const nextBindings = (data as { factor_bindings?: Record }).factor_bindings ?? {} - const removedNames = Object.entries(oldBindings) - .filter(([path, name]) => nextBindings[path] !== name) - .map(([, name]) => name) const nextNodes = nodes.map((node) => (node.id === nodeId ? { ...node, data } : node)) setNodes(nextNodes) - void removeUnboundFactors(nextNodes, removedNames) } // Client-side guardrail mirroring the backend's own connector validation @@ -1978,12 +2221,10 @@ export const ProtocolCanvas = forwardRef node.id === nodeId) const nextNodes = nodes.filter((node) => node.id !== nodeId) setNodes(nextNodes) setEdges((eds) => eds.filter((e) => e.source !== nodeId && e.target !== nodeId)) setSelectedNodeId(null) - removeFactorsForDeletedNodes(deleting, nextNodes) } // The node inspector's own Delete button calls deleteNode directly -- @@ -2020,6 +2261,13 @@ export const ProtocolCanvas = forwardRef { if (experimentLocked) return setAddPanelOpen(false) + if (modelFactorForNode(node.id)) { + setModelFactorFieldPath('config') + setModelFactorLevelIndex(undefined) + setModelFactorNodeId(node.id) + setSelectedNodeId(null) + return + } setSelectedNodeId(node.id) }} onPaneClick={() => setSelectedNodeId(null)} @@ -2122,7 +2370,7 @@ export const ProtocolCanvas = forwardRef {testRunIsRunning ? 'Test Run running…' : 'Test Run'} @@ -2220,8 +2468,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2250,8 +2497,6 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2282,30 +2527,28 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} + connectorFactorName={datasetFactorOwner(toPersistedGraph(nodes, edges), selectedNode.id)} /> ) : selectedNode?.type === 'skill' ? ( setSelectedNodeId(null)} /> ) : selectedNode?.type === 'okf_bundle' ? ( setSelectedNodeId(null)} /> ) : selectedNode?.type === 'okf_document' ? ( setSelectedNodeId(null)} @@ -2313,8 +2556,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} @@ -2327,8 +2569,6 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> @@ -2351,6 +2589,7 @@ export const ProtocolCanvas = forwardRef { if (pendingDelete.resolve) { - const deletedNodeIds = new Set(pendingDelete.nodes.map((node) => node.id)) - removeFactorsForDeletedNodes( - pendingDelete.nodes, - nodes.filter((node) => !deletedNodeIds.has(node.id)), - ) pendingDelete.resolve({ nodes: pendingDelete.nodes, edges: pendingDelete.edges }) } else { // requestDeleteNode's path -- no xyflow deletion pending, just @@ -2411,22 +2645,50 @@ export const ProtocolCanvas = forwardRef publishAndRunMutation.mutate({ scope: pendingRunConfirm, rowIndex: draftRow.rowIndex })} confirmLabel={pendingRunConfirm.type === 'graph' ? 'Test Run' : undefined} - confirmDisabled={!!publishedSourceError || (pendingHasRow && !!publishedRow.error)} - publishDisabled={!!draftRow.error} - additionalContent={(pendingHasRow || (publishedRevision !== null && publishedSourceError) || (showDraftInputs && draftRow.binding)) ?
+ confirmDisabled={!!publishedSourceError || !!publishedSubAgents.error || !!publishedScripts.error || !!publishedTools.error || !!publishedSkills.error || !!publishedKnowledge.error || !!publishedDatasets.error || (pendingHasRow && !!publishedRow.error)} + publishDisabled={!!draftRow.error || !!draftSubAgents.error || !!draftScripts.error || !!draftTools.error || !!draftSkills.error || !!draftKnowledge.error || !!draftDatasets.error} + additionalContent={
+ {publishedSubAgents.owners.length > 0 &&

Published Sub-Agents · run published v{publishedRevision}

} + {showDraftInputs && draftSubAgents.owners.length > 0 &&

Draft Sub-Agents · publish and run

} + {publishedSkills.owners.length > 0 &&

Published skills · run published v{publishedRevision}

} + {publishedTools.owners.length > 0 &&

Published tools · run published v{publishedRevision}

} + {publishedScripts.owners.length > 0 &&

Published scripts · run published v{publishedRevision}

} + {showDraftInputs && draftSkills.owners.length > 0 &&

Draft skills · publish and run

} + {showDraftInputs && draftTools.owners.length > 0 &&

Draft tools · publish and run

} + {showDraftInputs && draftScripts.owners.length > 0 &&

Draft scripts · publish and run

} + {publishedKnowledge.owners.length > 0 &&

Published knowledge · run published v{publishedRevision}

} + {showDraftInputs && draftKnowledge.owners.length > 0 &&

Draft knowledge · publish and run

} + {publishedDatasets.owners.length > 0 &&

Published datasets · run published v{publishedRevision}

} + {showDraftInputs && draftDatasets.owners.length > 0 &&

Draft datasets · publish and run

} {publishedRevision !== null && (pendingHasRow ?

Published source · run published v{publishedRevision}

{publishedRow.error &&

{publishedRow.error}

}
: publishedSourceError ?

{publishedSourceError}

: null)} {showDraftInputs && draftRow.binding &&

Draft source · publish and run

{draftRow.error &&

{draftRow.error}

}
} -
: undefined} +
} /> )} {factorPickerNodeId && ( { if (!open) setFactorPickerNodeId(null) }} factor={{ name: '', levels: [], level_type: 'string' }} pickableFields={factorPickerFields} + canvasAgentCount={nodes.filter(node => node.type === 'agent' || node.type === 'sub_agent').length} + onPickConnectorFactor={(field) => { + if (!field.connectorFactor) return + setFactorPickerNodeId(null) + setConnectorFactorChoice(field.connectorFactor) + if (field.connectorFactor.kind === 'agent') { setAgentFactorFieldPath(field.fieldPath); setAgentFactorNodeId(field.nodeId) } + else if (field.connectorFactor.kind === 'sub_agent') setSubAgentFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'tool') setToolFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'script') setScriptFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'skill') setSkillFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'dataset') setDatasetFactorNodeId(field.connectorFactor.nodeId) + else if (field.connectorFactor.kind === 'model') { setModelFactorLevelIndex(undefined); setModelFactorFieldPath(field.fieldPath); setModelFactorNodeId(field.connectorFactor.nodeId) } + else if (field.connectorFactor.kind === 'pattern') { setPatternFactorFieldPath(field.fieldPath); setPatternFactorNodeId(field.connectorFactor.nodeId) } + else setKnowledgeFactorNodeId(field.connectorFactor.nodeId) + }} existingNames={factorPickerExistingNames} emptyPickerMessage={`${(factorPickerNode?.data as { label?: string })?.label || 'This node'} has no fields that can be turned into a factor.`} revealHiddenServers={revealsHiddenMcpServers(nodes)} @@ -2436,8 +2698,346 @@ export const ProtocolCanvas = forwardRef )} + {patternFactorNodeId && nodes.some((node) => node.id === patternFactorNodeId) && node.id === patternFactorNodeId)!, edges, nodes)} + initialFieldPath={patternFactorFieldPath} + factors={factors} + onClose={() => setPatternFactorNodeId(null)} + onSave={async (factor, fieldPath, ownerId, previousName) => { + if (experimentLocked) throw new Error('Unlock the experiment before editing factors.') + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + const conflict = patternFactorConflict(patternFactorNodeId, fieldPath, nodes, edges, fieldPath === PATTERN_FACTOR_PATH ? ownerId : undefined) + if (conflict) throw new Error(conflict) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + if (fieldPath === PATTERN_FACTOR_PATH) setNodes((current) => graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, patternFactorNodeId, 'pattern', factor).nodes as unknown as Node[]) + else bindFactorOnNode(ownerId, fieldPath, factor.name) + setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath, ownerId) => { + if (experimentLocked) throw new Error('Unlock the experiment before editing factors.') + const fresh = await experimentsApi.get(experimentId!) + const shared = fieldPath !== PATTERN_FACTOR_PATH && nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== ownerId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => { + if (fieldPath === PATTERN_FACTOR_PATH ? (node.data.factor_bindings as Record | undefined)?.[PATTERN_FACTOR_PATH] !== name : node.id !== ownerId) return node + const data: Record = { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } + if (fieldPath === PATTERN_FACTOR_PATH) delete data.pattern_override + return { ...node, data } + })) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {agentFactorNodeId && nodes.some((node) => node.id === agentFactorNodeId && node.type === 'agent') && node.id === agentFactorNodeId)! as ProtocolNode & { data: AgentNodeData }} + factors={factors} + initialFieldPath={agentFactorFieldPath} + referenceScope={promptScopeFor(agentFactorNodeId)} + onClose={() => setAgentFactorNodeId(null)} + onSave={async (factor, fieldPath, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + bindFactorOnNode(agentFactorNodeId, fieldPath, factor.name) + if (fieldPath === 'active') { + // Legacy nodes omit active; make the bound field explicit without + // changing its current state to the factor's Disabled level. + setNodes((current) => current.map((node) => (node.data.factor_bindings as Record | undefined)?.active === factor.name ? { ...node, data: { ...node.data, active: node.data.active ?? true } } : node)) + } else setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath) => { + const fresh = await experimentsApi.get(experimentId!) + const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record ?? {}).some(([path, factorName]) => factorName === name && (node.id !== agentFactorNodeId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => node.id === agentFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {modelFactorNodeId && nodes.some((node) => node.id === modelFactorNodeId) && node.id === modelFactorNodeId)!} + nodeLabel={agentTracedLabel(nodes.find((node) => node.id === modelFactorNodeId)!, edges, nodes)} + initialFieldPath={modelFactorFieldPath} + initialLevelIndex={modelFactorLevelIndex} + factors={factors} + onClose={() => setModelFactorNodeId(null)} + onSave={async (factor, fieldPath, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const existing = fresh.design_spec?.factors ?? [] + if (existing.some((candidate) => candidate.name === factor.name && candidate.name !== previousName)) throw new Error('A factor with this name already exists.') + const bindings = nodes.find((node) => node.id === modelFactorNodeId)?.data.factor_bindings as Record | undefined + if (fieldPath === 'config' ? Object.keys(bindings ?? {}).some((key) => key.startsWith('config.')) : !!bindings?.config) throw new Error('Remove the conflicting model factor binding first.') + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...existing.filter((candidate) => candidate.name !== previousName), factor] } }) + if (previousName) renameFactorBindings(previousName, factor.name) + bindFactorOnNode(modelFactorNodeId, fieldPath, factor.name) + // Apply the first level to every binding, including the newly added one. + setNodes((current) => graphWithFactorBaseline({ nodes: current as unknown as ProtocolNode[], edges: [] }, factor.name, factor.levels[0]).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name, fieldPath) => { + const fresh = await experimentsApi.get(experimentId!) + const shared = nodes.some((node) => Object.entries(node.data.factor_bindings as Record | undefined ?? {}).some(([path, factorName]) => factorName === name && (node.id !== modelFactorNodeId || path !== fieldPath))) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + setNodes((current) => current.map((node) => node.id === modelFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record ?? {}).filter(([path]) => path !== fieldPath)) } } : node)) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {subAgentFactorNodeId && setSubAgentFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = subAgentFactorConflict(graph, fresh.design_spec?.factors ?? [], subAgentFactorNodeId, factor.level_type as 'boolean' | 'sub_agent_selection' | 'sub_agent_toggle', ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileSubAgentFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + active: next.levels[0], + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), active: next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, subAgentFactorNodeId, 'sub_agent', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individual = graph.nodes.find((node) => node.id === subAgentFactorNodeId)?.data.factor_bindings?.['active'] === name + const shared = individual && graph.nodes.some((node) => node.id !== subAgentFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === subAgentFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== 'active')) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {skillFactorNodeId && setSkillFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = skillFactorConflict(graph, fresh.design_spec?.factors ?? [], skillFactorNodeId, factor.level_type as 'boolean' | 'skill_selection' | 'skill_toggle', ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileSkillFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, skillFactorNodeId, 'skill', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individual = graph.nodes.find((node) => node.id === skillFactorNodeId)?.data.factor_bindings?.['config.enabled'] === name + const shared = individual && graph.nodes.some((node) => node.id !== skillFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === skillFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== 'config.enabled')) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {toolFactorNodeId && setToolFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = toolFactorConflict(graph, fresh.design_spec?.factors ?? [], toolFactorNodeId, factor.level_type as ToolFactorMode, ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = isIndividualToolMode(factor.level_type as ToolFactorMode) ? factor : reconcileToolFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => isIndividualToolMode(next.level_type as ToolFactorMode) ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : next.level_type === 'tool_names' ? { config: { ...(node.data.config as object), tool_names: next.levels[0] ?? [] } } : { tool_selection: next.levels[0] ?? [], tool_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [toolFactorPath(next.level_type as ToolFactorMode)]: next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, toolFactorNodeId, 'tool', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individualPath = Object.entries(graph.nodes.find((node) => node.id === toolFactorNodeId)?.data.factor_bindings ?? {}).find(([, value]) => value === name)?.[0] + const individual = !!individualPath + const shared = individual && graph.nodes.some((node) => node.id !== toolFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === toolFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== individualPath)) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {scriptFactorNodeId && setScriptFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = scriptFactorConflict(graph, fresh.design_spec?.factors ?? [], scriptFactorNodeId, factor.level_type as ScriptFactorMode, ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = isIndividualScriptMode(factor.level_type as ScriptFactorMode) ? factor : reconcileScriptFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => isIndividualScriptMode(next.level_type as ScriptFactorMode) ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + ...(next.level_type === 'boolean' ? { config: { ...(node.data.config as object), enabled: next.levels[0] } } : { script_selection: next.levels[0] ?? [], script_factor_mode: next.level_type }), + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), [scriptFactorPath(next.level_type as ScriptFactorMode)]: next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, scriptFactorNodeId, 'script', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individualPath = Object.entries(graph.nodes.find((node) => node.id === scriptFactorNodeId)?.data.factor_bindings ?? {}).find(([, value]) => value === name)?.[0] + const individual = !!individualPath + const shared = individual && graph.nodes.some((node) => node.id !== scriptFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === scriptFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== individualPath)) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {knowledgeFactorNodeId && setKnowledgeFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = knowledgeFactorConflict(graph, fresh.design_spec?.factors ?? [], knowledgeFactorNodeId, factor.level_type as 'boolean' | 'knowledge_selection' | 'knowledge_toggle', ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileKnowledgeFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, knowledgeFactorNodeId, 'knowledge', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individual = graph.nodes.find((node) => node.id === knowledgeFactorNodeId)?.data.factor_bindings?.['config.enabled'] === name + const shared = individual && graph.nodes.some((node) => node.id !== knowledgeFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === knowledgeFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== 'config.enabled')) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} + {datasetFactorNodeId && setDatasetFactorNodeId(null)} + onSave={async (factor, ownerId, previousName) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const conflict = datasetFactorConflict(graph, fresh.design_spec?.factors ?? [], datasetFactorNodeId, factor.level_type as 'boolean' | 'dataset_selection' | 'dataset_toggle', ownerId) + if (conflict) throw new Error(conflict) + if ((fresh.design_spec?.factors ?? []).some((existing) => existing.name === factor.name && existing.name !== previousName)) throw new Error('A factor with this name already exists.') + const next = factor.level_type === 'boolean' ? factor : reconcileDatasetFactor(factor, toPersistedGraph(nodes, edges), ownerId) + await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: [...(fresh.design_spec?.factors ?? []).filter((existing) => existing.name !== previousName), next] } }) + if (previousName) { + renameFactorBindings(previousName, next.name) + setFactorBaseline(next.name, next.levels[0]) + } + setNodes((current) => next.level_type === 'boolean' ? current.map((node) => node.id === ownerId ? { ...node, data: { + ...node.data, + config: { ...(node.data.config as object), enabled: next.levels[0] }, + factor_bindings: { ...(node.data.factor_bindings as Record | undefined), 'config.enabled': next.name }, + } } : node) : graphWithSharedFactor({ nodes: current as unknown as ProtocolNode[], edges }, datasetFactorNodeId, 'dataset', next).nodes as unknown as Node[]) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + onRemove={async (name) => { + const fresh = await experimentsApi.get(experimentId!) + const graph = toPersistedGraph(nodes, edges) + const individualPath = Object.entries(graph.nodes.find((node) => node.id === datasetFactorNodeId)?.data.factor_bindings ?? {}).find(([, factorName]) => factorName === name)?.[0] + const individual = !!individualPath + const shared = individual && graph.nodes.some((node) => node.id !== datasetFactorNodeId && Object.values(node.data.factor_bindings ?? {}).includes(name)) + if (!shared) await experimentsApi.update(experimentId!, { design_spec: { ...fresh.design_spec, factors: (fresh.design_spec?.factors ?? []).filter((factor) => factor.name !== name) } }) + if (individual) setNodes((current) => current.map((node) => node.id === datasetFactorNodeId ? { ...node, data: { ...node.data, factor_bindings: Object.fromEntries(Object.entries(node.data.factor_bindings as Record).filter(([path]) => path !== individualPath)) } } : node)) + else removeFactorBindings(name) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }} + />} {editingFactor && ( { if (!open) setEditingFactorName(null) diff --git a/frontend/src/components/protocol/ProtocolCanvasContext.tsx b/frontend/src/components/protocol/ProtocolCanvasContext.tsx index 5bd3e109..feaf8534 100644 --- a/frontend/src/components/protocol/ProtocolCanvasContext.tsx +++ b/frontend/src/components/protocol/ProtocolCanvasContext.tsx @@ -1,4 +1,5 @@ import { createContext, useContext } from 'react' +import type { DesignFactor } from '@/types/experiments' export type ConnectorSlot = | 'model' @@ -62,6 +63,17 @@ interface ProtocolCanvasActions { // node). Selecting one creates it near the requesting node, wires the // edge into the right handle, and opens its Inspector immediately. requestConnectorAdd: (request: ConnectorAddRequest) => void + // Opens the consolidated factor dialog from a Skill node's hover toolbar. + requestDatasetFactor?: (nodeId: string) => void + requestAgentFactor?: (nodeId: string) => void + requestKnowledgeFactor?: (nodeId: string) => void + requestModelFactor?: (nodeId: string, levelIndex?: number) => void + requestPatternFactor?: (nodeId: string) => void + modelFactorForNode?: (nodeId: string) => DesignFactor | undefined + requestSubAgentFactor?: (nodeId: string) => void + requestScriptFactor?: (nodeId: string) => void + requestToolFactor?: (nodeId: string) => void + requestSkillFactor?: (nodeId: string) => void // Same idea for the main pipeline handle -- always creates and wires // another Agent node (see MainEdgeAddRequest's own comment). requestMainEdgeAdd: (request: MainEdgeAddRequest) => void diff --git a/frontend/src/components/protocol/PythonCodeEditor.tsx b/frontend/src/components/protocol/PythonCodeEditor.tsx index c813927a..d9bc030c 100644 --- a/frontend/src/components/protocol/PythonCodeEditor.tsx +++ b/frontend/src/components/protocol/PythonCodeEditor.tsx @@ -62,13 +62,15 @@ export function PythonCodeEditor({ onChange, rows = 16, resizable = false, + height: editorHeight, }: { value: string onChange: (value: string) => void rows?: number resizable?: boolean + height?: string }) { - const height = `${rows * 1.35}em` + const height = editorHeight ?? `${rows * 1.35}em` return (
+

Make factor has moved to the node toolbar. Hover over the Pattern node and click the

- bindFactor('config.max_iterations', name)} - onUnbind={() => unbindFactor('config.max_iterations')} - > - {(trigger) => ( -
- - patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} - /> - {underIterated && ( -

- {truncatedAt != null - ? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — ` - : `This agent's wiring suggests at least ${suggestedIterations} — `} - each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '} - -

- )} -
+
+ + patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} + /> + {underIterated && ( +

+ {truncatedAt != null + ? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — ` + : `This agent's wiring suggests at least ${suggestedIterations} — `} + each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '} + +

)} - - ({ value: f, label: f }))} - boundFactorName={bindings['config.observation_format']} - onBind={(name) => bindFactor('config.observation_format', name)} - onUnbind={() => unbindFactor('config.observation_format')} - > - {(trigger) => ( -
- - -
- )} - +
+
+ + +
- bindFactor('config.include_scratchpad', name)} - onUnbind={() => unbindFactor('config.include_scratchpad')} - > - {(trigger) => ( -
-
- -

Carries a running record of prior reasoning/observations into each iteration.

-
- patchConfig({ include_scratchpad: checked })} - /> -
- )} - +
+
+ +

Carries a running record of prior reasoning/observations into each iteration.

+
+ patchConfig({ include_scratchpad: checked })} + /> +
{config.include_scratchpad && ( - bindFactor('config.scratchpad_window', name)} - onUnbind={() => unbindFactor('config.scratchpad_window')} - > - {(trigger) => ( -
- - patchConfig({ scratchpad_window: e.target.value === '' ? null : Number(e.target.value) })} - /> -
- )} - +
+ + patchConfig({ scratchpad_window: e.target.value === '' ? null : Number(e.target.value) })} + /> +
)} !open && setPendingCloseWarning(false)}> diff --git a/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx b/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx new file mode 100644 index 00000000..58ddf43b --- /dev/null +++ b/frontend/src/components/protocol/RegisterDatasetDialog.test.tsx @@ -0,0 +1,64 @@ +import { useState, type ReactNode } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen } from '@testing-library/react' +import { afterEach, expect, it, vi } from 'vitest' +import { datasetsApi } from '@/api/client' +import type { Dataset } from '@/types/datasets' +import { nodeDataForDataset } from './datasetCatalog' +import { DatasetNodeInspector } from './DatasetNodeInspector' +import { RegisterDatasetDialog } from './RegisterDatasetDialog' + +vi.mock('@/components/ui/dialog', () => { + const Container = ({ children }: { children: ReactNode }) =>
{children}
+ return { + Dialog: ({ open, children }: { open: boolean; children: ReactNode }) => open ?
{children}
: null, + DialogContent: Container, DialogDescription: Container, DialogFooter: Container, + DialogHeader: Container, DialogTitle: Container, + } +}) +vi.mock('./NodeInspectorDialog', () => ({ + NodeInspectorDialog: ({ children }: { children: ReactNode }) =>
{children}
, +})) +vi.mock('./FactorBindableField', () => ({ + FactorBindableField: ({ children }: { children: (trigger: ReactNode) => ReactNode }) => children(null), +})) +vi.mock('./SplitDatasetDialog', () => ({ SplitDatasetDialog: () => null })) + +afterEach(() => vi.restoreAllMocks()) + +it('shows the split action immediately after registration while the list refresh is pending', async () => { + const dataset: Dataset = { + id: 'dc6a940a-2de2-4639-b58d-17bd5ac9c045', name: 'New dataset', + raw_path: '/data/datasets/new/raw.csv', raw_sha256: 'a'.repeat(64), + train_path: null, test_path: null, train_sha256: null, test_sha256: null, + split_method: null, split_group_column: null, split_test_size: null, split_seed: null, + target_column: null, description: null, dictionary_json: null, created_at: null, + } + vi.spyOn(datasetsApi, 'create').mockResolvedValue(dataset) + vi.spyOn(datasetsApi, 'list').mockImplementation(() => new Promise(() => {})) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + client.setQueryData(['datasets'], [{ ...dataset, id: 'existing', name: 'Existing dataset' }]) + + function Handoff() { + const [created, setCreated] = useState(null) + return created ? ( + {}} onDelete={() => {}} onClose={() => {}} + /> + ) : {}} onCreated={setCreated} /> + } + + render() + fireEvent.change(screen.getByLabelText('Name'), { target: { value: dataset.name } }) + fireEvent.change(document.getElementById('dataset-file')!, { + target: { files: [new File(['value\n1\n'], 'source.csv', { type: 'text/csv' })] }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Register dataset' })) + + expect(await screen.findByRole('button', { name: 'Split dataset' })).toBeTruthy() + expect(screen.getByText(/has moved to the node toolbar/)).toHaveTextContent('Hover over the Dataset node') + expect(client.getQueryData(['datasets'])?.map((d) => d.id)).toEqual(['existing', dataset.id]) + client.clear() +}) diff --git a/frontend/src/components/protocol/RegisterDatasetDialog.tsx b/frontend/src/components/protocol/RegisterDatasetDialog.tsx index 7f267c9a..81b863d3 100644 --- a/frontend/src/components/protocol/RegisterDatasetDialog.tsx +++ b/frontend/src/components/protocol/RegisterDatasetDialog.tsx @@ -83,6 +83,12 @@ export function RegisterDatasetDialog({ }) }, onSuccess: (dataset) => { + // The inspector opens immediately and reads this list. Seed it before + // the handoff so its actions don't depend on the background refresh. + queryClient.setQueryData(['datasets'], (datasets = []) => [ + ...datasets.filter((existing) => existing.id !== dataset.id), + dataset, + ]) queryClient.invalidateQueries({ queryKey: ['datasets'] }) onCreated?.(dataset) reset() diff --git a/frontend/src/components/protocol/ResultJobFlows.test.tsx b/frontend/src/components/protocol/ResultJobFlows.test.tsx new file mode 100644 index 00000000..f5be4b75 --- /dev/null +++ b/frontend/src/components/protocol/ResultJobFlows.test.tsx @@ -0,0 +1,95 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { experimentsApi, protocolsApi } from '@/api/client' +import type { Protocol } from '@/types/protocols' +import type { ExecutionSlot, ResultJob } from '@/types/executionResults' +import { RunAllCellsButton } from './RunsTab' +import { ResultsPanel } from './ResultsPanel' + +vi.mock('./RunConfirmDialog', () => ({ RunConfirmDialog: ({ additionalContent, onConfirm, onPublishAndRun, confirmDisabled, publishDisabled }: { + additionalContent: React.ReactNode; onConfirm: () => void; onPublishAndRun: () => void; confirmDisabled: boolean; publishDisabled: boolean +}) =>
{additionalContent}
})) +vi.mock('./ExperimentDefinitionDownload', () => ({ ExperimentDefinitionDownload: () => null })) +const row: ExecutionSlot = { slot_id: 'stable-slot', cell_id: 'cell', cell_label: 'model=a', factor_values: {}, + replicate_id: 'parent', replicate_label: 'ambiguous-label', replicate_number: 1, row_result_id: null, row_index: null, + dataset_id: null, raw_sha256: null, run_id: 'attempt', status: 'completed', updated_at: null, error: null, result: {}, usage: null } +vi.mock('./ExecutionBrowser', () => ({ ExecutionBrowser: () => null, ExecutionRows: ({ actions }: { actions: (row: ExecutionSlot) => React.ReactNode }) =>
{actions(row)}
})) +const protocol = { id: 'protocol', published_revision_id: 'publication', published_revision: 1, graph: { nodes: [], edges: [] } } as unknown as Protocol +const pending: ResultJob = { id: 'job', kind: 'plan', status: 'pending', progress: {}, error: null, protocol_revision_id: 'publication' } + +function mount(children: React.ReactNode) { + return render({children}) +} +beforeEach(() => { + vi.spyOn(experimentsApi, 'createResultJob').mockResolvedValue(pending) + vi.spyOn(experimentsApi, 'getResultJob').mockResolvedValue({ ...pending, status: 'awaiting_confirmation', progress: { planned: 15786 } }) + vi.spyOn(experimentsApi, 'confirmResultJob').mockResolvedValue({ ...pending, status: 'queued' }) + vi.spyOn(experimentsApi, 'cancelResultJob').mockResolvedValue({ ...pending, status: 'cancelled' }) + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue({ ready: false, metric_types: {}, metric_directions: {} } as never) + vi.spyOn(experimentsApi, 'get').mockResolvedValue({ locked_at: 'already locked' } as never) + vi.spyOn(protocolsApi, 'runCells') + vi.spyOn(experimentsApi, 'downloadRunResultsCsv') +}) +afterEach(() => vi.restoreAllMocks()) + +describe('asynchronous result commands', () => { + it('prepares without executing, then confirms the sealed job independently of summary preparation', async () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + await screen.findByText('15786 executions will be queued.') + expect(protocolsApi.runCells).not.toHaveBeenCalled() + expect(experimentsApi.confirmResultJob).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Confirm batch' })) + await waitFor(() => expect(experimentsApi.confirmResultJob).toHaveBeenCalledWith('experiment', 'job')) + expect(experimentsApi.createResultJob).toHaveBeenCalledWith('experiment', + { protocol_id: 'protocol', protocol_revision_id: 'publication' }, + expect.objectContaining({ kind: 'plan' }), expect.any(AbortSignal)) + }) + + it('selects completed work by slot ID and abandons the previous preview safely', async () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + await screen.findByText('15786 executions will be queued.') + fireEvent.click(screen.getByRole('checkbox', { name: 'Run replicate 1 again' })) + await waitFor(() => expect(experimentsApi.createResultJob).toHaveBeenLastCalledWith('experiment', expect.anything(), + expect.objectContaining({ rerun_slot_ids: ['stable-slot'] }), expect.any(AbortSignal))) + expect(experimentsApi.cancelResultJob).toHaveBeenCalledWith('experiment', 'job', true) + expect(experimentsApi.confirmResultJob).not.toHaveBeenCalled() + }) + + it('offers iteration-limited work to run again instead of re-running it implicitly', async () => { + const status = row.status + row.status = 'limit_reached' + try { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + await screen.findByText('15786 executions will be queued.') + expect(screen.getByText(/including ones that hit their iteration limit, are skipped unless selected/)).toBeInTheDocument() + fireEvent.click(screen.getByRole('checkbox', { name: 'Run replicate 1 again' })) + await waitFor(() => expect(experimentsApi.createResultJob).toHaveBeenLastCalledWith('experiment', expect.anything(), + expect.objectContaining({ rerun_slot_ids: ['stable-slot'] }), expect.any(AbortSignal))) + } finally { + row.status = status + } + }) + + it('allows first publication while the batch preview is unavailable', () => { + mount() + fireEvent.click(screen.getByRole('button', { name: 'Run all cells' })) + expect(screen.getByRole('button', { name: 'Publish preview' })).toBeEnabled() + expect(screen.getByRole('button', { name: 'Confirm batch' })).toBeDisabled() + expect(experimentsApi.createResultJob).not.toHaveBeenCalled() + }) + + it('downloads a completed background export natively without constructing a browser CSV blob', async () => { + vi.mocked(experimentsApi.getResultJob).mockResolvedValue({ ...pending, kind: 'export', status: 'completed', + download_url: '/api/owned-export?token=signed', schema_url: '/api/owned-schema?token=signed' }) + const click = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {}) + mount(Summary

} cells={

Cells

} />) + fireEvent.click(screen.getByRole('button', { name: 'Download CSV' })) + await waitFor(() => expect(click).toHaveBeenCalledOnce()) + expect(experimentsApi.downloadRunResultsCsv).not.toHaveBeenCalled() + expect(screen.getByRole('link', { name: 'Download prepared CSV' })).toHaveAttribute('href', '/api/owned-export?token=signed') + }) +}) diff --git a/frontend/src/components/protocol/ResultJobProgress.tsx b/frontend/src/components/protocol/ResultJobProgress.tsx new file mode 100644 index 00000000..5792615f --- /dev/null +++ b/frontend/src/components/protocol/ResultJobProgress.tsx @@ -0,0 +1,23 @@ +import { useMutation } from '@tanstack/react-query' +import { useQueryClient } from '@tanstack/react-query' +import { useEffect } from 'react' +import { experimentsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { jobActive, useResultJob } from '@/lib/resultJobs' +import { executionKeys } from '@/lib/executionQueries' + +export function ResultJobProgress({ experimentId, jobId }: { experimentId: string; jobId?: string }) { + const job = useResultJob(experimentId, jobId) + const client = useQueryClient() + const checkpoint = job.data ? JSON.stringify([job.data.status, job.data.progress.created, job.data.progress.cancelled]) : undefined + useEffect(() => { + if (checkpoint) void client.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) + }, [checkpoint, client, experimentId]) + const cancel = useMutation({ mutationFn: () => experimentsApi.cancelResultJob(experimentId, jobId!), onSuccess: () => { void job.refetch() } }) + if (!jobId) return null + return
+

{job.data?.kind === 'cancel' ? 'Stopping executions' : 'Batch'} · {job.data?.status.replaceAll('_', ' ') ?? 'loading'} · {job.data?.progress.created ?? 0} queued · {job.data?.progress.skipped ?? 0} skipped{job.data?.progress.cancelled != null && ` · ${job.data.progress.cancelled} stopped`}

+ {job.data?.kind === 'plan' && jobActive(job.data) && } + {(job.data?.error || job.error || cancel.error) &&

{job.data?.error ?? job.error?.message ?? cancel.error?.message}

} +
+} diff --git a/frontend/src/components/protocol/ResultTimelineNode.test.tsx b/frontend/src/components/protocol/ResultTimelineNode.test.tsx new file mode 100644 index 00000000..0a87d69f --- /dev/null +++ b/frontend/src/components/protocol/ResultTimelineNode.test.tsx @@ -0,0 +1,15 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { ResultTimelineNode } from './ResultTimelineNode' + +describe('ResultTimelineNode', () => { + it('shows bypassed even when the node had no upstream output', () => { + render() + expect(screen.getByText('Bypassed')).toBeInTheDocument() + expect(screen.queryByText('completed')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/protocol/ResultTimelineNode.tsx b/frontend/src/components/protocol/ResultTimelineNode.tsx index 53fd585e..d792dbdb 100644 --- a/frontend/src/components/protocol/ResultTimelineNode.tsx +++ b/frontend/src/components/protocol/ResultTimelineNode.tsx @@ -1,8 +1,10 @@ import { useState } from 'react' import { ChevronDown, ChevronRight } from 'lucide-react' import { Badge } from '@/components/ui/badge' +import { nodeRunBadge } from '@/lib/protocolRun' import type { ResultNodeRun } from '@/types/experiments' import { ReceivedPromptPanel, RunStepTrace, UnresolvedReferencesNote } from './NodeRunOutputPanel' +import { ExecutionNodeText, ExecutionNodeTrace, type ExecutionContext } from './ExecutionNodeEvidence' function formatCurrency(value: number | null): string { return value === null || !Number.isFinite(value) ? 'Not reported' : new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) @@ -19,9 +21,10 @@ function nodeStatusClass(status: string): string { return 'border-transparent bg-muted text-muted-foreground' } -export function ResultTimelineNode({ node, defaultOpen = false }: { node: ResultNodeRun; defaultOpen?: boolean }) { +export function ResultTimelineNode({ node, defaultOpen = false, context }: { node: ResultNodeRun; defaultOpen?: boolean; context?: ExecutionContext }) { const [open, setOpen] = useState(defaultOpen) const hasDetails = Boolean(node.error || node.output_text || node.agent_run_id) + const bypassBadge = node.status === 'completed' && node.bypassed ? nodeRunBadge('completed', false, true) : null return (
  • @@ -43,22 +46,22 @@ export function ResultTimelineNode({ node, defaultOpen = false }: { node: Result {!hasDetails && No output details recorded} - {node.status} + {bypassBadge?.label ?? node.status} {open && (
    {node.error &&

    Error

    {node.error}

    } {/* Received before produced, so one node reads as the handoff it was: what it was given, then what it made of it. */} - {node.agent_run_id && } + {node.agent_run_id && (context ? : )} - {node.output_text ? ( + {context && node.output_text ? : node.output_text ? (

    Output

    {node.output_text}

    ) : !node.error &&

    No output was recorded for this node.

    } - {node.agent_run_id && } + {node.agent_run_id && (context ? : )}
    )}
    • diff --git a/frontend/src/components/protocol/ResultsPanel.tsx b/frontend/src/components/protocol/ResultsPanel.tsx index 364747bf..d8192313 100644 --- a/frontend/src/components/protocol/ResultsPanel.tsx +++ b/frontend/src/components/protocol/ResultsPanel.tsx @@ -1,9 +1,12 @@ -import { useEffect, useState, type ReactNode } from 'react' +import { useEffect, useRef, useState, type ReactNode } from 'react' +import { useMutation } from '@tanstack/react-query' import { Download, Maximize2, Minimize2, type LucideIcon } from 'lucide-react' import { experimentsApi, type ResultsScope } from '@/api/client' import { Button } from '@/components/ui/button' import { Card } from '@/components/ui/card' import { sanitizeFilename } from '@/lib/utils' +import { jobActive, useResultJob } from '@/lib/resultJobs' +import { newRequestKey } from '@/lib/requestKey' import { InfoTooltip } from './InfoTooltip' export function ResultsScorecard({ label, help, value, note, icon: Icon }: { label: string; help: string; value: string; note?: string; icon: LucideIcon }) { @@ -21,8 +24,21 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su comparison?: ReactNode; cells: ReactNode; details?: ReactNode }) { const [maximized, setMaximized] = useState(false) - const [downloading, setDownloading] = useState(false) - const [error, setError] = useState(null) + const [jobId, setJobId] = useState() + const downloaded = useRef(undefined) + const job = useResultJob(experimentId, jobId) + const createExport = useMutation({ mutationFn: () => experimentsApi.createResultJob(experimentId, scope, + { kind: 'export', request_key: newRequestKey() }), onSuccess: created => setJobId(created.id) }) + const downloading = createExport.isPending || (!!jobId && !job.data && !job.error) || jobActive(job.data) + const error = createExport.error?.message ?? job.error?.message ?? job.data?.error + useEffect(() => { + if (job.data?.status !== 'completed' || !job.data.download_url || downloaded.current === job.data.id) return + downloaded.current = job.data.id + const link = document.createElement('a') + link.href = job.data.download_url + link.download = `${sanitizeFilename(experimentName, 'experiment')}-results.csv` + link.click() + }, [job.data, experimentName]) useEffect(() => { if (!maximized) return const previous = document.body.style.overflow @@ -31,21 +47,6 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su document.addEventListener('keydown', close) return () => { document.body.style.overflow = previous; document.removeEventListener('keydown', close) } }, [maximized]) - async function download() { - setDownloading(true) - setError(null) - try { - const blob = await experimentsApi.downloadRunResultsCsv(experimentId, scope) - const url = URL.createObjectURL(blob) - try { - const link = document.createElement('a') - link.href = url - link.download = `${sanitizeFilename(experimentName, 'experiment')}-results.csv` - link.click() - } finally { URL.revokeObjectURL(url) } - } catch (error) { setError(error instanceof Error ? error.message : 'CSV unavailable') } - finally { setDownloading(false) } - } return
    {controls} @@ -53,12 +54,14 @@ export function ResultsPanel({ experimentId, experimentName, scope, controls, su {summary}
    - +
    {scorecards &&
    {scorecards}
    } {notice} {error &&

    {error}

    } + {downloading &&

    Preparing CSV · {job.data?.progress.bytes?.toLocaleString() ?? 0} bytes

    } + {job.data?.status === 'completed' && }
    {comparison}
    diff --git a/frontend/src/components/protocol/ResultsTab.test.tsx b/frontend/src/components/protocol/ResultsTab.test.tsx index 1680284d..31b1db1f 100644 --- a/frontend/src/components/protocol/ResultsTab.test.tsx +++ b/frontend/src/components/protocol/ResultsTab.test.tsx @@ -1,374 +1,142 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { fireEvent, render, screen, within } from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import { describe, expect, it, vi } from 'vitest' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { experimentsApi } from '@/api/client' -import type { Experiment, ExperimentRunResults } from '@/types/experiments' +import type { AttemptDetail, ExecutionCell, ExecutionSlot, ExecutionSummary } from '@/types/executionResults' +import type { Experiment } from '@/types/experiments' import { ResultsInspectorPanel, ResultsTab } from './ResultsTab' -vi.mock('@/api/client', async (importOriginal) => { +vi.mock('@/api/client', async importOriginal => { const actual = await importOriginal() - return { ...actual, experimentsApi: { ...actual.experimentsApi, getRunResults: vi.fn() } } + return { ...actual, experimentsApi: { ...actual.experimentsApi, + getExecutionSummary: vi.fn(), listExecutionCells: vi.fn(), listExecutions: vi.fn(), + listExecutionAttempts: vi.fn(), getExecutionAttempt: vi.fn(), getRunResults: vi.fn(), + } } }) - -function renderWithQuery(ui: React.ReactNode) { - const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) - return render({ui}) -} - -const overview = { - total_replicates: 1, - completed_replicates: 1, - running_replicates: 0, - queued_replicates: 0, - failed_replicates: 0, - not_started_replicates: 0, - obsolete_replicates: 0, - total_cost_usd: null, - total_input_tokens: null, - total_output_tokens: null, - total_tokens: null, - total_duration_seconds: null, - agent_run_count: 0, - reported_usage_count: 0, - reported_cost_count: 0, +vi.mock('./NodeRunOutputPanel', () => ({ ReceivedPromptPanel: () => null, RunStepTrace: () => null, UnresolvedReferencesNote: () => null })) + +const experiment = { id: 'experiment', name: 'Measurements', design_spec: { metrics: [] } } as unknown as Experiment +const summary: ExecutionSummary = { + scope_key: 'scope', unit: 'replicate', ready: true, protocol_id: 'protocol', protocol_revision_id: 'v1', design_revision_id: 'design', + generation: 1, change_id: 1, refreshing: false, updated_at: '', counts: { completed: 15786 }, total: 15786, + has_active: false, design_spec: null, metric_types: { duration_seconds: 'number' }, metric_aggregations: { duration_seconds: 'mean' }, + metric_directions: { duration_seconds: 'minimize' }, primary_metric: 'duration_seconds', primary_metric_direction: 'minimize', } - -const experiment = { - id: 'experiment-1', - name: 'Measurement UX', - design_spec: { metrics: [] }, -} as unknown as Experiment - -function orderedMetricsFixture() { - const orderedExperiment = { - ...experiment, - design_spec: { - metrics: [ - { id: 'clarity', name: 'Clarity', kind: 'custom', valueType: 'number', direction: 'maximize', primary: false }, - { id: 'safety', name: 'Safety', kind: 'custom', valueType: 'number', direction: 'maximize', primary: false }, - ], - }, - measurement_plan: { - metrics: [ - { id: 'safety', name: 'Safety', value_type: 'number', direction: 'maximize', aggregation: 'mean', primary: false }, - { id: 'clarity', name: 'Clarity', value_type: 'number', direction: 'maximize', aggregation: 'mean', primary: false }, - ], - producers: [], - inputs: [], - }, - } as unknown as Experiment - const results = { - overview, - metric_keys: ['Clarity', 'Safety'], - metric_types: { Clarity: 'number', Safety: 'number' }, - metric_aggregations: { Clarity: 'mean', Safety: 'mean' }, - metric_directions: { Clarity: 'maximize', Safety: 'maximize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 1, - completed_count: 1, current_completed_count: 1, obsolete_count: 0, - metric_means: { Clarity: 0.7, Safety: 0.9 }, metric_counts: { Clarity: 1, Safety: 1 }, - cost_usd: null, total_tokens: null, duration_seconds: null, - }], - replicates: [], - } satisfies ExperimentRunResults - return { orderedExperiment, results } +const cell: ExecutionCell = { cell_id: 'cell-1', cell_label: 'model=a', factor_values: { model: 'a' }, + replicate_count: 15786, total: 15786, counts: { completed: 15786 }, metric_values: { duration_seconds: 65 } } +const slot: ExecutionSlot = { slot_id: 'slot-1', cell_id: 'cell-1', cell_label: 'model=a', factor_values: { model: 'a' }, + replicate_id: 'parent-1', replicate_label: 'replicate-1', replicate_number: 1, row_result_id: null, row_index: null, + dataset_id: null, raw_sha256: null, run_id: 'run-1', status: 'completed', updated_at: '', error: null, result: {}, usage: null } +const detail: AttemptDetail = { run_id: 'run-1', status: 'completed', error: null, started_at: '', completed_at: '', + protocol_revision_id: 'v1', dataset_row: null, node_runs: {}, node_labels: {}, + usage: { cost_usd: 0.0123, total_tokens: 200, input_tokens: 100, output_tokens: 100, duration_seconds: 65, agent_run_count: 1 }, + attempt_result: { measurement: { observations: [], artifacts: [] } } } + +function mount(ui: React.ReactNode) { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + return { ...render({ui}), client } } +beforeEach(() => { + vi.mocked(experimentsApi.getExecutionSummary).mockResolvedValue(summary) + vi.mocked(experimentsApi.listExecutionCells).mockResolvedValue({ items: [cell], next_cursor: null }) + vi.mocked(experimentsApi.listExecutions).mockResolvedValue({ items: [slot], next_cursor: null }) + vi.mocked(experimentsApi.listExecutionAttempts).mockResolvedValue({ slot, items: [{ run_id: 'run-1', status: 'completed', + created_at: '', completed_at: '', error: null, current: true, result: {}, usage: null }], next_cursor: null }) + vi.mocked(experimentsApi.getExecutionAttempt).mockResolvedValue(detail) +}) +afterEach(() => { vi.clearAllMocks(); vi.useRealTimers() }) + +describe('bounded Results reads', () => { + it('renders cells independently of summary preparation and fetches records only on expansion', async () => { + vi.mocked(experimentsApi.getExecutionSummary).mockResolvedValue({ ...summary, ready: false }) + mount() + expect(await screen.findByText(/Preparing the version summary/)).toBeInTheDocument() + expect(await screen.findByRole('button', { name: 'View results' })).toBeInTheDocument() + expect(experimentsApi.listExecutions).not.toHaveBeenCalled() + expect(experimentsApi.getRunResults).not.toHaveBeenCalled() + expect(experimentsApi.getExecutionAttempt).not.toHaveBeenCalled() + }) -describe('results measurement states', () => { - it('uses the shared maximize controls for whole-dataset results', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - renderWithQuery() + it('uses server metric ordering and preserves the shared maximize controls', async () => { + mount() fireEvent.click(await screen.findByRole('button', { name: 'Maximize Results' })) expect(document.body.style.overflow).toBe('hidden') - expect(screen.getByRole('button', { name: 'Restore Results' })).toBeInTheDocument() fireEvent.keyDown(document, { key: 'Escape' }) expect(document.body.style.overflow).not.toBe('hidden') expect(screen.getByRole('button', { name: 'Download CSV' })).toBeInTheDocument() + await screen.findByRole('combobox', { name: 'Choose comparison metric' }) + await waitFor(() => expect(experimentsApi.listExecutionCells).toHaveBeenCalledWith('experiment', expect.anything(), + expect.objectContaining({ metric: 'duration_seconds', direction: 'minimize' }), expect.any(AbortSignal))) }) - it('shows explicit units for cost and duration in an individual replicate result', async () => { - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: { cost_usd: 0.0123, duration_seconds: 65 }, status: 'completed' as const, obsolete: false, - error: null, run_id: 'run-1', protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', - duration_seconds: 65, node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: 0.0123, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [], evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - } - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['cost_usd', 'duration_seconds'], - metric_types: { cost_usd: 'number', duration_seconds: 'number' }, - metric_aggregations: { cost_usd: 'sum', duration_seconds: 'sum' }, - metric_directions: { cost_usd: 'minimize', duration_seconds: 'minimize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [], - replicates: [replicate], - } satisfies ExperimentRunResults) - - renderWithQuery() - - expect(await screen.findByText('$0.01')).toBeInTheDocument() - expect(screen.getByText('1.1 min')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Outcome' })).not.toBeInTheDocument() - const usage = screen.getByRole('heading', { name: 'Usage' }).parentElement! - expect(within(usage).getAllByText(/^(Estimated cost|Duration|Total tokens|Agent calls)$/).map((label) => label.textContent)).toEqual([ - 'Estimated cost', - 'Duration', - 'Total tokens', - 'Agent calls', - ]) - }) - - it('shows a measured observation once in Outcome and keeps its provenance there', async () => { - const producer = { binding_id: 'judge', producer_id: 'asaree.judge', kind: 'reported' as const, version: '1' } - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: { Quality: 0.9 }, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', - protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: null, - node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [{ metric_id: 'quality', metric_name: 'Quality', value_type: 'number' as const, status: 'measured' as const, value: 0.9, error: null, attempt_id: 'run-1', producer, input_provenance: {} }], - evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - } - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['Quality'], metric_types: { Quality: 'number' }, metric_aggregations: { Quality: 'mean' }, - metric_directions: { Quality: 'maximize' }, primary_metric: null, primary_metric_direction: null, - cells: [], replicates: [replicate], - } satisfies ExperimentRunResults) - - renderWithQuery() - - expect(await screen.findByRole('heading', { name: 'Outcome' })).toBeInTheDocument() - expect(screen.getAllByText('0.9')).toHaveLength(1) - expect(screen.getByText('asaree.judge · v1')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Measurement observations' })).not.toBeInTheDocument() + it('paginates execution records on the server', async () => { + vi.mocked(experimentsApi.listExecutions).mockImplementation(async (_id, _scope, params) => + params?.cursor ? { items: [{ ...slot, slot_id: 'slot-2', replicate_number: 2 }], next_cursor: null } + : { items: [slot], next_cursor: 'page-2' }) + mount() + fireEvent.click(await screen.findByRole('button', { name: /model: a/ })) + await screen.findByText('Replicate 1') + const next = await screen.findAllByRole('button', { name: 'Next' }) + fireEvent.click(next.find(button => !button.hasAttribute('disabled'))!) + expect(await screen.findByText('Replicate 2')).toBeInTheDocument() + expect(experimentsApi.listExecutions).toHaveBeenCalledWith('experiment', expect.anything(), + { cell_id: 'cell-1', cursor: 'page-2' }, expect.any(AbortSignal)) }) - it('presents metrics in the measurement-plan order declared by the designer', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByRole('combobox', { name: 'Choose comparison metric' })).toHaveTextContent('Safety') - expect(screen.getAllByText(/Safety/).length).toBeGreaterThan(0) + it('labels per-row executions by source row and only inspects attempted slots', async () => { + vi.mocked(experimentsApi.getExecutionSummary).mockResolvedValue({ ...summary, unit: 'row', primary_metric: null }) + vi.mocked(experimentsApi.listExecutions).mockResolvedValue({ items: [ + { ...slot, row_result_id: 'slot-1', row_index: 0 }, + { ...slot, slot_id: 'unplanned:parent-1:1', replicate_number: 2, row_index: 1, run_id: null, status: 'not_started' }, + ], next_cursor: null }) + const select = vi.fn() + mount() + fireEvent.click(await screen.findByRole('button', { name: /model: a/ })) + expect(await screen.findByText(/Source row 1/)).toBeInTheDocument() + expect(screen.getByText(/Source row 2/).closest('button')).toBeDisabled() + fireEvent.click(screen.getByText(/Source row 1/).closest('button')!) + expect(select).toHaveBeenCalledWith(expect.objectContaining({ type: 'execution', slotId: 'slot-1' })) }) - it('uses measurement-plan order in the separate cell result details', async () => { - const { orderedExperiment, results } = orderedMetricsFixture() - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - const summary = await screen.findByRole('region', { name: 'Cell result summary' }) - expect(within(summary).getAllByText(/^(Safety|Clarity) · average$/).map((label) => label.textContent)).toEqual([ - 'Safety · average', - 'Clarity · average', - ]) - }) - - it('shows condition-level cost and duration once under Usage', async () => { - vi.mocked(experimentsApi.getRunResults).mockResolvedValue({ - overview, - metric_keys: ['cost_usd', 'duration_seconds'], - metric_types: { cost_usd: 'number', duration_seconds: 'number' }, - metric_aggregations: { cost_usd: 'sum', duration_seconds: 'sum' }, - metric_directions: { cost_usd: 'minimize', duration_seconds: 'minimize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 2, completed_count: 2, - current_completed_count: 2, obsolete_count: 0, metric_means: { cost_usd: 0.0123, duration_seconds: 65 }, - metric_counts: { cost_usd: 2, duration_seconds: 2 }, cost_usd: 0.0123, total_tokens: null, duration_seconds: 65, - }], - replicates: [], - } satisfies ExperimentRunResults) - - renderWithQuery() - + it('fetches the selected attempt independently of the experiment list', async () => { + mount() expect(await screen.findByText('$0.01')).toBeInTheDocument() expect(screen.getByText('1.1 min')).toBeInTheDocument() - expect(screen.queryByRole('heading', { name: 'Outcome metrics' })).not.toBeInTheDocument() + expect(experimentsApi.getExecutionAttempt).toHaveBeenCalledWith('experiment', 'run-1', + { protocol_revision_id: 'v1' }, expect.any(AbortSignal)) + expect(experimentsApi.getRunResults).not.toHaveBeenCalled() + expect(experimentsApi.listExecutions).not.toHaveBeenCalled() }) - it('ranks measured minimize values before missing values without skipping rank one', async () => { - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['loss'], - metric_types: { loss: 'number' }, - metric_aggregations: { loss: 'mean' }, - metric_directions: { loss: 'minimize' }, - primary_metric: 'loss', - primary_metric_direction: 'minimize', - cells: [ - { cell_label: 'missing', factor_values: { model: 'missing' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: {} as Record, metric_counts: {} as Record, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'measured', factor_values: { model: 'measured' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.2 }, metric_counts: { loss: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - const rows = await screen.findAllByRole('row') - expect(rows[1]).toHaveTextContent('model: measured') - expect(rows[1].firstElementChild).toHaveTextContent('1') - expect(rows[2]).toHaveTextContent('No measured observations') + it('preserves measured-null versus unavailable reported values and provenance', async () => { + vi.mocked(experimentsApi.getExecutionAttempt).mockResolvedValue({ ...detail, attempt_result: { measurement: { observations: [ + { metric_id: 'report', metric_name: 'Report', status: 'measured', value: null, error: null, attempt_id: 'run-1', + producer: { binding_id: 'reported', producer_id: 'asaree.agent_output', kind: 'reported', version: '1' }, input_provenance: {} }, + { metric_id: 'absent', metric_name: 'Absent', status: 'unavailable', value: null, error: 'Agent did not complete', attempt_id: 'run-1', + producer: { binding_id: 'reported', producer_id: 'asaree.agent_output', kind: 'reported', version: '1' }, input_provenance: {} }, + ], artifacts: [] } } }) + mount() + expect(await screen.findByText('Report')).toBeInTheDocument() + expect(screen.getByText('null')).toBeInTheDocument() + expect(screen.getByText('Agent did not complete')).toBeInTheDocument() + expect(screen.getAllByText('asaree.agent_output · v1')).toHaveLength(2) }) - it('summarizes unavailable condition observations instead of collapsing them into a blank', async () => { - const producer = { binding_id: 'python', producer_id: 'asaree.python_script', kind: 'reported' as const, version: '1' } - const results = { - overview, - metric_keys: ['ROC-AUC'], - metric_types: { 'ROC-AUC': 'number' }, - metric_aggregations: { 'ROC-AUC': 'mean' }, - metric_directions: { 'ROC-AUC': 'maximize' }, - primary_metric: 'ROC-AUC', - primary_metric_direction: 'maximize', - cells: [{ cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 2, completed_count: 2, current_completed_count: 2, obsolete_count: 0, metric_means: { 'ROC-AUC': 0.8 }, metric_counts: { 'ROC-AUC': 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }], - replicates: [{ - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, metric_values: {}, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: null, node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [{ metric_id: 'auc', metric_name: 'ROC-AUC', value_type: 'number' as const, status: 'unavailable' as const, value: null, error: 'Probability output is missing.', attempt_id: 'run-1', producer, input_provenance: {} }], - evaluation_artifacts: [], obsolete_runs: [], superseded_runs: [], - }], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Unavailable (1)')).toBeInTheDocument() - }) - - it('keeps a neutral scalar comparable without declaring a winner', async () => { - const results = { - overview, - metric_keys: ['variance'], - metric_types: { variance: 'number' }, - metric_aggregations: { variance: 'mean' }, - metric_directions: { variance: 'neutral' }, - primary_metric: null, - primary_metric_direction: null, - cells: [{ - cell_label: 'model_a', factor_values: { model: 'a' }, replicate_count: 1, - completed_count: 1, current_completed_count: 1, obsolete_count: 0, - metric_means: { variance: 0.2 }, metric_counts: { variance: 1 }, - cost_usd: null, total_tokens: null, duration_seconds: null, - }], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/No primary metric is declared/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getAllByText('0.2').length).toBeGreaterThan(0) + it('stops polling settled experiments', async () => { + vi.useFakeTimers() + mount() + await act(async () => { await vi.advanceTimersByTimeAsync(100) }) + const calls = vi.mocked(experimentsApi.getExecutionSummary).mock.calls.length + await act(async () => { await vi.advanceTimersByTimeAsync(15000) }) + expect(experimentsApi.getExecutionSummary).toHaveBeenCalledTimes(calls) }) - it('compares directional metrics without ranking or naming a winner when no primary is declared', async () => { - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['quality'], - metric_types: { quality: 'number' }, - metric_aggregations: { quality: 'mean' }, - metric_directions: { quality: 'maximize' }, - primary_metric: null, - primary_metric_direction: null, - cells: [ - { cell_label: 'b', factor_values: { model: 'b' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { quality: 0.9 }, metric_counts: { quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'a', factor_values: { model: 'a' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { quality: 0.2 }, metric_counts: { quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/No primary metric is declared/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getByRole('heading', { name: /Cell comparison/ })).toBeInTheDocument() - const rows = screen.getAllByRole('row') - expect(rows[1]).toHaveTextContent('model: a') - expect(rows[1].firstElementChild).toHaveTextContent('—') - }) - - it('does not rank an inspected metric that is not the declared primary', async () => { - const user = userEvent.setup() - const results = { - overview: { ...overview, total_replicates: 2, completed_replicates: 2 }, - metric_keys: ['loss', 'quality'], - metric_types: { loss: 'number', quality: 'number' }, - metric_aggregations: { loss: 'mean', quality: 'mean' }, - metric_directions: { loss: 'minimize', quality: 'maximize' }, - primary_metric: 'loss', - primary_metric_direction: 'minimize', - cells: [ - { cell_label: 'a', factor_values: { model: 'a' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.8, quality: 0.9 }, metric_counts: { loss: 1, quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - { cell_label: 'b', factor_values: { model: 'b' }, replicate_count: 1, completed_count: 1, current_completed_count: 1, obsolete_count: 0, metric_means: { loss: 0.2, quality: 0.2 }, metric_counts: { loss: 1, quality: 1 }, cost_usd: null, total_tokens: null, duration_seconds: null }, - ], - replicates: [], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - await screen.findByText('Best current result') - await user.click(screen.getByRole('combobox', { name: 'Choose comparison metric' })) - await user.click(screen.getByRole('option', { name: 'quality' })) - - expect(screen.getByText('Comparison only')).toBeInTheDocument() - expect(screen.getByText(/quality is not the declared primary metric/)).toBeInTheDocument() - expect(screen.queryByText('Best current result')).not.toBeInTheDocument() - expect(screen.getByRole('heading', { name: /Cell comparison/ })).toBeInTheDocument() - }) - - it('renders unavailable, failed, and not-applicable observations plus diagnostic artifacts', async () => { - const producer = { binding_id: 'python', producer_id: 'asaree.python_script', kind: 'reported' as const, version: '1' } - const replicate = { - replicate_label: 'replicate-1', replicate_number: 1, cell_label: 'model_a', factor_values: { model: 'a' }, - metric_values: {}, status: 'completed' as const, obsolete: false, error: null, run_id: 'run-1', - protocol_revision_id: 'revision-1', updated_at: '2026-01-01T00:00:00Z', duration_seconds: 1, - node_runs: [], input_tokens: null, output_tokens: null, total_tokens: null, cost_usd: null, - agent_run_count: 0, reported_usage_count: 0, reported_cost_count: 0, metric_evaluation: null, - metric_observations: [ - { metric_id: 'auc', metric_name: 'ROC-AUC', value_type: 'number' as const, status: 'unavailable' as const, value: null, error: 'Probability output is missing.', attempt_id: 'run-1', producer, input_provenance: {} }, - { metric_id: 'loss', metric_name: 'Log loss', value_type: 'number' as const, status: 'failed' as const, value: null, error: 'Evaluator crashed.', attempt_id: 'run-1', producer, input_provenance: {} }, - { metric_id: 'mcc', metric_name: 'MCC', value_type: 'number' as const, status: 'not_applicable' as const, value: null, error: 'Regression task.', attempt_id: 'run-1', producer, input_provenance: {} }, - ], - evaluation_artifacts: [ - { artifact_key: 'confusion_matrix', kind: 'confusion_matrix', payload: { labels: ['no', 'yes'], matrix: [[8, 1], [2, 9]] }, attempt_id: 'run-1', producer, input_provenance: {} }, - { artifact_key: 'calibration', kind: 'calibration', payload: { probability: [0.2], observed: [0.1] }, attempt_id: 'run-1', producer, input_provenance: {} }, - { artifact_key: 'per_class', kind: 'per_class', payload: [{ label: 'yes', precision: 0.9, recall: 0.82, f1: 0.86, support: 11 }], attempt_id: 'run-1', producer, input_provenance: {} }, - ], - obsolete_runs: [], superseded_runs: [], - } - const results = { - overview, - metric_keys: [], metric_types: {}, metric_aggregations: {}, metric_directions: {}, - primary_metric: null, primary_metric_direction: null, - cells: [], replicates: [replicate], - } satisfies ExperimentRunResults - vi.mocked(experimentsApi.getRunResults).mockResolvedValue(results) - - renderWithQuery() - - expect(await screen.findByText('Unavailable')).toBeInTheDocument() - expect(screen.getByText('Probability output is missing.')).toBeInTheDocument() - expect(screen.getByText('Failed')).toBeInTheDocument() - expect(screen.getByText('Evaluator crashed.')).toBeInTheDocument() - expect(screen.getByText('Not applicable')).toBeInTheDocument() - expect(screen.getByText('Regression task.')).toBeInTheDocument() - expect(screen.getByText('Confusion matrix')).toBeInTheDocument() - expect(screen.getByText('Calibration')).toBeInTheDocument() - expect(screen.getByText('Per-class report')).toBeInTheDocument() + it('aborts abandoned list requests', async () => { + vi.mocked(experimentsApi.listExecutionCells).mockImplementation(() => new Promise(() => {})) + const view = mount() + const signal = vi.mocked(experimentsApi.listExecutionCells).mock.calls[0][3]! + view.unmount() + expect(signal.aborted).toBe(true) }) }) diff --git a/frontend/src/components/protocol/ResultsTab.tsx b/frontend/src/components/protocol/ResultsTab.tsx index ad073497..ba694bfe 100644 --- a/frontend/src/components/protocol/ResultsTab.tsx +++ b/frontend/src/components/protocol/ResultsTab.tsx @@ -1,21 +1,15 @@ -import { useState } from 'react' -import { useQuery } from '@tanstack/react-query' -import { AlertTriangle, ChevronDown, ChevronRight, CircleDollarSign, Clock3, Coins, Cpu, Trophy, X } from 'lucide-react' -import { experimentsApi, type ResultsScope } from '@/api/client' +import { CircleDollarSign, Clock3, Coins, Cpu } from 'lucide-react' +import type { ResultsScope } from '@/api/client' import { Badge } from '@/components/ui/badge' -import { Button } from '@/components/ui/button' import { Card } from '@/components/ui/card' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { Skeleton } from '@/components/ui/skeleton' -import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' -import { displayFactorLevel, formatMetricLabel, formatMetricValue } from '@/lib/experiment' +import { formatMetricLabel, formatMetricValue } from '@/lib/experiment' import { OBSERVATION_LABELS } from '@/lib/measurementPlan' -import type { DesignMetric, EvaluationArtifact, Experiment, HistoricalRun, MetricObservation, ObsoleteRun, ResultCell, ResultReplicate, SupersededRun } from '@/types/experiments' -import { DatasetRowDetail, DatasetRowResults } from './DatasetRowResults' -import { ResultsPanel, ResultsScorecard as Scorecard } from './ResultsPanel' -import { InfoTooltip } from './InfoTooltip' -import { CellCard, ReplicateRow } from './CellPresentation' +import type { EvaluationArtifact, Experiment, MetricObservation, ResultReplicate } from '@/types/experiments' +import { ResultsScorecard as Scorecard } from './ResultsPanel' import { ResultTimelineNode } from './ResultTimelineNode' +import type { ExecutionContext } from './ExecutionNodeEvidence' +import { ExecutionBrowser } from './ExecutionBrowser' +import { ExecutionInspector } from './ExecutionInspector' function formatNumber(value: number | null, maximumFractionDigits = 0): string { if (value === null || !Number.isFinite(value)) return 'Not reported' @@ -34,28 +28,9 @@ function formatDuration(value: number | null): string { return `${(value / 3600).toFixed(1)} hr` } -function factorSummary( - values: Record, - designSpec: Experiment['design_spec'], - cellLabel?: string | null, -): string { - const entries = Object.entries(values) - if (entries.length === 0) return 'No varying factors' - return entries.map(([name, value]) => `${name.split(':').join(' · ')}: ${displayFactorLevel(designSpec, name, value, cellLabel)}`).join(' · ') -} -function numericMetricValue(replicate: ResultReplicate, metricKey: string | null): number | null { - if (!metricKey) return null - const value = replicate.metric_values[metricKey] - return typeof value === 'number' && Number.isFinite(value) ? value : null -} type ResultMetricTypes = Record -type ResultMetricAggregations = Record - -function isBinaryMetric(metricKey: string | null, metricTypes: ResultMetricTypes): boolean { - return metricKey !== null && metricTypes[metricKey] === 'boolean' -} function formatResultMetricValue(metricKey: string, value: unknown, metricTypes: ResultMetricTypes, individual = false): string { if (typeof value !== 'number') return '—' @@ -66,19 +41,8 @@ function formatResultMetricValue(metricKey: string, value: unknown, metricTypes: return formatMetricValue(metricKey, value) } -function binaryMetricNote(metricKey: string, value: number, count: number | undefined, metricTypes: ResultMetricTypes): string | null { - if (metricTypes[metricKey] !== 'boolean' || !count) return null - return `${Math.round(value * count)}/${count} passed` -} -function metricAggregationLabel(metricKey: string, metricTypes: ResultMetricTypes, metricAggregations: ResultMetricAggregations): string { - if (metricTypes[metricKey] === 'boolean') return 'pass rate' - return metricAggregations[metricKey] === 'sum' ? 'total' : 'average' -} -function metricDisplayLabel(metricKey: string, metricTypes: ResultMetricTypes, metricAggregations: ResultMetricAggregations): string { - return `${formatMetricLabel(metricKey)} · ${metricAggregationLabel(metricKey, metricTypes, metricAggregations)}` -} function statusLabel(status: ResultReplicate['status'], obsolete: boolean): string { if (obsolete) return 'Obsolete' @@ -122,23 +86,6 @@ function observationForMetric(replicate: ResultReplicate, metricId: string | und ) } -function missingObservationSummary( - replicates: ResultReplicate[], - metricId: string | undefined, - metricKey: string | null, -): string | null { - const counts = new Map() - for (const replicate of replicates) { - if (replicate.obsolete) continue - const status = observationForMetric(replicate, metricId, metricKey)?.status - if (status && status !== 'measured') counts.set(status, (counts.get(status) ?? 0) + 1) - } - if (counts.size === 0) return null - return (['unavailable', 'failed', 'timed_out', 'cancelled', 'not_applicable'] as const) - .filter((status) => counts.has(status)) - .map((status) => `${OBSERVATION_LABELS[status]} (${counts.get(status)})`) - .join(' · ') -} function ObservationCard({ observation }: { observation: MetricObservation }) { const problem = observation.status === 'failed' || observation.status === 'timed_out' @@ -151,7 +98,8 @@ function ObservationCard({ observation }: { observation: MetricObservation }) { : JSON.stringify(observation.value, null, 2) return

    {observation.metric_name}

    {OBSERVATION_LABELS[observation.status]}
    - {observation.status === 'measured' && 
    {measuredValue}
    } + {observation.status === 'measured' && 
    {observation.value_truncated ? observation.value_preview : measuredValue}
    } + {observation.value_truncated &&

    Preview · open complete recorded JSON for the original value.

    } {observation.error &&

    {observation.error}

    }

    {observation.producer.producer_id} · v{observation.producer.version}

     @@ -177,6 +125,7 @@ function CalibrationArtifact({ artifact }: { artifact: EvaluationArtifact }) { } function artifactPresentation(artifact: EvaluationArtifact) { + if (artifact.payload_truncated) return { title: formatMetricLabel(artifact.artifact_key), content: <>
    {artifact.payload_preview}

    Preview · open complete recorded JSON for the original artifact.

    } const kind = artifact.kind || artifact.artifact_key if (kind === 'confusion_matrix') return { title: 'Confusion matrix', content: } if (kind === 'binary_calibration_curve' || kind === 'calibration') return { title: 'Calibration', content: } @@ -191,60 +140,20 @@ function ArtifactCard({ artifact }: { artifact: EvaluationArtifact }) { function usageSummarizesMetric( key: string, - result: Pick, + result: Pick, ): boolean { return (key === 'cost_usd' && result.cost_usd !== null) || (key === 'total_tokens' && result.total_tokens !== null) || (key === 'duration_seconds' && result.duration_seconds !== null) } -function CellResultSummary({ cell, metricKeys, metricTypes, metricAggregations }: { cell: ResultCell; metricKeys: string[]; metricTypes: ResultMetricTypes; metricAggregations: ResultMetricAggregations }) { - const metrics = metricKeys.filter((key) => typeof cell.metric_means[key] === 'number' && !usageSummarizesMetric(key, cell)) - const hasUsage = cell.cost_usd !== null || cell.total_tokens !== null || cell.duration_seconds !== null - return ( -
    -
    -
    -

    Condition summaryMetrics are averages over current, non-obsolete replicates in this condition.

    -

    Current replicates only

    -
    - - {cell.current_completed_count}/{cell.replicate_count} complete - -
    - {metrics.length > 0 && ( -
    -

    Outcome metricsEach metric is labeled with its cell aggregation: average, total, or pass rate for Boolean outcomes.

    -
    - {metrics.map((key) => ( -
    -

    {metricDisplayLabel(key, metricTypes, metricAggregations)}

    -

    {formatResultMetricValue(key, cell.metric_means[key], metricTypes)}

    - {binaryMetricNote(key, cell.metric_means[key], cell.metric_counts[key], metricTypes) &&

    {binaryMetricNote(key, cell.metric_means[key], cell.metric_counts[key], metricTypes)}

    } -
    - ))} -
    -
    - )} - {hasUsage && ( -
    0 ? 'mt-4 border-t pt-3' : 'mt-4'}`}> -

    UsageTotals across current replicates in this condition. Provider telemetry can be unavailable for some calls.

    -
    - {cell.cost_usd !== null &&

    Cost

    {formatCurrency(cell.cost_usd)}

    } - {cell.duration_seconds !== null &&

    Duration

    {formatDuration(cell.duration_seconds)}

    } - {cell.total_tokens !== null &&

    Tokens

    {formatNumber(cell.total_tokens)}

    } -
    -
    - )} - {metrics.length === 0 && !hasUsage &&

    No current numeric results or provider usage have been reported yet.

    } -
    - ) -} -function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { +export function ReplicateResultDetail({ replicate, metricKeys, metricTypes, executionContext, metricLabels }: { replicate: ResultReplicate metricKeys: string[] metricTypes: ResultMetricTypes + executionContext?: ExecutionContext + metricLabels?: Record }) { const metrics = metricKeys.filter((key) => { if (typeof replicate.metric_values[key] !== 'number') return false @@ -273,9 +182,9 @@ function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: {

    Outcome

    {metrics.map((key) => { - const observation = observationForMetric(replicate, undefined, key) + const observation = observationForMetric(replicate, key, metricLabels?.[key] ?? key) return
    -

    {formatMetricLabel(key)}

    +

    {metricLabels?.[key] ?? formatMetricLabel(key)}

    {formatResultMetricValue(key, replicate.metric_values[key], metricTypes, true)}

    {observation?.status === 'measured' &&

    {observation.producer.producer_id} · v{observation.producer.version}

    }
    @@ -303,7 +212,7 @@ function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: {

    Run timeline

    {replicate.node_runs.length === 0 ?

    No node-level run details are available.

    : (
      - {replicate.node_runs.map((node) => )} + {replicate.node_runs.map((node) => )}
    )}
    @@ -311,401 +220,22 @@ function ReplicateResultDetail({ replicate, metricKeys, metricTypes }: { ) } -function historicalRunAsReplicate(replicate: ResultReplicate, historicalRun: HistoricalRun): ResultReplicate { - // The shared detail component needs the stable replicate identity/factors, - // while the historical run contributes the immutable execution facts. - return { - ...replicate, - ...historicalRun, - metric_values: historicalRun.metric_values ?? {}, - metric_evaluation: historicalRun.metric_evaluation, - obsolete_runs: [], - superseded_runs: [], - } -} - -function latestObsoleteRun(replicate: ResultReplicate | null): ObsoleteRun | null { - if (!replicate?.obsolete || !replicate.run_id) return null - return { - run_id: replicate.run_id, - metric_values: replicate.metric_values, - metric_evaluation: replicate.metric_evaluation, - metric_observations: replicate.metric_observations, - evaluation_artifacts: replicate.evaluation_artifacts, - legacy_values: replicate.legacy_values, - status: replicate.status, - obsolete: true, - error: replicate.error, - protocol_revision_id: replicate.protocol_revision_id, - updated_at: replicate.updated_at, - duration_seconds: replicate.duration_seconds, - node_runs: replicate.node_runs, - input_tokens: replicate.input_tokens, - output_tokens: replicate.output_tokens, - total_tokens: replicate.total_tokens, - cost_usd: replicate.cost_usd, - agent_run_count: replicate.agent_run_count, - reported_usage_count: replicate.reported_usage_count, - reported_cost_count: replicate.reported_cost_count, - } -} export type ResultsSelection = - | { type: 'cell'; cellLabel: string; scope?: ResultsScope } + | { type: 'cell'; cellLabel: string; cellId?: string; scope?: ResultsScope } | { type: 'replicate'; replicateLabel: string; scope?: ResultsScope } | { type: 'row'; rowResultId: string; scope: ResultsScope } + | { type: 'execution'; slotId: string; scope: ResultsScope } -function resultsMetricKey(metric: DesignMetric): string { - return metric.kind === 'runtime' && metric.catalogKey ? metric.catalogKey : metric.name -} - -function orderedResultsMetricKeys(metricKeys: string[], experiment: Experiment): string[] { - const available = new Set(metricKeys) - const designMetricsById = new Map((experiment.design_spec?.metrics ?? []).map((metric) => [metric.id, metric])) - const declared = (experiment.measurement_plan?.metrics ?? []).flatMap((definition) => { - const designMetric = designMetricsById.get(definition.id) - const key = designMetric ? resultsMetricKey(designMetric) : definition.name - return available.has(key) ? [key] : [] - }) - const declaredKeys = new Set(declared) - return [...declared, ...metricKeys.filter((key) => !declaredKeys.has(key))] -} +export function ResultsInspectorPanel(props: { + experimentId: string; experiment: Experiment; selection: ResultsSelection | null; onClose: () => void +}) { return } -// This lives alongside the canvas rather than inside the left Results panel. -// Keeping selection in the page lets the left panel remain usable while this -// inspector updates in place for every cell or replicate the user chooses. -export function ResultsInspectorPanel({ - experimentId, - experiment, - selection, - onClose, -}: { - experimentId: string - experiment: Experiment - selection: ResultsSelection | null - onClose: () => void +export function ResultsTab({ protocolId, experimentId, experiment, onSelectResult, versionId }: { + protocolId?: string; experimentId: string; experimentName: string; experiment: Experiment + onSelectResult: (selection: ResultsSelection) => void; versionId?: string }) { - const [expandedObsoleteRuns, setExpandedObsoleteRuns] = useState>(() => new Set()) - const resultsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'run-results', selection?.scope], - queryFn: () => experimentsApi.getRunResults(experimentId, selection?.scope), - enabled: selection !== null, - refetchInterval: 5000, - }) - if (!selection) return null - if (selection.type === 'row') { - const row = resultsQuery.data?.row_results?.find(item => item.row_result_id === selection.rowResultId) - if (!row) return - return - } - - const replicate = selection.type === 'replicate' - ? resultsQuery.data?.replicates.find((candidate) => candidate.replicate_label === selection.replicateLabel) ?? null - : null - const cell = selection.type === 'cell' - ? resultsQuery.data?.cells.find((candidate) => candidate.cell_label === selection.cellLabel) ?? null - : null - const title = replicate ? `Replicate ${replicate.replicate_number}` : 'Cell results' - const description = replicate - ? factorSummary(replicate.factor_values, experiment.design_spec, replicate.cell_label) - : cell - ? factorSummary(cell.factor_values, experiment.design_spec, cell.cell_label) - : undefined - // During a rolling frontend/backend restart, accept the prior response name - // too. The earlier API omitted the latest stale run from its history, so add - // it locally when necessary rather than hiding a real obsolete result. - const legacyObsoleteRuns = (replicate as (ResultReplicate & { previous_obsolete_runs?: ObsoleteRun[] }) | null) - ?.previous_obsolete_runs - const reportedObsoleteRuns = Array.isArray(replicate?.obsolete_runs) - ? replicate.obsolete_runs - : Array.isArray(legacyObsoleteRuns) ? legacyObsoleteRuns : [] - const latestStaleRun = latestObsoleteRun(replicate) - const obsoleteRuns = latestStaleRun && !reportedObsoleteRuns.some((run) => run.run_id === latestStaleRun.run_id) - ? [latestStaleRun, ...reportedObsoleteRuns] - : reportedObsoleteRuns - const supersededRuns: SupersededRun[] = Array.isArray(replicate?.superseded_runs) ? replicate.superseded_runs : [] - const metricKeys = resultsQuery.data ? orderedResultsMetricKeys(resultsQuery.data.metric_keys, experiment) : [] - - return ( - - ) -} - -export function ResultsTab({ - protocolId, - experimentId, - experimentName, - experiment, - onSelectResult, - versionId, -}: { - protocolId?: string - experimentId: string - experimentName: string - experiment: Experiment - onSelectResult: (selection: ResultsSelection) => void - versionId?: string -}) { - const [metricPreference, setMetricPreference] = useState(null) - const [expandedResultCells, setExpandedResultCells] = useState>(() => new Set()) - const scope: ResultsScope = { protocol_id: protocolId, protocol_revision_id: versionId || undefined } - const resultsQuery = useQuery({ queryKey: ['experiments', experimentId, 'run-results', scope], queryFn: () => experimentsApi.getRunResults(experimentId, scope), refetchInterval: 5000 }) - if (resultsQuery.isLoading) return
    - if (resultsQuery.isError || !resultsQuery.data) return

    {resultsQuery.error?.message ?? 'Could not load this experiment’s results.'}

    - - if (resultsQuery.data.consumption_mode === 'per_row') return onSelectResult({ type: 'row', rowResultId, scope })} /> - - const { overview, cells, replicates, metric_types: metricTypes, metric_aggregations: metricAggregations, metric_directions: metricDirections, primary_metric: primaryMetric, primary_metric_direction: primaryMetricDirection } = resultsQuery.data - const metricKeys = orderedResultsMetricKeys(resultsQuery.data.metric_keys, experiment) - const metricKey = metricPreference && metricKeys.includes(metricPreference) - ? metricPreference - : primaryMetric && metricKeys.includes(primaryMetric) - ? primaryMetric - : metricKeys[0] ?? null - const selectedDirection = metricKey ? (metricDirections?.[metricKey] ?? 'neutral') : null - const selectedMetricId = metricKey ? experiment.design_spec?.metrics?.find((metric) => { - return resultsMetricKey(metric) === metricKey - })?.id : undefined - const rankable = metricKey === primaryMetric && (selectedDirection === 'maximize' || selectedDirection === 'minimize') - const sortDirection = selectedDirection === 'minimize' ? -1 : 1 - const sortedCells = [...cells].sort((left, right) => { - if (!metricKey || !rankable) return left.cell_label.localeCompare(right.cell_label) - const leftValue = left.metric_means[metricKey] - const rightValue = right.metric_means[metricKey] - const leftMeasured = typeof leftValue === 'number' - const rightMeasured = typeof rightValue === 'number' - if (leftMeasured && rightMeasured) return sortDirection * (rightValue - leftValue) - if (leftMeasured) return -1 - if (rightMeasured) return 1 - return left.cell_label.localeCompare(right.cell_label) - }) - const replicatesByCell = new Map() - for (const replicate of replicates) { - const cellReplicates = replicatesByCell.get(replicate.cell_label) ?? [] - cellReplicates.push(replicate) - replicatesByCell.set(replicate.cell_label, cellReplicates) - } - for (const cellReplicates of replicatesByCell.values()) cellReplicates.sort((left, right) => left.replicate_number - right.replicate_number) - if (overview.total_replicates === 0) return

    Generate a design and run a replicate to see results here.

    - const bestCell = metricKey && rankable ? sortedCells.find((cell) => typeof cell.metric_means[metricKey] === 'number') ?? null : null - const directionLabel = selectedDirection === 'minimize' ? 'Lowest' : 'Highest' - - return ( - 0 && ( -
    - Inspect metric - -
    - )} - summary={bestCell && metricKey ? ( -
    -
    Best current result
    -

    {formatResultMetricValue(metricKey, bestCell.metric_means[metricKey], metricTypes)}

    - {binaryMetricNote(metricKey, bestCell.metric_means[metricKey], bestCell.metric_counts[metricKey], metricTypes) &&

    {binaryMetricNote(metricKey, bestCell.metric_means[metricKey], bestCell.metric_counts[metricKey], metricTypes)}

    } -

    {directionLabel} {metricDisplayLabel(metricKey, metricTypes, metricAggregations)} · {factorSummary(bestCell.factor_values, experiment.design_spec)}

    -
    - ) : metricKey ? ( -
    Comparison only

    {formatMetricLabel(metricKey)}

    {primaryMetric === null ? 'No primary metric is declared, so Results compares values without ranking cells or declaring a winner.' : metricKey !== primaryMetric ? `${formatMetricLabel(metricKey)} is not the declared primary metric, so Results compares its values without ranking cells or declaring a winner.` : `${formatMetricLabel(metricKey)} has no maximize/minimize direction, so Results compares values without declaring a winner.`}

    - ) : ( -

    Results are arriving

    Complete a run with reported metrics to rank conditions here.

    - )} - scorecards={<> - - - - - } - notice={overview.obsolete_replicates > 0 &&
    {overview.obsolete_replicates} run{overview.obsolete_replicates === 1 ? '' : 's'} used an older canvas version and are excluded from current totals.
    } - comparison={ -
    -

    {rankable ? 'Cell ranking' : 'Cell comparison'}{rankable ? 'Cells are ranked by the selected metric using current, non-obsolete replicates only.' : 'Metrics are shown for comparison without ordering cells or naming a winner.'}

    Current replicates only{rankable && isBinaryMetric(metricKey, metricTypes) ? ' · ranked by pass rate' : rankable && metricKey === primaryMetric ? ` · ${primaryMetricDirection === 'minimize' ? 'lower is better' : 'higher is better'}` : ''}

    - {metricKeys.length === 0 ?

    No scalar metrics are declared or reported yet. Run details and provider usage are still available below.

    :
    {sortedCells.map((cell, index) => { const value = metricKey ? cell.metric_means[metricKey] : undefined; const measured = typeof value === 'number'; const ranked = rankable && measured; const note = measured && metricKey ? binaryMetricNote(metricKey, value, cell.metric_counts[metricKey], metricTypes) : null; const missingSummary = missingObservationSummary(replicatesByCell.get(cell.cell_label) ?? [], selectedMetricId, metricKey); return })}
    {rankable ? 'Rank' : '—'}{rankable && Rank among conditions with a measured value for the selected metric.}ConditionThe factor levels used for this group of replicates.{metricKey ? metricDisplayLabel(metricKey, metricTypes, metricAggregations) : ''}CostDurationRunsCompleted current replicates out of all generated replicates for this condition.
    {ranked ? index + 1 : '—'}

    {factorSummary(cell.factor_values, experiment.design_spec)}

    {cell.obsolete_count > 0 && {cell.obsolete_count} obsolete}    		{measured ? <>{formatResultMetricValue(metricKey!, value, metricTypes)}{note && {note}}{missingSummary && {missingSummary}} : {missingSummary ?? 'No measured observations'}}{formatCurrency(cell.cost_usd)}{formatDuration(cell.duration_seconds)}{cell.current_completed_count}/{cell.replicate_count}
    } -
    - } - cells={ -
    - {sortedCells.map((cell) => { - const expanded = expandedResultCells.has(cell.cell_label) - const cellReplicates = replicatesByCell.get(cell.cell_label) ?? [] - const replicateListId = `result-cell-${cell.cell_label}-replicates` - const metric = metricKey ? cell.metric_means[metricKey] : undefined - const currentMetricValues = cellReplicates - .filter((replicate) => !replicate.obsolete) - .map((replicate) => numericMetricValue(replicate, metricKey)) - .filter((value): value is number => value !== null) - const replicateMetricTotal = currentMetricValues.length > 0 - ? currentMetricValues.reduce((sum, value) => sum + value, 0) - : null - const replicateMetricAverage = replicateMetricTotal !== null ? replicateMetricTotal / currentMetricValues.length : null - const replicateMetricDeviation = replicateMetricAverage !== null && currentMetricValues.length > 1 - ? Math.sqrt(currentMetricValues.reduce((sum, value) => sum + (value - replicateMetricAverage) ** 2, 0) / currentMetricValues.length) - : null - const orderedReplicates = [...cellReplicates].sort((left, right) => { - const leftValue = left.obsolete ? null : numericMetricValue(left, metricKey) - const rightValue = right.obsolete ? null : numericMetricValue(right, metricKey) - if (leftValue !== null && rightValue !== null) return sortDirection * (rightValue - leftValue) - if (leftValue !== null) return -1 - if (rightValue !== null) return 1 - return left.replicate_number - right.replicate_number - }) - return ( - setExpandedResultCells(current => { - const next = new Set(current) - if (next.has(cell.cell_label)) next.delete(cell.cell_label) - else next.add(cell.cell_label) - return next - })} - progress={<>{cell.current_completed_count}/{cell.replicate_count} current runs complete} - badges={cell.obsolete_count > 0 && {cell.obsolete_count} obsolete {cell.obsolete_count === 1 ? 'run' : 'runs'}} - metric={metric !== undefined && {metricDisplayLabel(metricKey!, metricTypes, metricAggregations)}{formatResultMetricValue(metricKey!, metric, metricTypes)}{binaryMetricNote(metricKey!, metric, cell.metric_counts[metricKey!], metricTypes) && {binaryMetricNote(metricKey!, metric, cell.metric_counts[metricKey!], metricTypes)}}} - actions={}> - {cellReplicates.length === 0 ?

    No replicate results are available for this cell.

    : ( - <> - {metricKey && replicateMetricTotal !== null && ( -
    -
    -

    Replicate comparison · {metricDisplayLabel(metricKey, metricTypes, metricAggregations)}

    -

    Current results only

    -
    - {isBinaryMetric(metricKey, metricTypes) ?

    {Math.round((replicateMetricAverage ?? 0) * currentMetricValues.length)}/{currentMetricValues.length} passed · {formatResultMetricValue(metricKey, replicateMetricAverage, metricTypes)}

    :

    {metricAggregations[metricKey] === 'sum' ? <>Total {formatMetricValue(metricKey, replicateMetricTotal)} : <>Average {formatMetricValue(metricKey, replicateMetricAverage)}} · Range {formatMetricValue(metricKey, Math.min(...currentMetricValues))}–{formatMetricValue(metricKey, Math.max(...currentMetricValues))}{replicateMetricDeviation !== null && <> · SD {formatMetricValue(metricKey, replicateMetricDeviation)}}

    } -
    - )} -

    Individual replicates

    Select one for full output

    -
      - {orderedReplicates.map((replicate) => { - const observation = observationForMetric(replicate, selectedMetricId, metricKey) - const value = numericMetricValue(replicate, metricKey) - return ( - onSelectResult({ type: 'replicate', replicateLabel: replicate.replicate_label })} - detail={<>{replicate.cost_usd !== null ? formatCurrency(replicate.cost_usd) : 'Cost not reported'}{replicate.duration_seconds !== null ? ` · ${formatDuration(replicate.duration_seconds)}` : ''}} - status={<>{value !== null && {formatResultMetricValue(metricKey!, value, metricTypes, true)}}{observation ? {OBSERVATION_LABELS[observation.status]} : evaluationStatusLabel(replicate) && {evaluationStatusLabel(replicate)}}{statusLabel(replicate.status, replicate.obsolete)}} - /> - )})} -
    - - )} -     - ) - })} -
    - } - /> - ) + const scope = { protocol_id: protocolId, protocol_revision_id: versionId } + return } diff --git a/frontend/src/components/protocol/RunConfirmDialog.tsx b/frontend/src/components/protocol/RunConfirmDialog.tsx index 1b862347..85c58b9e 100644 --- a/frontend/src/components/protocol/RunConfirmDialog.tsx +++ b/frontend/src/components/protocol/RunConfirmDialog.tsx @@ -50,6 +50,7 @@ export function RunConfirmDialog({ confirmLabel, confirmDisabled = false, publishDisabled = confirmDisabled, + publishLabel, isConfirming = false, confirmError = null, }: { @@ -75,6 +76,7 @@ export function RunConfirmDialog({ confirmLabel?: string confirmDisabled?: boolean publishDisabled?: boolean + publishLabel?: string isConfirming?: boolean confirmError?: string | null }) { @@ -222,7 +224,7 @@ export function RunConfirmDialog({ )} ) : ( diff --git a/frontend/src/components/protocol/RunsTab.test.tsx b/frontend/src/components/protocol/RunsTab.test.tsx index cbd19ee5..260372bf 100644 --- a/frontend/src/components/protocol/RunsTab.test.tsx +++ b/frontend/src/components/protocol/RunsTab.test.tsx @@ -10,6 +10,8 @@ describe('RunsTab factor discrepancy warning', () => { afterEach(() => vi.restoreAllMocks()) it('warns immediately and blocks runs when the canvas disagrees with generated factor levels', async () => { + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue({ ready: true, metric_types: {}, metric_directions: {} } as never) + vi.spyOn(experimentsApi, 'listExecutionCells').mockResolvedValue({ items: [], next_cursor: null }) vi.spyOn(experimentsApi, 'listReplicates').mockResolvedValue([{ id: 'replicate-1', cell_id: 'cell-1', cell_label: 'Prompt:original', replicate_label: 'Prompt:original__rep1', replicate_number: 1, design_revision_id: 'design-1', run_id: null, workspace_id: null, @@ -52,3 +54,33 @@ describe('RunsTab factor discrepancy warning', () => { expect(screen.getByRole('button', { name: 'Run all cells' })).toBeDisabled() }) }) + +describe('RunsTab stop controls', () => { + afterEach(() => vi.restoreAllMocks()) + const protocol = { id: 'protocol-1', published_revision_id: 'revision-1', published_revision: 1, has_unpublished_changes: false, + graph: { nodes: [], edges: [] } } as unknown as Protocol + const cell = (counts: Record) => ({ cell_id: 'cell-1', cell_label: 'model=a', factor_values: { model: 'a' }, + replicate_count: 3, total: 3, counts }) + + async function mount(hasActive: boolean, counts: Record) { + vi.spyOn(experimentsApi, 'getExecutionSummary').mockResolvedValue( + { ready: true, has_active: hasActive, counts, metric_types: {}, metric_directions: {} } as never) + vi.spyOn(experimentsApi, 'listExecutionCells').mockResolvedValue({ items: [cell(counts)], next_cursor: null } as never) + render( + + ) + await screen.findByRole('button', { name: 'Run cell' }) + } + + it('hides Stop all and Stop cell when nothing is pending or running', async () => { + await mount(false, { completed: 2, failed: 1 }) + expect(screen.queryByRole('button', { name: 'Stop all' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Stop cell' })).not.toBeInTheDocument() + }) + + it('offers Stop all and Stop cell while executions are active', async () => { + await mount(true, { completed: 1, running: 1, pending: 1 }) + expect(await screen.findByRole('button', { name: 'Stop all' })).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Stop cell' })).toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/protocol/RunsTab.tsx b/frontend/src/components/protocol/RunsTab.tsx index ab81dfb2..b6c40c90 100644 --- a/frontend/src/components/protocol/RunsTab.tsx +++ b/frontend/src/components/protocol/RunsTab.tsx @@ -1,531 +1,87 @@ import { useState } from 'react' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' -import { AlertTriangle, ChevronDown, Download, Lock, Square } from 'lucide-react' -import { Badge } from '@/components/ui/badge' +import { AlertTriangle, Square } from 'lucide-react' +import { ApiError, experimentsApi, protocolsApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' -import { Skeleton } from '@/components/ui/skeleton' -import { ApiError, experimentsApi, protocolsApi } from '@/api/client' -import { displayFactorCondition, factorValueKey, groupReplicatesIntoCells, type ExperimentalCell } from '@/lib/experiment' import { factorBindingDiscrepancies, type FactorBindingDiscrepancy } from '@/lib/factorBindings' +import { useBatchPreview } from '@/lib/resultJobs' +import { ResultJobProgress } from './ResultJobProgress' +import { executionKeys, useExecutionSummary } from '@/lib/executionQueries' import { protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' -import type { Experiment, ResultCell, ResultReplicate, Trial } from '@/types/experiments' +import type { Experiment } from '@/types/experiments' import type { Protocol } from '@/types/protocols' -import { DatasetRowRuns } from './DatasetRowRuns' +import type { ExecutionSlot } from '@/types/executionResults' +import { ExecutionBrowser, ExecutionRows } from './ExecutionBrowser' +import { runsAgainOnlyOnRequest } from '@/lib/protocolRun' +import { newRequestKey } from '@/lib/requestKey' +import { ExperimentDefinitionDownload } from './ExperimentDefinitionDownload' import { RunConfirmDialog } from './RunConfirmDialog' -import { CellCard, ReplicateRow, ReplicateRunAction, CellRunAction, ReplicateStatus } from './CellPresentation' -import { WarningBadge } from './nodes/WarningBadge' - -function factorEntries(cell: ExperimentalCell): [string, unknown][] { - // Factor ordering is meaningful when a user declared it, so preserve the - // object order here for the condition summary. Sorting the cells themselves - // below still gives a stable list no matter which replicate arrived first. - return Object.entries(cell.factorValues) -} - -function cellSortKey(cell: ExperimentalCell): string { - return factorEntries(cell) - .map(([name, value]) => `${name}:${factorValueKey(value)}`) - .join('|') -} - -function isActiveTrialStatus(status: Trial['status']): boolean { - return status === 'queued' || status === 'running' || status === 'finalizing' -} - -const OBSOLETE_TRIAL_BADGE = { - label: 'Obsolete', - className: 'border-transparent bg-[color:var(--chart-2)]/10 text-[color:var(--chart-2)]', -} - -// Ranked above the plain status badge for the same reason Obsolete is: the -// row's status really is "completed", and that is exactly the misreading -// worth preventing. Amber, the app's "finished, with a caveat" color. -const TRUNCATED_TRIAL_BADGE = { - label: 'Hit iteration limit', - className: 'border-transparent bg-[color:var(--chart-4)]/10 text-[color:var(--chart-4)]', -} +import { ReplicateRunAction } from './CellPresentation' +import type { ResultsSelection } from './ResultsTab' -const TRUNCATED_REPLICATE_HELP = - 'An agent in this replicate was stopped by its iteration limit, so it finished without finishing its work. It is not counted as scored. Raise Max iterations on the Reason + Act node and run it again.' - -function formatCurrency(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - return new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) -} - -function formatNumber(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - return new Intl.NumberFormat(undefined, { maximumFractionDigits: 0 }).format(value) -} - -function formatDuration(value: number | null): string | null { - if (value === null || !Number.isFinite(value)) return null - if (value < 60) return `${Math.round(value)} sec` - if (value < 3600) return `${(value / 60).toFixed(1)} min` - return `${(value / 3600).toFixed(1)} hr` -} - -function usageSummary(result: Pick | Pick): string[] { - const tokens = formatNumber(result.total_tokens) - return [ - formatCurrency(result.cost_usd), - tokens ? `${tokens} tokens` : null, - formatDuration(result.duration_seconds), - ].filter((value): value is string => !!value) -} - -// The panel's run control resumes only replicates without a current completed -// result. A completed run is meaningful even when it deliberately produced no -// score metrics, so it stays out of a later batch unless the user explicitly -// elects to re-run it (and therefore re-bill it). Obsolete results are current -// work again because they belong to an older canvas version. -export function RunAllCellsButton({ - protocol, - experimentId, - regenerationRequired, - unboundFactors, - bindingDiscrepancies, - replicateLabels, - label = 'Run all cells', - dialogTitle, - compact = false, - hasCompletedRun = false, -}: { - protocol: Protocol | undefined - experimentId: string - regenerationRequired: boolean - unboundFactors: string[] - bindingDiscrepancies: FactorBindingDiscrepancy[] - replicateLabels?: string[] - label?: string - dialogTitle?: string - compact?: boolean - hasCompletedRun?: boolean +export function RunAllCellsButton({ protocol, experimentId, regenerationRequired, unboundFactors, bindingDiscrepancies, label = 'Run all cells', dialogTitle, compact = false, cellId }: { + protocol: Protocol | undefined; experimentId: string; regenerationRequired: boolean; unboundFactors: string[] + bindingDiscrepancies: FactorBindingDiscrepancy[]; label?: string; dialogTitle?: string; compact?: boolean + cellId?: string }) { - const queryClient = useQueryClient() - const [dialogOpen, setDialogOpen] = useState(false) - const [selectedReruns, setSelectedReruns] = useState>(() => new Set()) - const [expandedCells, setExpandedCells] = useState>(() => new Set()) - const [lockBeforeRun, setLockBeforeRun] = useState(false) - const [downloadingDefinition, setDownloadingDefinition] = useState(false) - const [definitionDownloadError, setDefinitionDownloadError] = useState(null) - const experimentQuery = useQuery({ - queryKey: ['experiments', experimentId], - queryFn: () => experimentsApi.get(experimentId), - }) - const trialsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'runs'], - queryFn: () => experimentsApi.listTrials(experimentId), - refetchInterval: dialogOpen ? 3000 : false, - }) - const replicatesQuery = useQuery({ - queryKey: ['experiments', experimentId, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experimentId), - }) - const allReplicates = replicatesQuery.data ?? [] - const requestedLabels = replicateLabels ? new Set(replicateLabels) : null - const replicates = requestedLabels - ? allReplicates.filter((replicate) => requestedLabels.has(replicate.replicate_label)) - : allReplicates - const activeReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => isActiveTrialStatus(trial.status)) - .map((trial) => trial.replicate_label), - ) - const completedReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => trial.status === 'completed' && !trial.obsolete) - .map((trial) => trial.replicate_label), - ) - const obsoleteReplicateLabels = new Set( - (trialsQuery.data ?? []) - .filter((trial) => trial.obsolete) - .map((trial) => trial.replicate_label), - ) - const eligibleReplicateLabels = replicates - .filter((replicate) => !activeReplicateLabels.has(replicate.replicate_label)) - .map((replicate) => replicate.replicate_label) - // Replicates in THIS button's scope whose agent ran out of iterations. The - // dialog says so before a re-run because nothing about the cap has changed - // in between: the same run at the same cap truncates again, and comes back - // unscored again, having cost the same money. Named by count rather than by - // node -- this button only ever has the stored graph and the trial list, so - // it can't tell which node's cap did it (the canvas can; see - // RunConfirmDialog's `truncatedCaps`). - const truncatedLabels = new Set( - (trialsQuery.data ?? []).filter((trial) => trial.truncated && !trial.obsolete).map((trial) => trial.replicate_label), - ) - const truncatedInScope = replicates.filter((replicate) => truncatedLabels.has(replicate.replicate_label)).length - const cellCount = groupReplicatesIntoCells(replicates).length - const previouslyRunReplicateLabels = new Set( - replicates - .filter( - (replicate) => - !obsoleteReplicateLabels.has(replicate.replicate_label) && - (!!replicate.metric_values || completedReplicateLabels.has(replicate.replicate_label)), - ) - .map((replicate) => replicate.replicate_label), - ) - const previouslyRunReplicates = replicates.filter((replicate) => previouslyRunReplicateLabels.has(replicate.replicate_label)) - const pendingReplicateCount = replicates.filter( - (replicate) => - !previouslyRunReplicateLabels.has(replicate.replicate_label) && !activeReplicateLabels.has(replicate.replicate_label), - ).length - const previouslyRunCells = groupReplicatesIntoCells( - previouslyRunReplicates.filter((replicate) => !activeReplicateLabels.has(replicate.replicate_label)), - ).sort((a, b) => - cellSortKey(a).localeCompare(cellSortKey(b)), - ) - - const runMutation = useMutation({ - mutationFn: () => - protocolsApi.runCells(protocol!.id, { - // Do not launch a second concurrent attempt for an in-flight - // replicate. The Stop controls let users finish that attempt first. - replicateLabels: eligibleReplicateLabels, - rerunReplicateLabels: [...selectedReruns], - }), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'replicates'] }) - queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) - queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'runs'] }) - setDialogOpen(false) - }, - }) - const lockMutation = useMutation({ - mutationFn: () => experimentsApi.lock(experimentId), - onSuccess: (locked) => { - queryClient.setQueryData(['experiments', experimentId], locked) - queryClient.invalidateQueries({ queryKey: ['experiments'] }) - runMutation.mutate() - }, - }) - - function beginRun() { - if (lockBeforeRun && !experimentQuery.data?.locked_at) { - lockMutation.mutate() - return - } - runMutation.mutate() - } - - const publishAndRunMutation = useMutation({ + const client = useQueryClient() + const [open, setOpen] = useState(false) + const [lock, setLock] = useState(false) + const [reruns, setReruns] = useState>(() => new Set()) + const [rerunAll, setRerunAll] = useState(false) + const scope = { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined } + const [confirmedId, setConfirmedId] = useState() + const preview = useBatchPreview(experimentId, scope, { cell_id: cellId, rerun_slot_ids: [...reruns], rerun_completed: rerunAll }, open) + const summary = useExecutionSummary(experimentId, scope, open && !!protocol) + const experiment = useQuery({ queryKey: ['experiments', experimentId], queryFn: () => experimentsApi.get(experimentId), enabled: open }) + const run = useMutation({ mutationFn: async () => { - const live = queryClient.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol!.id)) - if (live) await protocolsApi.update(protocol!.id, { graph: toPersistedGraph(live.nodes, live.edges) }) - return protocolsApi.publish(protocol!.id) - }, - onSuccess: (published) => { - queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'revisions'] }) - queryClient.setQueryData(protocolForExperimentQueryKey(experimentId), published) - beginRun() + if (lock && !experiment.data?.locked_at) await experimentsApi.lock(experimentId) + return experimentsApi.confirmResultJob(experimentId, preview.data!.id) }, + onSuccess: job => { setConfirmedId(job.id); setOpen(false); void client.invalidateQueries({ queryKey: ['experiments', experimentId] }) }, }) - - function setReplicatesSelected(replicateLabels: string[], selected: boolean) { - setSelectedReruns((current) => { - const next = new Set(current) - for (const label of replicateLabels) { - if (selected) next.add(label) - else next.delete(label) - } - return next - }) - } - - function toggleExpanded(cellLabel: string) { - setExpandedCells((current) => { - const next = new Set(current) - if (next.has(cellLabel)) next.delete(cellLabel) - else next.add(cellLabel) - return next - }) - } - - function openDialog() { - // Defaulting all previous work to unchecked makes the safe (existing) - // choice explicit: previous results are skipped unless chosen again. - setSelectedReruns(new Set()) - setExpandedCells(new Set()) - setLockBeforeRun(false) - setDialogOpen(true) - } - - async function downloadDefinition() { - if (!protocol) return - setDownloadingDefinition(true) - setDefinitionDownloadError(null) - try { - // Read fresh rather than exporting whichever React Query snapshot - // happens to be on screen. A portable definition should describe one - // coherent point in time, including the currently published canvas. - const [experiment, canvas, designRevisions] = await Promise.all([ - experimentsApi.get(experimentId), - protocolsApi.get(protocol.id), - experimentsApi.listDesignRevisions(experimentId), - ]) - const revisionIds = [...new Set([canvas.published_revision_id, experiment.locked_protocol_revision_id].filter((id): id is string => !!id))] - const revisions = await Promise.all(revisionIds.map((revisionId) => protocolsApi.getRevision(canvas.id, revisionId))) - const revisionById = new Map(revisions.map((revision) => [revision.id, revision])) - const publishedCanvas = canvas.published_revision_id ? revisionById.get(canvas.published_revision_id) ?? null : null - const lockedCanvas = experiment.locked_protocol_revision_id ? revisionById.get(experiment.locked_protocol_revision_id) ?? null : null - const payload = { - format: 'asaree.experiment-definition', - schema_version: 1, - exported_at: new Date().toISOString(), - // The existing canvas importer consumes these two top-level fields. - // Keep them as the current draft while the richer envelope below - // carries immutable published/locked snapshots and provenance. - name: experiment.name, - description: canvas.description, - graph: canvas.graph, - design_spec: experiment.design_spec, - measurement_plan: experiment.measurement_plan, - experiment: { - source_id: experiment.id, - name: experiment.name, - description: experiment.description, - hypothesis: experiment.hypothesis, - design_type: experiment.design_type, - task_brief: experiment.task_brief, - design_spec: experiment.design_spec, - measurement_plan: experiment.measurement_plan, - dataset_ids: experiment.dataset_ids, - archived_at: experiment.archived_at, - created_at: experiment.created_at, - updated_at: experiment.updated_at, - lock: experiment.locked_at ? { - locked_at: experiment.locked_at, - source_protocol_revision_id: experiment.locked_protocol_revision_id, - design_spec: experiment.locked_design_spec, - measurement_plan: experiment.locked_measurement_plan, - canvas_revision: lockedCanvas ? { - source_id: lockedCanvas.id, - revision: lockedCanvas.revision, - published_at: lockedCanvas.published_at, - graph: lockedCanvas.graph, - } : null, - } : null, - }, - canvas: { - source_protocol_id: canvas.id, - name: canvas.name, - description: canvas.description, - draft: { graph: canvas.graph, updated_at: canvas.updated_at }, - published: publishedCanvas ? { - source_id: publishedCanvas.id, - revision: publishedCanvas.revision, - published_at: publishedCanvas.published_at, - graph: publishedCanvas.graph, - } : null, - has_unpublished_changes: canvas.has_unpublished_changes, - }, - design_revisions: designRevisions.map((revision) => ({ - source_id: revision.id, - revision: revision.revision, - superseded_at: revision.superseded_at, - design_spec: revision.design_spec, - cell_count: revision.cell_count, - replicate_count: revision.replicate_count, - scored_replicate_count: revision.scored_replicate_count, - created_at: revision.created_at, - })), - portability: { - excluded: ['provider credential secrets', 'replicate results and run history', 'dataset file contents', 'knowledge bundle/document contents', 'skill file contents'], - note: 'All canvas nodes and their configuration are included. Dataset, Knowledge, and Skill nodes retain their labels and resource IDs as references; recreate or map their external artifacts before importing into another workspace.', - }, - } - const name = experiment.name || canvas.name - const blob = new Blob([JSON.stringify(payload, null, 2)], { type: 'application/json' }) - const url = URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `${name.trim().replace(/[^a-z0-9]+/gi, '-').replace(/(^-|-$)/g, '') || 'experiment'}.json` - link.click() - URL.revokeObjectURL(url) - } catch { - setDefinitionDownloadError('Could not prepare the experiment definition. Please try again.') - } finally { - setDownloadingDefinition(false) - } - } - - const runBlocked = - !protocol || - replicatesQuery.isLoading || - replicates.length === 0 || - (pendingReplicateCount === 0 && previouslyRunCells.length === 0) || - regenerationRequired || - unboundFactors.length > 0 || - bindingDiscrepancies.length > 0 - const blockedTitle = regenerationRequired - ? 'Design changed — review and regenerate before running the experiment.' - : unboundFactors.length > 0 - ? `Rebind or remove: ${unboundFactors.join(', ')}.` - : bindingDiscrepancies.length > 0 - ? 'Canvas values and generated factor levels disagree. Resolve the warning in Runs before starting.' - : replicates.length === 0 - ? 'Generate design first — there are no cells to run yet.' - : pendingReplicateCount === 0 && previouslyRunCells.length === 0 - ? 'All selected replicates are already running.' - : undefined - const runnableReplicateCount = pendingReplicateCount + selectedReruns.size - const actionLabel = label ?? (hasCompletedRun ? 'Re-run all cells' : 'Run all cells') - const errorMessage = lockMutation.error instanceof ApiError && typeof lockMutation.error.detail === 'string' - ? lockMutation.error.detail - : lockMutation.isError - ? 'Could not lock the experiment before starting it.' - : runMutation.error instanceof ApiError && typeof runMutation.error.detail === 'string' - ? runMutation.error.detail - : runMutation.isError - ? 'Could not start the experiment.' - : null - - return ( - <> - - {compact ? {actionLabel} : } - - {dialogOpen && protocol && ( - 0 - ? `${truncatedInScope} replicate${truncatedInScope === 1 ? '' : 's'} here stopped at the iteration limit and came back unscored. Unless you raise Max iterations on the Reason + Act node first, this run will hit the same limit.` - : null - } - onCancel={() => setDialogOpen(false)} - onConfirm={beginRun} - hasUnpublishedChanges={protocol.has_unpublished_changes} - publishedRevision={protocol.published_revision} - isPublishing={publishAndRunMutation.isPending} - publishError={ - publishAndRunMutation.error instanceof ApiError && typeof publishAndRunMutation.error.detail === 'string' - ? publishAndRunMutation.error.detail - : publishAndRunMutation.isError - ? 'Could not publish the latest canvas.' - : null - } - onPublishAndRun={() => publishAndRunMutation.mutate()} - confirmLabel={actionLabel} - confirmDisabled={runnableReplicateCount === 0} - isConfirming={runMutation.isPending || lockMutation.isPending} - confirmError={errorMessage} - additionalContent={ -
    -
    - {!experimentQuery.data?.locked_at && ( -
    - setLockBeforeRun(checked === true)} - className="mt-0.5" - /> - -
    - )} - - {definitionDownloadError &&

    {definitionDownloadError}

    } -
    -
    -
    -

    Previously run cells

    -

    - Completed replicates are skipped by default. Check a cell or replicate to run it again. -

    -
    - {previouslyRunCells.length === 0 ? ( -

    No completed replicates to review.

    - ) : ( -
    - {previouslyRunCells.map((cell) => { - const labels = cell.replicates.map((replicate) => replicate.replicate_label) - const selectedCount = labels.filter((label) => selectedReruns.has(label)).length - const expanded = expandedCells.has(cell.label) - const summary = factorEntries(cell).map(([name, value]) => displayFactorCondition(name, value, experimentQuery.data?.design_spec, cell.label)).join(' · ') || 'Cell' - const listId = `rerun-cell-${cell.label}` - return ( -
    -
    - setReplicatesSelected(labels, checked === true)} - aria-label={`Run all replicates in ${summary} again`} - className="mt-0.5" - /> - -
    - {expanded && ( -
      - {cell.replicates.map((replicate) => { - const scored = !!replicate.metric_values - return ( -
    • - setReplicatesSelected([replicate.replicate_label], checked === true)} - aria-label={`Run replicate ${replicate.replicate_number} again`} - /> - Replicate {replicate.replicate_number} - - {scored ? 'Scored' : 'Completed without metrics'} - -
      • - ) - })} -
    - )} -
    - ) - })} -
    - )} - {runnableReplicateCount === 0 && ( -

    Select at least one previously run replicate to run again.

    - )} -
    -
    - } - /> - )} - - ) + const publish = useMutation({ mutationFn: async () => { + const live = client.getQueryData<{ nodes: Node[]; edges: Edge[] }>(protocolGraphQueryKey(protocol!.id)) + if (live) await protocolsApi.update(protocol!.id, { graph: toPersistedGraph(live.nodes, live.edges) }) + return protocolsApi.publish(protocol!.id) + }, onSuccess: published => { + client.setQueryData(protocolForExperimentQueryKey(experimentId), published) + void client.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) + // A new publication changes the execution scope. Refresh its counts before confirming. + setReruns(new Set()) + } }) + const blocked = !protocol || regenerationRequired || unboundFactors.length > 0 || bindingDiscrepancies.length > 0 + const pending = preview.data?.progress.planned ?? 0 + return <> + + {open && protocol && setOpen(false)} onConfirm={() => run.mutate()} + hasUnpublishedChanges={protocol.has_unpublished_changes} publishedRevision={protocol.published_revision} + onPublishAndRun={() => publish.mutate()} isPublishing={publish.isPending} publishError={publish.error?.message} + publishDisabled={publish.isPending || run.isPending} publishLabel="Publish & prepare preview" + confirmLabel={label} confirmDisabled={preview.data?.status !== 'awaiting_confirmation' || !pending || run.isPending || publish.isPending} + isConfirming={run.isPending} confirmError={run.error?.message} + additionalContent={
    + {preview.data?.status !== 'awaiting_confirmation' &&

    Preparing batch preview… {preview.data?.progress.scanned ?? 0} executions checked.

    } + {(preview.error || preview.data?.error) &&

    {preview.error?.message ?? preview.data?.error}

    } + {preview.data?.status === 'awaiting_confirmation' &&

    {pending} executions will be queued.

    } + {summary.error &&

    {summary.error.message}

    } + {summary.data?.unit === 'row' &&

    {summary.data.expected ?? '?'} expected row executions across {summary.data.replicate_count ?? 0} replicates.

    } + {!experiment.data?.locked_at && } + +

    Completed executions, including ones that hit their iteration limit, are skipped unless selected to run again.

    + + {}} actions={row => runsAgainOnlyOnRequest(row.status) && = 100} onCheckedChange={checked => setReruns(current => { const next = new Set(current); if (checked) next.add(row.slot_id); else next.delete(row.slot_id); return next })} />} /> +
    } />} + } -// A protocol run is the actual cancellable unit. A cell/experiment stop is -// simply this same operation applied to each of its active replicate runs. -// Queued runs are cancelled by the API immediately; active runs are flagged -// for a safe interruption point. Polling keeps the visible status in sync. function StopRunsButton({ protocol, experimentId, @@ -577,6 +133,7 @@ function RunReplicateButton({ regenerationRequired, unboundFactors, bindingDiscrepancies, + row, }: { protocol: Protocol | undefined experimentId: string @@ -590,12 +147,24 @@ function RunReplicateButton({ regenerationRequired: boolean unboundFactors: string[] bindingDiscrepancies: FactorBindingDiscrepancy[] + row?: ExecutionSlot }) { const queryClient = useQueryClient() const [dialogOpen, setDialogOpen] = useState(false) + const [confirmedId, setConfirmedId] = useState() + const preview = useBatchPreview(experimentId, + { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined }, + { replicate_ids: row ? [row.replicate_id] : undefined, + row_indices: row?.row_index != null ? [row.row_index] : undefined, + rerun_slot_ids: hasCompletedRun && row ? [row.slot_id] : [] }, dialogOpen && !!row) const runMutation = useMutation({ - mutationFn: () => protocolsApi.run(protocol!.id, replicateLabel), - onSuccess: () => { + mutationFn: async () => { + if (row) return experimentsApi.confirmResultJob(experimentId, preview.data!.id) + await protocolsApi.run(protocol!.id, replicateLabel) + }, + onSuccess: job => { + if (job) setConfirmedId(job.id) + queryClient.invalidateQueries({ queryKey: executionKeys.root(experimentId) }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'replicates'] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'runs'] }) queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'runs'] }) @@ -611,7 +180,6 @@ function RunReplicateButton({ onSuccess: (published) => { queryClient.invalidateQueries({ queryKey: ['protocols', protocol!.id, 'revisions'] }) queryClient.setQueryData(protocolForExperimentQueryKey(experimentId), published) - runMutation.mutate() }, }) const runBlocked = !protocol || regenerationRequired || unboundFactors.length > 0 || bindingDiscrepancies.length > 0 @@ -641,6 +209,7 @@ function RunReplicateButton({ {actionLabel} + {dialogOpen && protocol && ( publishAndRunMutation.mutate()} + publishDisabled={publishAndRunMutation.isPending || runMutation.isPending} + publishLabel={row ? 'Publish & prepare preview' : undefined} confirmLabel={`${actionLabel} replicate`} + confirmDisabled={!!row && (preview.data?.status !== 'awaiting_confirmation' || !preview.data.progress.planned)} + additionalContent={row &&

    {preview.data?.status === 'awaiting_confirmation' ? `${preview.data.progress.planned ?? 0} executions will be queued.` : preview.data?.error ?? preview.error?.message ?? 'Preparing batch preview…'}

    } isConfirming={runMutation.isPending} confirmError={errorMessage} /> @@ -675,283 +248,44 @@ function RunReplicateButton({ ) } -// This view deliberately begins at the design's natural unit: a cell is one -// unique factor combination, regardless of how many independently-run -// replicates it contains. A cell expands in place to reveal those replicates, -// preserving the high-level overview instead of replacing it with a dialog. -export function RunsTab({ - experimentId, - designSpec, - protocol, - regenerationRequired, - unboundFactors, - onViewResult, - onViewRowResult, -}: { - experimentId: string - designSpec: Experiment['design_spec'] - protocol: Protocol | undefined - regenerationRequired: boolean - unboundFactors: string[] - onViewRowResult?: (rowId: string) => void - onViewResult: (replicateLabel: string) => void +export function RunsTab({ experimentId, designSpec, protocol, regenerationRequired, unboundFactors, onViewResult, onViewRowResult, onSelectResult }: { + experimentId: string; designSpec: Experiment['design_spec']; protocol: Protocol | undefined + regenerationRequired: boolean; unboundFactors: string[]; onViewResult: (label: string) => void + onViewRowResult?: (id: string) => void; onSelectResult?: (selection: ResultsSelection) => void }) { - const [expandedCells, setExpandedCells] = useState>(() => new Set()) - const replicatesQuery = useQuery({ - queryKey: ['experiments', experimentId, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experimentId), - }) - const trialsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'runs'], - queryFn: () => experimentsApi.listTrials(experimentId), - // Obsolescence is surfaced at the Cells and cell-header levels, not only - // inside an expanded replicate list, so load this while Runs is visible. - enabled: true, - refetchInterval: 3000, - }) - const resultsQuery = useQuery({ - queryKey: ['experiments', experimentId, 'run-results', protocol?.id, protocol?.published_revision_id], - queryFn: () => experimentsApi.getRunResults(experimentId, { protocol_id: protocol?.id }), - refetchInterval: 5000, - }) - - if (protocol && resultsQuery.data?.consumption_mode === 'per_row') { - return 0 || factorBindingDiscrepancies(designSpec, protocol.graph).length > 0} onInspect={onViewRowResult} /> - } - - if (protocol && resultsQuery.isLoading) return - if (protocol && resultsQuery.isError) return

    Could not load the published execution scope.

    - - if (replicatesQuery.isLoading) { - return ( -
    - - - -
    - ) - } - - if (replicatesQuery.isError || !replicatesQuery.data) { - return

    Could not load this experiment’s cells.

    - } - - const cells = groupReplicatesIntoCells(replicatesQuery.data).sort((a, b) => cellSortKey(a).localeCompare(cellSortKey(b))) - const bindingDiscrepancies = factorBindingDiscrepancies(designSpec, protocol?.graph) - const trialsByLabel = new Map((trialsQuery.data ?? []).map((trial) => [trial.replicate_label, trial])) - const cellResultsByLabel = new Map((resultsQuery.data?.cells ?? []).map((cell) => [cell.cell_label, cell])) - const replicateResultsByLabel = new Map((resultsQuery.data?.replicates ?? []).map((replicate) => [replicate.replicate_label, replicate])) - const obsoleteCells = cells.filter((cell) => cell.replicates.some((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete)) - const obsoleteReplicateCount = cells.reduce( - (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete).length, - 0, - ) - const truncatedCells = cells.filter((cell) => cell.replicates.some((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated)) - const truncatedReplicateCount = cells.reduce( - (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length, - 0, - ) - // Runs stays operational rather than becoming a second Results dashboard: - // this one compact line answers whether there is work in flight, while - // comparison metrics, spend, and outputs stay in the Results rail item. - const trials = [...trialsByLabel.values()] - const currentTrialCount = trials.filter((trial) => !trial.obsolete).length - const completedCount = trials.filter((trial) => trial.status === 'completed' && !trial.obsolete).length - const runningCount = trials.filter((trial) => trial.status === 'running' || trial.status === 'finalizing').length - const queuedCount = trials.filter((trial) => trial.status === 'queued').length - const failedCount = trials.filter((trial) => trial.status === 'failed' || trial.status === 'cancelled').length - const overviewUsage = resultsQuery.data ? usageSummary({ - cost_usd: resultsQuery.data.overview.total_cost_usd, - total_tokens: resultsQuery.data.overview.total_tokens, - duration_seconds: resultsQuery.data.overview.total_duration_seconds, - }) : [] - const isActiveTrial = (trial: Trial | undefined): trial is Trial => - !!trial?.run_id && isActiveTrialStatus(trial.status) - const activeExperimentRunIds = trials.filter(isActiveTrial).map((trial) => trial.run_id) - // Completion is the user-visible truth here. A result can be completed - // from persisted metrics even when it has no ProtocolRun provenance (for - // example, imported or notebook-scored results), so run_id is not a valid - // test for whether this scope has already been run. - const hasFinishedRun = (trial: Trial | undefined) => - !!trial && trial.status !== 'not_started' && !isActiveTrial(trial) - const experimentHasCompletedRun = cells.length > 0 && cells.every((cell) => - cell.replicates.every((replicate) => hasFinishedRun(trialsByLabel.get(replicate.replicate_label))), - ) - - function toggleCell(cellLabel: string) { - setExpandedCells((current) => { - const next = new Set(current) - if (next.has(cellLabel)) next.delete(cellLabel) - else next.add(cellLabel) - return next - }) - } - - if (cells.length === 0) { - return

    No cells yet — generate this experiment’s design first.

    - } + const discrepancies = factorBindingDiscrepancies(designSpec, protocol?.graph) + const scope = { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined } + // Same query (and cache entry) the browser below already polls. + const summary = useExecutionSummary(experimentId, scope, !!protocol) + return <> + {discrepancies.length > 0 &&

    Canvas and generated runs disagree

    {discrepancies.map(issue =>

    {issue.nodeLabel} · {issue.factorName}: {issue.reason}

    )}
    } + { + if (onSelectResult) onSelectResult(selection) + else if (selection.type === 'row') onViewRowResult?.(selection.rowResultId) + else if (selection.type === 'replicate') onViewResult(selection.replicateLabel) + }} + controls={<>} + cellActions={cell => <> (cell.counts?.[status] ?? 0) > 0)} />} + actions={row => } + /> + +} - return ( -
    - {bindingDiscrepancies.length > 0 && ( -
    - -
    -

    Canvas and generated runs disagree

    -

    - Cell runs would replace these canvas values with saved factor levels. Update the factor levels and regenerate the design, or remove the bindings before running. -

    -
      - {bindingDiscrepancies.map((issue) => ( -
    • - {issue.nodeLabel} · {issue.factorName}: {issue.reason} -
      • - ))} -
    -
    -
    - )} -
    -
    -

    Cells

    - {obsoleteReplicateCount > 0 && ( - - )} - {truncatedReplicateCount > 0 && ( - - )} - - - {cells.length} -
    -

    - {completedCount}/{currentTrialCount || (trials.length === 0 ? replicatesQuery.data.length : 0)} current complete - {runningCount > 0 && ` · ${runningCount} running`} - {queuedCount > 0 && ` · ${queuedCount} queued`} - {failedCount > 0 && ` · ${failedCount} failed`} -

    - {overviewUsage.length > 0 && ( -

    - Current usage - {overviewUsage.map((value) => {value})} -

    - )} -
    -
    - {cells.map((cell) => { - const entries = factorEntries(cell) - const expanded = expandedCells.has(cell.label) - const replicateListId = `cell-${cell.label}-replicates` - const obsoleteCount = cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete).length - const truncatedCount = cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length - const cellResult = cellResultsByLabel.get(cell.label) - const cellUsage = cellResult ? usageSummary(cellResult) : [] - const activeCellRunIds = cell.replicates - .map((replicate) => trialsByLabel.get(replicate.replicate_label)) - .filter(isActiveTrial) - .map((trial) => trial.run_id) - const cellHasCompletedRun = cell.replicates.every((replicate) => - hasFinishedRun(trialsByLabel.get(replicate.replicate_label)), - ) - return ( - displayFactorCondition(name, value, designSpec, cell.label))} - replicateCount={cell.replicates.length} expanded={expanded} listId={replicateListId} - onToggle={() => toggleCell(cell.label)} - progress={cellResult && <>{cellResult.current_completed_count}/{cellResult.replicate_count} current complete{cellUsage.map(value => {value})}} - badges={<> - {obsoleteCount > 0 && } - {truncatedCount > 0 && } - } - actions={<> - replicate.replicate_label)} - label={cellHasCompletedRun ? 'Re-run all replicates' : 'Run all replicates'} - dialogTitle={cellHasCompletedRun ? 'Re-run all replicates?' : 'Run all replicates?'} - compact - /> - - }> - {trialsQuery.isLoading ? ( -
    - - -
    - ) : ( -
      - {cell.replicates.map((replicate) => { - const trial = trialsByLabel.get(replicate.replicate_label) - const replicateResult = replicateResultsByLabel.get(replicate.replicate_label) - const replicateUsage = replicateResult ? usageSummary(replicateResult) : [] - const badge = trial - ? trial.obsolete - ? OBSOLETE_TRIAL_BADGE - : trial.truncated - ? TRUNCATED_TRIAL_BADGE - : null - : null - return ( - - {trial?.obsolete && ( - - )} - {trial?.truncated && !trial.obsolete && ( - - )} - } - detail={replicateUsage.length > 0 && replicateUsage.map(value => {value})} - status={badge ? {badge.label} : trial ? : Status unavailable} - onView={trial?.run_id ? () => onViewResult(replicate.replicate_label) : undefined} - actions={ - - } - error={trial?.error} - /> - ) - })} -
    - )} -     - ) - })} -
    -
    - ) +const ACTIVE_STATUSES = ['pending', 'running', 'finalizing'] + +// Only while something can actually be stopped, like StopRunsButton for a single +// replicate. Once clicked it stays up so the cancellation's progress is visible. +function StopAllButton({ protocol, experimentId, cellId, active }: { protocol?: Protocol; experimentId: string; cellId?: string; active: boolean }) { + const [jobId, setJobId] = useState() + const stop = useMutation({ mutationFn: () => experimentsApi.createResultJob(experimentId, + { protocol_id: protocol?.id, protocol_revision_id: protocol?.published_revision_id ?? undefined }, + { kind: 'cancel', cell_id: cellId, request_key: newRequestKey() }), onSuccess: job => setJobId(job.id) }) + if (!active && !jobId && !stop.isPending) return null + return <> + + {stop.error &&

    {stop.error.message}

    } } diff --git a/frontend/src/components/protocol/ScriptFactorDialog.test.tsx b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx new file mode 100644 index 00000000..287ac70a --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorDialog.test.tsx @@ -0,0 +1,84 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { ScriptFactorDialog } from './ScriptFactorDialog' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'script', type: 'script', data: { label: 'Summarize', config: { name: 'Scoring', language: 'python', code: 'print(1)' }, factor_bindings: { 'config.enabled': 'Script enabled' } } }, + ], + edges: [{ source: 'script', target: 'agent', targetHandle: 'tool' }], +} as unknown as ProtocolGraph + +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Script enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Script levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + +it.each(['Script levels', 'All scripts on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Script enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Script’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This script on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Script enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected scripts', () => { + render() + expect(screen.getByRole('button', { name: 'This script on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Script levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Script to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('blocks off treatments for a Tool Step that requires script_code', () => { + const required = { ...graph, nodes: [ + { id: 'step', type: 'tool_step', data: { label: 'Fit model', config: { arguments: { code: { source: 'script_code' } } } } }, + { ...graph.nodes[1], data: { ...graph.nodes[1].data, factor_bindings: {} } }, + ], edges: [{ source: 'script', target: 'step', targetHandle: 'tool' }] } as unknown as ProtocolGraph + render() + expect(screen.getByRole('status')).toHaveTextContent('cannot include off') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This script on/off', 'Script levels'])('blocks %s when an Agent already has an all-scripts factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { script_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { name: 'Scoring', language: 'python', code: 'print(1)' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) diff --git a/frontend/src/components/protocol/ScriptFactorDialog.tsx b/frontend/src/components/protocol/ScriptFactorDialog.tsx new file mode 100644 index 00000000..4e3ed05f --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorDialog.tsx @@ -0,0 +1,118 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { connectedScripts, reconcileScriptFactor, requiredScriptIssues, scriptFactorConflict, scriptFactorModes as modes, SCRIPT_FACTOR_PATH, isIndividualScriptMode, scriptFactorPath, type ScriptFactorMode as Mode } from '@/lib/scriptFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +export function ScriptFactorDialog({ scriptNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { + scriptNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const script = graph.nodes.find((node) => node.id === scriptNodeId)! + const agents = factorRecipients(graph, scriptNodeId, 'script') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[SCRIPT_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, SCRIPT_FACTOR_PATH) ?? existingFor(scriptNodeId, 'config.enabled') + const initial = initialFactor && !isIndividualScriptMode(initialFactor.level_type as Mode) && initialFactor.level_type ? reconcileScriptFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = isIndividualScriptMode(nextMode) ? scriptNodeId : nextAgentId + const existing = existingFor(ownerId, scriptFactorPath(nextMode)) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level, index) => nextMode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Script'), nextMode === 'boolean' ? 'Enabled' : 'Scripts', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return isIndividualScriptMode(nextMode) ? factor : reconcileScriptFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`) } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = isIndividualScriptMode(mode) ? scriptNodeId : agentId + const current = existingFor(ownerId, scriptFactorPath(mode)) + const affected = isIndividualScriptMode(mode) ? [script] : connectedScripts(graph, agentId) + const unavailable = (value: Mode) => !isIndividualScriptMode(value) && !agentId + ? `${modes[value]} is unavailable because this Script is not connected to an Agent. Connect this Script to an Agent first.` + : scriptFactorConflict(graph, factors, scriptNodeId, value, agentId) + const previewGraph = { ...graph, nodes: graph.nodes.map((node) => node.id === ownerId ? { ...node, data: { ...node.data, factor_bindings: { ...node.data.factor_bindings, [scriptFactorPath(mode)]: draft.name } } } : node) } + const requiredIssues = requiredScriptIssues(previewGraph, [...factors.filter((factor) => factor.name !== current?.name), draft]) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the script factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Script factor + Choose how scripts vary across experimental cells. + +
    + {agents.length > 0 &&

    Applies to: {factorScopeLabel(graph, scriptNodeId, 'script')}

    } +
    {(Object.keys(modes) as Mode[]).map((value) => )}
    + {blockedReason ?

    {blockedReason}

    : <> +

    {mode === 'boolean' ? 'Enable or disable this Script node.' : mode === 'script_selection' ? 'Each cell receives exactly one connected script.' : 'Each cell enables every connected script, including deactivated scripts, or disables them all.'}

    +
    setDraft({ ...draft, name: event.target.value })} />
    + {!isIndividualScriptMode(mode) && affected.length < (mode === 'script_toggle' ? 1 : 2) &&

    Connect at least {mode === 'script_toggle' ? 'one script' : 'two different scripts'} before generating cells.

    } +
    {draft.levels.map((level, index) =>
    +
    +

    {mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'script_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable script')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

    + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + {mode !== 'boolean' && <>} +
    )}
    +

    Shared factors require matching Script connections. Changes require design review and regeneration.

    + {requiredIssues.map((issue) =>

    {issue}

    )} + } + {current && } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/ScriptFactorEditor.tsx b/frontend/src/components/protocol/ScriptFactorEditor.tsx new file mode 100644 index 00000000..b16a46b9 --- /dev/null +++ b/frontend/src/components/protocol/ScriptFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function ScriptFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the script factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'script_toggle' ? 'All scripts on/off' : 'Scripts as levels'} + {factor.level_type === 'script_toggle' ? 'Each cell receives all connected scripts or none.' : 'Each cell receives exactly one connected script.'} Add or disconnect Script nodes on the canvas to change the levels. + +
    + + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

    Incomplete: connect at least two different scripts before generating cells.

    } +
    + {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const script = graph?.nodes.find((node) => node.id === id) + return
    +
    +

    {factor.level_type === 'script_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : script?.data.label ?? labels[index] ?? 'Unavailable script'}{index === 0 ? ' · default test selection' : ''}

    + {factor.level_type !== 'script_toggle' && !!graph && !script &&

    Unavailable: disconnect or restore this script.

    } + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + + +
    + })} +
    +

    Individual script switches are controlled by this connector. Changes require design review and regeneration.

    + {onRemove &&

    Removing this factor enables all connected scripts and restores their individual controls.

    } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/ScriptNodeInspector.tsx b/frontend/src/components/protocol/ScriptNodeInspector.tsx index 5257f4be..990f4ffa 100644 --- a/frontend/src/components/protocol/ScriptNodeInspector.tsx +++ b/frontend/src/components/protocol/ScriptNodeInspector.tsx @@ -1,11 +1,13 @@ -import { Code2 } from 'lucide-react' +import { Code2, Power, Split, Trash2 } from 'lucide-react' import { nodeAccent } from '@/lib/nodeAccent' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' +import { Button } from '@/components/ui/button' +import { Switch } from '@/components/ui/switch' import { NodeInspectorDialog } from './NodeInspectorDialog' import { PythonCodeEditor } from './PythonCodeEditor' +import { useProtocolCanvasActions } from './ProtocolCanvasContext' import type { ScriptNodeConfig, ScriptNodeData, ProtocolNode } from '@/types/protocols' const ACCENT = nodeAccent('script') @@ -13,28 +15,22 @@ const ACCENT = nodeAccent('script') // Same floating-dialog shell as every other node inspector. Python-only for // v1 (see ScriptNodeData's own comment in types/protocols.ts) -- "Language" // is a fixed label, not a picker, so there's nothing to configure there yet. -// The whole node is also factor-bindable (bindableFields.ts's 'script_config' -// kind) -- comparing two hand-written scoring scripts as an experimental -// factor is a direct use of the same whole-node-config mechanism model_config/ -// tool_config/pattern already have. +// Script factor creation lives in the hover toolbar. The inspector edits the +// base code and exposes existing factor links and connector-controlled availability. export function ScriptNodeInspector({ node, - experimentId, - factorNodeLabel, + connectorFactorName, onChange, onDelete, onClose, }: { node: (ProtocolNode & { data: ScriptNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's - // agentTracedLabel) -- distinct from data.label, which is this node's own - // plain label shown in the header title. - factorNodeLabel: string + connectorFactorName?: string onChange: (nodeId: string, data: ScriptNodeData) => void onDelete: (nodeId: string) => void onClose: () => void }) { + const { requestEditFactor } = useProtocolCanvasActions() if (!node) return null const data = node.data const config = data.config @@ -44,10 +40,6 @@ export function ScriptNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -70,6 +62,24 @@ export function ScriptNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > +
    +

    + Make factor has moved to the node toolbar. Hover over the Script node and click the

    +
    + {connectorFactorName ?

    Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

    {Object.entries(bindings).map(([path, name]) =>

    Individual factor {name} conflicts with this connector.

    Its declaration remains in Design until you remove or rebind it.

    )}
    :
    +

    Off: the wired agent never sees this script at all.

    {bindings['config.enabled'] &&

    Factor: {bindings['config.enabled']}

    }
    + patchConfig({ enabled: checked })} /> +
    } +
    @@ -91,27 +101,7 @@ export function ScriptNodeInspector({ />
    - bindFactor('config', name)} - onUnbind={() => unbindFactor('config')} - > - {(trigger) => ( -
    - - patchConfig({ code })} rows={16} resizable /> -
    - )} -     +
    {bindings.config && } patchConfig({ code })} height="max(12rem, calc(100vh - 22rem))" />
    ) diff --git a/frontend/src/components/protocol/ScriptTestSelectors.tsx b/frontend/src/components/protocol/ScriptTestSelectors.tsx new file mode 100644 index 00000000..6b3d776e --- /dev/null +++ b/frontend/src/components/protocol/ScriptTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedScripts, scriptId, SCRIPT_FACTOR_PATH } from '@/lib/scriptFactors' +import type { useScriptTestSelection } from './useScriptTestSelection' + +export function ScriptTestSelectors({ selection }: { selection: ReturnType }) { + return
    {selection.owners.map((node) => { + const options = connectedScripts(selection.graph!, node.id) + if (node.data.script_factor_mode === 'script_toggle') return
    +

    {node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] || node.data.label}: scripts for this test

    + +
    + return
    +

    {node.data.factor_bindings?.[SCRIPT_FACTOR_PATH] || node.data.label}: script for this test

    + +
    + })}{selection.error &&

    {selection.error}

    }
    +} diff --git a/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx b/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx index 876d8a9a..65cb12cf 100644 --- a/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx +++ b/frontend/src/components/protocol/SingleAgentBaselinePatternNodeInspector.tsx @@ -1,9 +1,8 @@ -import { ArrowRight } from 'lucide-react' +import { ArrowRight, Split, RefreshCw } from 'lucide-react' import { nodeAccent } from '@/lib/nodeAccent' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { Switch } from '@/components/ui/switch' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { SingleAgentBaselinePatternConfig, SingleAgentBaselinePatternNodeData, ProtocolNode } from '@/types/protocols' @@ -20,17 +19,10 @@ const ACCENT = nodeAccent('pattern_single_agent_baseline') // (the node's own canvas hover toolbar), never a bare delete. export function SingleAgentBaselinePatternNodeInspector({ node, - experimentId, - factorNodeLabel, onChange, onClose, }: { node: (ProtocolNode & { data: SingleAgentBaselinePatternNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, which is this node's own plain label shown - // in the header title. - factorNodeLabel: string onChange: (nodeId: string, data: SingleAgentBaselinePatternNodeData) => void onClose: () => void }) { @@ -38,22 +30,11 @@ export function SingleAgentBaselinePatternNodeInspector({ if (!node) return null const data = node.data const config = data.config - const bindings = data.factor_bindings ?? {} function patchConfig(patch: Partial) { onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - - function unbindFactor(fieldPath: string) { - const next = { ...bindings } - delete next[fieldPath] - onChange(node!.id, { ...data, factor_bindings: next }) - } - return ( - bindFactor('config.max_iterations', name)} - onUnbind={() => unbindFactor('config.max_iterations')} - > - {(trigger) => ( -
    - - patchConfig({ max_iterations: Number(e.target.value) })} - /> -
    - )} -     +

    Make factor has moved to the node toolbar. Hover over the Pattern node and click the

    +
    + + patchConfig({ max_iterations: Number(e.target.value) })} + /> +
    - bindFactor('config.stop_on_first_success', name)} - onUnbind={() => unbindFactor('config.stop_on_first_success')} - > - {(trigger) => ( -
    -
    - -

    Off: keeps looping for the full iteration budget even after a successful pass.

    -
    - patchConfig({ stop_on_first_success: checked })} - /> -
    - )} -     +
    +
    + +

    Off: keeps looping for the full iteration budget even after a successful pass.

    +
    + patchConfig({ stop_on_first_success: checked })} + /> +
    ) } diff --git a/frontend/src/components/protocol/SkillFactorDialog.test.tsx b/frontend/src/components/protocol/SkillFactorDialog.test.tsx new file mode 100644 index 00000000..177a2e62 --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorDialog.test.tsx @@ -0,0 +1,115 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { SkillFactorDialog } from './SkillFactorDialog' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'skill', type: 'skill', data: { label: 'Summarize', config: { skill_id: 'a' }, factor_bindings: { 'config.enabled': 'Skill enabled' } } }, + ], + edges: [{ source: 'skill', target: 'agent', targetHandle: 'skill' }], +} as unknown as ProtocolGraph + +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Skill enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Skill levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + +it.each(['Skill levels', 'All skills on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Skill enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Skill’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This skill on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Skill enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected skills', () => { + render() + expect(screen.getByRole('button', { name: 'This skill on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Skill levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Skill to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This skill on/off', 'Skill levels'])('blocks %s when an Agent already has an all-skills factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { skill_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { skill_id: 'a' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('saves edits to an existing individual factor without removing other declarations', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Renamed' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Renamed', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'skill', 'Skill enabled')) +}) + +it('opens the existing group factor when a shared skill has multiple connected agents', () => { + const sharedGraph = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'First agent' } }, + { id: 'second', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Second agent', skill_selection: ['a'], skill_factor_mode: 'skill_toggle', factor_bindings: { skill_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { skill_id: 'a' } } }, + ], + edges: [...graph.edges, { id: 'second-edge', source: 'skill', target: 'second', targetHandle: 'skill' }], + } as ProtocolGraph + render() + expect(screen.queryByRole('combobox')).not.toBeInTheDocument() + expect(screen.getByText('Applies to: First agent, Second agent')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'All skills on/off' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByLabelText('Factor name')).toHaveValue('Group') +}) + +it('blocks an incomplete shared group while keeping its factor removable', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const sharedGraph = { + nodes: [ + ...['one', 'two'].map((id) => ({ id, type: 'agent', data: { label: id, skill_selection: ['a'], skill_factor_mode: 'skill_selection', factor_bindings: { skill_selection: 'Shared' } } })), + ...['a', 'b'].map((id) => ({ id, type: 'skill', data: { label: id, config: { skill_id: id } } })), + ], + edges: [{ source: 'a', target: 'one', targetHandle: 'skill' }, { source: 'a', target: 'two', targetHandle: 'skill' }, { source: 'b', target: 'one', targetHandle: 'skill' }], + } as unknown as ProtocolGraph + render() + expect(screen.getByRole('status')).toHaveTextContent('exactly the same Agents') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Shared')) +}) diff --git a/frontend/src/components/protocol/SkillFactorDialog.tsx b/frontend/src/components/protocol/SkillFactorDialog.tsx new file mode 100644 index 00000000..95c9d1e9 --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorDialog.tsx @@ -0,0 +1,115 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { connectedSkills, reconcileSkillFactor, skillFactorConflict, skillFactorModes as modes, SKILL_FACTOR_PATH, type SkillFactorMode as Mode } from '@/lib/skillFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +export function SkillFactorDialog({ skillNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { + skillNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const skill = graph.nodes.find((node) => node.id === skillNodeId)! + const agents = factorRecipients(graph, skillNodeId, 'skill') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[SKILL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, SKILL_FACTOR_PATH) ?? existingFor(skillNodeId, 'config.enabled') + const initial = initialFactor && initialFactor.level_type !== 'boolean' && initialFactor.level_type ? reconcileSkillFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = nextMode === 'boolean' ? skillNodeId : nextAgentId + const existing = existingFor(ownerId, nextMode === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level) => level ? 'Enabled' : 'Disabled'), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Skill'), nextMode === 'boolean' ? 'Enabled' : 'Skills', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return nextMode === 'boolean' ? factor : reconcileSkillFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level) => level ? 'Enabled' : 'Disabled') } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = mode === 'boolean' ? skillNodeId : agentId + const current = existingFor(ownerId, mode === 'boolean' ? 'config.enabled' : SKILL_FACTOR_PATH) + const affected = mode === 'boolean' ? [skill] : connectedSkills(graph, agentId) + const unavailable = (value: Mode) => value !== 'boolean' && !agentId + ? `${modes[value]} is unavailable because this Skill is not connected to an Agent. Connect this Skill to an Agent first.` + : skillFactorConflict(graph, factors, skillNodeId, value, agentId) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the skill factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Skill factor + Choose how skills vary across experimental cells. + +
    + {agents.length > 0 &&

    Applies to: {factorScopeLabel(graph, skillNodeId, 'skill')}

    } +
    {(Object.keys(modes) as Mode[]).map((value) => )}
    + {blockedReason ?

    {blockedReason}

    : <> +

    {mode === 'boolean' ? 'Enable or disable this Skill node.' : mode === 'skill_selection' ? 'Each cell receives exactly one connected skill.' : 'Each cell enables every connected skill, including deactivated skills, or disables them all.'}

    +
    setDraft({ ...draft, name: event.target.value })} />
    + {mode !== 'boolean' && affected.length < (mode === 'skill_toggle' ? 1 : 2) &&

    Connect at least {mode === 'skill_toggle' ? 'one skill' : 'two different skills'} before generating cells.

    } +
    {draft.levels.map((level, index) =>
    +
    +

    {mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'skill_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => (node.data.config as { skill_id?: string })?.skill_id === (level as string[])[0])?.data.label ?? 'Unavailable skill')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

    + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + {mode !== 'boolean' && <>} +
    )}
    +

    Shared factors require matching Skill connections. Changes require design review and regeneration.

    + } + {current && } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/SkillFactorEditor.tsx b/frontend/src/components/protocol/SkillFactorEditor.tsx new file mode 100644 index 00000000..b5f0fea1 --- /dev/null +++ b/frontend/src/components/protocol/SkillFactorEditor.tsx @@ -0,0 +1,76 @@ +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { useQuery } from '@tanstack/react-query' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { skillsApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +export function SkillFactorEditor({ factor, open, onOpenChange, onSave, onRemove }: { + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const skills = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list() }) + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the skill factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'skill_toggle' ? 'All skills on/off' : 'Skills as levels'} + {factor.level_type === 'skill_toggle' ? 'Each cell receives all connected skills or none.' : 'Each cell receives exactly one connected skill.'} Add or disconnect Skill nodes on the canvas to change the levels. + +
    + + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

    Incomplete: connect at least two different skills before generating cells.

    } +
    + {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const skill = skills.data?.find((skill) => skill.id === id) + return
    +
    +

    {factor.level_type === 'skill_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : skill?.name ?? labels[index] ?? 'Unavailable skill'}{index === 0 ? ' · default test selection' : ''}

    + {factor.level_type !== 'skill_toggle' && skills.isSuccess && !skill &&

    Unavailable: disconnect or restore this skill.

    } + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + + +
    + })} +
    + {skills.isError &&

    Could not load the skill library.

    } +

    Individual skill switches are controlled by this connector. Changes require design review and regeneration.

    + {onRemove &&

    Removing this factor enables all connected skills and restores their individual controls.

    } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/SkillNodeInspector.tsx b/frontend/src/components/protocol/SkillNodeInspector.tsx index 2d9939d2..796a1998 100644 --- a/frontend/src/components/protocol/SkillNodeInspector.tsx +++ b/frontend/src/components/protocol/SkillNodeInspector.tsx @@ -1,13 +1,13 @@ import { nodeAccent } from '@/lib/nodeAccent' import { useQuery } from '@tanstack/react-query' -import { ScrollText } from 'lucide-react' +import { Power, ScrollText, Split, Trash2 } from 'lucide-react' import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' import { Skeleton } from '@/components/ui/skeleton' import { Switch } from '@/components/ui/switch' import { skillsApi } from '@/api/client' import { EditableNodeTitle } from './EditableNodeTitle' -import { FactorBindableField } from './FactorBindableField' import { NodeInspectorDialog } from './NodeInspectorDialog' import type { ProtocolNode, SkillNodeData } from '@/types/protocols' @@ -27,20 +27,16 @@ const ACCENT = nodeAccent('skill') // across many protocols, so an inline edit would silently change all of them. export function SkillNodeInspector({ node, - experimentId, - factorNodeLabel, onChange, onDelete, onClose, + connectorFactorName, }: { node: (ProtocolNode & { data: SkillNodeData }) | null - experimentId: string | null - // The agent-traced display label (see bindableFields.ts's agentTracedLabel) - // -- distinct from data.label, this node's own plain header title. - factorNodeLabel: string onChange: (nodeId: string, data: SkillNodeData) => void onDelete: (nodeId: string) => void onClose: () => void + connectorFactorName?: string }) { const skillsQuery = useQuery({ queryKey: ['skills'], queryFn: () => skillsApi.list() }) @@ -54,10 +50,6 @@ export function SkillNodeInspector({ onChange(node!.id, { ...data, config: { ...config, ...patch } }) } - function bindFactor(fieldPath: string, factorName: string) { - onChange(node!.id, { ...data, factor_bindings: { ...bindings, [fieldPath]: factorName } }) - } - function unbindFactor(fieldPath: string) { const next = { ...bindings } delete next[fieldPath] @@ -80,29 +72,23 @@ export function SkillNodeInspector({ onDelete={() => onDelete(node.id)} onClose={onClose} > - bindFactor('config.enabled', name)} - onUnbind={() => unbindFactor('config.enabled')} - > - {(trigger) => ( -
    -
    - -

    Off: the wired agent never sees this skill at all.

    -
    - patchConfig({ enabled: checked })} /> -
    - )} -     +
    +

    + Make factor has moved to the node toolbar. Hover over the Skill node and click the

    +
    + {connectorFactorName ?

    Availability is controlled by connector factor {connectorFactorName}. Remove it to restore individual controls.

    {Object.entries(bindings).map(([path, name]) =>

    Individual factor {name} conflicts with this connector.

    Its declaration remains in Design until you remove or rebind it.

    )}
    :
    +

    Off: the wired agent never sees this skill at all.

    {bindings['config.enabled'] &&

    Factor: {bindings['config.enabled']}

    }
    + patchConfig({ enabled: checked })} /> +
    }
    diff --git a/frontend/src/components/protocol/SkillTestSelectors.tsx b/frontend/src/components/protocol/SkillTestSelectors.tsx new file mode 100644 index 00000000..c7680894 --- /dev/null +++ b/frontend/src/components/protocol/SkillTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedSkills, skillId, SKILL_FACTOR_PATH } from '@/lib/skillFactors' +import type { useSkillTestSelection } from './useSkillTestSelection' + +export function SkillTestSelectors({ selection }: { selection: ReturnType }) { + return
    {selection.owners.map((node) => { + const options = connectedSkills(selection.graph!, node.id) + if (node.data.skill_factor_mode === 'skill_toggle') return
    +

    {node.data.factor_bindings?.[SKILL_FACTOR_PATH] || node.data.label}: skills for this test

    + +
    + return
    +

    {node.data.factor_bindings?.[SKILL_FACTOR_PATH] || node.data.label}: skill for this test

    + +
    + })}{selection.error &&

    {selection.error}

    }
    +} diff --git a/frontend/src/components/protocol/StopRowRuns.tsx b/frontend/src/components/protocol/StopRowRuns.tsx deleted file mode 100644 index 27339ec9..00000000 --- a/frontend/src/components/protocol/StopRowRuns.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { useMutation, useQueryClient } from '@tanstack/react-query' -import { Square } from 'lucide-react' -import { protocolsApi } from '@/api/client' -import { Button } from '@/components/ui/button' -import type { Protocol } from '@/types/protocols' - -export function StopRowRuns({ protocol, runIds }: { protocol: Protocol; runIds: string[] }) { - const client = useQueryClient() - const stop = useMutation({ - mutationFn: async () => { - const outcomes = await Promise.allSettled([...new Set(runIds)].map(id => protocolsApi.cancelRun(protocol.id, id))) - if (outcomes.some(result => result.status === 'rejected')) throw new Error('Some runs could not be stopped. Try again.') - }, - onSettled: () => client.invalidateQueries({ queryKey: ['experiments', protocol.experiment_id] }), - }) - if (!runIds.length) return null - return
    {stop.error &&

    {stop.error.message}

    }
    -} diff --git a/frontend/src/components/protocol/SubAgentFactorDialog.test.tsx b/frontend/src/components/protocol/SubAgentFactorDialog.test.tsx new file mode 100644 index 00000000..1c7a8669 --- /dev/null +++ b/frontend/src/components/protocol/SubAgentFactorDialog.test.tsx @@ -0,0 +1,35 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { SubAgentFactorDialog } from './SubAgentFactorDialog' + +const graph = { + nodes: [ + { id: 'parent', type: 'agent', data: { label: 'Writer' } }, + { id: 'a', type: 'sub_agent', data: { label: 'Researcher', active: false, factor_bindings: { 'config.prompt': 'Prompt' } } }, + { id: 'b', type: 'sub_agent', data: { label: 'Reviewer', active: false } }, + ], + edges: ['a', 'b'].map((source) => ({ source, target: 'parent', targetHandle: 'sub_agents' })), +} as unknown as ProtocolGraph + +it.each(['Sub-Agent levels', 'All sub-agents on/off'])('creates %s with disabled workers while allowing prompt factors', async (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.queryByRole('status')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ + level_type: mode === 'Sub-Agent levels' ? 'sub_agent_selection' : 'sub_agent_toggle', + levels: mode === 'Sub-Agent levels' ? [['a'], ['b']] : [['a', 'b'], []], + }), 'parent', undefined)) +}) + +it('blocks parent factors only for individual on/off bindings', () => { + const conflicting = { ...graph, nodes: graph.nodes.map((node) => node.id === 'a' ? { ...node, data: { ...node.data, factor_bindings: { active: 'Enabled' } } } : node) } + render() + fireEvent.click(screen.getByRole('button', { name: 'Sub-Agent levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Enabled') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + fireEvent.click(screen.getByRole('button', { name: 'This sub-agent on/off' })) + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) diff --git a/frontend/src/components/protocol/SubAgentFactorDialog.tsx b/frontend/src/components/protocol/SubAgentFactorDialog.tsx new file mode 100644 index 00000000..c706d376 --- /dev/null +++ b/frontend/src/components/protocol/SubAgentFactorDialog.tsx @@ -0,0 +1,115 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' +import { useState } from 'react' +import { ArrowDown, ArrowUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { connectedSubAgents, reconcileSubAgentFactor, subAgentFactorConflict, subAgentFactorModes as modes, SUB_AGENT_FACTOR_PATH, type SubAgentFactorMode as Mode } from '@/lib/subAgentFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { computeFactorName } from './factorLevels' + +export function SubAgentFactorDialog({ subAgentNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { + subAgentNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const subAgent = graph.nodes.find((node) => node.id === subAgentNodeId)! + const agents = factorRecipients(graph, subAgentNodeId, 'sub_agent') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, SUB_AGENT_FACTOR_PATH) ?? existingFor(subAgentNodeId, 'active') + const initial = initialFactor && initialFactor.level_type !== 'boolean' && initialFactor.level_type ? reconcileSubAgentFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = nextMode === 'boolean' ? subAgentNodeId : nextAgentId + const existing = existingFor(ownerId, nextMode === 'boolean' ? 'active' : SUB_AGENT_FACTOR_PATH) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level) => level ? 'Enabled' : 'Disabled'), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Sub-Agent'), nextMode === 'boolean' ? 'Enabled' : 'Sub-Agents', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : undefined, + } + return nextMode === 'boolean' ? factor : reconcileSubAgentFactor(factor, graph, ownerId) + } + const [draft, setDraft] = useState(() => { + const factor = initialMode ? seed(mode, agentId) : initial ?? seed(mode, agentId) + return { ...factor, level_type: factor.level_type ?? 'boolean', level_labels: factor.level_labels ?? factor.levels.map((level) => level ? 'Enabled' : 'Disabled') } + }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = mode === 'boolean' ? subAgentNodeId : agentId + const current = existingFor(ownerId, mode === 'boolean' ? 'active' : SUB_AGENT_FACTOR_PATH) + const affected = mode === 'boolean' ? [subAgent] : connectedSubAgents(graph, agentId) + const unavailable = (value: Mode) => value !== 'boolean' && !agentId + ? `${modes[value]} is unavailable because this Sub-Agent is not connected to an Agent. Connect this Sub-Agent to an Agent first.` + : subAgentFactorConflict(graph, factors, subAgentNodeId, value, agentId) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setMode(nextMode) + setDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the sub-agent factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Sub-Agent factor + Choose how sub-agents vary across experimental cells. + +
    + {agents.length > 0 &&

    Applies to: {factorScopeLabel(graph, subAgentNodeId, 'sub_agent')}

    } +
    {(Object.keys(modes) as Mode[]).map((value) => )}
    + {blockedReason ?

    {blockedReason}

    : <> +

    {mode === 'boolean' ? 'Enable or disable this Sub-Agent node.' : mode === 'sub_agent_selection' ? 'Each cell receives exactly one connected sub-agent.' : 'Each cell enables every connected sub-agent, including deactivated sub-agents, or disables them all.'}

    +
    setDraft({ ...draft, name: event.target.value })} />
    + {mode !== 'boolean' && affected.length < (mode === 'sub_agent_toggle' ? 1 : 2) &&

    Connect at least {mode === 'sub_agent_toggle' ? 'one sub-agent' : 'two different sub-agents'} before generating cells.

    } +
    {draft.levels.map((level, index) =>
    +
    +

    {mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'sub_agent_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable sub-agent')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

    + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + {mode !== 'boolean' && <>} +
    )}
    +

    Shared factors require matching Sub-Agent connections. Changes require design review and regeneration.

    + } + {current && } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/SubAgentFactorEditor.tsx b/frontend/src/components/protocol/SubAgentFactorEditor.tsx new file mode 100644 index 00000000..a8fb29bc --- /dev/null +++ b/frontend/src/components/protocol/SubAgentFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function SubAgentFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the sub-agent factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'sub_agent_toggle' ? 'All sub-agents on/off' : 'Sub-Agents as levels'} + {factor.level_type === 'sub_agent_toggle' ? 'Each cell receives all connected sub-agents or none.' : 'Each cell receives exactly one connected sub-agent.'} Add or disconnect Sub-Agent nodes on the canvas to change the levels. + +
    + + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

    Incomplete: connect at least two different sub-agents before generating cells.

    } +
    + {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const subAgent = graph?.nodes.find((node) => node.id === id) + return
    +
    +

    {factor.level_type === 'sub_agent_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : subAgent?.data.label ?? labels[index] ?? 'Unavailable sub-agent'}{index === 0 ? ' · default test selection' : ''}

    + {factor.level_type !== 'sub_agent_toggle' && !!graph && !subAgent &&

    Unavailable: disconnect or restore this sub-agent.

    } + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + + +
    + })} +
    +

    Individual sub-agent switches are controlled by this connector. Changes require design review and regeneration.

    + {onRemove &&

    Removing this factor enables all connected sub-agents and restores their individual controls.

    } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/SubAgentTestSelectors.tsx b/frontend/src/components/protocol/SubAgentTestSelectors.tsx new file mode 100644 index 00000000..49961e73 --- /dev/null +++ b/frontend/src/components/protocol/SubAgentTestSelectors.tsx @@ -0,0 +1,17 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedSubAgents, SUB_AGENT_FACTOR_PATH } from '@/lib/subAgentFactors' +import type { useSubAgentTestSelection } from './useSubAgentTestSelection' + +export function SubAgentTestSelectors({ selection }: { selection: ReturnType }) { + return
    {selection.owners.map((node) => { + const options = connectedSubAgents(selection.graph!, node.id) + const toggle = node.data.sub_agent_factor_mode === 'sub_agent_toggle' + return
    +

    {node.data.factor_bindings?.[SUB_AGENT_FACTOR_PATH] || node.data.label}: {toggle ? 'sub-agents' : 'sub-agent'} for this test

    + +
    + })}{selection.error &&

    {selection.error}

    }
    +} diff --git a/frontend/src/components/protocol/TestRunResults.test.tsx b/frontend/src/components/protocol/TestRunResults.test.tsx index 63299072..30fb1d26 100644 --- a/frontend/src/components/protocol/TestRunResults.test.tsx +++ b/frontend/src/components/protocol/TestRunResults.test.tsx @@ -37,6 +37,23 @@ function testRun(overrides: Partial = {}): TestRun { } describe('TestRunResults', () => { + it('distinguishes bypassed nodes from completed execution', () => { + render() + + const progress = screen.getByRole('region', { name: 'Node progress' }) + expect(within(progress).getByText('Bypassed')).toBeInTheDocument() + expect(within(progress).getByText('Done')).toBeInTheDocument() + }) + it.each([ ['completed', 'Completed'], ['failed', 'Failed'], diff --git a/frontend/src/components/protocol/TestRunResults.tsx b/frontend/src/components/protocol/TestRunResults.tsx index 0d899c76..657cc139 100644 --- a/frontend/src/components/protocol/TestRunResults.tsx +++ b/frontend/src/components/protocol/TestRunResults.tsx @@ -188,7 +188,7 @@ export function TestRunResults({

    {group.label}

    {group.entries.map(([nodeId, nodeRun]) => { - const badge = nodeRunBadge(nodeRun.status, Boolean(nodeRun.truncation)) + const badge = nodeRunBadge(nodeRun.status, Boolean(nodeRun.truncation), nodeRun.bypassed) const label = nodeTypes.get(nodeId) === 'sub_agent' && nodeRun.status === 'skipped' ? (running ? 'Available' : 'Not invoked') : badge?.label diff --git a/frontend/src/components/protocol/ToolFactorDialog.test.tsx b/frontend/src/components/protocol/ToolFactorDialog.test.tsx new file mode 100644 index 00000000..6c262c98 --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorDialog.test.tsx @@ -0,0 +1,130 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import { expect, it, vi } from 'vitest' +import type { ProtocolGraph } from '@/types/protocols' +import { ToolFactorDialog } from './ToolFactorDialog' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { mcpServersApi } from '@/api/client' + +const graph = { + nodes: [ + { id: 'agent', type: 'agent', data: { label: 'Writer' } }, + { id: 'tool', type: 'mcp_tool', data: { label: 'Summarize', config: { server_id: 'a' }, factor_bindings: { 'config.enabled': 'Tool enabled' } } }, + ], + edges: [{ source: 'tool', target: 'agent', targetHandle: 'tool' }], +} as unknown as ProtocolGraph + +it('refreshes untouched levels after a refetch and preserves label pairing on save', async () => { + const save = vi.fn().mockResolvedValue(undefined) + const props = { toolNodeId: 'tool', graph, onClose: vi.fn(), onSave: save, onRemove: vi.fn() } + const factor = { name: 'Tool enabled', level_type: 'boolean' as const, levels: [true, false], level_labels: ['on', 'off'] } + const refreshed = { ...factor, levels: [false, true], level_labels: ['off', 'on'] } + const { rerender } = render() + rerender() + expect(screen.getByLabelText('Level 1 label')).toHaveValue('off') + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(save).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText('Level 1 label'), { target: { value: 'disabled' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledExactlyOnceWith({ ...refreshed, level_labels: ['disabled', 'on'] }, 'tool', 'Tool enabled')) +}) + +it('keeps the dialog open after removal and permits choosing another factor type', async () => { + const remove = vi.fn().mockResolvedValue(undefined) + const close = vi.fn() + const { rerender } = render() + fireEvent.click(screen.getByRole('button', { name: 'Remove factor' })) + await waitFor(() => expect(remove).toHaveBeenCalledWith('Tool enabled')) + await waitFor(() => expect(screen.getByRole('button', { name: 'Cancel' })).toBeEnabled()) + expect(close).not.toHaveBeenCalled() + const updated = { ...graph, nodes: graph.nodes.map((node) => ({ ...node, data: { ...node.data, factor_bindings: {} } })) } + rerender() + expect(screen.queryByRole('button', { name: 'Remove factor' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Tool levels' })) + expect(screen.getByLabelText('Factor name')).toBeEnabled() + expect(screen.queryByRole('status')).not.toBeInTheDocument() +}) + +it.each(['Tool levels', 'All tools on/off'])('explains why %s conflicts with individual bindings without offering replacement', (mode) => { + const save = vi.fn().mockResolvedValue(undefined) + render() + const option = screen.getByRole('button', { name: mode }) + expect(option).toHaveClass('bg-secondary', 'text-secondary-foreground', 'border-dashed') + expect(option).not.toHaveClass('opacity-50', 'text-muted-foreground') + fireEvent.click(option) + expect(option).toHaveClass('bg-primary', 'text-primary-foreground') + expect(option).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() + expect(screen.getByRole('status')).toHaveTextContent('Tool enabled') + expect(screen.getByRole('status')).toHaveTextContent('remove those bindings in each Tool’s factor dialog first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + expect(save).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'This tool on/off' })) + expect(screen.getByLabelText('Factor name')).toHaveValue('Tool enabled') + expect(screen.getByRole('button', { name: 'Remove factor' })).toBeEnabled() +}) + +it('only offers individual factors for disconnected tools', () => { + render() + expect(screen.getByRole('button', { name: 'This tool on/off' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.click(screen.getByRole('button', { name: 'Tool levels' })) + expect(screen.getByRole('status')).toHaveTextContent('Connect this Tool to an Agent first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it.each(['This tool on/off', 'Tool levels', 'Tools allowed'])('blocks %s when an Agent already has an all-tools factor', (mode) => { + const grouped = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'Writer', factor_bindings: { tool_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { server_id: 'a' } } }, + ], + } as ProtocolGraph + render() + fireEvent.click(screen.getByRole('button', { name: mode })) + expect(screen.getByRole('status')).toHaveTextContent('Group') + expect(screen.getByRole('status')).toHaveTextContent('remove it first') + expect(screen.queryByLabelText('Factor name')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Save factor' })).toBeDisabled() +}) + +it('edits allow-list levels while keeping the server pinned', async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + vi.spyOn(mcpServersApi, 'list').mockResolvedValue([{ id: 'a', capabilities: { tools: [{ name: 'search' }, { name: 'fetch' }] } }] as Awaited>) + const save = vi.fn().mockResolvedValue(undefined) + const unbound = { ...graph, nodes: graph.nodes.map((node) => node.id === 'tool' ? { ...node, data: { label: 'Search server', config: { server_id: 'a', tool_names: ['search'] } } } : node) } as ProtocolGraph + render() + await screen.findAllByText('fetch') + fireEvent.click(screen.getAllByRole('switch')[1]) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ level_type: 'tool_names', levels: [['search', 'fetch'], []] }), 'tool', undefined)) + expect((unbound.nodes[1].data.config as { server_id: string }).server_id).toBe('a') + client.clear() +}) + +it('saves edits to an existing individual factor without removing other declarations', async () => { + const save = vi.fn().mockResolvedValue(undefined) + render() + fireEvent.change(screen.getByLabelText('Factor name'), { target: { value: 'Renamed' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save factor' })) + await waitFor(() => expect(save).toHaveBeenCalledWith({ name: 'Renamed', level_type: 'boolean', levels: [false, true], level_labels: ['Disabled', 'Enabled'] }, 'tool', 'Tool enabled')) +}) + +it('opens the existing group factor when a shared tool has multiple connected agents', () => { + const sharedGraph = { + ...graph, + nodes: [ + { ...graph.nodes[0], data: { label: 'First agent' } }, + { id: 'second', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Second agent', tool_selection: ['a'], tool_factor_mode: 'tool_toggle', factor_bindings: { tool_selection: 'Group' } } }, + { ...graph.nodes[1], data: { label: 'Summarize', config: { server_id: 'a' } } }, + ], + edges: [...graph.edges, { id: 'second-edge', source: 'tool', target: 'second', targetHandle: 'tool' }], + } as ProtocolGraph + render() + expect(screen.queryByRole('combobox')).not.toBeInTheDocument() + expect(screen.getByText('Applies to: First agent, Second agent')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'All tools on/off' })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByLabelText('Factor name')).toHaveValue('Group') +}) diff --git a/frontend/src/components/protocol/ToolFactorDialog.tsx b/frontend/src/components/protocol/ToolFactorDialog.tsx new file mode 100644 index 00000000..1949f6f7 --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorDialog.tsx @@ -0,0 +1,122 @@ +import { factorScopeLabel, factorRecipients } from '@/lib/sharedFactors' +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { ArrowDown, ArrowUp, Split, Trash2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { connectedTools, reconcileToolFactor, toolFactorConflict, toolFactorModes as modes, TOOL_FACTOR_PATH, isIndividualToolMode, toolFactorPath, type ToolFactorMode as Mode } from '@/lib/toolFactors' +import type { DesignFactor } from '@/types/experiments' +import type { ProtocolGraph } from '@/types/protocols' +import { ToolNamesLevelRow } from './FactorEditorDialog' +import { computeFactorName } from './factorLevels' + +export function ToolFactorDialog({ toolNodeId, graph, factors, initialMode, onClose, onSave, onRemove }: { + toolNodeId: string + graph: ProtocolGraph + factors: DesignFactor[] + initialMode?: Mode + onClose: () => void + onSave: (factor: DesignFactor, ownerId: string, previousName?: string) => Promise + onRemove: (name: string) => Promise +}) { + const tool = graph.nodes.find((node) => node.id === toolNodeId)! + const agents = factorRecipients(graph, toolNodeId, 'tool') + const agentId = agents.find((agent) => agent.data.factor_bindings?.[TOOL_FACTOR_PATH])?.id ?? agents[0]?.id ?? '' + const existingFor = (ownerId: string, path: string) => factors.find((factor) => factor.name === graph.nodes.find((node) => node.id === ownerId)?.data.factor_bindings?.[path]) + const initialFactor = existingFor(agentId, TOOL_FACTOR_PATH) ?? existingFor(toolNodeId, 'config.enabled') ?? existingFor(toolNodeId, 'config.tool_names') + const initial = initialFactor && !isIndividualToolMode(initialFactor.level_type as Mode) && initialFactor.level_type ? reconcileToolFactor(initialFactor, graph, agentId) : initialFactor + const [mode, setMode] = useState(initialMode ?? (initial?.level_type as Mode) ?? 'boolean') + function seed(nextMode: Mode, nextAgentId: string): DesignFactor { + const ownerId = isIndividualToolMode(nextMode) ? toolNodeId : nextAgentId + const existing = existingFor(ownerId, toolFactorPath(nextMode)) + if (existing?.level_type === nextMode || (nextMode === 'boolean' && existing && !existing.level_type)) return { + ...existing, + level_type: nextMode, + level_labels: existing.level_labels ?? existing.levels.map((level, index) => nextMode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + } + const owner = graph.nodes.find((node) => node.id === ownerId) + const factor: DesignFactor = { + name: existing?.name ?? computeFactorName(String(owner?.data.label || 'Tool'), nextMode === 'boolean' ? 'Enabled' : nextMode === 'tool_names' ? 'Tools allowed' : 'Tools', factors.map((factor) => factor.name)), + level_type: nextMode, + levels: nextMode === 'boolean' ? [false, true] : nextMode === 'tool_names' ? [(tool.data.config as { tool_names?: string[] }).tool_names ?? [], []] : [], + level_labels: nextMode === 'boolean' ? ['Disabled', 'Enabled'] : nextMode === 'tool_names' ? ['Allowed tools', 'No tools'] : undefined, + } + return isIndividualToolMode(nextMode) ? factor : reconcileToolFactor(factor, graph, ownerId) + } + const initialDraft = !initialMode && initial && (initial.level_type ?? 'boolean') === mode ? initial : seed(mode, agentId) + const { draft, setDraft, reset: resetDraft } = useDialogDraft({ + ...initialDraft, + level_type: initialDraft.level_type ?? 'boolean', + level_labels: initialDraft.level_labels ?? initialDraft.levels.map((level, index) => mode === 'boolean' ? level ? 'Enabled' : 'Disabled' : `Level ${index + 1}`), + }, { sessionKey: mode }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const ownerId = isIndividualToolMode(mode) ? toolNodeId : agentId + const current = existingFor(ownerId, toolFactorPath(mode)) + const affected = isIndividualToolMode(mode) ? [tool] : connectedTools(graph, agentId) + const unavailable = (value: Mode) => !isIndividualToolMode(value) && !agentId + ? `${modes[value]} is unavailable because this Tool is not connected to an Agent. Connect this Tool to an Agent first.` + : toolFactorConflict(graph, factors, toolNodeId, value, agentId) + const blockedReason = unavailable(mode) + const labels = draft.level_labels ?? [] + function change(nextMode: Mode, nextAgentId = agentId) { + setMode(nextMode) + resetDraft(seed(nextMode, nextAgentId)) + setError('') + } + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + if (!remove && blockedReason) return + setSaving(true) + setError('') + try { + if (remove && current) await onRemove(current.name) + else { + await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }, ownerId, current?.name) + onClose() + } + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the tool factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && ownerId && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !open && !saving && onClose()}> + + + Tool factor + Choose how tools vary across experimental cells. + +
    + {agents.length > 0 &&

    Applies to: {factorScopeLabel(graph, toolNodeId, 'tool')}

    } +
    {(Object.keys(modes) as Mode[]).map((value) => )}
    + {blockedReason ?

    {blockedReason}

    : <> +

    {mode === 'boolean' ? 'Enable or disable this Tool node.' : mode === 'tool_names' ? 'Vary the allowed tools of this node’s pinned server.' : mode === 'tool_selection' ? 'Each cell receives exactly one connected tool.' : 'Each cell enables every connected tool, including deactivated tools, or disables them all.'}

    +
    setDraft({ ...draft, name: event.target.value })} />
    + {!isIndividualToolMode(mode) && affected.length < (mode === 'tool_toggle' ? 1 : 2) &&

    Connect at least {mode === 'tool_toggle' ? 'one tool' : 'two different tools'} before generating cells.

    } +
    {draft.levels.map((level, index) =>
    +
    +

    {mode === 'boolean' ? (level ? 'Enabled' : 'Disabled') : mode === 'tool_names' ? `${(level as string[]).length} allowed tools` : mode === 'tool_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : String(affected.find((node) => node.id === (level as string[])[0])?.data.label ?? 'Unavailable tool')}{index === 0 && mode !== 'boolean' ? ' · default test selection' : ''}

    + {mode === 'tool_names' && setDraft({ ...draft, levels: draft.levels.map((value, i) => i === index ? next : value) })} />} + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + {mode !== 'boolean' && <>} + {mode === 'tool_names' && } +
    )}
    + {mode === 'tool_names' && } +

    Shared factors require matching Tool connections. Changes require design review and regeneration.

    + } + {current && } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/ToolFactorEditor.tsx b/frontend/src/components/protocol/ToolFactorEditor.tsx new file mode 100644 index 00000000..8fed779e --- /dev/null +++ b/frontend/src/components/protocol/ToolFactorEditor.tsx @@ -0,0 +1,75 @@ +import { useState } from 'react' +import { useDialogDraft } from './useDialogDraft' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import type { DesignFactor } from '@/types/experiments' + +import type { ProtocolGraph } from '@/types/protocols' + +export function ToolFactorEditor({ graph, factor, open, onOpenChange, onSave, onRemove }: { + graph?: ProtocolGraph + factor: DesignFactor + open: boolean + onOpenChange: (open: boolean) => void + onSave: (factor: DesignFactor) => void | Promise + onRemove?: () => Promise +}) { + const { draft, setDraft } = useDialogDraft(factor, { open, sessionKey: factor.name }) + const [saving, setSaving] = useState(false) + const [error, setError] = useState('') + const labels = draft.level_labels ?? [] + function move(index: number, offset: number) { + const levels = [...draft.levels] + const nextLabels = [...labels] + ;[levels[index], levels[index + offset]] = [levels[index + offset], levels[index]] + ;[nextLabels[index], nextLabels[index + offset]] = [nextLabels[index + offset], nextLabels[index]] + setDraft({ ...draft, levels, level_labels: nextLabels }) + } + async function commit(remove = false) { + setSaving(true) + setError('') + try { + if (remove) await onRemove?.() + else await onSave({ ...draft, name: draft.name.trim(), level_labels: labels.map((label) => label.trim()) }) + onOpenChange(false) + } catch (error) { + setError(error instanceof Error ? error.message : 'Could not save the tool factor.') + } finally { setSaving(false) } + } + const valid = draft.name.trim() && labels.length === draft.levels.length && labels.every((label) => label.trim()) && new Set(labels.map((label) => label.trim())).size === labels.length + return !saving && onOpenChange(open)}> + + + {factor.level_type === 'tool_toggle' ? 'All tools on/off' : 'Tools as levels'} + {factor.level_type === 'tool_toggle' ? 'Each cell receives all connected tools or none.' : 'Each cell receives exactly one connected tool.'} Add or disconnect Tool nodes on the canvas to change the levels. + +
    + + setDraft({ ...draft, name: event.target.value })} /> + {draft.levels.length < 2 &&

    Incomplete: connect at least two different tools before generating cells.

    } +
    + {draft.levels.map((level, index) => { + const id = (level as string[])[0] + const tool = graph?.nodes.find((node) => node.id === id) + return
    +
    +

    {factor.level_type === 'tool_toggle' ? ((level as string[]).length ? 'All enabled' : 'All disabled') : tool?.data.label ?? labels[index] ?? 'Unavailable tool'}{index === 0 ? ' · default test selection' : ''}

    + {factor.level_type !== 'tool_toggle' && !!graph && !tool &&

    Unavailable: disconnect or restore this tool.

    } + setDraft({ ...draft, level_labels: labels.map((label, i) => i === index ? event.target.value : label) })} /> +
    + + +
    + })} +
    +

    Individual tool switches are controlled by this connector. Changes require design review and regeneration.

    + {onRemove &&

    Removing this factor enables all connected tools and restores their individual controls.

    } + {error &&

    {error}

    } +
    + +     +     +} diff --git a/frontend/src/components/protocol/ToolTestSelectors.tsx b/frontend/src/components/protocol/ToolTestSelectors.tsx new file mode 100644 index 00000000..f0929b98 --- /dev/null +++ b/frontend/src/components/protocol/ToolTestSelectors.tsx @@ -0,0 +1,26 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { connectedTools, toolId, TOOL_FACTOR_PATH } from '@/lib/toolFactors' +import type { useToolTestSelection } from './useToolTestSelection' + +export function ToolTestSelectors({ selection }: { selection: ReturnType }) { + return
    {selection.owners.map((node) => { + const options = connectedTools(selection.graph!, node.id) + if (node.data.tool_factor_mode === 'tool_toggle') return
    +

    {node.data.factor_bindings?.[TOOL_FACTOR_PATH] || node.data.label}: tools for this test

    + +
    + return
    +

    {node.data.factor_bindings?.[TOOL_FACTOR_PATH] || node.data.label}: tool for this test

    + +
    + })}{selection.error &&

    {selection.error}

    }
    +} diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 1f2f1933..813c1078 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -1,6 +1,9 @@ +import { sharedFactorGroup, factorScopeLabel, type SharedFactorKind } from '@/lib/sharedFactors' +import type { ProtocolGraph } from '@/types/protocols' import type { Edge, Node } from '@xyflow/react' import type { LevelType } from './factorLevels' import type { McpServer } from '@/types/mcpServers' +import { isPatternNode, patternFactorFields, PATTERN_FACTOR_PATH, patternFactorConflict } from './patternFactors' export interface BindableFieldSpec { fieldPath: string @@ -8,6 +11,12 @@ export interface BindableFieldSpec { levelType: LevelType } +export const agentFactorFields: BindableFieldSpec[] = [ + { fieldPath: 'active', label: 'This agent on/off', levelType: 'boolean' }, + { fieldPath: 'config.prompt', label: 'Prompt levels', levelType: 'text' }, + { fieldPath: 'config.system_prompt', label: 'System prompt levels', levelType: 'text' }, +] + // The bundled system servers (see the backend's // services/system_mcp_servers.py) that are hidden from every MCP Tool server // picker -- they're still registered, still connected, and still usable by @@ -155,6 +164,7 @@ const MODEL_NODE_TYPES = new Set([ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { switch (node.type) { case 'agent': + return [...agentFactorFields, { fieldPath: 'pattern_override', label: 'Execution pattern', levelType: 'pattern' }] case 'sub_agent': return [ // The run's own ask, and the field `{{...}}` references resolve in -- @@ -191,7 +201,7 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { // a connected Model node's whole config verbatim (never the node's // xyflow `type`), so replacing it wholesale per cell already works // with zero backend changes. - { fieldPath: 'config', label: 'Provider & model', levelType: 'model_config' }, + { fieldPath: 'config', label: 'Model levels', levelType: 'model_config' }, ] case 'mcp_tool': case 'mcp_scikit_learn': @@ -251,16 +261,11 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { ] case 'dataset': return [ - // No runtime effect beyond skipping the Dataset-context block in - // _build_user_input. + // Individual availability controls Dataset context and implicit tools. { fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }, // The whole node as a factor -- levels are entirely different - // datasets, which is the ONLY supported way to run one experiment - // across several of them. The Dataset connector is capped at one - // node per agent (see AgentNode.tsx) precisely because a cell's - // workspace is keyed by experiment_id/cell_label and therefore holds - // exactly one dataset; varying it per CELL is the shape that fits, - // and wiring several at once never was. + // datasets. Keep this field for editing legacy bindings; new creation + // uses factorCreationFields and the Dataset connector's levels. // // Unlike skill/okf_bundle above -- whose configs are also just // per-account pointers -- this one earns a structured level type @@ -271,36 +276,25 @@ export function bindableFieldsForNode(node: Node): BindableFieldSpec[] { { fieldPath: 'config', label: 'Dataset', levelType: 'dataset_config' }, ] case 'skill': - // Only `enabled` -- the natural "which skill" factor is the WHOLE node - // (levels = different skills), but unlike script/llm/tool_config, a - // skill node's config is just an id pointing at a row in the skill - // library, so a level would carry a per-account id rather than the - // thing itself. Comparing two skills today means two nodes, one - // enabled per cell; a real skill_config level type is deferred until - // the level editor can pick from the library the way the inspector - // does. + // Connector choices are listed by factorCreationFields. return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'okf_bundle': case 'okf_document': - // Only `enabled`, same reasoning as skill above: the node's config is a - // pointer at a per-account registration (a server name plus a path on - // the server), not the knowledge itself, so a level would carry a - // reference rather than the thing being compared. Comparing "with the - // knowledge vs. without" -- the question actually worth an experiment - // here -- is exactly what toggling `enabled` per cell does. + // Individual availability binds here; Knowledge connector choices are + // listed by factorCreationFields and edited in the dedicated dialog. return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }] case 'script': - // The whole node as a factor -- levels are entirely different scripts. - // _resolve_script_configs reads each wired script node's whole config - // verbatim, so comparing two scoring scripts already works with zero - // backend changes, same reasoning as model_config/tool_config. - return [{ fieldPath: 'config', label: 'Script', levelType: 'script_config' }] + // Keep config discoverable for editing existing variant bindings. + // New factors compare separate Script nodes through Script levels. + return [{ fieldPath: 'config.enabled', label: 'Enabled', levelType: 'boolean' }, { fieldPath: 'config', label: 'Script variants (existing factors)', levelType: 'script_config' }] default: return [] } } export interface UnboundField { + pickerGroup?: { id: string; label: string; category: string; componentId: string; componentLabel: string } + connectorFactor?: { kind: 'agent' | 'script' | 'tool' | 'skill' | 'sub_agent' | 'dataset' | 'knowledge' | 'model' | 'pattern'; nodeId: string; mode: LevelType; agentId?: string } nodeId: string nodeLabel: string fieldPath: string @@ -444,11 +438,15 @@ export function agentTracedLabel(node: Node, edges: Edge[], nodes: Node[]): stri export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundField[] { const result: UnboundField[] = [] for (const node of nodes) { + if (['okf_bundle', 'okf_document'].includes(node.type ?? '') && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'knowledge' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.knowledge_selection)) continue + if (node.type === 'skill' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'skill' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.skill_selection)) continue + if (node.type === 'dataset' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'dataset' && (nodes.find((target) => target.id === edge.target)?.data.factor_bindings as Record | undefined)?.dataset_selection)) continue const bindings = (node.data as { factor_bindings?: Record })?.factor_bindings ?? {} const label = agentTracedLabel(node, edges, nodes) const wholeConfigBound = !!bindings.config const configSubFieldBound = Object.keys(bindings).some((path) => path !== 'config' && path.startsWith('config.')) for (const field of bindableFieldsForNode(node)) { + if (node.type === 'sub_agent' && field.fieldPath === 'active' && edges.some((edge) => edge.source === node.id && edge.targetHandle === 'sub_agents' && (nodes.find((owner) => owner.id === edge.target)?.data.factor_bindings as Record | undefined)?.sub_agent_selection)) continue if (bindings[field.fieldPath]) continue if (field.fieldPath === 'config' && configSubFieldBound) continue if (field.fieldPath.startsWith('config.') && wholeConfigBound) continue @@ -472,3 +470,145 @@ export function unboundBindableFields(nodes: Node[], edges: Edge[]): UnboundFiel } return result } + +/** Creation entries route connector choices through their dedicated factor dialogs. */ +export function factorCreationFields(nodes: Node[], edges: Edge[]): UnboundField[] { + const fields = unboundBindableFields(nodes, edges).filter((field) => { + const kind = nodes.find((node) => node.id === field.nodeId)?.type + return !['script', 'mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool'].includes(kind ?? '') && kind !== 'skill' && kind !== 'dataset' && kind !== 'okf_bundle' && kind !== 'okf_document' + && !(kind === 'sub_agent' && field.fieldPath === 'active') + && !isPatternNode(kind) && field.fieldPath !== PATTERN_FACTOR_PATH + }) + for (const node of nodes.filter((node) => isPatternNode(node.type))) { + const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && edge.targetHandle === 'architectural_pattern')) + for (const owner of owners.length ? owners : [undefined]) { + for (const field of patternFactorFields(node.id, nodes, edges, owner?.id)) { + if ((nodes.find((node) => node.id === field.nodeId)?.data.factor_bindings as Record | undefined)?.[field.fieldPath]) continue + if (patternFactorConflict(node.id, field.fieldPath, nodes, edges, owner?.id)) continue + if (field.fieldPath !== PATTERN_FACTOR_PATH && fields.some((candidate) => candidate.nodeId === node.id && candidate.fieldPath === field.fieldPath)) continue + fields.push({ nodeId: field.nodeId, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: field.fieldPath, fieldLabel: field.label, levelType: field.levelType, currentValue: field.currentValue, + connectorFactor: { kind: 'pattern', nodeId: node.id, mode: field.levelType, agentId: owner?.id } }) + } + } + } + const connectorOwners = new Set() + for (const node of nodes.filter((node) => node.type === 'sub_agent')) { + const owners = nodes.filter((owner) => owner.type === 'agent' && edges.some((edge) => edge.source === node.id && edge.target === owner.id && edge.targetHandle === 'sub_agents')) + if (!(node.data.factor_bindings as Record | undefined)?.active) fields.push({ + nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'active', + fieldLabel: 'This sub-agent on/off', levelType: 'boolean', currentValue: node.data.active ?? true, + connectorFactor: { kind: 'sub_agent', nodeId: node.id, mode: 'boolean' }, + }) + for (const owner of owners) { + const key = `${owner.id}:sub_agent` + if (connectorOwners.has(key)) continue + connectorOwners.add(key) + for (const mode of ['selection', 'toggle'] as const) fields.push({ + nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: 'sub_agent_selection', + fieldLabel: mode === 'selection' ? 'Sub-Agent levels' : 'All sub-agents on/off', + levelType: `sub_agent_${mode}`, currentValue: undefined, + connectorFactor: { kind: 'sub_agent', nodeId: node.id, mode: `sub_agent_${mode}`, agentId: owner.id }, + }) + } + } + for (const node of nodes) { + if (!['script', 'mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool', 'skill', 'dataset', 'okf_bundle', 'okf_document'].includes(node.type ?? '')) continue + const kind = (node.type === 'okf_bundle' || node.type === 'okf_document' ? 'knowledge' : ['mcp_tool', 'mcp_scikit_learn', 'mcp_client_tool'].includes(node.type ?? '') ? 'tool' : node.type) as 'script' | 'tool' | 'skill' | 'dataset' | 'knowledge' + const handles = (kind === 'tool' || kind === 'script') ? ['tool'] : kind === 'skill' ? ['skill'] : kind === 'knowledge' ? ['knowledge'] : ['dataset', 'resource', 'tool'] + const owners = nodes.filter((owner) => ['agent', 'sub_agent'].includes(owner.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === owner.id && handles.includes(edge.targetHandle ?? ''))) + if (!(node.data.factor_bindings as Record | undefined)?.['config.enabled']) fields.push({ + nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.enabled', + fieldLabel: `This ${kind} on/off`, levelType: 'boolean', currentValue: undefined, + connectorFactor: { kind, nodeId: node.id, mode: 'boolean' }, + }) + if (kind === 'tool' && !(node.data.factor_bindings as Record | undefined)?.['config.tool_names']) fields.push({ nodeId: node.id, nodeLabel: agentTracedLabel(node, edges, nodes), fieldPath: 'config.tool_names', fieldLabel: 'Tools allowed', levelType: 'tool_names', currentValue: (node.data.config as { tool_names?: string[] })?.tool_names, connectorFactor: { kind, nodeId: node.id, mode: 'tool_names' } }) + for (const owner of owners) { + const key = `${owner.id}:${kind}` + if (connectorOwners.has(key)) continue + connectorOwners.add(key) + for (const mode of ['selection', 'toggle'] as const) { + const levelType = `${kind}_${mode}` as LevelType + fields.push({ + nodeId: owner.id, nodeLabel: String(owner.data.label || 'Agent'), fieldPath: `${kind}_selection`, + fieldLabel: mode === 'selection' ? `${kind === 'script' ? 'Script' : kind === 'tool' ? 'Tool' : kind === 'skill' ? 'Skill' : kind === 'knowledge' ? 'Knowledge' : 'Dataset'} levels` : `All ${kind === 'script' ? 'scripts' : kind === 'tool' ? 'tools' : kind === 'knowledge' ? 'knowledge' : `${kind}s`} on/off`, + levelType, currentValue: undefined, + connectorFactor: { kind, nodeId: node.id, mode: levelType, agentId: owner.id }, + }) + } + } + } + for (const field of fields) { + if (nodes.find((node) => node.id === field.nodeId)?.type === 'agent' && agentFactorFields.some((candidate) => candidate.fieldPath === field.fieldPath)) { + field.connectorFactor = { kind: 'agent', nodeId: field.nodeId, mode: field.levelType } + } + if (MODEL_NODE_TYPES.has(nodes.find((node) => node.id === field.nodeId)?.type ?? '')) { + field.connectorFactor = { kind: 'model', nodeId: field.nodeId, mode: field.levelType } + } + } + const graph = { nodes, edges } as unknown as ProtocolGraph + const sharedChoices = new Set() + return fields.filter((field) => { + const choice = field.connectorFactor + if (!choice || !['skill', 'dataset', 'knowledge', 'script', 'tool', 'sub_agent', 'pattern'].includes(choice.kind) || choice.mode === 'boolean' || choice.mode === 'tool_names' || choice.mode === 'script_config' || (choice.kind === 'pattern' && field.fieldPath !== PATTERN_FACTOR_PATH)) return true + const group = sharedFactorGroup(graph, choice.nodeId, choice.kind as SharedFactorKind) + const key = `${choice.kind}:${choice.mode}:${field.fieldPath}:${group.recipients.map((node) => node.id).sort().join(',')}` + if (sharedChoices.has(key)) return false + sharedChoices.add(key) + return true + }).map((field) => { + const node = nodes.find((candidate) => candidate.id === field.nodeId)! + const kind = field.connectorFactor?.kind ?? node.type ?? 'node' + const agentKinds = ['agent', 'sub_agent'] + const targets = nodes.filter((candidate) => agentKinds.includes(candidate.type ?? '') && edges.some((edge) => edge.source === node.id && edge.target === candidate.id)) + const owner = field.connectorFactor?.kind === 'sub_agent' && field.connectorFactor.mode === 'boolean' ? targets.length === 1 ? targets[0] : undefined : agentKinds.includes(node.type ?? '') ? node : targets.length === 1 ? targets[0] : undefined + const category = field.connectorFactor?.kind === 'script' ? 'Scripts' : field.connectorFactor?.kind === 'tool' ? 'Tools' : field.connectorFactor?.kind === 'sub_agent' ? 'Sub-Agents' + : field.connectorFactor?.kind === 'skill' ? 'Skills' + : field.connectorFactor?.kind === 'dataset' ? 'Datasets' + : field.connectorFactor?.kind === 'knowledge' ? 'Knowledge' + : field.connectorFactor?.kind === 'model' ? 'Model' + : field.connectorFactor?.kind === 'pattern' ? 'Pattern' + : kind === 'agent' ? 'Agent' + : agentKinds.includes(kind) ? 'Agent fields' + : MODEL_NODE_TYPES.has(kind) ? 'Model' + : kind.startsWith('pattern_') ? 'Pattern' + : ({ skill: 'Skills', dataset: 'Datasets', mcp_tool: 'Tools', mcp_scikit_learn: 'Tools', mcp_client_tool: 'Tools', script: 'Tools', okf_bundle: 'Knowledge', okf_document: 'Knowledge', memory: 'Memory', output_parser: 'Output parser', critic_gate: 'Critic gate' } as Record)[kind] ?? String(node.data.label || kind) + const connectorChoice = field.connectorFactor && field.connectorFactor.kind !== 'agent' && field.connectorFactor.kind !== 'model' && field.connectorFactor.kind !== 'pattern' && field.connectorFactor.mode !== 'boolean' && field.connectorFactor.mode !== 'tool_names' && field.connectorFactor.mode !== 'script_config' + const sharedScope = field.connectorFactor && ['skill', 'dataset', 'knowledge', 'script', 'tool', 'sub_agent', 'pattern'].includes(field.connectorFactor.kind) + ? factorScopeLabel(graph, field.connectorFactor.nodeId, field.connectorFactor.kind as SharedFactorKind) : undefined + const sharedRecipients = sharedScope && field.connectorFactor + ? sharedFactorGroup(graph, field.connectorFactor.nodeId, field.connectorFactor.kind as SharedFactorKind).recipients.map((node) => node.id).sort() : [] + const sharedScopeId = sharedRecipients.length === 1 ? sharedRecipients[0] : sharedRecipients.length ? `shared:${sharedRecipients.join(',')}` : undefined + return { + ...field, + pickerGroup: { + id: sharedScopeId ?? owner?.id ?? 'shared-unconnected', + label: sharedScope || (owner ? String(owner.data.label || 'Agent') : 'Shared or unconnected components'), + category, + componentId: field.connectorFactor?.kind === 'pattern' ? field.connectorFactor.nodeId : connectorChoice ? `${field.nodeId}:${kind}` : node.id, + componentLabel: field.connectorFactor?.kind === 'pattern' ? String(nodes.find((node) => node.id === field.connectorFactor?.nodeId)?.data.label || 'Pattern') : connectorChoice ? `${category} in this group` : String(node.data.label || kind), + }, + } + }) +} + +export function groupFactorCreationFields(fields: UnboundField[]) { + type ComponentGroup = { id: string; label: string; fields: UnboundField[] } + type ConnectorGroup = { label: string; components: Map } + const agents = new Map }>() + for (const field of fields) { + const group = field.pickerGroup ?? { id: 'fields', label: 'Canvas fields', category: 'Fields', componentId: field.nodeId, componentLabel: field.nodeLabel } + if (!agents.has(group.id)) agents.set(group.id, { id: group.id, label: group.label, connectors: new Map() }) + const agent = agents.get(group.id)! + if (!agent.connectors.has(group.category)) agent.connectors.set(group.category, { label: group.category, components: new Map() }) + const connector = agent.connectors.get(group.category)! + if (!connector.components.has(group.componentId)) connector.components.set(group.componentId, { id: group.componentId, label: group.componentLabel, fields: [] }) + connector.components.get(group.componentId)!.fields.push(field) + } + return [...agents.values()] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((agent) => ({ ...agent, connectors: [...agent.connectors.values()] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((connector) => ({ ...connector, components: [...connector.components.values()] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((component) => ({ ...component, fields: component.fields.sort((a, b) => a.fieldLabel.localeCompare(b.fieldLabel)) })) })) })) +} diff --git a/frontend/src/components/protocol/cells/AGENTS.md b/frontend/src/components/protocol/cells/AGENTS.md index 75202d72..273c83e5 100644 --- a/frontend/src/components/protocol/cells/AGENTS.md +++ b/frontend/src/components/protocol/cells/AGENTS.md @@ -17,9 +17,11 @@ earns its place. (`pickMetricColumns` — same preference order as the heatmap's default metric, capped because a real scored cell can have 6-7 numeric keys and showing all of them makes the table unreadably wide). One table row is one true cell (a unique factor combination): its metric - columns are replicate means and its status reports scored/total replicates. The API's legacy - `Replicate` rows are individual observations and must be grouped first via - `groupReplicatesIntoCells`. `table-fixed` isn't used since the column set is dynamic; instead: + columns are replicate means and its status reports scored/total replicates. The tab reads + `GET /experiments/{id}/cell-summaries` (one row per cell, metrics as sum/count, aggregated in + SQL), **never** `listReplicates`: a 15,786-replicate design shipped every replicate's + `artifacts` to the browser just to average them. Pool cells with `pooledMean` so a heatmap + square stays replicate-weighted. `table-fixed` isn't used since the column set is dynamic; instead: `truncate` on free-text cells, a shaded uppercase sortable header row, subtle zebra striping, `font-mono` on technical values. It's styled to the side panel's own compact table idiom (a plain `text-xs` `` in a bordered box, like `RunsTab`/`ResultsTab`), not diff --git a/frontend/src/components/protocol/cells/CellsHeatmap.tsx b/frontend/src/components/protocol/cells/CellsHeatmap.tsx index 9ba5ba31..30ebac9d 100644 --- a/frontend/src/components/protocol/cells/CellsHeatmap.tsx +++ b/frontend/src/components/protocol/cells/CellsHeatmap.tsx @@ -6,16 +6,15 @@ import { deriveFactors, displayFactorLevel, formatMetricLabel, - groupReplicatesIntoCells, heatColor, - meanMetric, metricValueSuffix, pickDefaultMetric, + pooledMean, scaledMetricValue, type FactorSpec, } from '@/lib/experiment' import { cn } from '@/lib/utils' -import type { Experiment, Replicate } from '@/types/experiments' +import type { CellSummary, Experiment } from '@/types/experiments' /** Why there's no grid here, said out loud in one dim line rather than * rendering nothing at all. The table below still carries every number, so @@ -48,20 +47,19 @@ function HeatmapUnavailable({ reason }: { reason: string }) { * three components to make that happen. The parent marks the container -- * see CellsTab's own `@container`. */ -export function CellsHeatmap({ experiment, cells }: { experiment: Experiment; cells: Replicate[] }) { +export function CellsHeatmap({ experiment, cells }: { experiment: Experiment; cells: CellSummary[] }) { const availableMetrics = useMemo(() => availableMetricKeys(cells), [cells]) const defaultMetric = useMemo(() => pickDefaultMetric(experiment, cells), [experiment, cells]) const [metricKey, setMetricKey] = useState(defaultMetric) const activeMetric = metricKey && availableMetrics.includes(metricKey) ? metricKey : defaultMetric const factors = useMemo(() => deriveFactors(cells, experiment.design_spec), [cells, experiment.design_spec]) - const cellCount = useMemo(() => groupReplicatesIntoCells(cells).length, [cells]) if (!factors || factors.length < 1) return if (factors.length > 3) return - if (cellCount < 2) return + if (cells.length < 2) return if (!activeMetric) return const rowFactor = factors[0] @@ -76,7 +74,7 @@ export function CellsHeatmap({ experiment, cells }: { experiment: Experiment; ce if (colFactor.name) match[colFactor.name] = colLevel if (facetFactor) match[facetFactor.name] = facetLevel const matched = replicatesMatching(cells, match) - return { value: meanMetric(matched, activeMetric), count: matched.length } + return { value: pooledMean(matched, activeMetric), count: matched.reduce((total, cell) => total + cell.replicate_count, 0) } }), ), ) diff --git a/frontend/src/components/protocol/cells/CellsTab.test.tsx b/frontend/src/components/protocol/cells/CellsTab.test.tsx new file mode 100644 index 00000000..dd1fe803 --- /dev/null +++ b/frontend/src/components/protocol/cells/CellsTab.test.tsx @@ -0,0 +1,27 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { render, screen } from '@testing-library/react' +import { afterEach, expect, it, vi } from 'vitest' +import { experimentsApi } from '@/api/client' +import type { CellSummary, Experiment } from '@/types/experiments' +import { CellsTab } from './CellsTab' + +const experiment = { id: 'e', name: 'Large', design_type: 'factorial', design_spec: null } as unknown as Experiment +const summary = (label: string, model: string, scored: number, sum: number): CellSummary => ({ + cell_id: label, cell_label: label, factor_values: { model }, replicate_count: 7893, scored_count: scored, + updated_at: '2026-10-01T00:00:00Z', metrics: { accuracy: { sum, count: scored } }, +}) + +afterEach(() => vi.restoreAllMocks()) + +it('renders large designs from per-cell aggregates without loading replicates', async () => { + const replicates = vi.spyOn(experimentsApi, 'listReplicates') + vi.spyOn(experimentsApi, 'listDesignRevisions').mockResolvedValue([]) + vi.spyOn(experimentsApi, 'listCellSummaries').mockResolvedValue([summary('a', 'a', 7893, 3946.5), summary('b', 'b', 100, 90)]) + render() + expect(await screen.findByText(/2 cells · 7993\/15786 replicates scored/)).toBeInTheDocument() + expect(screen.getByText('7893/7893 replicates scored')).toBeInTheDocument() + // Pooled from server sums: 3946.5 / 7893 and 90 / 100. + expect(screen.getAllByText('0.5').length).toBeGreaterThan(0) + expect(screen.getAllByText('0.9').length).toBeGreaterThan(0) + expect(replicates).not.toHaveBeenCalled() +}) diff --git a/frontend/src/components/protocol/cells/CellsTab.tsx b/frontend/src/components/protocol/cells/CellsTab.tsx index ccd01050..ffdf8662 100644 --- a/frontend/src/components/protocol/cells/CellsTab.tsx +++ b/frontend/src/components/protocol/cells/CellsTab.tsx @@ -4,9 +4,8 @@ import { ArrowLeft, Download, Maximize2, Minimize2 } from 'lucide-react' import { Button } from '@/components/ui/button' import { Skeleton } from '@/components/ui/skeleton' import { experimentsApi } from '@/api/client' -import { groupReplicatesIntoCells } from '@/lib/experiment' import { sanitizeFilename } from '@/lib/utils' -import type { Experiment, Replicate } from '@/types/experiments' +import type { CellSummary, Experiment } from '@/types/experiments' import { CellsHeatmap } from './CellsHeatmap' import { CellsTable } from './CellsTable' import { DesignHistory } from './DesignHistory' @@ -15,7 +14,7 @@ import { DesignHistory } from './DesignHistory' // than the viewport -- the same component renders in a drag-resizable ~320- // 1100px panel column and in a full-viewport overlay, and dragging the panel // wider has to visibly pay off without a width prop threaded through here. -function CellsBody({ experiment, cells }: { experiment: Experiment; cells: Replicate[] }) { +function CellsBody({ experiment, cells }: { experiment: Experiment; cells: CellSummary[] }) { return (
    @@ -59,15 +58,14 @@ export function CellsTab({ experiment, perRow = false }: { experiment: Experimen // and the CSV button can't disagree about which design they're showing. const [revisionId, setRevisionId] = useState(null) - // For the current design this is the same query key the canvas page itself - // uses, so it shares one cache entry with the top bar's cells readout and - // DesignTab's own invalidation after generating a design -- no second fetch, - // no chance of the two disagreeing about how many cells exist. A superseded - // revision gets its own key: it's a different set of rows, and it must not - // overwrite the shared entry every other reader is looking at. - const replicatesQuery = useQuery({ - queryKey: revisionId ? ['experiments', experiment.id, 'replicates', revisionId] : ['experiments', experiment.id, 'replicates'], - queryFn: () => experimentsApi.listReplicates(experiment.id, revisionId ?? undefined), + // Server-side per-cell aggregates, never one row per replicate: a design can + // carry tens of thousands of replicates. The key sits under + // ['experiments', id, 'replicates'] so DesignTab/RunsTab's existing + // invalidations after generating or running still refresh it, and a + // superseded revision gets its own entry so it never overwrites the current one. + const summariesQuery = useQuery({ + queryKey: ['experiments', experiment.id, 'replicates', 'cell-summaries', revisionId ?? 'current'], + queryFn: () => experimentsApi.listCellSummaries(experiment.id, revisionId ?? undefined), }) useEffect(() => { @@ -104,7 +102,7 @@ export function CellsTab({ experiment, perRow = false }: { experiment: Experimen ) } - if (replicatesQuery.isLoading) { + if (summariesQuery.isLoading) { return (
    @@ -113,9 +111,10 @@ export function CellsTab({ experiment, perRow = false }: { experiment: Experimen ) } - const cells = perRow ? replicatesQuery.data?.map(replicate => ({ ...replicate, metric_values: null })) : replicatesQuery.data - const cellCount = cells ? groupReplicatesIntoCells(cells).length : 0 - const scoredReplicates = cells?.filter((replicate) => replicate.metric_values).length ?? 0 + const cells = perRow ? summariesQuery.data?.map(cell => ({ ...cell, scored_count: 0, metrics: {} })) : summariesQuery.data + const cellCount = cells?.length ?? 0 + const replicateCount = cells?.reduce((total, cell) => total + cell.replicate_count, 0) ?? 0 + const scoredReplicates = cells?.reduce((total, cell) => total + cell.scored_count, 0) ?? 0 const viewingHistory = revisionId !== null // The history list stays mounted through the error/empty cases below: an @@ -127,7 +126,7 @@ export function CellsTab({ experiment, perRow = false }: { experiment: Experimen ) let body - if (replicatesQuery.isError || !cells) { + if (summariesQuery.isError || !cells) { body =

    Could not load this experiment's cells.

    } else if (cells.length === 0) { body = ( @@ -162,7 +161,7 @@ export function CellsTab({ experiment, perRow = false }: { experiment: Experimen

    - Cells — {cellCount} {cellCount === 1 ? 'cell' : 'cells'} · {scoredReplicates}/{cells?.length ?? 0} replicates scored + Cells — {cellCount} {cellCount === 1 ? 'cell' : 'cells'} · {scoredReplicates}/{replicateCount} replicates scored

    {paged.map((cell, i) => ( - - + {factors.map((f) => ( ))} {metricColumns.map((m) => ( ))} ))} diff --git a/frontend/src/components/protocol/connectionValidation.test.ts b/frontend/src/components/protocol/connectionValidation.test.ts index 03f023a6..d87f7187 100644 --- a/frontend/src/components/protocol/connectionValidation.test.ts +++ b/frontend/src/components/protocol/connectionValidation.test.ts @@ -5,6 +5,11 @@ import { isProtocolConnectionValid } from './connectionValidation' const target: Node = { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: {} } describe('isProtocolConnectionValid', () => { + it('allows the same Sub-Agent to connect to multiple parent Agents', () => { + const child: Node = { id: 'child', type: 'sub_agent', position: { x: 0, y: 0 }, data: {} } + const other = { ...target, id: 'other' } + expect(isProtocolConnectionValid({ source: 'child', sourceHandle: 'sub_agents', target: 'other', targetHandle: 'sub_agents' }, [child, target, other], [{ id: 'first', source: 'child', target: 'agent', targetHandle: 'sub_agents' }], false)).toBe(true) + }) const cappedSlots = [ ['model', 'model_openai'], ['memory', 'memory'], diff --git a/frontend/src/components/protocol/connectionValidation.ts b/frontend/src/components/protocol/connectionValidation.ts index dd218b69..e60898d0 100644 --- a/frontend/src/components/protocol/connectionValidation.ts +++ b/frontend/src/components/protocol/connectionValidation.ts @@ -81,8 +81,7 @@ export function isProtocolConnectionValid( case 'sub_agents': return ( sourceNode.type === 'sub_agent' && - targetNode.type === 'agent' && - !edges.some((edge) => edge.source === sourceNode.id && edge.targetHandle === 'sub_agents') + targetNode.type === 'agent' ) default: { const sourceCanFeedMainFlow = diff --git a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx index d4e95444..4963b31a 100644 --- a/frontend/src/components/protocol/edges/DatasetInputPanel.tsx +++ b/frontend/src/components/protocol/edges/DatasetInputPanel.tsx @@ -5,7 +5,7 @@ import { datasetsApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Checkbox } from '@/components/ui/checkbox' import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog' -import { datasetRowBindings, datasetRowTopologyError } from '@/lib/datasetRows' +import { datasetConfigIsFactorBound, datasetRowBindings, datasetRowFactorBindingError, datasetRowTopologyError } from '@/lib/datasetRows' import type { DatasetInput, ProtocolEdge, ProtocolNode } from '@/types/protocols' export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId: string; source: string; disabled?: boolean }) { @@ -22,12 +22,14 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId ? ['per_row', [...saved.columns].sort()] : ['whole_dataset'])) const inputKey = JSON.stringify(mode === 'per_row' ? [mode, [...columns].sort()] : [mode]) const hasChanges = inputKey !== appliedKey - const id = (nodes.find(node => node.id === source)?.data.config as { dataset_id?: string })?.dataset_id + const sourceNode = nodes.find(node => node.id === source) + const id = (sourceNode?.data.config as { dataset_id?: string })?.dataset_id + const factorBound = datasetConfigIsFactorBound(sourceNode) || !!nodes.find(node => node.id === edge?.target)?.data.factor_bindings?.dataset_selection const otherRowDriver = datasetRowBindings({ nodes, edges }).find(binding => binding.edge.id !== edgeId && binding.input.mode === 'per_row' && binding.config?.dataset_id?.toLowerCase() !== id?.toLowerCase(), ) - const schema = useQuery({ queryKey: ['datasets', id, 'row-schema'], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id }) + const schema = useQuery({ queryKey: ['datasets', id, 'row-schema'], queryFn: () => datasetsApi.getRowSchema(id!), enabled: !!id, refetchInterval: query => query.state.data?.ready === false ? 1000 : false }) const availableColumns = schema.data?.columns ?? [] const allColumnsSelected = availableColumns.length > 0 && availableColumns.every(column => columns.includes(column)) const displayedColumns = [...new Set([...availableColumns, ...columns])] @@ -42,7 +44,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId const input: DatasetInput = mode === 'per_row' ? { mode, columns } : { mode } const graph = { nodes, edges: edges.map(item => item.id === edgeId ? { ...item, data: { ...item.data, dataset_input: input } } : item) } - const error = !id ? 'Select a registered Dataset.' : schema.isError ? schema.error.message : schema.isLoading ? 'Loading original columns…' : !schema.data ? 'Source metadata unavailable.' : mode === 'per_row' && !schema.data.row_count ? 'The original Dataset has no rows.' : mode === 'per_row' && columns.some(column => !schema.data.columns.includes(column)) ? 'Saved columns are missing from the original CSV.' : datasetRowTopologyError(graph) + const error = !id ? 'Select a registered Dataset.' : schema.isError ? schema.error.message : schema.isLoading || schema.data?.ready === false ? 'Preparing original columns…' : !schema.data ? 'Source metadata unavailable.' : mode === 'per_row' && !schema.data.row_count ? 'The original Dataset has no rows.' : mode === 'per_row' && columns.some(column => !schema.data.columns.includes(column)) ? 'Saved columns are missing from the original CSV.' : datasetRowTopologyError(graph) function apply() { if (error || disabled || !hasChanges) return setEdges(current => current.map(item => { @@ -59,11 +61,12 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId

    Dataset input

    {(['whole_dataset', 'per_row'] as const).map(value => { - const optionDisabled = disabled || (value === 'per_row' && !!otherRowDriver) + const optionDisabled = disabled || (value === 'per_row' && (!!otherRowDriver || factorBound)) return })}
    {otherRowDriver &&

    Per row is unavailable because another dataset already uses it. All per-row inputs must use the same dataset. Use Whole dataset here, or switch the other dataset to Whole dataset first.

    } + {factorBound &&

    {datasetRowFactorBindingError}

    } {schema.data &&

    {schema.data.row_count} original rows. {mode === 'per_row' ? 'Each row runs one complete protocol; Agents share one driver and select their own columns.' : 'Each replicate runs the protocol with the whole dataset.'}

    } {mode === 'per_row' && (displayedColumns.length <= 10 ? columnPicker :

    {columns.length} of {availableColumns.length} columns selected

    @@ -77,7 +80,7 @@ export function DatasetInputPanel({ edgeId, source, disabled = false }: { edgeId
    )} - {error &&

    {error}

    } + {error && !(factorBound && error === datasetRowFactorBindingError) &&

    {error}

    } {applied ?

    {applied === 'per_row' ? 'Per row' : 'Whole dataset'} applied to the draft. Publish the experiment to update production runs and results.

    :

    Apply changes the draft connection. Publish the experiment to use it for production runs and results.

    } diff --git a/frontend/src/components/protocol/edges/InteractEdge.tsx b/frontend/src/components/protocol/edges/InteractEdge.tsx index 73332f12..631ac8e0 100644 --- a/frontend/src/components/protocol/edges/InteractEdge.tsx +++ b/frontend/src/components/protocol/edges/InteractEdge.tsx @@ -143,7 +143,7 @@ export function InteractEdge({ )}
    setHovered(true)} onMouseLeave={() => setHovered(false)} > @@ -152,14 +152,14 @@ export function InteractEdge({ type="button" aria-label="Add node on this connection" title="Add node on this connection" - className="flex size-5 cursor-pointer items-center justify-center rounded-full text-primary hover:bg-primary/10" + className="flex size-6 cursor-pointer items-center justify-center rounded-full text-primary hover:bg-primary/10" onClick={() => requestEdgeInsert({ edgeId: id, source, target })} > - + )} {canConfigureDataset && - }> + }> } {canNarrow && ( @@ -170,11 +170,11 @@ export function InteractEdge({ type="button" aria-label="What passes on this connection" title={handoffLabel ? `Passes ${handoffLabel}` : 'What passes: full output'} - className={`flex size-5 cursor-pointer items-center justify-center rounded-full hover:bg-primary/10 ${handoffLabel ? 'bg-primary/15 text-primary' : 'text-primary'}`} + className={`flex size-6 cursor-pointer items-center justify-center rounded-full hover:bg-primary/10 ${handoffLabel ? 'bg-primary/15 text-primary' : 'text-primary'}`} /> } > - + @@ -185,10 +185,10 @@ export function InteractEdge({ type="button" aria-label="Delete connection" title="Delete connection" - className="flex size-5 cursor-pointer items-center justify-center rounded-full text-destructive hover:bg-destructive/10" + className="flex size-6 cursor-pointer items-center justify-center rounded-full text-destructive hover:bg-destructive/10" onClick={() => setEdges((eds) => eds.filter((e) => e.id !== id))} > - +
    diff --git a/frontend/src/components/protocol/factorCreationFields.test.ts b/frontend/src/components/protocol/factorCreationFields.test.ts new file mode 100644 index 00000000..6a9157b7 --- /dev/null +++ b/frontend/src/components/protocol/factorCreationFields.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from 'vitest' +import type { Edge, Node } from '@xyflow/react' +import { factorCreationFields, groupFactorCreationFields } from './bindableFields' + +it('sorts agent groups, component groups, and factor choices alphabetically', () => { + const nodes: Node[] = [ + { id: 'z', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Zebra', config: {} } }, + { id: 'a', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Alpha', config: {} } }, + { id: 'shared', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Shared skill', config: { skill_id: 's' } } }, + { id: 'data', type: 'dataset', position: { x: 0, y: 0 }, data: { label: 'Cohort', config: { dataset_id: 'd' } } }, + { id: 'loose', type: 'skill', position: { x: 0, y: 0 }, data: { label: 'Unconnected skill', config: { skill_id: 'u' } } }, + ] + const edges: Edge[] = [ + { id: 's-z', source: 'shared', target: 'z', targetHandle: 'skill' }, + { id: 's-a', source: 'shared', target: 'a', targetHandle: 'skill' }, + { id: 'd-a', source: 'data', target: 'a', targetHandle: 'dataset' }, + ] + const groups = groupFactorCreationFields(factorCreationFields(nodes, edges)) + // A component shared by several agents gets its own group, named after them in alphabetical order. + expect(groups.map(group => group.label)).toEqual(['Alpha', 'Alpha, Zebra', 'Shared or unconnected components', 'Zebra']) + expect(groups[0].connectors.map(connector => connector.label)).toEqual(['Agent', 'Datasets']) + expect(groups[0].connectors.find(connector => connector.label === 'Datasets')?.components.map(component => component.label)).toEqual(['Cohort', 'Datasets in this group']) + expect(groups[1].connectors.map(connector => connector.label)).toEqual(['Skills']) + expect(groups[1].connectors[0].components.map(component => component.label)).toEqual(['Shared skill', 'Skills in this group']) + expect(groups[1].connectors[0].components[1].fields.map(field => field.fieldLabel)).toEqual(['All skills on/off', 'Skill levels']) + expect(groups[2].connectors[0].components.map(component => component.label)).toEqual(['Unconnected skill']) +}) + +it('uses connector labels for every model and MCP family and orders components and fields by name', () => { + const kinds = ['mcp_tool', 'model_openai', 'mcp_client_tool', 'model_local', 'mcp_scikit_learn', 'model_anthropic', 'model_openrouter', 'model_azure_foundry', 'pattern_reason_act'] + const nodes: Node[] = [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Writer' } }, + ...kinds.map((kind) => ({ id: kind, type: kind, position: { x: 0, y: 0 }, data: { label: kind, config: {} } })), + ] + const edges: Edge[] = kinds.map((kind) => ({ id: kind, source: kind, target: 'agent' })) + const groups = groupFactorCreationFields(factorCreationFields(nodes, edges)) + const connectors = groups[0].connectors + expect(connectors.map((connector) => connector.label)).toEqual(['Agent', 'Model', 'Pattern', 'Tools']) + expect(connectors.find((connector) => connector.label === 'Model')?.components.map((component) => component.label)).toEqual(['model_anthropic', 'model_azure_foundry', 'model_local', 'model_openai', 'model_openrouter']) + expect(connectors.find((connector) => connector.label === 'Pattern')?.components[0].fields.map((field) => field.fieldLabel)).toEqual(['Include scratchpad', 'Max iterations', 'Observation format', 'Scratchpad window']) +}) + +it('routes both knowledge kinds through the dialog and offers connector modes once per agent', () => { + const nodes: Node[] = [ + { id: 'agent', type: 'agent', position: { x: 0, y: 0 }, data: { label: 'Researcher' } }, + { id: 'bundle', type: 'okf_bundle', position: { x: 0, y: 0 }, data: { label: 'Bundle', config: { bundle_id: 'b' } } }, + { id: 'doc', type: 'okf_document', position: { x: 0, y: 0 }, data: { label: 'Document', config: { document_id: 'd' } } }, + ] + const edges: Edge[] = ['bundle', 'doc'].map((id) => ({ id, source: id, target: 'agent', targetHandle: 'knowledge' })) + const fields = factorCreationFields(nodes, edges).filter((field) => field.connectorFactor?.kind === 'knowledge') + expect(fields.map((field) => field.fieldLabel)).toEqual(['This knowledge on/off', 'Knowledge levels', 'All knowledge on/off', 'This knowledge on/off']) + expect(fields.every((field) => field.pickerGroup?.category === 'Knowledge')).toBe(true) + expect(fields.filter((field) => field.connectorFactor?.agentId === 'agent')).toHaveLength(2) +}) diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index 3828c7cb..f909f5c2 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -21,6 +21,18 @@ export type LevelType = | 'script_config' | 'dataset_config' | 'tool_names' + | 'script_selection' + | 'script_toggle' + | 'tool_selection' + | 'tool_toggle' + | 'skill_selection' + | 'skill_toggle' + | 'sub_agent_selection' + | 'sub_agent_toggle' + | 'dataset_selection' + | 'dataset_toggle' + | 'knowledge_selection' + | 'knowledge_toggle' export const LEVEL_TYPE_LABELS: Record = { string: 'String', @@ -30,9 +42,21 @@ export const LEVEL_TYPE_LABELS: Record = { model_config: 'Provider & model', tool_config: 'Server & tools', pattern: 'Execution pattern', - script_config: 'Script', + script_config: 'Script variants (legacy)', dataset_config: 'Dataset', tool_names: 'Tools allowed', + script_selection: 'Script levels', + script_toggle: 'All agent scripts on/off', + tool_selection: 'Tool levels', + tool_toggle: 'All agent tools on/off', + skill_selection: 'Skills as levels', + skill_toggle: 'All skills on/off', + sub_agent_selection: 'Sub-Agent levels', + sub_agent_toggle: 'All agent sub-agents on/off', + dataset_selection: 'Dataset levels', + dataset_toggle: 'All datasets on/off', + knowledge_selection: 'Knowledge levels', + knowledge_toggle: 'All knowledge on/off', } // Whether a level of this kind is a structured value (an object, or -- for @@ -47,7 +71,19 @@ export function isStructuredLevelType(type: LevelType): boolean { type === 'pattern' || type === 'script_config' || type === 'dataset_config' || - type === 'tool_names' + type === 'tool_names' || + type === 'script_selection' || + type === 'script_toggle' || + type === 'tool_selection' || + type === 'tool_toggle' || + type === 'skill_selection' || + type === 'skill_toggle' || + type === 'sub_agent_selection' || + type === 'sub_agent_toggle' || + type === 'dataset_selection' || + type === 'dataset_toggle' || + type === 'knowledge_selection' || + type === 'knowledge_toggle' ) } diff --git a/frontend/src/components/protocol/layout.ts b/frontend/src/components/protocol/layout.ts index 0be6cea8..abb02a32 100644 --- a/frontend/src/components/protocol/layout.ts +++ b/frontend/src/components/protocol/layout.ts @@ -23,7 +23,7 @@ const CONNECTOR_X: { agent: { architectural_pattern: 0.05, skill: 0.18, - dataset: 0.71, + dataset: 0.73, knowledge: 0.9, model: 0.08, sub_agents: 0.32, diff --git a/frontend/src/components/protocol/nodeConfigIssues.ts b/frontend/src/components/protocol/nodeConfigIssues.ts index e53c0f91..fcf44240 100644 --- a/frontend/src/components/protocol/nodeConfigIssues.ts +++ b/frontend/src/components/protocol/nodeConfigIssues.ts @@ -1,8 +1,9 @@ +import { modelCapabilities } from '@/lib/modelCapabilities' import type { QueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' import { toPersistedGraph } from '@/lib/protocolGraph' import { isUnderIterated, raiseForTruncation, suggestedMaxIterations } from '@/lib/reasonActIterations' -import type { LLMSettingModelsResponse } from '@/types/llmSettings' +import type { LLMSettingModelsResponse, ModelCapabilities } from '@/types/llmSettings' import type { OkfBundle, OkfDocument } from '@/types/okf' import type { Skill } from '@/types/skills' import type { @@ -17,7 +18,7 @@ import type { SkillNodeData, } from '@/types/protocols' import { PROVIDER_META } from './nodes/ModelNode' -import { providerModelsKey } from './useProviderModels' +import { modelCapabilitiesKey, providerModelsKey } from './useProviderModels' export interface NodeConfigIssue { nodeId: string @@ -41,9 +42,8 @@ export interface NodeConfigIssue { // among the provider's own discovered list (ModelNode.tsx's own richer // check) -- read from cache only, via the exact same queryKey ModelNode.tsx // already populates by rendering on this same canvas, so this never fires -// its own network request or makes clicking Run wait on one. A cache miss -// (that query never ran, or hasn't resolved yet) just means "can't tell," -// same as ModelNode.tsx's own empty-list case -- not treated as an issue. +// its own network request or makes clicking Run wait on one. An unresolved +// custom model is flagged until its sampling capabilities load. // `truncatedCaps` maps a Reason + Act node id to the `max_iterations` the last // run of its agent actually hit (ProtocolCanvas.tsx's truncationByPattern). // Optional because the callers that run from a stored `Protocol.graph` alone @@ -106,11 +106,14 @@ export function findNodeConfigIssues( } } if (config?.max_tokens == null) issues.push('Max tokens is required') - // Same "unrecognized model defaults to temperature-only" fallback as - // ModelNodeInspector.tsx's own showTemperature -- required whenever - // it's the field actually offered for this model, so it's never - // Motoro's own silent ModelConfig default (0.7) filling the gap. - if ((selectedModelInfo?.supports_temperature ?? true) && config?.temperature == null) { + const customKey = modelCapabilitiesKey(config?.provider, config?.model) + const resolved = config?.resolved_capabilities ?? selectedModelInfo ?? queryClient.getQueryData(customKey) + if (config?.model && !resolved) { + issues.push(queryClient.getQueryState(customKey)?.status === 'error' + ? 'Could not resolve model capabilities. Try again.' + : 'Model capabilities are still loading') + } + if (resolved && modelCapabilities(resolved).supports_temperature && config?.temperature == null) { issues.push('Temperature is required') } break diff --git a/frontend/src/components/protocol/nodes/AgentNode.tsx b/frontend/src/components/protocol/nodes/AgentNode.tsx index cef07cb9..6713f7aa 100644 --- a/frontend/src/components/protocol/nodes/AgentNode.tsx +++ b/frontend/src/components/protocol/nodes/AgentNode.tsx @@ -38,6 +38,7 @@ export function AgentNode({ // Paired with runStatus, never derivable from it: a truncated run is still // `completed` (see NodeRunState.truncation). runTruncated?: boolean + runBypassed?: boolean missingModel?: boolean missingOutputParser?: boolean canRunAlone?: boolean @@ -62,7 +63,7 @@ export function AgentNode({ }) { const isSubAgent = data.isSubAgent === true const accent = nodeAccent(isSubAgent ? 'sub_agent' : 'agent') - const badge = nodeRunBadge(data.runStatus, data.runTruncated) + const badge = nodeRunBadge(data.runStatus, data.runTruncated, data.runBypassed) // Peers are offered to the model as function schemas -- that is the only // channel a consultation can be *chosen* through -- so an agent on a model // that can't accept them would know its peers exist and never be able to @@ -82,8 +83,10 @@ export function AgentNode({ ...(peerNeedsToolCalling ? ["This model can't call tools, so this agent can't consult or delegate to connected agents"] : []), ] const { updateNodeData } = useReactFlow() - const { requestRunNode } = useProtocolCanvasActions() - const isActive = data.active ?? true + const { requestRunNode, requestAgentFactor, requestSubAgentFactor, requestEditFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = isSubAgent && !!data.subAgentFactorControlled + const isActive = controlled || (data.active ?? true) + const factorCount = boundFactorCount(data) + (controlled ? 1 : 0) // Ids are stored, labels are only ever displayed -- the same invariant the // inspector's editor upholds via `toDisplayPrompt`. Without this the summary @@ -127,7 +130,8 @@ export function AgentNode({ updateNodeData(id, { active: !isActive })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { active: !isActive })} + onMakeFactor={experimentLocked ? undefined : isSubAgent ? requestSubAgentFactor && (() => requestSubAgentFactor(id)) : requestAgentFactor && (() => requestAgentFactor(id))} runAlone={{ canRun: !!data.canRunAlone, onRun: () => requestRunNode(id) }} /> {/* Sits inside the top-right corner so it stays clear of the Knowledge @@ -145,7 +149,7 @@ export function AgentNode({ node" anyway. Top-center (on hover) is NodeHoverToolbar, and the Architectural Pattern connector's own label/stub live OUTSIDE the card on the left of this edge, so neither competes for this corner. */} - {hasBoundFactor(data) && } + {factorCount > 0 && } {/* Main flow is left-to-right -- the bottom sub-connectors stay on the bottom edge regardless, since a config source hangs below a node no matter which way the main flow runs. @@ -225,20 +229,16 @@ export function AgentNode({ w-72 rather than the w-60 it was with three: at 240px the margins are 64px each, which two centered captions won't fit in; at 288px they're 88px. Hence a Pattern/Skill pair in the left margin (5% / 18%) and a - Dataset/Knowledge pair in the right (71% / 90%) -- the numbers + Dataset/Knowledge pair in the right (73% / 90%) -- the numbers themselves live in layout.ts's CONNECTOR_X, because addNode() has to place a connector's node at that same x -- each spaced so neither their ~26px stubs nor their centered captions collide. Every caption is centered directly above its own handle, mirroring the bottom row (see ConnectorHandleLabel's side="top" branch). - Dataset is at 71% rather than the 74% it started at: "Dataset" and - "Knowledge" are the two longest captions on this edge, and centered - at 74/90 they were within a pixel or two of touching. 71% is about - as far left as it can go -- the toolbar's right edge is at ~69%, and - the stub's own padding already overlaps it slightly (harmless, the - toolbar is above the stub's z-index and only there on hover, and the - visible "+" glyph still clears it). + Dataset sits at 73%, nudged slightly right from its previous 71% + position on both Agent and Sub-Agent cards. Its handle, caption, + "+" stub and newly added dataset nodes share that same position. The bottom sub-connectors: required Model (exactly one), optional repeatable Sub-Agents on parent Agents, optional max-1 Memory (visual @@ -287,7 +287,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Skill - + {/* Dataset -- the data an agent works ON, as opposed to the Tool connector's "capabilities it works WITH". UNCAPPED, like Skill/Knowledge/Tool: a cell's workspace holds one dataset per named @@ -321,7 +321,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Dataset - + {/* Knowledge -- registered OKF bundles, each a directory of Markdown concepts on the SERVER's disk that the agent reads AND writes as it works (see OkfBundleNodeData). Its own slot rather than sharing @@ -351,7 +351,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Knowledge - + {/* Handle id `model`; graphs saved before the rename carry these edges on `ai` or `llm` -- ProtocolCanvas.tsx rewrites those on load (migrateLegacyHandles) and the backend keeps accepting both (see @@ -379,7 +379,7 @@ export function AgentNode({ className="!size-2 !border-2 !bg-background !border-[color:var(--card-accent)]" /> Sub-Agents - + )} Tool - + + {data.factor_bindings?.script_selection && } {/* Output Parser -- the field spec the agent's answer is written to and read back out of. Last on the bottom edge, at 95%: it's the only connector here whose work outlives the agent's own turn, so it sits diff --git a/frontend/src/components/protocol/nodes/CircleNode.tsx b/frontend/src/components/protocol/nodes/CircleNode.tsx index b3dbcae3..232648f8 100644 --- a/frontend/src/components/protocol/nodes/CircleNode.tsx +++ b/frontend/src/components/protocol/nodes/CircleNode.tsx @@ -29,6 +29,7 @@ export function CircleNode({ dimmed, isActive, onToggleActive, + onMakeFactor, swap, handlePosition = 'top', }: { @@ -64,6 +65,7 @@ export function CircleNode({ // AgentNode/CriticGateNode's own usage. isActive?: boolean onToggleActive?: () => void + onMakeFactor?: () => void // Passed straight through to NodeHoverToolbar -- see its own comment. swap?: { label: string; onSwap: () => void } // Which edge of the circle the connector handle sits on -- 'top' (default) @@ -80,7 +82,7 @@ export function CircleNode({ const metricCount = metricsForNode(id).length return (
    - +
    0 && !alwaysVisible) return null @@ -58,6 +65,11 @@ export function ConnectorAddStub({ // node, so the line needs to be its LAST child to land in the same // "closest to the node" spot. return ( +
    + {factorName && {factorLabel} · {factorOnOff ? 'on/off' : `${factorLevelCount ?? 0} levels`}} +
    ) } diff --git a/frontend/src/components/protocol/nodes/CriticGateNode.tsx b/frontend/src/components/protocol/nodes/CriticGateNode.tsx index 634d2cb9..995079b3 100644 --- a/frontend/src/components/protocol/nodes/CriticGateNode.tsx +++ b/frontend/src/components/protocol/nodes/CriticGateNode.tsx @@ -20,11 +20,11 @@ export function CriticGateNode({ id, data, selected, -}: NodeProps & { data: CriticGateNodeData & { runStatus?: NodeRunStatus; runTruncated?: boolean } }) { +}: NodeProps & { data: CriticGateNodeData & { runStatus?: NodeRunStatus; runTruncated?: boolean; runBypassed?: boolean } }) { const enabled = data.config?.enabled ?? true const Icon = enabled ? ShieldCheck : ShieldOff const summary = enabled ? `Up to ${data.config?.max_revisions ?? 1} revision(s)` : 'Gate disabled' - const badge = nodeRunBadge(data.runStatus, data.runTruncated) + const badge = nodeRunBadge(data.runStatus, data.runTruncated, data.runBypassed) const modelConnections = useNodeConnections({ id, handleType: 'target', handleId: 'model' }) const { updateNodeData } = useReactFlow() const { requestMakeFactor } = useProtocolCanvasActions() diff --git a/frontend/src/components/protocol/nodes/DatasetNode.tsx b/frontend/src/components/protocol/nodes/DatasetNode.tsx index 2874a544..362b6b65 100644 --- a/frontend/src/components/protocol/nodes/DatasetNode.tsx +++ b/frontend/src/components/protocol/nodes/DatasetNode.tsx @@ -4,6 +4,7 @@ import { Database } from 'lucide-react' import type { DatasetNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' import { CircleNode } from './CircleNode' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' // Declares which registered dataset an Agent's workspace tools operate on -- // a real runtime effect once wired (see DatasetNodeData's own comment in @@ -22,7 +23,9 @@ const ACCENT = nodeAccent('dataset') export function DatasetNode({ id, data, selected }: NodeProps & { data: DatasetNodeData }) { const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestDatasetFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.datasetFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) return ( requestDatasetFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/nodes/McpClientToolNode.tsx b/frontend/src/components/protocol/nodes/McpClientToolNode.tsx index 913e8a04..87d3b3dd 100644 --- a/frontend/src/components/protocol/nodes/McpClientToolNode.tsx +++ b/frontend/src/components/protocol/nodes/McpClientToolNode.tsx @@ -3,6 +3,7 @@ import { useReactFlow, type NodeProps } from '@xyflow/react' import { Plug } from 'lucide-react' import type { McpToolNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' // Its own hue, distinct from McpToolNode's: "a server the deployment @@ -21,7 +22,9 @@ export function McpClientToolNode({ id, data, selected }: NodeProps & { data: Mc const toolNames = data.config?.tool_names ?? [] const summary = toolNames.length > 0 ? `${data.config.server_name ?? '?'}: ${toolNames.join(', ')}` : null const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestToolFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.toolFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) const allowListIsFactor = !!data.factor_bindings?.['config.tool_names'] return ( @@ -37,10 +40,11 @@ export function McpClientToolNode({ id, data, selected }: NodeProps & { data: Mc // too, so the allow-list is the only thing that can be empty -- and // same suppression once that allow-list is itself a factor. warning={summary || allowListIsFactor ? undefined : 'Not configured -- allow at least one tool'} - factorCount={boundFactorCount(data)} + factorCount={controlled ? 1 : boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestToolFactor ? () => requestToolFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/nodes/McpToolNode.tsx b/frontend/src/components/protocol/nodes/McpToolNode.tsx index 12adf198..407a3111 100644 --- a/frontend/src/components/protocol/nodes/McpToolNode.tsx +++ b/frontend/src/components/protocol/nodes/McpToolNode.tsx @@ -3,6 +3,7 @@ import { useReactFlow, type NodeProps } from '@xyflow/react' import { Wrench } from 'lucide-react' import type { McpToolNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' // One hue for the kind, not per instance: all mcp_tool nodes share this, and @@ -23,7 +24,9 @@ export function McpToolNode({ id, data, selected }: NodeProps & { data: McpToolN const toolNames = data.config?.tool_names ?? [] const summary = toolNames.length > 0 ? `${data.config.server_name ?? '?'}: ${toolNames.join(', ')}` : null const { updateNodeData } = useReactFlow() - const enabled = data.config?.enabled ?? true + const { requestToolFactor, experimentLocked } = useProtocolCanvasActions() + const controlled = !!data.toolFactorControlled + const enabled = controlled || (data.config?.enabled ?? true) // Every MCP node already HAS its server (picked in the browser at creation // time), so telling its user to go pick one would send them looking for a // dropdown that isn't there -- only the allow-list can be empty. Not shown @@ -43,10 +46,11 @@ export function McpToolNode({ id, data, selected }: NodeProps & { data: McpToolN placeholder="MCP Tool" handleId="tool" warning={summary ? undefined : warning} - factorCount={boundFactorCount(data)} + factorCount={controlled ? 1 : boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestToolFactor ? () => requestToolFactor(id) : undefined} dimmed={!enabled} isActive={enabled} - onToggleActive={() => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} + onToggleActive={controlled ? undefined : () => updateNodeData(id, { config: { ...data.config, enabled: !enabled } })} /> ) } diff --git a/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx b/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx new file mode 100644 index 00000000..31a7416b --- /dev/null +++ b/frontend/src/components/protocol/nodes/ModelFactorGroup.test.tsx @@ -0,0 +1,50 @@ +import { fireEvent, render, screen } from '@testing-library/react' +import { beforeEach, expect, it, vi } from 'vitest' +import type { DesignFactor } from '@/types/experiments' +import { ModelFactorGroup } from './ModelFactorGroup' + +const { edit, updateInternals } = vi.hoisted(() => ({ edit: vi.fn(), updateInternals: vi.fn() })) +vi.mock('@xyflow/react', () => ({ + Handle: ({ id }: { id: string }) =>
    {id}
    , + Position: { Top: 'top' }, + useUpdateNodeInternals: () => updateInternals, +})) +vi.mock('../ProtocolCanvasContext', () => ({ useProtocolCanvasActions: () => ({ requestModelFactor: edit }) })) +vi.mock('./NodeHoverToolbar', () => ({ NodeHoverToolbar: () => null })) +vi.mock('../useProviderModels', () => ({ useProviderModels: () => ({ models: [], modelsQuery: {} }) })) + +const factor: DesignFactor = { + name: 'Writer:Models', level_type: 'model_config', level_labels: ['Baseline', 'Alternative'], + levels: [ + { provider: 'openai', model: 'gpt-5', temperature: 0.7, max_tokens: 1000 }, + { provider: 'anthropic', model: 'claude-sonnet', temperature: 1, max_tokens: 2000 }, + ], +} + +beforeEach(() => { localStorage.clear(); vi.clearAllMocks() }) + +it('expands derived configurations with one handle and opens the selected level', () => { + render() + expect(screen.getByText('Model factor · 2 levels')).toBeInTheDocument() + expect(screen.getByText('OpenAI')).toBeInTheDocument() + expect(screen.getByText('Anthropic')).toBeInTheDocument() + expect(screen.queryByText('model=gpt-5')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Expand model levels' })) + expect(screen.getByText('model=gpt-5')).toBeInTheDocument() + expect(screen.getByText('model=claude-sonnet')).toBeInTheDocument() + expect(screen.getByText(/default test selection/)).toBeInTheDocument() + expect(screen.getAllByTestId('handle')).toHaveLength(1) + fireEvent.click(screen.getByRole('button', { name: 'Edit model level 2: Alternative' })) + expect(edit).toHaveBeenCalledWith('model', 1) + expect(updateInternals).toHaveBeenCalledWith('model') +}) + +it('remembers expansion per node and keeps same-provider configurations separate', () => { + const { unmount } = render() + fireEvent.click(screen.getByRole('button', { name: 'Expand model levels' })) + unmount() + render() + expect(screen.getByRole('button', { name: 'Collapse model levels' })).toHaveAttribute('aria-expanded', 'true') + expect(screen.getAllByText('model=gpt-5')).toHaveLength(2) + expect(screen.getAllByTestId('handle')).toHaveLength(1) +}) diff --git a/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx new file mode 100644 index 00000000..78ec343a --- /dev/null +++ b/frontend/src/components/protocol/nodes/ModelFactorGroup.tsx @@ -0,0 +1,96 @@ +import type { ModelCapabilities } from '@/types/llmSettings' +import { modelCapabilities } from '@/lib/modelCapabilities' +import { useEffect, useState } from 'react' +import { Handle, Position, useUpdateNodeInternals } from '@xyflow/react' +import { ChevronDown, ChevronUp, Split } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Card } from '@/components/ui/card' +import { nodeAccent } from '@/lib/nodeAccent' +import { cardAccent } from '@/lib/utils' +import type { DesignFactor } from '@/types/experiments' +import { factorLevelLabels } from '../factorLevels' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' +import { useProviderModels } from '../useProviderModels' +import { PROVIDER_META } from './ModelNode' +import { NodeHoverToolbar } from './NodeHoverToolbar' +import { WarningBadge } from './WarningBadge' + +const expansionKey = (id: string) => `asaree:model-factor-expanded:${id}` + +function LevelPreview({ level, label, index, disabled, onEdit }: { + level: unknown + label: string + index: number + disabled: boolean + onEdit: () => void +}) { + const config = (level ?? {}) as Record + const provider = String(config.provider ?? '') + const meta = PROVIDER_META[provider] + const Icon = meta?.icon ?? Split + const { models, capabilities: resolvedCapabilities, capabilitiesPending, capabilitiesError } = useProviderModels(provider, config.model as string | undefined, config.resolved_capabilities as ModelCapabilities | undefined) + const model = models.find((model) => model.id === config.model) + const capabilities = modelCapabilities(resolvedCapabilities ?? model, config.resolved_capabilities as ModelCapabilities | undefined) + const effort = !capabilitiesPending && capabilities.supports_effort + const temperature = !capabilitiesPending && capabilities.supports_temperature + return + + +} + +/** Derived previews only: this group remains one graph node and one source handle. */ +export function ModelFactorGroup({ id, factor, selected, warning }: { + id: string + factor: DesignFactor + selected?: boolean + warning?: string[] +}) { + const { requestModelFactor, experimentLocked } = useProtocolCanvasActions() + const [expanded, setExpanded] = useState(() => { + try { return localStorage.getItem(expansionKey(id)) === 'true' } catch { return false } + }) + const updateNodeInternals = useUpdateNodeInternals() + useEffect(() => { updateNodeInternals(id) }, [id, expanded, factor.levels.length, updateNodeInternals]) + const providers = [...new Set(factor.levels.map((level) => String((level as Record)?.provider ?? '')))] + const labels = factorLevelLabels(factor) + const accent = nodeAccent('model') + return
    + requestModelFactor(id) : undefined} /> + + +
    + +

    {factor.name}

    Model factor · {factor.levels.length} levels

    + {!!warning?.length && } + +
    +
    {providers.map((provider) => { + const meta = PROVIDER_META[provider] + const Icon = meta?.icon ?? Split + return {meta?.label ?? provider} + })}
    + {expanded &&
    +

    One configuration per cell

    + {factor.levels.map((level, index) =>
    + requestModelFactor?.(id, index)} /> +
    )} +
    } +     +
    +} diff --git a/frontend/src/components/protocol/nodes/ModelNode.tsx b/frontend/src/components/protocol/nodes/ModelNode.tsx index 71f21501..94d53545 100644 --- a/frontend/src/components/protocol/nodes/ModelNode.tsx +++ b/frontend/src/components/protocol/nodes/ModelNode.tsx @@ -1,10 +1,13 @@ +import { modelCapabilities } from '@/lib/modelCapabilities' import type { NodeProps } from '@xyflow/react' import { Atom, Cloud, HardDrive, Route, Sparkles } from 'lucide-react' import { nodeAccent } from '@/lib/nodeAccent' import type { ModelNodeData } from '@/types/protocols' import { boundFactorCount } from '../bindableFields' import { useProviderModels } from '../useProviderModels' +import { useProtocolCanvasActions } from '../ProtocolCanvasContext' import { CircleNode } from './CircleNode' +import { ModelFactorGroup } from './ModelFactorGroup' // One shared card renderer for all three LLM provider node types // (model_anthropic/model_openai/model_azure_foundry -- see ModelNodeData's own @@ -31,6 +34,8 @@ export const PROVIDER_META: Record 0 && !selectedModelInfo) { warnings.push(`"${modelName}" isn't available on your ${meta.label} credential`) } + if (capabilitiesPending) warnings.push(capabilitiesError ?? 'Model capabilities are still loading') if (data.config?.max_tokens == null) warnings.push('Max tokens is required') - // Same "unrecognized model defaults to temperature-only" fallback as - // ModelNodeInspector.tsx's own showTemperature -- Temperature is required + // Same resolved capability metadata as ModelNodeInspector -- Temperature is required // (not left to Motoro's own silent 0.7 default) whenever it's the field // actually offered for this model. - if ((selectedModelInfo?.supports_temperature ?? true) && data.config?.temperature == null) { + if (!capabilitiesPending && modelCapabilities(resolvedCapabilities ?? selectedModelInfo, data.config?.resolved_capabilities).supports_temperature && data.config?.temperature == null) { warnings.push('Temperature is required') } + if (factor) return + return ( 0 ? warnings : undefined} factorCount={boundFactorCount(data)} + onMakeFactor={!experimentLocked && requestModelFactor ? () => requestModelFactor(id) : undefined} /> ) } diff --git a/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx b/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx index 39c5757c..76060ea9 100644 --- a/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx +++ b/frontend/src/components/protocol/nodes/NodeHoverToolbar.tsx @@ -18,6 +18,7 @@ function ToolbarIconButton({
    - {cell.label} +
    + {cell.cell_label} - {f.name in cell.factorValues ? displayFactorLevel(experiment.design_spec, f.name, cell.factorValues[f.name], cell.label) : '—'} + {f.name in cell.factor_values ? displayFactorLevel(experiment.design_spec, f.name, cell.factor_values[f.name], cell.cell_label) : '—'} - {formatMetricValue(m, meanMetric(cell.replicates, m))} + {formatMetricValue(m, pooledMean([cell], m))} - - {cell.scoredReplicateCount}/{cell.replicates.length} replicates scored + + {cell.scored_count}/{cell.replicate_count} replicates scored - {new Date(cell.updatedAt).toLocaleString()} + {cell.updated_at ? new Date(cell.updated_at).toLocaleString() : '—'}