Feat/factors

.dockerignore

@@ -10,6 +10,8 @@ __pycache__/
 dist/
 *.egg-info/
 data/
+# Database dumps (up to hundreds of MB) -- mounted at runtime, never built in.
+backups/
 # .git/ stays excluded even though the build has to read it. hatch-vcs derives
 # the package version from the repo's tags, so the final `uv sync` can't build
 # this project without that metadata -- but it arrives as a *named build context*

.github/workflows/ci.yml

@@ -44,6 +44,11 @@ jobs:
       DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro
       ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro
       ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/asaree
+      # Opt-in suites that skip without these. Results checks run every revision
+      # in a throwaway schema inside a rolled-back transaction; the migration
+      # re-run checks create and drop their own scratch database on this server.
+      ASAREE_RESULTS_TEST_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/asaree
+      ASAREE_MIGRATIONS_TEST_DATABASE_URL: postgresql+asyncpg://agentic:agentic@localhost:5432/motoro
 
     steps:
       - uses: actions/checkout@v6
@@ -146,6 +151,11 @@ jobs:
       - name: Type check
         run: npx tsc -b
 
+      # Component and lib tests (vitest + jsdom). Without this step a stale
+      # test can sit red on a branch with CI still green.
+      - name: Unit tests
+        run: npx vitest run
+
       # oxlint exits 0 on warnings, so this reports rather than gates. The
       # standing baseline is 9 warnings; it is here to make that number
       # visible and stop it climbing unnoticed.

.gitignore

@@ -10,7 +10,10 @@ dist/
 data/
 local_setup/
 local_files/
-backups/
+# Dumps stay local; the directory itself is tracked so a fresh clone has it,
+# owned by you, before compose bind-mounts it into db-backup.
+backups/*
+!backups/.gitkeep
 
 # Generated by hatch-vcs at build time from the git tag (see pyproject.toml).
 src/asaree/_version.py

Dockerfile

@@ -19,7 +19,10 @@ FROM python:3.13-slim AS application
 WORKDIR /app
 
 # git: needed by uv to resolve Motoro's pinned git dependency below.
-RUN apt-get update && apt-get install -y --no-install-recommends git \
+# postgresql-client: pg_dump/pg_restore for the pre-migration backup step
+# (asaree/migrations/backup.py). Debian's client is newer than the pg16 server,
+# which pg_dump supports; restore with this same image's pg_restore.
+RUN apt-get update && apt-get install -y --no-install-recommends git postgresql-client \
     && rm -rf /var/lib/apt/lists/*
 RUN pip install --no-cache-dir uv
 

README.md

@@ -183,6 +183,12 @@ servers (`asaree-sklearn-dc`, `-eda`, `-fs`, `-fte`, `-model`, `-stats`, from
 `mcp-servers/`) — ships as a dependency of the app and registers itself each
 time the app or worker starts (see `asaree.services.system_mcp_servers`).
 
+## Large experiment results
+
+For large experiment results, the dedicated result worker handles summaries,
+batch commands and exports. Deployment, recovery, API/SDK usage and pending
+PostgreSQL acceptance checks are documented in [RESULTS_OPERATIONS.md](RESULTS_OPERATIONS.md).
+
 ## Resetting your dev environment
 
 One Postgres server hosts two databases in a single volume — `motoro` (core's

RESULTS_OPERATIONS.md

@@ -0,0 +1,177 @@
+# Scalable execution results
+
+Runs and Results use `/api/experiments/{id}/execution-results`. The canvas uses
+`/api/protocols/{id}/run-heads?limit=1` to recover its latest execution. None of
+these list reads hydrates Motoro agents, traces, complete outputs, or histories.
+
+Execution and cell pages default to 20 records and cap at 100. Cursors bind the
+protocol publication, design revision and filters. Metric cursors also bind the
+published summary generation; refresh from the first page after it changes.
+Per-row pages generate only the requested window, including unstarted source
+rows once their count has been prepared. Stable slot IDs identify completed
+records. `unplanned:<replicate UUID>:<row index>` identifies a row without a
+persisted slot and cannot be used to inspect an attempt.
+
+Summaries are rebuildable projections. Database triggers record dirty scopes in
+`result_refreshes` in the same transaction as result changes. The result worker
+coalesces and rebuilds each scope from current facts, streams cell writes, and
+publishes cell/overview data atomically. Repeated deliveries do not double-count
+attempts. The UI shows records while a summary or usage backfill is preparing.
+It refreshes visible pages when a summary generation advances, retains loaded
+data on transient failures, and stops polling settled executions. Refresh or
+window focus discovers work started by another client after a scope settles.
+
+Operational usage belongs to each production `ProtocolRun`, independently of
+declared measurements. Accounting reads scalar, owner-scoped Motoro rows in
+batches of 200, including all causally attributed calls (not only final node
+pointers). A pointer fallback supports older records. Legacy calls without
+either attribution or a retained pointer cannot be recovered. Unreported or
+ambiguous default-zero provider fields remain unknown. Execution duration uses
+actual start/task completion, excluding queue time. Current execution usage
+and all-attempt spend are separate. Immutable measurement/evidence documents
+are never rewritten by accounting backfill.
+
+Only explicitly declared built-in runtime measurements enter scalar cell
+aggregates and ranking. Global runtime metrics use catalog keys; node runtime
+metrics use stable metric IDs with display labels. Display names do not identify
+measurements. Reported and legacy values remain opaque, retain
+their recorded provenance, and are not coerced or ranked. Per-row outcomes are
+never aggregated or ranked. Truncated attempts are excluded from scalar
+comparison. Primary ranks are maintained for the whole scope, including pages
+after the first.
+
+Selected attempts return bounded previews. Oversized observation values and
+artifact payloads carry explicit preview markers; a measured JSON null remains
+different from a deferred value. Exact JSON documents are available through
+`attempts/{run}/evidence` in chunks of at most 16,384 characters. Node output and
+received prompt text use the same chunk bound. Trace pages contain 20 metadata
+records; full step evidence loads only after selecting a step. Conversation
+messages are paged and large messages are marked previews; complete transcripts
+remain available through the recorded JSON reader.
+
+## Durable batch commands
+
+`POST execution-results/jobs` accepts `kind=plan|cancel|export`, a stable
+`request_key`, and the selected publication/design query parameters. Creation
+returns 202 immediately. Repeating the same owned request key and selection
+returns the same job; conflicting reuse is rejected.
+
+A plan first prepares a preview in bounded windows and stores exact parent,
+row, slot, and expected-attempt identities. `awaiting_confirmation` includes
+the exact planned execution count. Opening a dialog never launches an LLM.
+`POST jobs/{id}/confirm` queues that sealed selection. Each small transaction
+creates runs and advances their intent checkpoints together. Slots changed
+since preview are skipped, so a retry cannot silently re-bill a newer attempt.
+Publication/design changes require preparing a new batch. Runs already created
+remain pinned to their immutable publications.
+
+Completed and limit-reached executions (both finished with a result) are
+skipped unless explicitly selected by slot ID or `rerun_completed=true`.
+Explicit selections cap at 100 IDs; an all-scope command does not send a
+browser-sized ID list. Failed/cancelled work is retried automatically. A plan's cancel action stops remaining planning; already-created runs
+continue. The Runs tab's Stop all creates a scoped cancellation command, stops
+matching planners and cancels all active attempts, including earlier attempts
+that are no longer the latest slot result.
+
+Run cell and Stop cell use the same durable API with `cell_id`. Stopping one
+cell excludes it from existing broad planners while allowing their other cells
+to continue. Publication predicates also constrain the factor levels and
+replicate count: adding cells to a mutable design revision cannot expand the
+cohort belonging to an earlier immutable publication.
+
+SQL intent survives worker/Redis loss. Created pending runs are repeatedly
+offered to ARQ with the same run job ID; executor guards prevent completed or
+cancelled runs from executing again. Delivery continues for committed runs even
+if later planning fails or is cancelled. Preview-only cancellation cannot
+cancel a confirmed job. Unconfirmed previews expire after 24 hours.
+
+## Exports and source metadata
+
+An export is a background job, limited to one export at a time globally. It
+writes the researcher-facing analysis CSV -- the same columns and values as the
+synchronous `run-results.csv`: one column per factor (short level labels),
+`duration_seconds`, `input_tokens`, `output_tokens`, `total_tokens`,
+`cost_usd`, one column per metric, observation statuses and run metadata. Rows
+are built by the Results projection a chunk of replicates at a time (two
+passes: the first lays out columns from compact stubs, the second writes rows
+in factor-level order), so memory stays bounded at any size; 15,786 runs take
+about 20 seconds. Every chunk reads one REPEATABLE READ snapshot, so the file is
+consistent even while runs finish. The file is written to an owned temporary
+path and renamed into place; progress reports bytes, cancellation is checked
+between chunks, and a restart repeats the export. The schema sidecar is the same
+`result_rows_schema` document as `run-results.schema.json`. The synchronous
+endpoint keeps a 500-run cap and returns 413 above it, pointing to this job.
+
+Completed exports are downloaded natively in the browser or streamed to disk
+by the SDK. Download capabilities bind job/owner, expire after five minutes,
+and cannot supply filesystem paths. CSV/schema files expire after 24 hours.
+
+Original CSV rows are verified by streaming SHA-256 and indexed into an immutable
+SQLite cache. Construction uses batches of 1,000 rows and atomic rename.
+Subsequent planning windows use random indexed row access instead of reparsing
+the original CSV for every retry. Dataset row-schema HTTP reads return 202 with
+`ready=false`/`row_count=null` while the worker prepares metadata; they never
+read or parse source files in the request. The UI polls that small metadata
+record while preparing. Source preparation records hash/CSV validation failures
+without exposing paths. Count metadata describes the last verified registered
+hash; execution still verifies the pinned source identity before running.
+
+## Deployment and recovery
+
+0. Compose's `db-backup` runs first: when either database has a pending
+   migration, it writes `backups/{db}-{revision}-before-v{release}-{UTC}.dump`
+   (the database at that schema revision, saved before that release migrated it) and keeps the newest
+   five per database; manual dumps in `backups/` are left alone. If it fails, no
+   migration runs. Outside compose, run `python -m asaree.migrations backup`
+   before migrating. To restore one (stop the app first):
+   `docker compose run --rm --no-deps -e PGPASSWORD=... --entrypoint pg_restore db-backup -h motoro-postgres -U agentic --clean --if-exists --no-owner -d asaree /app/backups/<file>`.
+   One error about `transaction_timeout` is expected and harmless (the image's
+   client is newer than the server). These are troubleshooting snapshots taken
+   only at deploy time, not a backup strategy.
+1. Apply Motoro's migrations, then ASAREE's (through `718293a4b5c6`), using the
+   usual migration deployment. Historical JSON evidence is not rewritten in migration.
+2. The usage and trace reads go through Motoro's `list_run_summaries` and
+   `list_run_step_summaries`; their indexes (`ix_agent_runs_run_metadata`,
+   `ix_run_steps_run_sequence`) ship with Motoro's own migrations. ASAREE never
+   creates indexes on core tables.
+3. Run `arq asaree.worker.result_settings.ResultWorkerSettings` alongside the
+   production execution worker. Compose defines `asaree-results-worker`, which
+   waits for both migration steps. Both workers and the API must share the data
+   volume and configured export/row-index paths.
+4. Keep `ASAREE_RESULT_EXPORT_DIR` and `ASAREE_RESULT_ROW_INDEX_DIR` on persistent
+   shared storage (defaults are `./data/result-exports` and `./data/result-row-indexes`).
+5. To requeue all existing overview projections, run
+   `python scripts/rebuild_result_summaries.py`. The worker rebuilds from facts
+   and retries unfinished accounting; no evidence or execution timestamps change.
+
+Watch `result_refreshes.generation - processed_generation`, refresh `error`,
+unfinished `protocol_runs.usage_summary`, pending `result_jobs`, and worker logs
+for `result_summary_refresh_failed`, `result_usage_capture_failed`,
+`result_command_database_retry`, and `result_command_delivery_failed`.
+Size the dedicated result worker separately from LLM concurrency. Commands use
+bounded work windows; exports, usage capture and source preparation each have
+an independent global advisory lock. Read requests have a five-second SQL
+statement deadline and bounded page sizes. Read replicas are not required.
+
+SDK entry points: `execution_summary`, `iter_executions`, `create_result_job`,
+`result_job`, `confirm_result_job`, `cancel_result_job`, `iter_evidence_text`,
+and `download_result_export`. Supply a pinned publication in `scope` while
+iterating and keep the same `request_key` when retrying job creation.
+
+## Validation status
+
+Focused Python/SDK tests, frontend tests, TypeScript, Ruff and the new service
+type checks run locally without infrastructure. PostgreSQL validation is
+**pending**, as requested; no live migration or production-latency result is
+claimed. `tests/test_result_reads.py` applies the complete migration chain in
+a unique schema inside a rolled-back transaction when
+`ASAREE_RESULTS_TEST_DATABASE_URL` points to a disposable PostgreSQL database.
+
+The opt-in acceptance tests (`ASAREE_RESULTS_RUN_SCALE=1`) exercise 15,786,
+100,000 and 1,000,000 runs with 32 KiB node outputs and 8 KiB opaque reports.
+They require 20-record pages below 64 KiB, cached summaries below 16 KiB, and
+cached summary plus first-page latency below two seconds. Run these tests only
+on the disposable database; the million-run case intentionally creates a large
+fixture. Before rollout, execute these checks and inspect PostgreSQL plans for
+slot lookup and metric ranking, including concurrent execution/outbox updates,
+worker restart, Redis interruption, export COPY/download, and downgrade/upgrade.

compose.dev.yml

@@ -52,3 +52,10 @@ services:
       - ./src:/app/src
       - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro
       - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro
+
+  asaree-results-worker:
+    volumes:
+      - ./data:/app/data
+      - ./src:/app/src
+      - ${MOTORO_SRC:-../Motoro/src/motoro}:/app/.venv/lib/python3.13/site-packages/motoro:ro
+      - ./workspace-core/src/asaree_workspace_core:/app/workspace-core/src/asaree_workspace_core:ro

compose.yml

@@ -75,6 +75,32 @@ services:
   # Runs to completion and exits 0 -- `Exited (0)` in `docker compose ps` is
   # success, hence restart: "no". `--url` is passed explicitly because core's CLI
   # otherwise reads a bare DATABASE_URL, which nothing here sets.
+  # Snapshots both databases before either migration chain runs, only when a
+  # migration is pending: backups/{db}-{revision}-before-v{release}-{UTC}.dump.
+  # Troubleshooting aid, not a backup strategy -- see asaree/migrations/backup.py.
+  # Runs as root so it can always write the mount, then hands each dump to the
+  # owner of ./backups (you), whatever your UID is.
+  db-backup:
+    build:
+      context: .
+      secrets:
+        - gh_token
+      additional_contexts:
+        gitdir: ./.git
+    container_name: db-backup
+    depends_on:
+      motoro-postgres:
+        condition: service_healthy
+    env_file: .env
+    environment:
+      ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro
+      ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree
+    entrypoint: ["python", "-m", "asaree.migrations"]
+    command: ["backup", "--dir", "/app/backups"]
+    volumes:
+      - ./backups:/app/backups
+    restart: "no"
+
   motoro-migrate:
     build:
       context: .
@@ -88,6 +114,9 @@ services:
       # running initdb, which races the creation of the database being migrated.
       motoro-postgres:
         condition: service_healthy
+      # No migration runs until both databases are snapshotted (or need none).
+      db-backup:
+        condition: service_completed_successfully
     entrypoint: ["python", "-m", "motoro.migrations"]
     command:
       - deploy
@@ -218,6 +247,32 @@ services:
       - ./data:/app/data
     restart: unless-stopped
 
+  # Result maintenance and bulk commands never compete for LLM worker slots.
+  asaree-results-worker:
+    build:
+      context: .
+      secrets:
+        - gh_token
+      additional_contexts:
+        gitdir: ./.git
+    depends_on:
+      # asaree-migrate itself waits for motoro-migrate, which creates the core
+      # indexes the usage/trace reads rely on.
+      asaree-migrate:
+        condition: service_completed_successfully
+      motoro-redis:
+        condition: service_healthy
+    env_file: .env
+    environment:
+      ASAREE_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/motoro
+      ASAREE_PRODUCT_DATABASE_URL: postgresql+asyncpg://agentic:agentic@motoro-postgres:5432/asaree
+      ASAREE_REDIS_URL: redis://motoro-redis:6379/0
+      ASAREE_OKF_BUNDLE_ROOT: /app/data
+    command: ["arq", "asaree.worker.result_settings.ResultWorkerSettings"]
+    volumes:
+      - ./data:/app/data
+    restart: unless-stopped
+
   # Integration-test runner on the Compose network. The tests intentionally
   # use real Postgres (JSONB and core's vector schema rule out SQLite), so run
   # them beside the database instead of relying on a host-side localhost port.