From ac6d49617d6d02face126ffebb8f85378f4b3a9d Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 11:25:01 -0700 Subject: [PATCH 01/18] feat(sdk): bring client up to API parity --- sdk/README.md | 24 +- sdk/pyproject.toml | 2 +- sdk/src/asaree_client/__init__.py | 47 +++- sdk/src/asaree_client/client.py | 19 ++ sdk/src/asaree_client/models.py | 221 +++++++++++++++++- sdk/src/asaree_client/resources/datasets.py | 31 ++- .../asaree_client/resources/experiments.py | 164 +++++++++++-- .../asaree_client/resources/llm_settings.py | 24 +- sdk/src/asaree_client/resources/okf.py | 68 ++++++ sdk/src/asaree_client/resources/protocols.py | 58 +++-- sdk/src/asaree_client/resources/skills.py | 90 +++++++ sdk/src/asaree_client/resources/tools.py | 59 ++++- sdk/tests/test_current_api.py | 170 ++++++++++++++ 13 files changed, 913 insertions(+), 64 deletions(-) create mode 100644 sdk/src/asaree_client/resources/okf.py create mode 100644 sdk/src/asaree_client/resources/skills.py create mode 100644 sdk/tests/test_current_api.py diff --git a/sdk/README.md b/sdk/README.md index d67cc28f..36f3d4be 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -1,13 +1,13 @@ # asaree-client -A trimmed, synchronous SDK for ASAREE. It covers exactly the resources a -driver notebook needs to run a factorial experiment end to end — agents, -runs, experiments/cells/replicates, datasets, and MCP tool passthrough — not a full -mirror of every ASAREE endpoint. - -Deliberately not a copy of `ares_client`: ASAREE's runs execute inline -(`POST /runs` returns only once the run is terminal), so `runs.wait()` here -is a trivial re-fetch, not a poll loop. And the notebook's old +A synchronous SDK for ASAREE. It covers the complete programmatic research +workflow: agents and runs, experiments/designs/results, protocol publishing and +execution, datasets and workspace lineage, MCP servers, Agent Skills, OKF +knowledge, and per-user LLM settings. + +Deliberately not a copy of `ares_client`: ASAREE's runs execute through a +background worker (`POST /runs` returns while the run is pending), so +`runs.wait()` polls until a terminal result. The notebook's old `client.runs.update(run_id, metadata=...)` calls have no equivalent — that data now belongs on a `FactorialReplicateResult` row, written via `client.experiments.upsert_replicate(...)`. @@ -51,11 +51,5 @@ for replicate in replicates: run_id=run.id, metric_values={"roc_auc": 0.91}, ) -results = client.experiments.analyze( - experiment.id, - condition_factors=["tier"], - positive_levels={"tier": "critic"}, - reference_condition={"tier": "baseline"}, - primary_metric="roc_auc", -) +results = client.experiments.get_results(experiment.id) ``` diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index b31f697d..8e6a0f8d 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -6,7 +6,7 @@ name = "asaree-client" # __version__ -- which _transport sends as the User-Agent on every request -- # now names the exact build rather than a number someone forgot to bump. dynamic = ["version"] -description = "Trimmed synchronous SDK for ASAREE — just enough to drive a factorial experiment from a notebook." +description = "Synchronous Python SDK for ASAREE." readme = "README.md" requires-python = ">=3.12" diff --git a/sdk/src/asaree_client/__init__.py b/sdk/src/asaree_client/__init__.py index 3aa27875..f4a673e7 100644 --- a/sdk/src/asaree_client/__init__.py +++ b/sdk/src/asaree_client/__init__.py @@ -1,4 +1,4 @@ -"""Trimmed synchronous SDK for ASAREE.""" +"""Synchronous Python SDK for ASAREE.""" from __future__ import annotations @@ -19,16 +19,38 @@ ) from asaree_client.models import ( Agent, + CellRunBatch, + DesignImpact, DesignRevision, + DirectoryListing, + DiscoveredSkill, Experiment, ExperimentArtifact, + ExperimentResults, + ExperimentRunResults, + LLMConnectionCheck, + LLMModelInfo, + LLMModels, LLMSetting, MCPServer, + MeasurementCapabilities, + MeasurementPlanValidation, + MeasurementPlanValidationIssue, + OKFBundle, + OKFDocument, + PromptPreview, + Protocol, + ProtocolRevision, + ProtocolRun, RegisteredDataset, Replicate, Run, RunStep, + Skill, + SkillUrlPreview, + TestRun, ToolCallResult, + Trial, WorkspaceEvent, ) @@ -42,6 +64,7 @@ __version__ = "0.0.0+unknown" __all__ = [ + "__version__", "Agent", "AsareeAPIError", "AsareeAuthenticationError", @@ -55,16 +78,38 @@ "AsareeTimeoutError", "AsareeUnprocessableEntityError", "AsareeUpstreamError", + "CellRunBatch", + "DesignImpact", "DesignRevision", + "DirectoryListing", + "DiscoveredSkill", "Experiment", "ExperimentArtifact", + "ExperimentResults", + "ExperimentRunResults", + "LLMConnectionCheck", + "LLMModelInfo", + "LLMModels", "LLMSetting", "MCPServer", + "MeasurementCapabilities", + "MeasurementPlanValidation", + "MeasurementPlanValidationIssue", + "OKFBundle", + "OKFDocument", + "PromptPreview", + "Protocol", + "ProtocolRevision", + "ProtocolRun", "Replicate", "RegisteredDataset", "RetryPolicy", "Run", "RunStep", + "Skill", + "SkillUrlPreview", + "TestRun", "ToolCallResult", + "Trial", "WorkspaceEvent", ] diff --git a/sdk/src/asaree_client/client.py b/sdk/src/asaree_client/client.py index 09d4b3c1..950a43e9 100644 --- a/sdk/src/asaree_client/client.py +++ b/sdk/src/asaree_client/client.py @@ -13,8 +13,10 @@ from asaree_client.resources.datasets import Datasets from asaree_client.resources.experiments import Experiments from asaree_client.resources.llm_settings import LLMSettings +from asaree_client.resources.okf import OKF from asaree_client.resources.protocols import Protocols from asaree_client.resources.runs import Runs +from asaree_client.resources.skills import Skills from asaree_client.resources.tools import Tools @@ -68,6 +70,8 @@ def __init__( self.datasets = Datasets(self) self.tools = Tools(self) self.llm_settings = LLMSettings(self) + self.skills = Skills(self) + self.okf = OKF(self) def _request(self, method: str, path: str, **kwargs: Any) -> Any: response = self._http.request(method, f"/api{path}", **kwargs) @@ -79,6 +83,11 @@ def _request(self, method: str, path: str, **kwargs: Any) -> Any: def _get(self, path: str, **kwargs: Any) -> Any: return self._request("GET", path, **kwargs) + def _get_bytes(self, path: str, **kwargs: Any) -> bytes: + response = self._http.request("GET", f"/api{path}", **kwargs) + raise_for_status(response) + return response.content + def _post(self, path: str, **kwargs: Any) -> Any: return self._request("POST", path, **kwargs) @@ -91,6 +100,16 @@ def _patch(self, path: str, **kwargs: Any) -> Any: def _delete(self, path: str, **kwargs: Any) -> Any: return self._request("DELETE", path, **kwargs) + def version(self) -> str: + """Return the running ASAREE app version.""" + return str(self._get("/version")["version"]) + + def health(self) -> dict[str, str]: + """Return the server health payload.""" + response = self._http.request("GET", "/health") + raise_for_status(response) + return dict(response.json()) + def close(self) -> None: self._http.close() diff --git a/sdk/src/asaree_client/models.py b/sdk/src/asaree_client/models.py index 22030f24..8a7a6115 100644 --- a/sdk/src/asaree_client/models.py +++ b/sdk/src/asaree_client/models.py @@ -81,11 +81,18 @@ class Experiment(BaseModel): # an experiment can run against several since the Dataset connector was # uncapped. ``dataset_id`` is a read-only view of the first one, kept so # code written before that keeps working; it is no longer a stored column. - dataset_ids: list[uuid.UUID] = [] + dataset_ids: list[uuid.UUID] = Field(default_factory=list) dataset_id: uuid.UUID | None = None archived_at: datetime | None = None + metric_recommendations: dict[str, Any] | None = None + metric_recommendation_set_version: int = 1 + latest_test_run_id: uuid.UUID | None = None + locked_at: datetime | None = None + locked_protocol_revision_id: uuid.UUID | None = None + locked_design_spec: dict[str, Any] | None = None locked_measurement_plan: dict[str, Any] | None = None created_at: datetime + updated_at: datetime class Replicate(BaseModel): @@ -168,8 +175,12 @@ class ProtocolRun(BaseModel): design_revision_id: uuid.UUID | None = None protocol_revision_id: uuid.UUID | None = None target_node_id: str | None = None + conversation: dict[str, Any] | None = None + cancel_requested_at: datetime | None = None created_at: datetime updated_at: datetime + observations: list[dict[str, Any]] = Field(default_factory=list) + artifacts: list[dict[str, Any]] = Field(default_factory=list) class ProtocolRevision(BaseModel): @@ -188,8 +199,58 @@ class CellRunBatch(BaseModel): protocol_run_ids: list[uuid.UUID] replicate_labels: list[str] skipped: int - protocol_revision_id: uuid.UUID | None = None - protocol_revision: int | None = None + protocol_revision_id: uuid.UUID + protocol_revision: int + + +class PromptPreview(BaseModel): + text: str + + +class ResourceUsage(BaseModel): + duration_seconds: float | None + cost_usd: float | None + + +class TestRunResources(BaseModel): + task: ResourceUsage + evaluation: ResourceUsage + total: ResourceUsage + + +class TestRunFreshness(BaseModel): + out_of_date: bool + reasons: list[str] + + +class TestRunExecutionSummary(BaseModel): + node_runs: dict[str, Any] + started_at: datetime | None + completed_at: datetime | None + cancel_requested_at: datetime | None + + +class TestedPublishedRevision(BaseModel): + id: uuid.UUID + number: int + published_at: datetime + + +class TestRun(BaseModel): + id: uuid.UUID + protocol_id: uuid.UUID + status: str + error: str | None + protocol_revision_id: uuid.UUID | None + created_at: datetime + updated_at: datetime + observations: list[dict[str, Any]] + artifacts: list[dict[str, Any]] + conversation: dict[str, Any] | None + tested_published_revision: TestedPublishedRevision | None + freshness: TestRunFreshness + resources: TestRunResources + execution_summary: TestRunExecutionSummary class RegisteredDataset(BaseModel): @@ -205,6 +266,10 @@ class RegisteredDataset(BaseModel): test_path: str | None = None train_sha256: str | None = None test_sha256: str | None = None + split_method: str | None = None + split_group_column: str | None = None + split_test_size: float | None = None + split_seed: int | None = None target_column: str | None description: str | None = None dictionary_json: str | None = None @@ -214,6 +279,29 @@ class RegisteredDataset(BaseModel): class LLMSetting(BaseModel): provider: str api_base: str | None + azure_project_endpoint: str | None = None + + +class LLMModelInfo(BaseModel): + id: str + label: str | None + supports_temperature: bool + supports_effort: bool + effort_levels: list[str] + supports_tool_calling: bool | None + + +class LLMModels(BaseModel): + models: list[LLMModelInfo] + source: str + note: str | None + + +class LLMConnectionCheck(BaseModel): + provider: str + status: str + detail: str + endpoint: str | None class WorkspaceEvent(BaseModel): @@ -241,3 +329,130 @@ class MCPServer(BaseModel): class ToolCallResult(BaseModel): is_error: bool content: str + + +class Trial(BaseModel): + replicate_label: str + factor_values: dict[str, Any] + metric_values: dict[str, Any] + status: str + run_id: uuid.UUID | None + obsolete: bool + truncated: bool + error: str | None + updated_at: datetime + + +class ExperimentResults(BaseModel): + available: bool + reason: str | None + analysis: dict[str, Any] | None + best_condition: dict[str, Any] | None + + +class ExperimentRunResults(BaseModel): + overview: dict[str, Any] + metric_keys: list[str] + metric_types: dict[str, str] + metric_aggregations: dict[str, str] + metric_directions: dict[str, str] + primary_metric: str | None + primary_metric_direction: str | None + cells: list[dict[str, Any]] + replicates: list[dict[str, Any]] + + +class DesignImpact(BaseModel): + has_generated_design: bool + regeneration_required: bool + current_cell_count: int + proposed_cell_count: int + added_cell_count: int + retained_cell_count: int + removed_cell_count: int + current_replicate_count: int + proposed_replicate_count: int + added_replicate_count: int + retained_replicate_count: int + removed_replicate_count: int + regeneration_reasons: list[str] = Field(default_factory=list) + + +class MeasurementPlanValidationIssue(BaseModel): + code: str + message: str + path: str + blocking: bool = True + + +class MeasurementPlanValidation(BaseModel): + valid: bool + issues: list[MeasurementPlanValidationIssue] + + +class MeasurementCapabilities(BaseModel): + outputs: dict[str, list[str]] + + +class Skill(BaseModel): + id: uuid.UUID + name: str + description: str + body: str + frontmatter: dict[str, Any] + is_system: bool + source_filename: str | None + files: list[str] + created_at: datetime + updated_at: datetime + + +class DiscoveredSkill(BaseModel): + subdirectory: str + name: str + description: str + file_count: int + + +class SkillUrlPreview(BaseModel): + source: str + ref: str + skills: list[DiscoveredSkill] + + +class DirectoryEntry(BaseModel): + name: str + path: str + is_bundle: bool + + +class DirectoryListing(BaseModel): + path: str + absolute_path: str + parent: str | None + entries: list[DirectoryEntry] + + +class OKFBundle(BaseModel): + id: uuid.UUID + name: str + path: str | None + uploaded: bool + status: str + error_message: str | None + tool_names: list[str] + created_at: datetime + + +class OKFDocument(BaseModel): + id: uuid.UUID + name: str + title: str | None + description: str | None + concept_type: str | None + tags: list[str] + path: str | None + status: str + error_message: str | None + tool_names: list[str] + created_at: datetime diff --git a/sdk/src/asaree_client/resources/datasets.py b/sdk/src/asaree_client/resources/datasets.py index 6293e026..b216a831 100644 --- a/sdk/src/asaree_client/resources/datasets.py +++ b/sdk/src/asaree_client/resources/datasets.py @@ -13,7 +13,7 @@ import uuid from typing import Any -from asaree_client.models import RegisteredDataset +from asaree_client.models import RegisteredDataset, WorkspaceEvent ResourceId = uuid.UUID | str @@ -92,3 +92,32 @@ def list(self) -> builtins.list[RegisteredDataset]: def delete(self, dataset_id: ResourceId) -> None: self._client._delete(f"/datasets/{dataset_id}") + + def record_workspace_event( + self, + dataset_id: ResourceId, + *, + workspace_id: str, + stage: str, + event_type: str, + sha256_train: str | None = None, + sha256_test: str | None = None, + ) -> WorkspaceEvent: + payload: dict[str, Any] = { + "workspace_id": workspace_id, + "stage": stage, + "event_type": event_type, + } + if sha256_train is not None: + payload["sha256_train"] = sha256_train + if sha256_test is not None: + payload["sha256_test"] = sha256_test + data = self._client._post(f"/datasets/{dataset_id}/workspace-events", json=payload) + return WorkspaceEvent(**data) + + def list_workspace_events( + self, dataset_id: ResourceId, *, workspace_id: str | None = None + ) -> builtins.list[WorkspaceEvent]: + params = {"workspace_id": workspace_id} if workspace_id is not None else None + data = self._client._get(f"/datasets/{dataset_id}/workspace-events", params=params) + return [WorkspaceEvent(**event) for event in data] diff --git a/sdk/src/asaree_client/resources/experiments.py b/sdk/src/asaree_client/resources/experiments.py index 213bed36..8ee4c97b 100644 --- a/sdk/src/asaree_client/resources/experiments.py +++ b/sdk/src/asaree_client/resources/experiments.py @@ -13,7 +13,18 @@ from datetime import datetime from typing import Any -from asaree_client.models import DesignRevision, Experiment, ExperimentArtifact, Replicate +from asaree_client.models import ( + DesignImpact, + DesignRevision, + Experiment, + ExperimentArtifact, + ExperimentResults, + ExperimentRunResults, + MeasurementCapabilities, + MeasurementPlanValidation, + Replicate, + Trial, +) ResourceId = uuid.UUID | str # Distinguishes "omit this kwarg" (leave unchanged) from "pass None" @@ -29,15 +40,18 @@ def __init__(self, client: Any) -> None: def create( self, *, - name: str, + name: str | None = None, description: str | None = None, design_type: str = "factorial", task_brief: dict[str, Any] | None = None, factors: builtins.list[dict[str, Any]] | None = None, measurement_plan: dict[str, Any] | None = None, dataset_ids: builtins.list[ResourceId] | None = None, + dataset_id: ResourceId | None = None, ) -> Experiment: - payload: dict[str, Any] = {"name": name, "design_type": design_type} + payload: dict[str, Any] = {"design_type": design_type} + if name is not None: + payload["name"] = name if description is not None: payload["description"] = description if task_brief is not None: @@ -48,6 +62,8 @@ def create( payload["measurement_plan"] = measurement_plan if dataset_ids is not None: payload["dataset_ids"] = [str(d) for d in dataset_ids] + elif dataset_id is not None: + payload["dataset_id"] = str(dataset_id) data = self._client._post("/experiments", json=payload) return Experiment(**data) @@ -55,10 +71,40 @@ def get(self, experiment_id: ResourceId) -> Experiment: data = self._client._get(f"/experiments/{experiment_id}") return Experiment(**data) - def list(self) -> builtins.list[Experiment]: - data = self._client._get("/experiments") + def list(self, *, include_archived: bool = False) -> builtins.list[Experiment]: + params = {"include_archived": True} if include_archived else None + data = self._client._get("/experiments", params=params) return [Experiment(**e) for e in data] + def import_definition( + self, + *, + name: str, + graph: dict[str, Any], + description: str | None = None, + hypothesis: str | None = None, + design_type: str = "factorial", + task_brief: dict[str, Any] | None = None, + design_spec: dict[str, Any] | None = None, + measurement_plan: dict[str, Any] | None = None, + published_graph: dict[str, Any] | None = None, + protocol_description: str | None = None, + ) -> Experiment: + payload: dict[str, Any] = {"name": name, "graph": graph, "design_type": design_type} + for key, value in { + "description": description, + "hypothesis": hypothesis, + "task_brief": task_brief, + "design_spec": design_spec, + "measurement_plan": measurement_plan, + "published_graph": published_graph, + "protocol_description": protocol_description, + }.items(): + if value is not None: + payload[key] = value + data = self._client._post("/experiments/import-definition", json=payload) + return Experiment(**data) + def delete(self, experiment_id: ResourceId) -> None: self._client._delete(f"/experiments/{experiment_id}") @@ -73,6 +119,8 @@ def update( dataset_id: ResourceId | None = _UNSET, design_spec: dict[str, Any] | None = _UNSET, measurement_plan: dict[str, Any] | None = _UNSET, + measurement_validation_protocol_id: ResourceId | None = _UNSET, + metric_recommendations: dict[str, Any] | None = _UNSET, archived_at: datetime | None = _UNSET, ) -> Experiment: """Only the fields actually passed are sent (omit one to leave it @@ -107,12 +155,26 @@ def update( payload["design_spec"] = design_spec if measurement_plan is not _UNSET: payload["measurement_plan"] = measurement_plan + if measurement_validation_protocol_id is not _UNSET: + payload["measurement_validation_protocol_id"] = ( + str(measurement_validation_protocol_id) if measurement_validation_protocol_id else None + ) + if metric_recommendations is not _UNSET: + payload["metric_recommendations"] = metric_recommendations if archived_at is not _UNSET: payload["archived_at"] = archived_at.isoformat() if archived_at else None data = self._client._patch(f"/experiments/{experiment_id}", json=payload) return Experiment(**data) - def generate_design(self, experiment_id: ResourceId) -> builtins.list[Replicate]: + def generate_design( + self, + experiment_id: ResourceId, + *, + hypothesis: str | None = _UNSET, + design_spec: dict[str, Any] | None = _UNSET, + measurement_plan: dict[str, Any] | None = _UNSET, + measurement_validation_protocol_id: ResourceId | None = _UNSET, + ) -> builtins.list[Replicate]: """Materialize one cell per combination of the experiment's declared factors and their replicate results, returning the current replicates. @@ -123,9 +185,52 @@ def generate_design(self, experiment_id: ResourceId) -> builtins.list[Replicate] superseded revision as history (see ``list_design_revisions``). Nothing is deleted, and the returned list is always exactly the new design. """ - data = self._client._post(f"/experiments/{experiment_id}/generate-design") + payload: dict[str, Any] = {} + for key, value in { + "hypothesis": hypothesis, + "design_spec": design_spec, + "measurement_plan": measurement_plan, + }.items(): + if value is not _UNSET: + payload[key] = value + if measurement_validation_protocol_id is not _UNSET: + payload["measurement_validation_protocol_id"] = ( + str(measurement_validation_protocol_id) if measurement_validation_protocol_id else None + ) + kwargs = {"json": payload} if payload else {} + data = self._client._post(f"/experiments/{experiment_id}/generate-design", **kwargs) return [Replicate(**replicate) for replicate in data] + def get_design_impact(self, experiment_id: ResourceId) -> DesignImpact: + data = self._client._get(f"/experiments/{experiment_id}/design-impact") + return DesignImpact(**data) + + def validate_measurement_plan( + self, + experiment_id: ResourceId, + *, + measurement_plan: dict[str, Any] | None, + metrics: builtins.list[dict[str, Any]], + graph: dict[str, Any], + ) -> MeasurementPlanValidation: + data = self._client._post( + f"/experiments/{experiment_id}/measurement-plan/validate", + json={"measurement_plan": measurement_plan, "metrics": metrics, "graph": graph}, + ) + return MeasurementPlanValidation(**data) + + def get_measurement_capabilities(self, experiment_id: ResourceId) -> MeasurementCapabilities: + data = self._client._get(f"/experiments/{experiment_id}/measurement-capabilities") + return MeasurementCapabilities(**data) + + def lock(self, experiment_id: ResourceId) -> Experiment: + data = self._client._post(f"/experiments/{experiment_id}/lock") + return Experiment(**data) + + def unlock(self, experiment_id: ResourceId) -> Experiment: + data = self._client._post(f"/experiments/{experiment_id}/unlock") + return Experiment(**data) + def list_design_revisions(self, experiment_id: ResourceId) -> builtins.list[DesignRevision]: """Every generation of this experiment's design, newest first. The first entry (``superseded_at is None``) is the current design.""" @@ -143,25 +248,25 @@ def upsert_replicate( experiment_id: ResourceId, replicate_label: str, *, - run_id: ResourceId | None = None, - workspace_id: str | None = None, - factor_values: dict[str, Any] | None = None, - metric_values: dict[str, Any] | None = None, - artifacts: dict[str, Any] | None = None, + run_id: ResourceId | None = _UNSET, + workspace_id: str | None = _UNSET, + factor_values: dict[str, Any] | None = _UNSET, + metric_values: dict[str, Any] | None = _UNSET, + artifacts: dict[str, Any] | None = _UNSET, ) -> Replicate: """Merge fields onto a replicate-result row — pass just what changed; unset fields are left untouched (a pre-scoring call and a post-scoring call land on the same row without either erasing the other).""" payload: dict[str, Any] = {} - if run_id is not None: - payload["run_id"] = str(run_id) - if workspace_id is not None: + if run_id is not _UNSET: + payload["run_id"] = str(run_id) if run_id else None + if workspace_id is not _UNSET: payload["workspace_id"] = workspace_id - if factor_values is not None: + if factor_values is not _UNSET: payload["factor_values"] = factor_values - if metric_values is not None: + if metric_values is not _UNSET: payload["metric_values"] = metric_values - if artifacts is not None: + if artifacts is not _UNSET: payload["artifacts"] = artifacts data = self._client._put(f"/experiments/{experiment_id}/replicates/{replicate_label}", json=payload) return Replicate(**data) @@ -212,9 +317,7 @@ def analyze( payload["cost_keys"] = cost_keys return self._client._post(f"/experiments/{experiment_id}/analyze", json=payload) # type: ignore[no-any-return] - def create_artifact( - self, experiment_id: ResourceId, *, name: str, kind: str, content: str - ) -> ExperimentArtifact: + def create_artifact(self, experiment_id: ResourceId, *, name: str, kind: str, content: str) -> ExperimentArtifact: """A durable landing spot for anything worth keeping past one run -- e.g. ``analyze(...)``'s own result (``kind="analyze_result"``, ``content=json.dumps(result)``) or a flattened CSV export @@ -236,3 +339,22 @@ def get_artifact(self, experiment_id: ResourceId, artifact_id: ResourceId) -> Ex def delete_artifact(self, experiment_id: ResourceId, artifact_id: ResourceId) -> None: self._client._delete(f"/experiments/{experiment_id}/artifacts/{artifact_id}") + + def list_trials(self, experiment_id: ResourceId) -> builtins.list[Trial]: + data = self._client._get(f"/experiments/{experiment_id}/runs") + return [Trial(**trial) for trial in data] + + def get_results(self, experiment_id: ResourceId) -> ExperimentResults: + return ExperimentResults(**self._client._get(f"/experiments/{experiment_id}/results")) + + def get_run_results(self, experiment_id: ResourceId) -> ExperimentRunResults: + return ExperimentRunResults(**self._client._get(f"/experiments/{experiment_id}/run-results")) + + def get_run_results_schema(self, experiment_id: ResourceId) -> dict[str, Any]: + return self._client._get(f"/experiments/{experiment_id}/run-results.schema.json") # type: ignore[no-any-return] + + def download_run_results_csv(self, experiment_id: ResourceId) -> bytes: + return self._client._get_bytes(f"/experiments/{experiment_id}/run-results.csv") # type: ignore[no-any-return] + + def download_replicates_csv(self, experiment_id: ResourceId) -> bytes: + return self._client._get_bytes(f"/experiments/{experiment_id}/replicates.csv") # type: ignore[no-any-return] diff --git a/sdk/src/asaree_client/resources/llm_settings.py b/sdk/src/asaree_client/resources/llm_settings.py index 68417d87..cf43bbb2 100644 --- a/sdk/src/asaree_client/resources/llm_settings.py +++ b/sdk/src/asaree_client/resources/llm_settings.py @@ -4,14 +4,21 @@ from typing import Any -from asaree_client.models import LLMSetting +from asaree_client.models import LLMConnectionCheck, LLMModels, LLMSetting class LLMSettings: def __init__(self, client: Any) -> None: self._client = client - def set_key(self, provider: str, api_key: str, *, api_base: str | None = None) -> LLMSetting: + def set_key( + self, + provider: str, + api_key: str, + *, + api_base: str | None = None, + azure_project_endpoint: str | None = None, + ) -> LLMSetting: """Set (or replace) this user's credential for *provider*. For ``azure_foundry``, *api_base* is the Foundry resource name (or a @@ -22,9 +29,22 @@ def set_key(self, provider: str, api_key: str, *, api_base: str | None = None) - payload: dict[str, Any] = {"provider": provider, "api_key": api_key} if api_base is not None: payload["api_base"] = api_base + if azure_project_endpoint is not None: + payload["azure_project_endpoint"] = azure_project_endpoint data = self._client._put("/llm-settings", json=payload) return LLMSetting(**data) def list(self) -> list[LLMSetting]: data = self._client._get("/llm-settings") return [LLMSetting(**s) for s in data] + + def delete(self, provider: str) -> None: + self._client._delete(f"/llm-settings/{provider}") + + def check_connection(self, provider: str) -> LLMConnectionCheck: + data = self._client._get(f"/llm-settings/{provider}/connection") + return LLMConnectionCheck(**data) + + def list_models(self, provider: str) -> LLMModels: + data = self._client._get(f"/llm-settings/{provider}/models") + return LLMModels(**data) diff --git a/sdk/src/asaree_client/resources/okf.py b/sdk/src/asaree_client/resources/okf.py new file mode 100644 index 00000000..21d4747c --- /dev/null +++ b/sdk/src/asaree_client/resources/okf.py @@ -0,0 +1,68 @@ +"""Open Knowledge Format bundle and document resources.""" + +from __future__ import annotations + +import builtins +import uuid +from pathlib import Path +from typing import Any + +from asaree_client.models import DirectoryListing, OKFBundle, OKFDocument + +ResourceId = uuid.UUID | str + + +class OKF: + def __init__(self, client: Any) -> None: + self._client = client + + def browse(self, path: str = "") -> DirectoryListing: + return DirectoryListing(**self._client._get("/okf/browse", params={"path": path})) + + def list_bundles(self) -> builtins.list[OKFBundle]: + return [OKFBundle(**item) for item in self._client._get("/okf/bundles")] + + def register_bundle(self, path: str) -> OKFBundle: + return OKFBundle(**self._client._post("/okf/bundles", json={"path": path})) + + def upload_bundle(self, directory: str) -> OKFBundle: + root = Path(directory) + paths = sorted(path for path in root.rglob("*") if path.is_file()) + handles = [path.open("rb") for path in paths] + try: + files = [ + ("files", (f"{root.name}/{path.relative_to(root).as_posix()}", handle)) + for path, handle in zip(paths, handles, strict=True) + ] + data = self._client._post("/okf/bundles/upload", files=files) + finally: + for handle in handles: + handle.close() + return OKFBundle(**data) + + def refresh_bundle(self, bundle_id: ResourceId) -> OKFBundle: + return OKFBundle(**self._client._post(f"/okf/bundles/{bundle_id}/refresh")) + + def delete_bundle(self, bundle_id: ResourceId) -> None: + self._client._delete(f"/okf/bundles/{bundle_id}") + + def list_concepts(self, bundle_id: ResourceId) -> dict[str, Any]: + return self._client._get(f"/okf/bundles/{bundle_id}/concepts") # type: ignore[no-any-return] + + def list_documents(self) -> builtins.list[OKFDocument]: + return [OKFDocument(**item) for item in self._client._get("/okf/documents")] + + def upload_document(self, file_path: str) -> OKFDocument: + with open(file_path, "rb") as file: + data = self._client._post("/okf/documents", files={"file": (Path(file_path).name, file)}) + return OKFDocument(**data) + + def refresh_document(self, document_id: ResourceId) -> OKFDocument: + return OKFDocument(**self._client._post(f"/okf/documents/{document_id}/refresh")) + + def delete_document(self, document_id: ResourceId) -> None: + self._client._delete(f"/okf/documents/{document_id}") + + def get_document_markdown(self, document_id: ResourceId) -> str: + data = self._client._get(f"/okf/documents/{document_id}/markdown") + return str(data["markdown"]) diff --git a/sdk/src/asaree_client/resources/protocols.py b/sdk/src/asaree_client/resources/protocols.py index d6a0ac55..39e7a3d4 100644 --- a/sdk/src/asaree_client/resources/protocols.py +++ b/sdk/src/asaree_client/resources/protocols.py @@ -6,9 +6,10 @@ import uuid from typing import Any -from asaree_client.models import CellRunBatch, Protocol, ProtocolRevision, ProtocolRun +from asaree_client.models import CellRunBatch, PromptPreview, Protocol, ProtocolRevision, ProtocolRun, TestRun ResourceId = uuid.UUID | str +_UNSET: Any = object() class Protocols: @@ -46,19 +47,19 @@ def update( self, protocol_id: ResourceId, *, - name: str | None = None, - description: str | None = None, - experiment_id: ResourceId | None = None, - graph: dict[str, Any] | None = None, + name: str | None = _UNSET, + description: str | None = _UNSET, + experiment_id: ResourceId | None = _UNSET, + graph: dict[str, Any] | None = _UNSET, ) -> Protocol: payload: dict[str, Any] = {} - if name is not None: + if name is not _UNSET: payload["name"] = name - if description is not None: + if description is not _UNSET: payload["description"] = description - if experiment_id is not None: - payload["experiment_id"] = str(experiment_id) - if graph is not None: + if experiment_id is not _UNSET: + payload["experiment_id"] = str(experiment_id) if experiment_id else None + if graph is not _UNSET: payload["graph"] = graph data = self._client._patch(f"/protocols/{protocol_id}", json=payload) return Protocol(**data) @@ -98,14 +99,41 @@ def run_node(self, protocol_id: ResourceId, node_id: str) -> ProtocolRun: data = self._client._post(f"/protocols/{protocol_id}/nodes/{node_id}/run") return ProtocolRun(**data) - def run_cells(self, protocol_id: ResourceId) -> CellRunBatch: - """"Run all cells" -- one ProtocolRun per not-yet-completed replicate under + def preview_prompt( + self, protocol_id: ResourceId, node_id: str, *, graph: dict[str, Any] | None = None + ) -> PromptPreview: + payload = {"graph": graph} if graph is not None else {} + data = self._client._post(f"/protocols/{protocol_id}/nodes/{node_id}/prompt-preview", json=payload) + return PromptPreview(**data) + + def run_cells( + self, + protocol_id: ResourceId, + *, + replicate_labels: builtins.list[str] | None = None, + rerun_replicate_labels: builtins.list[str] | None = None, + ) -> CellRunBatch: + """ "Run all cells" -- one ProtocolRun per not-yet-completed replicate under this protocol's linked experiment, each replicate's cell factor_values substituted in. 422 if there's no linked experiment or the graph doesn't have exactly one final node.""" - data = self._client._post(f"/protocols/{protocol_id}/cell-runs") + payload: dict[str, Any] = {} + if replicate_labels is not None: + payload["replicate_labels"] = replicate_labels + if rerun_replicate_labels is not None: + payload["rerun_replicate_labels"] = rerun_replicate_labels + kwargs = {"json": payload} if payload else {} + data = self._client._post(f"/protocols/{protocol_id}/cell-runs", **kwargs) return CellRunBatch(**data) + def start_test_run(self, protocol_id: ResourceId) -> TestRun: + data = self._client._post(f"/protocols/{protocol_id}/test-runs") + return TestRun(**data) + + def get_latest_test_run(self, protocol_id: ResourceId) -> TestRun: + data = self._client._get(f"/protocols/{protocol_id}/test-runs/latest") + return TestRun(**data) + def get_run(self, protocol_id: ResourceId, run_id: ResourceId) -> ProtocolRun: data = self._client._get(f"/protocols/{protocol_id}/runs/{run_id}") return ProtocolRun(**data) @@ -113,3 +141,7 @@ def get_run(self, protocol_id: ResourceId, run_id: ResourceId) -> ProtocolRun: def list_runs(self, protocol_id: ResourceId) -> builtins.list[ProtocolRun]: data = self._client._get(f"/protocols/{protocol_id}/runs") return [ProtocolRun(**r) for r in data] + + def cancel_run(self, protocol_id: ResourceId, run_id: ResourceId) -> ProtocolRun: + data = self._client._post(f"/protocols/{protocol_id}/runs/{run_id}/cancel") + return ProtocolRun(**data) diff --git a/sdk/src/asaree_client/resources/skills.py b/sdk/src/asaree_client/resources/skills.py new file mode 100644 index 00000000..cb3c3afa --- /dev/null +++ b/sdk/src/asaree_client/resources/skills.py @@ -0,0 +1,90 @@ +"""Agent Skill library resource.""" + +from __future__ import annotations + +import builtins +import uuid +from pathlib import Path +from typing import Any + +from asaree_client.models import Skill, SkillUrlPreview + +ResourceId = uuid.UUID | str + + +class Skills: + def __init__(self, client: Any) -> None: + self._client = client + + def create(self, *, name: str, description: str, body: str) -> Skill: + data = self._client._post("/skills", json={"name": name, "description": description, "body": body}) + return Skill(**data) + + def upload(self, file_path: str, *, name: str | None = None, description: str | None = None) -> Skill: + form: dict[str, str] = {} + if name is not None: + form["name"] = name + if description is not None: + form["description"] = description + with open(file_path, "rb") as file: + data = self._client._post("/skills/upload", data=form, files={"file": (Path(file_path).name, file)}) + return Skill(**data) + + def upload_folder(self, directory: str) -> Skill: + return self._folder_request("POST", "/skills/upload-folder", directory) + + def replace_folder(self, skill_id: ResourceId, directory: str) -> Skill: + return self._folder_request("PUT", f"/skills/{skill_id}/folder", directory) + + def _folder_request(self, method: str, path: str, directory: str) -> Skill: + root = Path(directory) + paths = sorted(path for path in root.rglob("*") if path.is_file()) + handles = [path.open("rb") for path in paths] + try: + files = [ + ("files", (f"{root.name}/{path.relative_to(root).as_posix()}", handle)) + for path, handle in zip(paths, handles, strict=True) + ] + data = self._client._request(method, path, files=files) + finally: + for handle in handles: + handle.close() + return Skill(**data) + + def preview_url(self, url: str) -> SkillUrlPreview: + data = self._client._post("/skills/from-url/preview", json={"url": url}) + return SkillUrlPreview(**data) + + def create_from_url(self, url: str, subdirectory: str) -> Skill: + data = self._client._post("/skills/from-url", json={"url": url, "subdirectory": subdirectory}) + return Skill(**data) + + def list(self, *, limit: int = 100) -> builtins.list[Skill]: + data = self._client._get("/skills", params={"limit": limit}) + return [Skill(**item) for item in data["items"]] + + def get(self, skill_id: ResourceId) -> Skill: + return Skill(**self._client._get(f"/skills/{skill_id}")) + + def get_markdown(self, skill_id: ResourceId) -> str: + data = self._client._get(f"/skills/{skill_id}/markdown") + return str(data["markdown"]) + + def update( + self, + skill_id: ResourceId, + *, + name: str | None = None, + description: str | None = None, + body: str | None = None, + ) -> Skill: + payload = {k: v for k, v in {"name": name, "description": description, "body": body}.items() if v is not None} + return Skill(**self._client._patch(f"/skills/{skill_id}", json=payload)) + + def replace_markdown(self, skill_id: ResourceId, file_path: str) -> Skill: + with open(file_path, "rb") as file: + data = self._client._put(f"/skills/{skill_id}/markdown", files={"file": (Path(file_path).name, file)}) + return Skill(**data) + + def delete(self, skill_id: ResourceId) -> None: + self._client._delete(f"/skills/{skill_id}") diff --git a/sdk/src/asaree_client/resources/tools.py b/sdk/src/asaree_client/resources/tools.py index c9387009..b55f4c51 100644 --- a/sdk/src/asaree_client/resources/tools.py +++ b/sdk/src/asaree_client/resources/tools.py @@ -1,10 +1,4 @@ -"""MCP tool passthrough, matching asaree.api.mcp_servers. - -Registration itself is one-time environment setup (done once via curl, per -the SDK README), not part of the ongoing driver-script surface — this -resource is deliberately limited to what a notebook calls per experiment -run: discover registered servers, call a tool directly, reset a session. -""" +"""MCP server management and direct tool calls.""" from __future__ import annotations @@ -14,6 +8,7 @@ from asaree_client.models import MCPServer, ToolCallResult ResourceId = uuid.UUID | str +_UNSET: Any = object() class Tools: @@ -24,6 +19,56 @@ def list_servers(self) -> list[MCPServer]: data = self._client._get("/mcp-servers") return [MCPServer(**s) for s in data] + def create_server( + self, + *, + name: str, + transport: str, + command: str | None = None, + url: str | None = None, + headers: dict[str, str] | None = None, + ) -> MCPServer: + payload: dict[str, Any] = {"name": name, "transport": transport} + for key, value in {"command": command, "url": url, "headers": headers}.items(): + if value is not None: + payload[key] = value + return MCPServer(**self._client._post("/mcp-servers", json=payload)) + + def get_server(self, server_id: ResourceId) -> MCPServer: + return MCPServer(**self._client._get(f"/mcp-servers/{server_id}")) + + def update_server( + self, + server_id: ResourceId, + *, + name: str | None = _UNSET, + transport: str | None = _UNSET, + command: str | None = _UNSET, + url: str | None = _UNSET, + headers: dict[str, str] | None = _UNSET, + ) -> MCPServer: + payload = { + key: value + for key, value in { + "name": name, + "transport": transport, + "command": command, + "url": url, + "headers": headers, + }.items() + if value is not _UNSET + } + return MCPServer(**self._client._patch(f"/mcp-servers/{server_id}", json=payload)) + + def delete_server(self, server_id: ResourceId) -> None: + self._client._delete(f"/mcp-servers/{server_id}") + + def refresh_server(self, server_id: ResourceId) -> MCPServer: + return MCPServer(**self._client._post(f"/mcp-servers/{server_id}/refresh")) + + def reconnect_server(self, server_id: ResourceId) -> MCPServer: + return MCPServer(**self._client._post(f"/mcp-servers/{server_id}/reconnect")) + def call_tool( self, server_id: ResourceId, diff --git a/sdk/tests/test_current_api.py b/sdk/tests/test_current_api.py new file mode 100644 index 00000000..9115396f --- /dev/null +++ b/sdk/tests/test_current_api.py @@ -0,0 +1,170 @@ +"""Regression coverage for API features added after the original trimmed SDK.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from typing import Any + +from asaree_client.models import Experiment, RegisteredDataset, Trial +from asaree_client.resources.experiments import Experiments +from asaree_client.resources.protocols import Protocols + + +class RecordingClient: + def __init__(self, response: Any) -> None: + self.response = response + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + def _record(self, method: str, path: str, **kwargs: Any) -> Any: + self.calls.append((method, path, kwargs)) + return self.response + + def _get(self, path: str, **kwargs: Any) -> Any: + return self._record("GET", path, **kwargs) + + def _post(self, path: str, **kwargs: Any) -> Any: + return self._record("POST", path, **kwargs) + + def _patch(self, path: str, **kwargs: Any) -> Any: + return self._record("PATCH", path, **kwargs) + + +def _experiment_response() -> dict[str, Any]: + now = datetime.now(tz=UTC).isoformat() + return { + "id": str(uuid.uuid4()), + "name": "current", + "description": None, + "hypothesis": None, + "design_type": "factorial", + "task_brief": None, + "design_spec": None, + "measurement_plan": None, + "dataset_ids": [], + "dataset_id": None, + "archived_at": None, + "locked_at": None, + "locked_protocol_revision_id": None, + "locked_design_spec": None, + "locked_measurement_plan": None, + "created_at": now, + "updated_at": now, + } + + +def test_current_experiment_response_parses_lock_and_recommendation_state() -> None: + now = datetime.now(tz=UTC) + experiment = Experiment( + id=uuid.uuid4(), + name="current", + description=None, + hypothesis=None, + design_type="factorial", + task_brief=None, + design_spec=None, + measurement_plan=None, + metric_recommendations={"applied_version": 2}, + metric_recommendation_set_version=2, + latest_test_run_id=uuid.uuid4(), + dataset_ids=[], + dataset_id=None, + archived_at=None, + locked_at=now, + locked_protocol_revision_id=uuid.uuid4(), + locked_design_spec={"replicates": 3}, + locked_measurement_plan=None, + created_at=now, + updated_at=now, + ) + + assert experiment.locked_at == now + assert experiment.metric_recommendation_set_version == 2 + + +def test_current_dataset_response_parses_split_provenance() -> None: + dataset = RegisteredDataset( + id=uuid.uuid4(), + name="cohort", + target_column="outcome", + split_method="quick", + split_group_column="patient_id", + split_test_size=0.25, + split_seed=42, + ) + + assert dataset.split_group_column == "patient_id" + assert dataset.split_seed == 42 + + +def test_trial_parses_truncated_runs() -> None: + trial = Trial( + replicate_label="cell-1-r1", + factor_values={}, + metric_values={}, + status="completed", + run_id=uuid.uuid4(), + obsolete=False, + truncated=True, + error=None, + updated_at=datetime.now(tz=UTC), + ) + + assert trial.truncated is True + + +def test_experiment_update_sends_new_measurement_fields() -> None: + client = RecordingClient(response=_experiment_response()) + protocol_id = uuid.uuid4() + + Experiments(client).update( + "experiment-id", + measurement_plan=None, + measurement_validation_protocol_id=protocol_id, + metric_recommendations={"dismissed_version": 2}, + ) + + assert client.calls == [ + ( + "PATCH", + "/experiments/experiment-id", + { + "json": { + "measurement_plan": None, + "measurement_validation_protocol_id": str(protocol_id), + "metric_recommendations": {"dismissed_version": 2}, + } + }, + ) + ] + + +def test_run_cells_serializes_scoped_reruns() -> None: + client = RecordingClient( + response={ + "protocol_run_ids": [], + "replicate_labels": [], + "skipped": 0, + "protocol_revision_id": str(uuid.uuid4()), + "protocol_revision": 1, + } + ) + + Protocols(client).run_cells( + "protocol-id", + replicate_labels=["cell-1-r1"], + rerun_replicate_labels=["cell-1-r1"], + ) + + assert client.calls == [ + ( + "POST", + "/protocols/protocol-id/cell-runs", + { + "json": { + "replicate_labels": ["cell-1-r1"], + "rerun_replicate_labels": ["cell-1-r1"], + } + }, + ) + ] From 83de636ea0d61304c3cd9fb6f669f2dd51b9cd9c Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 13:37:50 -0700 Subject: [PATCH 02/18] feat(sdk): add account and token management --- sdk/README.md | 35 +++-- sdk/src/asaree_client/__init__.py | 16 +++ sdk/src/asaree_client/_transport.py | 14 +- sdk/src/asaree_client/client.py | 35 ++++- sdk/src/asaree_client/exceptions.py | 2 +- sdk/src/asaree_client/models.py | 58 ++++++++ sdk/src/asaree_client/resources/auth.py | 80 +++++++++++ .../asaree_client/resources/experiments.py | 6 - sdk/src/asaree_client/resources/protocols.py | 5 +- sdk/src/asaree_client/resources/users.py | 32 +++++ sdk/tests/test_auth.py | 135 ++++++++++++++++++ 11 files changed, 387 insertions(+), 31 deletions(-) create mode 100644 sdk/src/asaree_client/resources/auth.py create mode 100644 sdk/src/asaree_client/resources/users.py create mode 100644 sdk/tests/test_auth.py diff --git a/sdk/README.md b/sdk/README.md index 36f3d4be..007e42f2 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -15,20 +15,22 @@ data now belongs on a `FactorialReplicateResult` row, written via ## Auth bootstrap ASAREE has no static server-wide API key; each user is provisioned once and -issues their own token: +issues their own token through an unauthenticated client: -```bash -curl -X POST $ASAREE_BASE_URL/api/users -d '{"email": "...", "password": "..."}' -curl -X POST $ASAREE_BASE_URL/api/users/{user_id}/tokens -d '{"password": "..."}' -``` +```python +from asaree_client import AsareeClient -Every ASAREE route lives under `/api` (except `/health`) — the client -already knows this and prepends it to every request; you only need it -yourself for the one-time bootstrap above, made directly with curl. +with AsareeClient(base_url="http://localhost:8000") as bootstrap: + user = bootstrap.users.create(email="researcher@example.com", password="secure-password") + credential = bootstrap.users.issue_token(user.id, password="secure-password") + +print(credential.token) # Save once; the server never returns this value again. +``` -This is a one-time setup step, not something the SDK does — set the -resulting token as `ASAREE_API_KEY` (sent as `X-API-Key`) for everything -after that. +Set the resulting token as `ASAREE_API_KEY` (sent as `X-API-Key`) for +subsequent clients. Alternatively, use `client.auth.register()` and +`client.auth.login()` for a refreshable Bearer session; successful login and +refresh calls update that client automatically. ## Usage @@ -53,3 +55,14 @@ for replicate in replicates: results = client.experiments.get_results(experiment.id) ``` + +For a refreshable account session instead of an API key: + +```python +client = AsareeClient(base_url="http://localhost:8000") +client.auth.login(email="researcher@example.com", password="secure-password") + +profile = client.auth.get_profile() +client.auth.refresh() # Rotates both the access and refresh tokens in-place. +client.auth.logout() +``` diff --git a/sdk/src/asaree_client/__init__.py b/sdk/src/asaree_client/__init__.py index f4a673e7..db993bb4 100644 --- a/sdk/src/asaree_client/__init__.py +++ b/sdk/src/asaree_client/__init__.py @@ -19,6 +19,12 @@ ) from asaree_client.models import ( Agent, + ApiToken, + ApiTokenPage, + ApiTokenSummary, + AuthTokens, + BootstrapApiToken, + BootstrapUser, CellRunBatch, DesignImpact, DesignRevision, @@ -36,6 +42,7 @@ MeasurementCapabilities, MeasurementPlanValidation, MeasurementPlanValidationIssue, + Message, OKFBundle, OKFDocument, PromptPreview, @@ -51,6 +58,7 @@ TestRun, ToolCallResult, Trial, + UserAccount, WorkspaceEvent, ) @@ -66,6 +74,9 @@ __all__ = [ "__version__", "Agent", + "ApiToken", + "ApiTokenPage", + "ApiTokenSummary", "AsareeAPIError", "AsareeAuthenticationError", "AsareeBadRequestError", @@ -78,6 +89,9 @@ "AsareeTimeoutError", "AsareeUnprocessableEntityError", "AsareeUpstreamError", + "AuthTokens", + "BootstrapApiToken", + "BootstrapUser", "CellRunBatch", "DesignImpact", "DesignRevision", @@ -92,6 +106,7 @@ "LLMModels", "LLMSetting", "MCPServer", + "Message", "MeasurementCapabilities", "MeasurementPlanValidation", "MeasurementPlanValidationIssue", @@ -111,5 +126,6 @@ "TestRun", "ToolCallResult", "Trial", + "UserAccount", "WorkspaceEvent", ] diff --git a/sdk/src/asaree_client/_transport.py b/sdk/src/asaree_client/_transport.py index e3d7b0d5..f425c806 100644 --- a/sdk/src/asaree_client/_transport.py +++ b/sdk/src/asaree_client/_transport.py @@ -1,9 +1,8 @@ """Retry transport for the sync HTTP client. -Trimmed from ares_client._transport: sync-only (no AsyncRetryTransport), and -build_headers supports only X-API-Key — ASAREE's ``get_current_user`` dep -(asaree.deps) never reads an Authorization header, so there is no second -auth method to plumb through. +Trimmed from ares_client._transport: sync-only (no AsyncRetryTransport). +ASAREE accepts either a long-lived API key or a short-lived Bearer access +token, so both authentication modes are supported here. """ from __future__ import annotations @@ -134,12 +133,14 @@ def close(self) -> None: self._transport.close() -def build_headers(api_key: str | None) -> dict[str, str]: +def build_headers(api_key: str | None, access_token: str | None = None) -> dict[str, str]: from asaree_client import __version__ headers = {"User-Agent": f"asaree-client/{__version__}"} if api_key: headers["X-API-Key"] = api_key + elif access_token: + headers["Authorization"] = f"Bearer {access_token}" return headers @@ -154,11 +155,12 @@ def build_sync_client( api_key: str | None, timeout: float | httpx.Timeout, policy: RetryPolicy | None = None, + access_token: str | None = None, ) -> httpx.Client: transport = RetryTransport(httpx.HTTPTransport(), policy=policy or RetryPolicy()) return httpx.Client( base_url=base_url, - headers=build_headers(api_key), + headers=build_headers(api_key, access_token), timeout=_build_timeout(timeout), transport=transport, ) diff --git a/sdk/src/asaree_client/client.py b/sdk/src/asaree_client/client.py index 950a43e9..4fa1cc8c 100644 --- a/sdk/src/asaree_client/client.py +++ b/sdk/src/asaree_client/client.py @@ -10,6 +10,7 @@ from asaree_client._transport import RetryPolicy, build_sync_client, raise_for_status from asaree_client.exceptions import AsareeError from asaree_client.resources.agents import Agents +from asaree_client.resources.auth import Auth from asaree_client.resources.datasets import Datasets from asaree_client.resources.experiments import Experiments from asaree_client.resources.llm_settings import LLMSettings @@ -18,6 +19,7 @@ from asaree_client.resources.runs import Runs from asaree_client.resources.skills import Skills from asaree_client.resources.tools import Tools +from asaree_client.resources.users import Users def _resolve_base_url(base_url: str | None) -> str: @@ -39,12 +41,14 @@ def _resolve_timeout(timeout: float | httpx.Timeout | None) -> float | httpx.Tim class AsareeClient: """Synchronous ASAREE API client. - Authenticated with a per-user API token (sent as ``X-API-Key`` — - ASAREE has no server-wide key, see the SDK README's bootstrap section). + Authenticate with either a per-user API key (``X-API-Key``) or a session + access token (``Authorization: Bearer``). The auth resource can establish + and refresh a session on an initially unauthenticated client. Constructor arguments override environment variables: ``base_url`` reads - ``ASAREE_BASE_URL``, ``api_key`` reads ``ASAREE_API_KEY``, ``timeout`` - reads ``ASAREE_TIMEOUT``. + ``ASAREE_BASE_URL``, ``api_key`` reads ``ASAREE_API_KEY``, + ``access_token`` reads ``ASAREE_ACCESS_TOKEN``, and ``timeout`` reads + ``ASAREE_TIMEOUT``. """ def __init__( @@ -52,14 +56,24 @@ def __init__( base_url: str | None = None, *, api_key: str | None = None, + access_token: str | None = None, + refresh_token: str | None = None, timeout: float | httpx.Timeout | None = None, retry_policy: RetryPolicy | None = None, ) -> None: self.base_url = _resolve_base_url(base_url) - api_key = api_key or os.environ.get("ASAREE_API_KEY") + if api_key is not None and access_token is not None: + raise AsareeError("Pass either api_key or access_token, not both.") + if api_key is None and access_token is None: + api_key = os.environ.get("ASAREE_API_KEY") + access_token = os.environ.get("ASAREE_ACCESS_TOKEN") + if api_key and access_token: + raise AsareeError("Set either ASAREE_API_KEY or ASAREE_ACCESS_TOKEN, not both.") + self._refresh_token = refresh_token or os.environ.get("ASAREE_REFRESH_TOKEN") self._http = build_sync_client( base_url=self.base_url, api_key=api_key, + access_token=access_token, timeout=_resolve_timeout(timeout), policy=retry_policy, ) @@ -72,6 +86,17 @@ def __init__( self.llm_settings = LLMSettings(self) self.skills = Skills(self) self.okf = OKF(self) + self.auth = Auth(self) + self.users = Users(self) + + def _set_session_tokens(self, access_token: str, refresh_token: str) -> None: + self._http.headers.pop("X-API-Key", None) + self._http.headers["Authorization"] = f"Bearer {access_token}" + self._refresh_token = refresh_token + + def _clear_session_tokens(self) -> None: + self._http.headers.pop("Authorization", None) + self._refresh_token = None def _request(self, method: str, path: str, **kwargs: Any) -> Any: response = self._http.request(method, f"/api{path}", **kwargs) diff --git a/sdk/src/asaree_client/exceptions.py b/sdk/src/asaree_client/exceptions.py index 2a6930c2..9d975aa0 100644 --- a/sdk/src/asaree_client/exceptions.py +++ b/sdk/src/asaree_client/exceptions.py @@ -33,7 +33,7 @@ def __init__(self, detail: str, **kwargs: object) -> None: class AsareeAuthenticationError(AsareeAPIError): - """ASAREE API returned 401 Unauthorized — missing or invalid X-API-Key.""" + """ASAREE API returned 401 Unauthorized — invalid API key or session token.""" def __init__(self, detail: str, **kwargs: object) -> None: super().__init__(status_code=401, detail=detail, **kwargs) # type: ignore[arg-type] diff --git a/sdk/src/asaree_client/models.py b/sdk/src/asaree_client/models.py index 8a7a6115..563de6b8 100644 --- a/sdk/src/asaree_client/models.py +++ b/sdk/src/asaree_client/models.py @@ -456,3 +456,61 @@ class OKFDocument(BaseModel): error_message: str | None tool_names: list[str] created_at: datetime + + +class UserAccount(BaseModel): + id: uuid.UUID + email: str + display_name: str + is_active: bool + is_admin: bool + created_at: datetime + + +class AuthTokens(BaseModel): + access_token: str + refresh_token: str + token_type: str + expires_in: int + user: UserAccount + + +class Message(BaseModel): + message: str + + +class ApiToken(BaseModel): + id: uuid.UUID + name: str + token: str + token_prefix: str | None + expires_at: datetime | None + created_at: datetime + + +class ApiTokenSummary(BaseModel): + id: uuid.UUID + name: str + token_prefix: str | None + last_used_at: datetime | None + expires_at: datetime | None + is_revoked: bool + created_at: datetime + + +class ApiTokenPage(BaseModel): + items: list[ApiTokenSummary] + total: int + offset: int + limit: int + + +class BootstrapUser(BaseModel): + id: uuid.UUID + email: str + + +class BootstrapApiToken(BaseModel): + id: uuid.UUID + name: str + token: str diff --git a/sdk/src/asaree_client/resources/auth.py b/sdk/src/asaree_client/resources/auth.py new file mode 100644 index 00000000..d1d6c802 --- /dev/null +++ b/sdk/src/asaree_client/resources/auth.py @@ -0,0 +1,80 @@ +"""Account sessions, profiles, passwords, and API-token management.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from asaree_client.exceptions import AsareeAuthenticationError +from asaree_client.models import ApiToken, ApiTokenPage, AuthTokens, Message, UserAccount + +ResourceId = uuid.UUID | str + + +class Auth: + def __init__(self, client: Any) -> None: + self._client = client + + def register(self, *, email: str, password: str, display_name: str) -> UserAccount: + data = self._client._post( + "/auth/register", + json={"email": email, "password": password, "display_name": display_name}, + ) + return UserAccount(**data) + + def login(self, *, email: str, password: str) -> AuthTokens: + data = self._client._post("/auth/login", json={"email": email, "password": password}) + tokens = AuthTokens(**data) + self._client._set_session_tokens(tokens.access_token, tokens.refresh_token) + return tokens + + def refresh(self, refresh_token: str | None = None) -> AuthTokens: + token = refresh_token or self._client._refresh_token + if token is None: + raise AsareeAuthenticationError(detail="No refresh token available") + data = self._client._post( + "/auth/refresh", + headers={"Authorization": f"Bearer {token}"}, + ) + tokens = AuthTokens(**data) + self._client._set_session_tokens(tokens.access_token, tokens.refresh_token) + return tokens + + def logout(self) -> None: + self._client._post("/auth/logout") + self._client._clear_session_tokens() + + def forgot_password(self, email: str) -> Message: + return Message(**self._client._post("/auth/forgot-password", json={"email": email})) + + def reset_password(self, *, token: str, new_password: str) -> Message: + data = self._client._post("/auth/reset-password", json={"token": token, "new_password": new_password}) + return Message(**data) + + def get_profile(self) -> UserAccount: + return UserAccount(**self._client._get("/auth/me")) + + def update_profile(self, *, display_name: str | None = None, email: str | None = None) -> UserAccount: + payload = { + key: value for key, value in {"display_name": display_name, "email": email}.items() if value is not None + } + return UserAccount(**self._client._patch("/auth/me", json=payload)) + + def change_password(self, *, current_password: str, new_password: str) -> None: + self._client._post( + "/auth/me/password", + json={"current_password": current_password, "new_password": new_password}, + ) + + def create_api_token(self, *, name: str, expires_in_days: int | None = None) -> ApiToken: + payload: dict[str, Any] = {"name": name} + if expires_in_days is not None: + payload["expires_in_days"] = expires_in_days + return ApiToken(**self._client._post("/auth/me/tokens", json=payload)) + + def list_api_tokens(self, *, offset: int = 0, limit: int = 20) -> ApiTokenPage: + data = self._client._get("/auth/me/tokens", params={"offset": offset, "limit": limit}) + return ApiTokenPage(**data) + + def revoke_api_token(self, token_id: ResourceId) -> None: + self._client._delete(f"/auth/me/tokens/{token_id}") diff --git a/sdk/src/asaree_client/resources/experiments.py b/sdk/src/asaree_client/resources/experiments.py index 8ee4c97b..6a50d81e 100644 --- a/sdk/src/asaree_client/resources/experiments.py +++ b/sdk/src/asaree_client/resources/experiments.py @@ -46,8 +46,6 @@ def create( task_brief: dict[str, Any] | None = None, factors: builtins.list[dict[str, Any]] | None = None, measurement_plan: dict[str, Any] | None = None, - dataset_ids: builtins.list[ResourceId] | None = None, - dataset_id: ResourceId | None = None, ) -> Experiment: payload: dict[str, Any] = {"design_type": design_type} if name is not None: @@ -60,10 +58,6 @@ def create( payload["factors"] = factors if measurement_plan is not None: payload["measurement_plan"] = measurement_plan - if dataset_ids is not None: - payload["dataset_ids"] = [str(d) for d in dataset_ids] - elif dataset_id is not None: - payload["dataset_id"] = str(dataset_id) data = self._client._post("/experiments", json=payload) return Experiment(**data) diff --git a/sdk/src/asaree_client/resources/protocols.py b/sdk/src/asaree_client/resources/protocols.py index 39e7a3d4..adbc9ec4 100644 --- a/sdk/src/asaree_client/resources/protocols.py +++ b/sdk/src/asaree_client/resources/protocols.py @@ -83,8 +83,9 @@ def run( *, replicate_label: str | None = None, ) -> ProtocolRun: - """Compile and run this protocol's current graph -- 422 if it's - empty or has a cycle. Returns immediately with status "pending"; + """Run this protocol's immutable published revision -- 409 if it has + never been published, and 422 if the published graph is invalid, + empty, or has a cycle. Returns immediately with status "pending"; poll with get_run. ``replicate_label`` runs that one already-generated replicate (its cell's factor_values substituted in) instead of today's ad-hoc, un-substituted whole-graph run.""" diff --git a/sdk/src/asaree_client/resources/users.py b/sdk/src/asaree_client/resources/users.py new file mode 100644 index 00000000..ba448516 --- /dev/null +++ b/sdk/src/asaree_client/resources/users.py @@ -0,0 +1,32 @@ +"""Unauthenticated user and API-key bootstrap endpoints.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from asaree_client.models import BootstrapApiToken, BootstrapUser + +ResourceId = uuid.UUID | str + + +class Users: + def __init__(self, client: Any) -> None: + self._client = client + + def create(self, *, email: str, password: str) -> BootstrapUser: + data = self._client._post("/users", json={"email": email, "password": password}) + return BootstrapUser(**data) + + def issue_token( + self, + user_id: ResourceId, + *, + password: str, + name: str = "default", + ) -> BootstrapApiToken: + data = self._client._post( + f"/users/{user_id}/tokens", + json={"password": password, "name": name}, + ) + return BootstrapApiToken(**data) diff --git a/sdk/tests/test_auth.py b/sdk/tests/test_auth.py new file mode 100644 index 00000000..e9211a48 --- /dev/null +++ b/sdk/tests/test_auth.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from typing import Any + +import pytest + +from asaree_client._transport import build_headers +from asaree_client.exceptions import AsareeAuthenticationError +from asaree_client.resources.auth import Auth +from asaree_client.resources.users import Users + + +class AuthClient: + def __init__(self, responses: list[Any]) -> None: + self.responses = iter(responses) + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + self._refresh_token: str | None = None + self.session_tokens: tuple[str, str] | None = None + self.cleared = False + + def _request(self, method: str, path: str, **kwargs: Any) -> Any: + self.calls.append((method, path, kwargs)) + return next(self.responses) + + def _get(self, path: str, **kwargs: Any) -> Any: + return self._request("GET", path, **kwargs) + + def _post(self, path: str, **kwargs: Any) -> Any: + return self._request("POST", path, **kwargs) + + def _patch(self, path: str, **kwargs: Any) -> Any: + return self._request("PATCH", path, **kwargs) + + def _delete(self, path: str, **kwargs: Any) -> Any: + return self._request("DELETE", path, **kwargs) + + def _set_session_tokens(self, access_token: str, refresh_token: str) -> None: + self.session_tokens = (access_token, refresh_token) + self._refresh_token = refresh_token + + def _clear_session_tokens(self) -> None: + self.cleared = True + self._refresh_token = None + + +def _token_response() -> dict[str, Any]: + return { + "access_token": "access-2", + "refresh_token": "refresh-2", + "token_type": "bearer", + "expires_in": 900, + "user": { + "id": str(uuid.uuid4()), + "email": "researcher@example.com", + "display_name": "Researcher", + "is_active": True, + "is_admin": False, + "created_at": datetime.now(tz=UTC).isoformat(), + }, + } + + +def test_build_headers_supports_exactly_one_authentication_mode() -> None: + assert build_headers("api-key", "ignored")["X-API-Key"] == "api-key" + assert "Authorization" not in build_headers("api-key", "ignored") + assert build_headers(None, "access")["Authorization"] == "Bearer access" + + +def test_login_activates_returned_session() -> None: + client = AuthClient([_token_response()]) + + tokens = Auth(client).login(email="researcher@example.com", password="password123") + + assert client.calls == [ + ( + "POST", + "/auth/login", + {"json": {"email": "researcher@example.com", "password": "password123"}}, + ) + ] + assert client.session_tokens == (tokens.access_token, tokens.refresh_token) + + +def test_refresh_uses_and_rotates_stored_refresh_token() -> None: + client = AuthClient([_token_response()]) + client._refresh_token = "refresh-1" + + Auth(client).refresh() + + assert client.calls == [("POST", "/auth/refresh", {"headers": {"Authorization": "Bearer refresh-1"}})] + assert client.session_tokens == ("access-2", "refresh-2") + + +def test_refresh_requires_a_token() -> None: + with pytest.raises(AsareeAuthenticationError, match="No refresh token available"): + Auth(AuthClient([])).refresh() + + +def test_logout_clears_session_after_server_accepts_it() -> None: + client = AuthClient([None]) + + Auth(client).logout() + + assert client.calls == [("POST", "/auth/logout", {})] + assert client.cleared is True + + +def test_bootstrap_user_and_token_requests_are_json() -> None: + user_id = uuid.uuid4() + client = AuthClient( + [ + {"id": str(user_id), "email": "researcher@example.com"}, + {"id": str(uuid.uuid4()), "name": "automation", "token": "secret"}, + ] + ) + users = Users(client) + + user = users.create(email="researcher@example.com", password="password123") + token = users.issue_token(user.id, password="password123", name="automation") + + assert token.token == "secret" + assert client.calls == [ + ( + "POST", + "/users", + {"json": {"email": "researcher@example.com", "password": "password123"}}, + ), + ( + "POST", + f"/users/{user_id}/tokens", + {"json": {"password": "password123", "name": "automation"}}, + ), + ] From f653d4885edbc712db9cd8f53ba2601f3dbebda1 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 14:38:04 -0700 Subject: [PATCH 03/18] fix(sdk): close remaining API parity gaps --- sdk/src/asaree_client/__init__.py | 2 + sdk/src/asaree_client/_models/__init__.py | 1 + sdk/src/asaree_client/_models/agents.py | 62 ++ sdk/src/asaree_client/_models/auth.py | 64 ++ sdk/src/asaree_client/_models/datasets.py | 35 + sdk/src/asaree_client/_models/experiments.py | 130 ++++ sdk/src/asaree_client/_models/llm.py | 31 + sdk/src/asaree_client/_models/mcp.py | 24 + sdk/src/asaree_client/_models/okf.py | 44 ++ sdk/src/asaree_client/_models/protocols.py | 106 +++ sdk/src/asaree_client/_models/skills.py | 38 ++ sdk/src/asaree_client/_multipart.py | 25 + sdk/src/asaree_client/_sentinel.py | 15 + sdk/src/asaree_client/_transport.py | 15 +- sdk/src/asaree_client/models.py | 610 +++--------------- .../asaree_client/resources/experiments.py | 77 ++- sdk/src/asaree_client/resources/okf.py | 13 +- sdk/src/asaree_client/resources/protocols.py | 18 +- sdk/src/asaree_client/resources/skills.py | 22 +- sdk/src/asaree_client/resources/tools.py | 31 +- sdk/tests/test_sdk_gaps.py | 220 +++++++ 21 files changed, 977 insertions(+), 606 deletions(-) create mode 100644 sdk/src/asaree_client/_models/__init__.py create mode 100644 sdk/src/asaree_client/_models/agents.py create mode 100644 sdk/src/asaree_client/_models/auth.py create mode 100644 sdk/src/asaree_client/_models/datasets.py create mode 100644 sdk/src/asaree_client/_models/experiments.py create mode 100644 sdk/src/asaree_client/_models/llm.py create mode 100644 sdk/src/asaree_client/_models/mcp.py create mode 100644 sdk/src/asaree_client/_models/okf.py create mode 100644 sdk/src/asaree_client/_models/protocols.py create mode 100644 sdk/src/asaree_client/_models/skills.py create mode 100644 sdk/src/asaree_client/_multipart.py create mode 100644 sdk/src/asaree_client/_sentinel.py create mode 100644 sdk/tests/test_sdk_gaps.py diff --git a/sdk/src/asaree_client/__init__.py b/sdk/src/asaree_client/__init__.py index db993bb4..d1f3d681 100644 --- a/sdk/src/asaree_client/__init__.py +++ b/sdk/src/asaree_client/__init__.py @@ -54,6 +54,7 @@ Run, RunStep, Skill, + SkillPage, SkillUrlPreview, TestRun, ToolCallResult, @@ -122,6 +123,7 @@ "Run", "RunStep", "Skill", + "SkillPage", "SkillUrlPreview", "TestRun", "ToolCallResult", diff --git a/sdk/src/asaree_client/_models/__init__.py b/sdk/src/asaree_client/_models/__init__.py new file mode 100644 index 00000000..4e9cdbff --- /dev/null +++ b/sdk/src/asaree_client/_models/__init__.py @@ -0,0 +1 @@ +"""Private domain model implementations; public imports live in models.py.""" diff --git a/sdk/src/asaree_client/_models/agents.py b/sdk/src/asaree_client/_models/agents.py new file mode 100644 index 00000000..06190958 --- /dev/null +++ b/sdk/src/asaree_client/_models/agents.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class Agent(BaseModel): + id: uuid.UUID + name: str + goal: str + description: str + system_prompt: str + model_config_data: dict[str, Any] = Field(alias="model_config") + tool_config_data: dict[str, Any] = Field(alias="tool_config") + memory_config_data: dict[str, Any] = Field(alias="memory_config") + pattern_config: dict[str, Any] | None = None + skill_config: dict[str, Any] | None = None + output_contract: dict[str, Any] | None = None + is_system: bool = False + auto_eval_enabled: bool = True + auto_eval_model: str | None = None + source_plan_id: uuid.UUID | None = None + source_plan_title: str | None = None + budget_limit_usd: float | None = None + max_run_duration_seconds: int | None = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(populate_by_name=True) + + +class Run(BaseModel): + id: uuid.UUID + agent_id: uuid.UUID + status: str + input: str + output: str | None + output_text: str + payload: dict[str, Any] | None = None + error: str | None + token_usage: dict[str, Any] | None + cost_estimate: float | None + run_metadata: dict[str, Any] | None = None + pattern_overrides: dict[str, Any] | None = None + created_at: datetime + completed_at: datetime | None + + +class RunStep(BaseModel): + id: uuid.UUID + sequence: int + iteration: int | None + phase: str + input: dict[str, Any] | None + output: dict[str, Any] | None + llm_call: dict[str, Any] | None + tool_call: dict[str, Any] | None + started_at: datetime | None + completed_at: datetime | None diff --git a/sdk/src/asaree_client/_models/auth.py b/sdk/src/asaree_client/_models/auth.py new file mode 100644 index 00000000..cedbdf0e --- /dev/null +++ b/sdk/src/asaree_client/_models/auth.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel + + +class UserAccount(BaseModel): + id: uuid.UUID + email: str + display_name: str + is_active: bool + is_admin: bool + created_at: datetime + + +class AuthTokens(BaseModel): + access_token: str + refresh_token: str + token_type: str + expires_in: int + user: UserAccount + + +class Message(BaseModel): + message: str + + +class ApiToken(BaseModel): + id: uuid.UUID + name: str + token: str + token_prefix: str | None + expires_at: datetime | None + created_at: datetime + + +class ApiTokenSummary(BaseModel): + id: uuid.UUID + name: str + token_prefix: str | None + last_used_at: datetime | None + expires_at: datetime | None + is_revoked: bool + created_at: datetime + + +class ApiTokenPage(BaseModel): + items: list[ApiTokenSummary] + total: int + offset: int + limit: int + + +class BootstrapUser(BaseModel): + id: uuid.UUID + email: str + + +class BootstrapApiToken(BaseModel): + id: uuid.UUID + name: str + token: str diff --git a/sdk/src/asaree_client/_models/datasets.py b/sdk/src/asaree_client/_models/datasets.py new file mode 100644 index 00000000..8884515e --- /dev/null +++ b/sdk/src/asaree_client/_models/datasets.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel + + +class RegisteredDataset(BaseModel): + id: uuid.UUID + name: str + raw_path: str | None = None + raw_sha256: str | None = None + train_path: str | None = None + test_path: str | None = None + train_sha256: str | None = None + test_sha256: str | None = None + split_method: str | None = None + split_group_column: str | None = None + split_test_size: float | None = None + split_seed: int | None = None + target_column: str | None + description: str | None = None + dictionary_json: str | None = None + created_at: datetime | None = None + + +class WorkspaceEvent(BaseModel): + id: uuid.UUID + workspace_id: str + stage: str + event_type: str + sha256_train: str | None + sha256_test: str | None + created_at: datetime diff --git a/sdk/src/asaree_client/_models/experiments.py b/sdk/src/asaree_client/_models/experiments.py new file mode 100644 index 00000000..7dfb83ce --- /dev/null +++ b/sdk/src/asaree_client/_models/experiments.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + + +class Experiment(BaseModel): + id: uuid.UUID + name: str + description: str | None + hypothesis: str | None = None + design_type: str + task_brief: dict[str, Any] | None + design_spec: dict[str, Any] | None + measurement_plan: dict[str, Any] | None = None + dataset_ids: list[uuid.UUID] = Field(default_factory=list) + dataset_id: uuid.UUID | None = None + archived_at: datetime | None = None + metric_recommendations: dict[str, Any] | None = None + metric_recommendation_set_version: int = 1 + latest_test_run_id: uuid.UUID | None = None + locked_at: datetime | None = None + locked_protocol_revision_id: uuid.UUID | None = None + locked_design_spec: dict[str, Any] | None = None + locked_measurement_plan: dict[str, Any] | None = None + created_at: datetime + updated_at: datetime + + +class Replicate(BaseModel): + id: uuid.UUID + cell_id: uuid.UUID + cell_label: str + replicate_label: str + replicate_number: int + design_revision_id: uuid.UUID + run_id: uuid.UUID | None + workspace_id: str | None + factor_values: dict[str, Any] | None + metric_values: dict[str, Any] | None + artifacts: dict[str, Any] | None + created_at: datetime + updated_at: datetime + + +class DesignRevision(BaseModel): + id: uuid.UUID + revision: int + superseded_at: datetime | None + design_spec: dict[str, Any] | None + cell_count: int + replicate_count: int + scored_replicate_count: int + created_at: datetime + + +class ExperimentArtifact(BaseModel): + id: uuid.UUID + experiment_id: uuid.UUID + name: str + kind: str + content: str + created_at: datetime + updated_at: datetime + + +class Trial(BaseModel): + replicate_label: str + factor_values: dict[str, Any] + metric_values: dict[str, Any] + status: str + run_id: uuid.UUID | None + obsolete: bool + truncated: bool + error: str | None + updated_at: datetime + + +class ExperimentResults(BaseModel): + available: bool + reason: str | None + analysis: dict[str, Any] | None + best_condition: dict[str, Any] | None + + +class ExperimentRunResults(BaseModel): + overview: dict[str, Any] + metric_keys: list[str] + metric_types: dict[str, str] + metric_aggregations: dict[str, str] + metric_directions: dict[str, str] + primary_metric: str | None + primary_metric_direction: str | None + cells: list[dict[str, Any]] + replicates: list[dict[str, Any]] + + +class DesignImpact(BaseModel): + has_generated_design: bool + regeneration_required: bool + current_cell_count: int + proposed_cell_count: int + added_cell_count: int + retained_cell_count: int + removed_cell_count: int + current_replicate_count: int + proposed_replicate_count: int + added_replicate_count: int + retained_replicate_count: int + removed_replicate_count: int + regeneration_reasons: list[str] = Field(default_factory=list) + + +class MeasurementPlanValidationIssue(BaseModel): + code: str + message: str + path: str + blocking: bool = True + + +class MeasurementPlanValidation(BaseModel): + valid: bool + issues: list[MeasurementPlanValidationIssue] + + +class MeasurementCapabilities(BaseModel): + outputs: dict[str, list[str]] diff --git a/sdk/src/asaree_client/_models/llm.py b/sdk/src/asaree_client/_models/llm.py new file mode 100644 index 00000000..ffafea2f --- /dev/null +++ b/sdk/src/asaree_client/_models/llm.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LLMSetting(BaseModel): + provider: str + api_base: str | None + azure_project_endpoint: str | None = None + + +class LLMModelInfo(BaseModel): + id: str + label: str | None + supports_temperature: bool + supports_effort: bool + effort_levels: list[str] + supports_tool_calling: bool | None + + +class LLMModels(BaseModel): + models: list[LLMModelInfo] + source: str + note: str | None + + +class LLMConnectionCheck(BaseModel): + provider: str + status: str + detail: str + endpoint: str | None diff --git a/sdk/src/asaree_client/_models/mcp.py b/sdk/src/asaree_client/_models/mcp.py new file mode 100644 index 00000000..382e55b9 --- /dev/null +++ b/sdk/src/asaree_client/_models/mcp.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + + +class MCPServer(BaseModel): + id: uuid.UUID + name: str + transport: str + command: str | None + url: str | None + status: str + error_message: str | None + capabilities: dict[str, Any] | None + created_at: datetime + + +class ToolCallResult(BaseModel): + is_error: bool + content: str diff --git a/sdk/src/asaree_client/_models/okf.py b/sdk/src/asaree_client/_models/okf.py new file mode 100644 index 00000000..96982a04 --- /dev/null +++ b/sdk/src/asaree_client/_models/okf.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import BaseModel + + +class DirectoryEntry(BaseModel): + name: str + path: str + is_bundle: bool + + +class DirectoryListing(BaseModel): + path: str + absolute_path: str + parent: str | None + entries: list[DirectoryEntry] + + +class OKFBundle(BaseModel): + id: uuid.UUID + name: str + path: str | None + uploaded: bool + status: str + error_message: str | None + tool_names: list[str] + created_at: datetime + + +class OKFDocument(BaseModel): + id: uuid.UUID + name: str + title: str | None + description: str | None + concept_type: str | None + tags: list[str] + path: str | None + status: str + error_message: str | None + tool_names: list[str] + created_at: datetime diff --git a/sdk/src/asaree_client/_models/protocols.py b/sdk/src/asaree_client/_models/protocols.py new file mode 100644 index 00000000..78251bf0 --- /dev/null +++ b/sdk/src/asaree_client/_models/protocols.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + + +class Protocol(BaseModel): + id: uuid.UUID + name: str + description: str | None + experiment_id: uuid.UUID | None + graph: dict[str, Any] + published_revision_id: uuid.UUID | None = None + published_revision: int | None = None + has_unpublished_changes: bool = False + created_at: datetime + updated_at: datetime + + +class ProtocolRun(BaseModel): + id: uuid.UUID + protocol_id: uuid.UUID + status: str + node_runs: dict[str, Any] + error: str | None + replicate_label: str | None = None + replicate_result_id: uuid.UUID | None = None + factor_values: dict[str, Any] | None = None + design_revision_id: uuid.UUID | None = None + protocol_revision_id: uuid.UUID | None = None + target_node_id: str | None = None + conversation: dict[str, Any] | None = None + cancel_requested_at: datetime | None = None + created_at: datetime + updated_at: datetime + observations: list[dict[str, Any]] = Field(default_factory=list) + artifacts: list[dict[str, Any]] = Field(default_factory=list) + + +class ProtocolRevision(BaseModel): + id: uuid.UUID + protocol_id: uuid.UUID + revision: int + graph: dict[str, Any] + published_at: datetime + + +class CellRunBatch(BaseModel): + protocol_run_ids: list[uuid.UUID] + replicate_labels: list[str] + skipped: int + protocol_revision_id: uuid.UUID + protocol_revision: int + + +class PromptPreview(BaseModel): + text: str + + +class ResourceUsage(BaseModel): + duration_seconds: float | None + cost_usd: float | None + + +class TestRunResources(BaseModel): + task: ResourceUsage + evaluation: ResourceUsage + total: ResourceUsage + + +class TestRunFreshness(BaseModel): + out_of_date: bool + reasons: list[str] + + +class TestRunExecutionSummary(BaseModel): + node_runs: dict[str, Any] + started_at: datetime | None + completed_at: datetime | None + cancel_requested_at: datetime | None + + +class TestedPublishedRevision(BaseModel): + id: uuid.UUID + number: int + published_at: datetime + + +class TestRun(BaseModel): + id: uuid.UUID + protocol_id: uuid.UUID + status: str + error: str | None + protocol_revision_id: uuid.UUID | None + created_at: datetime + updated_at: datetime + observations: list[dict[str, Any]] + artifacts: list[dict[str, Any]] + conversation: dict[str, Any] | None + tested_published_revision: TestedPublishedRevision | None + freshness: TestRunFreshness + resources: TestRunResources + execution_summary: TestRunExecutionSummary diff --git a/sdk/src/asaree_client/_models/skills.py b/sdk/src/asaree_client/_models/skills.py new file mode 100644 index 00000000..f420ef2b --- /dev/null +++ b/sdk/src/asaree_client/_models/skills.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + + +class Skill(BaseModel): + id: uuid.UUID + name: str + description: str + body: str + frontmatter: dict[str, Any] + is_system: bool + source_filename: str | None + files: list[str] + created_at: datetime + updated_at: datetime + + +class SkillPage(BaseModel): + items: list[Skill] + total: int + + +class DiscoveredSkill(BaseModel): + subdirectory: str + name: str + description: str + file_count: int + + +class SkillUrlPreview(BaseModel): + source: str + ref: str + skills: list[DiscoveredSkill] diff --git a/sdk/src/asaree_client/_multipart.py b/sdk/src/asaree_client/_multipart.py new file mode 100644 index 00000000..adc54478 --- /dev/null +++ b/sdk/src/asaree_client/_multipart.py @@ -0,0 +1,25 @@ +"""Multipart directory uploads shared by folder-backed resources.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import BinaryIO + +MultipartFiles = list[tuple[str, tuple[str, BinaryIO]]] + + +@contextmanager +def multipart_directory(directory: str) -> Iterator[MultipartFiles]: + """Open every file below *directory* and preserve its picked-folder path.""" + root = Path(directory) + if not root.is_dir(): + raise ValueError(f"Not a directory: {directory}") + paths = sorted(path for path in root.rglob("*") if path.is_file()) + with ExitStack() as stack: + files: MultipartFiles = [] + for path in paths: + handle: BinaryIO = stack.enter_context(path.open("rb")) + files.append(("files", (f"{root.name}/{path.relative_to(root).as_posix()}", handle))) + yield files diff --git a/sdk/src/asaree_client/_sentinel.py b/sdk/src/asaree_client/_sentinel.py new file mode 100644 index 00000000..1753d4c3 --- /dev/null +++ b/sdk/src/asaree_client/_sentinel.py @@ -0,0 +1,15 @@ +"""Typed sentinel for PATCH-style omit-vs-null request fields.""" + +from __future__ import annotations + +from typing import Final + + +class UnsetType: + __slots__ = () + + def __repr__(self) -> str: + return "UNSET" + + +UNSET: Final = UnsetType() diff --git a/sdk/src/asaree_client/_transport.py b/sdk/src/asaree_client/_transport.py index f425c806..da6ca89c 100644 --- a/sdk/src/asaree_client/_transport.py +++ b/sdk/src/asaree_client/_transport.py @@ -28,6 +28,7 @@ ) _DEFAULT_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504}) +_IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"}) @dataclass @@ -90,16 +91,18 @@ def raise_for_status(response: httpx.Response) -> None: def _effective_max_retries(request: httpx.Request, policy: RetryPolicy) -> int: - """Per-request retry override: a truthy ``asaree_no_retry`` extension forces 0. + """Retry safe methods by default; require an explicit opt-in for others. - Callers set ``extensions={"asaree_no_retry": True}`` on a request (e.g. a - long, non-idempotent direct tool invocation like ``run_model_script``) to - opt out of automatic retries while keeping the client-wide policy for - everything else. + A timed-out POST may already have created or enqueued work server-side, so + replaying it can duplicate runs, uploads, artifacts, or tokens. Callers + may deliberately opt a known-safe non-idempotent method in with + ``asaree_allow_retry``; ``asaree_no_retry`` always wins. """ if request.extensions.get("asaree_no_retry"): return 0 - return policy.max_retries + if request.method in _IDEMPOTENT_METHODS or request.extensions.get("asaree_allow_retry"): + return policy.max_retries + return 0 class RetryTransport(httpx.BaseTransport): diff --git a/sdk/src/asaree_client/models.py b/sdk/src/asaree_client/models.py index 563de6b8..ee859f02 100644 --- a/sdk/src/asaree_client/models.py +++ b/sdk/src/asaree_client/models.py @@ -1,516 +1,100 @@ -"""Response models — one per resource, matching exactly what ASAREE's API returns. +"""Stable public imports for ASAREE API response models. -``model_config`` can't be a field name on a pydantic ``BaseModel`` (it's the -reserved settings attribute), so ``Agent`` follows the same workaround -``motoro.schemas.agent.AgentResponse`` uses on the server side: the -field is named ``model_config_data`` and populated from the wire key -``model_config`` via an alias. +Model implementations are grouped by domain in private modules so resources +can evolve independently without changing the public import path. """ -from __future__ import annotations - -import uuid -from datetime import datetime -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - - -class Agent(BaseModel): - id: uuid.UUID - name: str - goal: str - description: str - system_prompt: str - model_config_data: dict[str, Any] = Field(alias="model_config") - tool_config_data: dict[str, Any] = Field(alias="tool_config") - memory_config_data: dict[str, Any] = Field(alias="memory_config") - pattern_config: dict[str, Any] | None = None - output_contract: dict[str, Any] | None = None - budget_limit_usd: float | None = None - max_run_duration_seconds: int | None = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(populate_by_name=True) - - -class Run(BaseModel): - id: uuid.UUID - agent_id: uuid.UUID - status: str - input: str - output: str | None - # Derived from `output` server-side: the human-readable text (unwrapped - # from the output-contract envelope, if any) and the structured payload - # the envelope carries when the agent declares an output_contract. - output_text: str - payload: dict[str, Any] | None = None - error: str | None - token_usage: dict[str, Any] | None - cost_estimate: float | None - run_metadata: dict[str, Any] | None = None - pattern_overrides: dict[str, Any] | None = None - created_at: datetime - completed_at: datetime | None - - -class RunStep(BaseModel): - id: uuid.UUID - sequence: int - iteration: int | None - phase: str - input: dict[str, Any] | None - output: dict[str, Any] | None - llm_call: dict[str, Any] | None - tool_call: dict[str, Any] | None - started_at: datetime | None - completed_at: datetime | None - - -class Experiment(BaseModel): - id: uuid.UUID - name: str - description: str | None - hypothesis: str | None = None - design_type: str - task_brief: dict[str, Any] | None - design_spec: dict[str, Any] | None - measurement_plan: dict[str, Any] | None = None - # Every dataset attached to this experiment, in canvas wiring order -- - # an experiment can run against several since the Dataset connector was - # uncapped. ``dataset_id`` is a read-only view of the first one, kept so - # code written before that keeps working; it is no longer a stored column. - dataset_ids: list[uuid.UUID] = Field(default_factory=list) - dataset_id: uuid.UUID | None = None - archived_at: datetime | None = None - metric_recommendations: dict[str, Any] | None = None - metric_recommendation_set_version: int = 1 - latest_test_run_id: uuid.UUID | None = None - locked_at: datetime | None = None - locked_protocol_revision_id: uuid.UUID | None = None - locked_design_spec: dict[str, Any] | None = None - locked_measurement_plan: dict[str, Any] | None = None - created_at: datetime - updated_at: datetime - - -class Replicate(BaseModel): - id: uuid.UUID - cell_id: uuid.UUID - cell_label: str - replicate_label: str - replicate_number: int - # Which generation of the experiment's design this observation was made - # under. Replicates from a superseded design stay in the database as history, - # so a replicate is only part of the live design if this matches the - # experiment's current DesignRevision. - design_revision_id: uuid.UUID - run_id: uuid.UUID | None - workspace_id: str | None - factor_values: dict[str, Any] | None - metric_values: dict[str, Any] | None - artifacts: dict[str, Any] | None - created_at: datetime - updated_at: datetime - - -class DesignRevision(BaseModel): - """One generation of an experiment's factorial design. - - Regenerating a design that no longer produces the same set of cells - supersedes the current revision and opens a new one; the old cells (and - whatever was scored in them) are kept as history rather than deleted. - ``superseded_at is None`` marks the one revision that is current. - """ - - id: uuid.UUID - revision: int - superseded_at: datetime | None - design_spec: dict[str, Any] | None - cell_count: int - replicate_count: int - scored_replicate_count: int - created_at: datetime - - -class ExperimentArtifact(BaseModel): - """A durable, experiment-level (not per-cell) record -- an ``analyze`` - snapshot, a CSV export, or anything else a use case wants to keep past - one run. ``content`` is opaque to ASAREE -- a CSV export and a - JSON-encoded analyze result serialize completely differently, and - neither is interpreted server-side.""" - - id: uuid.UUID - experiment_id: uuid.UUID - name: str - kind: str - content: str - created_at: datetime - updated_at: datetime - - -class Protocol(BaseModel): - id: uuid.UUID - name: str - description: str | None - experiment_id: uuid.UUID | None - graph: dict[str, Any] - published_revision_id: uuid.UUID | None = None - published_revision: int | None = None - has_unpublished_changes: bool = False - created_at: datetime - updated_at: datetime - - -class ProtocolRun(BaseModel): - id: uuid.UUID - protocol_id: uuid.UUID - status: str - node_runs: dict[str, Any] - error: str | None - replicate_label: str | None = None - replicate_result_id: uuid.UUID | None = None - factor_values: dict[str, Any] | None = None - design_revision_id: uuid.UUID | None = None - protocol_revision_id: uuid.UUID | None = None - target_node_id: str | None = None - conversation: dict[str, Any] | None = None - cancel_requested_at: datetime | None = None - created_at: datetime - updated_at: datetime - observations: list[dict[str, Any]] = Field(default_factory=list) - artifacts: list[dict[str, Any]] = Field(default_factory=list) - - -class ProtocolRevision(BaseModel): - id: uuid.UUID - protocol_id: uuid.UUID - revision: int - graph: dict[str, Any] - published_at: datetime - - -class CellRunBatch(BaseModel): - """ "Run all cells" fanout result -- one ProtocolRun id per not-yet-completed - replicate; ``skipped`` is how many replicates already had metrics or a completed - run and were left alone (resume semantics).""" - - protocol_run_ids: list[uuid.UUID] - replicate_labels: list[str] - skipped: int - protocol_revision_id: uuid.UUID - protocol_revision: int - - -class PromptPreview(BaseModel): - text: str - - -class ResourceUsage(BaseModel): - duration_seconds: float | None - cost_usd: float | None - - -class TestRunResources(BaseModel): - task: ResourceUsage - evaluation: ResourceUsage - total: ResourceUsage - - -class TestRunFreshness(BaseModel): - out_of_date: bool - reasons: list[str] - - -class TestRunExecutionSummary(BaseModel): - node_runs: dict[str, Any] - started_at: datetime | None - completed_at: datetime | None - cancel_requested_at: datetime | None - - -class TestedPublishedRevision(BaseModel): - id: uuid.UUID - number: int - published_at: datetime - - -class TestRun(BaseModel): - id: uuid.UUID - protocol_id: uuid.UUID - status: str - error: str | None - protocol_revision_id: uuid.UUID | None - created_at: datetime - updated_at: datetime - observations: list[dict[str, Any]] - artifacts: list[dict[str, Any]] - conversation: dict[str, Any] | None - tested_published_revision: TestedPublishedRevision | None - freshness: TestRunFreshness - resources: TestRunResources - execution_summary: TestRunExecutionSummary - - -class RegisteredDataset(BaseModel): - id: uuid.UUID - name: str - raw_path: str | None = None - raw_sha256: str | None = None - # Null until a split is actually produced (Datasets.quick_split/ - # register_manual_split) -- registration itself only stores the raw - # file, it never splits (see RegisteredDataset's own comment in the - # backend model). - train_path: str | None = None - test_path: str | None = None - train_sha256: str | None = None - test_sha256: str | None = None - split_method: str | None = None - split_group_column: str | None = None - split_test_size: float | None = None - split_seed: int | None = None - target_column: str | None - description: str | None = None - dictionary_json: str | None = None - created_at: datetime | None = None - - -class LLMSetting(BaseModel): - provider: str - api_base: str | None - azure_project_endpoint: str | None = None - - -class LLMModelInfo(BaseModel): - id: str - label: str | None - supports_temperature: bool - supports_effort: bool - effort_levels: list[str] - supports_tool_calling: bool | None - - -class LLMModels(BaseModel): - models: list[LLMModelInfo] - source: str - note: str | None - - -class LLMConnectionCheck(BaseModel): - provider: str - status: str - detail: str - endpoint: str | None - - -class WorkspaceEvent(BaseModel): - id: uuid.UUID - workspace_id: str - stage: str - event_type: str - sha256_train: str | None - sha256_test: str | None - created_at: datetime - - -class MCPServer(BaseModel): - id: uuid.UUID - name: str - transport: str - command: str | None - url: str | None - status: str - error_message: str | None - capabilities: dict[str, Any] | None - created_at: datetime - - -class ToolCallResult(BaseModel): - is_error: bool - content: str - - -class Trial(BaseModel): - replicate_label: str - factor_values: dict[str, Any] - metric_values: dict[str, Any] - status: str - run_id: uuid.UUID | None - obsolete: bool - truncated: bool - error: str | None - updated_at: datetime - - -class ExperimentResults(BaseModel): - available: bool - reason: str | None - analysis: dict[str, Any] | None - best_condition: dict[str, Any] | None - - -class ExperimentRunResults(BaseModel): - overview: dict[str, Any] - metric_keys: list[str] - metric_types: dict[str, str] - metric_aggregations: dict[str, str] - metric_directions: dict[str, str] - primary_metric: str | None - primary_metric_direction: str | None - cells: list[dict[str, Any]] - replicates: list[dict[str, Any]] - - -class DesignImpact(BaseModel): - has_generated_design: bool - regeneration_required: bool - current_cell_count: int - proposed_cell_count: int - added_cell_count: int - retained_cell_count: int - removed_cell_count: int - current_replicate_count: int - proposed_replicate_count: int - added_replicate_count: int - retained_replicate_count: int - removed_replicate_count: int - regeneration_reasons: list[str] = Field(default_factory=list) - - -class MeasurementPlanValidationIssue(BaseModel): - code: str - message: str - path: str - blocking: bool = True - - -class MeasurementPlanValidation(BaseModel): - valid: bool - issues: list[MeasurementPlanValidationIssue] - - -class MeasurementCapabilities(BaseModel): - outputs: dict[str, list[str]] - - -class Skill(BaseModel): - id: uuid.UUID - name: str - description: str - body: str - frontmatter: dict[str, Any] - is_system: bool - source_filename: str | None - files: list[str] - created_at: datetime - updated_at: datetime - - -class DiscoveredSkill(BaseModel): - subdirectory: str - name: str - description: str - file_count: int - - -class SkillUrlPreview(BaseModel): - source: str - ref: str - skills: list[DiscoveredSkill] - - -class DirectoryEntry(BaseModel): - name: str - path: str - is_bundle: bool - - -class DirectoryListing(BaseModel): - path: str - absolute_path: str - parent: str | None - entries: list[DirectoryEntry] - - -class OKFBundle(BaseModel): - id: uuid.UUID - name: str - path: str | None - uploaded: bool - status: str - error_message: str | None - tool_names: list[str] - created_at: datetime - - -class OKFDocument(BaseModel): - id: uuid.UUID - name: str - title: str | None - description: str | None - concept_type: str | None - tags: list[str] - path: str | None - status: str - error_message: str | None - tool_names: list[str] - created_at: datetime - - -class UserAccount(BaseModel): - id: uuid.UUID - email: str - display_name: str - is_active: bool - is_admin: bool - created_at: datetime - - -class AuthTokens(BaseModel): - access_token: str - refresh_token: str - token_type: str - expires_in: int - user: UserAccount - - -class Message(BaseModel): - message: str - - -class ApiToken(BaseModel): - id: uuid.UUID - name: str - token: str - token_prefix: str | None - expires_at: datetime | None - created_at: datetime - - -class ApiTokenSummary(BaseModel): - id: uuid.UUID - name: str - token_prefix: str | None - last_used_at: datetime | None - expires_at: datetime | None - is_revoked: bool - created_at: datetime - - -class ApiTokenPage(BaseModel): - items: list[ApiTokenSummary] - total: int - offset: int - limit: int - - -class BootstrapUser(BaseModel): - id: uuid.UUID - email: str - - -class BootstrapApiToken(BaseModel): - id: uuid.UUID - name: str - token: str +from asaree_client._models.agents import Agent, Run, RunStep +from asaree_client._models.auth import ( + ApiToken, + ApiTokenPage, + ApiTokenSummary, + AuthTokens, + BootstrapApiToken, + BootstrapUser, + Message, + UserAccount, +) +from asaree_client._models.datasets import RegisteredDataset, WorkspaceEvent +from asaree_client._models.experiments import ( + DesignImpact, + DesignRevision, + Experiment, + ExperimentArtifact, + ExperimentResults, + ExperimentRunResults, + MeasurementCapabilities, + MeasurementPlanValidation, + MeasurementPlanValidationIssue, + Replicate, + Trial, +) +from asaree_client._models.llm import LLMConnectionCheck, LLMModelInfo, LLMModels, LLMSetting +from asaree_client._models.mcp import MCPServer, ToolCallResult +from asaree_client._models.okf import DirectoryEntry, DirectoryListing, OKFBundle, OKFDocument +from asaree_client._models.protocols import ( + CellRunBatch, + PromptPreview, + Protocol, + ProtocolRevision, + ProtocolRun, + ResourceUsage, + TestedPublishedRevision, + TestRun, + TestRunExecutionSummary, + TestRunFreshness, + TestRunResources, +) +from asaree_client._models.skills import DiscoveredSkill, Skill, SkillPage, SkillUrlPreview + +__all__ = [ + "Agent", + "ApiToken", + "ApiTokenPage", + "ApiTokenSummary", + "AuthTokens", + "BootstrapApiToken", + "BootstrapUser", + "CellRunBatch", + "DesignImpact", + "DesignRevision", + "DirectoryEntry", + "DirectoryListing", + "DiscoveredSkill", + "Experiment", + "ExperimentArtifact", + "ExperimentResults", + "ExperimentRunResults", + "LLMConnectionCheck", + "LLMModelInfo", + "LLMModels", + "LLMSetting", + "MCPServer", + "MeasurementCapabilities", + "MeasurementPlanValidation", + "MeasurementPlanValidationIssue", + "Message", + "OKFBundle", + "OKFDocument", + "PromptPreview", + "Protocol", + "ProtocolRevision", + "ProtocolRun", + "RegisteredDataset", + "Replicate", + "ResourceUsage", + "Run", + "RunStep", + "Skill", + "SkillPage", + "SkillUrlPreview", + "TestedPublishedRevision", + "TestRun", + "TestRunExecutionSummary", + "TestRunFreshness", + "TestRunResources", + "ToolCallResult", + "Trial", + "UserAccount", + "WorkspaceEvent", +] diff --git a/sdk/src/asaree_client/resources/experiments.py b/sdk/src/asaree_client/resources/experiments.py index 6a50d81e..522fbb39 100644 --- a/sdk/src/asaree_client/resources/experiments.py +++ b/sdk/src/asaree_client/resources/experiments.py @@ -13,6 +13,7 @@ from datetime import datetime from typing import Any +from asaree_client._sentinel import UNSET, UnsetType from asaree_client.models import ( DesignImpact, DesignRevision, @@ -27,10 +28,6 @@ ) ResourceId = uuid.UUID | str -# Distinguishes "omit this kwarg" (leave unchanged) from "pass None" -# (explicitly clear/detach) in update() below -- a plain default of None -# can't tell those apart. -_UNSET: Any = object() class Experiments: @@ -106,16 +103,16 @@ def update( self, experiment_id: ResourceId, *, - name: str | None = _UNSET, - description: str | None = _UNSET, - hypothesis: str | None = _UNSET, - dataset_ids: builtins.list[ResourceId] | None = _UNSET, - dataset_id: ResourceId | None = _UNSET, - design_spec: dict[str, Any] | None = _UNSET, - measurement_plan: dict[str, Any] | None = _UNSET, - measurement_validation_protocol_id: ResourceId | None = _UNSET, - metric_recommendations: dict[str, Any] | None = _UNSET, - archived_at: datetime | None = _UNSET, + name: str | None | UnsetType = UNSET, + description: str | None | UnsetType = UNSET, + hypothesis: str | None | UnsetType = UNSET, + dataset_ids: builtins.list[ResourceId] | None | UnsetType = UNSET, + dataset_id: ResourceId | None | UnsetType = UNSET, + design_spec: dict[str, Any] | None | UnsetType = UNSET, + measurement_plan: dict[str, Any] | None | UnsetType = UNSET, + measurement_validation_protocol_id: ResourceId | None | UnsetType = UNSET, + metric_recommendations: dict[str, Any] | None | UnsetType = UNSET, + archived_at: datetime | None | UnsetType = UNSET, ) -> Experiment: """Only the fields actually passed are sent (omit one to leave it unchanged; pass ``None`` explicitly to clear/detach it) -- @@ -135,27 +132,27 @@ def update( longer exposes ``delete()`` at all, to prevent accidental data loss; it's still here for scripted cleanup).""" payload: dict[str, Any] = {} - if name is not _UNSET: + if not isinstance(name, UnsetType): payload["name"] = name - if description is not _UNSET: + if not isinstance(description, UnsetType): payload["description"] = description - if hypothesis is not _UNSET: + if not isinstance(hypothesis, UnsetType): payload["hypothesis"] = hypothesis - if dataset_ids is not _UNSET: + if not isinstance(dataset_ids, UnsetType): payload["dataset_ids"] = [str(d) for d in dataset_ids] if dataset_ids else [] - if dataset_id is not _UNSET: + if not isinstance(dataset_id, UnsetType): payload["dataset_id"] = str(dataset_id) if dataset_id else None - if design_spec is not _UNSET: + if not isinstance(design_spec, UnsetType): payload["design_spec"] = design_spec - if measurement_plan is not _UNSET: + if not isinstance(measurement_plan, UnsetType): payload["measurement_plan"] = measurement_plan - if measurement_validation_protocol_id is not _UNSET: + if not isinstance(measurement_validation_protocol_id, UnsetType): payload["measurement_validation_protocol_id"] = ( str(measurement_validation_protocol_id) if measurement_validation_protocol_id else None ) - if metric_recommendations is not _UNSET: + if not isinstance(metric_recommendations, UnsetType): payload["metric_recommendations"] = metric_recommendations - if archived_at is not _UNSET: + if not isinstance(archived_at, UnsetType): payload["archived_at"] = archived_at.isoformat() if archived_at else None data = self._client._patch(f"/experiments/{experiment_id}", json=payload) return Experiment(**data) @@ -164,10 +161,10 @@ def generate_design( self, experiment_id: ResourceId, *, - hypothesis: str | None = _UNSET, - design_spec: dict[str, Any] | None = _UNSET, - measurement_plan: dict[str, Any] | None = _UNSET, - measurement_validation_protocol_id: ResourceId | None = _UNSET, + hypothesis: str | None | UnsetType = UNSET, + design_spec: dict[str, Any] | None | UnsetType = UNSET, + measurement_plan: dict[str, Any] | None | UnsetType = UNSET, + measurement_validation_protocol_id: ResourceId | None | UnsetType = UNSET, ) -> builtins.list[Replicate]: """Materialize one cell per combination of the experiment's declared factors and their replicate results, returning the current replicates. @@ -185,9 +182,9 @@ def generate_design( "design_spec": design_spec, "measurement_plan": measurement_plan, }.items(): - if value is not _UNSET: + if not isinstance(value, UnsetType): payload[key] = value - if measurement_validation_protocol_id is not _UNSET: + if not isinstance(measurement_validation_protocol_id, UnsetType): payload["measurement_validation_protocol_id"] = ( str(measurement_validation_protocol_id) if measurement_validation_protocol_id else None ) @@ -242,25 +239,25 @@ def upsert_replicate( experiment_id: ResourceId, replicate_label: str, *, - run_id: ResourceId | None = _UNSET, - workspace_id: str | None = _UNSET, - factor_values: dict[str, Any] | None = _UNSET, - metric_values: dict[str, Any] | None = _UNSET, - artifacts: dict[str, Any] | None = _UNSET, + run_id: ResourceId | None | UnsetType = UNSET, + workspace_id: str | None | UnsetType = UNSET, + factor_values: dict[str, Any] | None | UnsetType = UNSET, + metric_values: dict[str, Any] | None | UnsetType = UNSET, + artifacts: dict[str, Any] | None | UnsetType = UNSET, ) -> Replicate: """Merge fields onto a replicate-result row — pass just what changed; unset fields are left untouched (a pre-scoring call and a post-scoring call land on the same row without either erasing the other).""" payload: dict[str, Any] = {} - if run_id is not _UNSET: + if not isinstance(run_id, UnsetType): payload["run_id"] = str(run_id) if run_id else None - if workspace_id is not _UNSET: + if not isinstance(workspace_id, UnsetType): payload["workspace_id"] = workspace_id - if factor_values is not _UNSET: + if not isinstance(factor_values, UnsetType): payload["factor_values"] = factor_values - if metric_values is not _UNSET: + if not isinstance(metric_values, UnsetType): payload["metric_values"] = metric_values - if artifacts is not _UNSET: + if not isinstance(artifacts, UnsetType): payload["artifacts"] = artifacts data = self._client._put(f"/experiments/{experiment_id}/replicates/{replicate_label}", json=payload) return Replicate(**data) diff --git a/sdk/src/asaree_client/resources/okf.py b/sdk/src/asaree_client/resources/okf.py index 21d4747c..dc7ed82a 100644 --- a/sdk/src/asaree_client/resources/okf.py +++ b/sdk/src/asaree_client/resources/okf.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from asaree_client._multipart import multipart_directory from asaree_client.models import DirectoryListing, OKFBundle, OKFDocument ResourceId = uuid.UUID | str @@ -26,18 +27,8 @@ def register_bundle(self, path: str) -> OKFBundle: return OKFBundle(**self._client._post("/okf/bundles", json={"path": path})) def upload_bundle(self, directory: str) -> OKFBundle: - root = Path(directory) - paths = sorted(path for path in root.rglob("*") if path.is_file()) - handles = [path.open("rb") for path in paths] - try: - files = [ - ("files", (f"{root.name}/{path.relative_to(root).as_posix()}", handle)) - for path, handle in zip(paths, handles, strict=True) - ] + with multipart_directory(directory) as files: data = self._client._post("/okf/bundles/upload", files=files) - finally: - for handle in handles: - handle.close() return OKFBundle(**data) def refresh_bundle(self, bundle_id: ResourceId) -> OKFBundle: diff --git a/sdk/src/asaree_client/resources/protocols.py b/sdk/src/asaree_client/resources/protocols.py index adbc9ec4..d815c95b 100644 --- a/sdk/src/asaree_client/resources/protocols.py +++ b/sdk/src/asaree_client/resources/protocols.py @@ -6,10 +6,10 @@ import uuid from typing import Any +from asaree_client._sentinel import UNSET, UnsetType from asaree_client.models import CellRunBatch, PromptPreview, Protocol, ProtocolRevision, ProtocolRun, TestRun ResourceId = uuid.UUID | str -_UNSET: Any = object() class Protocols: @@ -47,19 +47,19 @@ def update( self, protocol_id: ResourceId, *, - name: str | None = _UNSET, - description: str | None = _UNSET, - experiment_id: ResourceId | None = _UNSET, - graph: dict[str, Any] | None = _UNSET, + name: str | None | UnsetType = UNSET, + description: str | None | UnsetType = UNSET, + experiment_id: ResourceId | None | UnsetType = UNSET, + graph: dict[str, Any] | None | UnsetType = UNSET, ) -> Protocol: payload: dict[str, Any] = {} - if name is not _UNSET: + if not isinstance(name, UnsetType): payload["name"] = name - if description is not _UNSET: + if not isinstance(description, UnsetType): payload["description"] = description - if experiment_id is not _UNSET: + if not isinstance(experiment_id, UnsetType): payload["experiment_id"] = str(experiment_id) if experiment_id else None - if graph is not _UNSET: + if not isinstance(graph, UnsetType): payload["graph"] = graph data = self._client._patch(f"/protocols/{protocol_id}", json=payload) return Protocol(**data) diff --git a/sdk/src/asaree_client/resources/skills.py b/sdk/src/asaree_client/resources/skills.py index cb3c3afa..a376d647 100644 --- a/sdk/src/asaree_client/resources/skills.py +++ b/sdk/src/asaree_client/resources/skills.py @@ -7,7 +7,8 @@ from pathlib import Path from typing import Any -from asaree_client.models import Skill, SkillUrlPreview +from asaree_client._multipart import multipart_directory +from asaree_client.models import Skill, SkillPage, SkillUrlPreview ResourceId = uuid.UUID | str @@ -37,31 +38,24 @@ def replace_folder(self, skill_id: ResourceId, directory: str) -> Skill: return self._folder_request("PUT", f"/skills/{skill_id}/folder", directory) def _folder_request(self, method: str, path: str, directory: str) -> Skill: - root = Path(directory) - paths = sorted(path for path in root.rglob("*") if path.is_file()) - handles = [path.open("rb") for path in paths] - try: - files = [ - ("files", (f"{root.name}/{path.relative_to(root).as_posix()}", handle)) - for path, handle in zip(paths, handles, strict=True) - ] + with multipart_directory(directory) as files: data = self._client._request(method, path, files=files) - finally: - for handle in handles: - handle.close() return Skill(**data) def preview_url(self, url: str) -> SkillUrlPreview: data = self._client._post("/skills/from-url/preview", json={"url": url}) return SkillUrlPreview(**data) - def create_from_url(self, url: str, subdirectory: str) -> Skill: + def create_from_url(self, url: str, subdirectory: str = "") -> Skill: data = self._client._post("/skills/from-url", json={"url": url, "subdirectory": subdirectory}) return Skill(**data) def list(self, *, limit: int = 100) -> builtins.list[Skill]: + return self.list_page(limit=limit).items + + def list_page(self, *, limit: int = 100) -> SkillPage: data = self._client._get("/skills", params={"limit": limit}) - return [Skill(**item) for item in data["items"]] + return SkillPage(**data) def get(self, skill_id: ResourceId) -> Skill: return Skill(**self._client._get(f"/skills/{skill_id}")) diff --git a/sdk/src/asaree_client/resources/tools.py b/sdk/src/asaree_client/resources/tools.py index b55f4c51..9bce686c 100644 --- a/sdk/src/asaree_client/resources/tools.py +++ b/sdk/src/asaree_client/resources/tools.py @@ -8,7 +8,6 @@ from asaree_client.models import MCPServer, ToolCallResult ResourceId = uuid.UUID | str -_UNSET: Any = object() class Tools: @@ -41,12 +40,18 @@ def update_server( self, server_id: ResourceId, *, - name: str | None = _UNSET, - transport: str | None = _UNSET, - command: str | None = _UNSET, - url: str | None = _UNSET, - headers: dict[str, str] | None = _UNSET, + name: str | None = None, + transport: str | None = None, + command: str | None = None, + url: str | None = None, + headers: dict[str, str] | None = None, ) -> MCPServer: + """Update supplied connection fields. + + ``None`` means unchanged, matching the server and Motoro contract. + The current endpoint cannot clear a saved command, URL, or headers; + replace the registration when that is required. + """ payload = { key: value for key, value in { @@ -56,7 +61,7 @@ def update_server( "url": url, "headers": headers, }.items() - if value is not _UNSET + if value is not None } return MCPServer(**self._client._patch(f"/mcp-servers/{server_id}", json=payload)) @@ -76,19 +81,19 @@ def call_tool( arguments: dict[str, Any] | None = None, *, timeout: float | None = None, - retry: bool = True, + retry: bool = False, ) -> ToolCallResult: """*timeout* overrides the client's default for just this call (e.g. a long-running direct tool invocation like ``run_model_script``). - *retry* set to ``False`` opts this call out of the client's automatic - retry policy — for a non-idempotent call where re-sending on a - transient failure could double-run something expensive. + Direct tool calls are not retried by default because a timeout may + happen after the tool already changed state. Set *retry* to ``True`` + only for a tool known to be safe to replay. """ kwargs: dict[str, Any] = {"json": {"arguments": arguments or {}}} if timeout is not None: kwargs["timeout"] = timeout - if not retry: - kwargs["extensions"] = {"asaree_no_retry": True} + if retry: + kwargs["extensions"] = {"asaree_allow_retry": True} data = self._client._post(f"/mcp-servers/{server_id}/tools/{tool_name}/call", **kwargs) return ToolCallResult(**data) diff --git a/sdk/tests/test_sdk_gaps.py b/sdk/tests/test_sdk_gaps.py new file mode 100644 index 00000000..159667d6 --- /dev/null +++ b/sdk/tests/test_sdk_gaps.py @@ -0,0 +1,220 @@ +"""Regression tests for parity gaps found in the SDK audit.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from asaree_client._multipart import multipart_directory +from asaree_client._transport import RetryPolicy, RetryTransport +from asaree_client.models import Agent +from asaree_client.resources.skills import Skills +from asaree_client.resources.tools import Tools + + +class SequenceTransport(httpx.BaseTransport): + def __init__(self, statuses: list[int]) -> None: + self.statuses = iter(statuses) + self.calls = 0 + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.calls += 1 + return httpx.Response(next(self.statuses), request=request) + + +class RecordingClient: + def __init__(self, response: Any) -> None: + self.response = response + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + def _get(self, path: str, **kwargs: Any) -> Any: + self.calls.append(("GET", path, kwargs)) + return self.response + + def _post(self, path: str, **kwargs: Any) -> Any: + self.calls.append(("POST", path, kwargs)) + return self.response + + def _patch(self, path: str, **kwargs: Any) -> Any: + self.calls.append(("PATCH", path, kwargs)) + return self.response + + +def _skill_response() -> dict[str, Any]: + now = datetime.now(tz=UTC).isoformat() + return { + "id": str(uuid.uuid4()), + "name": "audit", + "description": "Checks an experiment", + "body": "# Audit", + "frontmatter": {}, + "is_system": False, + "source_filename": None, + "files": [], + "created_at": now, + "updated_at": now, + } + + +def _mcp_response() -> dict[str, Any]: + return { + "id": str(uuid.uuid4()), + "name": "local", + "transport": "stdio", + "command": "server", + "url": None, + "status": "connected", + "error_message": None, + "capabilities": None, + "created_at": datetime.now(tz=UTC).isoformat(), + } + + +def test_agent_parses_current_response_fields() -> None: + now = datetime.now(tz=UTC) + source_plan_id = uuid.uuid4() + + agent = Agent.model_validate( + { + "id": uuid.uuid4(), + "name": "analyst", + "goal": "analyze", + "description": "", + "system_prompt": "Work carefully", + "model_config": {}, + "tool_config": {}, + "memory_config": {}, + "is_system": True, + "skill_config": {"skill_ids": []}, + "auto_eval_enabled": False, + "auto_eval_model": "gpt-5", + "source_plan_id": source_plan_id, + "source_plan_title": "Analysis plan", + "created_at": now, + "updated_at": now, + } + ) + + assert agent.is_system is True + assert agent.skill_config == {"skill_ids": []} + assert agent.auto_eval_enabled is False + assert agent.auto_eval_model == "gpt-5" + assert agent.source_plan_id == source_plan_id + assert agent.source_plan_title == "Analysis plan" + + +def test_post_is_not_retried_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + inner = SequenceTransport([500, 200]) + transport = RetryTransport(inner, RetryPolicy(max_retries=1, base_delay=0, jitter=False)) + monkeypatch.setattr("asaree_client._transport.time.sleep", lambda _seconds: None) + + response = transport.handle_request(httpx.Request("POST", "https://example.test/runs")) + + assert response.status_code == 500 + assert inner.calls == 1 + + +def test_get_retries_transient_status(monkeypatch: pytest.MonkeyPatch) -> None: + inner = SequenceTransport([500, 200]) + transport = RetryTransport(inner, RetryPolicy(max_retries=1, base_delay=0, jitter=False)) + monkeypatch.setattr("asaree_client._transport.time.sleep", lambda _seconds: None) + + response = transport.handle_request(httpx.Request("GET", "https://example.test/runs")) + + assert response.status_code == 200 + assert inner.calls == 2 + + +def test_post_can_explicitly_opt_in_to_retry(monkeypatch: pytest.MonkeyPatch) -> None: + inner = SequenceTransport([500, 200]) + transport = RetryTransport(inner, RetryPolicy(max_retries=1, base_delay=0, jitter=False)) + monkeypatch.setattr("asaree_client._transport.time.sleep", lambda _seconds: None) + request = httpx.Request( + "POST", + "https://example.test/tools/read-only/call", + extensions={"asaree_allow_retry": True}, + ) + + response = transport.handle_request(request) + + assert response.status_code == 200 + assert inner.calls == 2 + + +def test_skills_list_page_retains_total_and_list_remains_compatible() -> None: + response = {"items": [_skill_response()], "total": 7} + client = RecordingClient(response) + skills = Skills(client) + + page = skills.list_page(limit=25) + items = skills.list(limit=25) + + assert page.total == 7 + assert [skill.name for skill in page.items] == ["audit"] + assert [skill.name for skill in items] == ["audit"] + + +def test_create_skill_from_url_defaults_to_repository_root() -> None: + client = RecordingClient(_skill_response()) + + Skills(client).create_from_url("https://github.com/example/skills") + + assert client.calls == [ + ( + "POST", + "/skills/from-url", + {"json": {"url": "https://github.com/example/skills", "subdirectory": ""}}, + ) + ] + + +def test_mcp_update_omits_none_fields() -> None: + client = RecordingClient(_mcp_response()) + + Tools(client).update_server("server-id", name="renamed", command=None, headers=None) + + assert client.calls == [("PATCH", "/mcp-servers/server-id", {"json": {"name": "renamed"}})] + + +def test_tool_call_retry_is_explicit() -> None: + client = RecordingClient({"is_error": False, "content": "ok"}) + tools = Tools(client) + + tools.call_tool("server-id", "read_only") + tools.call_tool("server-id", "read_only", retry=True) + + assert client.calls == [ + ( + "POST", + "/mcp-servers/server-id/tools/read_only/call", + {"json": {"arguments": {}}}, + ), + ( + "POST", + "/mcp-servers/server-id/tools/read_only/call", + { + "json": {"arguments": {}}, + "extensions": {"asaree_allow_retry": True}, + }, + ), + ] + + +def test_directory_upload_preserves_relative_paths_and_closes_files(tmp_path: Path) -> None: + folder = tmp_path / "skill" + nested = folder / "references" + nested.mkdir(parents=True) + (folder / "SKILL.md").write_text("# Skill") + (nested / "notes.md").write_text("notes") + + with multipart_directory(str(folder)) as files: + assert [item[1][0] for item in files] == ["skill/SKILL.md", "skill/references/notes.md"] + handles = [item[1][1] for item in files] + assert all(not handle.closed for handle in handles) + + assert all(handle.closed for handle in handles) From c88afb86393aa384d76f24daf615f19882aaab31 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 16:01:28 -0700 Subject: [PATCH 04/18] feat(experiments): provision canvas on creation --- frontend/src/pages/ProtocolCanvasPage.tsx | 2 + sdk/README.md | 4 ++ .../asaree_client/resources/experiments.py | 1 + src/asaree/api/experiments.py | 9 +++ src/asaree/models/protocol.py | 10 +-- tests/test_experiment_creation_contract.py | 61 +++++++++++++++++++ tests/test_experiments.py | 7 +++ 7 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/test_experiment_creation_contract.py diff --git a/frontend/src/pages/ProtocolCanvasPage.tsx b/frontend/src/pages/ProtocolCanvasPage.tsx index 9d035ee6..bf60ac59 100644 --- a/frontend/src/pages/ProtocolCanvasPage.tsx +++ b/frontend/src/pages/ProtocolCanvasPage.tsx @@ -239,6 +239,8 @@ export function ProtocolCanvasPage() { queryFn: async () => { const existing = await protocolsApi.list(experimentId!) if (existing.length > 0) return existing[0] + // POST /experiments now creates this shell transactionally. Keep this + // fallback for experiments created before that behavior shipped. const experiment = await experimentsApi.get(experimentId!) // See generatedProtocolName for why the name is suffixed with the // experiment's shortid. Renaming the experiment later re-syncs this diff --git a/sdk/README.md b/sdk/README.md index 007e42f2..47e2bc5d 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -44,6 +44,10 @@ experiment = client.experiments.create(name="tier-x-effort", factors=[ {"name": "tier", "levels": ["baseline", "critic"]}, {"name": "effort", "levels": ["low", "high"]}, ]) +# Creation also provisions the experiment's empty protocol canvas. Retrieve it +# to build the graph programmatically; the same canvas is immediately visible +# when this user opens the experiment in the GUI. +protocol = client.protocols.list(experiment_id=experiment.id)[0] replicates = client.experiments.generate_design(experiment.id) for replicate in replicates: diff --git a/sdk/src/asaree_client/resources/experiments.py b/sdk/src/asaree_client/resources/experiments.py index 522fbb39..d0ef79a7 100644 --- a/sdk/src/asaree_client/resources/experiments.py +++ b/sdk/src/asaree_client/resources/experiments.py @@ -44,6 +44,7 @@ def create( factors: builtins.list[dict[str, Any]] | None = None, measurement_plan: dict[str, Any] | None = None, ) -> Experiment: + """Create an experiment and its linked empty protocol canvas atomically.""" payload: dict[str, Any] = {"design_type": design_type} if name is not None: payload["name"] = name diff --git a/src/asaree/api/experiments.py b/src/asaree/api/experiments.py index dd9ce16e..43fb44a9 100644 --- a/src/asaree/api/experiments.py +++ b/src/asaree/api/experiments.py @@ -436,6 +436,15 @@ async def create_experiment_endpoint( owner_id=experiment.owner_id, allow_preserved_bindings=False, ) + # An experiment's primary GUI is its protocol canvas. Create that durable + # shell in the same transaction so API/SDK-created experiments do not + # depend on somebody opening the GUI before their canvas exists. + await create_protocol( + db, + name=generated_protocol_name(experiment.name, experiment.id), + owner_id=experiment.owner_id, + experiment_id=experiment.id, + ) return _experiment_response(experiment, await get_experiment_dataset_ids(db, experiment.id)) diff --git a/src/asaree/models/protocol.py b/src/asaree/models/protocol.py index b73b3398..7ce2f0b4 100644 --- a/src/asaree/models/protocol.py +++ b/src/asaree/models/protocol.py @@ -54,11 +54,11 @@ class Protocol(Base, TimestampMixin): ) # Nullable, SET NULL on delete -- a protocol is a standalone reusable # object that MAY be tagged to the experiment it was built for; losing - # that experiment isn't a reason to lose the protocol. Which experiment - # a protocol belongs to (if any) is a UX convention the canvas page - # enforces (one protocol per experiment, created lazily on first visit), - # not a schema constraint -- reusing one protocol across experiments - # later needs no migration. + # that experiment isn't a reason to lose the protocol. New experiments + # receive one protocol transactionally at the API boundary; the canvas + # still repairs legacy experiments that predate that behavior. This is a + # UX invariant rather than a schema constraint, so standalone protocols + # and future protocol reuse need no migration. experiment_id: Mapped[uuid.UUID | None] = mapped_column( UUID(as_uuid=True), ForeignKey("research_experiments.id", ondelete="SET NULL"), nullable=True, index=True ) diff --git a/tests/test_experiment_creation_contract.py b/tests/test_experiment_creation_contract.py new file mode 100644 index 00000000..d7ff2c58 --- /dev/null +++ b/tests/test_experiment_creation_contract.py @@ -0,0 +1,61 @@ +"""Fast contract coverage for experiment creation side effects.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import AsyncMock + +import pytest + +from asaree.api import experiments as experiments_api +from asaree.services.protocols import generated_protocol_name + + +async def test_create_endpoint_provisions_linked_empty_canvas(monkeypatch: pytest.MonkeyPatch) -> None: + owner_id = uuid.uuid4() + experiment_id = uuid.uuid4() + now = datetime.now(UTC) + experiment = SimpleNamespace( + id=experiment_id, + name="SDK experiment", + description=None, + hypothesis=None, + design_type="factorial", + task_brief=None, + design_spec=None, + measurement_plan=None, + metric_recommendations=None, + latest_test_run_id=None, + archived_at=None, + locked_at=None, + locked_protocol_revision_id=None, + locked_design_spec=None, + locked_measurement_plan=None, + owner_id=owner_id, + created_at=now, + updated_at=now, + ) + create_protocol = AsyncMock() + monkeypatch.setattr(experiments_api, "get_experiment_by_name", AsyncMock(return_value=None)) + monkeypatch.setattr(experiments_api, "_validated_dataset_ids", AsyncMock(return_value=[])) + monkeypatch.setattr(experiments_api, "create_experiment", AsyncMock(return_value=experiment)) + monkeypatch.setattr(experiments_api, "create_protocol", create_protocol) + monkeypatch.setattr(experiments_api, "get_experiment_dataset_ids", AsyncMock(return_value=[])) + db = object() + + response = await experiments_api.create_experiment_endpoint( + experiments_api.CreateExperimentRequest(name="SDK experiment"), + cast(Any, SimpleNamespace(id=owner_id)), + cast(Any, db), + ) + + assert response.id == experiment_id + create_protocol.assert_awaited_once_with( + db, + name=generated_protocol_name(experiment.name, experiment_id), + owner_id=owner_id, + experiment_id=experiment_id, + ) diff --git a/tests/test_experiments.py b/tests/test_experiments.py index ec9bfef7..9fc68ef3 100644 --- a/tests/test_experiments.py +++ b/tests/test_experiments.py @@ -29,6 +29,7 @@ set_experiment_datasets, update_experiment, ) +from asaree.services.protocols import delete_protocol, generated_protocol_name, list_protocols @pytest_asyncio.fixture(autouse=True) @@ -51,6 +52,8 @@ async def owner_id() -> AsyncIterator[uuid.UUID]: uid = user.id yield uid async with get_session() as db: + for protocol in await list_protocols(db, owner_id=uid): + await delete_protocol(db, protocol.id) db_user = await db.get(User, uid) if db_user is not None: await db.delete(db_user) @@ -216,6 +219,10 @@ async def test_create_experiment_returns_a_persisted_recommended_measurement_pla "dismissed_version": None, "intentionally_removed_keys": [], } + protocols = await list_protocols(db, owner_id=owner_id, experiment_id=experiment_id) + assert len(protocols) == 1 + assert protocols[0].name == generated_protocol_name(response.name, experiment_id) + assert protocols[0].graph == {"nodes": [], "edges": []} finally: async with get_session() as db: persisted = await get_experiment(db, experiment_id) From b314f2ac5249764748a11b297ea35ef44f31755c Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 16:11:21 -0700 Subject: [PATCH 05/18] Keep factor baselines aligned with canvas values --- .../src/components/protocol/DesignTab.tsx | 3 + .../protocol/FactorBindableField.tsx | 6 +- .../protocol/FactorEditorDialog.tsx | 9 +- .../components/protocol/ProtocolCanvas.tsx | 51 +++++++++- frontend/src/lib/factorBindings.test.ts | 48 +++++++++- frontend/src/lib/factorBindings.ts | 92 ++++++++++++++++++- src/asaree/services/factor_bindings.py | 7 +- tests/test_factor_bindings.py | 22 ++++- 8 files changed, 225 insertions(+), 13 deletions(-) diff --git a/frontend/src/components/protocol/DesignTab.tsx b/frontend/src/components/protocol/DesignTab.tsx index 2a3f56b5..93c7e9b9 100644 --- a/frontend/src/components/protocol/DesignTab.tsx +++ b/frontend/src/components/protocol/DesignTab.tsx @@ -260,6 +260,9 @@ function FactorsEditor({ return { oldName, next } }, onSuccess: ({ oldName, next }) => { + if (next.level_type !== 'boolean' && next.levels.length > 0) { + canvasRef.current?.setFactorBaseline(oldName, next.levels[0]) + } if (next.name !== oldName) canvasRef.current?.renameFactorBindings(oldName, next.name) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id] }) queryClient.invalidateQueries({ queryKey: ['experiments', experiment.id, 'design-impact'] }) diff --git a/frontend/src/components/protocol/FactorBindableField.tsx b/frontend/src/components/protocol/FactorBindableField.tsx index 11dfa65b..863fde49 100644 --- a/frontend/src/components/protocol/FactorBindableField.tsx +++ b/frontend/src/components/protocol/FactorBindableField.tsx @@ -264,6 +264,7 @@ export function FactorBindableField({ {levels.map((level, i) => (
+ {i === 0 && Canvas baseline} {levelOptions && levelOptions.length > 0 ? ( { setLevels((ls) => ls.filter((_, j) => j !== i)) setLevelLabels((ls) => ls.filter((_, j) => j !== i)) @@ -807,6 +810,7 @@ export function FactorEditorDialog({ {levels.map((level, i) => (
+ {i === 0 && Canvas baseline} { setLevels((ls) => ls.filter((_, j) => j !== i)) setLevelLabels((ls) => ls.filter((_, j) => j !== i)) @@ -883,6 +888,7 @@ export function FactorEditorDialog({
{(levels as string[]).map((level, i) => (
+ {i === 0 && Canvas baseline} { setLevels((ls) => ls.filter((_, j) => j !== i)) setLevelLabels((ls) => ls.filter((_, j) => j !== i)) diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 9b27f014..41db3cee 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -21,6 +21,7 @@ import { Lock, Play, Plus, Square, X } from 'lucide-react' import { Button } from '@/components/ui/button' import { ApiError, experimentsApi, protocolsApi } from '@/api/client' import { CONNECTOR_HANDLES } from '@/lib/coordinationStrategy' +import { graphWithFactorBaseline, reconcileFactorBaselines } from '@/lib/factorBindings' import { newNodeId } from '@/lib/nodeId' import { handoffPeers, promptReferenceScope } from '@/lib/promptReferences' import { mergeProtocolSaveIntoCache, protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' @@ -385,6 +386,10 @@ export interface ProtocolCanvasHandle { // `newName` instead of being left referencing a name that no longer // resolves. renameFactorBindings: (oldName: string, newName: string) => void + // A non-boolean factor's first level is its canvas baseline. Editing that + // level from the sibling Design tab must update every field controlled by + // the factor, not leave the graph and design as two sources of truth. + setFactorBaseline: (factorName: string, baseline: unknown) => void } export const ProtocolCanvas = forwardRef { + setNodes((nds) => graphWithFactorBaseline( + { nodes: nds as unknown as ProtocolNode[], edges: [] }, + factorName, + baseline, + ).nodes as unknown as Node[]) + }, + [setNodes], + ) useImperativeHandle( canvasHandleRef, @@ -462,8 +477,9 @@ export const ProtocolCanvas = forwardRef(null) useEffect(() => { @@ -1402,6 +1418,9 @@ export const ProtocolCanvas = forwardRef { + if (next.level_type !== 'boolean' && next.levels.length > 0) { + setFactorBaseline(oldName, next.levels[0]) + } if (next.name !== oldName) renameFactorBindings(oldName, next.name) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) @@ -1793,6 +1812,36 @@ export const ProtocolCanvas = forwardRef { + if (!experimentId || !experimentQuery.data || experimentLocked || experimentQuery.data.locked_at) return + const graph = toPersistedGraph(nodes, edges) + const reconciled = reconcileFactorBaselines(experimentQuery.data.design_spec, graph) + if (JSON.stringify(reconciled) === JSON.stringify(factors)) return + + const seq = ++factorBaselineSyncSeqRef.current + const timer = setTimeout(() => { + void experimentsApi.get(experimentId).then(async (fresh) => { + if (seq !== factorBaselineSyncSeqRef.current) return + const nextFactors = reconcileFactorBaselines(fresh.design_spec, graph) + if (JSON.stringify(nextFactors) === JSON.stringify(fresh.design_spec?.factors ?? [])) return + await experimentsApi.update(experimentId, { + design_spec: { ...fresh.design_spec, factors: nextFactors }, + }) + if (seq !== factorBaselineSyncSeqRef.current) return + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId] }) + queryClient.invalidateQueries({ queryKey: ['experiments', experimentId, 'design-impact'] }) + }).catch(() => { + // Best-effort reconciliation. The publish/run validator remains the + // hard guard, and the next canvas change retries from fresh data. + }) + }, AUTOSAVE_DELAY_MS) + return () => clearTimeout(timer) + }, [edges, experimentId, experimentLocked, experimentQuery.data, factors, nodes, queryClient]) const lastSyncedDatasetIdsRef = useRef(JSON.stringify(datasetIdsInGraph(initialGraph.nodes as Node[]))) useEffect(() => { if (!experimentId) return diff --git a/frontend/src/lib/factorBindings.test.ts b/frontend/src/lib/factorBindings.test.ts index 65f33c79..1c5bc31e 100644 --- a/frontend/src/lib/factorBindings.test.ts +++ b/frontend/src/lib/factorBindings.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { factorBindingDiscrepancies } from './factorBindings' +import { factorBindingDiscrepancies, graphWithFactorBaseline, reconcileFactorBaselines } from './factorBindings' import type { ProtocolGraph } from '@/types/protocols' describe('factorBindingDiscrepancies', () => { @@ -26,7 +26,51 @@ describe('factorBindingDiscrepancies', () => { nodeLabel: 'Writer', fieldPath: 'config.system_prompt', factorName: 'Prompt', - reason: 'published value does not match any declared level', + reason: 'published value does not match the first (canvas baseline) level', }]) }) + + it('keeps the current canvas value first while preserving alternate levels', () => { + const graph = { + nodes: [{ + id: 'agent-1', + type: 'agent', + position: { x: 0, y: 0 }, + data: { + label: 'Writer', + factor_bindings: { 'config.system_prompt': 'Prompt' }, + config: { system_prompt: 'edited canvas prompt' }, + }, + }], + edges: [], + } as unknown as ProtocolGraph + + expect(reconcileFactorBaselines({ + factors: [{ + name: 'Prompt', + levels: ['old baseline', 'alternate'], + level_labels: ['Original', 'Alternative'], + }], + }, graph)).toEqual([{ + name: 'Prompt', + levels: ['edited canvas prompt', 'old baseline', 'alternate'], + level_labels: ['Baseline', 'Original', 'Alternative'], + }]) + }) + + it('mirrors a changed baseline into every field sharing the factor', () => { + const graph = { + nodes: [ + { id: 'agent-1', type: 'agent', position: { x: 0, y: 0 }, data: { factor_bindings: { 'config.prompt': 'Prompt' }, config: { prompt: 'old' } } }, + { id: 'agent-2', type: 'agent', position: { x: 0, y: 0 }, data: { factor_bindings: { 'config.prompt': 'Prompt' }, config: { prompt: 'old' } } }, + ], + edges: [], + } as unknown as ProtocolGraph + + const updated = graphWithFactorBaseline(graph, 'Prompt', 'new baseline') + expect(updated.nodes.map((node) => (node.data.config as { prompt: string }).prompt)).toEqual([ + 'new baseline', + 'new baseline', + ]) + }) }) diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index 7bc95485..bb0bd85b 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -1,4 +1,4 @@ -import type { DesignSpec } from '@/types/experiments' +import type { DesignFactor, DesignSpec } from '@/types/experiments' import type { ProtocolGraph } from '@/types/protocols' export interface FactorBindingDiscrepancy { @@ -6,12 +6,12 @@ export interface FactorBindingDiscrepancy { nodeLabel: string fieldPath: string factorName: string - reason: 'factor is no longer declared' | 'field no longer exists on the canvas' | 'published value does not match any declared level' + reason: 'factor is no longer declared' | 'field no longer exists on the canvas' | 'published value does not match the first (canvas baseline) level' } const MISSING = Symbol('missing factor-bound field') -function pathValue(root: Record, dottedPath: string): unknown | typeof MISSING { +export function pathValue(root: Record, dottedPath: string): unknown | typeof MISSING { let value: unknown = root for (const part of dottedPath.split('.')) { if (typeof value !== 'object' || value === null || !(part in value)) return MISSING @@ -24,6 +24,88 @@ function valuesEqual(left: unknown, right: unknown): boolean { return JSON.stringify(left) === JSON.stringify(right) } +function isBooleanFactor(factor: DesignFactor): boolean { + return factor.level_type === 'boolean' || ( + factor.level_type === undefined && + factor.levels.length === 2 && + factor.levels.every((level) => typeof level === 'boolean') + ) +} + +function withPathValue(root: Record, dottedPath: string, nextValue: unknown): Record { + const parts = dottedPath.split('.') + const next = { ...root } + let target = next + let source: Record = root + for (const part of parts.slice(0, -1)) { + const sourceChild = source[part] + const nextChild = typeof sourceChild === 'object' && sourceChild !== null + ? { ...(sourceChild as Record) } + : {} + target[part] = nextChild + target = nextChild + source = typeof sourceChild === 'object' && sourceChild !== null + ? sourceChild as Record + : {} + } + target[parts.at(-1)!] = nextValue + return next +} + +/** Keep a factor's current canvas value in the baseline (first) slot. */ +export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { + if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor + + const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) + const keptIndexes = factor.levels.flatMap((_, index) => index === currentIndex ? [] : [index]) + const levels = [currentValue, ...keptIndexes.map((index) => factor.levels[index])] + const labels = factor.level_labels + ? [currentIndex >= 0 ? factor.level_labels[currentIndex] ?? 'Baseline' : 'Baseline', ...keptIndexes.map((index) => factor.level_labels?.[index] ?? '')] + : undefined + return { ...factor, levels, ...(labels ? { level_labels: labels } : {}) } +} + +/** + * Reconcile stored factor baselines from the live canvas. A shared factor is + * only reconciled when every bound field currently agrees; otherwise the + * validator reports the ambiguity and the user must split the factor. + */ +export function reconcileFactorBaselines(designSpec: DesignSpec | null | undefined, graph: ProtocolGraph | undefined): DesignFactor[] { + const factors = designSpec?.factors ?? [] + const valuesByFactor = new Map() + for (const node of graph?.nodes ?? []) { + const data = node.data as unknown as Record + for (const [fieldPath, rawFactorName] of Object.entries((data.factor_bindings ?? {}) as Record)) { + const value = pathValue(data, fieldPath) + if (value === MISSING) continue + const name = rawFactorName.trim() + valuesByFactor.set(name, [...(valuesByFactor.get(name) ?? []), value]) + } + } + + return factors.map((factor) => { + const values = valuesByFactor.get(factor.name.trim()) ?? [] + if (values.length === 0 || values.some((value) => !valuesEqual(value, values[0]))) return factor + return factorWithCanvasBaseline(factor, values[0]) + }) +} + +/** Mirror a changed factor baseline back into every field bound to it. */ +export function graphWithFactorBaseline(graph: ProtocolGraph, factorName: string, baseline: unknown): ProtocolGraph { + return { + ...graph, + nodes: graph.nodes.map((node) => { + const data = node.data as unknown as Record + const paths = Object.entries((data.factor_bindings ?? {}) as Record) + .filter(([, name]) => name === factorName) + .map(([path]) => path) + if (paths.length === 0) return node + const nextData = paths.reduce((next, path) => withPathValue(next, path, baseline), data) + return { ...node, data: nextData as typeof node.data } + }), + } +} + /** Declared factors without a live canvas field to vary are not runnable. */ export function unboundFactorNames(designSpec: DesignSpec | null, graph: ProtocolGraph | undefined): string[] { const declared = designSpec?.factors?.map((factor) => factor.name.trim()).filter(Boolean) ?? [] @@ -53,9 +135,9 @@ export function factorBindingDiscrepancies( ? 'factor is no longer declared' : value === MISSING ? 'field no longer exists on the canvas' - : factor.levels.some((level) => valuesEqual(value, level)) + : isBooleanFactor(factor) || valuesEqual(value, factor.levels[0]) ? null - : 'published value does not match any declared level' + : 'published value does not match the first (canvas baseline) level' if (reason) discrepancies.push({ nodeId: node.id, nodeLabel, fieldPath, factorName, reason }) } } diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index 702c75c0..9575ad9a 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -55,10 +55,13 @@ def validate_factor_bindings(design_spec: dict[str, Any] | None, graph: dict[str "published canvas. Remove the stale binding or restore the field." ) levels = factor.get("levels") or [] - if not any(published_value == level for level in levels): + level_type = factor.get("level_type") or "string" + legacy_boolean = "level_type" not in factor and levels == [True, False] + if level_type != "boolean" and not legacy_boolean and (not levels or published_value != levels[0]): raise ValueError( f"{label!r} field {field_path!r} is bound to factor {factor_name!r}, but its published value " - "does not match any declared level. Update the factor levels and regenerate the design, or " + "does not match the first (canvas baseline) level. Update the factor levels and regenerate " + "the design, or " "remove the binding before publishing or running." ) diff --git a/tests/test_factor_bindings.py b/tests/test_factor_bindings.py index ddc9c40c..66110a21 100644 --- a/tests/test_factor_bindings.py +++ b/tests/test_factor_bindings.py @@ -37,7 +37,27 @@ def test_changed_published_value_must_be_reflected_in_bound_factor_levels() -> N "edges": [], } - with pytest.raises(ValueError, match="published value does not match any declared level"): + with pytest.raises(ValueError, match=r"published value does not match the first \(canvas baseline\) level"): + validate_factor_bindings(design_spec, graph) + + +def test_bound_canvas_value_must_be_the_first_level() -> None: + design_spec = {"factors": [{"name": "Prompt", "levels": ["baseline", "alternate"]}]} + graph = { + "nodes": [ + { + "id": "agent", + "data": { + "label": "Writer", + "config": {"system_prompt": "alternate"}, + "factor_bindings": {"config.system_prompt": "Prompt"}, + }, + } + ], + "edges": [], + } + + with pytest.raises(ValueError, match="canvas baseline"): validate_factor_bindings(design_spec, graph) From e1d7731c02eb38f856d57266b36f6879515cf437 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 16:18:39 -0700 Subject: [PATCH 06/18] Preserve payload-only protocol handoffs --- src/asaree/services/agent_messenger.py | 4 +- src/asaree/services/protocol_execution.py | 55 +++++++++++++++++---- tests/test_prompt_assembly.py | 18 ++++++- tests/test_protocol_execution.py | 60 +++++++++++++++++++++++ tests/test_supervisor_architecture.py | 22 +++++++-- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/src/asaree/services/agent_messenger.py b/src/asaree/services/agent_messenger.py index 315d7670..295e92d9 100644 --- a/src/asaree/services/agent_messenger.py +++ b/src/asaree/services/agent_messenger.py @@ -56,6 +56,7 @@ _build_user_input, _can_deliver_communication, _node_run_context, + _output_content, _run_agent_node, resolve_available_agents, ) @@ -753,7 +754,8 @@ def _supervisor_report(display_name: str, run: dict[str, Any]) -> str: """ status = str(run.get("status") or "skipped") if status == "completed": - return f"--- {display_name} ---\n{str(run.get('output_text') or '').strip() or '(no output)'}" + content = _output_content(str(run.get("output_text") or ""), run.get("payload") or {}).strip() + return f"--- {display_name} ---\n{content or '(no output)'}" if status == "cancelled": return f"--- {display_name} ---\n(this agent's turn was cancelled and produced nothing)" if status == "failed": diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index f9126d70..5c6697ee 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -2781,10 +2781,11 @@ def _upstream_context( for uid in ids: run = node_runs.get(uid) or {} text = run.get("output_text") - if not text: + payload = run.get("payload") or {} + if not text and not payload: continue label = f"{names[uid]} ({uid})" if names[uid] in ambiguous else names[uid] - blocks.append(f"[{label}]\n{_sender_block(str(text), run.get('payload') or {})}") + blocks.append(f"[{label}]\n{_sender_block(str(text or ''), payload)}") return "\n\n".join(blocks) @@ -2813,9 +2814,21 @@ def _sender_block(text: str, payload: dict[str, Any], *, raw: bool = False) -> s experimenter who references a predecessor by hand gets exactly what would have arrived anyway, only where they put it. """ - body = _reference_payload(text, raw=raw) + body = _reference_payload(text, raw=raw) if text else "" fields = _payload_fields_line(payload) - return f"{body}\n{fields}" if fields else body + return "\n".join(part for part in (body, fields) if part) + + +def _output_content(text: str, payload: dict[str, Any]) -> str: + """A run's complete handoff without sender framing. + + Critic and revision prompts provide their own framing, so they need the + same prose-plus-fields content as :func:`_sender_block` without adding an + inner fence. Keeping payload rendering here also makes payload-only output + a real handoff instead of an empty string. + """ + fields = _payload_fields_line(payload) + return "\n".join(part for part in (text, fields) if part) def _format_payload_value(value: Any) -> str: @@ -2897,10 +2910,11 @@ def _render_reference( for uid in ids: run = node_runs.get(uid) or {} text = run.get("output_text") - if not text: + payload = run.get("payload") or {} + if not text and not payload: unresolved.append(uid) continue - block = _sender_block(str(text), run.get("payload") or {}, raw=ref.raw) + block = _sender_block(str(text or ""), payload, raw=ref.raw) # Labelled only where the reference itself cannot say which sender is # which: a `{{previous}}` that expanded to several predecessors. A # single-node reference needs no label, because the experimenter named @@ -4018,6 +4032,19 @@ def _completed_worker_record( } +def _handoff_payload(extraction: dict[str, Any] | None) -> dict[str, Any]: + """The typed portion of a worker output that a transparent gate forwards. + + Caveats and truncation describe the worker run itself, so they stay on the + worker record. The payload is output data and must remain available to the + node after the gate even when Motoro intentionally left ``output_text`` + empty. + """ + if extraction is None or "payload" not in extraction: + return {} + return {"payload": extraction["payload"]} + + async def _run_gated_worker( worker: dict[str, Any], gate: dict[str, Any], @@ -4114,11 +4141,19 @@ async def _run_gated_worker( {"status": "skipped"}, ) assert output_text is not None, "_run_agent_node guarantees output_text when error is falsy" + handoff_payload = _handoff_payload(extraction) + complete_output = _output_content(output_text, handoff_payload.get("payload") or {}) if not enabled: return ( _completed_worker_record(output_text, attempt, run_id_str, extraction), - {"status": "completed", "output_text": output_text, "approved": None, "revisions_used": 0}, + { + "status": "completed", + "output_text": output_text, + "approved": None, + "revisions_used": 0, + **handoff_payload, + }, ) if attempt == max_revisions: @@ -4135,6 +4170,7 @@ async def _run_gated_worker( "feedback": last_verdict.get("feedback") if last_verdict else None, "rejection_scope": last_verdict.get("rejection_scope") if last_verdict else None, "run_id": last_critic_run_id, + **handoff_payload, }, ) @@ -4143,7 +4179,7 @@ async def _run_gated_worker( protocol_id=protocol_id, protocol_run_id=protocol_run_id, owner_id=owner_id, - worker_output=output_text, + worker_output=complete_output, graph=graph, ) if verdict_error == _AGENT_CANCELLED: @@ -4181,12 +4217,13 @@ async def _run_gated_worker( "feedback": verdict.get("feedback"), "rejection_scope": None, "run_id": critic_run_id, + **handoff_payload, }, ) last_verdict = verdict last_critic_run_id = critic_run_id - instruction = _build_revision_instruction(base_instruction, verdict, output_text) + instruction = _build_revision_instruction(base_instruction, verdict, complete_output) raise AssertionError("_run_gated_worker fell through its attempt loop") # unreachable diff --git a/tests/test_prompt_assembly.py b/tests/test_prompt_assembly.py index 6f320060..f42271d7 100644 --- a/tests/test_prompt_assembly.py +++ b/tests/test_prompt_assembly.py @@ -66,6 +66,16 @@ def test_a_direct_predecessors_output_arrives_because_the_edge_exists() -> None: assert text == f"Do the thing.\n\n[Analyst]\n{_fenced('Findings.')}" +def test_a_payload_only_predecessor_still_arrives_because_the_edge_exists() -> None: + graph = _two_step() + text = _prompt( + graph, + "b", + {"a": {"status": "completed", "output_text": "", "payload": {"n_rows": 4300}}}, + ) + assert text == "Do the thing.\n\n[Analyst]\nStructured fields: n_rows=4300" + + def test_nothing_the_platform_wrote_travels_with_the_output() -> None: """A guard on the prose that used to be here, by the phrases it used. @@ -722,6 +732,13 @@ def test_a_whole_node_reference_appends_the_extracted_fields() -> None: assert "4300 rows." in text +def test_a_payload_only_whole_node_reference_resolves_to_its_fields() -> None: + graph = _two_step() + graph["nodes"][1]["data"]["config"]["prompt"] = "{{node:a}}" + text = _prompt(graph, "b", {"a": _ran("", {"n_rows": 4300})}) + assert text == "Structured fields: n_rows=4300" + + def test_the_appended_fields_carry_no_frame_and_no_sender_name() -> None: """It is a continuation of the block it follows, which already carries both -- a second frame would read as a second sender.""" @@ -754,4 +771,3 @@ def test_previous_appends_each_senders_own_fields() -> None: assert "Structured fields: n_rows=4300" in text assert "Structured fields: auc=0.81" in text - diff --git a/tests/test_protocol_execution.py b/tests/test_protocol_execution.py index a15afa62..23695611 100644 --- a/tests/test_protocol_execution.py +++ b/tests/test_protocol_execution.py @@ -1225,6 +1225,46 @@ async def fake_run_critic(gate, **kwargs): assert critic_calls == ["worker output v1"] +async def test_gated_worker_sends_and_forwards_payload_only_output(monkeypatch: pytest.MonkeyPatch) -> None: + critic_calls = [] + + async def fake_run_agent_node(node, **kwargs): + return "", None, None, {"payload": {"score": 0.91}} + + async def fake_run_critic(gate, **kwargs): + critic_calls.append(kwargs["worker_output"]) + return {"approved": True, "feedback": "", "rejection_scope": ""}, None, "critic-run-1" + + monkeypatch.setattr(pe, "_run_agent_node", fake_run_agent_node) + monkeypatch.setattr(pe, "_run_critic", fake_run_critic) + + worker_run, gate_run = await _run(*_worker_gate()) + + assert critic_calls == ["Structured fields: score=0.91"] + assert worker_run["output_text"] == "" + assert worker_run["payload"] == {"score": 0.91} + assert gate_run["output_text"] == "" + assert gate_run["payload"] == {"score": 0.91} + + +@pytest.mark.parametrize(("max_revisions", "enabled"), [(0, True), (1, False)]) +async def test_gated_worker_forwards_payload_when_review_is_skipped( + monkeypatch: pytest.MonkeyPatch, max_revisions: int, enabled: bool +) -> None: + async def fake_run_agent_node(node, **kwargs): + return "", None, None, {"payload": {"score": 0.91}} + + async def unexpected_critic_call(gate, **kwargs): + raise AssertionError("the critic should not run") + + monkeypatch.setattr(pe, "_run_agent_node", fake_run_agent_node) + monkeypatch.setattr(pe, "_run_critic", unexpected_critic_call) + + _worker_run, gate_run = await _run(*_worker_gate(max_revisions=max_revisions, enabled=enabled)) + + assert gate_run["payload"] == {"score": 0.91} + + async def test_gated_worker_rejected_then_approved_on_revision(monkeypatch: pytest.MonkeyPatch) -> None: instructions = [] critic_calls = [] @@ -1268,6 +1308,26 @@ async def fake_run_critic(gate, *, worker_output, **_kwargs): assert "worker output v1" in instructions[1] # previous output included for reference +async def test_gated_worker_revision_includes_payload_only_previous_output(monkeypatch: pytest.MonkeyPatch) -> None: + instructions = [] + + async def fake_run_agent_node(node, *, user_input, **_kwargs): + instructions.append(user_input) + return "", None, None, {"payload": {"score": len(instructions)}} + + async def fake_run_critic(gate, **_kwargs): + if len(instructions) == 1: + return {"approved": False, "feedback": "raise the score", "rejection_scope": "partial"}, None, "critic-1" + return {"approved": True, "feedback": "", "rejection_scope": ""}, None, "critic-2" + + monkeypatch.setattr(pe, "_run_agent_node", fake_run_agent_node) + monkeypatch.setattr(pe, "_run_critic", fake_run_critic) + + await _run(*_worker_gate(max_revisions=2)) + + assert "Structured fields: score=1" in instructions[1] + + async def test_gated_worker_force_accepts_without_final_critic_call(monkeypatch: pytest.MonkeyPatch) -> None: attempts = [] critic_calls = [] diff --git a/tests/test_supervisor_architecture.py b/tests/test_supervisor_architecture.py index b9190cde..7bab0696 100644 --- a/tests/test_supervisor_architecture.py +++ b/tests/test_supervisor_architecture.py @@ -60,9 +60,9 @@ def stubs(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: "checkpoints": [], "node_runs": [], "cancel_requested_at": None, - # node_id -> (output_text, error). A node id absent from this answers - # with a generic completion; a callable is invoked with the - # _run_agent_node kwargs, which is how a test asserts mid-run. + # node_id -> (output_text, error[, extraction]). A node id absent from + # this answers with a generic completion; a callable is invoked with + # the _run_agent_node kwargs, which is how a test asserts mid-run. "answers": {}, "turns": [], "run_contexts": [], @@ -98,10 +98,12 @@ async def _run_agent_node( # indistinguishable from serial because nothing ever yields. await asyncio.sleep(0) answer = state["answers"].get(node_id, (f"{node_id} did its part.", None)) - output, error = answer(kwargs) if callable(answer) else answer + resolved = answer(kwargs) if callable(answer) else answer + output, error, *fields = resolved + extraction = fields[0] if fields else None finally: state["concurrent"] -= 1 - return output, error, uuid.uuid4() if error is None else None, None + return output, error, uuid.uuid4() if error is None else None, extraction async def _node_run_context(*_args: Any, **kwargs: Any) -> tuple[dict[str, Any], Any]: state["run_contexts"].append(kwargs) @@ -179,6 +181,16 @@ async def test_the_brief_reaches_every_worker(stubs: dict[str, Any]) -> None: assert "Split the cohort three ways." in _prompt_for(stubs, worker) +async def test_payload_only_turns_reach_supervisor_review_and_synthesis(stubs: dict[str, Any]) -> None: + stubs["answers"]["w1"] = ("", None, {"payload": {"finding": "cohort imbalance"}}) + + await _run(stubs) + + expected = 'Structured fields: finding="cohort imbalance"' + assert expected in _prompt_for(stubs, "qc") + assert expected in _prompt_for(stubs, "sup", occurrence=1) + + async def test_each_agent_is_told_its_role(stubs: dict[str, Any]) -> None: """The stages are forced, so the prompts have to say so -- a supervisor left to infer that its first turn is a brief writes the answer itself instead.""" From f27e924fd58ada41798ea6eab5ef1372eb5215d7 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 16:52:27 -0700 Subject: [PATCH 07/18] Update publication artifacts for v0.8.0 --- README.md | 4 +- .../src/components/protocol/bindableFields.ts | 2 +- .../{bioinformatics => BDM}/README.md | 100 ++++--- .../{bioinformatics => BDM}/dict_ZSN.json | 0 .../import_use_case.py | 137 ++++++---- .../{bioinformatics => BDM}/mi_ZSN.csv | 0 .../myocardial-anthropic-latest.json | 80 ++++-- .../myocardial-anthropic-v0.2.0.json | 0 .../myocardial-azure-foundry-latest.json | 80 ++++-- .../myocardial-azure-foundry-v0.2.0.json | 0 .../myocardial-openai-latest.json | 80 ++++-- .../myocardial-openai-v0.2.0.json | 0 .../{bioinformatics => BDM}/stats/REPORT.html | 0 .../{bioinformatics => BDM}/stats/analysis.py | 0 .../stats/analysis_log.txt | 0 .../stats/consolidate.py | 0 .../stats/figure_main.py | 0 .../{bioinformatics => BDM}/stats/figures.py | 0 .../stats/results.json | 0 .../stats/tables/all_effects.csv | 0 .../stats/tables/all_kruskal.csv | 0 .../stats/tables/all_pairwise.csv | 0 .../stats/tables/all_terms_multiplicity.csv | 0 .../stats/tables/all_terms_multiplicity.md | 0 .../stats/tables/analysis_frame.csv | 0 .../binomial_sensitivity_engineered_share.csv | 0 .../stats/tables/bootstrap_pr_auc.csv | 0 .../stats/tables/configuration_summary.csv | 0 .../stats/tables/cost_decomposition.csv | 0 .../stats/tables/effects_inner_cv_score.csv | 0 .../tables/effects_n_engineered_features.csv | 0 ...effects_n_engineered_features_selected.csv | 0 .../tables/effects_n_features_after_fs.csv | 0 .../stats/tables/effects_n_turns_total.csv | 0 .../stats/tables/effects_pr_auc.csv | 0 .../effects_prop_engineered_selected.csv | 0 .../stats/tables/effects_roc_auc.csv | 0 .../stats/tables/effects_total_tokens.csv | 0 .../stats/tables/effects_usd.csv | 0 .../stats/tables/effects_wallclock_min.csv | 0 .../stats/tables/endpoint_index.csv | 0 .../stats/tables/global_holm_sensitivity.csv | 0 .../stats/tables/interaction_sensitivity.csv | 0 .../stats/tables/interaction_sensitivity.md | 0 .../tables/pairwise_mwu_inner_cv_score.csv | 0 .../pairwise_mwu_n_engineered_features.csv | 0 ...ise_mwu_n_engineered_features_selected.csv | 0 .../pairwise_mwu_n_features_after_fs.csv | 0 .../tables/pairwise_mwu_n_turns_total.csv | 0 .../stats/tables/pairwise_mwu_pr_auc.csv | 0 .../pairwise_mwu_prop_engineered_selected.csv | 0 .../stats/tables/pairwise_mwu_roc_auc.csv | 0 .../tables/pairwise_mwu_total_tokens.csv | 0 .../stats/tables/pairwise_mwu_usd.csv | 0 .../tables/pairwise_mwu_wallclock_min.csv | 0 .../stats/validate_palette.py | 0 .../existing-tools-comparators.md | 245 ------------------ 57 files changed, 315 insertions(+), 413 deletions(-) rename publications/{bioinformatics => BDM}/README.md (67%) rename publications/{bioinformatics => BDM}/dict_ZSN.json (100%) rename publications/{bioinformatics => BDM}/import_use_case.py (57%) rename publications/{bioinformatics => BDM}/mi_ZSN.csv (100%) rename publications/{bioinformatics => BDM}/myocardial-anthropic-latest.json (74%) rename publications/{bioinformatics => BDM}/myocardial-anthropic-v0.2.0.json (100%) rename publications/{bioinformatics => BDM}/myocardial-azure-foundry-latest.json (74%) rename publications/{bioinformatics => BDM}/myocardial-azure-foundry-v0.2.0.json (100%) rename publications/{bioinformatics => BDM}/myocardial-openai-latest.json (74%) rename publications/{bioinformatics => BDM}/myocardial-openai-v0.2.0.json (100%) rename publications/{bioinformatics => BDM}/stats/REPORT.html (100%) rename publications/{bioinformatics => BDM}/stats/analysis.py (100%) rename publications/{bioinformatics => BDM}/stats/analysis_log.txt (100%) rename publications/{bioinformatics => BDM}/stats/consolidate.py (100%) rename publications/{bioinformatics => BDM}/stats/figure_main.py (100%) rename publications/{bioinformatics => BDM}/stats/figures.py (100%) rename publications/{bioinformatics => BDM}/stats/results.json (100%) rename publications/{bioinformatics => BDM}/stats/tables/all_effects.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/all_kruskal.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/all_pairwise.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/all_terms_multiplicity.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/all_terms_multiplicity.md (100%) rename publications/{bioinformatics => BDM}/stats/tables/analysis_frame.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/binomial_sensitivity_engineered_share.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/bootstrap_pr_auc.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/configuration_summary.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/cost_decomposition.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_inner_cv_score.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_n_engineered_features.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_n_engineered_features_selected.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_n_features_after_fs.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_n_turns_total.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_pr_auc.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_prop_engineered_selected.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_roc_auc.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_total_tokens.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_usd.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/effects_wallclock_min.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/endpoint_index.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/global_holm_sensitivity.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/interaction_sensitivity.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/interaction_sensitivity.md (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_inner_cv_score.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_n_engineered_features.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_n_engineered_features_selected.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_n_features_after_fs.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_n_turns_total.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_pr_auc.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_prop_engineered_selected.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_roc_auc.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_total_tokens.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_usd.csv (100%) rename publications/{bioinformatics => BDM}/stats/tables/pairwise_mwu_wallclock_min.csv (100%) rename publications/{bioinformatics => BDM}/stats/validate_palette.py (100%) delete mode 100644 publications/bioinformatics/existing-tools-comparators.md diff --git a/README.md b/README.md index a249b071..731bfc3a 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ docker compose up -d # picks up the changed .env ``` **5. Run an experiment.** Follow the worked myocardial-infarction use case in -[`publications/bioinformatics/README.md`](publications/bioinformatics/README.md) +[`publications/BDM/README.md`](publications/BDM/README.md) — five agents in series building a classifier on a public dataset. It picks up exactly where this step leaves off. @@ -201,4 +201,4 @@ You're now at true zero. To get back to a working state: 2. Re-add the LLM credential, then re-run a use case notebook's early setup cells (experiment, dataset, agent creation) — or, for the public myocardial-infarction use case, follow its walkthrough: - [`publications/bioinformatics/README.md`](publications/bioinformatics/README.md). + [`publications/BDM/README.md`](publications/BDM/README.md). diff --git a/frontend/src/components/protocol/bindableFields.ts b/frontend/src/components/protocol/bindableFields.ts index 95eccefe..1f2f1933 100644 --- a/frontend/src/components/protocol/bindableFields.ts +++ b/frontend/src/components/protocol/bindableFields.ts @@ -30,7 +30,7 @@ function isHiddenServerName(name: string): boolean { } // ...unless THIS canvas is already one of the pipelines built on them (the -// myocardial use cases in publications/bioinformatics), in which case hiding +// imported use cases with model factors), in which case hiding // them stops being tidiness and becomes a trap: delete one of the six sklearn // Tool nodes and there'd be no way to add it back. // diff --git a/publications/bioinformatics/README.md b/publications/BDM/README.md similarity index 67% rename from publications/bioinformatics/README.md rename to publications/BDM/README.md index 0f3a3998..b1b045d4 100644 --- a/publications/bioinformatics/README.md +++ b/publications/BDM/README.md @@ -15,7 +15,7 @@ protocol shape, on a dataset anyone can download. | `myocardial-azure-foundry-latest.json` | The experiment, wired to **Azure Foundry** — what `import_use_case.py` imports by default | | `myocardial-anthropic-latest.json` | The same, wired to **Anthropic** | | `myocardial-openai-latest.json` | The same, wired to **OpenAI** | -| `myocardial-*-v0.2.0.json` | Those three graphs frozen as the paper's runs used them — see "Versions" below | +| `myocardial-*-v0.2.0.json` | Archival graphs from the original runs — see "Versions" below | | `mi_ZSN.csv` | The dataset — 1700 admissions × 111 features, target `mi_ZSN` | | `dict_ZSN.json` | The data dictionary for those 111 columns | | `stats/` | The paper's analysis scripts and outputs for the spinal runs (not part of this walkthrough) | @@ -26,11 +26,11 @@ factors bound to it, because a node's provider is fixed when the node is created and can't be switched afterwards. Pick the one matching the API key you have: -| File | Model factor | Effort factor | Cells | +| File | Model factor | Effort factor | Design size | | --- | --- | --- | --- | -| `myocardial-anthropic-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 80 | -| `myocardial-openai-latest.json` | `gpt-5-mini`, `gpt-5` | `medium`, `high` | 80 | -| `myocardial-azure-foundry-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 80 | +| `myocardial-anthropic-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | +| `myocardial-openai-latest.json` | `gpt-5-mini`, `gpt-5` | `medium`, `high` | 8 cells / 80 replicates | +| `myocardial-azure-foundry-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | All three are 2 × 2 × 2 designs (model × effort × critic on/off) at 10 replicates, with the smaller/larger model of a family at the middle and top of @@ -47,12 +47,12 @@ factor bound to a temperature-based model varies nothing at runtime, silently. ## Versions -Each provider variant ships twice. `-v0.2.0` is the graph as the paper's runs -used it, frozen. Don't modernize it: its value is that it's the exact artifact -behind the published numbers, and a graph that drifts can't reproduce them. +Each provider variant ships twice. `-v0.2.0` is the graph retained from the +original runs. Don't modernize it: its value is as a historical artifact, not +as a v0.8.0 import target. -`-latest` is the maintained copy — the same design (same factors, replicates, -metrics, agents and critic gates), brought up to today's canvas: +`-latest` is the maintained v0.8.0 copy — the same factors, replicates, agents, +and critic gates, brought up to the current execution contract: - **Dataset connector.** Its edges use the current `dataset` handle rather than the legacy `resource` spelling, and the Dataset node now sits *above* the @@ -67,6 +67,13 @@ metrics, agents and critic gates), brought up to today's canvas: `open_workspace(stage=...)` instead of passing `experiment_id`/`cell_label`/ `name`, and Score calls `run_model_script` with **no** `code` argument instead of retyping the wired script. +- **Published execution.** The import helper publishes the localized graph as + an immutable protocol revision before generating or running replicates. +- **Declared measurement source.** The complete held-out scoring response is + captured as the opaque `Model evaluation` metric from MI-Score's exact + `asaree-sklearn-model.run_model_script` call. This retains the returned test + metrics and SHA-256 guards with producer provenance; an arbitrary successful + tool call is not promoted as a result. That `open_workspace(stage=...)` call is deliberately kept: seeding the workspace materializes `v0_raw`, but *not* a stage's `.scratch` input, which is @@ -74,12 +81,9 @@ what the `asaree-sklearn-*` servers read. The run context the agent receives says "do not call open_workspace" — true for the data, not for the scratch staging — so each prompt says so explicitly. -Nothing else about how a cell runs changed. The workspace tools the deleted +Nothing else about how a replicate runs changed. The workspace tools the deleted nodes allow-listed (`open_workspace`, `accept_stage`) are still reachable — -implicitly, along with the rest of `WORKSPACE_AGENT_TOOLS`. Current experiments -record scores through explicit measurement-plan producer bindings; arbitrary -successful `run_model_script` calls are not treated as authoritative metric -producers. +implicitly, along with the rest of `WORKSPACE_AGENT_TOOLS`. ## The dataset @@ -94,9 +98,7 @@ The column names are short Russian-derived codes (`nr11`, `zab_leg_01`, `asaree-sklearn-eda`'s `get_data_dictionary` serves back to an agent that asks what a column means. ASAREE itself never parses it. -## Walkthrough - -Everything below happens in the GUI. +## Walkthrough (ASAREE v0.8.0) **0. Get ASAREE running** — see the [root README](../../README.md), then open http://localhost:5173. Every MCP server this use case needs (`asaree-workspace` @@ -105,18 +107,27 @@ on startup; there is nothing to install or register by hand. **1. Register.** Create an account and sign in. -**2. Create an experiment.** The **+** button in the header makes one -(`Untitled Experiment 1`) and drops you straight onto its protocol canvas. -Rename it from the canvas's **⋮** menu → *Rename experiment*. +**2. Issue an API token.** In **Profile → API tokens**, create a token and copy +it when shown. Export it together with the API base URL: + +```bash +export ASAREE_BASE_URL=http://localhost:8000 +export ASAREE_API_KEY=... +``` -**3. Import the use case.** **⋮** → *Import from file…* → pick the JSON for -your provider. That lays out the whole graph and, because a factor is -meaningless without both halves of its binding, brings the design spec with it: -its factors, five metrics, 10 replicates, the critic-gate coordination strategy. +**3. Import the current definition.** From the repository root, choose the +provider file that matches the credential you will use: -Import **merges alongside, never replaces** — nothing you already have is -overwritten, but importing twice into the same canvas gives you two copies of -the graph. Use a fresh, empty experiment. +```bash +uv run --with ./sdk python publications/BDM/import_use_case.py \ + publications/BDM/myocardial-openai-latest.json +``` + +The helper registers and splits the dataset, maps deployment-specific dataset +and MCP-server identifiers, atomically creates the experiment and canvas, +attaches the dataset, validates the measurement plan, publishes the immutable +protocol revision, and generates the design. Re-running it updates the same +named experiment. **4. Add the LLM credential.** Open the shared LLM node — it feeds all five agents and all four critic gates, so it's the only place a model gets chosen. @@ -125,8 +136,11 @@ Once it's saved, the Model dropdown lists what that credential can actually reach, and the node shows Effort or Temperature depending on which one the selected model accepts. -**5. Register the dataset.** Open the *Myocardial Infarction Dataset* node → -*Register new dataset*: +The helper has already completed the two data steps below. They are recorded +here to make the experiment definition explicit. + +**5. Dataset registration.** The *Myocardial Infarction Dataset* node is bound +to: | Field | Value | | --- | --- | @@ -135,7 +149,7 @@ selected model accepts. | Target column | `mi_ZSN` | | Data dictionary | `dict_ZSN.json` | -It's selected on the node as soon as it's registered. The **Data dictionary** +The **Data dictionary** field is what makes this dataset workable: the agents are told to resolve every column's meaning with `get_data_dictionary` rather than guess from its name, and `dict_ZSN.json` is what that tool serves back. Nothing else to configure — @@ -143,7 +157,7 @@ column's meaning with `get_data_dictionary` rather than guess from its name, and directory when the pipeline opens it, so the reader finds it on the same shared filesystem it already reads the data from. -**6. Split it 70/30.** Same node inspector → *Split dataset* → **Quick split**: +**6. Train/test split.** The registered dataset uses this **Quick split**: | Field | Value | | --- | --- | @@ -160,25 +174,25 @@ The split is what the agents actually see. The workspace an agent opens is built from the train half; the test half is only touched by the final model script. Re-splitting at a different seed overwrites rather than accumulating. -**7. Generate the cells.** Side panel → **Design** tab → *Generate design*. It -reports the total: 2 × 2 × 2 factors × 10 replicates = **80 cells**. +**7. Inspect the generated design.** The **Design** and **Cells** tabs show +2 × 2 × 2 factor combinations = **8 cells**, with 10 replicates per cell +(**80 replicates** total). The imported canvas is already published. **8. Run.** Two ways: -- *Run cell* (top-right of the canvas) to pick one cell, then **Run** — do this - first. -- *Run all cells* in the top bar: every not-yet-scored cell at once, enqueued as - one batch. +- *Run cell* (top-right of the canvas) to pick one replicate, then **Run** — do + this first. +- *Run all cells* in the top bar enqueues every pending replicate as one batch. Watch progress on the canvas itself, or in the side panel's **Runs** tab; results land in **Cells** and **Results**. ## Cost -Generating the grid is free — cells are rows until a run starts — but 80 runs +Generating the grid is free — cells and replicates are rows until a run starts — but 80 runs of a five-agent pipeline, half of them at the top of the effort ladder, is a real bill. Run one cell -first, and if you only want a smoke test, lower **Replicates** on the Design -tab *before* generating: `Generate design` is additive, so it creates missing -combinations and never deletes. Lowering replicates afterwards leaves the extra -cells in place (harmless unless you press *Run all cells*). +first. If you only want a smoke test, lower **Replicates** on the Design tab and +regenerate before running; ASAREE opens a new design revision when the change +would remove existing replicate slots and retains the superseded revision in +design history. diff --git a/publications/bioinformatics/dict_ZSN.json b/publications/BDM/dict_ZSN.json similarity index 100% rename from publications/bioinformatics/dict_ZSN.json rename to publications/BDM/dict_ZSN.json diff --git a/publications/bioinformatics/import_use_case.py b/publications/BDM/import_use_case.py similarity index 57% rename from publications/bioinformatics/import_use_case.py rename to publications/BDM/import_use_case.py index 9fde78cd..e976b10b 100644 --- a/publications/bioinformatics/import_use_case.py +++ b/publications/BDM/import_use_case.py @@ -2,27 +2,25 @@ """Import a myocardial-*.json use case (and its dataset) into a running ASAREE. ASAREE_BASE_URL=http://localhost:8000 ASAREE_API_KEY=... \ - uv run --with ./sdk python publications/bioinformatics/import_use_case.py \ + uv run --with ./sdk python publications/BDM/import_use_case.py \ [myocardial-anthropic-latest.json] The API-side equivalent of README.md's GUI walkthrough, for rebuilding the -experiment from scratch repeatedly; the GUI is the documented path. Takes any -of the variants in this directory, defaulting to the Azure Foundry one the -paper's runs used -- in its `-latest` form; pass a `-v0.2.0` file explicitly to -import the frozen graph behind the published numbers (see README's "Versions"). -Idempotent: run it again and it reuses whatever already exists under the same -names rather than creating a second copy. +experiment from scratch repeatedly. It accepts the maintained `-latest` +provider variants and defaults to Azure Foundry. The `-v0.2.0` files are +archival artifacts of the paper's original runs, not v0.8.0 import targets. +Idempotent: run it again and it updates the existing experiment rather than +creating a second copy. -What it does, in order (the order matters -- an experiment has to exist before -a protocol can attach to it, and the dataset has to be registered before the -graph can point at it): +What it does, in order: 1. Register `myocardial_infarction` from mi_ZSN.csv + dict_ZSN.json. 2. Split it 70/30, stratified on the target, seed 42. -3. Create the experiment and attach the dataset. -4. Create the protocol from the file's `graph`, rewriting the dataset and MCP - server UUIDs baked into it to this deployment's own (see `_localize`). -5. Apply the file's `design_spec` and materialize its cells. +3. Rewrite the graph and measurement-source UUIDs for this deployment. +4. Atomically import the experiment and its canvas (or update an existing one), + then attach the dataset. +5. Publish an immutable protocol revision. +6. Apply the design and materialize its eight cells and 80 replicates. """ from __future__ import annotations @@ -51,21 +49,29 @@ SPLIT_SEED = 42 -def _localize(graph: dict[str, Any], *, dataset_id: str, server_ids: dict[str, str]) -> list[str]: - """Repoint the graph's baked-in UUIDs at this deployment's own rows. +def _localize( + graph: dict[str, Any], + measurement_plan: dict[str, Any], + *, + dataset_id: str, + server_ids: dict[str, str], +) -> list[str]: + """Repoint baked-in resource UUIDs at this deployment's own rows. Execution itself resolves by NAME, not UUID -- `protocol_execution` reads `server_name` off an mcp_tool node and `dataset_name` off the dataset node -- so a stale `server_id`/`dataset_id` doesn't stop a run. But the protocol canvas reads them to show which server/dataset a node is bound to, so an export from someone else's install renders as unresolved until they're - rewritten. Cheap to fix here; confusing to leave. + rewritten. Measurement producers also identify the exact MCP server whose + call is authoritative, so those references are localized with the graph. Returns the names of any servers the graph wants that aren't registered on this deployment -- empty on a normal install, since all six asaree-sklearn-* servers and asaree-workspace now ship with ASAREE itself. """ missing: list[str] = [] + server_id_by_node: dict[str, str] = {} for node in graph.get("nodes", []): config = node.get("data", {}).get("config", {}) if node.get("type") == "dataset": @@ -74,8 +80,14 @@ def _localize(graph: dict[str, Any], *, dataset_id: str, server_ids: dict[str, s name = config.get("server_name") if name in server_ids: config["server_id"] = server_ids[name] + server_id_by_node[str(node.get("id"))] = server_ids[name] elif name is not None and name not in missing: missing.append(name) + for producer in measurement_plan.get("producers", []): + config = producer.get("config", {}) + node_id = str(config.get("mcp_node_id", "")) + if node_id in server_id_by_node: + config["server_id"] = server_id_by_node[node_id] return missing @@ -94,6 +106,13 @@ def main(argv: list[str]) -> int: use_case = json.loads(use_case_file.read_text()) graph = use_case["graph"] + measurement_plan = use_case.get("measurement_plan") + if not isinstance(measurement_plan, dict): + print( + "ERROR: this is an archival definition without a v0.8.0 measurement plan; choose a -latest file.", + file=sys.stderr, + ) + return 2 with AsareeClient() as client: # --- 1. the dataset ------------------------------------------------- @@ -131,43 +150,73 @@ def main(argv: list[str]) -> int: ) print(f"split {1 - TEST_SIZE:.0%}/{TEST_SIZE:.0%} stratified, seed {SPLIT_SEED}") - # --- 3. the experiment ---------------------------------------------- + # --- 3. deployment-local references --------------------------------- + servers = {s.name: str(s.id) for s in client.tools.list_servers()} + missing = _localize( + graph, + measurement_plan, + dataset_id=str(dataset.id), + server_ids=servers, + ) + if missing: + print(f"ERROR: MCP servers not registered here: {', '.join(missing)}", file=sys.stderr) + return 2 + + # --- 4. experiment + canvas ----------------------------------------- name = use_case["name"] experiment = next((e for e in client.experiments.list() if e.name == name), None) + created = experiment is None if experiment is None: - experiment = client.experiments.create(name=name, description=use_case.get("description")) + experiment = client.experiments.import_definition( + name=name, + description=use_case.get("description"), + hypothesis=use_case.get("hypothesis"), + design_type=use_case.get("design_type", "factorial"), + task_brief=use_case.get("task_brief"), + design_spec=use_case["design_spec"], + measurement_plan=measurement_plan, + graph=graph, + published_graph=graph, + protocol_description=use_case.get("description"), + ) print(f"experiment created {name!r} ({experiment.id})") else: print(f"experiment reusing {name!r} ({experiment.id})") - experiment = client.experiments.update(experiment.id, dataset_id=dataset.id) - - # --- 4. the protocol ------------------------------------------------ - servers = {s.name: str(s.id) for s in client.tools.list_servers()} - missing = _localize(graph, dataset_id=str(dataset.id), server_ids=servers) - if missing: - # Not fatal -- the protocol still imports, and a run would still - # resolve these by name if they showed up later. But every tool - # call against them fails until they do, so say so loudly. - print(f"WARNING MCP servers not registered here: {', '.join(missing)}", file=sys.stderr) - - # Every protocol, not just this experiment's: names are unique per - # OWNER, so a same-named protocol parked under another experiment would - # make create() 409 rather than fall through to the update branch. - protocol = next((p for p in client.protocols.list() if p.name == name), None) - if protocol is None: - protocol = client.protocols.create( - name=name, description=use_case.get("description"), experiment_id=experiment.id, graph=graph + if experiment.locked_at is not None: + experiment = client.experiments.unlock(experiment.id) + print("experiment unlocked for update") + protocols = client.protocols.list(experiment_id=experiment.id) + if len(protocols) != 1: + print( + f"ERROR: expected one canvas for experiment {experiment.id}, found {len(protocols)}.", + file=sys.stderr, ) - print(f"protocol created ({protocol.id})") - else: + return 2 + protocol = protocols[0] + if not created: protocol = client.protocols.update(protocol.id, graph=graph) print(f"protocol graph updated ({protocol.id})") - # --- 5. the design -------------------------------------------------- - # A full replacement, not a merge -- see experiments.update's docstring. - experiment = client.experiments.update(experiment.id, design_spec=use_case["design_spec"]) - cells = client.experiments.generate_design(experiment.id) - print(f"design {len(cells)} cells") + # Full replacements, not merges. The protocol id asks the server to + # validate producer wiring against this exact localized graph. + experiment = client.experiments.update( + experiment.id, + description=use_case.get("description"), + hypothesis=use_case.get("hypothesis"), + dataset_ids=[dataset.id], + design_spec=use_case["design_spec"], + measurement_plan=measurement_plan, + measurement_validation_protocol_id=protocol.id, + ) + + # --- 5. immutable production revision ------------------------------- + protocol = client.protocols.publish(protocol.id) + print(f"protocol published revision {protocol.published_revision}") + + # --- 6. design ------------------------------------------------------ + replicates = client.experiments.generate_design(experiment.id) + cell_count = len({replicate.cell_id for replicate in replicates}) + print(f"design {cell_count} cells, {len(replicates)} replicates") print(f"\nDone. Open /experiments/{experiment.id}/protocol") return 0 diff --git a/publications/bioinformatics/mi_ZSN.csv b/publications/BDM/mi_ZSN.csv similarity index 100% rename from publications/bioinformatics/mi_ZSN.csv rename to publications/BDM/mi_ZSN.csv diff --git a/publications/bioinformatics/myocardial-anthropic-latest.json b/publications/BDM/myocardial-anthropic-latest.json similarity index 74% rename from publications/bioinformatics/myocardial-anthropic-latest.json rename to publications/BDM/myocardial-anthropic-latest.json index c6400659..70e4c1ba 100644 --- a/publications/bioinformatics/myocardial-anthropic-latest.json +++ b/publications/BDM/myocardial-anthropic-latest.json @@ -1,4 +1,16 @@ { + "format": "asaree.experiment-definition", + "schema_version": 1, + "design_type": "factorial", + "hypothesis": "LLM model capacity, reasoning effort, and critic gating influence predictive performance and execution cost in an agentic machine-learning pipeline.", + "task_brief": { + "domain": "acute myocardial infarction admissions", + "prediction_task": "binary classification", + "target_column": "mi_ZSN", + "positive_class": 1, + "expected_positive_rate": 0.232, + "primary_scientific_outcome": "average precision on a held-out stratified test split" + }, "name": "Myocardial Infarction Use Case (Anthropic)", "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", "graph": { @@ -55,7 +67,7 @@ "prompt": "", "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once \u2014 stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else \u2014 omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", "output_contract": { "name": "DCReport", "fields": [ @@ -125,7 +137,7 @@ "name": "dc-critic", "goal": "Review MI-DC's cleaning decisions for this stage only.", "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", + "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical — those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", "enabled": true, "max_revisions": 1 }, @@ -207,7 +219,7 @@ "prompt": "", "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", "output_contract": { "name": "FTEReport", "fields": [ @@ -334,7 +346,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": { "name": "FSReport", "fields": [ @@ -911,33 +923,49 @@ "replicates": 10, "metrics": [ { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", + "id": "model-evaluation", + "name": "Model evaluation", + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "kind": "custom", + "valueType": "opaque", "primary": false, - "direction": "maximize" + "direction": "neutral", + "aggregation": "none" } ], "coordination_strategy": { "slug": "critic_gate" } + }, + "measurement_plan": { + "metrics": [ + { + "id": "model-evaluation", + "name": "Model evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." + } + ], + "producers": [ + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "value": "model-evaluation" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script" + } + } + ], + "inputs": [] } } diff --git a/publications/bioinformatics/myocardial-anthropic-v0.2.0.json b/publications/BDM/myocardial-anthropic-v0.2.0.json similarity index 100% rename from publications/bioinformatics/myocardial-anthropic-v0.2.0.json rename to publications/BDM/myocardial-anthropic-v0.2.0.json diff --git a/publications/bioinformatics/myocardial-azure-foundry-latest.json b/publications/BDM/myocardial-azure-foundry-latest.json similarity index 74% rename from publications/bioinformatics/myocardial-azure-foundry-latest.json rename to publications/BDM/myocardial-azure-foundry-latest.json index 47754023..583db625 100644 --- a/publications/bioinformatics/myocardial-azure-foundry-latest.json +++ b/publications/BDM/myocardial-azure-foundry-latest.json @@ -1,4 +1,16 @@ { + "format": "asaree.experiment-definition", + "schema_version": 1, + "design_type": "factorial", + "hypothesis": "LLM model capacity, reasoning effort, and critic gating influence predictive performance and execution cost in an agentic machine-learning pipeline.", + "task_brief": { + "domain": "acute myocardial infarction admissions", + "prediction_task": "binary classification", + "target_column": "mi_ZSN", + "positive_class": 1, + "expected_positive_rate": 0.232, + "primary_scientific_outcome": "average precision on a held-out stratified test split" + }, "name": "Myocardial Infarction Use Case (Azure Foundry)", "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", "graph": { @@ -55,7 +67,7 @@ "prompt": "", "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once \u2014 stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else \u2014 omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", "output_contract": { "name": "DCReport", "fields": [ @@ -125,7 +137,7 @@ "name": "dc-critic", "goal": "Review MI-DC's cleaning decisions for this stage only.", "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", + "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical — those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", "enabled": true, "max_revisions": 1 }, @@ -207,7 +219,7 @@ "prompt": "", "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", "output_contract": { "name": "FTEReport", "fields": [ @@ -334,7 +346,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": { "name": "FSReport", "fields": [ @@ -911,33 +923,49 @@ "replicates": 10, "metrics": [ { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", + "id": "model-evaluation", + "name": "Model evaluation", + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "kind": "custom", + "valueType": "opaque", "primary": false, - "direction": "maximize" + "direction": "neutral", + "aggregation": "none" } ], "coordination_strategy": { "slug": "critic_gate" } + }, + "measurement_plan": { + "metrics": [ + { + "id": "model-evaluation", + "name": "Model evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." + } + ], + "producers": [ + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "value": "model-evaluation" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script" + } + } + ], + "inputs": [] } } diff --git a/publications/bioinformatics/myocardial-azure-foundry-v0.2.0.json b/publications/BDM/myocardial-azure-foundry-v0.2.0.json similarity index 100% rename from publications/bioinformatics/myocardial-azure-foundry-v0.2.0.json rename to publications/BDM/myocardial-azure-foundry-v0.2.0.json diff --git a/publications/bioinformatics/myocardial-openai-latest.json b/publications/BDM/myocardial-openai-latest.json similarity index 74% rename from publications/bioinformatics/myocardial-openai-latest.json rename to publications/BDM/myocardial-openai-latest.json index c29e6c39..3a8ad508 100644 --- a/publications/bioinformatics/myocardial-openai-latest.json +++ b/publications/BDM/myocardial-openai-latest.json @@ -1,4 +1,16 @@ { + "format": "asaree.experiment-definition", + "schema_version": 1, + "design_type": "factorial", + "hypothesis": "LLM model capacity, reasoning effort, and critic gating influence predictive performance and execution cost in an agentic machine-learning pipeline.", + "task_brief": { + "domain": "acute myocardial infarction admissions", + "prediction_task": "binary classification", + "target_column": "mi_ZSN", + "positive_class": 1, + "expected_positive_rate": 0.232, + "primary_scientific_outcome": "average precision on a held-out stratified test split" + }, "name": "Myocardial Infarction Use Case (OpenAI)", "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", "graph": { @@ -55,7 +67,7 @@ "prompt": "", "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once \u2014 stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else \u2014 omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", "output_contract": { "name": "DCReport", "fields": [ @@ -125,7 +137,7 @@ "name": "dc-critic", "goal": "Review MI-DC's cleaning decisions for this stage only.", "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", + "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical — those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", "enabled": true, "max_revisions": 1 }, @@ -207,7 +219,7 @@ "prompt": "", "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", + "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", "output_contract": { "name": "FTEReport", "fields": [ @@ -334,7 +346,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": { "name": "FSReport", "fields": [ @@ -911,33 +923,49 @@ "replicates": 10, "metrics": [ { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", + "id": "model-evaluation", + "name": "Model evaluation", + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "kind": "custom", + "valueType": "opaque", "primary": false, - "direction": "maximize" + "direction": "neutral", + "aggregation": "none" } ], "coordination_strategy": { "slug": "critic_gate" } + }, + "measurement_plan": { + "metrics": [ + { + "id": "model-evaluation", + "name": "Model evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." + } + ], + "producers": [ + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "value": "model-evaluation" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script" + } + } + ], + "inputs": [] } } diff --git a/publications/bioinformatics/myocardial-openai-v0.2.0.json b/publications/BDM/myocardial-openai-v0.2.0.json similarity index 100% rename from publications/bioinformatics/myocardial-openai-v0.2.0.json rename to publications/BDM/myocardial-openai-v0.2.0.json diff --git a/publications/bioinformatics/stats/REPORT.html b/publications/BDM/stats/REPORT.html similarity index 100% rename from publications/bioinformatics/stats/REPORT.html rename to publications/BDM/stats/REPORT.html diff --git a/publications/bioinformatics/stats/analysis.py b/publications/BDM/stats/analysis.py similarity index 100% rename from publications/bioinformatics/stats/analysis.py rename to publications/BDM/stats/analysis.py diff --git a/publications/bioinformatics/stats/analysis_log.txt b/publications/BDM/stats/analysis_log.txt similarity index 100% rename from publications/bioinformatics/stats/analysis_log.txt rename to publications/BDM/stats/analysis_log.txt diff --git a/publications/bioinformatics/stats/consolidate.py b/publications/BDM/stats/consolidate.py similarity index 100% rename from publications/bioinformatics/stats/consolidate.py rename to publications/BDM/stats/consolidate.py diff --git a/publications/bioinformatics/stats/figure_main.py b/publications/BDM/stats/figure_main.py similarity index 100% rename from publications/bioinformatics/stats/figure_main.py rename to publications/BDM/stats/figure_main.py diff --git a/publications/bioinformatics/stats/figures.py b/publications/BDM/stats/figures.py similarity index 100% rename from publications/bioinformatics/stats/figures.py rename to publications/BDM/stats/figures.py diff --git a/publications/bioinformatics/stats/results.json b/publications/BDM/stats/results.json similarity index 100% rename from publications/bioinformatics/stats/results.json rename to publications/BDM/stats/results.json diff --git a/publications/bioinformatics/stats/tables/all_effects.csv b/publications/BDM/stats/tables/all_effects.csv similarity index 100% rename from publications/bioinformatics/stats/tables/all_effects.csv rename to publications/BDM/stats/tables/all_effects.csv diff --git a/publications/bioinformatics/stats/tables/all_kruskal.csv b/publications/BDM/stats/tables/all_kruskal.csv similarity index 100% rename from publications/bioinformatics/stats/tables/all_kruskal.csv rename to publications/BDM/stats/tables/all_kruskal.csv diff --git a/publications/bioinformatics/stats/tables/all_pairwise.csv b/publications/BDM/stats/tables/all_pairwise.csv similarity index 100% rename from publications/bioinformatics/stats/tables/all_pairwise.csv rename to publications/BDM/stats/tables/all_pairwise.csv diff --git a/publications/bioinformatics/stats/tables/all_terms_multiplicity.csv b/publications/BDM/stats/tables/all_terms_multiplicity.csv similarity index 100% rename from publications/bioinformatics/stats/tables/all_terms_multiplicity.csv rename to publications/BDM/stats/tables/all_terms_multiplicity.csv diff --git a/publications/bioinformatics/stats/tables/all_terms_multiplicity.md b/publications/BDM/stats/tables/all_terms_multiplicity.md similarity index 100% rename from publications/bioinformatics/stats/tables/all_terms_multiplicity.md rename to publications/BDM/stats/tables/all_terms_multiplicity.md diff --git a/publications/bioinformatics/stats/tables/analysis_frame.csv b/publications/BDM/stats/tables/analysis_frame.csv similarity index 100% rename from publications/bioinformatics/stats/tables/analysis_frame.csv rename to publications/BDM/stats/tables/analysis_frame.csv diff --git a/publications/bioinformatics/stats/tables/binomial_sensitivity_engineered_share.csv b/publications/BDM/stats/tables/binomial_sensitivity_engineered_share.csv similarity index 100% rename from publications/bioinformatics/stats/tables/binomial_sensitivity_engineered_share.csv rename to publications/BDM/stats/tables/binomial_sensitivity_engineered_share.csv diff --git a/publications/bioinformatics/stats/tables/bootstrap_pr_auc.csv b/publications/BDM/stats/tables/bootstrap_pr_auc.csv similarity index 100% rename from publications/bioinformatics/stats/tables/bootstrap_pr_auc.csv rename to publications/BDM/stats/tables/bootstrap_pr_auc.csv diff --git a/publications/bioinformatics/stats/tables/configuration_summary.csv b/publications/BDM/stats/tables/configuration_summary.csv similarity index 100% rename from publications/bioinformatics/stats/tables/configuration_summary.csv rename to publications/BDM/stats/tables/configuration_summary.csv diff --git a/publications/bioinformatics/stats/tables/cost_decomposition.csv b/publications/BDM/stats/tables/cost_decomposition.csv similarity index 100% rename from publications/bioinformatics/stats/tables/cost_decomposition.csv rename to publications/BDM/stats/tables/cost_decomposition.csv diff --git a/publications/bioinformatics/stats/tables/effects_inner_cv_score.csv b/publications/BDM/stats/tables/effects_inner_cv_score.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_inner_cv_score.csv rename to publications/BDM/stats/tables/effects_inner_cv_score.csv diff --git a/publications/bioinformatics/stats/tables/effects_n_engineered_features.csv b/publications/BDM/stats/tables/effects_n_engineered_features.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_n_engineered_features.csv rename to publications/BDM/stats/tables/effects_n_engineered_features.csv diff --git a/publications/bioinformatics/stats/tables/effects_n_engineered_features_selected.csv b/publications/BDM/stats/tables/effects_n_engineered_features_selected.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_n_engineered_features_selected.csv rename to publications/BDM/stats/tables/effects_n_engineered_features_selected.csv diff --git a/publications/bioinformatics/stats/tables/effects_n_features_after_fs.csv b/publications/BDM/stats/tables/effects_n_features_after_fs.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_n_features_after_fs.csv rename to publications/BDM/stats/tables/effects_n_features_after_fs.csv diff --git a/publications/bioinformatics/stats/tables/effects_n_turns_total.csv b/publications/BDM/stats/tables/effects_n_turns_total.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_n_turns_total.csv rename to publications/BDM/stats/tables/effects_n_turns_total.csv diff --git a/publications/bioinformatics/stats/tables/effects_pr_auc.csv b/publications/BDM/stats/tables/effects_pr_auc.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_pr_auc.csv rename to publications/BDM/stats/tables/effects_pr_auc.csv diff --git a/publications/bioinformatics/stats/tables/effects_prop_engineered_selected.csv b/publications/BDM/stats/tables/effects_prop_engineered_selected.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_prop_engineered_selected.csv rename to publications/BDM/stats/tables/effects_prop_engineered_selected.csv diff --git a/publications/bioinformatics/stats/tables/effects_roc_auc.csv b/publications/BDM/stats/tables/effects_roc_auc.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_roc_auc.csv rename to publications/BDM/stats/tables/effects_roc_auc.csv diff --git a/publications/bioinformatics/stats/tables/effects_total_tokens.csv b/publications/BDM/stats/tables/effects_total_tokens.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_total_tokens.csv rename to publications/BDM/stats/tables/effects_total_tokens.csv diff --git a/publications/bioinformatics/stats/tables/effects_usd.csv b/publications/BDM/stats/tables/effects_usd.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_usd.csv rename to publications/BDM/stats/tables/effects_usd.csv diff --git a/publications/bioinformatics/stats/tables/effects_wallclock_min.csv b/publications/BDM/stats/tables/effects_wallclock_min.csv similarity index 100% rename from publications/bioinformatics/stats/tables/effects_wallclock_min.csv rename to publications/BDM/stats/tables/effects_wallclock_min.csv diff --git a/publications/bioinformatics/stats/tables/endpoint_index.csv b/publications/BDM/stats/tables/endpoint_index.csv similarity index 100% rename from publications/bioinformatics/stats/tables/endpoint_index.csv rename to publications/BDM/stats/tables/endpoint_index.csv diff --git a/publications/bioinformatics/stats/tables/global_holm_sensitivity.csv b/publications/BDM/stats/tables/global_holm_sensitivity.csv similarity index 100% rename from publications/bioinformatics/stats/tables/global_holm_sensitivity.csv rename to publications/BDM/stats/tables/global_holm_sensitivity.csv diff --git a/publications/bioinformatics/stats/tables/interaction_sensitivity.csv b/publications/BDM/stats/tables/interaction_sensitivity.csv similarity index 100% rename from publications/bioinformatics/stats/tables/interaction_sensitivity.csv rename to publications/BDM/stats/tables/interaction_sensitivity.csv diff --git a/publications/bioinformatics/stats/tables/interaction_sensitivity.md b/publications/BDM/stats/tables/interaction_sensitivity.md similarity index 100% rename from publications/bioinformatics/stats/tables/interaction_sensitivity.md rename to publications/BDM/stats/tables/interaction_sensitivity.md diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_inner_cv_score.csv b/publications/BDM/stats/tables/pairwise_mwu_inner_cv_score.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_inner_cv_score.csv rename to publications/BDM/stats/tables/pairwise_mwu_inner_cv_score.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_n_engineered_features.csv b/publications/BDM/stats/tables/pairwise_mwu_n_engineered_features.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_n_engineered_features.csv rename to publications/BDM/stats/tables/pairwise_mwu_n_engineered_features.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_n_engineered_features_selected.csv b/publications/BDM/stats/tables/pairwise_mwu_n_engineered_features_selected.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_n_engineered_features_selected.csv rename to publications/BDM/stats/tables/pairwise_mwu_n_engineered_features_selected.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_n_features_after_fs.csv b/publications/BDM/stats/tables/pairwise_mwu_n_features_after_fs.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_n_features_after_fs.csv rename to publications/BDM/stats/tables/pairwise_mwu_n_features_after_fs.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_n_turns_total.csv b/publications/BDM/stats/tables/pairwise_mwu_n_turns_total.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_n_turns_total.csv rename to publications/BDM/stats/tables/pairwise_mwu_n_turns_total.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_pr_auc.csv b/publications/BDM/stats/tables/pairwise_mwu_pr_auc.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_pr_auc.csv rename to publications/BDM/stats/tables/pairwise_mwu_pr_auc.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_prop_engineered_selected.csv b/publications/BDM/stats/tables/pairwise_mwu_prop_engineered_selected.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_prop_engineered_selected.csv rename to publications/BDM/stats/tables/pairwise_mwu_prop_engineered_selected.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_roc_auc.csv b/publications/BDM/stats/tables/pairwise_mwu_roc_auc.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_roc_auc.csv rename to publications/BDM/stats/tables/pairwise_mwu_roc_auc.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_total_tokens.csv b/publications/BDM/stats/tables/pairwise_mwu_total_tokens.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_total_tokens.csv rename to publications/BDM/stats/tables/pairwise_mwu_total_tokens.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_usd.csv b/publications/BDM/stats/tables/pairwise_mwu_usd.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_usd.csv rename to publications/BDM/stats/tables/pairwise_mwu_usd.csv diff --git a/publications/bioinformatics/stats/tables/pairwise_mwu_wallclock_min.csv b/publications/BDM/stats/tables/pairwise_mwu_wallclock_min.csv similarity index 100% rename from publications/bioinformatics/stats/tables/pairwise_mwu_wallclock_min.csv rename to publications/BDM/stats/tables/pairwise_mwu_wallclock_min.csv diff --git a/publications/bioinformatics/stats/validate_palette.py b/publications/BDM/stats/validate_palette.py similarity index 100% rename from publications/bioinformatics/stats/validate_palette.py rename to publications/BDM/stats/validate_palette.py diff --git a/publications/bioinformatics/existing-tools-comparators.md b/publications/bioinformatics/existing-tools-comparators.md deleted file mode 100644 index df00366e..00000000 --- a/publications/bioinformatics/existing-tools-comparators.md +++ /dev/null @@ -1,245 +0,0 @@ -# Existing-tool comparators for ASAREE - -Research date: 2026-09-25. This is a primary-source scan prompted by the review -comment that ASAREE is "a potentially useful, openly available analytical -sandbox whose practical value is plausible, but whose advantage over existing -tools has not been demonstrated." Tools released after the manuscript's -comparison cut-off should be labeled as such rather than presented as omissions -from the original submission. - -## What ASAREE is claiming - -The repository describes ASAREE as a workbench for running LLM agents as -designed experiments: users visually assemble a protocol, bind factors to node -configuration, materialize a full factorial design, run replicates, and compare -recorded metrics. It also versions datasets and production protocol snapshots -and exposes tools through MCP ([project README](../../README.md)). The public -use case is an agent sequence for tabular biomedical ML, varied in a 2 × 2 × 2 -model/effort/critic design with ten replicates -([use-case README](README.md)). - -That description touches four established product/research categories. The -reviewer may mean any of them; the first two are the most direct. - -## 1. Evaluation and experiment platforms — closest functional comparators - -### Arize Phoenix (highest-priority comparator) - -Phoenix is open source and explicitly positions itself as a platform for -"experimentation, evaluation, and troubleshooting" of AI/LLM applications. It -supports versioned datasets, side-by-side experiments, deterministic and -LLM-judge evaluators, and repeated runs; its client API exposes a `repetitions` -argument and stores experiment/evaluation results for comparison -([official overview](https://arize.com/docs/phoenix/), -[dataset concepts](https://arize.com/docs/phoenix/learn/datasets-and-experiments/datasets-concepts), -[experiment API](https://arize-phoenix.readthedocs.io/projects/client/api/experiments.html)). - -Why the reviewer may have it in mind: this is the clearest existing open-source -answer to "systematically run a stochastic agent repeatedly against versioned -data and compare scores." ASAREE needs to demonstrate more than the generic -ability to store datasets, traces, repetitions, and scores. Its plausible -distinction is the first-class *factorial design over fields inside a visual -multi-agent protocol*, including factor binding, generated cells/replicates, -immutable executable protocol revisions, and biomedical data-workspace lineage. - -### MLflow GenAI evaluation - -MLflow evaluates tool-using agents from datasets or traces, records the result -as an experiment run, and supplies agent-specific scorers such as tool-call -correctness and efficiency. It also supports deterministic code scorers and LLM -judges; classic MLflow separately evaluates classification and regression models -([agent evaluation](https://mlflow.org/docs/latest/genai/eval-monitor/running-evaluation/agents/), -[scorers](https://mlflow.org/docs/latest/genai/eval-monitor/scorers/), -[classic model evaluation](https://mlflow.org/docs/latest/ml/evaluation)). - -Why it matters: ASAREE's use case produces ordinary supervised-ML outcomes as -well as agent runtime outcomes. A convincing comparison should show what ASAREE -adds beyond an MLflow-tracked loop over model/agent configurations. - -### LangSmith, Braintrust, W&B Weave, and Promptfoo - -- LangSmith supports offline benchmarks over datasets, comparisons among - application versions, code and model-judge evaluators, pairwise/summary - evaluators, and agent examples that verify expected tool calls - ([official evaluation docs](https://docs.langchain.com/langsmith/evaluation-types)). -- Braintrust defines an evaluation as data + task + scores and stores offline - evaluations as experiments. Its datasets are versioned and can be populated - from production, staging, evaluations, or manual examples - ([experiments](https://www.braintrust.dev/docs/guides/experiments), - [datasets](https://www.braintrust.dev/docs/guides/datasets)). -- W&B Weave logs datasets, model outputs, per-example scores and summary - metrics, then compares multiple evaluations in its UI - ([official EvaluationLogger guide](https://weave-docs.wandb.ai/guides/evaluation/evaluation_logger)). -- Promptfoo is an open-source CLI/library that crosses prompts/models with test - cases and evaluates them through deterministic assertions, custom code, or - model-graded assertions; it explicitly covers agent quality and trajectory - goal success - ([getting started](https://www.promptfoo.dev/docs/getting-started/), - [assertions and metrics](https://www.promptfoo.dev/docs/configuration/expected-outputs/)). - -These are direct comparators for the evaluation/result-management layer, but -less direct for ASAREE's visual protocol construction and factorial-design -semantics. They are still likely to be named by an agent-evaluation reviewer. - -### Inspect AI - -The UK AI Security Institute's open-source Inspect framework models evaluations -as datasets, solvers/agents, tools, sandboxes, and scorers; it supports -multi-agent primitives, arbitrary external agents, and several isolated -execution backends ([official documentation](https://inspect.aisi.org.uk/)). - -Inspect is a direct comparator if ASAREE is framed as an *agent evaluation -harness*. It is more benchmark/code oriented than ASAREE's GUI and biomedical -workflow, but its mature sandboxing and task/scorer abstraction set a baseline -for claims about reproducibility and safe execution. - -## 2. Visual agent workflow builders — closest interface comparators - -### AutoGen Studio - -AutoGen Studio offers a drag-and-drop/JSON interface for teams, agents, tools, -models, and termination conditions, plus a playground for running, inspecting, -and debugging multi-agent sessions. Its paper explicitly describes it as a -no-code tool for prototyping, debugging, and evaluating multi-agent workflows -([official docs](https://microsoft.github.io/autogen/0.7.1/user-guide/autogenstudio-user-guide/index.html), -[Microsoft Research publication](https://www.microsoft.com/en-us/research/publication/autogen-studio-a-no-code-developer-tool-for-building-and-debugging-multi-agent-systems/)). - -This is probably the first tool a reviewer will cite against the protocol -canvas. ASAREE should not claim novelty merely for drag-and-drop multi-agent -composition; the defensible comparison is whether a visual workflow can be -turned into a controlled factorial experiment with replicated, statistically -comparable outcomes. - -### Flowise and Langflow - -Flowise is an open-source visual platform for single- and multi-agent workflows, -with models, branching/loops, MCP, traces, datasets, evaluators, and evaluation -runs. Its packaged evaluation feature is documented as Cloud/Enterprise -functionality ([overview](https://docs.flowiseai.com/), -[evaluations](https://docs.flowiseai.com/using-flowise/evaluations)). Langflow's -visual editor connects prompts, models, data, agents, MCP servers, and tools; -flows can be run in a playground, exported as JSON, served through APIs, or -exposed as MCP tools -([visual editor](https://docs.langflow.org/concepts-overview), -[agents](https://docs.langflow.org/components-agents)). - -These are direct comparators for practical visual authoring and MCP integration. -Neither cited documentation foregrounds designed experiments over internal node -fields; that potential gap should be demonstrated with a feature matrix and a -worked head-to-head task rather than asserted. - -## 3. Biomedical/data-science agents — task-level comparators - -### BioMedAgent, Agentomics, Biomni, AIDE, MLAgentBench, and STELLA - -BioMedAgent is a particularly important task-level comparator. It is a -self-evolving multi-agent framework that accepts natural-language biomedical -analysis tasks, learns to use and chain bioinformatics tools, and covers -cross-omics analysis, ML modeling, and pathology image segmentation. Its -authors report a 77% success rate on the 327-task BioMed-AQA benchmark and -external evaluation on BixBench -([Nature Biomedical Engineering article](https://doi.org/10.1038/s41551-026-01634-6), -[official repository](https://github.com/BOBQWERA/BioMedAgent)). It postdates -ASAREE v0.2.0, but is now one of the clearest practical biomedical-agent -comparators and should be acknowledged in a current revision. - -Agentomics is an end-to-end biomedical ML agent system with validation -checkpoints, containerized execution, multiple model providers, and support for -biomedical foundation models. Its Bioinformatics paper benchmarks it across 20 -datasets against human solutions and four agent systems: AIDE, MLAgentBench, -STELLA, and Biomni -([Bioinformatics article](https://academic.oup.com/bioinformatics/article/42/Supplement_1/btag250/8726289)). -The paper characterizes AIDE and MLAgentBench as generalist ML coding agents, -STELLA as a biomedical multi-agent system with manager/developer/critic/tool -agents, and Biomni as a general-purpose biomedical agent with an expert-vetted -tool environment. - -These systems do not have ASAREE's same purpose: they seek a strong ML solution, -whereas ASAREE measures the effects of agent/protocol choices. Nevertheless, -they are important practical baselines for the myocardial/spinal-style use -case. If ASAREE claims usefulness for producing biomedical classifiers, a -reviewer can reasonably ask how its output quality, success rate, wall time, and -cost compare with at least one autonomous ML agent and a non-agent baseline. - -### BRAD and Coala - -BRAD is an open agentic bioinformatics system that connects LLMs to literature, -databases, custom software, and user data, records detailed interaction logs, -and demonstrates an automated biomarker/enrichment workflow -([Bioinformatics article](https://academic.oup.com/bioinformatics/article/41/5/btaf159/8125018)). -Coala converts CWL-described command-line tools into MCP tools and executes them -in containers, targeting reproducible local bioinformatics analysis -([Bioinformatics article](https://academic.oup.com/bioinformatics/article/42/9/btag641/8771239)). - -They are adjacent rather than direct: BRAD overlaps in agentic biomedical -analysis and provenance, while Coala overlaps in MCP tool integration and -reproducible execution. Both weaken broad claims that ASAREE uniquely makes -bioinformatics tools accessible to LLM agents. - -## 4. Conventional analytics, workflow, and AutoML systems — necessary baselines - -- Galaxy is an open web platform explicitly built for accessible, - reproducible, transparent computational biomedical research; it records the - information needed to repeat complete analyses - ([Galaxy project](https://galaxyproject.org/galaxy-project/)). -- KNIME is an open-source visual workflow system for data access, - transformation, analysis, modeling, and visualization - ([official documentation](https://docs.knime.com/ap/latest/)). -- Orange is open-source visual programming for machine learning and data - visualization, with canvas-connected widgets and no-code workflows - ([official site](https://orangedatamining.com/)). -- AutoGluon trains and ranks model families from tabular data and exposes a - model leaderboard; its paper describes highly accurate tabular AutoML from a - single line of Python - ([official tutorial](https://auto.gluon.ai/stable/tutorials/tabular/tabular-quick-start.html), - [paper](https://arxiv.org/abs/2003.06505)). The Agentomics paper also names - auto-sklearn and H2O alongside AutoGluon as established AutoML frameworks - ([Bioinformatics article](https://academic.oup.com/bioinformatics/article/42/Supplement_1/btag250/8726289)). - -These are not agent-experiment platforms. They matter because "analytical -sandbox" and the paper's visual tabular-ML use case overlap with capabilities -biomedical users already recognize. The practical advantage should therefore -be stated narrowly: ASAREE is for causal/comparative study of stochastic agent -protocol configurations, not simply visual analytics, workflow reproducibility, -or automated classifier search. - -## Likely reviewer shortlist - -If space permits only a compact comparison, prioritize: - -1. **Phoenix** — strongest open-source experiment/evaluation overlap, including - versioned datasets and repetitions. -2. **AutoGen Studio** — strongest visual multi-agent-workflow overlap. -3. **MLflow** — strongest bridge between agent evaluation and conventional ML - experiment tracking/evaluation. -4. **BioMedAgent, Agentomics, and Biomni** — strongest task-level overlap for - autonomous biomedical analysis/ML; AIDE and MLAgentBench are useful - generalist agent baselines. -5. **Galaxy or KNIME** — the established biomedical/visual analytical workflow - baseline. - -Flowise/Langflow, LangSmith, Braintrust, Weave, Promptfoo, Inspect, BRAD, Coala, -Orange, and AutoGluon belong in a broader related-work table or supplement. - -## What would actually answer the criticism - -A feature inventory alone will probably not satisfy "advantage ... has not been -demonstrated." A focused revision should: - -1. Add a capability table whose rows distinguish visual multi-agent authoring, - arbitrary internal-field factors, full-factorial cell generation, - repetitions, statistical comparison, immutable executable revisions, - versioned biomedical datasets, deterministic + model-judge metrics, MCP, - self-hosting, and run-level cost/time capture. -2. Recreate one ASAREE experiment in the nearest feasible baseline—preferably - Phoenix or MLflow around an equivalent agent workflow—and report setup/code - burden, completeness of provenance, and whether the factorial comparison can - be expressed natively or only through bespoke orchestration. -3. Add a non-agent practical baseline such as AutoGluon for predictive quality, - time, and cost. This prevents evidence that one agent configuration beats - another from being mistaken for evidence that the sandbox improves the - biomedical analysis. -4. Frame the advantage as **designed experimentation on agent protocols**, not - as visual workflow construction, generic LLM evaluation, AutoML, or a new - bioinformatics agent. Existing tools already substantiate all four broader - categories. From b515775dc6c4e2167b01008f9c72057bbece2a76 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 17:21:22 -0700 Subject: [PATCH 08/18] feat: add MCP server authentication Closes #2 --- .env.example | 8 + frontend/src/App.tsx | 2 + frontend/src/api/client.ts | 11 +- .../protocol/ConnectMcpServerDialog.tsx | 156 +++++++++++++++- .../protocol/CustomMetricDialogs.test.tsx | 5 +- .../protocol/ManageMcpCredentialsDialog.tsx | 169 +++++++++++++++++ .../protocol/McpCredentialFields.test.ts | 20 ++ .../protocol/McpCredentialFields.tsx | 57 ++++++ .../protocol/McpToolNodeInspector.tsx | 54 ++++-- frontend/src/lib/mcpCredentials.ts | 17 ++ frontend/src/lib/mcpOAuth.test.ts | 17 ++ frontend/src/lib/mcpOAuth.ts | 70 +++++++ frontend/src/pages/McpOAuthCallbackPage.tsx | 24 +++ frontend/src/types/mcpServers.ts | 15 ++ pyproject.toml | 2 +- src/asaree/api/mcp_servers.py | 175 ++++++++++++++++-- src/asaree/config.py | 11 ++ tests/test_mcp_server_auth.py | 172 +++++++++++++++++ uv.lock | 4 +- 19 files changed, 950 insertions(+), 39 deletions(-) create mode 100644 frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx create mode 100644 frontend/src/components/protocol/McpCredentialFields.test.ts create mode 100644 frontend/src/components/protocol/McpCredentialFields.tsx create mode 100644 frontend/src/lib/mcpCredentials.ts create mode 100644 frontend/src/lib/mcpOAuth.test.ts create mode 100644 frontend/src/lib/mcpOAuth.ts create mode 100644 frontend/src/pages/McpOAuthCallbackPage.tsx create mode 100644 tests/test_mcp_server_auth.py diff --git a/.env.example b/.env.example index f88cb528..f83dcb53 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,14 @@ OPENAI_API_KEY= # python3 -c "import secrets; print(secrets.token_urlsafe(32))" ASAREE_ENCRYPTION_KEY=VSrs26eW-By3hinbUIc8wTIdL8AP8VTm9_as2tqkgXs +# Exact URLs registered during MCP OAuth. Set both to the externally reachable +# origins in deployed environments; providers reject redirect URI mismatches. +ASAREE_MCP_OAUTH_CALLBACK_URL=http://localhost:8000/api/mcp-servers/oauth/callback +ASAREE_FRONTEND_URL=http://localhost:5173 +# Optional HTTPS Client ID Metadata Document. Leave blank to use dynamic +# client registration with providers that support it. +# ASAREE_MCP_OAUTH_CLIENT_METADATA_URL=https://example.org/asaree/oauth-client.json + # Where the workspace MCP server (asaree.mcp_servers.workspace_server) stores # per-cell versioned workspaces. MUST be an absolute path, not the relative # default (asaree_workspace_core.workspace.WORKSPACE_ROOT falls back to diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 44b34ab3..55479272 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -6,6 +6,7 @@ import { RegisterPage } from '@/pages/RegisterPage' import { ProfilePage } from '@/pages/ProfilePage' import { ExperimentsPage } from '@/pages/ExperimentsPage' import { ProtocolCanvasPage } from '@/pages/ProtocolCanvasPage' +import { McpOAuthCallbackPage } from '@/pages/McpOAuthCallbackPage' export function App() { return ( @@ -16,6 +17,7 @@ export function App() { } /> } /> + } /> }> } /> } /> diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index b59166f2..cd8ee2c5 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -27,7 +27,7 @@ import type { Trial, } from '@/types/experiments' import type { LLMConnectionCheck, LLMProvider, LLMSetting, LLMSettingModelsResponse } from '@/types/llmSettings' -import type { McpServer } from '@/types/mcpServers' +import type { McpOAuthAuthorization, McpServer } from '@/types/mcpServers' import type { OkfBundle, OkfDocument } from '@/types/okf' import type { CellRunBatch, PromptPreview, Protocol, ProtocolGraph, ProtocolRevision, ProtocolRun, TestRun } from '@/types/protocols' import type { Run, RunStep } from '@/types/runs' @@ -397,14 +397,21 @@ export const mcpServersApi = { // existing scope (system servers like asaree-workspace aren't listed here // either; not something the MCP Tool node picker widens). list: () => request('/mcp-servers'), + get: (id: string) => request(`/mcp-servers/${id}`), // Registers a connection the user typed in themselves -- backs the MCP // Client Tool node (ConnectMcpServerDialog). The response already carries // the discovered tools: core connects and lists them synchronously during // registration, so a 201 whose `status` is 'error' means "row saved, server // unreachable", not a failure to save. 409 on a duplicate `name`, 422 when // the stdio allowlist or the SSRF guard rejects it. - create: (data: { name: string; transport: string; command?: string | null; url?: string | null; headers?: Record | null }) => + create: (data: { name: string; transport: string; command?: string | null; url?: string | null; headers?: Record | null; server_env?: Record | null }) => request('/mcp-servers', { method: 'POST', body: data }), + update: (id: string, data: { headers?: Record | null; server_env?: Record | null }) => + request(`/mcp-servers/${id}`, { method: 'PATCH', body: data }), + beginOAuth: (id: string, scope?: string | null) => + request(`/mcp-servers/${id}/oauth/start`, { method: 'POST', body: { scope: scope || null } }), + clearCredentials: (id: string, revoke = false) => + request(`/mcp-servers/${id}/credentials/clear`, { method: 'POST', body: { revoke } }), // Re-dials and re-discovers tools. The repair path for a server registered // while it happened to be down. reconnect: (id: string) => request(`/mcp-servers/${id}/reconnect`, { method: 'POST' }), diff --git a/frontend/src/components/protocol/ConnectMcpServerDialog.tsx b/frontend/src/components/protocol/ConnectMcpServerDialog.tsx index fa8bca61..b6059c92 100644 --- a/frontend/src/components/protocol/ConnectMcpServerDialog.tsx +++ b/frontend/src/components/protocol/ConnectMcpServerDialog.tsx @@ -1,12 +1,15 @@ import { useState } from 'react' import { useMutation, useQueryClient } from '@tanstack/react-query' -import { Plug, Terminal } from 'lucide-react' +import { KeyRound, Plug, ShieldCheck, Terminal } from 'lucide-react' import { ApiError, mcpServersApi } from '@/api/client' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' import { HUD_ACCENT_RING_CLASSNAME } from '@/lib/utils' +import { authorizeMcpServer } from '@/lib/mcpOAuth' +import { credentialsAreValid, credentialsRecord, emptyCredentialEntry, type CredentialEntry } from '@/lib/mcpCredentials' +import { McpCredentialFields } from './McpCredentialFields' import type { McpServer } from '@/types/mcpServers' // The two transports an MCP client can speak here. Core's MCPTransport also @@ -37,6 +40,15 @@ const TRANSPORTS = [ // than discovered by submitting a command and reading a 422 back. const ALLOWED_EXECUTABLES = 'python, python3, python3.11, python3.12, uv, node, npx, npm' +type AuthMode = 'none' | 'bearer' | 'headers' | 'oauth' | 'env' + +const HTTP_AUTH_MODES: { value: AuthMode; label: string; hint: string }[] = [ + { value: 'none', label: 'None', hint: 'Connect without credentials.' }, + { value: 'bearer', label: 'Bearer token', hint: 'Send an Authorization: Bearer header.' }, + { value: 'headers', label: 'Headers', hint: 'Send one or more API-key or custom authentication headers.' }, + { value: 'oauth', label: 'OAuth', hint: 'Authorize through the server’s standards-based browser flow.' }, +] + // Registers a brand-new MCP server connection the user types in, then hands it // back so the caller can place an MCP Client Tool node bound to it. // @@ -59,11 +71,21 @@ export function ConnectMcpServerDialog({ const [transport, setTransport] = useState('stdio') const [command, setCommand] = useState('') const [url, setUrl] = useState('') + const [authMode, setAuthMode] = useState('none') + const [bearerToken, setBearerToken] = useState('') + const [credentials, setCredentials] = useState([emptyCredentialEntry()]) + const [oauthScope, setOauthScope] = useState('') + const [oauthServer, setOauthServer] = useState(null) + const [oauthPending, setOauthPending] = useState(false) + const [oauthError, setOauthError] = useState(null) const queryClient = useQueryClient() const isStdio = transport === 'stdio' const endpoint = isStdio ? command.trim() : url.trim() - const canSubmit = name.trim().length > 0 && endpoint.length > 0 + const authIsValid = authMode === 'none' || authMode === 'oauth' + || (authMode === 'bearer' && bearerToken.length > 0) + || ((authMode === 'headers' || authMode === 'env') && credentialsAreValid(credentials)) + const canSubmit = name.trim().length > 0 && endpoint.length > 0 && authIsValid const connectMutation = useMutation({ mutationFn: () => @@ -72,9 +94,17 @@ export function ConnectMcpServerDialog({ transport, command: isStdio ? command.trim() : null, url: isStdio ? null : url.trim(), + headers: !isStdio && authMode === 'bearer' + ? { Authorization: `Bearer ${bearerToken}` } + : !isStdio && authMode === 'headers' ? credentialsRecord(credentials) : null, + server_env: isStdio && authMode === 'env' ? credentialsRecord(credentials) : null, }), onSuccess: (server) => { queryClient.invalidateQueries({ queryKey: ['mcp-servers'] }) + if (authMode === 'oauth') { + setOauthServer(server) + return + } onConnected?.(server) reset() onOpenChange(false) @@ -86,9 +116,34 @@ export function ConnectMcpServerDialog({ setTransport('stdio') setCommand('') setUrl('') + setAuthMode('none') + setBearerToken('') + setCredentials([emptyCredentialEntry()]) + setOauthScope('') + setOauthServer(null) + setOauthPending(false) + setOauthError(null) connectMutation.reset() } + async function authorizeOAuth() { + if (!oauthServer || oauthPending) return + setOauthPending(true) + setOauthError(null) + try { + await authorizeMcpServer(oauthServer.id, oauthScope.trim() || undefined) + const connected = await mcpServersApi.get(oauthServer.id) + queryClient.invalidateQueries({ queryKey: ['mcp-servers'] }) + onConnected?.(connected) + reset() + onOpenChange(false) + } catch (error) { + setOauthError(error instanceof Error ? error.message : 'MCP authorization failed.') + } finally { + setOauthPending(false) + } + } + const errorMessage = !connectMutation.isError ? null : connectMutation.error instanceof ApiError && typeof connectMutation.error.detail === 'string' @@ -119,6 +174,7 @@ export function ConnectMcpServerDialog({ if (canSubmit && !connectMutation.isPending) connectMutation.mutate() }} > + {!oauthServer && <>
setTransport(option.value)} - className={`flex cursor-pointer items-center gap-2 rounded-lg border px-3 py-2 text-left text-sm transition-colors ${ + onClick={() => { + setTransport(option.value) + setAuthMode('none') + setCredentials([emptyCredentialEntry()]) + }} + className={`flex cursor-pointer items-center gap-2 rounded-lg border px-3 py-2 text-left text-sm shadow-[0_0_16px_-6px_var(--primary)] transition-colors ${ active ? 'border-primary bg-primary/10 ring-1 ring-primary/40' : 'bg-background hover:bg-muted' }`} > @@ -162,8 +222,9 @@ export function ConnectMcpServerDialog({

{TRANSPORTS.find((t) => t.value === transport)?.hint}

+ } - {isStdio ? ( + {!oauthServer && (isStdio ? (

The server’s streamable HTTP endpoint.

+ ))} + + {!oauthServer && ( +
+ +
+ {(isStdio + ? [ + { value: 'none' as const, label: 'None', hint: 'Launch without additional credentials.' }, + { value: 'env' as const, label: 'Environment', hint: 'Pass encrypted environment credentials to the subprocess.' }, + ] + : HTTP_AUTH_MODES + ).map((option) => ( + + ))} +
+

+ {(isStdio + ? authMode === 'env' ? 'Secrets are encrypted at rest and passed only to this subprocess.' : 'Launch without additional credentials.' + : HTTP_AUTH_MODES.find((mode) => mode.value === authMode)?.hint)} +

+ {authMode === 'bearer' && ( + setBearerToken(event.target.value)} + /> + )} + {(authMode === 'headers' || authMode === 'env') && ( + + )} + {authMode === 'oauth' && ( +
+ setOauthScope(event.target.value)} + /> +

Leave blank to use the scopes advertised by the server.

+
+ )} +
+ )} + + {oauthServer && ( +
+
+ +
+

Authorize {oauthServer.name}

+

The server is saved. Finish authorization in the provider window.

+
+
+ +
)} - {errorMessage &&

{errorMessage}

} + {(errorMessage || oauthError) &&

{oauthError ?? errorMessage}

} - + {!oauthServer && - + } diff --git a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx index 320841e9..cd6f7572 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.test.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.test.tsx @@ -89,7 +89,10 @@ describe('CustomMetricFlow', () => { const user = userEvent.setup() vi.spyOn(mcpServersApi, 'list').mockResolvedValue([{ id: 'server-1', name: 'Quality server', transport: 'stdio', command: 'quality', url: null, status: 'connected', error_message: null, - capabilities: { tools: [{ name: 'score', input_schema: { properties: {} } }, { name: 'explain', input_schema: { properties: {} } }] }, created_at: '', + capabilities: { tools: [{ name: 'score', input_schema: { properties: {} } }, { name: 'explain', input_schema: { properties: {} } }] }, + authentication: { auth_mode: 'none', configured: false, authorization_required: false, static_headers_configured: false, stdio_env_configured: false, stdio_env_names: [] }, + credential_management_allowed: true, + created_at: '', }]) const graph = { nodes: [ diff --git a/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx b/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx new file mode 100644 index 00000000..e300acdc --- /dev/null +++ b/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx @@ -0,0 +1,169 @@ +import { useState } from 'react' +import { useQueryClient } from '@tanstack/react-query' +import { ApiError, mcpServersApi } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' +import { Label } from '@/components/ui/label' +import { authorizeMcpServer } from '@/lib/mcpOAuth' +import { credentialsAreValid, credentialsRecord, emptyCredentialEntry, type CredentialEntry } from '@/lib/mcpCredentials' +import { HUD_ACCENT_RING_CLASSNAME } from '@/lib/utils' +import type { McpServer } from '@/types/mcpServers' +import { McpCredentialFields } from './McpCredentialFields' + +type Mode = 'bearer' | 'headers' | 'env' | 'oauth' + +export function ManageMcpCredentialsDialog({ + server, + open, + onOpenChange, + onUpdated, +}: { + server: McpServer + open: boolean + onOpenChange: (open: boolean) => void + onUpdated: (server: McpServer) => void +}) { + const initialMode: Mode = server.transport === 'stdio' + ? 'env' + : server.authentication.auth_mode === 'oauth' ? 'oauth' : 'bearer' + const [mode, setMode] = useState(initialMode) + const [bearerToken, setBearerToken] = useState('') + const [entries, setEntries] = useState([emptyCredentialEntry()]) + const [scope, setScope] = useState('') + const [pending, setPending] = useState(false) + const [error, setError] = useState(null) + const queryClient = useQueryClient() + + const isHttp = server.transport === 'http' + const canSave = mode === 'oauth' + || (mode === 'bearer' && bearerToken.length > 0) + || ((mode === 'headers' || mode === 'env') && credentialsAreValid(entries)) + const lockedToOAuth = server.authentication.auth_mode === 'oauth' + const lockedToStatic = server.authentication.auth_mode === 'static_headers' + const hasCredentials = server.authentication.configured || server.authentication.authorization_required + + async function finish(updated: McpServer) { + await queryClient.invalidateQueries({ queryKey: ['mcp-servers'] }) + onUpdated(updated) + onOpenChange(false) + } + + async function save() { + if (!canSave || pending) return + setPending(true) + setError(null) + try { + if (mode === 'oauth') { + await authorizeMcpServer(server.id, scope.trim() || undefined) + await finish(await mcpServersApi.get(server.id)) + } else if (mode === 'env') { + await finish(await mcpServersApi.update(server.id, { server_env: credentialsRecord(entries) })) + } else { + const headers = mode === 'bearer' + ? { Authorization: `Bearer ${bearerToken}` } + : credentialsRecord(entries) + await finish(await mcpServersApi.update(server.id, { headers })) + } + } catch (caught) { + setError(caught instanceof ApiError && typeof caught.detail === 'string' + ? caught.detail + : caught instanceof Error ? caught.message : 'Could not update credentials.') + } finally { + setPending(false) + } + } + + async function clear() { + if (pending) return + setPending(true) + setError(null) + try { + await finish(await mcpServersApi.clearCredentials(server.id, server.authentication.auth_mode === 'oauth')) + } catch (caught) { + setError(caught instanceof Error ? caught.message : 'Could not clear credentials.') + } finally { + setPending(false) + } + } + + const modes: { value: Mode; label: string }[] = server.transport === 'stdio' + ? [{ value: 'env', label: 'Environment' }] + : lockedToOAuth + ? [{ value: 'oauth', label: 'OAuth' }] + : lockedToStatic + ? [{ value: 'bearer', label: 'Bearer token' }, { value: 'headers', label: 'Headers' }] + : [{ value: 'bearer', label: 'Bearer token' }, { value: 'headers', label: 'Headers' }, { value: 'oauth', label: 'OAuth' }] + + return ( + + + + Manage MCP credentials + + Saved secret values are never displayed. Enter replacements, reauthorize, or clear them. + + +
+
+ +
+ {modes.map((candidate) => ( + + ))} +
+ {(lockedToOAuth || lockedToStatic) && ( +

Clear the current credentials before switching to a different authentication family.

+ )} +
+ {mode === 'bearer' && ( + setBearerToken(event.target.value)} + /> + )} + {(mode === 'headers' || mode === 'env') && ( + + )} + {mode === 'oauth' && isHttp && ( +
+ setScope(event.target.value)} + /> +

Saving opens the provider authorization window.

+
+ )} + {error &&

{error}

} +
+ + + + + + + ) +} diff --git a/frontend/src/components/protocol/McpCredentialFields.test.ts b/frontend/src/components/protocol/McpCredentialFields.test.ts new file mode 100644 index 00000000..6e3ae35d --- /dev/null +++ b/frontend/src/components/protocol/McpCredentialFields.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { credentialsAreValid, credentialsRecord, type CredentialEntry } from '@/lib/mcpCredentials' + +const entries = (...values: [string, string][]): CredentialEntry[] => + values.map(([name, value], id) => ({ id, name, value })) + +describe('MCP credential fields', () => { + it('builds a replacement map without trimming secret values', () => { + expect(credentialsRecord(entries([' X-API-Key ', ' secret '], ['X-Tenant', 'lab']))).toEqual({ + 'X-API-Key': ' secret ', + 'X-Tenant': 'lab', + }) + }) + + it('requires complete, case-insensitively unique names', () => { + expect(credentialsAreValid(entries(['X-API-Key', 'secret']))).toBe(true) + expect(credentialsAreValid(entries(['X-API-Key', 'one'], ['x-api-key', 'two']))).toBe(false) + expect(credentialsAreValid(entries(['API_TOKEN', '']))).toBe(false) + }) +}) diff --git a/frontend/src/components/protocol/McpCredentialFields.tsx b/frontend/src/components/protocol/McpCredentialFields.tsx new file mode 100644 index 00000000..86ac5f5e --- /dev/null +++ b/frontend/src/components/protocol/McpCredentialFields.tsx @@ -0,0 +1,57 @@ +import { Plus, Trash2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { emptyCredentialEntry, type CredentialEntry } from '@/lib/mcpCredentials' + +export function McpCredentialFields({ + entries, + onChange, + namePlaceholder, +}: { + entries: CredentialEntry[] + onChange: (entries: CredentialEntry[]) => void + namePlaceholder: string +}) { + function patch(id: number, change: Partial) { + onChange(entries.map((entry) => (entry.id === id ? { ...entry, ...change } : entry))) + } + + return ( +
+ {entries.map((entry) => ( +
+ patch(entry.id, { name: event.target.value })} + /> + patch(entry.id, { value: event.target.value })} + /> + +
+ ))} + +
+ ) +} diff --git a/frontend/src/components/protocol/McpToolNodeInspector.tsx b/frontend/src/components/protocol/McpToolNodeInspector.tsx index 3ae0a75f..1b0936f6 100644 --- a/frontend/src/components/protocol/McpToolNodeInspector.tsx +++ b/frontend/src/components/protocol/McpToolNodeInspector.tsx @@ -1,6 +1,7 @@ +import { useState } from 'react' import { nodeAccent } from '@/lib/nodeAccent' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { Plug, RefreshCw, Wrench } from 'lucide-react' +import { KeyRound, Plug, RefreshCw, Wrench } from 'lucide-react' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' @@ -11,6 +12,7 @@ import { EditableNodeTitle } from './EditableNodeTitle' import { FactorBindableField } from './FactorBindableField' import { MCP_CLIENT_TOOL_NODE_TYPE } from './mcpServerCatalog' import { NodeInspectorDialog } from './NodeInspectorDialog' +import { ManageMcpCredentialsDialog } from './ManageMcpCredentialsDialog' import type { McpToolNodeData, ProtocolNode } from '@/types/protocols' // Kept in step with McpToolNode/McpClientToolNode's own accents -- the same @@ -64,6 +66,7 @@ export function McpToolNodeInspector({ }) { const serversQuery = useQuery({ queryKey: ['mcp-servers'], queryFn: () => mcpServersApi.list() }) const queryClient = useQueryClient() + const [credentialsOpen, setCredentialsOpen] = useState(false) const serverId = node?.data.config.server_id ?? null // Re-dials and re-discovers tools, then writes the fresh list onto the node @@ -92,6 +95,7 @@ export function McpToolNodeInspector({ const instructions = selectedServer?.capabilities?.instructions?.trim() const selectedTools = config.tool_names ?? [] const isClientTool = node.type === MCP_CLIENT_TOOL_NODE_TYPE + const canManageCredentials = selectedServer?.credential_management_allowed ?? false const accent = isClientTool ? CLIENT_ACCENT : ACCENT const Icon = isClientTool ? Plug : Wrench @@ -165,16 +169,29 @@ export function McpToolNodeInspector({
- +
+ {canManageCredentials && ( + + )} + +
@@ -186,6 +203,9 @@ export function McpToolNodeInspector({ {selectedServer.status} )} + {selectedServer.authentication.authorization_required && ( + authorization required + )}
{/* dir="rtl" so a long command/URL truncates at the FRONT -- the distinguishing part of either one is at the end. */} @@ -288,6 +308,18 @@ export function McpToolNodeInspector({ )}
+ {selectedServer && canManageCredentials && credentialsOpen && ( + { + const discovered = updated.capabilities?.tools?.map((tool) => tool.name) ?? [] + patchConfig({ tool_names: selectedTools.filter((name) => discovered.includes(name)) }) + }} + /> + )} + ) } diff --git a/frontend/src/lib/mcpCredentials.ts b/frontend/src/lib/mcpCredentials.ts new file mode 100644 index 00000000..6e9b7d24 --- /dev/null +++ b/frontend/src/lib/mcpCredentials.ts @@ -0,0 +1,17 @@ +export type CredentialEntry = { id: number; name: string; value: string } + +let nextEntryId = 1 + +export function emptyCredentialEntry(): CredentialEntry { + return { id: nextEntryId++, name: '', value: '' } +} + +export function credentialsRecord(entries: CredentialEntry[]): Record { + return Object.fromEntries(entries.map((entry) => [entry.name.trim(), entry.value])) +} + +export function credentialsAreValid(entries: CredentialEntry[]): boolean { + if (entries.length === 0 || entries.some((entry) => !entry.name.trim() || !entry.value)) return false + const names = entries.map((entry) => entry.name.trim().toLowerCase()) + return new Set(names).size === names.length +} diff --git a/frontend/src/lib/mcpOAuth.test.ts b/frontend/src/lib/mcpOAuth.test.ts new file mode 100644 index 00000000..1894d843 --- /dev/null +++ b/frontend/src/lib/mcpOAuth.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from 'vitest' +import { postMcpOAuthResult } from './mcpOAuth' + +describe('MCP OAuth callback', () => { + it('posts only safe completion metadata to the opener origin', () => { + const postMessage = vi.fn() + vi.stubGlobal('opener', { postMessage }) + + postMcpOAuthResult(new URLSearchParams('status=success&server_id=server-1')) + + expect(postMessage).toHaveBeenCalledWith( + { type: 'asaree:mcp-oauth', status: 'success', serverId: 'server-1', code: undefined }, + window.location.origin, + ) + vi.unstubAllGlobals() + }) +}) diff --git a/frontend/src/lib/mcpOAuth.ts b/frontend/src/lib/mcpOAuth.ts new file mode 100644 index 00000000..1909bb95 --- /dev/null +++ b/frontend/src/lib/mcpOAuth.ts @@ -0,0 +1,70 @@ +import { mcpServersApi } from '@/api/client' + +const MESSAGE_TYPE = 'asaree:mcp-oauth' + +type OAuthMessage = { + type: typeof MESSAGE_TYPE + status: 'success' | 'error' + serverId?: string + code?: string +} + +export async function authorizeMcpServer(serverId: string, scope?: string): Promise { + const popup = window.open('', 'asaree-mcp-oauth', 'popup,width=640,height=760') + if (!popup) throw new Error('Allow popups to authorize this MCP server.') + const oauthWindow = popup + + let authorization + try { + authorization = await mcpServersApi.beginOAuth(serverId, scope) + } catch (error) { + oauthWindow.close() + throw error + } + oauthWindow.location.assign(authorization.authorization_url) + + await new Promise((resolve, reject) => { + const remaining = Math.max(1_000, new Date(authorization.expires_at).getTime() - Date.now()) + const timeout = window.setTimeout(() => finish(new Error('MCP authorization timed out.')), remaining) + const closed = window.setInterval(() => { + if (oauthWindow.closed) finish(new Error('MCP authorization was cancelled.')) + }, 500) + + function finish(error?: Error) { + window.clearTimeout(timeout) + window.clearInterval(closed) + window.removeEventListener('message', onMessage) + if (!oauthWindow.closed) oauthWindow.close() + if (error) reject(error) + else resolve() + } + + function onMessage(event: MessageEvent) { + if (event.origin !== window.location.origin || event.source !== oauthWindow || event.data?.type !== MESSAGE_TYPE) return + if (event.data.status === 'success' && event.data.serverId === serverId) finish() + else finish(new Error(oauthErrorMessage(event.data.code))) + } + + window.addEventListener('message', onMessage) + }) +} + +export function postMcpOAuthResult(params: URLSearchParams): void { + const status = params.get('status') === 'success' ? 'success' : 'error' + window.opener?.postMessage( + { + type: MESSAGE_TYPE, + status, + serverId: params.get('server_id') ?? undefined, + code: params.get('code') ?? undefined, + } satisfies OAuthMessage, + window.location.origin, + ) +} + +function oauthErrorMessage(code?: string): string { + if (code === 'authorization_rejected') return 'Authorization was rejected by the provider.' + if (code === 'invalid_state') return 'The authorization request expired or was already used.' + if (code === 'token_exchange_failed') return 'The provider rejected the authorization code.' + return 'MCP authorization failed.' +} diff --git a/frontend/src/pages/McpOAuthCallbackPage.tsx b/frontend/src/pages/McpOAuthCallbackPage.tsx new file mode 100644 index 00000000..a5f38802 --- /dev/null +++ b/frontend/src/pages/McpOAuthCallbackPage.tsx @@ -0,0 +1,24 @@ +import { useEffect } from 'react' +import { useSearchParams } from 'react-router-dom' +import { AuthLayout } from '@/components/AuthLayout' +import { postMcpOAuthResult } from '@/lib/mcpOAuth' + +export function McpOAuthCallbackPage() { + const [params] = useSearchParams() + const succeeded = params.get('status') === 'success' + + useEffect(() => { + postMcpOAuthResult(params) + const timer = window.setTimeout(() => window.close(), 150) + return () => window.clearTimeout(timer) + }, [params]) + + return ( + +

You can close this window.

+ + ) +} diff --git a/frontend/src/types/mcpServers.ts b/frontend/src/types/mcpServers.ts index 02224044..117cacd5 100644 --- a/frontend/src/types/mcpServers.ts +++ b/frontend/src/types/mcpServers.ts @@ -27,5 +27,20 @@ export interface McpServer { // such a row fills it in). Not a column of its own for the same reason // `tools` isn't: it's protocol-shaped data, not schema. capabilities: { tools?: McpToolCapability[]; instructions?: string } | null + authentication: { + auth_mode: 'none' | 'static_headers' | 'stdio_env' | 'oauth' + configured: boolean + authorization_required: boolean + static_headers_configured: boolean + stdio_env_configured: boolean + stdio_env_names: string[] + } + credential_management_allowed: boolean created_at: string } + +export interface McpOAuthAuthorization { + authorization_url: string + expires_at: string + transaction_id: string +} diff --git a/pyproject.toml b/pyproject.toml index d4839d0b..138aa7ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ dev = [ ] [tool.uv.sources] -motoro = { git = "https://github.com/EpistasisLab/motoro.git", tag = "v0.8.0" } +motoro = { git = "https://github.com/EpistasisLab/motoro.git", rev = "ca69a35b79413e6291e7759ffe1b9c5e605f6468" } asaree-workspace-core = { path = "workspace-core", editable = true } # Every mcp-servers/ package is declared here, including asaree-sklearn-core # (which the six servers each depend on): uv resolves `tool.uv.sources` from diff --git a/src/asaree/api/mcp_servers.py b/src/asaree/api/mcp_servers.py index e3b0a5c1..cf5018c2 100644 --- a/src/asaree/api/mcp_servers.py +++ b/src/asaree/api/mcp_servers.py @@ -20,13 +20,30 @@ import uuid from datetime import datetime from typing import Any - -from fastapi import APIRouter, HTTPException +from urllib.parse import urlencode + +from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import RedirectResponse +from motoro.mcp.oauth import ( + MCPAuthConfigurationError, + MCPAuthenticationError, + MCPOAuthCallbackError, + MCPOAuthDiscoveryError, + MCPOAuthStateError, + MCPReauthorizationRequiredError, +) from motoro.mcp.registry import get_registry +from motoro.security.mcp_credentials import MCPCredentialValidationError from motoro.services import mcp_service -from motoro.services.mcp_service import MCPServerNameConflictError -from pydantic import BaseModel - +from motoro.services.mcp_service import ( + UNSET, + MCPOAuthClientMetadata, + MCPServerNameConflictError, + MCPServerNotFoundError, +) +from pydantic import BaseModel, Field + +from asaree.config import get_settings from asaree.deps import CurrentUser router = APIRouter(prefix="/mcp-servers", tags=["mcp-servers"]) @@ -58,6 +75,7 @@ class RegisterServerRequest(BaseModel): command: str | None = None url: str | None = None headers: dict[str, str] | None = None + server_env: dict[str, str] | None = None class UpdateServerRequest(BaseModel): @@ -66,6 +84,30 @@ class UpdateServerRequest(BaseModel): command: str | None = None url: str | None = None headers: dict[str, str] | None = None + server_env: dict[str, str] | None = None + + +class AuthenticationStatusResponse(BaseModel): + auth_mode: str + configured: bool + authorization_required: bool + static_headers_configured: bool + stdio_env_configured: bool + stdio_env_names: list[str] + + +class OAuthStartRequest(BaseModel): + scope: str | None = Field(default=None, max_length=2000) + + +class OAuthStartResponse(BaseModel): + authorization_url: str + expires_at: datetime + transaction_id: str + + +class ClearCredentialsRequest(BaseModel): + revoke: bool = False class ServerResponse(BaseModel): @@ -77,6 +119,8 @@ class ServerResponse(BaseModel): status: str error_message: str | None capabilities: dict[str, Any] | None + authentication: AuthenticationStatusResponse + credential_management_allowed: bool created_at: datetime @@ -124,7 +168,10 @@ async def _capabilities_with_tool_annotations(config: Any, *, refresh: bool = Fa return capabilities -async def _to_response(config: Any, *, refresh_annotations: bool = False) -> ServerResponse: +async def _to_response( + config: Any, *, viewer_id: uuid.UUID | None = None, refresh_annotations: bool = False +) -> ServerResponse: + auth = await mcp_service.get_authentication_status(config.id) return ServerResponse( id=config.id, name=config.name, @@ -134,10 +181,33 @@ async def _to_response(config: Any, *, refresh_annotations: bool = False) -> Ser status=config.status.value, error_message=config.error_message, capabilities=await _capabilities_with_tool_annotations(config, refresh=refresh_annotations), + authentication=AuthenticationStatusResponse( + auth_mode=auth.auth_mode, + configured=auth.configured, + authorization_required=auth.authorization_required, + static_headers_configured=auth.static_headers_configured, + stdio_env_configured=auth.stdio_env_configured, + stdio_env_names=list(auth.stdio_env_names), + ), + credential_management_allowed=viewer_id is not None and config.owner_id == viewer_id, created_at=config.created_at, ) +def _raise_auth_http_error(exc: Exception) -> None: + if isinstance(exc, MCPServerNotFoundError): + raise HTTPException(status_code=404, detail="No such server") from exc + if isinstance(exc, MCPReauthorizationRequiredError): + raise HTTPException(status_code=409, detail="MCP OAuth authorization is required") from exc + if isinstance(exc, MCPOAuthDiscoveryError): + raise HTTPException(status_code=502, detail=str(exc)) from exc + if isinstance(exc, (MCPAuthConfigurationError, MCPCredentialValidationError)): + raise HTTPException(status_code=422, detail=str(exc)) from exc + if isinstance(exc, MCPAuthenticationError): + raise HTTPException(status_code=400, detail=str(exc)) from exc + raise exc + + @router.post("", response_model=ServerResponse, status_code=201) async def register_server_endpoint(body: RegisterServerRequest, user: CurrentUser) -> ServerResponse: if await mcp_service.get_server_by_name(body.name, owner_id=user.id) is not None: @@ -149,19 +219,52 @@ async def register_server_endpoint(body: RegisterServerRequest, user: CurrentUse command=body.command, url=body.url, headers=body.headers, + server_env=body.server_env, owner_id=user.id, ) except MCPServerNameConflictError as exc: raise HTTPException(status_code=409, detail="A server with this name already exists") from exc + except (MCPAuthenticationError, MCPCredentialValidationError, MCPServerNotFoundError) as exc: + _raise_auth_http_error(exc) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc - return await _to_response(config, refresh_annotations=True) + return await _to_response(config, viewer_id=user.id, refresh_annotations=True) @router.get("", response_model=list[ServerResponse]) async def list_servers_endpoint(user: CurrentUser) -> list[ServerResponse]: servers = await mcp_service.list_servers(owner_id=user.id) - return list(await asyncio.gather(*(_to_response(s) for s in servers))) + return list(await asyncio.gather(*(_to_response(s, viewer_id=user.id) for s in servers))) + + +@router.get("/oauth/callback", name="complete_mcp_oauth_callback") +async def complete_oauth_callback( + code: str | None = Query(default=None), + state: str | None = Query(default=None), + iss: str | None = Query(default=None), + error: str | None = Query(default=None), +) -> RedirectResponse: + """Consume the provider callback and return to the frontend popup route. + + This endpoint is intentionally unauthenticated: the random, one-use OAuth + state is the callback credential. Motoro binds it to the owner and server. + """ + settings = get_settings() + params: dict[str, str] + if error or not code or not state: + params = {"status": "error", "code": "authorization_rejected"} + else: + try: + config = await mcp_service.complete_oauth_authorization(code=code, state=state, iss=iss) + params = {"status": "success", "server_id": str(config.id)} + except MCPOAuthStateError: + params = {"status": "error", "code": "invalid_state"} + except MCPOAuthCallbackError: + params = {"status": "error", "code": "token_exchange_failed"} + except (MCPAuthenticationError, MCPServerNotFoundError, MCPReauthorizationRequiredError): + params = {"status": "error", "code": "authorization_failed"} + target = f"{settings.frontend_url.rstrip('/')}/mcp/oauth/callback?{urlencode(params)}" + return RedirectResponse(target, status_code=303) @router.get("/{server_id}", response_model=ServerResponse) @@ -169,7 +272,7 @@ async def get_server_endpoint(server_id: uuid.UUID, user: CurrentUser) -> Server config = await mcp_service.get_server(server_id) if config is None or not _readable(config, user): raise HTTPException(status_code=404, detail="No such server") - return await _to_response(config) + return await _to_response(config, viewer_id=user.id) @router.patch("/{server_id}", response_model=ServerResponse) @@ -184,14 +287,58 @@ async def update_server_endpoint(server_id: uuid.UUID, body: UpdateServerRequest transport=body.transport, command=body.command, url=body.url, - headers=body.headers, + headers=body.headers if "headers" in body.model_fields_set else UNSET, + server_env=body.server_env if "server_env" in body.model_fields_set else UNSET, ) except MCPServerNameConflictError as exc: raise HTTPException(status_code=409, detail="A server with this name already exists") from exc + except (MCPAuthenticationError, MCPCredentialValidationError, MCPServerNotFoundError) as exc: + _raise_auth_http_error(exc) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc assert config is not None # existence already checked above - return await _to_response(config, refresh_annotations=True) + return await _to_response(config, viewer_id=user.id, refresh_annotations=True) + + +@router.post("/{server_id}/oauth/start", response_model=OAuthStartResponse) +async def start_oauth_endpoint( + server_id: uuid.UUID, body: OAuthStartRequest, user: CurrentUser +) -> OAuthStartResponse: + settings = get_settings() + try: + authorization = await mcp_service.begin_oauth_authorization( + server_id, + redirect_uri=settings.mcp_oauth_callback_url, + client_metadata=MCPOAuthClientMetadata( + client_name="ASAREE", + scope=body.scope, + client_uri=settings.frontend_url, + ), + client_metadata_url=settings.mcp_oauth_client_metadata_url, + owner_id=user.id, + ) + except (MCPAuthenticationError, MCPCredentialValidationError, MCPServerNotFoundError) as exc: + _raise_auth_http_error(exc) + raise AssertionError("unreachable") from exc + return OAuthStartResponse( + authorization_url=authorization.authorization_url, + expires_at=authorization.expires_at, + transaction_id=authorization.transaction_id, + ) + + +@router.post("/{server_id}/credentials/clear", response_model=ServerResponse) +async def clear_credentials_endpoint( + server_id: uuid.UUID, body: ClearCredentialsRequest, user: CurrentUser +) -> ServerResponse: + try: + if body.revoke: + await mcp_service.clear_oauth_credentials(server_id, owner_id=user.id, revoke=True) + config = await mcp_service.clear_server_credentials(server_id, owner_id=user.id) + except (MCPAuthenticationError, MCPServerNotFoundError) as exc: + _raise_auth_http_error(exc) + raise AssertionError("unreachable") from exc + return await _to_response(config, viewer_id=user.id) @router.delete("/{server_id}", status_code=204) @@ -209,7 +356,7 @@ async def refresh_server_endpoint(server_id: uuid.UUID, user: CurrentUser) -> Se raise HTTPException(status_code=404, detail="No such server") config = await mcp_service.refresh_server(server_id) assert config is not None - return await _to_response(config, refresh_annotations=True) + return await _to_response(config, viewer_id=user.id, refresh_annotations=True) @router.post("/{server_id}/reconnect", response_model=ServerResponse) @@ -219,7 +366,7 @@ async def reconnect_server_endpoint(server_id: uuid.UUID, user: CurrentUser) -> raise HTTPException(status_code=404, detail="No such server") config = await mcp_service.reconnect_server(server_id) assert config is not None - return await _to_response(config, refresh_annotations=True) + return await _to_response(config, viewer_id=user.id, refresh_annotations=True) @router.post("/{server_id}/tools/{tool_name}/call", response_model=CallToolResponse) @@ -238,6 +385,8 @@ async def call_tool_endpoint( raise HTTPException(status_code=404, detail="No such server") try: outcome = await mcp_service.call_server_tool(server_id, tool_name, body.arguments) + except MCPReauthorizationRequiredError as exc: + raise HTTPException(status_code=409, detail="MCP OAuth authorization is required") from exc except RuntimeError as exc: raise HTTPException(status_code=502, detail=str(exc)) from exc if outcome is None: diff --git a/src/asaree/config.py b/src/asaree/config.py index db1f09fd..5363e4e5 100644 --- a/src/asaree/config.py +++ b/src/asaree/config.py @@ -45,6 +45,17 @@ class AsareeSettings(CoreSettings): # browsers refuse to combine with "*". Defaults to the Vite dev server. cors_allowed_origins: str = "http://localhost:5173" + # MCP OAuth uses an exact, server-reachable callback URI. Keep it explicit + # rather than deriving it from a request Host header, which is unreliable + # behind reverse proxies and unsafe as OAuth client metadata. + mcp_oauth_callback_url: str = "http://localhost:8000/api/mcp-servers/oauth/callback" + # Where the callback sends the browser after Motoro has consumed the code. + # The small frontend route reports the result to the opener and closes. + frontend_url: str = "http://localhost:5173" + # Optional HTTPS Client ID Metadata Document (CIMD). When absent, Motoro + # falls back to dynamic client registration where the provider supports it. + mcp_oauth_client_metadata_url: str | None = None + # Local disk, matching ARES's current approach — see # project_plan/core_asaree_use_case.md §9: metadata (this database) and # hashes are what matter; where the bytes physically live is a separate, diff --git a/tests/test_mcp_server_auth.py b/tests/test_mcp_server_auth.py new file mode 100644 index 00000000..711faa70 --- /dev/null +++ b/tests/test_mcp_server_auth.py @@ -0,0 +1,172 @@ +import uuid +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any, cast + +import pytest +from motoro.services.mcp_service import UNSET + +from asaree.api import mcp_servers + + +def _config(owner_id: uuid.UUID) -> SimpleNamespace: + return SimpleNamespace(id=uuid.uuid4(), owner_id=owner_id) + + +@pytest.mark.asyncio +async def test_update_leaves_omitted_credentials_unchanged(monkeypatch: pytest.MonkeyPatch) -> None: + owner_id = uuid.uuid4() + config = _config(owner_id) + captured: dict[str, object] = {} + + async def update(server_id: uuid.UUID, **kwargs: object) -> SimpleNamespace: + captured.update(kwargs) + return config + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.get_server", lambda _server_id: _async(config)) + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.update_server", update) + monkeypatch.setattr(mcp_servers, "_to_response", lambda _config, **_kwargs: _async("response")) + + result = cast(Any, await mcp_servers.update_server_endpoint( + config.id, + mcp_servers.UpdateServerRequest(name="renamed"), + cast(Any, SimpleNamespace(id=owner_id)), + )) + + assert result == "response" + assert captured["headers"] is UNSET + assert captured["server_env"] is UNSET + + +@pytest.mark.asyncio +async def test_update_passes_explicit_credential_clear(monkeypatch: pytest.MonkeyPatch) -> None: + owner_id = uuid.uuid4() + config = _config(owner_id) + captured: dict[str, object] = {} + + async def update(server_id: uuid.UUID, **kwargs: object) -> SimpleNamespace: + captured.update(kwargs) + return config + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.get_server", lambda _server_id: _async(config)) + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.update_server", update) + monkeypatch.setattr(mcp_servers, "_to_response", lambda _config, **_kwargs: _async("response")) + + await mcp_servers.update_server_endpoint( + config.id, + mcp_servers.UpdateServerRequest(headers=None, server_env={}), + cast(Any, SimpleNamespace(id=owner_id)), + ) + + assert captured["headers"] is None + assert captured["server_env"] == {} + + +@pytest.mark.asyncio +async def test_oauth_start_uses_configured_callback_and_owner(monkeypatch: pytest.MonkeyPatch) -> None: + owner_id = uuid.uuid4() + server_id = uuid.uuid4() + captured: dict[str, object] = {} + + async def begin(server_id_arg: uuid.UUID, **kwargs: object) -> SimpleNamespace: + captured["server_id"] = server_id_arg + captured.update(kwargs) + return SimpleNamespace( + authorization_url="https://auth.example/authorize", + expires_at=datetime.now(UTC), + transaction_id="transaction", + ) + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.begin_oauth_authorization", begin) + monkeypatch.setattr( + mcp_servers, + "get_settings", + lambda: SimpleNamespace( + mcp_oauth_callback_url="https://asaree.example/api/mcp-servers/oauth/callback", + mcp_oauth_client_metadata_url=None, + frontend_url="https://asaree.example", + ), + ) + + response = await mcp_servers.start_oauth_endpoint( + server_id, + mcp_servers.OAuthStartRequest(scope="tools.read"), + cast(Any, SimpleNamespace(id=owner_id)), + ) + + assert response.authorization_url == "https://auth.example/authorize" + assert captured["server_id"] == server_id + assert captured["redirect_uri"] == "https://asaree.example/api/mcp-servers/oauth/callback" + assert captured["owner_id"] == owner_id + assert cast(Any, captured["client_metadata"]).scope == "tools.read" + + +@pytest.mark.asyncio +async def test_oauth_callback_completes_and_redirects_without_exposing_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + server_id = uuid.uuid4() + captured: dict[str, object] = {} + + async def complete(**kwargs: object) -> SimpleNamespace: + captured.update(kwargs) + return SimpleNamespace(id=server_id) + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.complete_oauth_authorization", complete) + monkeypatch.setattr(mcp_servers, "get_settings", lambda: SimpleNamespace(frontend_url="https://asaree.example")) + + response = await mcp_servers.complete_oauth_callback( + code="secret-code", state="secret-state", iss="https://issuer", error=None + ) + + assert response.status_code == 303 + assert response.headers["location"] == ( + f"https://asaree.example/mcp/oauth/callback?status=success&server_id={server_id}" + ) + assert "secret-code" not in response.headers["location"] + assert captured == {"code": "secret-code", "state": "secret-state", "iss": "https://issuer"} + + +@pytest.mark.asyncio +async def test_server_response_exposes_only_safe_authentication_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + server_id = uuid.uuid4() + owner_id = uuid.uuid4() + config = SimpleNamespace( + id=server_id, + owner_id=owner_id, + name="private-server", + transport=SimpleNamespace(value="http"), + command=None, + url="https://mcp.example/mcp", + status=SimpleNamespace(value="connected"), + error_message=None, + capabilities={"tools": []}, + created_at=datetime.now(UTC), + headers_encrypted="encrypted-secret-material", + ) + status = SimpleNamespace( + auth_mode="static_headers", + configured=True, + authorization_required=False, + static_headers_configured=True, + stdio_env_configured=False, + stdio_env_names=(), + ) + monkeypatch.setattr( + "asaree.api.mcp_servers.mcp_service.get_authentication_status", + lambda _server_id: _async(status), + ) + + response = await mcp_servers._to_response(config, viewer_id=owner_id) + payload = response.model_dump(mode="json") + + assert payload["authentication"]["auth_mode"] == "static_headers" + assert payload["credential_management_allowed"] is True + assert "headers" not in payload + assert "encrypted-secret-material" not in str(payload) + + +async def _async(value: object) -> object: + return value diff --git a/uv.lock b/uv.lock index 3ec1cd2d..07ddd87a 100644 --- a/uv.lock +++ b/uv.lock @@ -252,7 +252,7 @@ requires-dist = [ { name = "bcrypt", specifier = ">=4.2.0" }, { name = "email-validator", specifier = ">=2.2.0" }, { name = "fastapi", specifier = ">=0.121.0" }, - { name = "motoro", git = "https://github.com/EpistasisLab/motoro.git?tag=v0.8.0" }, + { name = "motoro", git = "https://github.com/EpistasisLab/motoro.git?rev=ca69a35b79413e6291e7759ffe1b9c5e605f6468" }, { name = "pandas", specifier = ">=2.2.0" }, { name = "pyarrow", specifier = ">=18.0.0" }, { name = "pydantic", specifier = ">=2.10.0" }, @@ -1877,7 +1877,7 @@ wheels = [ [[package]] name = "motoro" version = "0.8.0" -source = { git = "https://github.com/EpistasisLab/motoro.git?tag=v0.8.0#123cdbbe273d88e39bd5cfaeaf47b63f80c60370" } +source = { git = "https://github.com/EpistasisLab/motoro.git?rev=ca69a35b79413e6291e7759ffe1b9c5e605f6468#ca69a35b79413e6291e7759ffe1b9c5e605f6468" } dependencies = [ { name = "alembic" }, { name = "asyncpg" }, From 3f1cc2d6bd164e3f0d95a528f22f2c37c00abb16 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 21:48:22 -0700 Subject: [PATCH 09/18] chore: pin Motoro v0.9.0 --- pyproject.toml | 2 +- uv.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 138aa7ed..9cbc1927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,7 @@ dev = [ ] [tool.uv.sources] -motoro = { git = "https://github.com/EpistasisLab/motoro.git", rev = "ca69a35b79413e6291e7759ffe1b9c5e605f6468" } +motoro = { git = "https://github.com/EpistasisLab/motoro.git", tag = "v0.9.0" } asaree-workspace-core = { path = "workspace-core", editable = true } # Every mcp-servers/ package is declared here, including asaree-sklearn-core # (which the six servers each depend on): uv resolves `tool.uv.sources` from diff --git a/uv.lock b/uv.lock index 07ddd87a..98d35a9e 100644 --- a/uv.lock +++ b/uv.lock @@ -252,7 +252,7 @@ requires-dist = [ { name = "bcrypt", specifier = ">=4.2.0" }, { name = "email-validator", specifier = ">=2.2.0" }, { name = "fastapi", specifier = ">=0.121.0" }, - { name = "motoro", git = "https://github.com/EpistasisLab/motoro.git?rev=ca69a35b79413e6291e7759ffe1b9c5e605f6468" }, + { name = "motoro", git = "https://github.com/EpistasisLab/motoro.git?tag=v0.9.0" }, { name = "pandas", specifier = ">=2.2.0" }, { name = "pyarrow", specifier = ">=18.0.0" }, { name = "pydantic", specifier = ">=2.10.0" }, @@ -1876,8 +1876,8 @@ wheels = [ [[package]] name = "motoro" -version = "0.8.0" -source = { git = "https://github.com/EpistasisLab/motoro.git?rev=ca69a35b79413e6291e7759ffe1b9c5e605f6468#ca69a35b79413e6291e7759ffe1b9c5e605f6468" } +version = "0.9.0" +source = { git = "https://github.com/EpistasisLab/motoro.git?tag=v0.9.0#0dec7882afd5002eab0590899f7435cd822b9d6c" } dependencies = [ { name = "alembic" }, { name = "asyncpg" }, From 9c3fef18c19d77b525e7f80e7b94976e011aac4c Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 21:55:30 -0700 Subject: [PATCH 10/18] Version myocardial use case for ASAREE v0.8.0 --- publications/BDM/README.md | 25 +- publications/BDM/import_use_case.py | 14 +- .../BDM/myocardial-anthropic-v0.2.0.json | 1027 ----------------- ....json => myocardial-anthropic-v0.8.0.json} | 0 .../BDM/myocardial-azure-foundry-v0.2.0.json | 1027 ----------------- ...n => myocardial-azure-foundry-v0.8.0.json} | 0 .../BDM/myocardial-openai-v0.2.0.json | 1027 ----------------- ...est.json => myocardial-openai-v0.8.0.json} | 0 8 files changed, 15 insertions(+), 3105 deletions(-) delete mode 100644 publications/BDM/myocardial-anthropic-v0.2.0.json rename publications/BDM/{myocardial-anthropic-latest.json => myocardial-anthropic-v0.8.0.json} (100%) delete mode 100644 publications/BDM/myocardial-azure-foundry-v0.2.0.json rename publications/BDM/{myocardial-azure-foundry-latest.json => myocardial-azure-foundry-v0.8.0.json} (100%) delete mode 100644 publications/BDM/myocardial-openai-v0.2.0.json rename publications/BDM/{myocardial-openai-latest.json => myocardial-openai-v0.8.0.json} (100%) diff --git a/publications/BDM/README.md b/publications/BDM/README.md index b1b045d4..340ae2ff 100644 --- a/publications/BDM/README.md +++ b/publications/BDM/README.md @@ -12,10 +12,9 @@ protocol shape, on a dataset anyone can download. | File | | | --- | --- | -| `myocardial-azure-foundry-latest.json` | The experiment, wired to **Azure Foundry** — what `import_use_case.py` imports by default | -| `myocardial-anthropic-latest.json` | The same, wired to **Anthropic** | -| `myocardial-openai-latest.json` | The same, wired to **OpenAI** | -| `myocardial-*-v0.2.0.json` | Archival graphs from the original runs — see "Versions" below | +| `myocardial-azure-foundry-v0.8.0.json` | The experiment, wired to **Azure Foundry** — what `import_use_case.py` imports by default | +| `myocardial-anthropic-v0.8.0.json` | The same, wired to **Anthropic** | +| `myocardial-openai-v0.8.0.json` | The same, wired to **OpenAI** | | `mi_ZSN.csv` | The dataset — 1700 admissions × 111 features, target `mi_ZSN` | | `dict_ZSN.json` | The data dictionary for those 111 columns | | `stats/` | The paper's analysis scripts and outputs for the spinal runs (not part of this walkthrough) | @@ -28,9 +27,9 @@ have: | File | Model factor | Effort factor | Design size | | --- | --- | --- | --- | -| `myocardial-anthropic-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | -| `myocardial-openai-latest.json` | `gpt-5-mini`, `gpt-5` | `medium`, `high` | 8 cells / 80 replicates | -| `myocardial-azure-foundry-latest.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | +| `myocardial-anthropic-v0.8.0.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | +| `myocardial-openai-v0.8.0.json` | `gpt-5-mini`, `gpt-5` | `medium`, `high` | 8 cells / 80 replicates | +| `myocardial-azure-foundry-v0.8.0.json` | `claude-sonnet-5`, `claude-opus-5` | `medium`, `xhigh` | 8 cells / 80 replicates | All three are 2 × 2 × 2 designs (model × effort × critic on/off) at 10 replicates, with the smaller/larger model of a family at the middle and top of @@ -45,14 +44,10 @@ declared in `motoro.services.model_capabilities`, not a per-provider one, so if you swap a model in, check which of the two the node then offers: an effort factor bound to a temperature-based model varies nothing at runtime, silently. -## Versions +## ASAREE v0.8.0 execution contract -Each provider variant ships twice. `-v0.2.0` is the graph retained from the -original runs. Don't modernize it: its value is as a historical artifact, not -as a v0.8.0 import target. - -`-latest` is the maintained v0.8.0 copy — the same factors, replicates, agents, -and critic gates, brought up to the current execution contract: +The three provider variants use the same factors, replicates, agents, and +critic gates, with these v0.8.0 execution details: - **Dataset connector.** Its edges use the current `dataset` handle rather than the legacy `resource` spelling, and the Dataset node now sits *above* the @@ -120,7 +115,7 @@ provider file that matches the credential you will use: ```bash uv run --with ./sdk python publications/BDM/import_use_case.py \ - publications/BDM/myocardial-openai-latest.json + publications/BDM/myocardial-openai-v0.8.0.json ``` The helper registers and splits the dataset, maps deployment-specific dataset diff --git a/publications/BDM/import_use_case.py b/publications/BDM/import_use_case.py index e976b10b..0d9ee6c8 100644 --- a/publications/BDM/import_use_case.py +++ b/publications/BDM/import_use_case.py @@ -3,12 +3,11 @@ ASAREE_BASE_URL=http://localhost:8000 ASAREE_API_KEY=... \ uv run --with ./sdk python publications/BDM/import_use_case.py \ - [myocardial-anthropic-latest.json] + [myocardial-anthropic-v0.8.0.json] The API-side equivalent of README.md's GUI walkthrough, for rebuilding the -experiment from scratch repeatedly. It accepts the maintained `-latest` -provider variants and defaults to Azure Foundry. The `-v0.2.0` files are -archival artifacts of the paper's original runs, not v0.8.0 import targets. +experiment from scratch repeatedly. It accepts the v0.8.0 provider variants +and defaults to Azure Foundry. Idempotent: run it again and it updates the existing experiment rather than creating a second copy. @@ -36,10 +35,7 @@ HERE = Path(__file__).resolve().parent -# The maintained one, not the frozen one: `-v0.2.0` is the exact graph the -# paper's runs used and is kept byte-stable for reproduction, while `-latest` -# tracks the current canvas (see README.md). -DEFAULT_USE_CASE_FILE = HERE / "myocardial-azure-foundry-latest.json" +DEFAULT_USE_CASE_FILE = HERE / "myocardial-azure-foundry-v0.8.0.json" DATA_FILE = HERE / "mi_ZSN.csv" DICTIONARY_FILE = HERE / "dict_ZSN.json" @@ -109,7 +105,7 @@ def main(argv: list[str]) -> int: measurement_plan = use_case.get("measurement_plan") if not isinstance(measurement_plan, dict): print( - "ERROR: this is an archival definition without a v0.8.0 measurement plan; choose a -latest file.", + "ERROR: this definition lacks the v0.8.0 measurement plan.", file=sys.stderr, ) return 2 diff --git a/publications/BDM/myocardial-anthropic-v0.2.0.json b/publications/BDM/myocardial-anthropic-v0.2.0.json deleted file mode 100644 index dd8d1755..00000000 --- a/publications/BDM/myocardial-anthropic-v0.2.0.json +++ /dev/null @@ -1,1027 +0,0 @@ -{ - "name": "Myocardial Infarction Use Case (Anthropic)", - "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", - "graph": { - "nodes": [ - { - "id": "llm-shared", - "type": "model_anthropic", - "position": { - "x": 1140, - "y": 680 - }, - "data": { - "label": "Anthropic (shared)", - "config": { - "provider": "anthropic", - "model": "claude-sonnet-5", - "temperature": null, - "effort": "medium", - "max_tokens": 128000 - }, - "factor_bindings": { - "config.model": "Anthropic:Model", - "config.effort": "Anthropic:Effort" - } - } - }, - { - "id": "dataset-mi", - "type": "dataset", - "position": { - "x": 724, - "y": 460 - }, - "data": { - "label": "Myocardial Infarction Dataset", - "config": { - "dataset_id": "56d2bbd7-624f-44e3-bd1f-a070b41432b1", - "dataset_name": "myocardial_infarction", - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-dc", - "type": "agent", - "position": { - "x": 0, - "y": 0 - }, - "data": { - "label": "MI-DC", - "config": { - "prompt": "", - "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: identify THIS cell's workspace; pass them to\n open_workspace. Your workspace identity then travels with every tool call\n automatically, so you never pass a workspace id or dataset id to the cleaning tools.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(experiment_id, cell_label, name, stage=\"dc\"): STEP 1, once \u2014 open/\n resume this cell's workspace AND stage your scratch input; every later call is\n scoped to it automatically. Always pass stage=\"dc\" \u2014 omitting it means your\n other tools will find no scratch input and fail.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"dc\").\n It opens (or resumes) this cell's workspace and stages your scratch input. When you need a column's meaning, type, or\n codes, call get_data_dictionary(name, columns=\"col1,col2\") for just those\n columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-dc", - "type": "critic_gate", - "position": { - "x": 300, - "y": 0 - }, - "data": { - "label": "Critic (DC)", - "config": { - "name": "dc-critic", - "goal": "Review MI-DC's cleaning decisions for this stage only.", - "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-dc", - "type": "pattern_reason_act", - "position": { - "x": 8, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-dc", - "type": "mcp_tool", - "position": { - "x": -16, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-dc", - "type": "mcp_tool", - "position": { - "x": 164, - "y": 240 - }, - "data": { - "label": "EDA (get_data_dictionary)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-dc", - "type": "mcp_tool", - "position": { - "x": 344, - "y": 240 - }, - "data": { - "label": "DC (inspect/coerce/impute)", - "config": { - "server_id": "80699456-f1cd-4957-b97b-6dd0e2013cfb", - "server_name": "asaree-sklearn-dc", - "tool_names": [ - "inspect_columns", - "apply_coercions", - "drop_and_impute" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fte", - "type": "agent", - "position": { - "x": 560, - "y": 0 - }, - "data": { - "label": "MI-FTE", - "config": { - "prompt": "", - "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id to build_features and fit_preprocessor.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fte\").\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT\n pre-clean it; it is already clean and complete.\n get_data_dictionary(name, columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fte", - "type": "critic_gate", - "position": { - "x": 860, - "y": 0 - }, - "data": { - "label": "Critic (FTE)", - "config": { - "name": "fte-critic", - "goal": "Review MI-FTE's engineering/encoding decisions for this stage only.", - "description": "Shared Critic reviewing the FTE stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FTE (feature transformation/engineering) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (encoders, and any statistic used by an engineered feature such as a group median or bin edge) is computed on the training split only; the test split is never touched\n- engineered features are emitted as a TYPED, REPLAYABLE recipe (each entry has an op, its inputs, and an explicit deterministic output name), not as one-shot mutations of the training frame\n- every train-derived statistic an engineered feature depends on (bin edges, group median, training prevalence) is FROZEN as an explicit recipe param computed on the training fold, so the same value is applied to test without recomputing (this is the train/test leakage + skew guard)\n- engineered output names are explicit and deterministic so the FS selector and the test partition align on the same column names; group aggregates declare a fallback for group values unseen at fit time\n- encoding appropriateness: each column has an encoding decision (binarize, ordinal with a stated order, one-hot, leave-numeric, or bin) justified from column semantics OR data spread, not a blanket rule\n- every engineered feature carries a one-line rationale naming a specific reason it should help THIS target, not a generic statement\n- engineering is proportionate: the agent engineered as much as is justified and no more; the critic flags both bloat (features with no specific hypothesis) and under-engineering (obvious, well-supported features left out). Judged on rationale quality, not a count\n- interactions (multiplicative or logical) are individually reasoned, not a blanket multiply of all pairs\n- clinical meta-features, if any, are clinically coherent and non-leaky; any composite or index does NOT silently discard component signal (components are retained alongside the composite unless their removal is justified)\n- the engineered + encoded matrix is committed to the workspace as v2_fte, fully numeric and model-ready: build_features (recipe) then fit_preprocessor (encoding_map) are BOTH called, so the encoded matrix is the committed output the FS stage and the runner read (workspace identity is ambient; persistence is automatic, not a save_artifact step)\n- the per-column encoding_map was passed to fit_preprocessor via encoding_map_json (not just described): the map the agent reports equals the map it fit, so test is encoded with the same per-column decisions and handle_unknown protects against unseen test categories\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: engineering emitted as one-shot frame mutations instead of a typed,\n replayable recipe; a train-derived statistic (bin edge, group median,\n prevalence) recomputed at transform time instead of frozen into the recipe;\n non-deterministic engineered names that would desync the FS selector on test; a\n group aggregate with no unseen-group fallback; lossy composites that discard\n component signal without justification; leaky meta-features computed downstream\n of the outcome; generic rationales; bloat or under-engineering; build_features\n or fit_preprocessor did not commit the encoded v2_fte matrix, or the encoding_map\n only described but not passed via encoding_map_json.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fte\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fte", - "type": "pattern_reason_act", - "position": { - "x": 568, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fte", - "type": "mcp_tool", - "position": { - "x": 544, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fte", - "type": "mcp_tool", - "position": { - "x": 724, - "y": 240 - }, - "data": { - "label": "EDA (dictionary + distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary", - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fte", - "type": "mcp_tool", - "position": { - "x": 904, - "y": 240 - }, - "data": { - "label": "FTE (build/fit)", - "config": { - "server_id": "03d4b07d-ffdc-497c-8779-a79126ada1db", - "server_name": "asaree-sklearn-fte", - "tool_names": [ - "build_features", - "fit_preprocessor" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fs", - "type": "agent", - "position": { - "x": 1120, - "y": 0 - }, - "data": { - "label": "MI-FS", - "config": { - "prompt": "", - "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id; the FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fs\").\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a\n preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fs", - "type": "critic_gate", - "position": { - "x": 1420, - "y": 0 - }, - "data": { - "label": "Critic (FS)", - "config": { - "name": "fs-critic", - "goal": "Review MI-FS's feature-selection decisions for this stage only.", - "description": "Shared Critic reviewing the FS stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FS (feature selection) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- profiling completeness: every retained column is profiled with dtype, n_unique, and summary stats (data arrives clean and already encoded from DC/FTE; missingness and encoding are NOT this stage's job)\n- selection runs on the encoded feature space the workspace already holds (v2_fte), so the selected names align with the matrix the runner scores; no preprocessor is re-applied here (the input is already model-ready)\n- variance filter: a variance threshold (and the method) is stated and low/near-zero-variance features are removed against it\n- correlation filter: highly correlated pairs are found (method + |rho| threshold stated) and one of each pair is removed with a defensible UNSUPERVISED tie-break (e.g. lower missingness then higher variance)\n- supervised filter is CORRECT for the task: f_classif or mutual_info_classif for classification (NOT f_regression, which assumes a continuous target), and is rank-based / conservative (keep top-k) rather than an aggressive p-value cut that could discard features carrying signal only in interaction\n- selection covers the FULL expanded space: original survivors and FTE meta-features are filtered on equal footing; nothing bypasses selection\n- observed class distribution is checked against task_brief (expected_positive_rate for binary, num_classes for multiclass)\n- no leakage: all fits (variance, correlation, supervised filter) are computed on the training split only; the test split is never touched\n- the selected features are committed to the workspace as v3_fs by the final selection call (workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed columns must be a subset of the v2_fte columns\n- selection is proportionate: it did not strip features the downstream tree model could plausibly exploit, nor retain obvious redundancy\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: a cut so aggressive it likely discards interaction-only signal the tree\n could use; a correlation tie-break that leaks the label; meta-features exempted\n from selection; unstated thresholds; the final selection call did not commit\n v3_fs; any fit touching the test split.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fs\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fs", - "type": "pattern_reason_act", - "position": { - "x": 1128, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fs", - "type": "mcp_tool", - "position": { - "x": 1104, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fs", - "type": "mcp_tool", - "position": { - "x": 1284, - "y": 240 - }, - "data": { - "label": "EDA (get_feature_distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fs", - "type": "mcp_tool", - "position": { - "x": 1464, - "y": 240 - }, - "data": { - "label": "FS (variance/corr/supervised/balance)", - "config": { - "server_id": "570f42e8-2eff-42ce-8d3f-733b9ee04ff2", - "server_name": "asaree-sklearn-fs", - "tool_names": [ - "variance_filter", - "compute_correlations", - "supervised_filter", - "check_class_balance" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-mlm", - "type": "agent", - "position": { - "x": 1680, - "y": 0 - }, - "data": { - "label": "MI-MLM", - "config": { - "prompt": "", - "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", - "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-mlm", - "type": "critic_gate", - "position": { - "x": 1980, - "y": 0 - }, - "data": { - "label": "Critic (MLM)", - "config": { - "name": "mlm-critic", - "goal": "Review MI-MLM's proposed hyperparameter payload for this stage only.", - "description": "Shared Critic reviewing the MLM stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the MLM (hyperparameter planning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- the payload is ONE JSON object with exactly the keys search_space, fixed_params, n_trials, and rationale; it is not code and does not attempt to set any harness-fixed policy (CV folds, seeds, objective metric, calibration, threshold, or scale_pos_weight)\n- schema validity: search_space tunes at least 3 DISTINCT params drawn from the nine allowed XGBoost params, each with low < high and [low, high] inside that param's hard bounds; fixed_params values are inside their hard bounds; every one of the nine params appears exactly once across search_space and fixed_params (none unspecified, none in both); n_trials is an integer in [1, 100]\n- the choices are defensible for class-imbalanced target optimized on PR-AUC: the search is not degenerate (ranges not collapsed to a point, not the full hard bound on everything), depth and regularization are not pinned to values that force gross over- or under-fitting, and n_trials is adequate for the number of tuned dimensions\n- every non-default choice carries a specific one-line rationale tied to THIS imbalanced PR-AUC task, not a generic statement\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: the output is not a single valid JSON payload; search_space tunes fewer\n than 3 params or names a param outside the nine allowed; a range falls\n outside its hard bounds or has low >= high; a param is unspecified or appears\n in both search_space and fixed_params; n_trials outside [1, 100]; the agent\n tries to set harness-fixed policy (CV folds, seeds, objective metric,\n calibration, threshold, scale_pos_weight); a degenerate search (point ranges\n or full-bound-on-all) or rationales generic rather than tied to the\n imbalanced PR-AUC target.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"mlm\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-mlm", - "type": "pattern_single_agent_baseline", - "position": { - "x": 1688, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 10, - "stop_on_first_success": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-score", - "type": "agent", - "position": { - "x": 2240, - "y": 0 - }, - "data": { - "label": "MI-Score", - "config": { - "prompt": "", - "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with code set to the wired Script node's text verbatim and payload_json set to the approved payload (as compact JSON) from the upstream context -- do not alter either. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", - "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", - "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "pattern-score", - "type": "pattern_reason_act", - "position": { - "x": 2248, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-model-score", - "type": "mcp_tool", - "position": { - "x": 2314, - "y": 240 - }, - "data": { - "label": "Model (run_model_script)", - "config": { - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "server_name": "asaree-sklearn-model", - "tool_names": [ - "run_model_script" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "script-model", - "type": "script", - "position": { - "x": 2494, - "y": 240 - }, - "data": { - "label": "Fixed Model Script", - "config": { - "name": "fixed_model_script", - "language": "python", - "code": "CV_FOLDS = 5\nCV_FOLD_SEED = 20260701\nOPTUNA_SAMPLER_SEED = 20260702\nXGB_SEED = 20260703\nVAL_SPLIT_SEED = 20260704\nVAL_FRACTION = 0.15\n\n# ==================================================================\n# ASAREE fixed modeling script (harness-owned; the MLM agent does NOT\n# write this). The agent supplies only the typed payload, bound as hp.\n# Everything here is deterministic given hp and the seeds baked above,\n# so replicate variance reflects LLM stochasticity in the payload only.\n#\n# Pre-bound by run_model_script: hp, X_train, y_train, np, pd. The test\n# split is NEVER bound; this script defines predict_proba(X) and\n# chosen_threshold, and the runner scores the held-out 30 itself.\n#\n# Harness-fixed policy (not agent-controllable):\n# inner CV 5-fold StratifiedKFold, fixed global fold assignment\n# objective PR-AUC (average_precision) on the training folds only\n# pos weighting scale_pos_weight = n_neg/n_pos on the fitted rows\n# calibration isotonic on one held-out training val fold\n# threshold argmax-F1 on the val fold; Brier reported on the val fold\n# no early stopping\n# Collaborator note: isotonic is monotonic, so calibration cannot change\n# PR-AUC or ROC-AUC (both rank-based); it only moves Brier and the\n# threshold-dependent secondaries, none of which is a tested outcome. The\n# scored model is fit on train-minus-val, so its PR-AUC/ROC-AUC equal the\n# base model ranking and the descriptive-only claim holds exactly.\n# ==================================================================\nimport optuna\nfrom optuna.samplers import TPESampler\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import average_precision_score, f1_score, brier_score_loss\nfrom sklearn.isotonic import IsotonicRegression\nfrom xgboost import XGBClassifier\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\n_INT_PARAMS = {\"n_estimators\", \"max_depth\"}\n_LOG_PARAMS = {\"learning_rate\", \"reg_lambda\", \"reg_alpha\"}\n\n_search = {}\nfor _e in hp[\"search_space\"]:\n _search[_e[\"param\"]] = (float(_e[\"low\"]), float(_e[\"high\"]))\n_fixed = dict(hp[\"fixed_params\"])\nfor _p in _INT_PARAMS:\n if _p in _fixed:\n _fixed[_p] = int(round(float(_fixed[_p])))\n_n_trials = int(hp[\"n_trials\"])\n\n_Xtr = X_train.reset_index(drop=True)\n_ytr = np.asarray(y_train).astype(int).ravel()\n\n\ndef _spw(y):\n pos = int((y == 1).sum())\n neg = int((y == 0).sum())\n return (neg / pos) if pos > 0 else 1.0\n\n\ndef _suggest(trial):\n params = dict(_fixed)\n for name, bounds in _search.items():\n lo, hi = bounds\n if name in _INT_PARAMS:\n params[name] = trial.suggest_int(name, int(round(lo)), int(round(hi)))\n elif name in _LOG_PARAMS:\n params[name] = trial.suggest_float(name, lo, hi, log=True)\n else:\n params[name] = trial.suggest_float(name, lo, hi)\n return params\n\n\ndef _make_model(params, spw):\n return XGBClassifier(\n objective=\"binary:logistic\",\n eval_metric=\"logloss\",\n tree_method=\"hist\",\n n_jobs=1,\n random_state=XGB_SEED,\n scale_pos_weight=spw,\n **params,\n )\n\n\n_cv = StratifiedKFold(n_splits=CV_FOLDS, shuffle=True, random_state=CV_FOLD_SEED)\n\n\ndef _objective(trial):\n params = _suggest(trial)\n scores = []\n for tr_idx, va_idx in _cv.split(_Xtr, _ytr):\n Xt = _Xtr.iloc[tr_idx]\n Xv = _Xtr.iloc[va_idx]\n yt = _ytr[tr_idx]\n yv = _ytr[va_idx]\n model = _make_model(params, _spw(yt))\n model.fit(Xt, yt)\n proba = model.predict_proba(Xv)[:, 1]\n scores.append(average_precision_score(yv, proba))\n return float(np.mean(scores))\n\n\n_study = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=OPTUNA_SAMPLER_SEED))\n_study.optimize(_objective, n_trials=_n_trials, show_progress_bar=False)\n\n_best = dict(_fixed)\n_best.update(_study.best_params)\nfor _p in _INT_PARAMS:\n if _p in _best:\n _best[_p] = int(round(float(_best[_p])))\n\n_idx = np.arange(len(_Xtr))\n_fit_idx, _val_idx = train_test_split(\n _idx, test_size=VAL_FRACTION, stratify=_ytr, random_state=VAL_SPLIT_SEED\n)\n_Xfit = _Xtr.iloc[_fit_idx]\n_Xval = _Xtr.iloc[_val_idx]\n_yfit = _ytr[_fit_idx]\n_yval = _ytr[_val_idx]\n\n_base = _make_model(_best, _spw(_yfit))\n_base.fit(_Xfit, _yfit)\n\n_val_raw = _base.predict_proba(_Xval)[:, 1]\n_iso = IsotonicRegression(out_of_bounds=\"clip\")\n_iso.fit(_val_raw, _yval)\n\n\ndef predict_proba(X):\n raw = _base.predict_proba(X)[:, 1]\n return _iso.predict(raw)\n\n\n_val_cal = _iso.predict(_val_raw)\n_grid = np.unique(np.concatenate([np.array([0.0, 1.0]), _val_cal]))\n_best_t, _best_f1 = 0.5, -1.0\nfor _t in _grid:\n _f1 = f1_score(_yval, (_val_cal >= _t).astype(int), zero_division=0)\n if _f1 > _best_f1:\n _best_f1, _best_t = _f1, float(_t)\nchosen_threshold = float(_best_t)\n\n_check = predict_proba(_Xtr.head(5))\nassert _check.shape[0] == 5\n\nresult = {\n \"best_hyperparameters\": {k: (int(v) if k in _INT_PARAMS else float(v)) for k, v in _best.items()},\n \"tuned_params\": sorted(_search.keys()),\n \"n_trials\": _n_trials,\n \"optuna_best_value\": float(_study.best_value),\n \"cv_metric\": \"average_precision\",\n \"cv_folds\": CV_FOLDS,\n \"calibration\": \"isotonic\",\n \"calibration_brier_val\": float(brier_score_loss(_yval, _val_cal)),\n \"val_fraction\": VAL_FRACTION,\n \"chosen_threshold\": chosen_threshold,\n \"chosen_threshold_val_f1\": float(_best_f1),\n \"scale_pos_weight_fit\": float(_spw(_yfit)),\n \"n_train_fit\": int(len(_fit_idx)),\n \"n_val\": int(len(_val_idx)),\n}\n" - }, - "factor_bindings": {} - } - } - ], - "edges": [ - { - "id": "e-dc-to-gatedc", - "source": "agent-dc", - "target": "gate-dc" - }, - { - "id": "e-gatedc-to-fte", - "source": "gate-dc", - "target": "agent-fte" - }, - { - "id": "e-fte-to-gatefte", - "source": "agent-fte", - "target": "gate-fte" - }, - { - "id": "e-gatefte-to-fs", - "source": "gate-fte", - "target": "agent-fs" - }, - { - "id": "e-fs-to-gatefs", - "source": "agent-fs", - "target": "gate-fs" - }, - { - "id": "e-gatefs-to-mlm", - "source": "gate-fs", - "target": "agent-mlm" - }, - { - "id": "e-mlm-to-gatemlm", - "source": "agent-mlm", - "target": "gate-mlm" - }, - { - "id": "e-llm-dc", - "source": "llm-shared", - "target": "agent-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fte", - "source": "llm-shared", - "target": "agent-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fs", - "source": "llm-shared", - "target": "agent-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-mlm", - "source": "llm-shared", - "target": "agent-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatedc", - "source": "llm-shared", - "target": "gate-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefte", - "source": "llm-shared", - "target": "gate-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefs", - "source": "llm-shared", - "target": "gate-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatemlm", - "source": "llm-shared", - "target": "gate-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-dc", - "source": "pattern-dc", - "target": "agent-dc", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fte", - "source": "pattern-fte", - "target": "agent-fte", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fs", - "source": "pattern-fs", - "target": "agent-fs", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-mlm", - "source": "pattern-mlm", - "target": "agent-mlm", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-workspace-dc", - "source": "tool-workspace-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-dc", - "source": "tool-eda-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-dc", - "source": "tool-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fte", - "source": "tool-workspace-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fte", - "source": "tool-eda-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fte", - "source": "tool-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fs", - "source": "tool-workspace-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fs", - "source": "tool-eda-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fs", - "source": "tool-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-dataset-agent-dc", - "source": "dataset-mi", - "target": "agent-dc", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fte", - "source": "dataset-mi", - "target": "agent-fte", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fs", - "source": "dataset-mi", - "target": "agent-fs", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-gatemlm-to-score", - "source": "gate-mlm", - "target": "agent-score" - }, - { - "id": "e-llm-score", - "source": "llm-shared", - "target": "agent-score", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-score", - "source": "pattern-score", - "target": "agent-score", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-model-score", - "source": "tool-model-score", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-script-model-score", - "source": "script-model", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - } - ] - }, - "design_spec": { - "factors": [ - { - "name": "Anthropic:Model", - "levels": [ - "claude-sonnet-5", - "claude-opus-5" - ], - "level_type": "string" - }, - { - "name": "Anthropic:Effort", - "levels": [ - "medium", - "xhigh" - ], - "level_type": "string" - }, - { - "name": "Critic enabled", - "levels": [ - true, - false - ], - "level_type": "boolean" - } - ], - "replicates": 10, - "metrics": [ - { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", - "primary": false, - "direction": "maximize" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - } -} diff --git a/publications/BDM/myocardial-anthropic-latest.json b/publications/BDM/myocardial-anthropic-v0.8.0.json similarity index 100% rename from publications/BDM/myocardial-anthropic-latest.json rename to publications/BDM/myocardial-anthropic-v0.8.0.json diff --git a/publications/BDM/myocardial-azure-foundry-v0.2.0.json b/publications/BDM/myocardial-azure-foundry-v0.2.0.json deleted file mode 100644 index 419112df..00000000 --- a/publications/BDM/myocardial-azure-foundry-v0.2.0.json +++ /dev/null @@ -1,1027 +0,0 @@ -{ - "name": "Myocardial Infarction Use Case (Azure Foundry)", - "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", - "graph": { - "nodes": [ - { - "id": "llm-shared", - "type": "model_azure_foundry", - "position": { - "x": 1140, - "y": 680 - }, - "data": { - "label": "Azure Foundry (shared)", - "config": { - "provider": "azure_foundry", - "model": "claude-sonnet-5", - "temperature": null, - "effort": "medium", - "max_tokens": 128000 - }, - "factor_bindings": { - "config.model": "Azure Foundry:Model", - "config.effort": "Azure Foundry:Effort" - } - } - }, - { - "id": "dataset-mi", - "type": "dataset", - "position": { - "x": 724, - "y": 460 - }, - "data": { - "label": "Myocardial Infarction Dataset", - "config": { - "dataset_id": "56d2bbd7-624f-44e3-bd1f-a070b41432b1", - "dataset_name": "myocardial_infarction", - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-dc", - "type": "agent", - "position": { - "x": 0, - "y": 0 - }, - "data": { - "label": "MI-DC", - "config": { - "prompt": "", - "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: identify THIS cell's workspace; pass them to\n open_workspace. Your workspace identity then travels with every tool call\n automatically, so you never pass a workspace id or dataset id to the cleaning tools.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(experiment_id, cell_label, name, stage=\"dc\"): STEP 1, once \u2014 open/\n resume this cell's workspace AND stage your scratch input; every later call is\n scoped to it automatically. Always pass stage=\"dc\" \u2014 omitting it means your\n other tools will find no scratch input and fail.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"dc\").\n It opens (or resumes) this cell's workspace and stages your scratch input. When you need a column's meaning, type, or\n codes, call get_data_dictionary(name, columns=\"col1,col2\") for just those\n columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-dc", - "type": "critic_gate", - "position": { - "x": 300, - "y": 0 - }, - "data": { - "label": "Critic (DC)", - "config": { - "name": "dc-critic", - "goal": "Review MI-DC's cleaning decisions for this stage only.", - "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-dc", - "type": "pattern_reason_act", - "position": { - "x": 8, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-dc", - "type": "mcp_tool", - "position": { - "x": -16, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-dc", - "type": "mcp_tool", - "position": { - "x": 164, - "y": 240 - }, - "data": { - "label": "EDA (get_data_dictionary)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-dc", - "type": "mcp_tool", - "position": { - "x": 344, - "y": 240 - }, - "data": { - "label": "DC (inspect/coerce/impute)", - "config": { - "server_id": "80699456-f1cd-4957-b97b-6dd0e2013cfb", - "server_name": "asaree-sklearn-dc", - "tool_names": [ - "inspect_columns", - "apply_coercions", - "drop_and_impute" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fte", - "type": "agent", - "position": { - "x": 560, - "y": 0 - }, - "data": { - "label": "MI-FTE", - "config": { - "prompt": "", - "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id to build_features and fit_preprocessor.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fte\").\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT\n pre-clean it; it is already clean and complete.\n get_data_dictionary(name, columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fte", - "type": "critic_gate", - "position": { - "x": 860, - "y": 0 - }, - "data": { - "label": "Critic (FTE)", - "config": { - "name": "fte-critic", - "goal": "Review MI-FTE's engineering/encoding decisions for this stage only.", - "description": "Shared Critic reviewing the FTE stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FTE (feature transformation/engineering) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (encoders, and any statistic used by an engineered feature such as a group median or bin edge) is computed on the training split only; the test split is never touched\n- engineered features are emitted as a TYPED, REPLAYABLE recipe (each entry has an op, its inputs, and an explicit deterministic output name), not as one-shot mutations of the training frame\n- every train-derived statistic an engineered feature depends on (bin edges, group median, training prevalence) is FROZEN as an explicit recipe param computed on the training fold, so the same value is applied to test without recomputing (this is the train/test leakage + skew guard)\n- engineered output names are explicit and deterministic so the FS selector and the test partition align on the same column names; group aggregates declare a fallback for group values unseen at fit time\n- encoding appropriateness: each column has an encoding decision (binarize, ordinal with a stated order, one-hot, leave-numeric, or bin) justified from column semantics OR data spread, not a blanket rule\n- every engineered feature carries a one-line rationale naming a specific reason it should help THIS target, not a generic statement\n- engineering is proportionate: the agent engineered as much as is justified and no more; the critic flags both bloat (features with no specific hypothesis) and under-engineering (obvious, well-supported features left out). Judged on rationale quality, not a count\n- interactions (multiplicative or logical) are individually reasoned, not a blanket multiply of all pairs\n- clinical meta-features, if any, are clinically coherent and non-leaky; any composite or index does NOT silently discard component signal (components are retained alongside the composite unless their removal is justified)\n- the engineered + encoded matrix is committed to the workspace as v2_fte, fully numeric and model-ready: build_features (recipe) then fit_preprocessor (encoding_map) are BOTH called, so the encoded matrix is the committed output the FS stage and the runner read (workspace identity is ambient; persistence is automatic, not a save_artifact step)\n- the per-column encoding_map was passed to fit_preprocessor via encoding_map_json (not just described): the map the agent reports equals the map it fit, so test is encoded with the same per-column decisions and handle_unknown protects against unseen test categories\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: engineering emitted as one-shot frame mutations instead of a typed,\n replayable recipe; a train-derived statistic (bin edge, group median,\n prevalence) recomputed at transform time instead of frozen into the recipe;\n non-deterministic engineered names that would desync the FS selector on test; a\n group aggregate with no unseen-group fallback; lossy composites that discard\n component signal without justification; leaky meta-features computed downstream\n of the outcome; generic rationales; bloat or under-engineering; build_features\n or fit_preprocessor did not commit the encoded v2_fte matrix, or the encoding_map\n only described but not passed via encoding_map_json.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fte\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fte", - "type": "pattern_reason_act", - "position": { - "x": 568, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fte", - "type": "mcp_tool", - "position": { - "x": 544, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fte", - "type": "mcp_tool", - "position": { - "x": 724, - "y": 240 - }, - "data": { - "label": "EDA (dictionary + distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary", - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fte", - "type": "mcp_tool", - "position": { - "x": 904, - "y": 240 - }, - "data": { - "label": "FTE (build/fit)", - "config": { - "server_id": "03d4b07d-ffdc-497c-8779-a79126ada1db", - "server_name": "asaree-sklearn-fte", - "tool_names": [ - "build_features", - "fit_preprocessor" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fs", - "type": "agent", - "position": { - "x": 1120, - "y": 0 - }, - "data": { - "label": "MI-FS", - "config": { - "prompt": "", - "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id; the FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fs\").\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a\n preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fs", - "type": "critic_gate", - "position": { - "x": 1420, - "y": 0 - }, - "data": { - "label": "Critic (FS)", - "config": { - "name": "fs-critic", - "goal": "Review MI-FS's feature-selection decisions for this stage only.", - "description": "Shared Critic reviewing the FS stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FS (feature selection) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- profiling completeness: every retained column is profiled with dtype, n_unique, and summary stats (data arrives clean and already encoded from DC/FTE; missingness and encoding are NOT this stage's job)\n- selection runs on the encoded feature space the workspace already holds (v2_fte), so the selected names align with the matrix the runner scores; no preprocessor is re-applied here (the input is already model-ready)\n- variance filter: a variance threshold (and the method) is stated and low/near-zero-variance features are removed against it\n- correlation filter: highly correlated pairs are found (method + |rho| threshold stated) and one of each pair is removed with a defensible UNSUPERVISED tie-break (e.g. lower missingness then higher variance)\n- supervised filter is CORRECT for the task: f_classif or mutual_info_classif for classification (NOT f_regression, which assumes a continuous target), and is rank-based / conservative (keep top-k) rather than an aggressive p-value cut that could discard features carrying signal only in interaction\n- selection covers the FULL expanded space: original survivors and FTE meta-features are filtered on equal footing; nothing bypasses selection\n- observed class distribution is checked against task_brief (expected_positive_rate for binary, num_classes for multiclass)\n- no leakage: all fits (variance, correlation, supervised filter) are computed on the training split only; the test split is never touched\n- the selected features are committed to the workspace as v3_fs by the final selection call (workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed columns must be a subset of the v2_fte columns\n- selection is proportionate: it did not strip features the downstream tree model could plausibly exploit, nor retain obvious redundancy\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: a cut so aggressive it likely discards interaction-only signal the tree\n could use; a correlation tie-break that leaks the label; meta-features exempted\n from selection; unstated thresholds; the final selection call did not commit\n v3_fs; any fit touching the test split.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fs\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fs", - "type": "pattern_reason_act", - "position": { - "x": 1128, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fs", - "type": "mcp_tool", - "position": { - "x": 1104, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fs", - "type": "mcp_tool", - "position": { - "x": 1284, - "y": 240 - }, - "data": { - "label": "EDA (get_feature_distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fs", - "type": "mcp_tool", - "position": { - "x": 1464, - "y": 240 - }, - "data": { - "label": "FS (variance/corr/supervised/balance)", - "config": { - "server_id": "570f42e8-2eff-42ce-8d3f-733b9ee04ff2", - "server_name": "asaree-sklearn-fs", - "tool_names": [ - "variance_filter", - "compute_correlations", - "supervised_filter", - "check_class_balance" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-mlm", - "type": "agent", - "position": { - "x": 1680, - "y": 0 - }, - "data": { - "label": "MI-MLM", - "config": { - "prompt": "", - "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", - "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-mlm", - "type": "critic_gate", - "position": { - "x": 1980, - "y": 0 - }, - "data": { - "label": "Critic (MLM)", - "config": { - "name": "mlm-critic", - "goal": "Review MI-MLM's proposed hyperparameter payload for this stage only.", - "description": "Shared Critic reviewing the MLM stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the MLM (hyperparameter planning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- the payload is ONE JSON object with exactly the keys search_space, fixed_params, n_trials, and rationale; it is not code and does not attempt to set any harness-fixed policy (CV folds, seeds, objective metric, calibration, threshold, or scale_pos_weight)\n- schema validity: search_space tunes at least 3 DISTINCT params drawn from the nine allowed XGBoost params, each with low < high and [low, high] inside that param's hard bounds; fixed_params values are inside their hard bounds; every one of the nine params appears exactly once across search_space and fixed_params (none unspecified, none in both); n_trials is an integer in [1, 100]\n- the choices are defensible for class-imbalanced target optimized on PR-AUC: the search is not degenerate (ranges not collapsed to a point, not the full hard bound on everything), depth and regularization are not pinned to values that force gross over- or under-fitting, and n_trials is adequate for the number of tuned dimensions\n- every non-default choice carries a specific one-line rationale tied to THIS imbalanced PR-AUC task, not a generic statement\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: the output is not a single valid JSON payload; search_space tunes fewer\n than 3 params or names a param outside the nine allowed; a range falls\n outside its hard bounds or has low >= high; a param is unspecified or appears\n in both search_space and fixed_params; n_trials outside [1, 100]; the agent\n tries to set harness-fixed policy (CV folds, seeds, objective metric,\n calibration, threshold, scale_pos_weight); a degenerate search (point ranges\n or full-bound-on-all) or rationales generic rather than tied to the\n imbalanced PR-AUC target.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"mlm\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-mlm", - "type": "pattern_single_agent_baseline", - "position": { - "x": 1688, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 10, - "stop_on_first_success": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-score", - "type": "agent", - "position": { - "x": 2240, - "y": 0 - }, - "data": { - "label": "MI-Score", - "config": { - "prompt": "", - "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with code set to the wired Script node's text verbatim and payload_json set to the approved payload (as compact JSON) from the upstream context -- do not alter either. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", - "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", - "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "pattern-score", - "type": "pattern_reason_act", - "position": { - "x": 2248, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-model-score", - "type": "mcp_tool", - "position": { - "x": 2314, - "y": 240 - }, - "data": { - "label": "Model (run_model_script)", - "config": { - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "server_name": "asaree-sklearn-model", - "tool_names": [ - "run_model_script" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "script-model", - "type": "script", - "position": { - "x": 2494, - "y": 240 - }, - "data": { - "label": "Fixed Model Script", - "config": { - "name": "fixed_model_script", - "language": "python", - "code": "CV_FOLDS = 5\nCV_FOLD_SEED = 20260701\nOPTUNA_SAMPLER_SEED = 20260702\nXGB_SEED = 20260703\nVAL_SPLIT_SEED = 20260704\nVAL_FRACTION = 0.15\n\n# ==================================================================\n# ASAREE fixed modeling script (harness-owned; the MLM agent does NOT\n# write this). The agent supplies only the typed payload, bound as hp.\n# Everything here is deterministic given hp and the seeds baked above,\n# so replicate variance reflects LLM stochasticity in the payload only.\n#\n# Pre-bound by run_model_script: hp, X_train, y_train, np, pd. The test\n# split is NEVER bound; this script defines predict_proba(X) and\n# chosen_threshold, and the runner scores the held-out 30 itself.\n#\n# Harness-fixed policy (not agent-controllable):\n# inner CV 5-fold StratifiedKFold, fixed global fold assignment\n# objective PR-AUC (average_precision) on the training folds only\n# pos weighting scale_pos_weight = n_neg/n_pos on the fitted rows\n# calibration isotonic on one held-out training val fold\n# threshold argmax-F1 on the val fold; Brier reported on the val fold\n# no early stopping\n# Collaborator note: isotonic is monotonic, so calibration cannot change\n# PR-AUC or ROC-AUC (both rank-based); it only moves Brier and the\n# threshold-dependent secondaries, none of which is a tested outcome. The\n# scored model is fit on train-minus-val, so its PR-AUC/ROC-AUC equal the\n# base model ranking and the descriptive-only claim holds exactly.\n# ==================================================================\nimport optuna\nfrom optuna.samplers import TPESampler\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import average_precision_score, f1_score, brier_score_loss\nfrom sklearn.isotonic import IsotonicRegression\nfrom xgboost import XGBClassifier\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\n_INT_PARAMS = {\"n_estimators\", \"max_depth\"}\n_LOG_PARAMS = {\"learning_rate\", \"reg_lambda\", \"reg_alpha\"}\n\n_search = {}\nfor _e in hp[\"search_space\"]:\n _search[_e[\"param\"]] = (float(_e[\"low\"]), float(_e[\"high\"]))\n_fixed = dict(hp[\"fixed_params\"])\nfor _p in _INT_PARAMS:\n if _p in _fixed:\n _fixed[_p] = int(round(float(_fixed[_p])))\n_n_trials = int(hp[\"n_trials\"])\n\n_Xtr = X_train.reset_index(drop=True)\n_ytr = np.asarray(y_train).astype(int).ravel()\n\n\ndef _spw(y):\n pos = int((y == 1).sum())\n neg = int((y == 0).sum())\n return (neg / pos) if pos > 0 else 1.0\n\n\ndef _suggest(trial):\n params = dict(_fixed)\n for name, bounds in _search.items():\n lo, hi = bounds\n if name in _INT_PARAMS:\n params[name] = trial.suggest_int(name, int(round(lo)), int(round(hi)))\n elif name in _LOG_PARAMS:\n params[name] = trial.suggest_float(name, lo, hi, log=True)\n else:\n params[name] = trial.suggest_float(name, lo, hi)\n return params\n\n\ndef _make_model(params, spw):\n return XGBClassifier(\n objective=\"binary:logistic\",\n eval_metric=\"logloss\",\n tree_method=\"hist\",\n n_jobs=1,\n random_state=XGB_SEED,\n scale_pos_weight=spw,\n **params,\n )\n\n\n_cv = StratifiedKFold(n_splits=CV_FOLDS, shuffle=True, random_state=CV_FOLD_SEED)\n\n\ndef _objective(trial):\n params = _suggest(trial)\n scores = []\n for tr_idx, va_idx in _cv.split(_Xtr, _ytr):\n Xt = _Xtr.iloc[tr_idx]\n Xv = _Xtr.iloc[va_idx]\n yt = _ytr[tr_idx]\n yv = _ytr[va_idx]\n model = _make_model(params, _spw(yt))\n model.fit(Xt, yt)\n proba = model.predict_proba(Xv)[:, 1]\n scores.append(average_precision_score(yv, proba))\n return float(np.mean(scores))\n\n\n_study = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=OPTUNA_SAMPLER_SEED))\n_study.optimize(_objective, n_trials=_n_trials, show_progress_bar=False)\n\n_best = dict(_fixed)\n_best.update(_study.best_params)\nfor _p in _INT_PARAMS:\n if _p in _best:\n _best[_p] = int(round(float(_best[_p])))\n\n_idx = np.arange(len(_Xtr))\n_fit_idx, _val_idx = train_test_split(\n _idx, test_size=VAL_FRACTION, stratify=_ytr, random_state=VAL_SPLIT_SEED\n)\n_Xfit = _Xtr.iloc[_fit_idx]\n_Xval = _Xtr.iloc[_val_idx]\n_yfit = _ytr[_fit_idx]\n_yval = _ytr[_val_idx]\n\n_base = _make_model(_best, _spw(_yfit))\n_base.fit(_Xfit, _yfit)\n\n_val_raw = _base.predict_proba(_Xval)[:, 1]\n_iso = IsotonicRegression(out_of_bounds=\"clip\")\n_iso.fit(_val_raw, _yval)\n\n\ndef predict_proba(X):\n raw = _base.predict_proba(X)[:, 1]\n return _iso.predict(raw)\n\n\n_val_cal = _iso.predict(_val_raw)\n_grid = np.unique(np.concatenate([np.array([0.0, 1.0]), _val_cal]))\n_best_t, _best_f1 = 0.5, -1.0\nfor _t in _grid:\n _f1 = f1_score(_yval, (_val_cal >= _t).astype(int), zero_division=0)\n if _f1 > _best_f1:\n _best_f1, _best_t = _f1, float(_t)\nchosen_threshold = float(_best_t)\n\n_check = predict_proba(_Xtr.head(5))\nassert _check.shape[0] == 5\n\nresult = {\n \"best_hyperparameters\": {k: (int(v) if k in _INT_PARAMS else float(v)) for k, v in _best.items()},\n \"tuned_params\": sorted(_search.keys()),\n \"n_trials\": _n_trials,\n \"optuna_best_value\": float(_study.best_value),\n \"cv_metric\": \"average_precision\",\n \"cv_folds\": CV_FOLDS,\n \"calibration\": \"isotonic\",\n \"calibration_brier_val\": float(brier_score_loss(_yval, _val_cal)),\n \"val_fraction\": VAL_FRACTION,\n \"chosen_threshold\": chosen_threshold,\n \"chosen_threshold_val_f1\": float(_best_f1),\n \"scale_pos_weight_fit\": float(_spw(_yfit)),\n \"n_train_fit\": int(len(_fit_idx)),\n \"n_val\": int(len(_val_idx)),\n}\n" - }, - "factor_bindings": {} - } - } - ], - "edges": [ - { - "id": "e-dc-to-gatedc", - "source": "agent-dc", - "target": "gate-dc" - }, - { - "id": "e-gatedc-to-fte", - "source": "gate-dc", - "target": "agent-fte" - }, - { - "id": "e-fte-to-gatefte", - "source": "agent-fte", - "target": "gate-fte" - }, - { - "id": "e-gatefte-to-fs", - "source": "gate-fte", - "target": "agent-fs" - }, - { - "id": "e-fs-to-gatefs", - "source": "agent-fs", - "target": "gate-fs" - }, - { - "id": "e-gatefs-to-mlm", - "source": "gate-fs", - "target": "agent-mlm" - }, - { - "id": "e-mlm-to-gatemlm", - "source": "agent-mlm", - "target": "gate-mlm" - }, - { - "id": "e-llm-dc", - "source": "llm-shared", - "target": "agent-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fte", - "source": "llm-shared", - "target": "agent-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fs", - "source": "llm-shared", - "target": "agent-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-mlm", - "source": "llm-shared", - "target": "agent-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatedc", - "source": "llm-shared", - "target": "gate-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefte", - "source": "llm-shared", - "target": "gate-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefs", - "source": "llm-shared", - "target": "gate-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatemlm", - "source": "llm-shared", - "target": "gate-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-dc", - "source": "pattern-dc", - "target": "agent-dc", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fte", - "source": "pattern-fte", - "target": "agent-fte", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fs", - "source": "pattern-fs", - "target": "agent-fs", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-mlm", - "source": "pattern-mlm", - "target": "agent-mlm", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-workspace-dc", - "source": "tool-workspace-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-dc", - "source": "tool-eda-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-dc", - "source": "tool-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fte", - "source": "tool-workspace-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fte", - "source": "tool-eda-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fte", - "source": "tool-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fs", - "source": "tool-workspace-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fs", - "source": "tool-eda-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fs", - "source": "tool-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-dataset-agent-dc", - "source": "dataset-mi", - "target": "agent-dc", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fte", - "source": "dataset-mi", - "target": "agent-fte", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fs", - "source": "dataset-mi", - "target": "agent-fs", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-gatemlm-to-score", - "source": "gate-mlm", - "target": "agent-score" - }, - { - "id": "e-llm-score", - "source": "llm-shared", - "target": "agent-score", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-score", - "source": "pattern-score", - "target": "agent-score", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-model-score", - "source": "tool-model-score", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-script-model-score", - "source": "script-model", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - } - ] - }, - "design_spec": { - "factors": [ - { - "name": "Azure Foundry:Model", - "levels": [ - "claude-sonnet-5", - "claude-opus-5" - ], - "level_type": "string" - }, - { - "name": "Azure Foundry:Effort", - "levels": [ - "medium", - "xhigh" - ], - "level_type": "string" - }, - { - "name": "Critic enabled", - "levels": [ - true, - false - ], - "level_type": "boolean" - } - ], - "replicates": 10, - "metrics": [ - { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", - "primary": false, - "direction": "maximize" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - } -} diff --git a/publications/BDM/myocardial-azure-foundry-latest.json b/publications/BDM/myocardial-azure-foundry-v0.8.0.json similarity index 100% rename from publications/BDM/myocardial-azure-foundry-latest.json rename to publications/BDM/myocardial-azure-foundry-v0.8.0.json diff --git a/publications/BDM/myocardial-openai-v0.2.0.json b/publications/BDM/myocardial-openai-v0.2.0.json deleted file mode 100644 index 3b229c04..00000000 --- a/publications/BDM/myocardial-openai-v0.2.0.json +++ /dev/null @@ -1,1027 +0,0 @@ -{ - "name": "Myocardial Infarction Use Case (OpenAI)", - "description": "MI-DC -> MI-FTE -> MI-FS -> MI-MLM -> MI-Score: five agents in series on the UCI Myocardial Infarction Complications (ZSN) dataset, each handing a versioned dataset workspace to the next, the first four behind a Critic Gate (MI-Score is intentionally ungated -- it only executes a fixed scoring script, so there is no design decision left to review). The pipeline's methodology (cleaning, feature engineering, selection, hyperparameter planning, scoring) and the critics' review criteria are dataset-agnostic; everything dataset-specific lives in each worker's own task_brief block. Three facts are stated there rather than left for an agent to discover cold: (1) the dataset carries an explicit ID column (role=id, 1..1700, all-unique) that must be dropped as a non-feature record identifier; (2) every feature is RAW -- a mix of integer/real/binary/ordinal/categorical types with real missingness -- so DC's and FTE's inspect-first instructions do real work here; (3) the column names are short Russian-derived codes (nr11, np01, zab_leg_01), not descriptive English, so every column's meaning must be resolved via get_data_dictionary rather than guessed from its name. expected_positive_rate (0.232) is 394 positive of 1700. See README.md in this directory for how to import and run this.", - "graph": { - "nodes": [ - { - "id": "llm-shared", - "type": "model_openai", - "position": { - "x": 1140, - "y": 680 - }, - "data": { - "label": "OpenAI (shared)", - "config": { - "provider": "openai", - "model": "gpt-5-mini", - "temperature": null, - "effort": "medium", - "max_tokens": 128000 - }, - "factor_bindings": { - "config.model": "OpenAI:Model", - "config.effort": "OpenAI:Effort" - } - } - }, - { - "id": "dataset-mi", - "type": "dataset", - "position": { - "x": 724, - "y": 460 - }, - "data": { - "label": "Myocardial Infarction Dataset", - "config": { - "dataset_id": "56d2bbd7-624f-44e3-bd1f-a070b41432b1", - "dataset_name": "myocardial_infarction", - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-dc", - "type": "agent", - "position": { - "x": 0, - "y": 0 - }, - "data": { - "label": "MI-DC", - "config": { - "prompt": "", - "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", - "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: identify THIS cell's workspace; pass them to\n open_workspace. Your workspace identity then travels with every tool call\n automatically, so you never pass a workspace id or dataset id to the cleaning tools.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here \u2014 the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS \u2014 the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) \u2014 inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(experiment_id, cell_label, name, stage=\"dc\"): STEP 1, once \u2014 open/\n resume this cell's workspace AND stage your scratch input; every later call is\n scoped to it automatically. Always pass stage=\"dc\" \u2014 omitting it means your\n other tools will find no scratch input and fail.\n- get_data_dictionary(name, columns=...): read-only, any time \u2014 a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once \u2014 the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 \u2014 coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal \u2014 drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns \u2192 apply_coercions \u2192 drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"dc\").\n It opens (or resumes) this cell's workspace and stages your scratch input. When you need a column's meaning, type, or\n codes, call get_data_dictionary(name, columns=\"col1,col2\") for just those\n columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split \u2014 nothing is changed or committed \u2014\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS \u2014 so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and \u2014\n later \u2014 what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN \u2014 the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column \u2014 the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE \u2014 the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped \u2014 pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy \u2014 mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n\u2014 do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-dc", - "type": "critic_gate", - "position": { - "x": 300, - "y": 0 - }, - "data": { - "label": "Critic (DC)", - "config": { - "name": "dc-critic", - "goal": "Review MI-DC's cleaning decisions for this stage only.", - "description": "Shared Critic reviewing the DC stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the DC (data cleaning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (domain rules, IQR bounds, imputation statistics) is computed on the training split only; the test split is never touched\n- domain/type validation is run on ALL columns (continuous included), and values violating the column type or domain (wrong type, out-of-set category, impossible sign/range) are coerced to NaN with the rule stated\n- a missingness threshold is set WITH a rationale and gates RAW missingness only; columns above it are dropped citing the measured missingness\n- outlier detection (IQR) is run on continuous columns only and is treated as a review flag, not an automatic delete: physiologically plausible but statistically extreme values are kept; deletions are reserved for implausible values and each deletion states why\n- imputation is appropriate and train-only: mode for binary/categorical, mean or median for continuous with the mean-vs-median choice justified by skew; ALL surviving NaNs are imputed, including those created by domain fixes and outlier removal\n- the cleaned matrices are committed to the workspace as v1_dc by apply_coercions and drop_and_impute (the cell workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed output must have zero remaining missing values and identical train/test columns\n- cleaning is proportionate: the agent did not over-clean (deleting real signal) or under-clean (leaving integrity errors); judged on the stated reasons, not on a count\n- text-typed numeric columns handled: any column that is semantically continuous but loaded as text/object because a MINORITY of cells hold non-numeric tokens (censored \"x\" values, unit/flag suffixes, or free-text lab comments) is NOT left as text or treated as categorical \u2014 those tokens are coerced to NaN so the column parses as numeric, with the rule and coerced count stated\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: IQR deletion of physiologically plausible extremes (real clinical signal\n thrown away); domain/type validation skipped on continuous columns; the\n missingness threshold re-triggering on outlier-induced NaNs instead of raw\n missingness; NaNs left unimputed after domain fixes or outlier removal; a\n semantically-continuous column left as text or mis-typed as categorical because\n a few non-numeric tokens (censored \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-dc", - "type": "pattern_reason_act", - "position": { - "x": 8, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-dc", - "type": "mcp_tool", - "position": { - "x": -16, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-dc", - "type": "mcp_tool", - "position": { - "x": 164, - "y": 240 - }, - "data": { - "label": "EDA (get_data_dictionary)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-dc", - "type": "mcp_tool", - "position": { - "x": 344, - "y": 240 - }, - "data": { - "label": "DC (inspect/coerce/impute)", - "config": { - "server_id": "80699456-f1cd-4957-b97b-6dd0e2013cfb", - "server_name": "asaree-sklearn-dc", - "tool_names": [ - "inspect_columns", - "apply_coercions", - "drop_and_impute" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fte", - "type": "agent", - "position": { - "x": 560, - "y": 0 - }, - "data": { - "label": "MI-FTE", - "config": { - "prompt": "", - "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", - "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you \u2014 you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test \u2014 all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id to build_features and fit_preprocessor.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fte\").\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT\n pre-clean it; it is already clean and complete.\n get_data_dictionary(name, columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix \u2014 confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary \u2014 the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero \u2014 how many inputs are non-zero\n count_equal(value) \u2014 how many inputs equal a literal value\n count_threshold(op, value) \u2014 how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing \u2014 how many of the inputs are missing\n weighted_sum(weights) \u2014 \u03a3 weight\u1d62\u00b7input\u1d62, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std \u2014 arithmetic across inputs\n n_distinct \u2014 count of distinct values across inputs\n which_max \u2014 NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor \u2014 boolean combination over binary inputs\n multiply(a, b, ...) \u2014 multiplicative interaction\n ratio(num, den) \u2014 a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) \u2014 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing \u2014 1 if the value is missing else 0\n log1p / sqrt / square / abs \u2014 monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) \u2014 quantile bins from the train distribution\n group_agg(group_col, value_col, stat) \u2014 per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) \u2014 replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders \u2014 substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary \u2014 build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 \u2014 the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix \u2014 the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fte", - "type": "critic_gate", - "position": { - "x": 860, - "y": 0 - }, - "data": { - "label": "Critic (FTE)", - "config": { - "name": "fte-critic", - "goal": "Review MI-FTE's engineering/encoding decisions for this stage only.", - "description": "Shared Critic reviewing the FTE stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FTE (feature transformation/engineering) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- no leakage: every fit (encoders, and any statistic used by an engineered feature such as a group median or bin edge) is computed on the training split only; the test split is never touched\n- engineered features are emitted as a TYPED, REPLAYABLE recipe (each entry has an op, its inputs, and an explicit deterministic output name), not as one-shot mutations of the training frame\n- every train-derived statistic an engineered feature depends on (bin edges, group median, training prevalence) is FROZEN as an explicit recipe param computed on the training fold, so the same value is applied to test without recomputing (this is the train/test leakage + skew guard)\n- engineered output names are explicit and deterministic so the FS selector and the test partition align on the same column names; group aggregates declare a fallback for group values unseen at fit time\n- encoding appropriateness: each column has an encoding decision (binarize, ordinal with a stated order, one-hot, leave-numeric, or bin) justified from column semantics OR data spread, not a blanket rule\n- every engineered feature carries a one-line rationale naming a specific reason it should help THIS target, not a generic statement\n- engineering is proportionate: the agent engineered as much as is justified and no more; the critic flags both bloat (features with no specific hypothesis) and under-engineering (obvious, well-supported features left out). Judged on rationale quality, not a count\n- interactions (multiplicative or logical) are individually reasoned, not a blanket multiply of all pairs\n- clinical meta-features, if any, are clinically coherent and non-leaky; any composite or index does NOT silently discard component signal (components are retained alongside the composite unless their removal is justified)\n- the engineered + encoded matrix is committed to the workspace as v2_fte, fully numeric and model-ready: build_features (recipe) then fit_preprocessor (encoding_map) are BOTH called, so the encoded matrix is the committed output the FS stage and the runner read (workspace identity is ambient; persistence is automatic, not a save_artifact step)\n- the per-column encoding_map was passed to fit_preprocessor via encoding_map_json (not just described): the map the agent reports equals the map it fit, so test is encoded with the same per-column decisions and handle_unknown protects against unseen test categories\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: engineering emitted as one-shot frame mutations instead of a typed,\n replayable recipe; a train-derived statistic (bin edge, group median,\n prevalence) recomputed at transform time instead of frozen into the recipe;\n non-deterministic engineered names that would desync the FS selector on test; a\n group aggregate with no unseen-group fallback; lossy composites that discard\n component signal without justification; leaky meta-features computed downstream\n of the outcome; generic rationales; bloat or under-engineering; build_features\n or fit_preprocessor did not commit the encoded v2_fte matrix, or the encoding_map\n only described but not passed via encoding_map_json.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fte\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fte", - "type": "pattern_reason_act", - "position": { - "x": 568, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fte", - "type": "mcp_tool", - "position": { - "x": 544, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fte", - "type": "mcp_tool", - "position": { - "x": 724, - "y": 240 - }, - "data": { - "label": "EDA (dictionary + distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_data_dictionary", - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fte", - "type": "mcp_tool", - "position": { - "x": 904, - "y": 240 - }, - "data": { - "label": "FTE (build/fit)", - "config": { - "server_id": "03d4b07d-ffdc-497c-8779-a79126ada1db", - "server_name": "asaree-sklearn-fte", - "tool_names": [ - "build_features", - "fit_preprocessor" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-fs", - "type": "agent", - "position": { - "x": 1120, - "y": 0 - }, - "data": { - "label": "MI-FS", - "config": { - "prompt": "", - "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you \u2014 you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- experiment_id, cell_label: pass them to open_workspace. Your workspace identity\n then travels with every tool call automatically, so you never pass a workspace id\n or dataset id; the FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(experiment_id, cell_label, name=, stage=\"fs\").\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a\n preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT \u2014 do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-fs", - "type": "critic_gate", - "position": { - "x": 1420, - "y": 0 - }, - "data": { - "label": "Critic (FS)", - "config": { - "name": "fs-critic", - "goal": "Review MI-FS's feature-selection decisions for this stage only.", - "description": "Shared Critic reviewing the FS stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the FS (feature selection) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- profiling completeness: every retained column is profiled with dtype, n_unique, and summary stats (data arrives clean and already encoded from DC/FTE; missingness and encoding are NOT this stage's job)\n- selection runs on the encoded feature space the workspace already holds (v2_fte), so the selected names align with the matrix the runner scores; no preprocessor is re-applied here (the input is already model-ready)\n- variance filter: a variance threshold (and the method) is stated and low/near-zero-variance features are removed against it\n- correlation filter: highly correlated pairs are found (method + |rho| threshold stated) and one of each pair is removed with a defensible UNSUPERVISED tie-break (e.g. lower missingness then higher variance)\n- supervised filter is CORRECT for the task: f_classif or mutual_info_classif for classification (NOT f_regression, which assumes a continuous target), and is rank-based / conservative (keep top-k) rather than an aggressive p-value cut that could discard features carrying signal only in interaction\n- selection covers the FULL expanded space: original survivors and FTE meta-features are filtered on equal footing; nothing bypasses selection\n- observed class distribution is checked against task_brief (expected_positive_rate for binary, num_classes for multiclass)\n- no leakage: all fits (variance, correlation, supervised filter) are computed on the training split only; the test split is never touched\n- the selected features are committed to the workspace as v3_fs by the final selection call (workspace identity is ambient; persistence is automatic, not a save_artifact step); the committed columns must be a subset of the v2_fte columns\n- selection is proportionate: it did not strip features the downstream tree model could plausibly exploit, nor retain obvious redundancy\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: a cut so aggressive it likely discards interaction-only signal the tree\n could use; a correlation tie-break that leaks the label; meta-features exempted\n from selection; unstated thresholds; the final selection call did not commit\n v3_fs; any fit touching the test split.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"fs\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-fs", - "type": "pattern_reason_act", - "position": { - "x": 1128, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-workspace-fs", - "type": "mcp_tool", - "position": { - "x": 1104, - "y": 240 - }, - "data": { - "label": "Workspace (open_workspace, accept_stage)", - "config": { - "server_id": "f7d33b44-61f1-4e8b-b70c-5ee71366e553", - "server_name": "asaree-workspace", - "tool_names": [ - "open_workspace", - "accept_stage" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-eda-fs", - "type": "mcp_tool", - "position": { - "x": 1284, - "y": 240 - }, - "data": { - "label": "EDA (get_feature_distributions)", - "config": { - "server_id": "d344d0de-e13b-4d6b-b3bb-007a73a072f0", - "server_name": "asaree-sklearn-eda", - "tool_names": [ - "get_feature_distributions" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "tool-fs", - "type": "mcp_tool", - "position": { - "x": 1464, - "y": 240 - }, - "data": { - "label": "FS (variance/corr/supervised/balance)", - "config": { - "server_id": "570f42e8-2eff-42ce-8d3f-733b9ee04ff2", - "server_name": "asaree-sklearn-fs", - "tool_names": [ - "variance_filter", - "compute_correlations", - "supervised_filter", - "check_class_balance" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-mlm", - "type": "agent", - "position": { - "x": 1680, - "y": 0 - }, - "data": { - "label": "MI-MLM", - "config": { - "prompt": "", - "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", - "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", - "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "gate-mlm", - "type": "critic_gate", - "position": { - "x": 1980, - "y": 0 - }, - "data": { - "label": "Critic (MLM)", - "config": { - "name": "mlm-critic", - "goal": "Review MI-MLM's proposed hyperparameter payload for this stage only.", - "description": "Shared Critic reviewing the MLM stage -- approves or requests revision with feedback and a partial/full rejection scope.", - "system_prompt": "\nYou are a methodology reviewer for ONE stage of a biomedical ML pipeline (DC, FTE,\nFS, or MLM). You gate the pipeline: your verdict decides whether the stage agent\nmust revise before the pipeline continues. There is a SINGLE revision loop, so be\nprecise and actionable, and do not waste it on nitpicks.\n\nYou hold the combined expertise of all four stages: data cleaning and clinical\ndata integrity, feature transformation and encoding, feature selection\nstatistics, and XGBoost training / calibration. Reason from first principles.\n\nYou are reviewing the MLM (hyperparameter planning) stage specifically -- this Critic Gate is wired to only that one stage, never any other. ASAREE's canvas sends your instruction per run with only the stage agent's output to review -- no task_brief and no dynamically-injected checklist -- so this stage's own checklist is baked in below instead of arriving with each run:\n\n- the payload is ONE JSON object with exactly the keys search_space, fixed_params, n_trials, and rationale; it is not code and does not attempt to set any harness-fixed policy (CV folds, seeds, objective metric, calibration, threshold, or scale_pos_weight)\n- schema validity: search_space tunes at least 3 DISTINCT params drawn from the nine allowed XGBoost params, each with low < high and [low, high] inside that param's hard bounds; fixed_params values are inside their hard bounds; every one of the nine params appears exactly once across search_space and fixed_params (none unspecified, none in both); n_trials is an integer in [1, 100]\n- the choices are defensible for class-imbalanced target optimized on PR-AUC: the search is not degenerate (ranges not collapsed to a point, not the full hard bound on everything), depth and regularization are not pinned to values that force gross over- or under-fitting, and n_trials is adequate for the number of tuned dimensions\n- every non-default choice carries a specific one-line rationale tied to THIS imbalanced PR-AUC task, not a generic statement\n\nEvaluate the output against EACH criterion, citing concrete evidence from the\noutput (field names, values). A criterion passes only if the output clearly\nsatisfies it; if you cannot find evidence, it fails.\n\nFailure modes to watch for this stage: the output is not a single valid JSON payload; search_space tunes fewer\n than 3 params or names a param outside the nine allowed; a range falls\n outside its hard bounds or has low >= high; a param is unspecified or appears\n in both search_space and fixed_params; n_trials outside [1, 100]; the agent\n tries to set harness-fixed policy (CV folds, seeds, objective metric,\n calibration, threshold, scale_pos_weight); a degenerate search (point ranges\n or full-bound-on-all) or rationales generic rather than tied to the\n imbalanced PR-AUC target.\n\nDO NOT BE TRIGGER-HAPPY. Enforce a revision ONLY when something is genuinely\nnon-optimal, clinically irrelevant, leaky, or incorrect. Do not fail a stage over\nstylistic preferences, defensible judgment calls, or choices you would merely\nhave made differently. On a true tie, approve. Do not invent problems.\n\nYour ENTIRE response is this verdict -- the FIRST character you output MUST be `{`.\nDo NOT write any preamble, plan, or meta-commentary about the task (e.g. \"this is\na review task...\", \"the reviewer should...\"); render the verdict itself, not a\ndescription of how you would review. Return EXACTLY ONE JSON object (followed only\nby the machine-readable verdict line described in the Rules) and nothing else:\n{\n \"stage\": \"mlm\",\n \"approved\": true,\n \"criteria_results\": [\n {\"criterion\": \"\", \"pass\": true,\n \"evidence\": \"what in the output supports this\"}\n ],\n \"feedback\": \"\",\n \"rejection_scope\": \"\"\n}\n\nRules:\n- \"approved\" is true ONLY if every criterion passes.\n- When \"approved\" is false, \"feedback\" MUST be specific, actionable instructions\n the stage agent can follow to fix the failing criteria in a SINGLE revision:\n name the concrete fields, decisions, or tool calls to change and why. Do NOT\n rewrite the output yourself; tell the agent what to do.\n- When \"approved\" is true, \"feedback\" is an empty string.\n- When \"approved\" is false, \"rejection_scope\" says how much of the output the\n stage agent should rebuild. It changes what the agent is told to PRESERVE:\n \"partial\" -- the failing criteria are LOCALIZED. Every other decision in the\n output is sound, and the agent will be instructed to reproduce it exactly.\n Use this for a specific, fixable defect: a wrong encoding, a missed\n coercion, one column dropped that should have been kept.\n \"full\" -- the stage's APPROACH is wrong at its foundation, so preserving the\n surrounding decisions would preserve the problem. The agent will be told to\n reconsider the stage from first principles.\n Choose \"partial\" unless the approach ITSELF is the defect; \"full\" discards\n sound work and costs the stage its only revision. An omitted or unrecognized\n value is treated as \"full\".\n- When \"approved\" is true, \"rejection_scope\" is an empty string.\n- Be strict but fair: approve work that meets the criteria.\n- Emit the JSON object as the very first content of your response; never precede\n it with commentary. If every criterion is satisfied, still return the complete\n verdict object -- silence or a task description is not a valid response.\n- After the JSON object, on its own final line, also emit a single machine-readable\n verdict anchor -- literally `VERDICT: APPROVED` or `VERDICT: REJECTED` -- that\n matches the \"approved\" field. On a rejection, follow it with a second anchor\n line -- literally `SCOPE: PARTIAL` or `SCOPE: FULL` -- matching\n \"rejection_scope\". The JSON object stays the source of truth; these lines are\n only fallback anchors for automated parsing.\n", - "enabled": true, - "max_revisions": 1 - }, - "factor_bindings": { - "config.enabled": "Critic enabled" - } - } - }, - { - "id": "pattern-mlm", - "type": "pattern_single_agent_baseline", - "position": { - "x": 1688, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 10, - "stop_on_first_success": true - }, - "factor_bindings": {} - } - }, - { - "id": "agent-score", - "type": "agent", - "position": { - "x": 2240, - "y": 0 - }, - "data": { - "label": "MI-Score", - "config": { - "prompt": "", - "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with code set to the wired Script node's text verbatim and payload_json set to the approved payload (as compact JSON) from the upstream context -- do not alter either. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", - "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", - "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, - "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 - }, - "factor_bindings": {} - } - }, - { - "id": "pattern-score", - "type": "pattern_reason_act", - "position": { - "x": 2248, - "y": -220 - }, - "data": { - "label": "", - "config": { - "max_iterations": 12, - "include_scratchpad": true, - "scratchpad_window": 10, - "observation_format": "raw" - }, - "factor_bindings": {} - } - }, - { - "id": "tool-model-score", - "type": "mcp_tool", - "position": { - "x": 2314, - "y": 240 - }, - "data": { - "label": "Model (run_model_script)", - "config": { - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "server_name": "asaree-sklearn-model", - "tool_names": [ - "run_model_script" - ], - "enabled": true - }, - "factor_bindings": {} - } - }, - { - "id": "script-model", - "type": "script", - "position": { - "x": 2494, - "y": 240 - }, - "data": { - "label": "Fixed Model Script", - "config": { - "name": "fixed_model_script", - "language": "python", - "code": "CV_FOLDS = 5\nCV_FOLD_SEED = 20260701\nOPTUNA_SAMPLER_SEED = 20260702\nXGB_SEED = 20260703\nVAL_SPLIT_SEED = 20260704\nVAL_FRACTION = 0.15\n\n# ==================================================================\n# ASAREE fixed modeling script (harness-owned; the MLM agent does NOT\n# write this). The agent supplies only the typed payload, bound as hp.\n# Everything here is deterministic given hp and the seeds baked above,\n# so replicate variance reflects LLM stochasticity in the payload only.\n#\n# Pre-bound by run_model_script: hp, X_train, y_train, np, pd. The test\n# split is NEVER bound; this script defines predict_proba(X) and\n# chosen_threshold, and the runner scores the held-out 30 itself.\n#\n# Harness-fixed policy (not agent-controllable):\n# inner CV 5-fold StratifiedKFold, fixed global fold assignment\n# objective PR-AUC (average_precision) on the training folds only\n# pos weighting scale_pos_weight = n_neg/n_pos on the fitted rows\n# calibration isotonic on one held-out training val fold\n# threshold argmax-F1 on the val fold; Brier reported on the val fold\n# no early stopping\n# Collaborator note: isotonic is monotonic, so calibration cannot change\n# PR-AUC or ROC-AUC (both rank-based); it only moves Brier and the\n# threshold-dependent secondaries, none of which is a tested outcome. The\n# scored model is fit on train-minus-val, so its PR-AUC/ROC-AUC equal the\n# base model ranking and the descriptive-only claim holds exactly.\n# ==================================================================\nimport optuna\nfrom optuna.samplers import TPESampler\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import average_precision_score, f1_score, brier_score_loss\nfrom sklearn.isotonic import IsotonicRegression\nfrom xgboost import XGBClassifier\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\n_INT_PARAMS = {\"n_estimators\", \"max_depth\"}\n_LOG_PARAMS = {\"learning_rate\", \"reg_lambda\", \"reg_alpha\"}\n\n_search = {}\nfor _e in hp[\"search_space\"]:\n _search[_e[\"param\"]] = (float(_e[\"low\"]), float(_e[\"high\"]))\n_fixed = dict(hp[\"fixed_params\"])\nfor _p in _INT_PARAMS:\n if _p in _fixed:\n _fixed[_p] = int(round(float(_fixed[_p])))\n_n_trials = int(hp[\"n_trials\"])\n\n_Xtr = X_train.reset_index(drop=True)\n_ytr = np.asarray(y_train).astype(int).ravel()\n\n\ndef _spw(y):\n pos = int((y == 1).sum())\n neg = int((y == 0).sum())\n return (neg / pos) if pos > 0 else 1.0\n\n\ndef _suggest(trial):\n params = dict(_fixed)\n for name, bounds in _search.items():\n lo, hi = bounds\n if name in _INT_PARAMS:\n params[name] = trial.suggest_int(name, int(round(lo)), int(round(hi)))\n elif name in _LOG_PARAMS:\n params[name] = trial.suggest_float(name, lo, hi, log=True)\n else:\n params[name] = trial.suggest_float(name, lo, hi)\n return params\n\n\ndef _make_model(params, spw):\n return XGBClassifier(\n objective=\"binary:logistic\",\n eval_metric=\"logloss\",\n tree_method=\"hist\",\n n_jobs=1,\n random_state=XGB_SEED,\n scale_pos_weight=spw,\n **params,\n )\n\n\n_cv = StratifiedKFold(n_splits=CV_FOLDS, shuffle=True, random_state=CV_FOLD_SEED)\n\n\ndef _objective(trial):\n params = _suggest(trial)\n scores = []\n for tr_idx, va_idx in _cv.split(_Xtr, _ytr):\n Xt = _Xtr.iloc[tr_idx]\n Xv = _Xtr.iloc[va_idx]\n yt = _ytr[tr_idx]\n yv = _ytr[va_idx]\n model = _make_model(params, _spw(yt))\n model.fit(Xt, yt)\n proba = model.predict_proba(Xv)[:, 1]\n scores.append(average_precision_score(yv, proba))\n return float(np.mean(scores))\n\n\n_study = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=OPTUNA_SAMPLER_SEED))\n_study.optimize(_objective, n_trials=_n_trials, show_progress_bar=False)\n\n_best = dict(_fixed)\n_best.update(_study.best_params)\nfor _p in _INT_PARAMS:\n if _p in _best:\n _best[_p] = int(round(float(_best[_p])))\n\n_idx = np.arange(len(_Xtr))\n_fit_idx, _val_idx = train_test_split(\n _idx, test_size=VAL_FRACTION, stratify=_ytr, random_state=VAL_SPLIT_SEED\n)\n_Xfit = _Xtr.iloc[_fit_idx]\n_Xval = _Xtr.iloc[_val_idx]\n_yfit = _ytr[_fit_idx]\n_yval = _ytr[_val_idx]\n\n_base = _make_model(_best, _spw(_yfit))\n_base.fit(_Xfit, _yfit)\n\n_val_raw = _base.predict_proba(_Xval)[:, 1]\n_iso = IsotonicRegression(out_of_bounds=\"clip\")\n_iso.fit(_val_raw, _yval)\n\n\ndef predict_proba(X):\n raw = _base.predict_proba(X)[:, 1]\n return _iso.predict(raw)\n\n\n_val_cal = _iso.predict(_val_raw)\n_grid = np.unique(np.concatenate([np.array([0.0, 1.0]), _val_cal]))\n_best_t, _best_f1 = 0.5, -1.0\nfor _t in _grid:\n _f1 = f1_score(_yval, (_val_cal >= _t).astype(int), zero_division=0)\n if _f1 > _best_f1:\n _best_f1, _best_t = _f1, float(_t)\nchosen_threshold = float(_best_t)\n\n_check = predict_proba(_Xtr.head(5))\nassert _check.shape[0] == 5\n\nresult = {\n \"best_hyperparameters\": {k: (int(v) if k in _INT_PARAMS else float(v)) for k, v in _best.items()},\n \"tuned_params\": sorted(_search.keys()),\n \"n_trials\": _n_trials,\n \"optuna_best_value\": float(_study.best_value),\n \"cv_metric\": \"average_precision\",\n \"cv_folds\": CV_FOLDS,\n \"calibration\": \"isotonic\",\n \"calibration_brier_val\": float(brier_score_loss(_yval, _val_cal)),\n \"val_fraction\": VAL_FRACTION,\n \"chosen_threshold\": chosen_threshold,\n \"chosen_threshold_val_f1\": float(_best_f1),\n \"scale_pos_weight_fit\": float(_spw(_yfit)),\n \"n_train_fit\": int(len(_fit_idx)),\n \"n_val\": int(len(_val_idx)),\n}\n" - }, - "factor_bindings": {} - } - } - ], - "edges": [ - { - "id": "e-dc-to-gatedc", - "source": "agent-dc", - "target": "gate-dc" - }, - { - "id": "e-gatedc-to-fte", - "source": "gate-dc", - "target": "agent-fte" - }, - { - "id": "e-fte-to-gatefte", - "source": "agent-fte", - "target": "gate-fte" - }, - { - "id": "e-gatefte-to-fs", - "source": "gate-fte", - "target": "agent-fs" - }, - { - "id": "e-fs-to-gatefs", - "source": "agent-fs", - "target": "gate-fs" - }, - { - "id": "e-gatefs-to-mlm", - "source": "gate-fs", - "target": "agent-mlm" - }, - { - "id": "e-mlm-to-gatemlm", - "source": "agent-mlm", - "target": "gate-mlm" - }, - { - "id": "e-llm-dc", - "source": "llm-shared", - "target": "agent-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fte", - "source": "llm-shared", - "target": "agent-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-fs", - "source": "llm-shared", - "target": "agent-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-mlm", - "source": "llm-shared", - "target": "agent-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatedc", - "source": "llm-shared", - "target": "gate-dc", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefte", - "source": "llm-shared", - "target": "gate-fte", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatefs", - "source": "llm-shared", - "target": "gate-fs", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-llm-gatemlm", - "source": "llm-shared", - "target": "gate-mlm", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-dc", - "source": "pattern-dc", - "target": "agent-dc", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fte", - "source": "pattern-fte", - "target": "agent-fte", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-fs", - "source": "pattern-fs", - "target": "agent-fs", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-pattern-mlm", - "source": "pattern-mlm", - "target": "agent-mlm", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-workspace-dc", - "source": "tool-workspace-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-dc", - "source": "tool-eda-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-dc", - "source": "tool-dc", - "target": "agent-dc", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fte", - "source": "tool-workspace-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fte", - "source": "tool-eda-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fte", - "source": "tool-fte", - "target": "agent-fte", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-workspace-fs", - "source": "tool-workspace-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-eda-fs", - "source": "tool-eda-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-tool-fs", - "source": "tool-fs", - "target": "agent-fs", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-dataset-agent-dc", - "source": "dataset-mi", - "target": "agent-dc", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fte", - "source": "dataset-mi", - "target": "agent-fte", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-dataset-agent-fs", - "source": "dataset-mi", - "target": "agent-fs", - "sourceHandle": "resource", - "targetHandle": "resource" - }, - { - "id": "e-gatemlm-to-score", - "source": "gate-mlm", - "target": "agent-score" - }, - { - "id": "e-llm-score", - "source": "llm-shared", - "target": "agent-score", - "sourceHandle": "model", - "targetHandle": "model" - }, - { - "id": "e-pattern-score", - "source": "pattern-score", - "target": "agent-score", - "sourceHandle": "architectural_pattern", - "targetHandle": "architectural_pattern" - }, - { - "id": "e-tool-model-score", - "source": "tool-model-score", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - }, - { - "id": "e-script-model-score", - "source": "script-model", - "target": "agent-score", - "sourceHandle": "tool", - "targetHandle": "tool" - } - ] - }, - "design_spec": { - "factors": [ - { - "name": "OpenAI:Model", - "levels": [ - "gpt-5-mini", - "gpt-5" - ], - "level_type": "string" - }, - { - "name": "OpenAI:Effort", - "levels": [ - "medium", - "high" - ], - "level_type": "string" - }, - { - "name": "Critic enabled", - "levels": [ - true, - false - ], - "level_type": "boolean" - } - ], - "replicates": 10, - "metrics": [ - { - "name": "average_precision", - "primary": true, - "direction": "maximize" - }, - { - "name": "roc_auc", - "primary": false, - "direction": "maximize" - }, - { - "name": "f1", - "primary": false, - "direction": "maximize" - }, - { - "name": "balanced_accuracy", - "primary": false, - "direction": "maximize" - }, - { - "name": "accuracy", - "primary": false, - "direction": "maximize" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - } -} diff --git a/publications/BDM/myocardial-openai-latest.json b/publications/BDM/myocardial-openai-v0.8.0.json similarity index 100% rename from publications/BDM/myocardial-openai-latest.json rename to publications/BDM/myocardial-openai-v0.8.0.json From bb49dcf169f6c372e69da337811f57b3643a96af Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 22:03:09 -0700 Subject: [PATCH 11/18] Localize MCP servers when importing experiments --- src/asaree/api/experiments.py | 56 ++++++++- tests/test_experiment_creation_contract.py | 134 +++++++++++++++++++++ 2 files changed, 186 insertions(+), 4 deletions(-) diff --git a/src/asaree/api/experiments.py b/src/asaree/api/experiments.py index 43fb44a9..83c86fd6 100644 --- a/src/asaree/api/experiments.py +++ b/src/asaree/api/experiments.py @@ -15,6 +15,7 @@ from typing import Any from fastapi import APIRouter, HTTPException, Response +from motoro.services import mcp_service from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.exc import IntegrityError @@ -280,6 +281,43 @@ def _reject_locked_mutation(experiment: Any, fields: dict[str, Any]) -> None: ) +def _localize_imported_mcp_references( + graph: dict[str, Any], + measurement_plan: dict[str, Any] | None, + server_ids_by_name: dict[str, str], +) -> tuple[dict[str, Any], dict[str, Any] | None]: + """Replace installation-specific MCP UUIDs using portable server names.""" + localized_graph = deepcopy(graph) + localized_plan = deepcopy(measurement_plan) + server_id_by_node: dict[str, str] = {} + + for node in localized_graph.get("nodes", []): + if not isinstance(node, dict) or node.get("type") not in {"mcp_tool", "mcp_scikit_learn", "mcp_client_tool"}: + continue + data = node.get("data") + config = data.get("config") if isinstance(data, dict) else None + if not isinstance(config, dict): + continue + server_id = server_ids_by_name.get(str(config.get("server_name") or "")) + if server_id is None: + continue + config["server_id"] = server_id + server_id_by_node[str(node.get("id"))] = server_id + + if localized_plan is not None: + for producer in localized_plan.get("producers", []): + if not isinstance(producer, dict) or producer.get("producer_id") != "asaree.mcp_tool": + continue + config = producer.get("config") + if not isinstance(config, dict): + continue + server_id = server_id_by_node.get(str(config.get("mcp_node_id") or "")) + if server_id is not None: + config["server_id"] = server_id + + return localized_graph, localized_plan + + async def _validated_dataset_ids(dataset_ids: list[uuid.UUID], db: DbSession, user: CurrentUser) -> list[uuid.UUID]: for dataset_id in dataset_ids: dataset = await get_dataset(db, dataset_id) @@ -473,15 +511,25 @@ async def import_experiment_definition_endpoint( graph = normalize_protocol_graph(body.graph) published_graph = normalize_protocol_graph(body.published_graph) if body.published_graph is not None else None + servers = await mcp_service.list_servers(owner_id=user.id) + server_ids_by_name = {str(server.name): str(server.id) for server in servers} + if published_graph is None: + graph, measurement_plan = _localize_imported_mcp_references(graph, body.measurement_plan, server_ids_by_name) + else: + graph, _ = _localize_imported_mcp_references(graph, None, server_ids_by_name) + published_graph, measurement_plan = _localize_imported_mcp_references( + published_graph, body.measurement_plan, server_ids_by_name + ) try: design_spec = normalize_design_spec(body.design_spec, validate_metrics=True) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc - # Do not attach source dataset IDs here. They refer to source-workspace - # artifacts and may not exist for this owner; their node configuration is - # retained in the graph so the user can supply/reconnect them after import. + # Do not attach source dataset IDs here. They refer to source-workspace + # artifacts and may not exist for this owner. MCP references are different: + # server names are portable, so matching visible registrations above replace + # source-install UUIDs before measurement-plan validation. try: # The lookup above gives the dialog its friendly suggested name. This # savepoint covers the unavoidable race where another tab takes it @@ -496,7 +544,7 @@ async def import_experiment_definition_endpoint( design_type=body.design_type, task_brief=body.task_brief, design_spec=design_spec, - measurement_plan=body.measurement_plan, + measurement_plan=measurement_plan, ) # `publish_protocol` can only freeze the protocol's current draft. # Start from the source published snapshot when one exists, freeze diff --git a/tests/test_experiment_creation_contract.py b/tests/test_experiment_creation_contract.py index d7ff2c58..d4e9eb79 100644 --- a/tests/test_experiment_creation_contract.py +++ b/tests/test_experiment_creation_contract.py @@ -9,11 +9,25 @@ from unittest.mock import AsyncMock import pytest +from fastapi import HTTPException from asaree.api import experiments as experiments_api from asaree.services.protocols import generated_protocol_name +class _NestedTransaction: + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_args: object) -> None: + return None + + +class _FakeDb: + def begin_nested(self) -> _NestedTransaction: + return _NestedTransaction() + + async def test_create_endpoint_provisions_linked_empty_canvas(monkeypatch: pytest.MonkeyPatch) -> None: owner_id = uuid.uuid4() experiment_id = uuid.uuid4() @@ -59,3 +73,123 @@ async def test_create_endpoint_provisions_linked_empty_canvas(monkeypatch: pytes owner_id=owner_id, experiment_id=experiment_id, ) + + +async def test_import_definition_localizes_mcp_server_ids_by_name(monkeypatch: pytest.MonkeyPatch) -> None: + owner_id = uuid.uuid4() + experiment_id = uuid.uuid4() + source_server_id = uuid.uuid4() + local_server_id = uuid.uuid4() + now = datetime.now(UTC) + experiment = SimpleNamespace( + id=experiment_id, + name="Portable experiment", + description=None, + hypothesis=None, + design_type="factorial", + task_brief=None, + design_spec={"metrics": []}, + measurement_plan=None, + metric_recommendations=None, + latest_test_run_id=None, + archived_at=None, + locked_at=None, + locked_protocol_revision_id=None, + locked_design_spec=None, + locked_measurement_plan=None, + owner_id=owner_id, + created_at=now, + updated_at=now, + ) + graph = { + "nodes": [ + {"id": "agent", "type": "agent", "data": {"label": "Agent"}}, + { + "id": "tool", + "type": "mcp_tool", + "data": { + "label": "Scorer", + "config": { + "server_id": str(source_server_id), + "server_name": "asaree-sklearn-model", + "tool_names": ["run_model_script"], + "enabled": True, + }, + }, + }, + ], + "edges": [{"source": "tool", "target": "agent", "targetHandle": "tool"}], + } + measurement_plan = { + "metrics": [ + { + "id": "model-evaluation", + "name": "Model evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": False, + } + ], + "producers": [ + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": {"value": "model-evaluation"}, + "artifacts": [], + "config": { + "agent_node_id": "agent", + "mcp_node_id": "tool", + "server_id": str(source_server_id), + "tool_name": "run_model_script", + }, + } + ], + "inputs": [], + } + experiment.measurement_plan = measurement_plan + + async def require_local_server(_db: object, **kwargs: Any) -> None: + imported_graph = kwargs["graph"] + imported_plan = kwargs["document"] + node_server_id = imported_graph["nodes"][1]["data"]["config"]["server_id"] + producer_server_id = imported_plan["producers"][0]["config"]["server_id"] + if node_server_id != str(local_server_id) or producer_server_id != str(local_server_id): + raise HTTPException(status_code=422, detail="The registered MCP Server is unavailable.") + + async def create_imported_experiment(_db: object, **kwargs: Any) -> SimpleNamespace: + experiment.measurement_plan = kwargs["measurement_plan"] + return experiment + + monkeypatch.setattr(experiments_api, "get_experiment_by_name", AsyncMock(return_value=None)) + monkeypatch.setattr(experiments_api, "create_experiment", create_imported_experiment) + monkeypatch.setattr(experiments_api, "create_protocol", AsyncMock(return_value=SimpleNamespace())) + monkeypatch.setattr(experiments_api, "_require_valid_measurement_plan", require_local_server) + monkeypatch.setattr(experiments_api, "get_experiment_dataset_ids", AsyncMock(return_value=[])) + monkeypatch.setattr( + experiments_api.mcp_service, + "list_servers", + AsyncMock( + return_value=[ + SimpleNamespace( + id=local_server_id, + name="asaree-sklearn-model", + owner_id=None, + is_system=True, + ) + ] + ), + ) + response = await experiments_api.import_experiment_definition_endpoint( + experiments_api.ImportExperimentDefinitionRequest( + name="Portable experiment", + design_spec={"metrics": []}, + measurement_plan=measurement_plan, + graph=graph, + ), + cast(Any, SimpleNamespace(id=owner_id)), + cast(Any, _FakeDb()), + ) + + assert response.id == experiment_id From 58bfff8e0704c3539710e124b6906875df626654 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 22:10:14 -0700 Subject: [PATCH 12/18] Remove canvas baseline factor badges --- frontend/src/components/protocol/FactorBindableField.tsx | 1 - frontend/src/components/protocol/FactorEditorDialog.tsx | 3 --- 2 files changed, 4 deletions(-) diff --git a/frontend/src/components/protocol/FactorBindableField.tsx b/frontend/src/components/protocol/FactorBindableField.tsx index 863fde49..ef0b0948 100644 --- a/frontend/src/components/protocol/FactorBindableField.tsx +++ b/frontend/src/components/protocol/FactorBindableField.tsx @@ -264,7 +264,6 @@ export function FactorBindableField({ {levels.map((level, i) => (
- {i === 0 && Canvas baseline} {levelOptions && levelOptions.length > 0 ? ( (
- {i === 0 && Canvas baseline} {(levels as string[]).map((level, i) => (
- {i === 0 && Canvas baseline} Date: Mon, 28 Sep 2026 22:19:12 -0700 Subject: [PATCH 13/18] Migrate BDM output contracts to parser nodes --- publications/BDM/README.md | 4 + .../BDM/myocardial-anthropic-v0.8.0.json | 490 +++++++++++------- .../BDM/myocardial-azure-foundry-v0.8.0.json | 490 +++++++++++------- .../BDM/myocardial-openai-v0.8.0.json | 490 +++++++++++------- 4 files changed, 919 insertions(+), 555 deletions(-) diff --git a/publications/BDM/README.md b/publications/BDM/README.md index 340ae2ff..d7fa219a 100644 --- a/publications/BDM/README.md +++ b/publications/BDM/README.md @@ -64,6 +64,10 @@ critic gates, with these v0.8.0 execution details: instead of retyping the wired script. - **Published execution.** The import helper publishes the localized graph as an immutable protocol revision before generating or running replicates. +- **Visible output contracts.** Each agent's declared output shape lives in a + connected Output Parser node rather than the legacy hidden + `config.output_contract` field. The contracts and runtime behavior are + unchanged; the canvas now exposes where each structured payload is defined. - **Declared measurement source.** The complete held-out scoring response is captured as the opaque `Model evaluation` metric from MI-Score's exact `asaree-sklearn-model.run_model_script` call. This retains the returned test diff --git a/publications/BDM/myocardial-anthropic-v0.8.0.json b/publications/BDM/myocardial-anthropic-v0.8.0.json index 70e4c1ba..a4fdff19 100644 --- a/publications/BDM/myocardial-anthropic-v0.8.0.json +++ b/publications/BDM/myocardial-anthropic-v0.8.0.json @@ -68,58 +68,10 @@ "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -220,33 +172,10 @@ "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -347,63 +276,10 @@ "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -505,33 +381,10 @@ "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -588,33 +441,10 @@ "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with NO `code` argument -- the wired Script node reaches the tool as a path in ambient run context and executes byte-for-byte as written; retyping it is the one thing that could corrupt it. Pass only payload_json, set to the approved payload (as compact JSON) from the upstream context -- do not alter it. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script (the script arrives as an ambient path, never pasted into the prompt) and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -673,6 +503,261 @@ }, "factor_bindings": {} } + }, + { + "id": "output-parser-dc", + "type": "output_parser", + "position": { + "x": 236, + "y": 160 + }, + "data": { + "label": "MI-DC Output Parser", + "config": { + "output_contract": { + "name": "DCReport", + "fields": [ + { + "name": "n_rows", + "type": "integer", + "description": "Row count after cleaning" + }, + { + "name": "n_cols_in", + "type": "integer", + "description": "Column count before cleaning" + }, + { + "name": "n_cols_out", + "type": "integer", + "description": "Column count after cleaning" + }, + { + "name": "domain_violations", + "type": "array", + "description": "Domain-rule violations found and corrected" + }, + { + "name": "missingness_threshold", + "type": "number", + "description": "Threshold used to drop high-missingness columns" + }, + { + "name": "dropped_for_missingness", + "type": "array", + "description": "Columns dropped for exceeding the missingness threshold" + }, + { + "name": "outliers", + "type": "object", + "description": "Outlier handling summary per column" + }, + { + "name": "imputation", + "type": "object", + "description": "Imputation strategy used per column" + }, + { + "name": "notes_for_fte", + "type": "string", + "description": "Handoff notes for the MI-FTE stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fte", + "type": "output_parser", + "position": { + "x": 796, + "y": 160 + }, + "data": { + "label": "MI-FTE Output Parser", + "config": { + "output_contract": { + "name": "FTEReport", + "fields": [ + { + "name": "engineering_recipe", + "type": "object", + "description": "Feature-engineering steps applied" + }, + { + "name": "encoding_map", + "type": "object", + "description": "Categorical encoding scheme per column" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after engineering/encoding" + }, + { + "name": "notes_for_fs", + "type": "string", + "description": "Handoff notes for the MI-FS stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fs", + "type": "output_parser", + "position": { + "x": 1356, + "y": 160 + }, + "data": { + "label": "MI-FS Output Parser", + "config": { + "output_contract": { + "name": "FSReport", + "fields": [ + { + "name": "n_features_in", + "type": "integer", + "description": "Feature count before selection" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after selection" + }, + { + "name": "variance_threshold", + "type": "number", + "description": "Variance filter threshold used" + }, + { + "name": "correlation", + "type": "object", + "description": "Correlation-filtering summary" + }, + { + "name": "supervised_filter", + "type": "object", + "description": "Supervised-signal filtering summary" + }, + { + "name": "dropped_features", + "type": "array", + "description": "Features dropped during selection" + }, + { + "name": "selected_features", + "type": "array", + "description": "Final selected feature set" + }, + { + "name": "observed_class_distribution", + "type": "object", + "description": "Observed target class distribution" + }, + { + "name": "class_balance_check", + "type": "boolean", + "description": "Whether class balance passed the configured check" + }, + { + "name": "notes_for_mlm", + "type": "string", + "description": "Handoff notes for the MI-MLM stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-mlm", + "type": "output_parser", + "position": { + "x": 1916, + "y": 160 + }, + "data": { + "label": "MI-MLM Output Parser", + "config": { + "output_contract": { + "name": "MLMPayload", + "fields": [ + { + "name": "search_space", + "type": "object", + "description": "Hyperparameter search space proposal" + }, + { + "name": "fixed_params", + "type": "object", + "description": "Fixed (non-searched) XGBoost params" + }, + { + "name": "n_trials", + "type": "integer", + "description": "Proposed Optuna trial count" + }, + { + "name": "rationale", + "type": "string", + "description": "Rationale for the proposed search space" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-score", + "type": "output_parser", + "position": { + "x": 2476, + "y": 160 + }, + "data": { + "label": "MI-Score Output Parser", + "config": { + "output_contract": { + "name": "ScoreReport", + "fields": [ + { + "name": "test_metrics", + "type": "object", + "description": "Held-out test metrics computed server-side" + }, + { + "name": "code_sha256", + "type": "string", + "description": "Hash of the executed script, for the verbatim-execution guard" + }, + { + "name": "payload_sha256", + "type": "string", + "description": "Hash of the scored payload, for the approved-payload guard" + }, + { + "name": "result", + "type": "object", + "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } } ], "edges": [ @@ -890,6 +975,41 @@ "target": "agent-score", "sourceHandle": "tool", "targetHandle": "tool" + }, + { + "id": "e-output-parser-dc", + "source": "output-parser-dc", + "target": "agent-dc", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fte", + "source": "output-parser-fte", + "target": "agent-fte", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fs", + "source": "output-parser-fs", + "target": "agent-fs", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-mlm", + "source": "output-parser-mlm", + "target": "agent-mlm", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-score", + "source": "output-parser-score", + "target": "agent-score", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" } ] }, diff --git a/publications/BDM/myocardial-azure-foundry-v0.8.0.json b/publications/BDM/myocardial-azure-foundry-v0.8.0.json index 583db625..073fbc9e 100644 --- a/publications/BDM/myocardial-azure-foundry-v0.8.0.json +++ b/publications/BDM/myocardial-azure-foundry-v0.8.0.json @@ -68,58 +68,10 @@ "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -220,33 +172,10 @@ "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -347,63 +276,10 @@ "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -505,33 +381,10 @@ "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -588,33 +441,10 @@ "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with NO `code` argument -- the wired Script node reaches the tool as a path in ambient run context and executes byte-for-byte as written; retyping it is the one thing that could corrupt it. Pass only payload_json, set to the approved payload (as compact JSON) from the upstream context -- do not alter it. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script (the script arrives as an ambient path, never pasted into the prompt) and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -673,6 +503,261 @@ }, "factor_bindings": {} } + }, + { + "id": "output-parser-dc", + "type": "output_parser", + "position": { + "x": 236, + "y": 160 + }, + "data": { + "label": "MI-DC Output Parser", + "config": { + "output_contract": { + "name": "DCReport", + "fields": [ + { + "name": "n_rows", + "type": "integer", + "description": "Row count after cleaning" + }, + { + "name": "n_cols_in", + "type": "integer", + "description": "Column count before cleaning" + }, + { + "name": "n_cols_out", + "type": "integer", + "description": "Column count after cleaning" + }, + { + "name": "domain_violations", + "type": "array", + "description": "Domain-rule violations found and corrected" + }, + { + "name": "missingness_threshold", + "type": "number", + "description": "Threshold used to drop high-missingness columns" + }, + { + "name": "dropped_for_missingness", + "type": "array", + "description": "Columns dropped for exceeding the missingness threshold" + }, + { + "name": "outliers", + "type": "object", + "description": "Outlier handling summary per column" + }, + { + "name": "imputation", + "type": "object", + "description": "Imputation strategy used per column" + }, + { + "name": "notes_for_fte", + "type": "string", + "description": "Handoff notes for the MI-FTE stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fte", + "type": "output_parser", + "position": { + "x": 796, + "y": 160 + }, + "data": { + "label": "MI-FTE Output Parser", + "config": { + "output_contract": { + "name": "FTEReport", + "fields": [ + { + "name": "engineering_recipe", + "type": "object", + "description": "Feature-engineering steps applied" + }, + { + "name": "encoding_map", + "type": "object", + "description": "Categorical encoding scheme per column" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after engineering/encoding" + }, + { + "name": "notes_for_fs", + "type": "string", + "description": "Handoff notes for the MI-FS stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fs", + "type": "output_parser", + "position": { + "x": 1356, + "y": 160 + }, + "data": { + "label": "MI-FS Output Parser", + "config": { + "output_contract": { + "name": "FSReport", + "fields": [ + { + "name": "n_features_in", + "type": "integer", + "description": "Feature count before selection" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after selection" + }, + { + "name": "variance_threshold", + "type": "number", + "description": "Variance filter threshold used" + }, + { + "name": "correlation", + "type": "object", + "description": "Correlation-filtering summary" + }, + { + "name": "supervised_filter", + "type": "object", + "description": "Supervised-signal filtering summary" + }, + { + "name": "dropped_features", + "type": "array", + "description": "Features dropped during selection" + }, + { + "name": "selected_features", + "type": "array", + "description": "Final selected feature set" + }, + { + "name": "observed_class_distribution", + "type": "object", + "description": "Observed target class distribution" + }, + { + "name": "class_balance_check", + "type": "boolean", + "description": "Whether class balance passed the configured check" + }, + { + "name": "notes_for_mlm", + "type": "string", + "description": "Handoff notes for the MI-MLM stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-mlm", + "type": "output_parser", + "position": { + "x": 1916, + "y": 160 + }, + "data": { + "label": "MI-MLM Output Parser", + "config": { + "output_contract": { + "name": "MLMPayload", + "fields": [ + { + "name": "search_space", + "type": "object", + "description": "Hyperparameter search space proposal" + }, + { + "name": "fixed_params", + "type": "object", + "description": "Fixed (non-searched) XGBoost params" + }, + { + "name": "n_trials", + "type": "integer", + "description": "Proposed Optuna trial count" + }, + { + "name": "rationale", + "type": "string", + "description": "Rationale for the proposed search space" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-score", + "type": "output_parser", + "position": { + "x": 2476, + "y": 160 + }, + "data": { + "label": "MI-Score Output Parser", + "config": { + "output_contract": { + "name": "ScoreReport", + "fields": [ + { + "name": "test_metrics", + "type": "object", + "description": "Held-out test metrics computed server-side" + }, + { + "name": "code_sha256", + "type": "string", + "description": "Hash of the executed script, for the verbatim-execution guard" + }, + { + "name": "payload_sha256", + "type": "string", + "description": "Hash of the scored payload, for the approved-payload guard" + }, + { + "name": "result", + "type": "object", + "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } } ], "edges": [ @@ -890,6 +975,41 @@ "target": "agent-score", "sourceHandle": "tool", "targetHandle": "tool" + }, + { + "id": "e-output-parser-dc", + "source": "output-parser-dc", + "target": "agent-dc", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fte", + "source": "output-parser-fte", + "target": "agent-fte", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fs", + "source": "output-parser-fs", + "target": "agent-fs", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-mlm", + "source": "output-parser-mlm", + "target": "agent-mlm", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-score", + "source": "output-parser-score", + "target": "agent-score", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" } ] }, diff --git a/publications/BDM/myocardial-openai-v0.8.0.json b/publications/BDM/myocardial-openai-v0.8.0.json index 3a8ad508..ae79876b 100644 --- a/publications/BDM/myocardial-openai-v0.8.0.json +++ b/publications/BDM/myocardial-openai-v0.8.0.json @@ -68,58 +68,10 @@ "goal": "Clean the training matrix; commit v1_dc. After drop_and_impute succeeds, call accept_stage(stage=\"dc\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Inspects columns, applies domain coercions, drops/imputes for missingness, and commits the cleaned matrix as workspace version v1_dc.", "system_prompt": "\nYou are a DATA CLEANING agent for biomedical tabular data. You run FIRST. Your\nsingle responsibility is to hand the downstream transform/engineering agent a\nCLEAN, COMPLETE training matrix. You do this in three tool calls: INSPECT every\ncolumn, COERCE bad/implausible values to NaN, then DROP over-sparse columns and\nIMPUTE what remains. You do NOT encode, engineer, select, or model. You do NOT\ntouch the test split, and you never see the target column.\n\nWORKSPACE MODEL (read this first): pipeline state is handed to the next stage as\nversioned train/test files on disk, NOT as artifacts you save by key. Each cleaning\ntool fits on the TRAINING split only, applies the same frozen fit to BOTH splits,\nand writes this stage's dataset version (v1_dc); the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. The test split is transformed for you but NEVER shown to you (leakage is\nstructural). There is no save_artifact step and there are no artifact keys to echo.\n\nRUN CONTEXT\nYou will receive:\n- task_brief: { domain, prediction_task (\"binary\" | \"multiclass\"), target_column,\n positive_class, expected_positive_rate, num_classes, group_column, notes }.\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the cleaning tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"dc\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive: they encode the clinical concept and the feature\ntype (lab, diagnosis, medication, demographic, vital, procedure, etc.). Use\nthem, plus the data dictionary, to know each column's expected type and domain.\nYOU own every judgment here — the tools are mechanisms, not decision-makers. They\ncoerce, drop, and impute exactly the columns and bounds you name and nothing else;\nnone of them delete a value or a column on their own. Ground every decision in the\ndata; if a field is missing or contradictory, flag it in notes_for_fte, do not\ninvent.\n\nYOUR TOOLS — the five you have, and exactly when to call each. There are NO other DC\ntools: do NOT call a separate domain-check, outlier, or missingness tool (they were\nremoved) — inspect_columns and apply_coercions already cover domain, outliers, and\nmissingness. Calling anything not listed here fails.\n- open_workspace(stage=\"dc\"): STEP 1, once — stage your scratch input into the\n already-open workspace; every later call is scoped to it automatically. Pass\n stage=\"dc\" and nothing else — omitting stage means your other tools will find\n no scratch input and fail; passing anything else overrides what the run\n already knows.\n- get_data_dictionary(name, columns=...): read-only, any time — a column's meaning,\n type, or codes; never changes data.\n- inspect_columns(): STEP 2, once — the SINGLE read-only per-column report (missingness,\n inferred type, IQR outlier flags, non-numeric-token counts). Your evidence base.\n- apply_coercions(rules_json): STEP 3 — coerce integrity errors AND implausible outliers\n to NaN; commits v1_dc and returns the post-coercion missingness you threshold on next.\n- drop_and_impute(drop_json, strategy_json, protect_json): STEP 4, terminal — drop the\n over-sparse columns you chose and impute every remaining NaN; commits v1_dc.\nFixed order: inspect_columns → apply_coercions → drop_and_impute. inspect_columns is\nread-only; the other two mutate and commit v1_dc.\n\nSTEPS (this order matters; do not reorder)\n1. Call open_workspace(stage=\"dc\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It stages your scratch input from the already-seeded workspace. When you need a\n column's meaning, type, or codes, call get_data_dictionary(columns=\"col1,col2\")\n for just those columns (omit columns for a compact index of every column).\n\n2. INSPECT every column. Call inspect_columns(). It returns ONE read-only\n per-column report on the TRAINING split — nothing is changed or committed —\n with, per column: missingness (count + pct), inferred type, and for NUMERIC\n columns summary stats plus an IQR block (Q1/Q3, the lower/upper fences, and the\n count of values outside them); for OBJECT columns cardinality, the top values,\n and a count of NON-NUMERIC TOKENS — so you can spot a semantically-continuous\n column loaded as text only because a minority of cells hold censored values\n (\"< 5\", \"<0.2\", \">100\"), a unit/flag suffix (\"123 L\"), or free-text (\"SEE\n NOTE\"). Read this report and decide, per column, what needs coercing and —\n later — what is too sparse to keep. This is the ONLY inspection call; it\n replaces separate domain, outlier, and missingness passes.\n\n3. COERCE bad and implausible values to NaN. Call apply_coercions(rules_json)\n with a per-column rule list. Each rule is an object:\n {\"feature\": , \"type\": \"numeric\"|\"categorical\" (optional),\n \"min\": (optional), \"max\": (optional), \"nonneg\": true (optional),\n \"allowed\": [] (optional), \"reason\": \"<1-line why>\"}.\n Any value violating a rule is coerced to NaN — the CELL only; its row and every\n other column are preserved (nothing here drops a row). A numeric \"type\" also\n parses a text-typed numeric column to float, sending non-numeric tokens to NaN.\n Use rules for BOTH kinds of bad value:\n - DATA-INTEGRITY errors: a string in a numeric column, a category outside the\n known level set, a negative count or negative lab, an impossible value.\n - IMPLAUSIBLE OUTLIERS: an IQR flag from step 2 is a REVIEW signal, not an\n automatic delete, and the decision is YOURS. In clinical data the extreme\n tail is often the important part (a lactate of 12, a creatinine of 9, a\n heart rate of 180 are real and predictive). KEEP a flagged value that is\n physiologically plausible even if statistically extreme; DELETE only values\n that are physiologically implausible (impossible vitals, out-of-range labs,\n data-entry artifacts), and express the delete as a per-column min/max bound\n (never a row index) so it replays leak-free on the test split. State a\n one-line reason per bound.\n apply_coercions commits the coerced matrix and RETURNS the post-coercion\n missingness per column — the real numbers (native NaNs plus everything you just\n coerced) you threshold on in step 4. Coerced/censored cells are imputed in\n step 4 like any other NaN.\n\n4. DROP over-sparse columns, then IMPUTE — the terminal step. Using the\n post-coercion missingness from step 3, SET A MISSINGNESS THRESHOLD YOURSELF and\n STATE its rationale; it is NOT hardcoded and no tool picks it for you. Then call\n drop_and_impute(drop_json, strategy_json, protect_json):\n - drop_json: the explicit list of columns whose measured missingness exceeds\n your threshold (cite each value). ONLY the columns you name are dropped.\n - protect_json: columns that must NEVER be dropped — pass task_brief's\n group_column and any identifier columns here (omit if group_column is null).\n - strategy_json: per-column impute strategy — mode for binary/categorical,\n mean OR median for continuous (median when skewed, mean when roughly\n symmetric; state which and why). Columns you do not list get a sensible\n default for their type.\n This tool drops the named columns, imputes EVERY remaining NaN (train-fit only,\n applied to both splits), and commits the cleaned, fully-imputed matrices as this\n stage's output (v1_dc). No missing values may remain and train/test columns must\n match. It refuses to coerce a still-text numeric column (if so, add a numeric\n \"type\" rule in step 3 first) and refuses to drop every column.\n\nCONCISENESS: report only what you changed or flagged. List dropped, coerced, or\nnon-default-handled columns explicitly, but summarize untouched columns by COUNT\n— do NOT enumerate all columns. Keep every rationale to one line. The committed\ndataset version already carries the full per-column detail; the runner reads it,\nso your report does not need to restate it.\n\nOUTPUT - return ONE JSON report:\n{\n \"n_rows\": ..., \"n_cols_in\": ..., \"n_cols_out\": ...,\n \"domain_violations\": [\n {\"feature\": \"...\", \"rule\": \"expected type/domain\", \"n_coerced\": 0}\n ],\n \"missingness_threshold\": 0.0,\n \"dropped_for_missingness\": [[\"feature\", 0.0], ...], # [name, raw missing pct]\n \"outliers\": [\n {\"feature\": \"...\", \"method\": \"IQR\", \"n_flagged\": 0, \"n_deleted\": 0,\n \"reason\": \"why deleted ones were implausible / why others were kept\"}\n ],\n \"imputation\": [\n {\"feature_or_group\": \"...\", \"dtype\": \"binary|categorical|continuous\",\n \"strategy\": \"mode|mean|median\", \"reason\": \"skew/symmetry note\"}\n ],\n \"notes_for_fte\": \"data-quality concerns, anything FTE or the critic should \"\n \"know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "DCReport", - "fields": [ - { - "name": "n_rows", - "type": "integer", - "description": "Row count after cleaning" - }, - { - "name": "n_cols_in", - "type": "integer", - "description": "Column count before cleaning" - }, - { - "name": "n_cols_out", - "type": "integer", - "description": "Column count after cleaning" - }, - { - "name": "domain_violations", - "type": "array", - "description": "Domain-rule violations found and corrected" - }, - { - "name": "missingness_threshold", - "type": "number", - "description": "Threshold used to drop high-missingness columns" - }, - { - "name": "dropped_for_missingness", - "type": "array", - "description": "Columns dropped for exceeding the missingness threshold" - }, - { - "name": "outliers", - "type": "object", - "description": "Outlier handling summary per column" - }, - { - "name": "imputation", - "type": "object", - "description": "Imputation strategy used per column" - }, - { - "name": "notes_for_fte", - "type": "string", - "description": "Handoff notes for the MI-FTE stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -220,33 +172,10 @@ "goal": "Engineer and encode features; commit v2_fte. After fit_preprocessor succeeds, call accept_stage(stage=\"fte\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Builds and fits feature-engineering/encoding transforms on top of v1_dc, commits the result as workspace version v2_fte.", "system_prompt": "\nYou are a feature TRANSFORMATION and ENGINEERING agent for biomedical tabular\ndata. You run SECOND, on the CLEAN, COMPLETE dataset produced by the data\ncleaning agent (no missing values, integrity errors already fixed). Your job is\nto decide per-column encodings and engineer a well-justified set of features,\nexpressed as a REPLAYABLE RECIPE so the same columns can be recreated on the\nheld-out test set. You do NOT clean, impute, drop, or select. You do NOT touch\nthe test split.\n\nWORKSPACE MODEL (read this first): the DC stage already committed a CLEANED\ntrain/test dataset version (v1_dc). Opening the workspace loads that cleaned\nTRAINING matrix for you — you do NOT re-clean and you do NOT pass any DC keys.\nbuild_features and fit_preprocessor each fit on\nthe training split only, apply the frozen fit to BOTH splits, and write this\nstage's dataset version (v2_fte); the cell's workspace identity travels with every\ntool call automatically, so you never pass a workspace id or dataset id. The test split is transformed for you but never\nshown to you. There is no save_artifact step and there are no artifact keys.\n\nWHY A RECIPE: the test set is held out and its engineered columns must be\nrecreated from the same train-frozen statistics. So you do not write one-shot\ncode; you DECLARE each feature as a typed recipe entry, and build_features\nmaterializes it on train, freezes any train statistic into the recipe, and\napplies it to test — all in one call. Anything that depends on a training\nstatistic (a bin edge, a group median, a prevalence) is frozen at fit time and\nnever recomputed, which is what prevents leakage and train/test skew.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved DC report (context; the cleaned data is already on disk).\n- The registered dataset name, the data dictionary, and the random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which build_features and fit_preprocessor read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fte\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n\nColumn names are descriptive (clinical concept + feature type). Use them, or the\ndata spread (cardinality, distribution) when the name is uninformative.\n\nSTEPS\n1. Call open_workspace(stage=\"fte\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the CLEANED training matrix (the DC output) into your scratch input.\n Do NOT pre-clean it; it is already clean and complete.\n get_data_dictionary(columns=\"...\") gives column semantics on demand.\n2. Inspect columns (get_feature_distributions()) for dtype, cardinality,\n and distribution. This informs encoding and bin edges only.\n3. DECLARE the engineering RECIPE: a list of typed entries. Each entry has a\n unique explicit output name (deterministic; FS selects on these names and the\n test partition replays them), an op, its inputs, and any frozen params. EVERY column named in \"inputs\" MUST already exist in the cleaned v1_dc matrix — confirm it against get_data_dictionary / get_feature_distributions first. Entries do NOT chain: a recipe output name cannot be reused as another entry's input, and build_features rejects any input column not present in the matrix. Row-wise\n ops are stateless and safe; statistic-bearing ops must freeze their statistic.\n Vocabulary — the COMPLETE, CLOSED set (build_features REJECTS any op not listed here; do NOT invent ops):\n META-FEATURE / COMPOSITE (row-wise, across a group of related inputs)\n count_nonzero — how many inputs are non-zero\n count_equal(value) — how many inputs equal a literal value\n count_threshold(op, value) — how many inputs satisfy op in\n {gt,ge,lt,le,eq,ne} vs value (e.g. \"# labs above a cutoff\")\n count_missing — how many of the inputs are missing\n weighted_sum(weights) — Σ weightᵢ·inputᵢ, a weighted clinical score\n (Charlson/Elixhauser-style); weights aligns 1:1 with inputs\n sum / mean / min / max / range / std — arithmetic across inputs\n n_distinct — count of distinct values across inputs\n which_max — NAME of the largest-valued input (a categorical\n \"dominant signal\"; it is encoded downstream)\n and / or / xor / nor — boolean combination over binary inputs\n multiply(a, b, ...) — multiplicative interaction\n ratio(num, den) — a rate/ratio (2 inputs)\n SINGLE-COLUMN TRANSFORM (row-wise, one input)\n threshold(op, value) — 0/1 flag from a numeric comparison (the\n \"abnormal vs reference cutoff\" flag; count_equal only matches a\n literal, threshold does the numeric comparison)\n is_missing — 1 if the value is missing else 0\n log1p / sqrt / square / abs — monotone math (log1p and sqrt guard\n their domain -> NaN)\n STATISTIC-BEARING (auto-frozen from train, replayed on test)\n bin(input; n_bins or edges) — quantile bins from the train distribution\n group_agg(group_col, value_col, stat) — per-group stat frozen from train,\n with a fallback for groups unseen at fit\n frequency_encode(input) — replace each category by its TRAIN frequency\n CLINICAL META-FEATURES expressed through these ops, for example:\n dual_diagnosis = and(depression, sud);\n polysubstance_use = or(tobacco, stimulant, ...);\n comorbidity_count = count_nonzero(pmh_*);\n comorbidity bucket = or(...) over a Charlson/Elixhauser-style group;\n shock_index = ratio(heart_rate, systolic_bp);\n charlson_score = weighted_sum(, weights=[...]);\n n_abnormal_labs = count_threshold(, op=\"gt\", value=);\n polypharmacy_flag from a distinct drug-class count.\n In each example the LEFT name is the NEW output column; the inputs (depression,\n sud, pmh_*, heart_rate, ...) are ILLUSTRATIVE placeholders — substitute the\n ACTUAL column names present in THIS dataset. Never pass an output name (e.g.\n comorbidity_count) or an as-yet-uncreated feature as an input.\n NOTE: the list above is the COMPLETE, closed vocabulary — build_features\n rejects any op not on it, so do NOT invent one (there is no polynomial-degree,\n datetime, text, target-encoding, or free-form-scaling op; per-column scaling\n and one-hot/ordinal encoding are fit_preprocessor's job, step 5). Use threshold\n for a numeric \"abnormal lab\" cutoff and count_equal only to match a literal\n value already present in a column.\n On composites: a single index can wash out component signal. Do NOT default to\n one lossy index; when you build a composite, KEEP its components too unless you\n justify removing them.\n PARSIMONY: declare a feature only when you can state a specific hypothesis for\n why it helps THIS target. Prefer a small high-justification set over breadth.\n No fixed cap: decide when you have enough. The critic may push back on bloat\n (no specific reason) OR under-engineering (obvious features omitted).\n4. Call build_features(recipe_json), where recipe_json is a JSON ARRAY of the\n entries you declared in step 3 — the array itself, e.g. [{...}, {...}], not\n the full report object (a wrapping object is tolerated, the array preferred). It\n materializes the engineered columns on the (already-clean) training split,\n freezes train-derived statistics into the recipe, applies them to test, and\n commits them as v2_fte.\n5. Call fit_preprocessor to encode/scale the engineered matrix into the fully\n numeric, model-ready v2_fte. Pass your per-column decisions as\n encoding_map_json (the JSON array from your encoding_map); each entry is\n {\"feature\", \"encoding\": leave-numeric|binarize|ordinal|onehot|bin,\n \"order_or_bins\"?}. It fits on the training split only and applies the frozen\n transform to BOTH splits. This is the FINAL FTE call: it OVERWRITES v2_fte with\n the encoded matrix — the terminal FTE output the FS stage and the runner read.\n Do NOT select features or drop columns; that is the FS stage's job and you do\n not have a selection tool.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON recipe report:\n{\n \"engineering_recipe\": [\n {\"name\": \"...\", \"op\": \"...\", \"inputs\": [\"...\"],\n \"params\": { ... frozen train statistics if any ... },\n \"rationale\": \"1-line specific reason it should help THIS target\",\n \"components_retained\": true}\n ],\n \"encoding_map\": [\n {\"feature\": \"...\", \"encoding\": \"leave-numeric|binarize|ordinal|onehot|bin\",\n \"order_or_bins\": \"... (only for ordinal/bin)\",\n \"reason\": \"1-line semantic or distributional reason\"}\n ],\n \"n_features_out\": 0,\n \"notes_for_fs\": \"tool limitations hit, dtype quirks, anything FS or the critic \"\n \"should know (empty string if nothing).\"\n}\n", - "output_contract": { - "name": "FTEReport", - "fields": [ - { - "name": "engineering_recipe", - "type": "object", - "description": "Feature-engineering steps applied" - }, - { - "name": "encoding_map", - "type": "object", - "description": "Categorical encoding scheme per column" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after engineering/encoding" - }, - { - "name": "notes_for_fs", - "type": "string", - "description": "Handoff notes for the MI-FS stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -347,63 +276,10 @@ "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", - "output_contract": { - "name": "FSReport", - "fields": [ - { - "name": "n_features_in", - "type": "integer", - "description": "Feature count before selection" - }, - { - "name": "n_features_out", - "type": "integer", - "description": "Feature count after selection" - }, - { - "name": "variance_threshold", - "type": "number", - "description": "Variance filter threshold used" - }, - { - "name": "correlation", - "type": "object", - "description": "Correlation-filtering summary" - }, - { - "name": "supervised_filter", - "type": "object", - "description": "Supervised-signal filtering summary" - }, - { - "name": "dropped_features", - "type": "array", - "description": "Features dropped during selection" - }, - { - "name": "selected_features", - "type": "array", - "description": "Final selected feature set" - }, - { - "name": "observed_class_distribution", - "type": "object", - "description": "Observed target class distribution" - }, - { - "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" - }, - { - "name": "notes_for_mlm", - "type": "string", - "description": "Handoff notes for the MI-MLM stage" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -505,33 +381,10 @@ "goal": "Emit an XGBoost hyperparameter search payload only -- no tools, no code.\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Reads v3_fs's report and proposes a bounded hyperparameter search space plus fixed params for the deterministic downstream scoring script; never executes anything itself.", "system_prompt": "\nCRITICAL -- READ FIRST:\nYour ENTIRE response is a single JSON object and nothing else. You are NOT writing\na modeling script. The harness already owns the one fixed script; you only choose\nits hyperparameters. If your output contains ANY code -- a ```python block,\n`import`, `def`, `class`, XGBClassifier(...), an Optuna objective, or a\nload/split/fit call -- it is discarded and your stage FAILS. Output only the typed\npayload described under OUTPUT FORMAT.\n\nYou are a HYPERPARAMETER PLANNER for biomedical tabular prediction. You do NOT\nwrite code. The harness owns a single fixed XGBoost modeling script; your only\njob is to choose its hyperparameter search and return it as ONE typed JSON\npayload. A reviewer gates your payload; if rejected you revise it. After approval\nthe fixed script runs with your payload bound as `hp`, on the training data only.\n\nWHAT IS FIXED BY THE HARNESS (you do NOT control these; do not fight them):\n- The model is XGBoost, objective branched on the task (binary:logistic here).\n- Inner tuning is 5-fold stratified cross-validation on the TRAINING data only,\n with a fixed global fold assignment. You never see or affect the held-out test.\n- The tuning objective is PR-AUC (average precision), fixed on clinical grounds\n for this imbalanced minority-class target.\n- scale_pos_weight is computed by the harness as n_neg/n_pos on the fitted rows.\n- Calibration (isotonic), the operating threshold (argmax F1 on a held-out\n training fold), and the Brier score are computed by the harness and are\n DESCRIPTIVE ONLY: isotonic is monotonic, so it does not change PR-AUC or\n ROC-AUC. Do not try to set them.\n- All seeds are fixed by the harness, so the run is deterministic given your\n payload.\n\nWHAT YOU CONTROL - the typed payload:\n- search_space: a list of at least 3 DISTINCT parameters to tune, each an object\n {\"param\": , \"low\": , \"high\": }. Require low < high and a\n range that lies inside the hard bounds below.\n- fixed_params: an object mapping every parameter you are NOT tuning to a single\n fixed value inside its hard bounds. Keys may ONLY be names drawn from the nine\n parameters listed below. NEVER include objective, eval_metric, tree_method,\n n_jobs, random_state/seed, scale_pos_weight, cv, calibration, or threshold --\n those are harness-fixed; any such key is silently DROPPED and the harness\n value stands (you never override them).\n- n_trials: an integer in [1, 100] - the Optuna trial budget.\n- rationale: a short string with one specific reason per non-default choice, tied\n to this roughly, imbalanced, PR-AUC-optimized target.\n\nCOMPLETENESS RULE: account for every one of the nine parameters below EXACTLY\nONCE -- either in search_space (tuned) or in fixed_params (held constant), and\nnever in both. Aim to omit none: an omitted parameter is not failed, but it\nsilently reverts to the fixed script's default and you lose all control of it, so\na complete payload is the only way to fully determine the model. The reviewer\nstill holds you to this even though it is no longer enforced by failure.\n\nTHE NINE XGBOOST PARAMETERS AND THEIR HARD BOUNDS (do not exceed):\n n_estimators integer [100, 1000]\n max_depth integer [3, 10]\n learning_rate float [1e-3, 0.3] (searched on a log scale)\n min_child_weight float [1, 20]\n gamma float [0, 5]\n subsample float [0.5, 1.0]\n colsample_bytree float [0.5, 1.0]\n reg_lambda float [1e-3, 10] (searched on a log scale)\n reg_alpha float [1e-3, 10] (searched on a log scale)\nThe harness applies the integer and log-scale handling; you only give low/high\nfor tuned params or a fixed value within the bounds. n_estimators and max_depth\ntake integer fixed values. The three log-scale params (learning_rate, reg_lambda,\nreg_alpha) must have BOTH low and high strictly positive (>= 1e-3): a low of 0 is\ninvalid on a log scale.\n\nRUN CONTEXT (informs your CHOICES; not bound to the script):\n- task_brief (target, prediction_task, selection_metric, num_classes, notes).\n- A derived pipeline brief: the final selected feature set, engineered features,\n encodings, observed class balance, and DC/FS caveats. Use the feature count and\n class balance to size the search (how deep, how much regularization, how many\n trials).\n\nMake the search purposeful: tune the parameters that matter most for an\nimbalanced tree model (commonly depth, learning_rate, and regularization or\nsubsampling), give them sensible ranges rather than the full hard bound on\neverything, fix the rest at defensible values, and pick n_trials appropriate to\nthe number of tuned dimensions. Do NOT collapse a tuned range to a single point.\n\nENFORCEMENT: a deterministic pre-flight check sanitizes your payload BEFORE it\nruns -- it does NOT fail the stage. Any suggestion that breaks a rule above -- an\nunknown or harness-fixed key (e.g. objective), a search entry missing \"param\" or\nkeyed \"name\", a range outside the hard bounds (including a log-scale low of 0),\nor a param in both blocks -- is DROPPED, and that parameter silently falls back\nto the fixed script's default, so you lose all control of it. n_trials is\nclamped into [1, 100]. A dropped suggestion means your intent is discarded and\nthe reviewer still sees the mistake -- so get every rule right the first time.\n\nOUTPUT FORMAT -- the FIRST character you output MUST be `{` and the LAST `}`.\nEmit one raw JSON object: no code fence, no prose, no preamble or plan. It is\nparsed verbatim as your payload. Example (structure only -- choose your own\nvalues, do NOT copy these):\n\n{\"search_space\": [{\"param\": \"max_depth\", \"low\": 3, \"high\": 8}, {\"param\": \"learning_rate\", \"low\": 0.01, \"high\": 0.3}, {\"param\": \"reg_lambda\", \"low\": 0.1, \"high\": 10.0}], \"fixed_params\": {\"n_estimators\": 500, \"min_child_weight\": 5.0, \"gamma\": 0.0, \"subsample\": 0.9, \"colsample_bytree\": 0.9, \"reg_alpha\": 0.001}, \"n_trials\": 40, \"rationale\": \"...\"}\n\nBEFORE YOU RESPOND, verify ALL of these (anything you get wrong is dropped and that param falls back to its default, so your intent is lost):\n1. Output is a single JSON object -- no code, no fence, no prose.\n2. Exactly four keys: search_space, fixed_params, n_trials, rationale.\n3. Each of the nine params appears EXACTLY once (search_space or fixed_params,\n never both, never omitted).\n4. Every low/high and every fixed value is INSIDE its hard bound -- re-check each\n against the table above (e.g. max_depth low >= 3 and high <= 10).\n5. n_trials is an integer in [1, 100].\n6. Every search_space entry is keyed \"param\" (never \"name\"), with numeric low\n and high; the log-scale params (learning_rate, reg_lambda, reg_alpha) keep\n low >= 1e-3.\n", - "output_contract": { - "name": "MLMPayload", - "fields": [ - { - "name": "search_space", - "type": "object", - "description": "Hyperparameter search space proposal" - }, - { - "name": "fixed_params", - "type": "object", - "description": "Fixed (non-searched) XGBoost params" - }, - { - "name": "n_trials", - "type": "integer", - "description": "Proposed Optuna trial count" - }, - { - "name": "rationale", - "type": "string", - "description": "Rationale for the proposed search space" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -588,33 +441,10 @@ "goal": "Score MI-MLM's approved hyperparameter payload against the held-out test split. Call run_model_script with NO `code` argument -- the wired Script node reaches the tool as a path in ambient run context and executes byte-for-byte as written; retyping it is the one thing that could corrupt it. Pass only payload_json, set to the approved payload (as compact JSON) from the upstream context -- do not alter it. Report the returned test_metrics and the code_sha256/payload_sha256 guards verbatim; never compute or estimate a metric yourself.", "description": "Runs the harness-fixed XGBoost+Optuna scoring script against MI-MLM's approved payload via run_model_script (the script arrives as an ambient path, never pasted into the prompt) and reports the held-out test metrics. Unlike the real notebook (a direct, non-agent call with SHA guards), this is an ASAREE agent so the canvas can demonstrate the Script node end-to-end; the same SHA guards are still enforced server-side and reported back verbatim.", "system_prompt": "", - "output_contract": { - "name": "ScoreReport", - "fields": [ - { - "name": "test_metrics", - "type": "object", - "description": "Held-out test metrics computed server-side" - }, - { - "name": "code_sha256", - "type": "string", - "description": "Hash of the executed script, for the verbatim-execution guard" - }, - { - "name": "payload_sha256", - "type": "string", - "description": "Hash of the scored payload, for the approved-payload guard" - }, - { - "name": "result", - "type": "object", - "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" - } - ] - }, + "output_contract": null, "budget_limit_usd": 200.0, - "max_run_duration_seconds": 86400 + "max_run_duration_seconds": 86400, + "require_output_parser": true }, "factor_bindings": {} } @@ -673,6 +503,261 @@ }, "factor_bindings": {} } + }, + { + "id": "output-parser-dc", + "type": "output_parser", + "position": { + "x": 236, + "y": 160 + }, + "data": { + "label": "MI-DC Output Parser", + "config": { + "output_contract": { + "name": "DCReport", + "fields": [ + { + "name": "n_rows", + "type": "integer", + "description": "Row count after cleaning" + }, + { + "name": "n_cols_in", + "type": "integer", + "description": "Column count before cleaning" + }, + { + "name": "n_cols_out", + "type": "integer", + "description": "Column count after cleaning" + }, + { + "name": "domain_violations", + "type": "array", + "description": "Domain-rule violations found and corrected" + }, + { + "name": "missingness_threshold", + "type": "number", + "description": "Threshold used to drop high-missingness columns" + }, + { + "name": "dropped_for_missingness", + "type": "array", + "description": "Columns dropped for exceeding the missingness threshold" + }, + { + "name": "outliers", + "type": "object", + "description": "Outlier handling summary per column" + }, + { + "name": "imputation", + "type": "object", + "description": "Imputation strategy used per column" + }, + { + "name": "notes_for_fte", + "type": "string", + "description": "Handoff notes for the MI-FTE stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fte", + "type": "output_parser", + "position": { + "x": 796, + "y": 160 + }, + "data": { + "label": "MI-FTE Output Parser", + "config": { + "output_contract": { + "name": "FTEReport", + "fields": [ + { + "name": "engineering_recipe", + "type": "object", + "description": "Feature-engineering steps applied" + }, + { + "name": "encoding_map", + "type": "object", + "description": "Categorical encoding scheme per column" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after engineering/encoding" + }, + { + "name": "notes_for_fs", + "type": "string", + "description": "Handoff notes for the MI-FS stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-fs", + "type": "output_parser", + "position": { + "x": 1356, + "y": 160 + }, + "data": { + "label": "MI-FS Output Parser", + "config": { + "output_contract": { + "name": "FSReport", + "fields": [ + { + "name": "n_features_in", + "type": "integer", + "description": "Feature count before selection" + }, + { + "name": "n_features_out", + "type": "integer", + "description": "Feature count after selection" + }, + { + "name": "variance_threshold", + "type": "number", + "description": "Variance filter threshold used" + }, + { + "name": "correlation", + "type": "object", + "description": "Correlation-filtering summary" + }, + { + "name": "supervised_filter", + "type": "object", + "description": "Supervised-signal filtering summary" + }, + { + "name": "dropped_features", + "type": "array", + "description": "Features dropped during selection" + }, + { + "name": "selected_features", + "type": "array", + "description": "Final selected feature set" + }, + { + "name": "observed_class_distribution", + "type": "object", + "description": "Observed target class distribution" + }, + { + "name": "class_balance_check", + "type": "boolean", + "description": "Whether class balance passed the configured check" + }, + { + "name": "notes_for_mlm", + "type": "string", + "description": "Handoff notes for the MI-MLM stage" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-mlm", + "type": "output_parser", + "position": { + "x": 1916, + "y": 160 + }, + "data": { + "label": "MI-MLM Output Parser", + "config": { + "output_contract": { + "name": "MLMPayload", + "fields": [ + { + "name": "search_space", + "type": "object", + "description": "Hyperparameter search space proposal" + }, + { + "name": "fixed_params", + "type": "object", + "description": "Fixed (non-searched) XGBoost params" + }, + { + "name": "n_trials", + "type": "integer", + "description": "Proposed Optuna trial count" + }, + { + "name": "rationale", + "type": "string", + "description": "Rationale for the proposed search space" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } + }, + { + "id": "output-parser-score", + "type": "output_parser", + "position": { + "x": 2476, + "y": 160 + }, + "data": { + "label": "MI-Score Output Parser", + "config": { + "output_contract": { + "name": "ScoreReport", + "fields": [ + { + "name": "test_metrics", + "type": "object", + "description": "Held-out test metrics computed server-side" + }, + { + "name": "code_sha256", + "type": "string", + "description": "Hash of the executed script, for the verbatim-execution guard" + }, + { + "name": "payload_sha256", + "type": "string", + "description": "Hash of the scored payload, for the approved-payload guard" + }, + { + "name": "result", + "type": "object", + "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" + } + ] + }, + "enabled": true + }, + "factor_bindings": {} + } } ], "edges": [ @@ -890,6 +975,41 @@ "target": "agent-score", "sourceHandle": "tool", "targetHandle": "tool" + }, + { + "id": "e-output-parser-dc", + "source": "output-parser-dc", + "target": "agent-dc", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fte", + "source": "output-parser-fte", + "target": "agent-fte", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-fs", + "source": "output-parser-fs", + "target": "agent-fs", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-mlm", + "source": "output-parser-mlm", + "target": "agent-mlm", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" + }, + { + "id": "e-output-parser-score", + "source": "output-parser-score", + "target": "agent-score", + "sourceHandle": "output_parser", + "targetHandle": "output_parser" } ] }, From 27fde457df34c1231d84ec7912f8ab3197e8290a Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 22:31:56 -0700 Subject: [PATCH 14/18] Use meaningful factor level labels --- .../protocol/FactorBindableField.tsx | 16 ++++++++-------- .../components/protocol/FactorEditorDialog.tsx | 14 +++++++------- .../protocol/ProtocolCanvas.test.tsx | 2 +- .../components/protocol/factorLevels.test.ts | 17 +++++++++++++++++ .../src/components/protocol/factorLevels.ts | 18 +++++++++++++----- publications/BDM/README.md | 4 ++++ .../BDM/myocardial-anthropic-v0.8.0.json | 8 ++++++-- .../BDM/myocardial-azure-foundry-v0.8.0.json | 16 ++++++++++++++-- publications/BDM/myocardial-openai-v0.8.0.json | 8 ++++++-- src/asaree/services/metrics.py | 10 +++++++++- tests/test_experiment_run_results.py | 7 ++++++- tests/test_metrics.py | 6 ++++++ 12 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 frontend/src/components/protocol/factorLevels.test.ts diff --git a/frontend/src/components/protocol/FactorBindableField.tsx b/frontend/src/components/protocol/FactorBindableField.tsx index ef0b0948..519c9d40 100644 --- a/frontend/src/components/protocol/FactorBindableField.tsx +++ b/frontend/src/components/protocol/FactorBindableField.tsx @@ -89,7 +89,7 @@ export function FactorBindableField({ nodeLabel: string levelType: LevelType // The field's own current value, e.g. config.system_prompt -- omitted for - // a boolean field, since its levels are always the fixed [true, false]. + // a boolean field, since its levels are always the fixed [false, true]. currentValue?: unknown // The field's own already-fetched choices (e.g. ModelNodeInspector's model/ // effort lists) -- when given, each level row renders as a Select over @@ -114,7 +114,7 @@ export function FactorBindableField({ const { requestEditFactor } = useProtocolCanvasActions() const [open, setOpen] = useState(false) const [levels, setLevels] = useState(() => seedLevels(currentValue)) - const [levelLabels, setLevelLabels] = useState(() => defaultFactorLevelLabels(`${nodeLabel}:${defaultLabel}`, 2)) + const [levelLabels, setLevelLabels] = useState(() => defaultFactorLevelLabels(`${nodeLabel}:${defaultLabel}`, 2, levelType)) const queryClient = useQueryClient() // This component only ever renders inside a node inspector on the canvas, // so the graph is right there in context -- no prop-drilling needed. @@ -220,7 +220,7 @@ export function FactorBindableField({ factor={{ name: factorName, levels: structured ? seedStructuredLevels(currentValue, levelType) : seedLevels(currentValue), - level_labels: defaultFactorLevelLabels(factorName, 2), + level_labels: defaultFactorLevelLabels(factorName, 2, levelType), level_type: levelType, }} onSave={(next) => saveMutation.mutateAsync(next)} @@ -237,7 +237,7 @@ export function FactorBindableField({ if (next) { const nextLevels = seedLevels(currentValue) setLevels(nextLevels) - setLevelLabels(defaultFactorLevelLabels(factorName, nextLevels.length)) + setLevelLabels(defaultFactorLevelLabels(factorName, nextLevels.length, levelType)) } }} > @@ -258,7 +258,7 @@ export function FactorBindableField({

{factorName}

{levelType === 'boolean' ? ( -

Levels: true, false

+

Levels: false, true

) : (
@@ -328,7 +328,7 @@ export function FactorBindableField({ size="sm" onClick={() => { setLevels((ls) => [...ls, '']) - setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(factorName, ls.length + 1)[ls.length]]) + setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(factorName, ls.length + 1, levelType)[ls.length]]) }} > Add level @@ -344,8 +344,8 @@ export function FactorBindableField({ ? [0, 1] : levels.flatMap((level, index) => index === 0 || level.trim() ? [index] : []) const parsedLevels = - levelType === 'boolean' ? [true, false] : selectedIndexes.map((index) => parseLevelValue(levels[index], levelType)) - const defaults = defaultFactorLevelLabels(factorName, parsedLevels.length) + levelType === 'boolean' ? [false, true] : selectedIndexes.map((index) => parseLevelValue(levels[index], levelType)) + const defaults = defaultFactorLevelLabels(factorName, parsedLevels.length, levelType) const labels = selectedIndexes.map((index, outputIndex) => levelLabels[index]?.trim() || defaults[outputIndex]) saveMutation.mutate({ name: factorName, levels: parsedLevels, level_labels: labels, level_type: levelType }) }} diff --git a/frontend/src/components/protocol/FactorEditorDialog.tsx b/frontend/src/components/protocol/FactorEditorDialog.tsx index cd77bed9..f93604d8 100644 --- a/frontend/src/components/protocol/FactorEditorDialog.tsx +++ b/frontend/src/components/protocol/FactorEditorDialog.tsx @@ -597,7 +597,7 @@ export function FactorEditorDialog({ ? seedStructuredLevels(field.currentValue, field.levelType) : seedLevels(field.currentValue) setLevels(nextLevels) - setLevelLabels(defaultFactorLevelLabels(nextName, nextLevels.length)) + setLevelLabels(defaultFactorLevelLabels(nextName, nextLevels.length, field.levelType)) } // Computed, not user-typed (see factorLevels.ts's computeFactorName) -- @@ -627,9 +627,9 @@ export function FactorEditorDialog({ isStructuredLevelType(draftLevelType) ? draftLevels : draftLevelType === 'boolean' - ? [true, false] + ? [false, true] : selectedIndexes.map((index) => parseLevelValue((draftLevels as string[])[index], draftLevelType)) - const defaults = defaultFactorLevelLabels(draftName, parsedLevels.length) + const defaults = defaultFactorLevelLabels(draftName, parsedLevels.length, draftLevelType) const labels = selectedIndexes.map((index, outputIndex) => draftLevelLabels[index]?.trim() || defaults[outputIndex]) return { factor: { name: draftName, levels: parsedLevels, level_labels: labels, level_type: draftLevelType } satisfies DesignFactor, field } } @@ -743,7 +743,7 @@ export function FactorEditorDialog({
{levelType === 'boolean' ? ( -

Levels: true, false

+

Levels: false, true

) : levelType === 'text' ? (
{promptScope && ( @@ -798,7 +798,7 @@ export function FactorEditorDialog({ size="sm" onClick={() => { setLevels((ls) => [...ls, '']) - setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1)[ls.length]]) + setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1, levelType)[ls.length]]) }} > Add level @@ -876,7 +876,7 @@ export function FactorEditorDialog({ size="sm" onClick={() => { setLevels((ls) => [...ls, emptyStructuredLevel(levelType)]) - setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1)[ls.length]]) + setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1, levelType)[ls.length]]) }} > Add level @@ -917,7 +917,7 @@ export function FactorEditorDialog({ size="sm" onClick={() => { setLevels((ls) => [...ls, '']) - setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1)[ls.length]]) + setLevelLabels((ls) => [...ls, defaultFactorLevelLabels(name, ls.length + 1, levelType)[ls.length]]) }} > Add level diff --git a/frontend/src/components/protocol/ProtocolCanvas.test.tsx b/frontend/src/components/protocol/ProtocolCanvas.test.tsx index cba5297e..784f4afa 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.test.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.test.tsx @@ -302,7 +302,7 @@ describe('ProtocolCanvas connector adds', () => { await user.click(screen.getAllByRole('button', { name: 'Make experimental factor' })[0]) expect(await screen.findByText('Writer:Active')).toBeInTheDocument() - expect(screen.getByText('Levels: true, false')).toBeInTheDocument() + expect(screen.getByText('Levels: false, true')).toBeInTheDocument() expect(screen.queryByText('Bind to a field on the canvas')).not.toBeInTheDocument() }) diff --git a/frontend/src/components/protocol/factorLevels.test.ts b/frontend/src/components/protocol/factorLevels.test.ts new file mode 100644 index 00000000..6ab951b2 --- /dev/null +++ b/frontend/src/components/protocol/factorLevels.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { defaultFactorLevelLabels, factorLevelLabels } from './factorLevels' + +describe('factor level labels', () => { + it('defaults boolean treatments to their values', () => { + expect(defaultFactorLevelLabels('Critic enabled', 2, 'boolean')).toEqual(['false', 'true']) + expect(factorLevelLabels({ name: 'Critic enabled', levels: [false, true] })).toEqual([ + 'false', + 'true', + ]) + expect(factorLevelLabels({ name: 'Legacy order', levels: [true, false] })).toEqual(['true', 'false']) + }) + + it('keeps numbered defaults for other factor types', () => { + expect(defaultFactorLevelLabels('Azure Foundry:Model', 2, 'string')).toEqual(['level1', 'level2']) + }) +}) diff --git a/frontend/src/components/protocol/factorLevels.ts b/frontend/src/components/protocol/factorLevels.ts index 71614e52..3828c7cb 100644 --- a/frontend/src/components/protocol/factorLevels.ts +++ b/frontend/src/components/protocol/factorLevels.ts @@ -52,15 +52,21 @@ export function isStructuredLevelType(type: LevelType): boolean { } export function levelTypeOf(factor: DesignFactor): LevelType { - return factor.level_type ?? 'string' + if (factor.level_type) return factor.level_type + return factor.levels.length > 0 && factor.levels.every((level) => typeof level === 'boolean') ? 'boolean' : 'string' } -export function defaultFactorLevelLabels(_factorName: string, count: number): string[] { +export function defaultFactorLevelLabels(_factorName: string, count: number, type?: LevelType): string[] { + if (type === 'boolean') return ['false', 'true'].slice(0, count) return Array.from({ length: count }, (_, index) => `level${index + 1}`) } export function factorLevelLabels(factor: DesignFactor): string[] { - const defaults = defaultFactorLevelLabels(factor.name, factor.levels.length) + const type = levelTypeOf(factor) + const defaults = + type === 'boolean' + ? factor.levels.map((level, index) => (typeof level === 'boolean' ? String(level) : `level${index + 1}`)) + : defaultFactorLevelLabels(factor.name, factor.levels.length, type) return factor.level_labels?.length === factor.levels.length ? factor.level_labels.map((label, index) => label.trim() || defaults[index]) : defaults @@ -121,13 +127,15 @@ export function emptyStructuredLevel(type: LevelType): unknown { } } -// A boolean factor's levels are always exactly [true, false] -- there's +// A boolean factor's levels are always exactly [false, true] -- there's // nothing to type in, so switching to it replaces whatever was there. +// False is first because factorial analysis treats the first level as the +// reference/-1 condition and the second as treatment/+1. // Every other type starts with two blank rows, matching this app's existing // "start with a couple of empty slots" convention (FactorBindableField's // own popover). export function defaultLevelsForType(type: LevelType): unknown[] { - if (type === 'boolean') return [true, false] + if (type === 'boolean') return [false, true] if (isStructuredLevelType(type)) return [emptyStructuredLevel(type), emptyStructuredLevel(type)] return ['', ''] } diff --git a/publications/BDM/README.md b/publications/BDM/README.md index d7fa219a..692ecbd8 100644 --- a/publications/BDM/README.md +++ b/publications/BDM/README.md @@ -37,6 +37,10 @@ its provider's effort ladder. The two ladders aren't the same length: OpenAI's `reasoning_effort` stops at `high`, so `high` is that variant's counterpart to Anthropic's `xhigh`, not a rung below it. +Factor levels are ordered reference first, treatment second: smaller model → +larger model, medium → highest effort, and critic off → critic on. This matches +the Results analysis's −1/+1 coding and the paper's stated contrasts. + Model levels are editable on the Design tab after importing, and the model field accepts any id you type — the levels above are just what the catalog can vouch for. Whether a model gets an Effort or a Temperature control is a per-model fact diff --git a/publications/BDM/myocardial-anthropic-v0.8.0.json b/publications/BDM/myocardial-anthropic-v0.8.0.json index a4fdff19..c7b61df7 100644 --- a/publications/BDM/myocardial-anthropic-v0.8.0.json +++ b/publications/BDM/myocardial-anthropic-v0.8.0.json @@ -1034,8 +1034,12 @@ { "name": "Critic enabled", "levels": [ - true, - false + false, + true + ], + "level_labels": [ + "false", + "true" ], "level_type": "boolean" } diff --git a/publications/BDM/myocardial-azure-foundry-v0.8.0.json b/publications/BDM/myocardial-azure-foundry-v0.8.0.json index 073fbc9e..6cbf9293 100644 --- a/publications/BDM/myocardial-azure-foundry-v0.8.0.json +++ b/publications/BDM/myocardial-azure-foundry-v0.8.0.json @@ -1021,6 +1021,10 @@ "claude-sonnet-5", "claude-opus-5" ], + "level_labels": [ + "sonnet-5", + "opus-5" + ], "level_type": "string" }, { @@ -1029,13 +1033,21 @@ "medium", "xhigh" ], + "level_labels": [ + "medium", + "xhigh" + ], "level_type": "string" }, { "name": "Critic enabled", "levels": [ - true, - false + false, + true + ], + "level_labels": [ + "false", + "true" ], "level_type": "boolean" } diff --git a/publications/BDM/myocardial-openai-v0.8.0.json b/publications/BDM/myocardial-openai-v0.8.0.json index ae79876b..da2d4f65 100644 --- a/publications/BDM/myocardial-openai-v0.8.0.json +++ b/publications/BDM/myocardial-openai-v0.8.0.json @@ -1034,8 +1034,12 @@ { "name": "Critic enabled", "levels": [ - true, - false + false, + true + ], + "level_labels": [ + "false", + "true" ], "level_type": "boolean" } diff --git a/src/asaree/services/metrics.py b/src/asaree/services/metrics.py index b3e4f4c3..d17a5fd1 100644 --- a/src/asaree/services/metrics.py +++ b/src/asaree/services/metrics.py @@ -333,7 +333,15 @@ def normalize_design_spec( levels = factor.get("levels") if not isinstance(levels, list): continue - defaults = [f"level{index}" for index in range(1, len(levels) + 1)] + boolean_levels = bool(levels) and all(isinstance(level, bool) for level in levels) + defaults = ( + [ + str(level).lower() if isinstance(level, bool) else f"level{index}" + for index, level in enumerate(levels, 1) + ] + if factor.get("level_type") == "boolean" or boolean_levels + else [f"level{index}" for index in range(1, len(levels) + 1)] + ) supplied = factor.get("level_labels") if isinstance(supplied, list) and len(supplied) == len(levels): factor["level_labels"] = [ diff --git a/tests/test_experiment_run_results.py b/tests/test_experiment_run_results.py index 013af7a4..7133e14f 100644 --- a/tests/test_experiment_run_results.py +++ b/tests/test_experiment_run_results.py @@ -348,7 +348,12 @@ def test_results_csv_projects_categorical_factors_to_short_level_labels() -> Non "metric_values": {"accuracy": 0.9, "passed": False}, }, ], - {"factors": [{"name": "model", "levels": ["small", "large"], "level_labels": ["small", "large"]}]}, + { + "factors": [ + {"name": "critic", "levels": [True, False], "level_labels": ["true", "false"]}, + {"name": "model", "levels": ["small", "large"], "level_labels": ["small", "large"]}, + ] + }, ) rows = list(csv.DictReader(io.StringIO(csv_text))) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 94dc507e..1c7beef8 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -222,6 +222,12 @@ def test_design_spec_adds_short_default_labels_for_legacy_factor_levels() -> Non } +def test_design_spec_defaults_boolean_level_labels_to_values() -> None: + spec = normalize_design_spec({"factors": [{"name": "Critic enabled", "levels": [False, True]}]}) + + assert spec["factors"][0]["level_labels"] == ["false", "true"] + + def test_design_spec_normalizes_legacy_model_factor_kind() -> None: spec = normalize_design_spec( {"factors": [{"name": "Agent:Model", "level_type": "llm_config", "levels": [{"model": "gpt-5"}]}]} From 490f0fbf86b0cc8948af19c42e094c91a990a9ec Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 22:36:10 -0700 Subject: [PATCH 15/18] Raise BDM agent iteration limits --- publications/BDM/myocardial-anthropic-v0.8.0.json | 8 ++++---- publications/BDM/myocardial-azure-foundry-v0.8.0.json | 8 ++++---- publications/BDM/myocardial-openai-v0.8.0.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/publications/BDM/myocardial-anthropic-v0.8.0.json b/publications/BDM/myocardial-anthropic-v0.8.0.json index c7b61df7..5b8db096 100644 --- a/publications/BDM/myocardial-anthropic-v0.8.0.json +++ b/publications/BDM/myocardial-anthropic-v0.8.0.json @@ -108,7 +108,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -212,7 +212,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -316,7 +316,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -459,7 +459,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" diff --git a/publications/BDM/myocardial-azure-foundry-v0.8.0.json b/publications/BDM/myocardial-azure-foundry-v0.8.0.json index 6cbf9293..04b63603 100644 --- a/publications/BDM/myocardial-azure-foundry-v0.8.0.json +++ b/publications/BDM/myocardial-azure-foundry-v0.8.0.json @@ -108,7 +108,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -212,7 +212,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -316,7 +316,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -459,7 +459,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" diff --git a/publications/BDM/myocardial-openai-v0.8.0.json b/publications/BDM/myocardial-openai-v0.8.0.json index da2d4f65..20913f3b 100644 --- a/publications/BDM/myocardial-openai-v0.8.0.json +++ b/publications/BDM/myocardial-openai-v0.8.0.json @@ -108,7 +108,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -212,7 +212,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -316,7 +316,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" @@ -459,7 +459,7 @@ "data": { "label": "", "config": { - "max_iterations": 12, + "max_iterations": 15, "include_scratchpad": true, "scratchpad_window": 10, "observation_format": "raw" From b5abf8fd639673fc25626969298c5ea7202289a7 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 28 Sep 2026 23:18:54 -0700 Subject: [PATCH 16/18] feat: add typed reported metric projections --- .../protocol/CustomMetricDialogs.tsx | 4 +- .../src/components/protocol/MetricsEditor.tsx | 10 +- frontend/src/lib/experiment.test.ts | 31 +- frontend/src/lib/experiment.ts | 2 +- frontend/src/lib/metricCatalog.ts | 12 +- publications/BDM/README.md | 12 +- .../BDM/myocardial-anthropic-v0.8.0.json | 1572 ++++++++++++++++- .../BDM/myocardial-azure-foundry-v0.8.0.json | 1572 ++++++++++++++++- .../BDM/myocardial-openai-v0.8.0.json | 1572 ++++++++++++++++- src/asaree/services/csv_export.py | 4 +- src/asaree/services/experiment_run_results.py | 14 +- src/asaree/services/measurement_engine.py | 6 +- src/asaree/services/measurement_migration.py | 11 +- src/asaree/services/metrics.py | 11 +- src/asaree/services/protocol_runs.py | 16 +- src/asaree/services/reported_metrics.py | 127 +- src/asaree/services/runtime_metrics.py | 8 +- tests/test_bdm_use_cases.py | 101 ++ tests/test_experiment_measurements.py | 21 +- tests/test_experiment_run_results.py | 42 +- tests/test_measurement_migration.py | 40 + tests/test_metrics.py | 19 +- tests/test_protocol_runs.py | 46 + tests/test_reported_metrics.py | 143 ++ 24 files changed, 5198 insertions(+), 198 deletions(-) create mode 100644 tests/test_bdm_use_cases.py diff --git a/frontend/src/components/protocol/CustomMetricDialogs.tsx b/frontend/src/components/protocol/CustomMetricDialogs.tsx index 6586e356..6cc0c483 100644 --- a/frontend/src/components/protocol/CustomMetricDialogs.tsx +++ b/frontend/src/components/protocol/CustomMetricDialogs.tsx @@ -223,7 +223,9 @@ export function CustomMetricFlow({ metric, binding, graph, existingMetrics, sour || !producer || (producer === 'agent' ? agentSourceError : producer === 'python' ? pythonSourceError : mcpSourceError || mcpMappingError), ) - const nextMetric = { ...metric, name: name.trim(), description: metric.description, direction: 'neutral' as const, valueType: 'opaque' as const, aggregation: 'none' as const, primary: false } + // Imported measurement plans can carry typed reported metrics. Editing the + // source/name must not silently downgrade those declarations to opaque. + const nextMetric = { ...metric, name: name.trim(), description: metric.description } const nextConfig: CustomMetricProducerConfig | null = producer === 'agent' && selectedAgentSource ? { producer, agentNodeId: selectedAgentSource.agentNodeId } : producer === 'mcp' && selectedMcpSource diff --git a/frontend/src/components/protocol/MetricsEditor.tsx b/frontend/src/components/protocol/MetricsEditor.tsx index 667d81a1..82e37fd4 100644 --- a/frontend/src/components/protocol/MetricsEditor.tsx +++ b/frontend/src/components/protocol/MetricsEditor.tsx @@ -18,10 +18,6 @@ import { CustomMetricFlow, MetricNodeLabel, type MetricNodeDisplay } from './Cus import { NodeInspectorDialog } from './NodeInspectorDialog' import { useDialogAutosave } from './useDialogAutosave' -function withoutRanking(metrics: DesignMetric[]): DesignMetric[] { - return metrics.map((metric) => ({ ...metric, direction: 'neutral', primary: false })) -} - function synchronizeMeasurementPlanMetrics(plan: MeasurementPlan | null, metrics: DesignMetric[]) { if (!plan) return null const definitionsById = new Map(plan.metrics.map((definition) => [definition.id, definition])) @@ -359,7 +355,7 @@ function MetricsDialog({ - {binding && } + {binding && metric.valueType === 'opaque' && }
})}
} @@ -500,7 +496,7 @@ export function MetricsEditor({ } } for (const metric of selected) nextPlan = upsertRuntimeMetric(nextPlan, metric) - const nextMetrics = withoutRanking([...preserved, ...selected]) + const nextMetrics = [...preserved, ...selected] const byId = new Map(nextMetrics.map((metric) => [metric.id, metric])) const synchronizedPlan = nextPlan && { ...nextPlan, @@ -564,7 +560,7 @@ export function MetricsEditor({ const customById = new Map(draft.metrics.filter((metric) => metric.kind === 'custom' && metric.id).map((metric) => [metric.id!, metric])) const orderedCustomMetrics = customMetricIds.flatMap((id) => customById.get(id) ?? []) draft.metrics = [...draft.metrics.filter((metric) => metric.kind !== 'custom'), ...orderedCustomMetrics] - const normalizedDraftMetrics = withoutRanking(draft.metrics) + const normalizedDraftMetrics = draft.metrics draft = { metrics: normalizedDraftMetrics, measurementPlan: synchronizeMeasurementPlanMetrics(draft.measurementPlan, normalizedDraftMetrics), diff --git a/frontend/src/lib/experiment.test.ts b/frontend/src/lib/experiment.test.ts index 551f76c8..56fc1be1 100644 --- a/frontend/src/lib/experiment.test.ts +++ b/frontend/src/lib/experiment.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from 'vitest' -import { displayFactorLevel } from './experiment' +import type { Experiment } from '@/types/experiments' +import { displayFactorLevel, primaryMetric } from './experiment' +import { normalizeDesignMetrics } from './metricCatalog' describe('displayFactorLevel', () => { it('uses the label paired with a matching raw level', () => { @@ -33,3 +35,30 @@ describe('displayFactorLevel', () => { )).toBe('Cited technical response') }) }) + +describe('reported scalar metrics', () => { + it('preserves an explicitly typed custom metric and uses it as primary', () => { + const metrics = normalizeDesignMetrics([{ + id: 'pr-auc', + name: 'pr_auc', + kind: 'custom', + valueType: 'number', + direction: 'maximize', + aggregation: 'mean', + primary: true, + }]) + const experiment = { design_spec: { metrics } } as Experiment + + expect(metrics[0]).toMatchObject({ valueType: 'number', direction: 'maximize', primary: true }) + expect(primaryMetric(experiment)).toEqual({ key: 'pr_auc', direction: 'maximize', valueType: 'number' }) + }) + + it('keeps an ordinary custom draft opaque and non-primary', () => { + expect(normalizeDesignMetrics([{ + name: 'Reviewer report', + kind: 'custom', + direction: 'maximize', + primary: true, + }])[0]).toMatchObject({ valueType: 'opaque', direction: 'neutral', primary: false }) + }) +}) diff --git a/frontend/src/lib/experiment.ts b/frontend/src/lib/experiment.ts index 776185c2..811d8d44 100644 --- a/frontend/src/lib/experiment.ts +++ b/frontend/src/lib/experiment.ts @@ -303,7 +303,7 @@ export function pickMetricColumns(experiment: Experiment | undefined, replicates * must never choose the canvas top-bar result. */ export function primaryMetric(experiment: Experiment | undefined): { key: string; direction: 'maximize' | 'minimize'; valueType?: 'number' | 'boolean' | 'string' } | null { const metric = experiment?.design_spec?.metrics?.find((candidate) => candidate.primary && candidate.name) - if (!metric || metric.kind !== 'runtime' || metric.direction === 'neutral' || metric.valueType === 'string' || metric.valueType === 'opaque') return null + if (!metric || metric.direction === 'neutral' || metric.valueType === 'string' || metric.valueType === 'opaque') return null const key = metric.kind === 'runtime' && typeof metric.catalogKey === 'string' ? metric.catalogKey : metric.name return { key, direction: metric.direction, valueType: metric.valueType } } diff --git a/frontend/src/lib/metricCatalog.ts b/frontend/src/lib/metricCatalog.ts index 9d6a6def..06255364 100644 --- a/frontend/src/lib/metricCatalog.ts +++ b/frontend/src/lib/metricCatalog.ts @@ -65,6 +65,8 @@ export function normalizeDesignMetrics(metrics: DesignMetric[] | undefined): Des if (!raw || typeof raw.name !== 'string' || !raw.name.trim()) return [] const catalog = metricCatalogEntry(raw.catalogKey) if (raw.kind && !['runtime', 'custom'].includes(raw.kind)) return [] + const kind = raw.kind ?? catalog?.kind ?? 'custom' + const scalarCustom = kind === 'custom' && raw.kind === 'custom' && (raw.valueType === 'number' || raw.valueType === 'boolean') return [{ ...raw, id: typeof raw.id === 'string' && raw.id ? raw.id : legacyId(raw), @@ -73,13 +75,13 @@ export function normalizeDesignMetrics(metrics: DesignMetric[] | undefined): Des description: typeof raw.description === 'string' && raw.description.trim() ? raw.description.trim() : catalog?.shortDescription ?? `Legacy metric declaration for ${raw.name.trim()}.`, - kind: raw.kind ?? catalog?.kind ?? 'custom', - valueType: (raw.kind ?? catalog?.kind) === 'custom' ? 'opaque' : raw.valueType ?? catalog?.valueType ?? 'number', - direction: (raw.kind ?? catalog?.kind) === 'custom' ? 'neutral' : raw.direction === 'maximize' || raw.direction === 'minimize' || raw.direction === 'neutral' + kind, + valueType: kind === 'custom' && !scalarCustom ? 'opaque' : raw.valueType ?? catalog?.valueType ?? 'number', + direction: kind === 'custom' && !scalarCustom ? 'neutral' : raw.direction === 'maximize' || raw.direction === 'minimize' || raw.direction === 'neutral' ? raw.direction : (catalog?.defaultDirection ?? 'maximize'), - aggregation: (raw.kind ?? catalog?.kind) === 'custom' ? 'none' : raw.valueType === 'boolean' ? 'mean' : (raw.aggregation === 'sum' ? 'sum' : (catalog?.aggregation ?? 'mean')), - primary: (raw.kind ?? catalog?.kind) === 'custom' ? false : Boolean(raw.primary), + aggregation: kind === 'custom' && !scalarCustom ? 'none' : raw.valueType === 'boolean' ? 'mean' : (raw.aggregation === 'sum' ? 'sum' : raw.aggregation === 'none' ? 'none' : (catalog?.aggregation ?? 'mean')), + primary: kind === 'custom' && !scalarCustom ? false : Boolean(raw.primary), unit: raw.unit ?? catalog?.unit, scoring: undefined, }] diff --git a/publications/BDM/README.md b/publications/BDM/README.md index 692ecbd8..24ee0649 100644 --- a/publications/BDM/README.md +++ b/publications/BDM/README.md @@ -72,11 +72,13 @@ critic gates, with these v0.8.0 execution details: connected Output Parser node rather than the legacy hidden `config.output_contract` field. The contracts and runtime behavior are unchanged; the canvas now exposes where each structured payload is defined. -- **Declared measurement source.** The complete held-out scoring response is - captured as the opaque `Model evaluation` metric from MI-Score's exact - `asaree-sklearn-model.run_model_script` call. This retains the returned test - metrics and SHA-256 guards with producer provenance; an arbitrary successful - tool call is not promoted as a result. +- **Declared measurements.** The complete held-out scoring response is retained + as an opaque provenance record, while typed projections expose PR-AUC (the + primary metric), ROC-AUC, Brier and operating-point diagnostics, pipeline + feature counts, Optuna/XGBoost decisions, SHA-256 guards, and the built-in + runtime metrics as analysis-ready Results and CSV columns. Agent-stage values + come from their connected Output Parser payloads; scoring values come from + MI-Score's exact `asaree-sklearn-model.run_model_script` call. That `open_workspace(stage=...)` call is deliberately kept: seeding the workspace materializes `v0_raw`, but *not* a stage's `.scratch` input, which is diff --git a/publications/BDM/myocardial-anthropic-v0.8.0.json b/publications/BDM/myocardial-anthropic-v0.8.0.json index 5b8db096..4ce99ce5 100644 --- a/publications/BDM/myocardial-anthropic-v0.8.0.json +++ b/publications/BDM/myocardial-anthropic-v0.8.0.json @@ -275,7 +275,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\n7. Compare selected_features with the output names in the upstream FTE\n engineering_recipe. Report their intersection count as\n n_engineered_features_selected and 100 times that count divided by\n n_features_out as pct_engineered_features_selected.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"n_engineered_features_selected\": ...,\n \"pct_engineered_features_selected\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": null, "budget_limit_usd": 200.0, "max_run_duration_seconds": 86400, @@ -549,12 +549,12 @@ }, { "name": "outliers", - "type": "object", + "type": "array", "description": "Outlier handling summary per column" }, { "name": "imputation", - "type": "object", + "type": "array", "description": "Imputation strategy used per column" }, { @@ -584,7 +584,7 @@ "fields": [ { "name": "engineering_recipe", - "type": "object", + "type": "array", "description": "Feature-engineering steps applied" }, { @@ -632,6 +632,16 @@ "type": "integer", "description": "Feature count after selection" }, + { + "name": "n_engineered_features_selected", + "type": "integer", + "description": "Selected features created by the upstream engineering recipe" + }, + { + "name": "pct_engineered_features_selected", + "type": "number", + "description": "Percentage of selected features created by the upstream engineering recipe" + }, { "name": "variance_threshold", "type": "number", @@ -664,8 +674,8 @@ }, { "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" + "type": "string", + "description": "Whether observed class balance matches the task brief, with divergence details when it does not" }, { "name": "notes_for_mlm", @@ -694,7 +704,7 @@ "fields": [ { "name": "search_space", - "type": "object", + "type": "array", "description": "Hyperparameter search space proposal" }, { @@ -748,7 +758,7 @@ "description": "Hash of the scored payload, for the approved-payload guard" }, { - "name": "result", + "name": "model_decisions", "type": "object", "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" } @@ -1046,50 +1056,1528 @@ ], "replicates": 10, "metrics": [ + { + "id": "runtime-cost", + "catalogKey": "cost_usd", + "name": "Cost", + "description": "Estimated provider cost for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "USD" + }, + { + "id": "runtime-duration", + "catalogKey": "duration_seconds", + "name": "Duration", + "description": "Wall-clock duration of the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "catalogKey": "input_tokens", + "name": "Input tokens", + "description": "Tokens sent to the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "catalogKey": "output_tokens", + "name": "Output tokens", + "description": "Tokens generated by the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "catalogKey": "total_tokens", + "name": "Total tokens", + "description": "Combined input and output tokens for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "catalogKey": "tool_calls", + "name": "Tool calls", + "description": "Recorded tool-call attempts across attributed Agent runs.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "catalogKey": "tool_error_rate", + "name": "Tool error rate", + "description": "Failed tool calls divided by all tool-call attempts.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "catalogKey": "agent_loop_iterations", + "name": "Agent-loop iterations", + "description": "Distinct Agent reasoning-loop iterations in the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "catalogKey": "critic_rejections", + "name": "Critic rejections", + "description": "Rejected critic-gate reviews during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "catalogKey": "critic_approvals", + "name": "Critic approvals", + "description": "Critic gates ending in an actual approval.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "sum", + "unit": "reviews" + }, { "id": "model-evaluation", - "name": "Model evaluation", - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "name": "model_evaluation", + "description": "Complete held-out scoring response from run_model_script.", "kind": "custom", "valueType": "opaque", "primary": false, "direction": "neutral", "aggregation": "none" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - }, - "measurement_plan": { - "metrics": [ + }, { - "id": "model-evaluation", - "name": "Model evaluation", - "value_type": "opaque", + "id": "pr-auc", + "name": "pr_auc", + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "kind": "custom", + "valueType": "number", + "primary": true, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "description": "Held-out ROC-AUC.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "description": "Held-out Brier score.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "description": "Training-selected operating threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, "direction": "neutral", - "aggregation": "none", + "aggregation": "mean" + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "description": "Observed positive-class rate in the held-out test split.", + "kind": "custom", + "valueType": "number", "primary": false, - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." - } - ], - "producers": [ + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, { - "id": "model-evaluation-source", - "producer_id": "asaree.mcp_tool", - "kind": "reported", - "outputs": { - "value": "model-evaluation" - }, - "artifacts": [], - "config": { - "agent_node_id": "agent-score", - "mcp_node_id": "tool-model-score", - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "tool_name": "run_model_script" - } - } - ], - "inputs": [] + "id": "n-test-samples", + "name": "n_test_samples", + "description": "Held-out test sample count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "description": "Positive-class sample count in the held-out test split.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "description": "accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "description": "balanced accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "description": "precision at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "description": "recall at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "description": "f1 at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "description": "accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "description": "balanced accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "description": "precision at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "description": "recall at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "description": "f1 at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "description": "accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "description": "balanced accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "description": "precision at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "description": "recall at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "description": "f1 at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "description": "Feature count after data cleaning.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "description": "Data-cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "description": "Feature-engineering recipe operation count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "description": "Feature count after engineering and encoding.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "description": "Features surviving selection and reaching modeling.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "description": "Engineered features surviving feature selection.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "description": "Engineered features as a percentage of selected features.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "description": "Feature-selection variance threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "selected-features", + "name": "selected_features", + "description": "Final selected feature names.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "description": "Optuna trial count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "description": "Best inner-CV average precision found by Optuna.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "description": "Validation-fold Brier score after isotonic calibration.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "description": "Final XGBoost n_estimators value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "description": "Final XGBoost max_depth value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "description": "Final XGBoost learning_rate value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "description": "Final XGBoost min_child_weight value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "description": "Final XGBoost subsample value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "description": "Final XGBoost colsample_bytree value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "description": "Final XGBoost reg_lambda value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "description": "Final XGBoost reg_alpha value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "description": "Final XGBoost gamma value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "code-sha256", + "name": "code_sha256", + "description": "SHA-256 of the executed fixed modeling script.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "description": "SHA-256 of the approved hyperparameter payload.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "data-sha256", + "name": "data_sha256", + "description": "SHA-256 provenance hash for the scored data.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + } + ], + "coordination_strategy": { + "slug": "critic_gate" + } + }, + "measurement_plan": { + "metrics": [ + { + "id": "runtime-cost", + "name": "Cost", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Estimated provider cost for the run.", + "unit": "USD" + }, + { + "id": "runtime-duration", + "name": "Duration", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Wall-clock duration of the run.", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "name": "Input tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens sent to the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "name": "Output tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens generated by the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "name": "Total tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Combined input and output tokens for the run.", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "name": "Tool calls", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Recorded tool-call attempts across attributed Agent runs.", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "name": "Tool error rate", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Failed tool calls divided by all tool-call attempts.", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "name": "Agent-loop iterations", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Distinct Agent reasoning-loop iterations in the run.", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "name": "Critic rejections", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Rejected critic-gate reviews during the run.", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "name": "Critic approvals", + "value_type": "number", + "direction": "neutral", + "aggregation": "sum", + "primary": false, + "description": "Critic gates ending in an actual approval.", + "unit": "reviews" + }, + { + "id": "model-evaluation", + "name": "model_evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Complete held-out scoring response from run_model_script." + }, + { + "id": "pr-auc", + "name": "pr_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": true, + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Held-out ROC-AUC.", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Held-out Brier score.", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Training-selected operating threshold." + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Observed positive-class rate in the held-out test split.", + "unit": "proportion" + }, + { + "id": "n-test-samples", + "name": "n_test_samples", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Held-out test sample count.", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive-class sample count in the held-out test split.", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the 0 5 operating point." + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the 0 5 operating point." + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the 0 5 operating point." + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the 0 5 operating point." + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the 0 5 operating point." + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the chosen threshold operating point." + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the chosen threshold operating point." + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the chosen threshold operating point." + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the chosen threshold operating point." + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the chosen threshold operating point." + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the prevalence operating point." + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the prevalence operating point." + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the prevalence operating point." + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the prevalence operating point." + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the prevalence operating point." + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after data cleaning.", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Data-cleaning missingness threshold.", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-engineering recipe operation count.", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after engineering and encoding.", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Features surviving selection and reaching modeling.", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features surviving feature selection.", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features as a percentage of selected features.", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-selection variance threshold." + }, + { + "id": "selected-features", + "name": "selected_features", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Final selected feature names." + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Optuna trial count.", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Best inner-CV average precision found by Optuna.", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Validation-fold Brier score after isotonic calibration.", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost n_estimators value." + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost max_depth value." + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost learning_rate value." + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost min_child_weight value." + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost subsample value." + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost colsample_bytree value." + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_lambda value." + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_alpha value." + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost gamma value." + }, + { + "id": "code-sha256", + "name": "code_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the executed fixed modeling script." + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the approved hyperparameter payload." + }, + { + "id": "data-sha256", + "name": "data_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 provenance hash for the scored data." + } + ], + "producers": [ + { + "id": "runtime", + "producer_id": "asaree.runtime", + "kind": "runtime", + "outputs": { + "cost_usd": "runtime-cost", + "duration_seconds": "runtime-duration", + "input_tokens": "runtime-input-tokens", + "output_tokens": "runtime-output-tokens", + "total_tokens": "runtime-total-tokens", + "tool_calls": "runtime-tool-calls", + "tool_error_rate": "runtime-tool-error-rate", + "agent_loop_iterations": "runtime-agent-loop-iterations", + "critic_rejections": "runtime-critic-rejections", + "critic_approvals": "runtime-critic-approvals" + }, + "artifacts": [], + "config": {} + }, + { + "id": "dc-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_dc": "n-features-after-dc", + "missingness_threshold": "missingness-threshold", + "n_dropped_for_missingness": "n-dropped-for-missingness" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-dc", + "projections": { + "n_features_after_dc": { + "path": "n_cols_out" + }, + "missingness_threshold": { + "path": "missingness_threshold" + }, + "n_dropped_for_missingness": { + "path": "dropped_for_missingness", + "transform": "length" + } + } + } + }, + { + "id": "fte-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_engineered_features": "n-engineered-features", + "n_features_after_fte": "n-features-after-fte" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fte", + "projections": { + "n_engineered_features": { + "path": "engineering_recipe", + "transform": "length" + }, + "n_features_after_fte": { + "path": "n_features_out" + } + } + } + }, + { + "id": "fs-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_fs": "n-features-after-fs", + "n_engineered_features_selected": "n-engineered-features-selected", + "pct_engineered_features_selected": "pct-engineered-features-selected", + "variance_threshold": "variance-threshold", + "selected_features": "selected-features" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fs", + "projections": { + "n_features_after_fs": { + "path": "n_features_out" + }, + "n_engineered_features_selected": { + "path": "n_engineered_features_selected" + }, + "pct_engineered_features_selected": { + "path": "pct_engineered_features_selected" + }, + "variance_threshold": { + "path": "variance_threshold" + }, + "selected_features": { + "path": "selected_features" + } + } + } + }, + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "model_evaluation": "model-evaluation", + "pr_auc": "pr-auc", + "roc_auc": "roc-auc", + "brier_score": "brier-score", + "chosen_threshold": "chosen-threshold", + "test_prevalence": "test-prevalence", + "n_test_samples": "n-test-samples", + "n_pos_test": "n-pos-test", + "accuracy_at_0_5": "accuracy-0-5", + "balanced_accuracy_at_0_5": "balanced_accuracy-0-5", + "precision_at_0_5": "precision-0-5", + "recall_at_0_5": "recall-0-5", + "f1_at_0_5": "f1-0-5", + "accuracy_at_chosen_threshold": "accuracy-chosen", + "balanced_accuracy_at_chosen_threshold": "balanced_accuracy-chosen", + "precision_at_chosen_threshold": "precision-chosen", + "recall_at_chosen_threshold": "recall-chosen", + "f1_at_chosen_threshold": "f1-chosen", + "accuracy_at_prevalence": "accuracy-prevalence", + "balanced_accuracy_at_prevalence": "balanced_accuracy-prevalence", + "precision_at_prevalence": "precision-prevalence", + "recall_at_prevalence": "recall-prevalence", + "f1_at_prevalence": "f1-prevalence", + "n_pred_pos_at_0_5": "n-pred-pos-0-5", + "n_pred_pos_at_chosen_threshold": "n-pred-pos-chosen", + "n_pred_pos_at_prevalence": "n-pred-pos-prevalence", + "optuna_n_trials": "optuna-n-trials", + "optuna_best_inner_cv_score": "optuna-best-inner-cv-score", + "calibration_brier_val": "calibration-brier-val", + "code_sha256": "code-sha256", + "payload_sha256": "payload-sha256", + "data_sha256": "data-sha256", + "hp_n_estimators": "hp-n-estimators", + "hp_max_depth": "hp-max-depth", + "hp_learning_rate": "hp-learning-rate", + "hp_min_child_weight": "hp-min-child-weight", + "hp_subsample": "hp-subsample", + "hp_colsample_bytree": "hp-colsample-bytree", + "hp_reg_lambda": "hp-reg-lambda", + "hp_reg_alpha": "hp-reg-alpha", + "hp_gamma": "hp-gamma" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script", + "projections": { + "pr_auc": { + "path": "test_metrics.average_precision" + }, + "roc_auc": { + "path": "test_metrics.roc_auc" + }, + "brier_score": { + "path": "test_metrics.brier_score" + }, + "chosen_threshold": { + "path": "test_metrics.chosen_threshold" + }, + "test_prevalence": { + "path": "test_metrics.test_prevalence" + }, + "n_test_samples": { + "path": "test_metrics.n_test_samples" + }, + "n_pos_test": { + "path": "test_metrics.n_pos_test" + }, + "accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/accuracy" + }, + "balanced_accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/balanced_accuracy" + }, + "precision_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/precision" + }, + "recall_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/recall" + }, + "f1_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/f1" + }, + "accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.accuracy" + }, + "balanced_accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.balanced_accuracy" + }, + "precision_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.precision" + }, + "recall_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.recall" + }, + "f1_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.f1" + }, + "accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.accuracy" + }, + "balanced_accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.balanced_accuracy" + }, + "precision_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.precision" + }, + "recall_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.recall" + }, + "f1_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.f1" + }, + "n_pred_pos_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/n_pred_pos" + }, + "n_pred_pos_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.n_pred_pos" + }, + "n_pred_pos_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.n_pred_pos" + }, + "optuna_n_trials": { + "path": "model_decisions.n_trials" + }, + "optuna_best_inner_cv_score": { + "path": "model_decisions.optuna_best_value" + }, + "calibration_brier_val": { + "path": "model_decisions.calibration_brier_val" + }, + "code_sha256": { + "path": "code_sha256" + }, + "payload_sha256": { + "path": "payload_sha256" + }, + "data_sha256": { + "path": "data_sha256" + }, + "hp_n_estimators": { + "path": "model_decisions.best_hyperparameters.n_estimators" + }, + "hp_max_depth": { + "path": "model_decisions.best_hyperparameters.max_depth" + }, + "hp_learning_rate": { + "path": "model_decisions.best_hyperparameters.learning_rate" + }, + "hp_min_child_weight": { + "path": "model_decisions.best_hyperparameters.min_child_weight" + }, + "hp_subsample": { + "path": "model_decisions.best_hyperparameters.subsample" + }, + "hp_colsample_bytree": { + "path": "model_decisions.best_hyperparameters.colsample_bytree" + }, + "hp_reg_lambda": { + "path": "model_decisions.best_hyperparameters.reg_lambda" + }, + "hp_reg_alpha": { + "path": "model_decisions.best_hyperparameters.reg_alpha" + }, + "hp_gamma": { + "path": "model_decisions.best_hyperparameters.gamma" + } + } + } + } + ], + "inputs": [ + { + "producer_binding_id": "runtime", + "input_key": "facts", + "source_key": "attempt.runtime" + } + ] } } diff --git a/publications/BDM/myocardial-azure-foundry-v0.8.0.json b/publications/BDM/myocardial-azure-foundry-v0.8.0.json index 04b63603..8d00b467 100644 --- a/publications/BDM/myocardial-azure-foundry-v0.8.0.json +++ b/publications/BDM/myocardial-azure-foundry-v0.8.0.json @@ -275,7 +275,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\n7. Compare selected_features with the output names in the upstream FTE\n engineering_recipe. Report their intersection count as\n n_engineered_features_selected and 100 times that count divided by\n n_features_out as pct_engineered_features_selected.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"n_engineered_features_selected\": ...,\n \"pct_engineered_features_selected\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": null, "budget_limit_usd": 200.0, "max_run_duration_seconds": 86400, @@ -549,12 +549,12 @@ }, { "name": "outliers", - "type": "object", + "type": "array", "description": "Outlier handling summary per column" }, { "name": "imputation", - "type": "object", + "type": "array", "description": "Imputation strategy used per column" }, { @@ -584,7 +584,7 @@ "fields": [ { "name": "engineering_recipe", - "type": "object", + "type": "array", "description": "Feature-engineering steps applied" }, { @@ -632,6 +632,16 @@ "type": "integer", "description": "Feature count after selection" }, + { + "name": "n_engineered_features_selected", + "type": "integer", + "description": "Selected features created by the upstream engineering recipe" + }, + { + "name": "pct_engineered_features_selected", + "type": "number", + "description": "Percentage of selected features created by the upstream engineering recipe" + }, { "name": "variance_threshold", "type": "number", @@ -664,8 +674,8 @@ }, { "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" + "type": "string", + "description": "Whether observed class balance matches the task brief, with divergence details when it does not" }, { "name": "notes_for_mlm", @@ -694,7 +704,7 @@ "fields": [ { "name": "search_space", - "type": "object", + "type": "array", "description": "Hyperparameter search space proposal" }, { @@ -748,7 +758,7 @@ "description": "Hash of the scored payload, for the approved-payload guard" }, { - "name": "result", + "name": "model_decisions", "type": "object", "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" } @@ -1054,50 +1064,1528 @@ ], "replicates": 10, "metrics": [ + { + "id": "runtime-cost", + "catalogKey": "cost_usd", + "name": "Cost", + "description": "Estimated provider cost for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "USD" + }, + { + "id": "runtime-duration", + "catalogKey": "duration_seconds", + "name": "Duration", + "description": "Wall-clock duration of the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "catalogKey": "input_tokens", + "name": "Input tokens", + "description": "Tokens sent to the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "catalogKey": "output_tokens", + "name": "Output tokens", + "description": "Tokens generated by the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "catalogKey": "total_tokens", + "name": "Total tokens", + "description": "Combined input and output tokens for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "catalogKey": "tool_calls", + "name": "Tool calls", + "description": "Recorded tool-call attempts across attributed Agent runs.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "catalogKey": "tool_error_rate", + "name": "Tool error rate", + "description": "Failed tool calls divided by all tool-call attempts.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "catalogKey": "agent_loop_iterations", + "name": "Agent-loop iterations", + "description": "Distinct Agent reasoning-loop iterations in the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "catalogKey": "critic_rejections", + "name": "Critic rejections", + "description": "Rejected critic-gate reviews during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "catalogKey": "critic_approvals", + "name": "Critic approvals", + "description": "Critic gates ending in an actual approval.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "sum", + "unit": "reviews" + }, { "id": "model-evaluation", - "name": "Model evaluation", - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "name": "model_evaluation", + "description": "Complete held-out scoring response from run_model_script.", "kind": "custom", "valueType": "opaque", "primary": false, "direction": "neutral", "aggregation": "none" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - }, - "measurement_plan": { - "metrics": [ + }, { - "id": "model-evaluation", - "name": "Model evaluation", - "value_type": "opaque", + "id": "pr-auc", + "name": "pr_auc", + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "kind": "custom", + "valueType": "number", + "primary": true, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "description": "Held-out ROC-AUC.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "description": "Held-out Brier score.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "description": "Training-selected operating threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, "direction": "neutral", - "aggregation": "none", + "aggregation": "mean" + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "description": "Observed positive-class rate in the held-out test split.", + "kind": "custom", + "valueType": "number", "primary": false, - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." - } - ], - "producers": [ + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, { - "id": "model-evaluation-source", - "producer_id": "asaree.mcp_tool", - "kind": "reported", - "outputs": { - "value": "model-evaluation" - }, - "artifacts": [], - "config": { - "agent_node_id": "agent-score", - "mcp_node_id": "tool-model-score", - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "tool_name": "run_model_script" - } - } - ], - "inputs": [] + "id": "n-test-samples", + "name": "n_test_samples", + "description": "Held-out test sample count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "description": "Positive-class sample count in the held-out test split.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "description": "accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "description": "balanced accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "description": "precision at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "description": "recall at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "description": "f1 at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "description": "accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "description": "balanced accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "description": "precision at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "description": "recall at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "description": "f1 at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "description": "accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "description": "balanced accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "description": "precision at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "description": "recall at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "description": "f1 at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "description": "Feature count after data cleaning.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "description": "Data-cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "description": "Feature-engineering recipe operation count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "description": "Feature count after engineering and encoding.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "description": "Features surviving selection and reaching modeling.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "description": "Engineered features surviving feature selection.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "description": "Engineered features as a percentage of selected features.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "description": "Feature-selection variance threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "selected-features", + "name": "selected_features", + "description": "Final selected feature names.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "description": "Optuna trial count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "description": "Best inner-CV average precision found by Optuna.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "description": "Validation-fold Brier score after isotonic calibration.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "description": "Final XGBoost n_estimators value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "description": "Final XGBoost max_depth value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "description": "Final XGBoost learning_rate value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "description": "Final XGBoost min_child_weight value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "description": "Final XGBoost subsample value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "description": "Final XGBoost colsample_bytree value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "description": "Final XGBoost reg_lambda value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "description": "Final XGBoost reg_alpha value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "description": "Final XGBoost gamma value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "code-sha256", + "name": "code_sha256", + "description": "SHA-256 of the executed fixed modeling script.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "description": "SHA-256 of the approved hyperparameter payload.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "data-sha256", + "name": "data_sha256", + "description": "SHA-256 provenance hash for the scored data.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + } + ], + "coordination_strategy": { + "slug": "critic_gate" + } + }, + "measurement_plan": { + "metrics": [ + { + "id": "runtime-cost", + "name": "Cost", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Estimated provider cost for the run.", + "unit": "USD" + }, + { + "id": "runtime-duration", + "name": "Duration", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Wall-clock duration of the run.", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "name": "Input tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens sent to the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "name": "Output tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens generated by the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "name": "Total tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Combined input and output tokens for the run.", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "name": "Tool calls", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Recorded tool-call attempts across attributed Agent runs.", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "name": "Tool error rate", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Failed tool calls divided by all tool-call attempts.", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "name": "Agent-loop iterations", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Distinct Agent reasoning-loop iterations in the run.", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "name": "Critic rejections", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Rejected critic-gate reviews during the run.", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "name": "Critic approvals", + "value_type": "number", + "direction": "neutral", + "aggregation": "sum", + "primary": false, + "description": "Critic gates ending in an actual approval.", + "unit": "reviews" + }, + { + "id": "model-evaluation", + "name": "model_evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Complete held-out scoring response from run_model_script." + }, + { + "id": "pr-auc", + "name": "pr_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": true, + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Held-out ROC-AUC.", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Held-out Brier score.", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Training-selected operating threshold." + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Observed positive-class rate in the held-out test split.", + "unit": "proportion" + }, + { + "id": "n-test-samples", + "name": "n_test_samples", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Held-out test sample count.", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive-class sample count in the held-out test split.", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the 0 5 operating point." + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the 0 5 operating point." + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the 0 5 operating point." + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the 0 5 operating point." + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the 0 5 operating point." + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the chosen threshold operating point." + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the chosen threshold operating point." + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the chosen threshold operating point." + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the chosen threshold operating point." + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the chosen threshold operating point." + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the prevalence operating point." + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the prevalence operating point." + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the prevalence operating point." + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the prevalence operating point." + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the prevalence operating point." + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after data cleaning.", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Data-cleaning missingness threshold.", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-engineering recipe operation count.", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after engineering and encoding.", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Features surviving selection and reaching modeling.", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features surviving feature selection.", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features as a percentage of selected features.", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-selection variance threshold." + }, + { + "id": "selected-features", + "name": "selected_features", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Final selected feature names." + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Optuna trial count.", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Best inner-CV average precision found by Optuna.", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Validation-fold Brier score after isotonic calibration.", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost n_estimators value." + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost max_depth value." + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost learning_rate value." + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost min_child_weight value." + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost subsample value." + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost colsample_bytree value." + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_lambda value." + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_alpha value." + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost gamma value." + }, + { + "id": "code-sha256", + "name": "code_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the executed fixed modeling script." + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the approved hyperparameter payload." + }, + { + "id": "data-sha256", + "name": "data_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 provenance hash for the scored data." + } + ], + "producers": [ + { + "id": "runtime", + "producer_id": "asaree.runtime", + "kind": "runtime", + "outputs": { + "cost_usd": "runtime-cost", + "duration_seconds": "runtime-duration", + "input_tokens": "runtime-input-tokens", + "output_tokens": "runtime-output-tokens", + "total_tokens": "runtime-total-tokens", + "tool_calls": "runtime-tool-calls", + "tool_error_rate": "runtime-tool-error-rate", + "agent_loop_iterations": "runtime-agent-loop-iterations", + "critic_rejections": "runtime-critic-rejections", + "critic_approvals": "runtime-critic-approvals" + }, + "artifacts": [], + "config": {} + }, + { + "id": "dc-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_dc": "n-features-after-dc", + "missingness_threshold": "missingness-threshold", + "n_dropped_for_missingness": "n-dropped-for-missingness" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-dc", + "projections": { + "n_features_after_dc": { + "path": "n_cols_out" + }, + "missingness_threshold": { + "path": "missingness_threshold" + }, + "n_dropped_for_missingness": { + "path": "dropped_for_missingness", + "transform": "length" + } + } + } + }, + { + "id": "fte-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_engineered_features": "n-engineered-features", + "n_features_after_fte": "n-features-after-fte" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fte", + "projections": { + "n_engineered_features": { + "path": "engineering_recipe", + "transform": "length" + }, + "n_features_after_fte": { + "path": "n_features_out" + } + } + } + }, + { + "id": "fs-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_fs": "n-features-after-fs", + "n_engineered_features_selected": "n-engineered-features-selected", + "pct_engineered_features_selected": "pct-engineered-features-selected", + "variance_threshold": "variance-threshold", + "selected_features": "selected-features" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fs", + "projections": { + "n_features_after_fs": { + "path": "n_features_out" + }, + "n_engineered_features_selected": { + "path": "n_engineered_features_selected" + }, + "pct_engineered_features_selected": { + "path": "pct_engineered_features_selected" + }, + "variance_threshold": { + "path": "variance_threshold" + }, + "selected_features": { + "path": "selected_features" + } + } + } + }, + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "model_evaluation": "model-evaluation", + "pr_auc": "pr-auc", + "roc_auc": "roc-auc", + "brier_score": "brier-score", + "chosen_threshold": "chosen-threshold", + "test_prevalence": "test-prevalence", + "n_test_samples": "n-test-samples", + "n_pos_test": "n-pos-test", + "accuracy_at_0_5": "accuracy-0-5", + "balanced_accuracy_at_0_5": "balanced_accuracy-0-5", + "precision_at_0_5": "precision-0-5", + "recall_at_0_5": "recall-0-5", + "f1_at_0_5": "f1-0-5", + "accuracy_at_chosen_threshold": "accuracy-chosen", + "balanced_accuracy_at_chosen_threshold": "balanced_accuracy-chosen", + "precision_at_chosen_threshold": "precision-chosen", + "recall_at_chosen_threshold": "recall-chosen", + "f1_at_chosen_threshold": "f1-chosen", + "accuracy_at_prevalence": "accuracy-prevalence", + "balanced_accuracy_at_prevalence": "balanced_accuracy-prevalence", + "precision_at_prevalence": "precision-prevalence", + "recall_at_prevalence": "recall-prevalence", + "f1_at_prevalence": "f1-prevalence", + "n_pred_pos_at_0_5": "n-pred-pos-0-5", + "n_pred_pos_at_chosen_threshold": "n-pred-pos-chosen", + "n_pred_pos_at_prevalence": "n-pred-pos-prevalence", + "optuna_n_trials": "optuna-n-trials", + "optuna_best_inner_cv_score": "optuna-best-inner-cv-score", + "calibration_brier_val": "calibration-brier-val", + "code_sha256": "code-sha256", + "payload_sha256": "payload-sha256", + "data_sha256": "data-sha256", + "hp_n_estimators": "hp-n-estimators", + "hp_max_depth": "hp-max-depth", + "hp_learning_rate": "hp-learning-rate", + "hp_min_child_weight": "hp-min-child-weight", + "hp_subsample": "hp-subsample", + "hp_colsample_bytree": "hp-colsample-bytree", + "hp_reg_lambda": "hp-reg-lambda", + "hp_reg_alpha": "hp-reg-alpha", + "hp_gamma": "hp-gamma" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script", + "projections": { + "pr_auc": { + "path": "test_metrics.average_precision" + }, + "roc_auc": { + "path": "test_metrics.roc_auc" + }, + "brier_score": { + "path": "test_metrics.brier_score" + }, + "chosen_threshold": { + "path": "test_metrics.chosen_threshold" + }, + "test_prevalence": { + "path": "test_metrics.test_prevalence" + }, + "n_test_samples": { + "path": "test_metrics.n_test_samples" + }, + "n_pos_test": { + "path": "test_metrics.n_pos_test" + }, + "accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/accuracy" + }, + "balanced_accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/balanced_accuracy" + }, + "precision_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/precision" + }, + "recall_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/recall" + }, + "f1_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/f1" + }, + "accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.accuracy" + }, + "balanced_accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.balanced_accuracy" + }, + "precision_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.precision" + }, + "recall_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.recall" + }, + "f1_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.f1" + }, + "accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.accuracy" + }, + "balanced_accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.balanced_accuracy" + }, + "precision_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.precision" + }, + "recall_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.recall" + }, + "f1_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.f1" + }, + "n_pred_pos_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/n_pred_pos" + }, + "n_pred_pos_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.n_pred_pos" + }, + "n_pred_pos_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.n_pred_pos" + }, + "optuna_n_trials": { + "path": "model_decisions.n_trials" + }, + "optuna_best_inner_cv_score": { + "path": "model_decisions.optuna_best_value" + }, + "calibration_brier_val": { + "path": "model_decisions.calibration_brier_val" + }, + "code_sha256": { + "path": "code_sha256" + }, + "payload_sha256": { + "path": "payload_sha256" + }, + "data_sha256": { + "path": "data_sha256" + }, + "hp_n_estimators": { + "path": "model_decisions.best_hyperparameters.n_estimators" + }, + "hp_max_depth": { + "path": "model_decisions.best_hyperparameters.max_depth" + }, + "hp_learning_rate": { + "path": "model_decisions.best_hyperparameters.learning_rate" + }, + "hp_min_child_weight": { + "path": "model_decisions.best_hyperparameters.min_child_weight" + }, + "hp_subsample": { + "path": "model_decisions.best_hyperparameters.subsample" + }, + "hp_colsample_bytree": { + "path": "model_decisions.best_hyperparameters.colsample_bytree" + }, + "hp_reg_lambda": { + "path": "model_decisions.best_hyperparameters.reg_lambda" + }, + "hp_reg_alpha": { + "path": "model_decisions.best_hyperparameters.reg_alpha" + }, + "hp_gamma": { + "path": "model_decisions.best_hyperparameters.gamma" + } + } + } + } + ], + "inputs": [ + { + "producer_binding_id": "runtime", + "input_key": "facts", + "source_key": "attempt.runtime" + } + ] } } diff --git a/publications/BDM/myocardial-openai-v0.8.0.json b/publications/BDM/myocardial-openai-v0.8.0.json index 20913f3b..5c8a81e7 100644 --- a/publications/BDM/myocardial-openai-v0.8.0.json +++ b/publications/BDM/myocardial-openai-v0.8.0.json @@ -275,7 +275,7 @@ "prompt": "", "goal": "Prune the feature space; commit v3_fs. After your final filter/balance-check tool call succeeds, call accept_stage(stage=\"fs\") yourself to promote your work to workspace HEAD before finishing -- this run's Critic Gate reviews your report, not this promotion, so call it on every attempt (a later revision's call simply supersedes an earlier one).\n\nRandom seed: 42\n\ntask_brief: {\"domain\": \"acute myocardial infarction admissions, UCI Myocardial Infarction Complications (ZSN) dataset\", \"prediction_task\": \"binary\", \"target_column\": \"mi_ZSN\", \"positive_class\": 1, \"expected_positive_rate\": 0.232, \"selection_metric\": \"average_precision\", \"num_classes\": 2, \"group_column\": null, \"notes\": \"Predict development of chronic heart failure (ZSN) as a post-MI complication. Dataset: 1700 admissions x 113 raw columns (1 ID + 111 features + target mi_ZSN); target is binary (0=no ZSN, 1=ZSN), observed positive rate 23.2%. Column `ID` (role=id, values 1..1700, all unique) is a non-feature record identifier -- exclude it from the training matrix entirely; it must never be imputed, engineered, selected, or scored as a feature. Unlike some ASAREE datasets, features here are RAW, not pre-encoded: types are a mix of integer, real, binary, ordinal, and categorical, and several are known to have real missingness (this is the original UCI dataset, not a cleaned derivative) -- confirm actual missingness yourself via inspect_columns rather than assuming any column is complete. Column names are NOT descriptive English words; they are short Russian-derived codes (e.g. `nr11`, `np01`, `zab_leg_01`, `S_AD_KBRIG`) -- do not infer meaning from the name alone, always confirm via get_data_dictionary, which carries a description and, for categorical/ordinal columns, its code mapping. FEATURE FAMILIES: demographics (AGE, SEX); cardiac history (INF_ANAM prior-MI count, STENOK_AN/FK_STENOK angina, GB/SIM_GIPERT/DLIT_AG hypertension, ZSN_A prior heart failure, IBS_POST/IBS_NASL coronary disease, nr*/np* prior arrhythmia and conduction-block history); comorbidities (endocr_* diabetes/obesity/thyroid, zab_leg_* chronic pulmonary disease); admission vitals and ICU-admission complications (S_AD_KBRIG/D_AD_KBRIG/S_AD_ORIT/D_AD_ORIT blood pressure, O_L_POST/K_SH_POST/MP_TP_POST/SVT_POST/GT_POST/FIB_G_POST); infarct localization (ant_im/lat_im/inf_im/post_im/IM_PG_P); admission ECG rhythm and conduction findings (ritm_ecg_p_*, n_r_ecg_p_*, n_p_ecg_p_*); labs (K_BLOOD, Na_BLOOD, ALT_BLOOD, AST_BLOOD, KFK_BLOOD, L_BLOOD, ROE) with their derived abnormal-value flags (GIPO_K, GIPER_Na); and treatment given before/during ICU admission (fibr_ter_* fibrinolytics, NA_KB/NA_R_*_n opioids, NOT_NA_*_n/NOT_NA_KB NSAIDs, LID_KB/LID_S_n lidocaine, B_BLOK_S_n beta-blockers, ANT_CA_S_n calcium-channel blockers, GEPAR_S_n heparin, ASP_S_n aspirin, TIKL_S_n/TRENT_S_n antiplatelets, NITR_S nitrates). A per-column data dictionary (description + codes) is available via get_data_dictionary.\"}", "description": "Filters by variance/correlation/supervised signal and checks class balance on top of v2_fte, commits the result as workspace version v3_fs.", - "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", + "system_prompt": "\nYou are a feature SELECTION agent for biomedical tabular data. You run THIRD, on\nthe engineered + encoded dataset from FTE (already clean, complete, and fully\nnumeric; missingness, engineering, and encoding were handled upstream). Your job\nis to profile the feature space and prune it through unsupervised filters plus one\nsupervised filter, over the FULL space of original survivors and engineered\nmeta-features alike. You do NOT clean, impute, engineer, or model. You do NOT\ntouch the test split.\n\nWORKSPACE MODEL (read this first): the FTE stage already committed the encoded,\nmodel-ready train/test matrix (v2_fte). Opening the workspace loads that encoded\nTRAINING matrix for you — you do NOT re-apply any preprocessor; the input is\nalready the space the runner will score. A selection tool computes on the\ntraining split only and writes this stage's dataset version (v3_fs) = the input\nrestricted to the columns you keep, on BOTH splits; the cell's workspace identity\ntravels with every tool call automatically, so you never pass a workspace id or a\ndataset id. There is no save_artifact step and there are no artifact keys.\n\nRUN CONTEXT\nYou will receive:\n- task_brief (same schema as DC's).\n- The approved FTE recipe report (context; the encoded data is already on disk).\n- The registered dataset name and random seed.\n- Your cell's workspace is ALREADY OPEN, seeded from the wired dataset before your\n first turn -- your run context says so. What that does NOT do is materialize this\n stage's scratch input, which the selection tools read, so you still make one\n argument-free-except-stage open_workspace(stage=\"fs\") call. You never pass a\n workspace id, an experiment id, a cell label or a dataset id to anything: the\n workspace identity travels with every tool call as ambient run context.\n The FINAL selection call commits the selected columns as v3_fs.\n\nReproduce the same random_seed in every tool call. Every drop must cite a\nspecific, measurable criterion.\n\nSTEPS\n1. Call open_workspace(stage=\"fs\") -- that argument and nothing else; which\n workspace and which dataset both arrive as ambient run context.\n It loads the ENCODED, model-ready TRAINING matrix (the FTE output) into your\n scratch input. Do NOT re-apply a preprocessor; selection runs directly on this\n already-encoded space, so the selected names align with what the runner scores.\n2. Profile every column (get_feature_distributions()): dtype, n_unique,\n summary stats. (Data is already complete and numeric; this is selection context.)\n3. Variance filter: variance_filter(threshold=)\n (wraps sklearn VarianceThreshold): set and STATE a threshold; drop low /\n near-zero-variance features. State your scaling assumption, since the threshold\n interacts with scale.\n4. Correlation filter (compute_correlations(method='spearman')): find\n pairs with |rho| above a stated threshold (e.g. 0.9). For each pair remove ONE\n member using an UNSUPERVISED tie-break: keep the lower-missingness member, then\n the higher-variance member; drop whichever is harder to interpret on a true\n tie. Do not use the label here.\n5. Supervised filter: choose the survivors of steps 3-4 and call\n supervised_filter(method=\"f_classif\" | \"mutual_info_classif\",\n k=, features_json=).\n This is the FINAL selection call: it commits the selected\n columns as this stage's output (v3_fs). The committed columns must be a subset\n of the v2_fte columns.\n6. Call check_class_balance() and verify the observed rate matches\n expected_positive_rate (binary) or the class count matches num_classes.\n\n7. Compare selected_features with the output names in the upstream FTE\n engineering_recipe. Report their intersection count as\n n_engineered_features_selected and 100 times that count divided by\n n_features_out as pct_engineered_features_selected.\n\nCONCISENESS: report only what you changed or flagged. Summarize untouched columns\nby COUNT — do NOT enumerate all columns. Keep every rationale to one line. The\ncommitted dataset version carries the full detail; the runner reads it.\n\nOUTPUT - return ONE JSON dossier:\n{\n \"n_features_in\": ..., \"n_features_out\": ...,\n \"n_engineered_features_selected\": ...,\n \"pct_engineered_features_selected\": ...,\n \"variance_threshold\": 0.0,\n \"correlation\": {\"method\": \"spearman\", \"rho_threshold\": 0.9,\n \"pairs_removed\": [[\"kept\", \"dropped\", 0.95], ...]},\n \"supervised_filter\": {\"method\": \"f_classif|mutual_info_classif\", \"k\": 0,\n \"top_scores\": [[\"feature\", score], ...]},\n \"dropped_features\": [{\"feature\": \"...\", \"reason\": \"criterion + measured evidence\"}],\n \"selected_features\": [\"...\"],\n \"observed_class_distribution\": {\"\": 0.0, ...},\n \"class_balance_check\": \"matches task_brief\" | \"diverges: ...\",\n \"notes_for_mlm\": \"caveats, dtype quirks, anything MLM should know.\"\n}\n", "output_contract": null, "budget_limit_usd": 200.0, "max_run_duration_seconds": 86400, @@ -549,12 +549,12 @@ }, { "name": "outliers", - "type": "object", + "type": "array", "description": "Outlier handling summary per column" }, { "name": "imputation", - "type": "object", + "type": "array", "description": "Imputation strategy used per column" }, { @@ -584,7 +584,7 @@ "fields": [ { "name": "engineering_recipe", - "type": "object", + "type": "array", "description": "Feature-engineering steps applied" }, { @@ -632,6 +632,16 @@ "type": "integer", "description": "Feature count after selection" }, + { + "name": "n_engineered_features_selected", + "type": "integer", + "description": "Selected features created by the upstream engineering recipe" + }, + { + "name": "pct_engineered_features_selected", + "type": "number", + "description": "Percentage of selected features created by the upstream engineering recipe" + }, { "name": "variance_threshold", "type": "number", @@ -664,8 +674,8 @@ }, { "name": "class_balance_check", - "type": "boolean", - "description": "Whether class balance passed the configured check" + "type": "string", + "description": "Whether observed class balance matches the task brief, with divergence details when it does not" }, { "name": "notes_for_mlm", @@ -694,7 +704,7 @@ "fields": [ { "name": "search_space", - "type": "object", + "type": "array", "description": "Hyperparameter search space proposal" }, { @@ -748,7 +758,7 @@ "description": "Hash of the scored payload, for the approved-payload guard" }, { - "name": "result", + "name": "model_decisions", "type": "object", "description": "Train-side decisions the script itself reported (tuning, calibration, threshold)" } @@ -1046,50 +1056,1528 @@ ], "replicates": 10, "metrics": [ + { + "id": "runtime-cost", + "catalogKey": "cost_usd", + "name": "Cost", + "description": "Estimated provider cost for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "USD" + }, + { + "id": "runtime-duration", + "catalogKey": "duration_seconds", + "name": "Duration", + "description": "Wall-clock duration of the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "catalogKey": "input_tokens", + "name": "Input tokens", + "description": "Tokens sent to the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "catalogKey": "output_tokens", + "name": "Output tokens", + "description": "Tokens generated by the model during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "catalogKey": "total_tokens", + "name": "Total tokens", + "description": "Combined input and output tokens for the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "catalogKey": "tool_calls", + "name": "Tool calls", + "description": "Recorded tool-call attempts across attributed Agent runs.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "catalogKey": "tool_error_rate", + "name": "Tool error rate", + "description": "Failed tool calls divided by all tool-call attempts.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "catalogKey": "agent_loop_iterations", + "name": "Agent-loop iterations", + "description": "Distinct Agent reasoning-loop iterations in the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "catalogKey": "critic_rejections", + "name": "Critic rejections", + "description": "Rejected critic-gate reviews during the run.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "sum", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "catalogKey": "critic_approvals", + "name": "Critic approvals", + "description": "Critic gates ending in an actual approval.", + "kind": "runtime", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "sum", + "unit": "reviews" + }, { "id": "model-evaluation", - "name": "Model evaluation", - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards.", + "name": "model_evaluation", + "description": "Complete held-out scoring response from run_model_script.", "kind": "custom", "valueType": "opaque", "primary": false, "direction": "neutral", "aggregation": "none" - } - ], - "coordination_strategy": { - "slug": "critic_gate" - } - }, - "measurement_plan": { - "metrics": [ + }, { - "id": "model-evaluation", - "name": "Model evaluation", - "value_type": "opaque", + "id": "pr-auc", + "name": "pr_auc", + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "kind": "custom", + "valueType": "number", + "primary": true, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "description": "Held-out ROC-AUC.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "description": "Held-out Brier score.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "description": "Training-selected operating threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, "direction": "neutral", - "aggregation": "none", + "aggregation": "mean" + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "description": "Observed positive-class rate in the held-out test split.", + "kind": "custom", + "valueType": "number", "primary": false, - "description": "The complete held-out scoring result reported by the wired asaree-sklearn-model run_model_script call, including test metrics and SHA-256 guards." - } - ], - "producers": [ + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, { - "id": "model-evaluation-source", - "producer_id": "asaree.mcp_tool", - "kind": "reported", - "outputs": { - "value": "model-evaluation" - }, - "artifacts": [], - "config": { - "agent_node_id": "agent-score", - "mcp_node_id": "tool-model-score", - "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", - "tool_name": "run_model_script" - } - } - ], - "inputs": [] + "id": "n-test-samples", + "name": "n_test_samples", + "description": "Held-out test sample count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "description": "Positive-class sample count in the held-out test split.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "description": "accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "description": "balanced accuracy at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "description": "precision at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "description": "recall at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "description": "f1 at the 0 5 operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "description": "accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "description": "balanced accuracy at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "description": "precision at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "description": "recall at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "description": "f1 at the chosen threshold operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "description": "accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "description": "balanced accuracy at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "description": "precision at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "description": "recall at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "description": "f1 at the prevalence operating point.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean" + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "description": "Feature count after data cleaning.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "description": "Data-cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "description": "Feature-engineering recipe operation count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "description": "Feature count after engineering and encoding.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "description": "Features surviving selection and reaching modeling.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "description": "Engineered features surviving feature selection.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "description": "Engineered features as a percentage of selected features.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "description": "Feature-selection variance threshold.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "selected-features", + "name": "selected_features", + "description": "Final selected feature names.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "description": "Optuna trial count.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "description": "Best inner-CV average precision found by Optuna.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "maximize", + "aggregation": "mean", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "description": "Validation-fold Brier score after isotonic calibration.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "minimize", + "aggregation": "mean", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "description": "Final XGBoost n_estimators value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "description": "Final XGBoost max_depth value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "description": "Final XGBoost learning_rate value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "description": "Final XGBoost min_child_weight value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "description": "Final XGBoost subsample value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "description": "Final XGBoost colsample_bytree value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "description": "Final XGBoost reg_lambda value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "description": "Final XGBoost reg_alpha value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "description": "Final XGBoost gamma value.", + "kind": "custom", + "valueType": "number", + "primary": false, + "direction": "neutral", + "aggregation": "mean" + }, + { + "id": "code-sha256", + "name": "code_sha256", + "description": "SHA-256 of the executed fixed modeling script.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "description": "SHA-256 of the approved hyperparameter payload.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + }, + { + "id": "data-sha256", + "name": "data_sha256", + "description": "SHA-256 provenance hash for the scored data.", + "kind": "custom", + "valueType": "opaque", + "primary": false, + "direction": "neutral", + "aggregation": "none" + } + ], + "coordination_strategy": { + "slug": "critic_gate" + } + }, + "measurement_plan": { + "metrics": [ + { + "id": "runtime-cost", + "name": "Cost", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Estimated provider cost for the run.", + "unit": "USD" + }, + { + "id": "runtime-duration", + "name": "Duration", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Wall-clock duration of the run.", + "unit": "seconds" + }, + { + "id": "runtime-input-tokens", + "name": "Input tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens sent to the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-output-tokens", + "name": "Output tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Tokens generated by the model during the run.", + "unit": "tokens" + }, + { + "id": "runtime-total-tokens", + "name": "Total tokens", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Combined input and output tokens for the run.", + "unit": "tokens" + }, + { + "id": "runtime-tool-calls", + "name": "Tool calls", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Recorded tool-call attempts across attributed Agent runs.", + "unit": "calls" + }, + { + "id": "runtime-tool-error-rate", + "name": "Tool error rate", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Failed tool calls divided by all tool-call attempts.", + "unit": "rate" + }, + { + "id": "runtime-agent-loop-iterations", + "name": "Agent-loop iterations", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Distinct Agent reasoning-loop iterations in the run.", + "unit": "iterations" + }, + { + "id": "runtime-critic-rejections", + "name": "Critic rejections", + "value_type": "number", + "direction": "minimize", + "aggregation": "sum", + "primary": false, + "description": "Rejected critic-gate reviews during the run.", + "unit": "reviews" + }, + { + "id": "runtime-critic-approvals", + "name": "Critic approvals", + "value_type": "number", + "direction": "neutral", + "aggregation": "sum", + "primary": false, + "description": "Critic gates ending in an actual approval.", + "unit": "reviews" + }, + { + "id": "model-evaluation", + "name": "model_evaluation", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Complete held-out scoring response from run_model_script." + }, + { + "id": "pr-auc", + "name": "pr_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": true, + "description": "Held-out average precision (PR-AUC), the prespecified primary endpoint.", + "unit": "AUPRC" + }, + { + "id": "roc-auc", + "name": "roc_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Held-out ROC-AUC.", + "unit": "ROC-AUC" + }, + { + "id": "brier-score", + "name": "brier_score", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Held-out Brier score.", + "unit": "score" + }, + { + "id": "chosen-threshold", + "name": "chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Training-selected operating threshold." + }, + { + "id": "test-prevalence", + "name": "test_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Observed positive-class rate in the held-out test split.", + "unit": "proportion" + }, + { + "id": "n-test-samples", + "name": "n_test_samples", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Held-out test sample count.", + "unit": "samples" + }, + { + "id": "n-pos-test", + "name": "n_pos_test", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive-class sample count in the held-out test split.", + "unit": "samples" + }, + { + "id": "accuracy-0-5", + "name": "accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the 0 5 operating point." + }, + { + "id": "balanced_accuracy-0-5", + "name": "balanced_accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the 0 5 operating point." + }, + { + "id": "precision-0-5", + "name": "precision_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the 0 5 operating point." + }, + { + "id": "recall-0-5", + "name": "recall_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the 0 5 operating point." + }, + { + "id": "f1-0-5", + "name": "f1_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the 0 5 operating point." + }, + { + "id": "accuracy-chosen", + "name": "accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the chosen threshold operating point." + }, + { + "id": "balanced_accuracy-chosen", + "name": "balanced_accuracy_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the chosen threshold operating point." + }, + { + "id": "precision-chosen", + "name": "precision_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the chosen threshold operating point." + }, + { + "id": "recall-chosen", + "name": "recall_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the chosen threshold operating point." + }, + { + "id": "f1-chosen", + "name": "f1_at_chosen_threshold", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the chosen threshold operating point." + }, + { + "id": "accuracy-prevalence", + "name": "accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "accuracy at the prevalence operating point." + }, + { + "id": "balanced_accuracy-prevalence", + "name": "balanced_accuracy_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "balanced accuracy at the prevalence operating point." + }, + { + "id": "precision-prevalence", + "name": "precision_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "precision at the prevalence operating point." + }, + { + "id": "recall-prevalence", + "name": "recall_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "recall at the prevalence operating point." + }, + { + "id": "f1-prevalence", + "name": "f1_at_prevalence", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "f1 at the prevalence operating point." + }, + { + "id": "n-pred-pos-0-5", + "name": "n_pred_pos_at_0_5", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the 0.5 operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-chosen", + "name": "n_pred_pos_at_chosen_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the training-selected operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-pred-pos-prevalence", + "name": "n_pred_pos_at_prevalence", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Positive predictions at the prevalence operating point; zero indicates undefined precision and F1.", + "unit": "predictions" + }, + { + "id": "n-features-after-dc", + "name": "n_features_after_dc", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after data cleaning.", + "unit": "features" + }, + { + "id": "missingness-threshold", + "name": "missingness_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Data-cleaning missingness threshold.", + "unit": "proportion" + }, + { + "id": "n-dropped-for-missingness", + "name": "n_dropped_for_missingness", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Columns dropped for exceeding the cleaning missingness threshold.", + "unit": "features" + }, + { + "id": "n-engineered-features", + "name": "n_engineered_features", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-engineering recipe operation count.", + "unit": "features" + }, + { + "id": "n-features-after-fte", + "name": "n_features_after_fte", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature count after engineering and encoding.", + "unit": "features" + }, + { + "id": "n-features-after-fs", + "name": "n_features_after_fs", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Features surviving selection and reaching modeling.", + "unit": "features" + }, + { + "id": "n-engineered-features-selected", + "name": "n_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features surviving feature selection.", + "unit": "features" + }, + { + "id": "pct-engineered-features-selected", + "name": "pct_engineered_features_selected", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Engineered features as a percentage of selected features.", + "unit": "percent" + }, + { + "id": "variance-threshold", + "name": "variance_threshold", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Feature-selection variance threshold." + }, + { + "id": "selected-features", + "name": "selected_features", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "Final selected feature names." + }, + { + "id": "optuna-n-trials", + "name": "optuna_n_trials", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Optuna trial count.", + "unit": "trials" + }, + { + "id": "optuna-best-inner-cv-score", + "name": "optuna_best_inner_cv_score", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": false, + "description": "Best inner-CV average precision found by Optuna.", + "unit": "AUPRC" + }, + { + "id": "calibration-brier-val", + "name": "calibration_brier_val", + "value_type": "number", + "direction": "minimize", + "aggregation": "mean", + "primary": false, + "description": "Validation-fold Brier score after isotonic calibration.", + "unit": "score" + }, + { + "id": "hp-n-estimators", + "name": "hp_n_estimators", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost n_estimators value." + }, + { + "id": "hp-max-depth", + "name": "hp_max_depth", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost max_depth value." + }, + { + "id": "hp-learning-rate", + "name": "hp_learning_rate", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost learning_rate value." + }, + { + "id": "hp-min-child-weight", + "name": "hp_min_child_weight", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost min_child_weight value." + }, + { + "id": "hp-subsample", + "name": "hp_subsample", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost subsample value." + }, + { + "id": "hp-colsample-bytree", + "name": "hp_colsample_bytree", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost colsample_bytree value." + }, + { + "id": "hp-reg-lambda", + "name": "hp_reg_lambda", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_lambda value." + }, + { + "id": "hp-reg-alpha", + "name": "hp_reg_alpha", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost reg_alpha value." + }, + { + "id": "hp-gamma", + "name": "hp_gamma", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + "primary": false, + "description": "Final XGBoost gamma value." + }, + { + "id": "code-sha256", + "name": "code_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the executed fixed modeling script." + }, + { + "id": "payload-sha256", + "name": "payload_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 of the approved hyperparameter payload." + }, + { + "id": "data-sha256", + "name": "data_sha256", + "value_type": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": false, + "description": "SHA-256 provenance hash for the scored data." + } + ], + "producers": [ + { + "id": "runtime", + "producer_id": "asaree.runtime", + "kind": "runtime", + "outputs": { + "cost_usd": "runtime-cost", + "duration_seconds": "runtime-duration", + "input_tokens": "runtime-input-tokens", + "output_tokens": "runtime-output-tokens", + "total_tokens": "runtime-total-tokens", + "tool_calls": "runtime-tool-calls", + "tool_error_rate": "runtime-tool-error-rate", + "agent_loop_iterations": "runtime-agent-loop-iterations", + "critic_rejections": "runtime-critic-rejections", + "critic_approvals": "runtime-critic-approvals" + }, + "artifacts": [], + "config": {} + }, + { + "id": "dc-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_dc": "n-features-after-dc", + "missingness_threshold": "missingness-threshold", + "n_dropped_for_missingness": "n-dropped-for-missingness" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-dc", + "projections": { + "n_features_after_dc": { + "path": "n_cols_out" + }, + "missingness_threshold": { + "path": "missingness_threshold" + }, + "n_dropped_for_missingness": { + "path": "dropped_for_missingness", + "transform": "length" + } + } + } + }, + { + "id": "fte-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_engineered_features": "n-engineered-features", + "n_features_after_fte": "n-features-after-fte" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fte", + "projections": { + "n_engineered_features": { + "path": "engineering_recipe", + "transform": "length" + }, + "n_features_after_fte": { + "path": "n_features_out" + } + } + } + }, + { + "id": "fs-report-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": { + "n_features_after_fs": "n-features-after-fs", + "n_engineered_features_selected": "n-engineered-features-selected", + "pct_engineered_features_selected": "pct-engineered-features-selected", + "variance_threshold": "variance-threshold", + "selected_features": "selected-features" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-fs", + "projections": { + "n_features_after_fs": { + "path": "n_features_out" + }, + "n_engineered_features_selected": { + "path": "n_engineered_features_selected" + }, + "pct_engineered_features_selected": { + "path": "pct_engineered_features_selected" + }, + "variance_threshold": { + "path": "variance_threshold" + }, + "selected_features": { + "path": "selected_features" + } + } + } + }, + { + "id": "model-evaluation-source", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": { + "model_evaluation": "model-evaluation", + "pr_auc": "pr-auc", + "roc_auc": "roc-auc", + "brier_score": "brier-score", + "chosen_threshold": "chosen-threshold", + "test_prevalence": "test-prevalence", + "n_test_samples": "n-test-samples", + "n_pos_test": "n-pos-test", + "accuracy_at_0_5": "accuracy-0-5", + "balanced_accuracy_at_0_5": "balanced_accuracy-0-5", + "precision_at_0_5": "precision-0-5", + "recall_at_0_5": "recall-0-5", + "f1_at_0_5": "f1-0-5", + "accuracy_at_chosen_threshold": "accuracy-chosen", + "balanced_accuracy_at_chosen_threshold": "balanced_accuracy-chosen", + "precision_at_chosen_threshold": "precision-chosen", + "recall_at_chosen_threshold": "recall-chosen", + "f1_at_chosen_threshold": "f1-chosen", + "accuracy_at_prevalence": "accuracy-prevalence", + "balanced_accuracy_at_prevalence": "balanced_accuracy-prevalence", + "precision_at_prevalence": "precision-prevalence", + "recall_at_prevalence": "recall-prevalence", + "f1_at_prevalence": "f1-prevalence", + "n_pred_pos_at_0_5": "n-pred-pos-0-5", + "n_pred_pos_at_chosen_threshold": "n-pred-pos-chosen", + "n_pred_pos_at_prevalence": "n-pred-pos-prevalence", + "optuna_n_trials": "optuna-n-trials", + "optuna_best_inner_cv_score": "optuna-best-inner-cv-score", + "calibration_brier_val": "calibration-brier-val", + "code_sha256": "code-sha256", + "payload_sha256": "payload-sha256", + "data_sha256": "data-sha256", + "hp_n_estimators": "hp-n-estimators", + "hp_max_depth": "hp-max-depth", + "hp_learning_rate": "hp-learning-rate", + "hp_min_child_weight": "hp-min-child-weight", + "hp_subsample": "hp-subsample", + "hp_colsample_bytree": "hp-colsample-bytree", + "hp_reg_lambda": "hp-reg-lambda", + "hp_reg_alpha": "hp-reg-alpha", + "hp_gamma": "hp-gamma" + }, + "artifacts": [], + "config": { + "agent_node_id": "agent-score", + "mcp_node_id": "tool-model-score", + "server_id": "b16f01eb-90af-4558-af3e-77c6fdcb7b22", + "tool_name": "run_model_script", + "projections": { + "pr_auc": { + "path": "test_metrics.average_precision" + }, + "roc_auc": { + "path": "test_metrics.roc_auc" + }, + "brier_score": { + "path": "test_metrics.brier_score" + }, + "chosen_threshold": { + "path": "test_metrics.chosen_threshold" + }, + "test_prevalence": { + "path": "test_metrics.test_prevalence" + }, + "n_test_samples": { + "path": "test_metrics.n_test_samples" + }, + "n_pos_test": { + "path": "test_metrics.n_pos_test" + }, + "accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/accuracy" + }, + "balanced_accuracy_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/balanced_accuracy" + }, + "precision_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/precision" + }, + "recall_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/recall" + }, + "f1_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/f1" + }, + "accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.accuracy" + }, + "balanced_accuracy_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.balanced_accuracy" + }, + "precision_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.precision" + }, + "recall_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.recall" + }, + "f1_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.f1" + }, + "accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.accuracy" + }, + "balanced_accuracy_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.balanced_accuracy" + }, + "precision_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.precision" + }, + "recall_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.recall" + }, + "f1_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.f1" + }, + "n_pred_pos_at_0_5": { + "path": "/test_metrics/metrics_at_0.5/n_pred_pos" + }, + "n_pred_pos_at_chosen_threshold": { + "path": "test_metrics.metrics_at_chosen_threshold.n_pred_pos" + }, + "n_pred_pos_at_prevalence": { + "path": "test_metrics.metrics_at_prevalence.n_pred_pos" + }, + "optuna_n_trials": { + "path": "model_decisions.n_trials" + }, + "optuna_best_inner_cv_score": { + "path": "model_decisions.optuna_best_value" + }, + "calibration_brier_val": { + "path": "model_decisions.calibration_brier_val" + }, + "code_sha256": { + "path": "code_sha256" + }, + "payload_sha256": { + "path": "payload_sha256" + }, + "data_sha256": { + "path": "data_sha256" + }, + "hp_n_estimators": { + "path": "model_decisions.best_hyperparameters.n_estimators" + }, + "hp_max_depth": { + "path": "model_decisions.best_hyperparameters.max_depth" + }, + "hp_learning_rate": { + "path": "model_decisions.best_hyperparameters.learning_rate" + }, + "hp_min_child_weight": { + "path": "model_decisions.best_hyperparameters.min_child_weight" + }, + "hp_subsample": { + "path": "model_decisions.best_hyperparameters.subsample" + }, + "hp_colsample_bytree": { + "path": "model_decisions.best_hyperparameters.colsample_bytree" + }, + "hp_reg_lambda": { + "path": "model_decisions.best_hyperparameters.reg_lambda" + }, + "hp_reg_alpha": { + "path": "model_decisions.best_hyperparameters.reg_alpha" + }, + "hp_gamma": { + "path": "model_decisions.best_hyperparameters.gamma" + } + } + } + } + ], + "inputs": [ + { + "producer_binding_id": "runtime", + "input_key": "facts", + "source_key": "attempt.runtime" + } + ] } } diff --git a/src/asaree/services/csv_export.py b/src/asaree/services/csv_export.py index 07c1dd39..d60f659a 100644 --- a/src/asaree/services/csv_export.py +++ b/src/asaree/services/csv_export.py @@ -423,9 +423,7 @@ def result_rows_schema( "value_type": ( "string" if key in script_columns - else "opaque" - if key in reported_keys - else (metric_types or {}).get(key, "number") + else (metric_types or {}).get(key, "opaque" if key in reported_keys else "number") ), **( {} diff --git a/src/asaree/services/experiment_run_results.py b/src/asaree/services/experiment_run_results.py index 83948790..42641f58 100644 --- a/src/asaree/services/experiment_run_results.py +++ b/src/asaree/services/experiment_run_results.py @@ -139,9 +139,9 @@ def _primary_metric(design_spec: dict[str, Any] | None) -> tuple[str | None, str for metric in metrics: if ( isinstance(metric, dict) - and metric.get("kind") == "runtime" and metric.get("primary") and isinstance(metric.get("name"), str) + and metric.get("valueType", "number") in {"number", "boolean"} ): direction = metric.get("direction") # Catalog runtime metrics are stored under their telemetry key @@ -177,9 +177,7 @@ def _declared_metric_types(design_spec: dict[str, Any] | None) -> dict[str, str] """Map Results metric keys to their declared numeric outcome type.""" types: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): - if metric["kind"] != "runtime": - continue - key = metric.get("catalogKey") + key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: types[key] = metric["valueType"] return types @@ -189,9 +187,7 @@ def _declared_metric_aggregations(design_spec: dict[str, Any] | None) -> dict[st """Map Results metric keys to their declared per-cell aggregation.""" aggregations: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): - if metric["kind"] != "runtime": - continue - key = metric.get("catalogKey") + key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: aggregations[key] = metric["aggregation"] return aggregations @@ -201,9 +197,7 @@ def _declared_metric_directions(design_spec: dict[str, Any] | None) -> dict[str, """Map scalar Results keys to whether higher, lower, or neither is preferred.""" directions: dict[str, str] = {} for metric in normalize_metrics((design_spec or {}).get("metrics")): - if metric["kind"] != "runtime": - continue - key = metric.get("catalogKey") + key = metric.get("catalogKey") if metric["kind"] == "runtime" else metric.get("name") if isinstance(key, str) and metric["valueType"] in {"number", "boolean"}: directions[key] = metric["direction"] return directions diff --git a/src/asaree/services/measurement_engine.py b/src/asaree/services/measurement_engine.py index 71afe16e..9a36618f 100644 --- a/src/asaree/services/measurement_engine.py +++ b/src/asaree/services/measurement_engine.py @@ -29,9 +29,9 @@ "cancelled", "not_applicable", ] -# Runtime metrics remain scalar. Reported custom metrics deliberately retain -# the tool's opaque result, so observations must be able to carry any -# JSON-compatible value without coercing or validating it. +# Reported metrics can retain a producer's opaque result or project typed +# scalar fields from structured output, so observations must carry any +# JSON-compatible value before declaration-driven validation. ScalarValue = Any diff --git a/src/asaree/services/measurement_migration.py b/src/asaree/services/measurement_migration.py index d8554b34..053dee18 100644 --- a/src/asaree/services/measurement_migration.py +++ b/src/asaree/services/measurement_migration.py @@ -59,11 +59,18 @@ def normalize_experiment_measurement_plan(document: Any, metrics: Any) -> dict[s reported_bindings = tuple(binding for binding in plan.producers if binding.producer_id in reported_ids) reported_binding_ids = {binding.id for binding in reported_bindings} reported_metric_ids = {metric_id for binding in reported_bindings for metric_id in binding.outputs.values()} + projected_metric_ids = { + metric_id + for binding in reported_bindings + for output_key, metric_id in binding.outputs.items() + if isinstance(binding.config.get("projections"), Mapping) + and output_key in binding.config["projections"] + } normalized = replace( plan, metrics=tuple( replace(metric, value_type="opaque", direction="neutral", aggregation="none", primary=False) - if metric.id in reported_metric_ids + if metric.id in reported_metric_ids and metric.id not in projected_metric_ids else metric for metric in plan.metrics ), @@ -86,7 +93,7 @@ def _legacy_metric_id(key: str, metrics: Any) -> tuple[str, str]: def _is_declared_custom_metric(key: str, metrics: Any) -> bool: - """Whether *key* belongs to a current opaque custom-metric declaration.""" + """Whether *key* belongs to a current reported-metric declaration.""" return any( metric["kind"] == "custom" and metric["name"] == key for metric in normalize_metrics(metrics) ) diff --git a/src/asaree/services/metrics.py b/src/asaree/services/metrics.py index d17a5fd1..c2621b7b 100644 --- a/src/asaree/services/metrics.py +++ b/src/asaree/services/metrics.py @@ -300,7 +300,16 @@ def normalize_metrics(metrics: Any, *, validate_custom_names: bool = False) -> l if catalog else "mean" ) - if metric["kind"] == "custom": + # Explicit scalar custom declarations are produced by structured + # reported-output projections. Legacy/custom drafts without an + # explicit scalar type keep the original opaque display-only + # semantics. + scalar_custom = ( + metric["kind"] == "custom" + and raw.get("kind") == "custom" + and raw.get("valueType") in {"number", "boolean"} + ) + if metric["kind"] == "custom" and not scalar_custom: metric["valueType"] = "opaque" metric["direction"] = "neutral" metric["aggregation"] = "none" diff --git a/src/asaree/services/protocol_runs.py b/src/asaree/services/protocol_runs.py index 58c29de3..182be238 100644 --- a/src/asaree/services/protocol_runs.py +++ b/src/asaree/services/protocol_runs.py @@ -381,8 +381,19 @@ async def update_attempt_result( return run +def _required_metric_was_measured(evaluation: MeasurementEvaluation, metric_id: str | None) -> bool: + return metric_id is None or any( + observation.metric_id == metric_id and observation.status == "measured" + for observation in evaluation.observations + ) + + async def record_measurement_evaluation( - db: AsyncSession, protocol_run_id: uuid.UUID, evaluation: MeasurementEvaluation + db: AsyncSession, + protocol_run_id: uuid.UUID, + evaluation: MeasurementEvaluation, + *, + required_metric_id: str | None = None, ) -> ProtocolRun | None: """Freeze one engine result on its attempt and update only its current projection. @@ -409,6 +420,7 @@ async def record_measurement_evaluation( for observation in evaluation.observations if observation.status == "measured" and observation.producer.kind != "runtime" } + required_metric_measured = _required_metric_was_measured(evaluation, required_metric_id) if measured_values: attempt_result["metric_values"] = { **(attempt_result.get("metric_values") or {}), @@ -449,7 +461,7 @@ async def record_measurement_evaluation( # The numbers are not lost: `attempt_result["metric_values"]` above and # `artifacts["measurement"]` here both keep the full document, so the # run stays inspectable and a re-run at a workable cap scores normally. - if measured_values and node_run_truncation(run.node_runs) is None: + if measured_values and required_metric_measured and node_run_truncation(run.node_runs) is None: replicate.metric_values = {**(replicate.metric_values or {}), **measured_values} await db.flush() await db.refresh(run) diff --git a/src/asaree/services/reported_metrics.py b/src/asaree/services/reported_metrics.py index 84b252f6..214c8028 100644 --- a/src/asaree/services/reported_metrics.py +++ b/src/asaree/services/reported_metrics.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json import uuid from collections.abc import Mapping, Sequence from copy import deepcopy @@ -25,6 +26,7 @@ ProducerProvenance, ValidationIssue, ValidationReport, + is_scalar_value, preserved_binding_severity, ) from asaree.services.protocol_graph import directly_connected_tool_pair, node_map @@ -38,6 +40,52 @@ _SCRIPT_TOOL = "run_wired_script" +def _structured_value(value: Any) -> Any: + """Decode a JSON transport string while preserving already-typed values.""" + if not isinstance(value, str): + return value + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + +def _project_reported_value(value: Any, projection: Any) -> tuple[Any, str | None]: + """Select one typed value from a reported JSON document. + + Projection paths use dot-separated object keys (and integer list indexes). + ``length`` is intentionally the only transform: it covers collection counts + without turning measurement plans into an expression language. + """ + if not isinstance(projection, Mapping): + return None, "The reported metric projection is invalid." + path = projection.get("path") + if not isinstance(path, str): + return None, "The reported metric projection has no path." + current = _structured_value(value) + segments = ( + [part.replace("~1", "/").replace("~0", "~") for part in path.removeprefix("/").split("/")] + if path.startswith("/") + else [part for part in path.split(".") if part] + ) + for segment in segments: + if isinstance(current, Mapping) and segment in current: + current = current[segment] + elif isinstance(current, Sequence) and not isinstance(current, str): + try: + current = current[int(segment)] + except (ValueError, IndexError): + return None, f"Reported metric path {path!r} is unavailable." + else: + return None, f"Reported metric path {path!r} is unavailable." + transform = projection.get("transform") + if transform is None: + return current, None + if transform == "length" and isinstance(current, Mapping | Sequence) and not isinstance(current, str): + return len(current), None + return None, f"Reported metric transform {transform!r} cannot be applied." + + def _tool_matches(recorded: Any, configured: str) -> bool: return isinstance(recorded, str) and (recorded == configured or recorded.endswith(f".{configured}")) @@ -142,11 +190,16 @@ async def collect_reported_metrics( if isinstance(node_run, Mapping) else None ) + structured_agent_output = ( + node_run.get("last_successful_payload", node_run.get("payload")) + if isinstance(node_run, Mapping) + else None + ) agent_output_available = ( binding.producer_id == AGENT_OUTPUT_PRODUCER_ID and isinstance(node_run, Mapping) and (node_run.get("status") == "completed" or "last_successful_output_text" in node_run) - and successful_output is not None + and (successful_output is not None or structured_agent_output is not None) ) call = ( None if binding.producer_id == AGENT_OUTPUT_PRODUCER_ID else await _last_matching_call(run, binding, graph) @@ -172,25 +225,43 @@ async def collect_reported_metrics( ), }, ) - for metric_id in binding.outputs.values(): + projections = binding.config.get("projections") + projections = projections if isinstance(projections, Mapping) else {} + for output_key, metric_id in binding.outputs.items(): metric = metrics.get(metric_id) if metric is None: continue + raw_value = ( + structured_agent_output + if agent_output_available and output_key in projections and structured_agent_output is not None + else successful_output + if agent_output_available + else call.get("result") + if call is not None + else None + ) + value, projection_error = ( + _project_reported_value(raw_value, projections[output_key]) + if output_key in projections + else (raw_value, None) + ) + type_error = ( + None + if projection_error is not None or not measured or is_scalar_value(value, metric.value_type) + else f"Projected value does not match declared type {metric.value_type!r}." + ) + observation_error = projection_error or type_error observations.append( MetricObservation( metric_id=metric.id, metric_name=metric.name, value_type=metric.value_type, - value=( - successful_output - if agent_output_available - else call.get("result") - if call is not None - else None - ), - status="measured" if measured else "unavailable", + value=value if observation_error is None else None, + status="measured" if measured and observation_error is None else "unavailable", error=( - None + observation_error + if observation_error is not None + else None if measured else "The Agent did not produce a completed final output." if binding.producer_id == AGENT_OUTPUT_PRODUCER_ID @@ -285,10 +356,44 @@ async def validate_reported_measurement_plan( ) -> ValidationReport: """Validate reported-source identity and wiring without interpreting values.""" issues: list[ValidationIssue] = [] + metrics = {metric.id: metric for metric in plan.metrics} for index, binding in enumerate(plan.producers): if binding.producer_id not in REPORTED_PRODUCER_IDS: continue path = f"producers[{index}]" + raw_projections = binding.config.get("projections") + projections = raw_projections if isinstance(raw_projections, Mapping) else {} + if raw_projections is not None and not isinstance(raw_projections, Mapping): + issues.append( + ValidationIssue( + "reported_projections_invalid", + "Reported metric projections must be an object keyed by producer output.", + f"{path}.config.projections", + ) + ) + for output_key, metric_id in binding.outputs.items(): + metric = metrics.get(metric_id) + projection = projections.get(output_key) + if metric is not None and metric.value_type != "opaque" and projection is None: + issues.append( + ValidationIssue( + "reported_projection_missing", + f"Scalar reported metric {metric.name!r} requires a structured field projection.", + f"{path}.outputs.{output_key}", + ) + ) + if projection is not None and ( + not isinstance(projection, Mapping) + or not isinstance(projection.get("path"), str) + or projection.get("transform") not in {None, "length"} + ): + issues.append( + ValidationIssue( + "reported_projection_invalid", + "A reported metric projection needs a path and an optional supported transform.", + f"{path}.config.projections.{output_key}", + ) + ) if agent_issue := _agent_issue(binding, index, graph, preserved_binding_ids): issues.append(agent_issue) if binding.producer_id == AGENT_OUTPUT_PRODUCER_ID: diff --git a/src/asaree/services/runtime_metrics.py b/src/asaree/services/runtime_metrics.py index 6ba00a28..2590053c 100644 --- a/src/asaree/services/runtime_metrics.py +++ b/src/asaree/services/runtime_metrics.py @@ -670,7 +670,13 @@ async def finalize_attempt_measurement( attempt_result = dict(run.attempt_result or {}) attempt_result["evaluation_summary"] = {"cost_usd": 0.0} run.attempt_result = attempt_result - await record_measurement_evaluation(db, run.id, evaluation) + primary_metric_id = next((metric.id for metric in declared.metrics if metric.primary), None) + await record_measurement_evaluation( + db, + run.id, + evaluation, + required_metric_id=primary_metric_id, + ) completed_result = dict(run.attempt_result or {}) completed_result["evaluation_state"] = "completed" run.attempt_result = completed_result diff --git a/tests/test_bdm_use_cases.py b/tests/test_bdm_use_cases.py new file mode 100644 index 00000000..b93e8b10 --- /dev/null +++ b/tests/test_bdm_use_cases.py @@ -0,0 +1,101 @@ +import csv +import io +import json +from pathlib import Path + +from asaree.services.csv_export import result_rows_to_csv +from asaree.services.measurement_migration import normalize_experiment_measurement_plan +from asaree.services.metrics import ( + design_metrics_from_measurement_plan, + normalize_metrics, +) + +BDM = Path(__file__).parents[1] / "publications" / "BDM" +USE_CASES = sorted(BDM.glob("myocardial-*-v0.8.0.json")) + + +def test_myocardial_use_cases_declare_the_same_analysis_ready_measurements() -> None: + documents = [json.loads(path.read_text()) for path in USE_CASES] + expected_metrics = documents[0]["design_spec"]["metrics"] + expected_plan_metrics = documents[0]["measurement_plan"]["metrics"] + + assert len(USE_CASES) == 3 + for document in documents: + metrics = document["design_spec"]["metrics"] + plan = document["measurement_plan"] + assert metrics == expected_metrics + assert plan["metrics"] == expected_plan_metrics + assert normalize_metrics(metrics) == design_metrics_from_measurement_plan(plan) + assert normalize_experiment_measurement_plan(plan, metrics) == plan + + primary = [metric for metric in metrics if metric["primary"]] + assert [(metric["name"], metric["direction"]) for metric in primary] == [("pr_auc", "maximize")] + runtime = next(producer for producer in plan["producers"] if producer["producer_id"] == "asaree.runtime") + assert set(runtime["outputs"]) == { + "cost_usd", + "duration_seconds", + "input_tokens", + "output_tokens", + "total_tokens", + "tool_calls", + "tool_error_rate", + "agent_loop_iterations", + "critic_rejections", + "critic_approvals", + } + score = next(producer for producer in plan["producers"] if producer["id"] == "model-evaluation-source") + assert score["config"]["projections"]["pr_auc"] == {"path": "test_metrics.average_precision"} + assert score["config"]["projections"]["accuracy_at_0_5"] == { + "path": "/test_metrics/metrics_at_0.5/accuracy" + } + + +def test_myocardial_output_parser_types_match_the_prompted_payloads() -> None: + document = json.loads(USE_CASES[0].read_text()) + parsers = { + node["id"]: {field["name"]: field["type"] for field in node["data"]["config"]["output_contract"]["fields"]} + for node in document["graph"]["nodes"] + if node["type"] == "output_parser" + } + + assert parsers["output-parser-dc"]["outliers"] == "array" + assert parsers["output-parser-dc"]["imputation"] == "array" + assert parsers["output-parser-fte"]["engineering_recipe"] == "array" + assert parsers["output-parser-fs"]["class_balance_check"] == "string" + assert parsers["output-parser-fs"]["n_engineered_features_selected"] == "integer" + assert parsers["output-parser-mlm"]["search_space"] == "array" + assert "model_decisions" in parsers["output-parser-score"] + assert "result" not in parsers["output-parser-score"] + + +def test_myocardial_results_export_has_analysis_ready_custom_columns() -> None: + document = json.loads(USE_CASES[0].read_text()) + csv_text = result_rows_to_csv( + [ + { + "cell_label": "cell-1", + "replicate_number": 1, + "factor_values": {"Critic enabled": True}, + "metric_values": {}, + } + ], + design_spec=document["design_spec"], + ) + + header = next(csv.reader(io.StringIO(csv_text))) + assert { + "pr_auc", + "roc_auc", + "brier_score", + "n_pred_pos_at_0_5", + "n_features_after_dc", + "n_engineered_features", + "n_features_after_fs", + "n_engineered_features_selected", + "pct_engineered_features_selected", + "optuna_best_inner_cv_score", + "hp_n_estimators", + "code_sha256", + "payload_sha256", + "data_sha256", + } <= set(header) diff --git a/tests/test_experiment_measurements.py b/tests/test_experiment_measurements.py index 372d9dc6..67657bb6 100644 --- a/tests/test_experiment_measurements.py +++ b/tests/test_experiment_measurements.py @@ -607,10 +607,10 @@ async def test_python_script_metric_requires_a_ready_direct_agent_script_pair() "id": "quality", "name": "Quality", "kind": "custom", - "valueType": "number", - "direction": "maximize", - "aggregation": "mean", - "primary": True, + "valueType": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": False, } ], graph=graph, @@ -638,10 +638,10 @@ async def test_python_script_metric_accepts_a_ready_direct_agent_script_pair() - "id": "quality", "name": "Quality", "kind": "custom", - "valueType": "number", - "direction": "maximize", - "aggregation": "mean", - "primary": True, + "valueType": "opaque", + "direction": "neutral", + "aggregation": "none", + "primary": False, } ], graph=graph, @@ -671,7 +671,10 @@ async def test_agent_output_metric_accepts_an_active_agent_without_an_input_bind "producer_id": "asaree.agent_output", "kind": "reported", "outputs": {"value": "quality"}, - "config": {"agent_node_id": "agent-1"}, + "config": { + "agent_node_id": "agent-1", + "projections": {"value": {"path": "quality"}}, + }, } ], "inputs": [], diff --git a/tests/test_experiment_run_results.py b/tests/test_experiment_run_results.py index 7133e14f..2010d73b 100644 --- a/tests/test_experiment_run_results.py +++ b/tests/test_experiment_run_results.py @@ -78,7 +78,7 @@ def test_declared_runtime_metrics_are_projected_from_execution_telemetry() -> No assert _primary_metric(spec) == ("cost_usd", "maximize") -def test_only_builtin_metrics_declare_aggregations() -> None: +def test_builtin_and_reported_scalar_metrics_declare_aggregations() -> None: assert _declared_metric_aggregations( { "metrics": [ @@ -92,10 +92,10 @@ def test_only_builtin_metrics_declare_aggregations() -> None: }, ] } - ) == {"cost_usd": "sum"} + ) == {"Quality": "mean", "cost_usd": "sum"} -def test_only_builtin_metrics_declare_ranking_directions() -> None: +def test_builtin_and_reported_scalar_metrics_declare_ranking_directions() -> None: assert _declared_metric_directions( { "metrics": [ @@ -109,7 +109,7 @@ def test_only_builtin_metrics_declare_ranking_directions() -> None: }, ] } - ) == {"cost_usd": "minimize"} + ) == {"Quality": "maximize", "cost_usd": "minimize"} def test_cell_metric_aggregations_apply_the_declared_operation() -> None: @@ -169,6 +169,40 @@ def test_results_csv_keeps_an_empty_column_for_an_unreported_custom_metric() -> assert reported == {"name": "Reviewer report", "role": "reported", "value_type": "opaque"} +def test_results_csv_describes_a_reported_scalar_as_numeric() -> None: + design_spec = { + "metrics": [ + { + "id": "pr-auc", + "name": "pr_auc", + "kind": "custom", + "valueType": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": True, + } + ] + } + rows = [ + { + "cell_label": "cell", + "replicate_number": 1, + "factor_values": {}, + "metric_values": {"pr_auc": 0.72}, + } + ] + + schema = result_rows_schema(rows, metric_types={"pr_auc": "number"}, design_spec=design_spec) + reported = next(column for column in schema["columns"] if column["name"] == "pr_auc") + + assert reported == { + "name": "pr_auc", + "role": "reported", + "value_type": "number", + } + assert _primary_metric(design_spec) == ("pr_auc", "maximize") + + def test_results_csv_projects_script_stdout_and_execution_metadata() -> None: envelope = { "code_sha256": "abc123", diff --git a/tests/test_measurement_migration.py b/tests/test_measurement_migration.py index b17a7af2..3bc6f7e0 100644 --- a/tests/test_measurement_migration.py +++ b/tests/test_measurement_migration.py @@ -16,6 +16,46 @@ def test_legacy_declarations_are_not_migrated_into_measurement_plans() -> None: assert normalize_experiment_measurement_plan(None, metrics) == {"metrics": [], "producers": [], "inputs": []} +def test_projected_reported_metrics_keep_their_scalar_contract() -> None: + plan = normalize_experiment_measurement_plan( + { + "metrics": [ + { + "id": "pr-auc", + "name": "pr_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": True, + } + ], + "producers": [ + { + "id": "score", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": {"pr_auc": "pr-auc"}, + "config": { + "agent_node_id": "agent", + "projections": {"pr_auc": {"path": "test_metrics.average_precision"}}, + }, + } + ], + "inputs": [], + }, + (), + ) + + assert plan["metrics"][0] == { + "id": "pr-auc", + "name": "pr_auc", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + "primary": True, + } + + def test_historical_values_and_artifacts_gain_legacy_unknown_provenance() -> None: metrics = [{"name": "Clinical quality", "kind": "custom", "direction": "maximize"}] diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 1c7beef8..befbefb1 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -169,7 +169,7 @@ def test_design_metrics_are_derived_from_a_caller_supplied_plan() -> None: ] -def test_custom_metric_values_are_retained_without_type_validation() -> None: +def test_explicit_scalar_custom_metric_values_are_type_checked() -> None: metrics = [ {"name": "Passed", "kind": "custom", "valueType": "boolean", "primary": True}, {"name": "Score", "kind": "custom", "valueType": "number", "primary": False}, @@ -178,16 +178,23 @@ def test_custom_metric_values_are_retained_without_type_validation() -> None: "Passed": True, "Score": 0.8, } - assert validate_metric_values(metrics, {"Passed": 1, "Score": {"grade": "A"}}) == { - "Passed": 1, - "Score": {"grade": "A"}, - } + with pytest.raises(ValueError, match="must be Boolean"): + validate_metric_values(metrics, {"Passed": 1}) + with pytest.raises(ValueError, match="finite number"): + validate_metric_values(metrics, {"Score": {"grade": "A"}}) -def test_custom_boolean_declarations_normalize_to_opaque_display_only_metrics() -> None: +def test_explicit_custom_boolean_declarations_remain_scalar() -> None: metrics = normalize_metrics( [{"name": "Passed", "kind": "custom", "valueType": "boolean", "aggregation": "sum", "primary": True}] ) + assert metrics[0]["valueType"] == "boolean" + assert metrics[0]["aggregation"] == "mean" + assert metrics[0]["primary"] is True + + +def test_custom_draft_without_an_explicit_scalar_type_remains_opaque() -> None: + metrics = normalize_metrics([{"name": "Reviewer report", "kind": "custom", "primary": True}]) assert metrics[0]["valueType"] == "opaque" assert metrics[0]["aggregation"] == "none" assert metrics[0]["primary"] is False diff --git a/tests/test_protocol_runs.py b/tests/test_protocol_runs.py index a2e35ecf..b8ce8f52 100644 --- a/tests/test_protocol_runs.py +++ b/tests/test_protocol_runs.py @@ -26,6 +26,7 @@ ) from asaree.services.protocol_revisions import publish_protocol from asaree.services.protocol_runs import ( + _required_metric_was_measured, create_protocol_run, create_test_run, fail_protocol_run, @@ -78,6 +79,51 @@ async def protocol_id(owner_id: uuid.UUID) -> AsyncIterator[uuid.UUID]: await delete_protocol(db, pid) +def test_primary_measurement_is_required_before_projecting_a_scored_replicate() -> None: + evaluation = MeasurementEvaluation( + replicate_id="replicate", + attempt_id="attempt", + observations=( + MetricObservation( + metric_id="pipeline-count", + metric_name="n_features_after_fs", + value_type="number", + status="measured", + value=20, + error=None, + attempt_id="attempt", + producer=ProducerProvenance( + binding_id="fs", + producer_id="asaree.agent_output", + kind="reported", + version="1", + ), + input_provenance={}, + ), + MetricObservation( + metric_id="pr-auc", + metric_name="pr_auc", + value_type="number", + status="unavailable", + value=None, + error="The scoring tool did not return average precision.", + attempt_id="attempt", + producer=ProducerProvenance( + binding_id="score", + producer_id="asaree.mcp_tool", + kind="reported", + version="1", + ), + input_provenance={}, + ), + ), + artifacts=(), + ) + + assert _required_metric_was_measured(evaluation, None) + assert not _required_metric_was_measured(evaluation, "pr-auc") + + async def test_create_get_and_node_run_progress(owner_id: uuid.UUID, protocol_id: uuid.UUID) -> None: async with get_session() as db: run = await create_protocol_run(db, protocol_id=protocol_id, owner_id=owner_id) diff --git a/tests/test_reported_metrics.py b/tests/test_reported_metrics.py index 0ca82674..d8958dac 100644 --- a/tests/test_reported_metrics.py +++ b/tests/test_reported_metrics.py @@ -198,6 +198,115 @@ async def test_agent_output_report_captures_completed_final_output(output: str) assert result.observations[0].producer.evaluation["source"] == "agent_final_output" +@pytest.mark.asyncio +async def test_agent_output_projection_reads_the_structured_parser_payload() -> None: + plan = parse_measurement_plan( + { + "metrics": [ + { + "id": "feature-count", + "name": "n_engineered_features", + "value_type": "number", + "direction": "neutral", + "aggregation": "mean", + } + ], + "producers": [ + { + "id": "fte-output", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": {"feature_count": "feature-count"}, + "config": { + "agent_node_id": "agent", + "projections": { + "feature_count": {"path": "engineering_recipe", "transform": "length"} + }, + }, + } + ], + "inputs": [], + } + ) + run = SimpleNamespace( + id=uuid4(), + replicate_result_id=None, + node_runs={ + "agent": { + "status": "completed", + "output_text": "summary", + "payload": {"engineering_recipe": [{"name": "a"}, {"name": "b"}]}, + } + }, + ) + + result = await collect_reported_metrics(run, plan, {"nodes": [], "edges": []}) + + assert result.observations[0].status == "measured" + assert result.observations[0].value == 2 + + +@pytest.mark.asyncio +async def test_mcp_projection_supports_json_pointer_keys_containing_dots( + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = parse_measurement_plan( + { + "metrics": [ + { + "id": "accuracy", + "name": "accuracy_at_0_5", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + } + ], + "producers": [ + { + "id": "score", + "producer_id": "asaree.mcp_tool", + "kind": "reported", + "outputs": {"accuracy": "accuracy"}, + "config": { + "agent_node_id": "agent", + "mcp_node_id": "mcp", + "server_id": str(uuid4()), + "tool_name": "score", + "projections": { + "accuracy": {"path": "/test_metrics/metrics_at_0.5/accuracy"} + }, + }, + } + ], + "inputs": [], + } + ) + monkeypatch.setattr( + "asaree.services.reported_metrics.get_run_steps", + lambda _run_id: _async_value( + [ + SimpleNamespace( + tool_call={ + "server": "scorer", + "tool": "score", + "result": '{"test_metrics":{"metrics_at_0.5":{"accuracy":0.81}}}', + } + ) + ] + ), + ) + monkeypatch.setattr( + "asaree.services.reported_metrics.mcp_service.get_server", + lambda _server_id: _async_value(SimpleNamespace(name="scorer")), + ) + run = SimpleNamespace(id=uuid4(), replicate_result_id=None, node_runs={"agent": {"run_id": str(uuid4())}}) + + result = await collect_reported_metrics(run, plan, {"nodes": [], "edges": []}) + + assert result.observations[0].status == "measured" + assert result.observations[0].value == 0.81 + + @pytest.mark.asyncio async def test_agent_output_report_is_unavailable_when_agent_did_not_complete() -> None: run = SimpleNamespace( @@ -264,5 +373,39 @@ async def test_agent_output_plan_requires_an_active_agent() -> None: assert sub_agent.valid +@pytest.mark.asyncio +async def test_scalar_reported_metric_requires_a_projection() -> None: + plan = parse_measurement_plan( + { + "metrics": [ + { + "id": "quality", + "name": "Quality", + "value_type": "number", + "direction": "maximize", + "aggregation": "mean", + } + ], + "producers": [ + { + "id": "quality-source", + "producer_id": "asaree.agent_output", + "kind": "reported", + "outputs": {"value": "quality"}, + "config": {"agent_node_id": "agent"}, + } + ], + "inputs": [], + } + ) + + report = await validate_reported_measurement_plan( + plan, + {"nodes": [{"id": "agent", "type": "agent", "data": {}}]}, + ) + + assert [issue.code for issue in report.issues] == ["reported_projection_missing"] + + async def _async_value(value): return value From 86d546029af0e6aacd1923f009ad9d2b5922251d Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 29 Sep 2026 15:12:50 -0700 Subject: [PATCH 17/18] fix: close review gaps in SDK parity, factor baselines, and MCP OAuth - Replace a factor's canvas baseline level instead of prepending, so pauses while typing in a bound field no longer accumulate levels. - Accept either ordering of legacy two-boolean factors in the backend validator, matching the frontend. - Forward payload-only outputs through deactivated nodes and payload-only supervisor briefs. - SDK: expose MCP authentication status, server_env, omit-vs-clear credential updates, begin_oauth, and clear_credentials. - MCP OAuth popup: broadcast callback results on a same-origin channel and keep waiting (with cancel) when a provider's COOP detaches the popup. - Add callback error, credential clear/revoke, and owner-scope tests. --- .../protocol/ConnectMcpServerDialog.tsx | 30 +++++- .../protocol/ManageMcpCredentialsDialog.tsx | 37 +++++-- frontend/src/lib/factorBindings.test.ts | 34 ++++++- frontend/src/lib/factorBindings.ts | 12 ++- frontend/src/lib/mcpOAuth.test.ts | 90 ++++++++++++++++- frontend/src/lib/mcpOAuth.ts | 92 ++++++++++++++--- sdk/README.md | 16 +++ sdk/src/asaree_client/__init__.py | 4 + sdk/src/asaree_client/_models/mcp.py | 23 ++++- sdk/src/asaree_client/models.py | 4 +- sdk/src/asaree_client/resources/tools.py | 58 ++++++++--- sdk/tests/test_sdk_gaps.py | 56 ++++++++++- src/asaree/services/agent_messenger.py | 2 +- src/asaree/services/factor_bindings.py | 6 +- src/asaree/services/protocol_execution.py | 11 ++- tests/test_factor_bindings.py | 16 +++ tests/test_mcp_server_auth.py | 98 ++++++++++++++++++- tests/test_protocol_execution.py | 9 ++ tests/test_supervisor_architecture.py | 7 ++ 19 files changed, 545 insertions(+), 60 deletions(-) diff --git a/frontend/src/components/protocol/ConnectMcpServerDialog.tsx b/frontend/src/components/protocol/ConnectMcpServerDialog.tsx index b6059c92..d6317007 100644 --- a/frontend/src/components/protocol/ConnectMcpServerDialog.tsx +++ b/frontend/src/components/protocol/ConnectMcpServerDialog.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { useMutation, useQueryClient } from '@tanstack/react-query' import { KeyRound, Plug, ShieldCheck, Terminal } from 'lucide-react' import { ApiError, mcpServersApi } from '@/api/client' @@ -78,6 +78,9 @@ export function ConnectMcpServerDialog({ const [oauthServer, setOauthServer] = useState(null) const [oauthPending, setOauthPending] = useState(false) const [oauthError, setOauthError] = useState(null) + const [oauthDetached, setOauthDetached] = useState(false) + const oauthAbortRef = useRef(null) + useEffect(() => () => oauthAbortRef.current?.abort(), []) const queryClient = useQueryClient() const isStdio = transport === 'stdio' @@ -112,6 +115,7 @@ export function ConnectMcpServerDialog({ }) function reset() { + oauthAbortRef.current?.abort() setName('') setTransport('stdio') setCommand('') @@ -130,16 +134,26 @@ export function ConnectMcpServerDialog({ if (!oauthServer || oauthPending) return setOauthPending(true) setOauthError(null) + setOauthDetached(false) + const controller = new AbortController() + oauthAbortRef.current = controller try { - await authorizeMcpServer(oauthServer.id, oauthScope.trim() || undefined) + await authorizeMcpServer(oauthServer.id, oauthScope.trim() || undefined, { + signal: controller.signal, + onDetached: () => setOauthDetached(true), + }) const connected = await mcpServersApi.get(oauthServer.id) queryClient.invalidateQueries({ queryKey: ['mcp-servers'] }) onConnected?.(connected) reset() onOpenChange(false) } catch (error) { - setOauthError(error instanceof Error ? error.message : 'MCP authorization failed.') + if (!controller.signal.aborted) { + setOauthError(error instanceof Error ? error.message : 'MCP authorization failed.') + } } finally { + if (oauthAbortRef.current === controller) oauthAbortRef.current = null + setOauthDetached(false) setOauthPending(false) } } @@ -327,6 +341,16 @@ export function ConnectMcpServerDialog({ {oauthPending ? 'Waiting for authorization…' : 'Open authorization'} + {oauthPending && oauthDetached && ( +
+

+ If you closed the authorization window, cancel and try again. +

+ +
+ )}
)} diff --git a/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx b/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx index e300acdc..546dc522 100644 --- a/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx +++ b/frontend/src/components/protocol/ManageMcpCredentialsDialog.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { useQueryClient } from '@tanstack/react-query' import { ApiError, mcpServersApi } from '@/api/client' import { Button } from '@/components/ui/button' @@ -33,6 +33,9 @@ export function ManageMcpCredentialsDialog({ const [scope, setScope] = useState('') const [pending, setPending] = useState(false) const [error, setError] = useState(null) + const [oauthDetached, setOauthDetached] = useState(false) + const oauthAbortRef = useRef(null) + useEffect(() => () => oauthAbortRef.current?.abort(), []) const queryClient = useQueryClient() const isHttp = server.transport === 'http' @@ -55,7 +58,13 @@ export function ManageMcpCredentialsDialog({ setError(null) try { if (mode === 'oauth') { - await authorizeMcpServer(server.id, scope.trim() || undefined) + const controller = new AbortController() + oauthAbortRef.current = controller + setOauthDetached(false) + await authorizeMcpServer(server.id, scope.trim() || undefined, { + signal: controller.signal, + onDetached: () => setOauthDetached(true), + }) await finish(await mcpServersApi.get(server.id)) } else if (mode === 'env') { await finish(await mcpServersApi.update(server.id, { server_env: credentialsRecord(entries) })) @@ -66,10 +75,13 @@ export function ManageMcpCredentialsDialog({ await finish(await mcpServersApi.update(server.id, { headers })) } } catch (caught) { + if (oauthAbortRef.current?.signal.aborted) return setError(caught instanceof ApiError && typeof caught.detail === 'string' ? caught.detail : caught instanceof Error ? caught.message : 'Could not update credentials.') } finally { + oauthAbortRef.current = null + setOauthDetached(false) setPending(false) } } @@ -81,7 +93,9 @@ export function ManageMcpCredentialsDialog({ try { await finish(await mcpServersApi.clearCredentials(server.id, server.authentication.auth_mode === 'oauth')) } catch (caught) { - setError(caught instanceof Error ? caught.message : 'Could not clear credentials.') + setError(caught instanceof ApiError && typeof caught.detail === 'string' + ? caught.detail + : caught instanceof Error ? caught.message : 'Could not clear credentials.') } finally { setPending(false) } @@ -153,12 +167,23 @@ export function ManageMcpCredentialsDialog({

Saving opens the provider authorization window.

)} + {pending && oauthDetached && ( +

+ Waiting for the provider to finish. If you closed the authorization window, cancel and try again. +

+ )} {error &&

{error}

}
- + {pending && mode === 'oauth' ? ( + + ) : ( + + )} diff --git a/frontend/src/lib/factorBindings.test.ts b/frontend/src/lib/factorBindings.test.ts index 1c5bc31e..1fb96d3b 100644 --- a/frontend/src/lib/factorBindings.test.ts +++ b/frontend/src/lib/factorBindings.test.ts @@ -30,7 +30,7 @@ describe('factorBindingDiscrepancies', () => { }]) }) - it('keeps the current canvas value first while preserving alternate levels', () => { + it('replaces the baseline with an edited canvas value, keeping alternates and labels', () => { const graph = { nodes: [{ id: 'agent-1', @@ -53,8 +53,36 @@ describe('factorBindingDiscrepancies', () => { }], }, graph)).toEqual([{ name: 'Prompt', - levels: ['edited canvas prompt', 'old baseline', 'alternate'], - level_labels: ['Baseline', 'Original', 'Alternative'], + levels: ['edited canvas prompt', 'alternate'], + level_labels: ['Original', 'Alternative'], + }]) + }) + + it('promotes a canvas value that matches an alternate level', () => { + const graph = { + nodes: [{ + id: 'agent-1', + type: 'agent', + position: { x: 0, y: 0 }, + data: { + label: 'Writer', + factor_bindings: { 'config.system_prompt': 'Prompt' }, + config: { system_prompt: 'alternate' }, + }, + }], + edges: [], + } as unknown as ProtocolGraph + + expect(reconcileFactorBaselines({ + factors: [{ + name: 'Prompt', + levels: ['old baseline', 'alternate'], + level_labels: ['Original', 'Alternative'], + }], + }, graph)).toEqual([{ + name: 'Prompt', + levels: ['alternate', 'old baseline'], + level_labels: ['Alternative', 'Original'], }]) }) diff --git a/frontend/src/lib/factorBindings.ts b/frontend/src/lib/factorBindings.ts index bb0bd85b..ebd369cd 100644 --- a/frontend/src/lib/factorBindings.ts +++ b/frontend/src/lib/factorBindings.ts @@ -52,15 +52,23 @@ function withPathValue(root: Record, dottedPath: string, nextVa return next } -/** Keep a factor's current canvas value in the baseline (first) slot. */ +/** + * Keep a factor's current canvas value in the baseline (first) slot. A value + * matching an alternate level is promoted; any other value *replaces* the old + * baseline. Bound fields save on every keystroke, so keeping the old baseline + * as an extra level would turn each pause mid-edit into a new treatment. + */ export function factorWithCanvasBaseline(factor: DesignFactor, currentValue: unknown): DesignFactor { if (isBooleanFactor(factor) || valuesEqual(factor.levels[0], currentValue)) return factor const currentIndex = factor.levels.findIndex((level) => valuesEqual(level, currentValue)) + if (currentIndex < 0) { + return { ...factor, levels: [currentValue, ...factor.levels.slice(1)] } + } const keptIndexes = factor.levels.flatMap((_, index) => index === currentIndex ? [] : [index]) const levels = [currentValue, ...keptIndexes.map((index) => factor.levels[index])] const labels = factor.level_labels - ? [currentIndex >= 0 ? factor.level_labels[currentIndex] ?? 'Baseline' : 'Baseline', ...keptIndexes.map((index) => factor.level_labels?.[index] ?? '')] + ? [factor.level_labels[currentIndex] ?? 'Baseline', ...keptIndexes.map((index) => factor.level_labels?.[index] ?? '')] : undefined return { ...factor, levels, ...(labels ? { level_labels: labels } : {}) } } diff --git a/frontend/src/lib/mcpOAuth.test.ts b/frontend/src/lib/mcpOAuth.test.ts index 1894d843..604e2944 100644 --- a/frontend/src/lib/mcpOAuth.test.ts +++ b/frontend/src/lib/mcpOAuth.test.ts @@ -1,5 +1,35 @@ -import { describe, expect, it, vi } from 'vitest' -import { postMcpOAuthResult } from './mcpOAuth' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { McpServer } from '@/types/mcpServers' +import { authorizeMcpServer, postMcpOAuthResult } from './mcpOAuth' + +const api = vi.hoisted(() => ({ get: vi.fn(), beginOAuth: vi.fn() })) +vi.mock('@/api/client', () => ({ mcpServersApi: api })) + +function server(authorized: boolean): McpServer { + return { + id: 'server-1', + authentication: { + auth_mode: authorized ? 'oauth' : 'none', + configured: authorized, + authorization_required: false, + static_headers_configured: false, + stdio_env_configured: false, + stdio_env_names: [], + }, + } as unknown as McpServer +} + +function fakePopup() { + return { closed: false, close: vi.fn(), location: { assign: vi.fn() } } +} + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + vi.restoreAllMocks() + api.get.mockReset() + api.beginOAuth.mockReset() +}) describe('MCP OAuth callback', () => { it('posts only safe completion metadata to the opener origin', () => { @@ -12,6 +42,60 @@ describe('MCP OAuth callback', () => { { type: 'asaree:mcp-oauth', status: 'success', serverId: 'server-1', code: undefined }, window.location.origin, ) - vi.unstubAllGlobals() + }) + + it('also broadcasts the result for openers severed by the provider', () => { + const posted: unknown[] = [] + class FakeChannel { + postMessage(message: unknown) { posted.push(message) } + close() {} + } + vi.stubGlobal('opener', null) + vi.stubGlobal('BroadcastChannel', FakeChannel) + + postMcpOAuthResult(new URLSearchParams('status=error&code=invalid_state')) + + expect(posted).toEqual([{ type: 'asaree:mcp-oauth', status: 'error', serverId: undefined, code: 'invalid_state' }]) + }) +}) + +describe('authorizeMcpServer', () => { + it('keeps waiting after the popup detaches and resolves once the server is authorized', async () => { + vi.useFakeTimers() + const popup = fakePopup() + vi.spyOn(window, 'open').mockReturnValue(popup as unknown as Window) + api.get.mockResolvedValueOnce(server(false)).mockResolvedValueOnce(server(true)) + api.beginOAuth.mockResolvedValue({ + authorization_url: 'https://auth.example/authorize', + expires_at: new Date(Date.now() + 600_000).toISOString(), + transaction_id: 't', + }) + const onDetached = vi.fn() + + const done = authorizeMcpServer('server-1', undefined, { onDetached }) + await vi.advanceTimersByTimeAsync(0) + popup.closed = true + await vi.advanceTimersByTimeAsync(2_500) + + await expect(done).resolves.toBeUndefined() + expect(onDetached).toHaveBeenCalledOnce() + }) + + it('rejects when the caller cancels', async () => { + vi.useFakeTimers() + vi.spyOn(window, 'open').mockReturnValue(fakePopup() as unknown as Window) + api.get.mockResolvedValue(server(false)) + api.beginOAuth.mockResolvedValue({ + authorization_url: 'https://auth.example/authorize', + expires_at: new Date(Date.now() + 600_000).toISOString(), + transaction_id: 't', + }) + const controller = new AbortController() + + const done = authorizeMcpServer('server-1', undefined, { signal: controller.signal }) + await vi.advanceTimersByTimeAsync(0) + controller.abort() + + await expect(done).rejects.toThrow('cancelled') }) }) diff --git a/frontend/src/lib/mcpOAuth.ts b/frontend/src/lib/mcpOAuth.ts index 1909bb95..28c6d64c 100644 --- a/frontend/src/lib/mcpOAuth.ts +++ b/frontend/src/lib/mcpOAuth.ts @@ -1,6 +1,12 @@ import { mcpServersApi } from '@/api/client' +import type { McpServer } from '@/types/mcpServers' const MESSAGE_TYPE = 'asaree:mcp-oauth' +// Same-origin fallback for providers whose Cross-Origin-Opener-Policy severs +// `window.opener`: the callback page can then no longer postMessage to us, and +// the popup handle reports `closed` while the user is still authorizing. +const CHANNEL_NAME = 'asaree:mcp-oauth' +const STATUS_POLL_MS = 2_000 type OAuthMessage = { type: typeof MESSAGE_TYPE @@ -9,57 +15,115 @@ type OAuthMessage = { code?: string } -export async function authorizeMcpServer(serverId: string, scope?: string): Promise { +export type AuthorizeMcpServerOptions = { + signal?: AbortSignal + // The popup handle stopped reporting (closed by the user, or detached by the + // provider's COOP). Authorization keeps waiting until it completes, expires, + // or `signal` aborts, so the caller should offer a way to cancel. + onDetached?: () => void +} + +export function isOAuthAuthorized(server: McpServer): boolean { + const auth = server.authentication + return auth.auth_mode === 'oauth' && auth.configured && !auth.authorization_required +} + +export async function authorizeMcpServer( + serverId: string, + scope?: string, + { signal, onDetached }: AuthorizeMcpServerOptions = {}, +): Promise { const popup = window.open('', 'asaree-mcp-oauth', 'popup,width=640,height=760') if (!popup) throw new Error('Allow popups to authorize this MCP server.') const oauthWindow = popup let authorization + let alreadyAuthorized try { + // Status polling can only detect a *change* to authorized; a server whose + // existing tokens are still valid must wait for the callback message. + alreadyAuthorized = isOAuthAuthorized(await mcpServersApi.get(serverId)) authorization = await mcpServersApi.beginOAuth(serverId, scope) } catch (error) { oauthWindow.close() throw error } + if (signal?.aborted) { + oauthWindow.close() + throw new Error('MCP authorization was cancelled.') + } oauthWindow.location.assign(authorization.authorization_url) await new Promise((resolve, reject) => { + const channel = typeof BroadcastChannel === 'undefined' ? null : new BroadcastChannel(CHANNEL_NAME) const remaining = Math.max(1_000, new Date(authorization.expires_at).getTime() - Date.now()) const timeout = window.setTimeout(() => finish(new Error('MCP authorization timed out.')), remaining) + let poll: number | undefined + let settled = false const closed = window.setInterval(() => { - if (oauthWindow.closed) finish(new Error('MCP authorization was cancelled.')) + if (!oauthWindow.closed) return + window.clearInterval(closed) + onDetached?.() + if (!alreadyAuthorized) poll = window.setInterval(checkStatus, STATUS_POLL_MS) }, 500) function finish(error?: Error) { + if (settled) return + settled = true window.clearTimeout(timeout) window.clearInterval(closed) + window.clearInterval(poll) window.removeEventListener('message', onMessage) + signal?.removeEventListener('abort', onAbort) + channel?.close() if (!oauthWindow.closed) oauthWindow.close() if (error) reject(error) else resolve() } + function onAbort() { + finish(new Error('MCP authorization was cancelled.')) + } + + async function checkStatus() { + try { + if (isOAuthAuthorized(await mcpServersApi.get(serverId))) finish() + } catch { + // Transient; the next tick, the callback message, or the timeout decides. + } + } + + function onResult(data: OAuthMessage | undefined) { + if (data?.type !== MESSAGE_TYPE) return + if (data.status === 'success' && data.serverId === serverId) finish() + else finish(new Error(oauthErrorMessage(data.code))) + } + function onMessage(event: MessageEvent) { - if (event.origin !== window.location.origin || event.source !== oauthWindow || event.data?.type !== MESSAGE_TYPE) return - if (event.data.status === 'success' && event.data.serverId === serverId) finish() - else finish(new Error(oauthErrorMessage(event.data.code))) + if (event.origin !== window.location.origin || event.source !== oauthWindow) return + onResult(event.data) } window.addEventListener('message', onMessage) + if (channel) channel.onmessage = (event: MessageEvent) => onResult(event.data) + signal?.addEventListener('abort', onAbort) }) } export function postMcpOAuthResult(params: URLSearchParams): void { const status = params.get('status') === 'success' ? 'success' : 'error' - window.opener?.postMessage( - { - type: MESSAGE_TYPE, - status, - serverId: params.get('server_id') ?? undefined, - code: params.get('code') ?? undefined, - } satisfies OAuthMessage, - window.location.origin, - ) + const message = { + type: MESSAGE_TYPE, + status, + serverId: params.get('server_id') ?? undefined, + code: params.get('code') ?? undefined, + } satisfies OAuthMessage + window.opener?.postMessage(message, window.location.origin) + if (typeof BroadcastChannel !== 'undefined') { + const channel = new BroadcastChannel(CHANNEL_NAME) + channel.postMessage(message) + channel.close() + } } function oauthErrorMessage(code?: string): string { diff --git a/sdk/README.md b/sdk/README.md index 47e2bc5d..f3dad008 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -70,3 +70,19 @@ profile = client.auth.get_profile() client.auth.refresh() # Rotates both the access and refresh tokens in-place. client.auth.logout() ``` + +MCP server credentials are write-only: responses report `server.authentication` +metadata but never the stored values. + +```python +server = client.tools.create_server( + name="search", transport="http", url="https://mcp.example/mcp", + headers={"Authorization": "Bearer ..."}, +) +client.tools.update_server(server.id, headers=None) # None clears; omit to keep. + +# OAuth: open the URL in any browser; ASAREE stores the tokens on callback. +authorization = client.tools.begin_oauth(server.id, scope="tools.read") +print(authorization.authorization_url) +client.tools.clear_credentials(server.id, revoke=True) +``` diff --git a/sdk/src/asaree_client/__init__.py b/sdk/src/asaree_client/__init__.py index d1f3d681..4c5bcfa4 100644 --- a/sdk/src/asaree_client/__init__.py +++ b/sdk/src/asaree_client/__init__.py @@ -38,6 +38,8 @@ LLMModelInfo, LLMModels, LLMSetting, + MCPAuthenticationStatus, + MCPOAuthAuthorization, MCPServer, MeasurementCapabilities, MeasurementPlanValidation, @@ -106,6 +108,8 @@ "LLMModelInfo", "LLMModels", "LLMSetting", + "MCPAuthenticationStatus", + "MCPOAuthAuthorization", "MCPServer", "Message", "MeasurementCapabilities", diff --git a/sdk/src/asaree_client/_models/mcp.py b/sdk/src/asaree_client/_models/mcp.py index 382e55b9..8425f5f6 100644 --- a/sdk/src/asaree_client/_models/mcp.py +++ b/sdk/src/asaree_client/_models/mcp.py @@ -2,11 +2,30 @@ import uuid from datetime import datetime -from typing import Any +from typing import Any, Literal from pydantic import BaseModel +class MCPAuthenticationStatus(BaseModel): + """Non-secret credential metadata; stored secret values are never returned.""" + + auth_mode: Literal["none", "static_headers", "stdio_env", "oauth"] + configured: bool + authorization_required: bool + static_headers_configured: bool + stdio_env_configured: bool + stdio_env_names: list[str] + + +class MCPOAuthAuthorization(BaseModel): + """Open ``authorization_url`` in a browser; the server consumes the callback.""" + + authorization_url: str + expires_at: datetime + transaction_id: str + + class MCPServer(BaseModel): id: uuid.UUID name: str @@ -16,6 +35,8 @@ class MCPServer(BaseModel): status: str error_message: str | None capabilities: dict[str, Any] | None + authentication: MCPAuthenticationStatus + credential_management_allowed: bool created_at: datetime diff --git a/sdk/src/asaree_client/models.py b/sdk/src/asaree_client/models.py index ee859f02..a8148760 100644 --- a/sdk/src/asaree_client/models.py +++ b/sdk/src/asaree_client/models.py @@ -30,7 +30,7 @@ Trial, ) from asaree_client._models.llm import LLMConnectionCheck, LLMModelInfo, LLMModels, LLMSetting -from asaree_client._models.mcp import MCPServer, ToolCallResult +from asaree_client._models.mcp import MCPAuthenticationStatus, MCPOAuthAuthorization, MCPServer, ToolCallResult from asaree_client._models.okf import DirectoryEntry, DirectoryListing, OKFBundle, OKFDocument from asaree_client._models.protocols import ( CellRunBatch, @@ -69,6 +69,8 @@ "LLMModelInfo", "LLMModels", "LLMSetting", + "MCPAuthenticationStatus", + "MCPOAuthAuthorization", "MCPServer", "MeasurementCapabilities", "MeasurementPlanValidation", diff --git a/sdk/src/asaree_client/resources/tools.py b/sdk/src/asaree_client/resources/tools.py index 9bce686c..a916aced 100644 --- a/sdk/src/asaree_client/resources/tools.py +++ b/sdk/src/asaree_client/resources/tools.py @@ -5,7 +5,8 @@ import uuid from typing import Any -from asaree_client.models import MCPServer, ToolCallResult +from asaree_client._sentinel import UNSET, UnsetType +from asaree_client.models import MCPOAuthAuthorization, MCPServer, ToolCallResult ResourceId = uuid.UUID | str @@ -26,9 +27,16 @@ def create_server( command: str | None = None, url: str | None = None, headers: dict[str, str] | None = None, + server_env: dict[str, str] | None = None, ) -> MCPServer: + """Register a server. + + *headers* carries static HTTP credentials (e.g. ``{"Authorization": + "Bearer ..."}``); *server_env* carries encrypted environment + credentials for a stdio server. Neither is ever returned by the API. + """ payload: dict[str, Any] = {"name": name, "transport": transport} - for key, value in {"command": command, "url": url, "headers": headers}.items(): + for key, value in {"command": command, "url": url, "headers": headers, "server_env": server_env}.items(): if value is not None: payload[key] = value return MCPServer(**self._client._post("/mcp-servers", json=payload)) @@ -44,27 +52,47 @@ def update_server( transport: str | None = None, command: str | None = None, url: str | None = None, - headers: dict[str, str] | None = None, + headers: dict[str, str] | None | UnsetType = UNSET, + server_env: dict[str, str] | None | UnsetType = UNSET, ) -> MCPServer: - """Update supplied connection fields. + """Update supplied connection fields and credentials. - ``None`` means unchanged, matching the server and Motoro contract. - The current endpoint cannot clear a saved command, URL, or headers; - replace the registration when that is required. + For *name*, *transport*, *command* and *url*, ``None`` means unchanged: + the endpoint cannot clear them. Credentials distinguish omission from + clearing: leave *headers*/*server_env* unset to keep the stored values, + pass a mapping to replace them, or ``None`` to clear them. """ - payload = { + payload: dict[str, Any] = { key: value - for key, value in { - "name": name, - "transport": transport, - "command": command, - "url": url, - "headers": headers, - }.items() + for key, value in {"name": name, "transport": transport, "command": command, "url": url}.items() if value is not None } + if not isinstance(headers, UnsetType): + payload["headers"] = headers + if not isinstance(server_env, UnsetType): + payload["server_env"] = server_env return MCPServer(**self._client._patch(f"/mcp-servers/{server_id}", json=payload)) + def begin_oauth(self, server_id: ResourceId, *, scope: str | None = None) -> MCPOAuthAuthorization: + """Start an OAuth authorization for an HTTP server. + + Open the returned ``authorization_url`` in a browser before + ``expires_at``. The provider redirects to the ASAREE callback, which + stores the tokens server-side; poll :meth:`get_server` + and check ``authentication.configured`` to observe completion. + """ + data = self._client._post(f"/mcp-servers/{server_id}/oauth/start", json={"scope": scope}) + return MCPOAuthAuthorization(**data) + + def clear_credentials(self, server_id: ResourceId, *, revoke: bool = False) -> MCPServer: + """Clear every stored credential kind without deleting the registration. + + With *revoke*, ASAREE first asks the OAuth provider to revoke its + tokens; local credentials are cleared whether or not that succeeds. + """ + data = self._client._post(f"/mcp-servers/{server_id}/credentials/clear", json={"revoke": revoke}) + return MCPServer(**data) + def delete_server(self, server_id: ResourceId) -> None: self._client._delete(f"/mcp-servers/{server_id}") diff --git a/sdk/tests/test_sdk_gaps.py b/sdk/tests/test_sdk_gaps.py index 159667d6..b6eded7e 100644 --- a/sdk/tests/test_sdk_gaps.py +++ b/sdk/tests/test_sdk_gaps.py @@ -71,6 +71,15 @@ def _mcp_response() -> dict[str, Any]: "status": "connected", "error_message": None, "capabilities": None, + "authentication": { + "auth_mode": "stdio_env", + "configured": True, + "authorization_required": False, + "static_headers_configured": False, + "stdio_env_configured": True, + "stdio_env_names": ["API_TOKEN"], + }, + "credential_management_allowed": True, "created_at": datetime.now(tz=UTC).isoformat(), } @@ -173,12 +182,55 @@ def test_create_skill_from_url_defaults_to_repository_root() -> None: ] -def test_mcp_update_omits_none_fields() -> None: +def test_mcp_update_omits_unset_fields() -> None: client = RecordingClient(_mcp_response()) - Tools(client).update_server("server-id", name="renamed", command=None, headers=None) + server = Tools(client).update_server("server-id", name="renamed", command=None) assert client.calls == [("PATCH", "/mcp-servers/server-id", {"json": {"name": "renamed"}})] + assert server.authentication.stdio_env_names == ["API_TOKEN"] + + +def test_mcp_update_sends_explicit_credential_clear_and_replacement() -> None: + client = RecordingClient(_mcp_response()) + + Tools(client).update_server("server-id", headers=None, server_env={"API_TOKEN": "secret"}) + + assert client.calls == [ + ("PATCH", "/mcp-servers/server-id", {"json": {"headers": None, "server_env": {"API_TOKEN": "secret"}}}) + ] + + +def test_mcp_create_sends_stdio_credentials() -> None: + client = RecordingClient(_mcp_response()) + + Tools(client).create_server(name="local", transport="stdio", command="server", server_env={"API_TOKEN": "s"}) + + assert client.calls == [ + ( + "POST", + "/mcp-servers", + {"json": {"name": "local", "transport": "stdio", "command": "server", "server_env": {"API_TOKEN": "s"}}}, + ) + ] + + +def test_mcp_oauth_start_and_credential_clear() -> None: + expires_at = datetime.now(tz=UTC).isoformat() + client = RecordingClient( + {"authorization_url": "https://auth.example/authorize", "expires_at": expires_at, "transaction_id": "t"} + ) + tools = Tools(client) + + authorization = tools.begin_oauth("server-id", scope="tools.read") + client.response = _mcp_response() + tools.clear_credentials("server-id", revoke=True) + + assert authorization.authorization_url == "https://auth.example/authorize" + assert client.calls == [ + ("POST", "/mcp-servers/server-id/oauth/start", {"json": {"scope": "tools.read"}}), + ("POST", "/mcp-servers/server-id/credentials/clear", {"json": {"revoke": True}}), + ] def test_tool_call_retry_is_explicit() -> None: diff --git a/src/asaree/services/agent_messenger.py b/src/asaree/services/agent_messenger.py index 295e92d9..4c2fc845 100644 --- a/src/asaree/services/agent_messenger.py +++ b/src/asaree/services/agent_messenger.py @@ -949,7 +949,7 @@ async def _skip(node_id: str, reason: str) -> dict[str, Any]: await messenger.checkpoint() return dispatch, "cancelled" if state == "canceled" else "failed" - brief = str(dispatch["output_text"] or "") + brief = _output_content(str(dispatch["output_text"] or ""), dispatch.get("payload") or {}) # -- 2. every worker, none skipped --------------------------------- async def _worker(node_id: str) -> tuple[str, dict[str, Any]]: diff --git a/src/asaree/services/factor_bindings.py b/src/asaree/services/factor_bindings.py index 9575ad9a..e6192a9b 100644 --- a/src/asaree/services/factor_bindings.py +++ b/src/asaree/services/factor_bindings.py @@ -56,7 +56,11 @@ def validate_factor_bindings(design_spec: dict[str, Any] | None, graph: dict[str ) levels = factor.get("levels") or [] level_type = factor.get("level_type") or "string" - legacy_boolean = "level_type" not in factor and levels == [True, False] + # Mirrors the frontend's isBooleanFactor: legacy factors predate + # level_type, and either ordering of the two booleans is boolean. + legacy_boolean = ( + "level_type" not in factor and len(levels) == 2 and all(isinstance(level, bool) for level in levels) + ) if level_type != "boolean" and not legacy_boolean and (not levels or published_value != levels[0]): raise ValueError( f"{label!r} field {field_path!r} is bound to factor {factor_name!r}, but its published value " diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 5c6697ee..aa65cab9 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -2670,10 +2670,15 @@ def _upstream_output_text(graph: dict[str, Any], node_id: str, node_runs: dict[s """What a deactivated node's own output becomes: its upstream context, verbatim, with no goal/prompt mixed in -- the literal "pass the input straight through unchanged" semantic of a disabled node. Empty - string for a start node (nothing upstream to pass through).""" + string for a start node (nothing upstream to pass through). A + payload-only predecessor passes through as its rendered fields, so + deactivating a node never silently drops a typed handoff.""" upstream_ids = _upstream_ids(graph, node_id) - parts = [node_runs[uid]["output_text"] for uid in upstream_ids if node_runs.get(uid, {}).get("output_text")] - return "\n\n".join(parts) + parts = [ + _output_content(str(run.get("output_text") or ""), run.get("payload") or {}) + for run in (node_runs.get(uid) or {} for uid in upstream_ids) + ] + return "\n\n".join(part for part in parts if part) def _node_seed_prompt(node: dict[str, Any]) -> str: diff --git a/tests/test_factor_bindings.py b/tests/test_factor_bindings.py index 66110a21..46a5ecd6 100644 --- a/tests/test_factor_bindings.py +++ b/tests/test_factor_bindings.py @@ -61,6 +61,22 @@ def test_bound_canvas_value_must_be_the_first_level() -> None: validate_factor_bindings(design_spec, graph) +@pytest.mark.parametrize("levels", [[True, False], [False, True]]) +def test_legacy_boolean_factor_accepts_either_level_order(levels: list[bool]) -> None: + design_spec = {"factors": [{"name": "Gate", "levels": levels}]} + graph = { + "nodes": [ + { + "id": "gate", + "data": {"config": {"enabled": True}, "factor_bindings": {"config.enabled": "Gate"}}, + } + ], + "edges": [], + } + + validate_factor_bindings(design_spec, graph) + + def test_binding_to_missing_published_field_is_rejected() -> None: design_spec = {"factors": [{"name": "Prompt", "levels": ["one", "two"]}]} graph = { diff --git a/tests/test_mcp_server_auth.py b/tests/test_mcp_server_auth.py index 711faa70..bb1a213a 100644 --- a/tests/test_mcp_server_auth.py +++ b/tests/test_mcp_server_auth.py @@ -4,6 +4,7 @@ from typing import Any, cast import pytest +from motoro.mcp.oauth import MCPOAuthCallbackError, MCPOAuthStateError from motoro.services.mcp_service import UNSET from asaree.api import mcp_servers @@ -27,11 +28,14 @@ async def update(server_id: uuid.UUID, **kwargs: object) -> SimpleNamespace: monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.update_server", update) monkeypatch.setattr(mcp_servers, "_to_response", lambda _config, **_kwargs: _async("response")) - result = cast(Any, await mcp_servers.update_server_endpoint( - config.id, - mcp_servers.UpdateServerRequest(name="renamed"), - cast(Any, SimpleNamespace(id=owner_id)), - )) + result = cast( + Any, + await mcp_servers.update_server_endpoint( + config.id, + mcp_servers.UpdateServerRequest(name="renamed"), + cast(Any, SimpleNamespace(id=owner_id)), + ), + ) assert result == "response" assert captured["headers"] is UNSET @@ -168,5 +172,89 @@ async def test_server_response_exposes_only_safe_authentication_metadata( assert "encrypted-secret-material" not in str(payload) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("kwargs", "failure", "expected_code"), + [ + ({"code": None, "state": None, "error": "access_denied"}, None, "authorization_rejected"), + ({"code": "c", "state": None, "error": None}, None, "authorization_rejected"), + ({"code": "c", "state": "s", "error": None}, MCPOAuthStateError("stale"), "invalid_state"), + ({"code": "c", "state": "s", "error": None}, MCPOAuthCallbackError("rejected"), "token_exchange_failed"), + ( + {"code": "c", "state": "s", "error": None}, + mcp_servers.MCPServerNotFoundError("gone"), + "authorization_failed", + ), + ], +) +async def test_oauth_callback_redirects_errors_without_detail( + monkeypatch: pytest.MonkeyPatch, kwargs: dict[str, Any], failure: Exception | None, expected_code: str +) -> None: + async def complete(**_kwargs: object) -> SimpleNamespace: + if failure is None: + raise AssertionError("an incomplete callback must not reach Motoro") + raise failure + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.complete_oauth_authorization", complete) + monkeypatch.setattr(mcp_servers, "get_settings", lambda: SimpleNamespace(frontend_url="https://asaree.example/")) + + response = await mcp_servers.complete_oauth_callback(iss=None, **kwargs) + + assert response.status_code == 303 + assert response.headers["location"] == ( + f"https://asaree.example/mcp/oauth/callback?status=error&code={expected_code}" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("revoke", [True, False]) +async def test_clear_credentials_is_owner_scoped_and_revokes_only_on_request( + monkeypatch: pytest.MonkeyPatch, revoke: bool +) -> None: + owner_id = uuid.uuid4() + config = _config(owner_id) + calls: list[tuple[str, dict[str, object]]] = [] + + async def clear_oauth(server_id: uuid.UUID, **kwargs: object) -> bool: + calls.append(("oauth", kwargs)) + return True + + async def clear_all(server_id: uuid.UUID, **kwargs: object) -> SimpleNamespace: + calls.append(("all", kwargs)) + return config + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.clear_oauth_credentials", clear_oauth) + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.clear_server_credentials", clear_all) + monkeypatch.setattr(mcp_servers, "_to_response", lambda _config, **_kwargs: _async("response")) + + result = cast( + Any, + await mcp_servers.clear_credentials_endpoint( + config.id, + mcp_servers.ClearCredentialsRequest(revoke=revoke), + cast(Any, SimpleNamespace(id=owner_id)), + ), + ) + + assert result == "response" + expected = [("oauth", {"owner_id": owner_id, "revoke": True})] if revoke else [] + assert calls == [*expected, ("all", {"owner_id": owner_id})] + + +@pytest.mark.asyncio +async def test_clear_credentials_for_another_owners_server_is_not_found(monkeypatch: pytest.MonkeyPatch) -> None: + async def clear_all(server_id: uuid.UUID, **kwargs: object) -> SimpleNamespace: + raise mcp_servers.MCPServerNotFoundError("MCP server was not found") + + monkeypatch.setattr("asaree.api.mcp_servers.mcp_service.clear_server_credentials", clear_all) + + with pytest.raises(mcp_servers.HTTPException) as exc_info: + await mcp_servers.clear_credentials_endpoint( + uuid.uuid4(), mcp_servers.ClearCredentialsRequest(), cast(Any, SimpleNamespace(id=uuid.uuid4())) + ) + + assert exc_info.value.status_code == 404 + + async def _async(value: object) -> object: return value diff --git a/tests/test_protocol_execution.py b/tests/test_protocol_execution.py index 23695611..33522614 100644 --- a/tests/test_protocol_execution.py +++ b/tests/test_protocol_execution.py @@ -408,6 +408,15 @@ def test_a_deactivated_node_passes_its_input_through_with_no_label_of_its_own() assert pe._upstream_output_text(graph, "b", {"a": {"output_text": "draft text here"}}) == "draft text here" +def test_a_deactivated_node_passes_a_payload_only_input_through_as_its_fields() -> None: + a = _node("a", "agent", {"prompt": "Count it"}, label="Counter") + b = _node("b", "agent", {"prompt": "Polish it"}, label="Editor") + b["data"]["active"] = False + graph = {"nodes": [a, b], "edges": _edges(("a", "b"))} + node_runs = {"a": {"output_text": "", "payload": {"n_rows": 4300}}} + assert pe._upstream_output_text(graph, "b", node_runs) == "Structured fields: n_rows=4300" + + def test_a_reader_downstream_of_a_deactivated_node_sees_that_nodes_name() -> None: """Attribution follows the graph the run actually walked, not the graph the user would have drawn with the node removed. Naming the deactivated node is diff --git a/tests/test_supervisor_architecture.py b/tests/test_supervisor_architecture.py index 7bab0696..81a40198 100644 --- a/tests/test_supervisor_architecture.py +++ b/tests/test_supervisor_architecture.py @@ -181,6 +181,13 @@ async def test_the_brief_reaches_every_worker(stubs: dict[str, Any]) -> None: assert "Split the cohort three ways." in _prompt_for(stubs, worker) +async def test_a_payload_only_brief_still_reaches_every_worker(stubs: dict[str, Any]) -> None: + stubs["answers"]["sup"] = ("", None, {"payload": {"split": "three ways"}}) + await _run(stubs) + for worker in ("w1", "w2", "w3"): + assert 'Structured fields: split="three ways"' in _prompt_for(stubs, worker) + + async def test_payload_only_turns_reach_supervisor_review_and_synthesis(stubs: dict[str, Any]) -> None: stubs["answers"]["w1"] = ("", None, {"payload": {"finding": "cohort imbalance"}}) From 868d1a9bc2ffb2be8eb83c7480c10eec37538898 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 29 Sep 2026 15:28:49 -0700 Subject: [PATCH 18/18] fix: keep the version badge current and accurate - Add git commit/tags to uv's cache keys so a new commit or tag rebuilds the project; previously uv reused its cached build and the Docker image kept reporting a stale version. - Use hatch-vcs's no-guess-dev scheme so dev builds report the last real tag (0.7.0.post1.dev17) instead of an invented next patch; the badge shows it as v0.7.0+17. - Retry the badge's version fetch instead of giving up when the backend is still starting. --- frontend/src/components/AppVersionBadge.tsx | 26 ++++++++++++--------- pyproject.toml | 22 ++++++++++++++--- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/AppVersionBadge.tsx b/frontend/src/components/AppVersionBadge.tsx index 2161582e..89b7b64f 100644 --- a/frontend/src/components/AppVersionBadge.tsx +++ b/frontend/src/components/AppVersionBadge.tsx @@ -8,16 +8,17 @@ import { versionApi } from '@/api/client' * tag `v0.3.0a` comes back as `0.3.0a0`. Dropping that padding shows the tag * that was actually cut rather than a number nobody typed. * - * `0.2.1.dev50+g8366ca1` -> `v0.2.1-dev50`: between tags hatch-vcs guesses the - * next patch release and counts the commits since. Kept (rather than rounded to - * the last real release) because "which build am I looking at" is the whole - * point of the badge, and a dev build is exactly the case where the tag alone - * would be a lie. The `+g` local part is dropped from the badge and kept - * in its tooltip -- it's the part you only want when you're already suspicious. + * `0.2.0.post1.dev50+g8366ca1` -> `v0.2.0+50`: between tags hatch-vcs (with + * pyproject's `no-guess-dev` scheme) keeps the last real tag and counts the + * commits since. The count is kept (rather than rounded to the tag) because + * "which build am I looking at" is the whole point of the badge, and a dev + * build is exactly the case where the tag alone would be a lie. The `+g` + * local part is dropped from the badge and kept in its tooltip -- it's the part + * you only want when you're already suspicious. */ function formatVersion(raw: string): string { const [publicPart] = raw.split('+') - return `v${publicPart.replace(/\.dev/, '-dev').replace(/(a|b|rc)0$/, '$1')}` + return `v${publicPart.replace(/\.post\d+\.dev(\d+)$/, '+$1').replace(/(a|b|rc)0(?=$|\+)/, '$1')}` } /** The running build's version, pinned to the very top-right of the viewport. @@ -36,12 +37,15 @@ export function AppVersionBadge() { const { data } = useQuery({ queryKey: ['app-version'], queryFn: versionApi.get, - // A build can't change under a running tab, so this is fetched once per - // session and never revalidated. Failure is silent: a missing badge is a - // better outcome than an error toast over the login form. + // A build can't change under a running tab, so once fetched this is never + // revalidated. Failure is silent (a missing badge beats an error toast over + // the login form) but not final: a tab opened while the backend is still + // starting -- e.g. right after `docker compose up` -- would otherwise keep + // that one refused request, and no badge, until a manual reload. Retry + // indefinitely on react-query's default backoff (capped at 30s). staleTime: Infinity, gcTime: Infinity, - retry: false, + retry: true, }) if (!data?.version) return null diff --git a/pyproject.toml b/pyproject.toml index 9cbc1927..29db2094 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,9 +2,9 @@ name = "asaree" # Derived from the git tag by hatch-vcs (see [tool.hatch.version]), same as # Motoro. Nothing to edit when cutting a release: tag it and the version -# follows. Between tags this reads as a dev version (0.2.1.dev4+g), which -# is the point -- it names the exact build rather than whatever number was last -# typed here. +# follows. Between tags this reads as a post-release dev version +# (0.2.0.post1.dev4+g), which is the point -- it names the exact build +# rather than whatever number was last typed here. dynamic = ["version"] description = "ASAREE — the product built on Motoro." readme = "README.md" @@ -116,6 +116,22 @@ build-backend = "hatchling.build" [tool.hatch.version] source = "vcs" +# `no-guess-dev` rather than the default `guess-next-dev`: between tags the +# default invents the next patch (v0.7.0 + 17 commits -> 0.7.1.dev17), naming a +# release nobody cut. This keeps the last real tag (0.7.0.post1.dev17), which +# still sorts after 0.7.0 under PEP 440. +raw-options = { version_scheme = "no-guess-dev" } + +# uv reuses its cached build of this project until a cache key changes, and by +# default the only keys are pyproject.toml/setup.py/setup.cfg. The version lives +# in git, not in any of those files, so without the git keys a new commit or tag +# silently reused the old build -- and the Docker image kept reporting whatever +# version was stamped the last time pyproject.toml changed. +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { git = { commit = true, tags = true } }, +] # Generated at build time, gitignored, and shipped in the wheel so an installed # app can report its own version without the git metadata it was built from.