diff --git a/config/trinity-meta-prompt/prompt.md b/config/trinity-meta-prompt/prompt.md index 50411e4807..1c9a1aff3f 100644 --- a/config/trinity-meta-prompt/prompt.md +++ b/config/trinity-meta-prompt/prompt.md @@ -80,7 +80,7 @@ Until it is removed, an entry appended to the `requests` array of `~/.trinity/op } ``` -The operator's answer is written back into the entry: `status: "responded"` with `response`, `responded_by` and `responded_at`. An item that has waited past the operator's aging bound carries a `platform.aging_since` timestamp written by Trinity — read it, never write to `platform`. After processing a response, update the item's status to `"acknowledged"`. Keep only `pending` and `responded` items plus up to 3 recent `acknowledged` items. The platform database is the permanent record. An ID you raised with `ask_operator` is never read from the file. +The operator's answer is written back into the entry: `status: "responded"` with `response`, `response_text` and `responded_at` — never who answered. An item that has waited past the operator's aging bound carries a `platform.aging_since` timestamp written by Trinity — read it, never write to `platform`. After processing a response, update the item's status to `"acknowledged"`. Keep only `pending` and `responded` items plus up to 3 recent `acknowledged` items. The platform database is the permanent record. An ID you raised with `ask_operator` is never read from the file. ### When to Use diff --git a/docker/base-image/agent_server/mcp_validator.py b/docker/base-image/agent_server/mcp_validator.py index 3cfec4c458..37c43ce52a 100644 --- a/docker/base-image/agent_server/mcp_validator.py +++ b/docker/base-image/agent_server/mcp_validator.py @@ -87,6 +87,10 @@ class McpValidationError(ValueError): # documented default for fresh installs. _TRINITY_DEFAULT_URL = "http://mcp-server:8080/mcp" _TRINITY_BEARER_RE = re.compile(r"^Bearer\s+trinity_mcp_[A-Za-z0-9_-]{1,200}$") +# #2392: the injected entry also carries the spawning execution's id, expanded +# by Claude Code from its process env. Only this exact literal is canonical. +_TRINITY_EXEC_ID_HEADER = "X-Trinity-Execution-Id" +_TRINITY_EXEC_ID_VALUE = "${TRINITY_EXECUTION_ID:-manual}" def _is_canonical_trinity_entry(server: dict) -> bool: @@ -94,7 +98,8 @@ def _is_canonical_trinity_entry(server: dict) -> bool: entry: only ``type``, ``url``, ``headers`` keys; ``type=http``; ``url`` matches the configured Trinity MCP URL; ``headers`` contains only ``Authorization`` with a ``Bearer trinity_mcp_…`` token of the right - shape. + shape, optionally plus ``X-Trinity-Execution-Id`` whose value is exactly + ``${TRINITY_EXECUTION_ID:-manual}`` (#2392). Strict allowlist — any extra key, any wrong value, any extra header, any non-bearer auth scheme is rejected. Owners who want to redefine @@ -114,7 +119,12 @@ def _is_canonical_trinity_entry(server: dict) -> bool: return False headers = server.get("headers") - if not isinstance(headers, dict) or list(headers.keys()) != ["Authorization"]: + if not isinstance(headers, dict): + return False + if set(headers.keys()) == {"Authorization", _TRINITY_EXEC_ID_HEADER}: + if headers[_TRINITY_EXEC_ID_HEADER] != _TRINITY_EXEC_ID_VALUE: + return False + elif list(headers.keys()) != ["Authorization"]: return False auth = headers.get("Authorization") if not isinstance(auth, str) or not _TRINITY_BEARER_RE.match(auth): diff --git a/docker/base-image/agent_server/services/trinity_mcp.py b/docker/base-image/agent_server/services/trinity_mcp.py index 0fd24dcff1..55064aad5f 100644 --- a/docker/base-image/agent_server/services/trinity_mcp.py +++ b/docker/base-image/agent_server/services/trinity_mcp.py @@ -12,6 +12,13 @@ logger = logging.getLogger(__name__) +# #2392: every Trinity MCP call carries the id of the execution that spawned the +# CLI, so the backend can attribute tool calls. The value is expanded by the CLI +# itself from its own process env (TRINITY_EXECUTION_ID is set per spawn by the +# agent server), so one config file serves every concurrent execution. +EXECUTION_ID_HEADER = "X-Trinity-Execution-Id" +EXECUTION_ID_HEADER_VALUE = "${TRINITY_EXECUTION_ID:-manual}" + def inject_trinity_mcp_if_configured() -> bool: """ @@ -51,7 +58,9 @@ def _inject_claude_mcp(trinity_mcp_url: str, trinity_mcp_api_key: str) -> bool: "type": "http", "url": trinity_mcp_url, "headers": { - "Authorization": f"Bearer {trinity_mcp_api_key}" + "Authorization": f"Bearer {trinity_mcp_api_key}", + # Claude Code expands ${VAR:-default} in http MCP headers. + EXECUTION_ID_HEADER: EXECUTION_ID_HEADER_VALUE, } } } @@ -118,7 +127,10 @@ def _inject_gemini_mcp(trinity_mcp_url: str, trinity_mcp_api_key: str) -> bool: settings["mcpServers"]["trinity"] = { "url": trinity_mcp_url, "headers": { - "Authorization": f"Bearer {trinity_mcp_api_key}" + "Authorization": f"Bearer {trinity_mcp_api_key}", + # Gemini CLI expands $VAR / ${VAR} / ${VAR:-default} across + # settings.json at load (packages/cli/src/utils/envVarResolver.ts). + EXECUTION_ID_HEADER: EXECUTION_ID_HEADER_VALUE, } } @@ -401,6 +413,11 @@ def _inject_codex_mcp(trinity_mcp_url: str, trinity_mcp_api_key: str) -> bool: server = { "url": trinity_mcp_url, "bearer_token_env_var": "TRINITY_MCP_API_KEY", + # #2392: Codex has no ${VAR} expansion; env_http_headers maps header -> + # env var name, read from the codex process env. When the var is unset + # or blank Codex OMITS the header (no `manual` default); the backend + # treats an absent id as absent. + "env_http_headers": {EXECUTION_ID_HEADER: "TRINITY_EXECUTION_ID"}, } if _upsert_codex_mcp_servers({"trinity": server}): logger.info("Injected Trinity MCP server into codex config.toml") diff --git a/docs/memory/architecture/agent-lifecycle.md b/docs/memory/architecture/agent-lifecycle.md index 7bebb27d00..9f297eb8cc 100644 --- a/docs/memory/architecture/agent-lifecycle.md +++ b/docs/memory/architecture/agent-lifecycle.md @@ -174,6 +174,8 @@ An agent's Claude Code marketplace-plugin selection is a first-class, **committe **Backend side** (details in [git-sync-health.md](../feature-flows/git-sync-health.md)): `SyncHealthService` polls git-enabled agents every 60s (#2742: from the `synchealth:leader` lease-holder only, so `--workers 2` runs one poll not two — **fail-open**, because an observability feed must not go dark when Redis is; the cadence is `SYNC_HEALTH_POLL_INTERVAL_SECONDS`, default unchanged at 60 s. One poller instead of two halves the rate `consecutive_failures` climbs, so `sync_failing` now fires at ~3 min rather than ~90 s), upserts `agent_sync_state` (`consecutive_failures` ++ on fail / reset on success; `ahead_working`/`behind_working` expose working-branch divergence, P6; `git_dir_bytes` #1596 + `pack_count`/`loose_objects`/`maintenance_failures` #1595 carry the repo-health curve — all agent-supplied ints coerced at the boundary, the `/health clone_status` convention), emits `sync_failing` operator-queue entries at ≥3 failures (S1) and edge-triggered `git_bloat` entries when `git_dir_bytes` crosses `GIT_DIR_ALERT_BYTES` (default 10 GiB) or maintenance fails 3× running (#1595 — the failure class was previously silent until the disk filled). Powers the dashboard sync dot, `GET /api/agents/sync-health` (now incl. `git_dir_bytes`), and `GET /api/fleet/sync-audit` — whose `duplicate_binding` flag marks agents sharing a `(github_repo, working_branch)` pair (§P5 silent-clobber setup) (S6, #390). +**Divergence age and the divergence freeze (trinity-enterprise#706).** Health measures whether the agent and its repository *agree*, not only whether the last push succeeded. The poller also keeps four columns: `diverged_since` (first poll at which the working tuple — origin on the agent's own branch, #2105 — showed ahead or behind; set once, cleared when both are known 0 or ahead is 0 with no upstream, **kept** when ahead could not be computed, so unknown never starts or clears it), `dirty_files` (the status call's `changes_count`) with its own `dirty_since` clock, and `last_successful_push_at` (the later of the heartbeat's #3011 record, an operator Push's `agent_git_config.last_sync_at` and the stored value; agent timestamps must carry an offset and not be in the future). **One policy module decides everything:** `services/sync_freeze_policy.py::classify` (stdlib-only, never raises) returns `state` / `reason` / `recommendation` / `binding` / `freeze` / `freeze_reason`; it is vendored **byte-identically** to `src/scheduler/sync_freeze_policy.py` (the #1088 precedent, Invariant #5; `test_ent706_sync_policy_parity.py`), and `services/sync_health_view.py::sync_view` is the one backend call site (it resolves `git_service.is_push_denied` so the leaf stays pure). **Work agent** = `source_mode = 0 OR auto_sync_enabled = 1` — fork-to-own keeps `source_mode = 1` in the DB but auto-syncs to its own fork. **Freeze** = `freeze_schedules_if_sync_failing AND (sync_failing OR (work agent AND diverged > 24 h AND last_check_at within 15 min))`; the freshness clause fails open because the poller writes nothing for an unreachable agent. Enforced in two places over the one rule: the scheduler's direct-DB `SchedulerDatabase.sync_freeze_reason` (fail-open; a reason string, never a truthy `(bool, reason)` tuple) writes `Git sync frozen: ` into the skipped row — **one row per cron tick** while the freeze lasts, pruned by execution retention — and `GET /api/internal/agents/{name}/sync-health-status` reports the same decision with `freeze_reason`. While a divergence freeze is in force the poller raises **one** `sync_diverged` operator-queue item per episode under its lease, id `sync-diverged-{agent}-{diverged_since}` (reserved prefix; the `(agent_name, request_id)` conflict target collapses repeats and a fail-open double leader), with counts and the recommendation and no agent-authored error text; it is not auto-resolved. Un-freezing is automatic when divergence clears. The dot (`utils/syncHealth.js`) renders the backend's `state` and `reason` and holds no threshold. No backfill: every clock starts at the first poll after upgrade, which is the 24 h soak before any divergence freeze. Requirements §11.19. + ### Post-Creation Repo Binding (ent#109) "Bind this agent to a GitHub repo you own" — the ownership retrofit ent#123 left open. A tokenless public-template agent (the default Cornelius) accumulates a knowledge base it cannot push anywhere, and the only documented escape was "create a new agent with fork-to-own and import your data", which discards the agent's identity, its 180-day name reservation and its history. `POST /api/agents/{name}/git/bind-to-own-repo` creates a user-owned repo from the agent's **current workspace volume** (not from its template — that is the reason it cannot share ent#93's copy step), repoints `origin` in place, and re-bakes the container env. It is a **rebind, not a fork verb**: an already-writable agent is an ordinary rebind, which is what makes AC #3 ("works for any agent") literally true. Owner-only **and** human-only (`reject_agent_principal` — an agent-scoped key resolves to its owner *carrying the owner's role*, so a role gate alone is satisfied by any agent's injected key on a default admin-owned install). Requirements §11.12; flow [agent-repo-binding.md](../feature-flows/agent-repo-binding.md). diff --git a/docs/memory/architecture/api-endpoints.md b/docs/memory/architecture/api-endpoints.md index 4c5a1e4ab9..1915bc5a88 100644 --- a/docs/memory/architecture/api-endpoints.md +++ b/docs/memory/architecture/api-endpoints.md @@ -16,7 +16,7 @@ | GET | `/api/agents` | List all agents | | GET | `/api/agents/context-stats` | Context & activity state for all agents | | GET | `/api/agents/autonomy-status` | Autonomy status for all accessible agents | -| GET | `/api/agents/sync-health` | Per-agent git sync health for dashboard dots (#389) | +| GET | `/api/agents/sync-health` | Per-agent git sync health for dashboard dots (#389); carries the backend's `state` / `reason` / `recommendation` / `binding` / `freeze` (ent#706) | | GET | `/api/agents/subscription-pressure` | Batch subscription-pressure rows for dashboard badges (#471): per accessible agent (pure-DB `visible_agent_names`, ent#384 helper) — `auth_mode` (`AgentAuthStatus` vocabulary), `subscription_name`, `failure_events_24h` + its auth-kind slice `auth_failures_24h` (#2352), one-gate `rate_limited_now`, the probe's own `token_status` (`ok`|`invalid_token`|`rate_limited`|`error`), fresh `utilization_5h_pct`. Explicit `response_model`; registered before `/{agent_name}` (Invariant #4) | | POST | `/api/agents` | Create agent. Accepts an optional `display_label` (ent#1640) — the human-facing name set at creation (normalized + named-error validated like `PUT /label`); omit/blank → renders under the slug. Accepts `import_intent: fork\|copy\|clone` for `github:` templates (ent#15 — copy = backend-materialized snapshot, no sync/row/PAT; see [github-import-intents.md](../feature-flows/github-import-intents.md)) and an `Idempotency-Key` header (Invariant #18, scope `agent_create:{user_id}`) | | POST | `/api/agents/deploy-local` | Deploy a packaged local agent (base64 tar.gz; `require_role("creator")`). #2060 integrity contract: embedded `.trinity-manifest.json` verified post-extract + post-copy (fail-closed `MANIFEST_DRIFT`/`MANIFEST_REQUIRED`), in-root symlinks preserved, caps carry observed+limit, evidence-bearing response (`verified`/counts/`compatibility_hard_count`), `Idempotency-Key` (scope `agent_deploy:{user_id}`) + per-base-name `agent:deploy_op:` lock, failed deploys compensate (residue removed, workspace volume reclaimed, stopped previous version restarted). See [local-agent-deploy.md](../feature-flows/local-agent-deploy.md) | @@ -145,7 +145,7 @@ The per-agent VoIP config + voice-picker UI lives in the agent Settings/Sharing | GET/PUT/DELETE | `/api/agents/{name}/github-pat` | PAT config status / set per-agent PAT (validated, encrypted) / clear (revert to global) | | GET/PUT | `/api/agents/{name}/git/auto-sync` | Per-agent 15-min auto-sync heartbeat flag | | GET/PUT | `/api/agents/{name}/git/freeze-schedules-if-failing` | Freeze-on-sync-failure flag | -| GET | `/api/agents/{name}/git/sync-state` | Persisted sync-state row | +| GET | `/api/agents/{name}/git/sync-state` | Persisted sync-state row plus the policy verdict (ent#706) | | POST | `/api/agents/{name}/git/reset-to-main-preserve-state` | Recovery reset — see [Git Sync Health](agent-lifecycle.md#git-sync-health-389390) | | POST | `/api/agents/{name}/git/bind-to-own-repo` | **Bind to a repo the caller owns** (ent#109) — create it if needed, push the agent's CURRENT workspace history, repoint `origin`, persist the per-agent PAT, re-bake the container env. `OwnedAgentByName` **+ `reject_agent_principal`** (human-only), `Idempotency-Key` verb-folded. See [Post-Creation Repo Binding](agent-lifecycle.md#post-creation-repo-binding-ent109) | | GET | `/api/agents/{name}/git/bind-to-own-repo/status` | Resolve a binding whose HTTP response was lost — reports the DB row vs the live container's `origin` (`origin_in_sync`) rather than a remembered request (ent#109) | @@ -265,7 +265,7 @@ Generalises #868 to agent scope (`db/schedules.py:get_agent_analytics`); read-on Bulk ops scope writes to the caller's accessible agents (tri-state: admin = no filter, empty set = no-op). The sync service write-back also propagates `cancelled`/`expired` status into agent queue files (in-place flips of still-`pending` entries only) so agents stop waiting on cleared items and stale file entries can't resurrect purged rows (#1017). The Operations UI exposes these as a per-tab **Clear All** button (`notifications` tab uses `POST /api/notifications/dismiss-all` — bulk pending+acknowledged → dismissed, same accessible-set scoping). -WebSocket events: `operator_queue_new`, `operator_queue_responded` / `operator_queue_cancelled` (thin `{id, agent_name}` since trinity-enterprise#611 — listeners refetch), `operator_queue_acknowledged`, `operator_queue_cleared` (one per bulk op, #1017; count only — `notifications_cleared` for the notifications variant), `operator_queue_sync` (#2915, one per poll cycle; announces expiry too). Backed by the 5s Operator Queue Sync background service. Every ending — answer, cancel, bulk cancel, expiry — goes through one sink, `services/ask_service.py` (trinity-enterprise#611: CAS + endings ledger → one audit row per transition → one thin trigger → the registered ending observers); the machine keys never see the new person fields (`disposed_by_email`, `resolved_to`) on get/list. See [operating-room.md → Endings](../feature-flows/operating-room.md#endings-trinity-enterprise611). +WebSocket events: `operator_queue_new`, `operator_queue_responded` / `operator_queue_cancelled` (thin `{id, agent_name}` since trinity-enterprise#611 — listeners refetch), `operator_queue_acknowledged`, `operator_queue_cleared` (one per bulk op, #1017; count only — `notifications_cleared` for the notifications variant), `operator_queue_sync` (#2915, one per poll cycle; announces expiry too). Backed by the 5s Operator Queue Sync background service. Every ending — answer, cancel, bulk cancel, expiry — goes through one sink, `services/ask_service.py` (trinity-enterprise#611: CAS + endings ledger → one audit row per transition → one thin trigger → the registered ending observers); the machine keys never see a person's identity on get/list — an allowlist projection (`_MACHINE_ROW_FIELDS`: no `responded_by_*`, `addressed_to_email`, `disposed_by_email`, `resolved_to`), and the platform's heads-ups about a person are not returned to them at all (trinity-enterprise#715). See [operating-room.md → Endings](../feature-flows/operating-room.md#endings-trinity-enterprise611). **Addressed asks (ent#364).** `operator_queue.addressed_to_email` names the HUMAN an item is for; `agent_name` only ever said which agent. NULL — every pre-ent#364 row — means "the operator answers", so the column is additive by construction. It is a **column and not a key in `context`** because it is an authorization decision (who may answer, whose Workspace sidebar it appears in) and `context` is agent-authored JSON whose clamp bounds size and type only. Validated at the one agent-authored boundary (`operator_queue_service._validated_addressee`, inside the #1632 clamp) against `client_portal.service.agent_on_roster(agent, email, include_owned=False)` — an agent may address only someone already on its roster — and **fails closed**: malformed, off-roster, or an unreadable roster all drop to NULL. The read/answer surface is **OSS core** (ent#428): `client_portal/asks/`, mounted unconditionally in `main.py` at `/api/enterprise/client-portal/asks` — a prefix that is retained history, like the `enterprise_`-prefixed portal tables, not a licensing claim. It re-checks the roster at read time so a revoked share stops showing an already-raised ask, projects an explicit client-facing shape (never `context`), and answers through the same ask sink as the operator respond route (`services/ask_service.py`, trinity-enterprise#611), so the ledger, write-back, audit and the WS trigger are unforked, and it is person-only like respond (a non-person platform principal gets 403 `person_required`). `responded_by_id` stays NULL for a client answerer — there is no `users` row, and inventing one would be a lie in the audit trail. **The list fails loud (trinity-enterprise#610 PR A0):** `GET /api/enterprise/client-portal/asks` returns **503 `{code: "asks_unavailable", message}`** with `Retry-After: 20` (the Workspace poll interval) when any of its queue reads raises (the agent set, the count or the page, #3059) or the read-time roster re-check cannot read the roster (`asks/service.AsksUnavailable`, strict `_on_roster`); it never answers an outage with `[]`. `answer_ask`'s roster failure stays the uniform 404. diff --git a/docs/memory/architecture/backend.md b/docs/memory/architecture/backend.md index b86c56ed97..a5cd310734 100644 --- a/docs/memory/architecture/backend.md +++ b/docs/memory/architecture/backend.md @@ -173,6 +173,8 @@ - `heartbeat_service.py` - Agent push-heartbeat liveness layer — see [Heartbeat Liveness](reliability.md#heartbeat-liveness-reliability-004-307) - `operator_queue_service.py` - Operating Room sync with agent containers (OPS-001); the agent-authored ingestion boundary enforces the per-agent depth/rate/size caps + reserved-id guard + leader lock (#1632) - `sync_health_service.py` - Git sync health polling — see [Git Sync Health](agent-lifecycle.md#git-sync-health-389390) +- `sync_freeze_policy.py` - The one sync-health rule (state / reason / recommendation / freeze), stdlib leaf, vendored byte-identically to `src/scheduler/` (ent#706) — see [Git Sync Health](agent-lifecycle.md#git-sync-health-389390) +- `sync_health_view.py` - `sync_view(row, config)`: the backend's single call into the policy, used by the poller and every sync-health read (ent#706) - `canary_service.py` - Orchestration-invariant watcher — see [Canary Harness](reliability.md#canary-invariant-harness-canary-001-411) - `compatibility/` - Agent compatibility validation package (spec/collector/static_checks/ai_checks/fixes) — see [Agent Compatibility Validation](agent-lifecycle.md#agent-compatibility-validation-668) diff --git a/docs/memory/architecture/background-services.md b/docs/memory/architecture/background-services.md index 3d507e8f94..fd5719b620 100644 --- a/docs/memory/architecture/background-services.md +++ b/docs/memory/architecture/background-services.md @@ -18,7 +18,7 @@ Services that run continuously in the backend process: |---------|--------|-------------| | **Cleanup Service** | `cleanup_service.py` | Every 5 min: active watchdog reconciliation against agent process registries (orphan recovery, auto-terminate timeouts) + passive stale recovery (CLEANUP-001, #129). Also runs retention + soft-delete purge sweeps, the **expired-SSH sweep** (`_sweep_expired_ssh_credentials` → `SshService.cleanup_expired_credentials` — removes an expired ephemeral key's line from the container `authorized_keys` sshd reads; TTL was previously enforced only on Redis metadata, #1616), and the #740 startup orphan-loop hook, and the **agent_reminders retention sweep** (`_sweep_agent_reminders_retention` — DELETEs terminal `fired`/`cancelled`/`failed` reminders past `agent_reminders_retention_days`, #1296) — see [Soft Delete & Retention](reliability.md#soft-delete-retention--recovery-834-772). Runs the additive **lease-reaper** (`lease_reaper_service`) each cycle — re-queues (preserving `execution_id`) or poison-parks expired pull leases (#1081 Phase 3, #429/#1402; inert until an agent is piloted). #1804: every recovery path also closes its execution's dispatch activity (`_close_bulk_swept_activities` for the bulk sweeps, the shared helper elsewhere), counted in `activities_closed_on_recovery`; the 120-minute activity backstop now runs **last** in the cycle. **#2433:** orphan = the agent does not know the execution **and** no live backend dispatcher owns it — see [In-Flight Dispatch Proof-of-Life](execution.md#in-flight-dispatch-proof-of-life-2433) | | **Operator Queue Sync** | `operator_queue_service.py` | Polls running agents every 5s, reads `~/.trinity/operator-queue.json`, syncs to DB, writes responses back (OPS-001). The item `id` is a platform-minted uuid; the agent's correlation string is `request_id` with `(agent_name, request_id)` uniqueness, so all sync reads/writes (exists, acknowledge, response write-back) are agent-scoped and two agents can't collide (#1631). **Leader-locked (#1632):** only the holder of `opqueue:leader` (SET NX, TTL `max(3×interval, 30s)` floor so a slow-write cycle can't flap leadership, own-lease refresh, fail-open — mirror monitoring #1464) runs a cycle, so `--workers 2` doesn't double-charge the ingestion rate limiter or double-broadcast the flood alert. Ingestion is capped per agent (depth + rate + fleet + field hygiene, #1632) — see [Operator Queue](api-endpoints.md#operator-queue-ops-001). **Honest since #2915:** the cycle reads the tri-state `agent_container_states()` (`None` ⇒ nothing swept or synced — the #2196 class), runs expiry BEFORE the no-running-agents return (since trinity-enterprise#611 through the ask sink, `ask_service.expire()`: per-id CAS with the endings ledger `disposed_by = 'timeout'`, one `expired` audit row per ask, the ending wake for opted-in running filers; `_changed_this_cycle` is reset above it so an expiry rides the cycle's one trigger, even when Docker is unreadable), sweeps open rows of stopped agents to `unconfirmed:agent_not_running` and their owed answers / terminal flips to `undelivered:agent_not_running` (both sweeps skip platform-minted rows in SQL), reconciles every file entry against the row (`sync_state`/`sync_detail`, edge-triggered by rowcount, 3-cycle hysteresis on read failures, no write-back on an unparseable or wrong-shape file, first entry wins on a duplicated id), delivers an answer only into a matching still-pending entry after a pre-write re-read with `if_match` (412 = retry, never clobber), records every delivery outcome, audits accountability transitions only, and broadcasts ONE `operator_queue_sync` per cycle — see [operating-room.md → Sync honesty](../feature-flows/operating-room.md#sync-honesty-2915). **Native asks (trinity-enterprise#611):** a row raised over the platform (`channel=mcp`) is outside the file contract — the sync index names its ids as `foreign`, and the cycle skips a file entry re-using one before any branch reads it (logged once per agent and id); the cycle creates through `create_operator_queue_item_with_outcome` and never counts, announces or audits a row it did not insert (a native ask raised between the index read and the create); the first file ingest per agent per process logs the file channel's deprecation (`ask_operator`, two releases) | -| **Sync Health Service** | `sync_health_service.py` | Polls git-enabled agents every 60s (`SYNC_HEALTH_POLL_INTERVAL_SECONDS`, default unchanged) — see [Git Sync Health](agent-lifecycle.md#git-sync-health-389390). **Leader-locked (#2742):** `synchealth:leader`, SET NX, TTL `max(3×interval, 30s)`, own-lease refresh, compare-and-delete release (a GET-then-DEL lets an expired worker delete a sibling's fresh lease), **fail-open** — Redis down ⇒ every worker polls, i.e. the pre-#2742 behaviour, and the agent-side lock-free + single-flighted status (#2742) makes a duplicated poll harmless. Failing closed was rejected: this feed is how `sync_failing` is ever raised, and darkening it exactly when infra is degraded is the worse error. **Named side effect:** `upsert_sync_state` *increments* `consecutive_failures` per failed poll, so it is not idempotent — one leader instead of two doubles time-to-`sync_failing` from ~90s to ~180s (arguably making the counter mean what its name says: 3 consecutive failed polls = 3 minutes) | +| **Sync Health Service** | `sync_health_service.py` | Polls git-enabled agents every 60s (`SYNC_HEALTH_POLL_INTERVAL_SECONDS`, default unchanged) — see [Git Sync Health](agent-lifecycle.md#git-sync-health-389390). **Leader-locked (#2742):** `synchealth:leader`, SET NX, TTL `max(3×interval, 30s)`, own-lease refresh, compare-and-delete release (a GET-then-DEL lets an expired worker delete a sibling's fresh lease), **fail-open** — Redis down ⇒ every worker polls, i.e. the pre-#2742 behaviour, and the agent-side lock-free + single-flighted status (#2742) makes a duplicated poll harmless. Failing closed was rejected: this feed is how `sync_failing` is ever raised, and darkening it exactly when infra is degraded is the worse error. **Named side effect:** `upsert_sync_state` *increments* `consecutive_failures` per failed poll, so it is not idempotent — one leader instead of two doubles time-to-`sync_failing` from ~90s to ~180s (arguably making the counter mean what its name says: 3 consecutive failed polls = 3 minutes). **Also raises the `sync_diverged` episode item (trinity-enterprise#706)** under the same lease; its id is deterministic per episode (`sync-diverged-{agent}-{diverged_since}`), so a fail-open double poll converges on one row through the `(agent_name, request_id)` conflict target | | **Skills Library Sync** | `skills_sync_service.py` | Scheduled skills-library `git pull` + optional fleet-wide skill re-inject (ent#236). Runs in every worker but only the `skills:sync:leader` lease-holder performs a cycle (fail-open, mirrors #1464); self-gates on the default-OFF `skills_library_auto_sync_enabled` setting, re-read each cycle so an interval change needs no restart. A sweep fires only on a changed library commit, targets running non-ghost agents at `SKILLS_FLEET_INJECT_CONCURRENCY` (5), and persists an honest per-agent report + raises an operator alarm on any failure | | **Subscription Recovery Probe** | `subscription_recovery_service.py` | #447: every `SUBSCRIPTION_RECOVERY_PROBE_SECONDS` (300s) re-probes only the subscriptions currently PRESENTED as rate-limited, so a recovered one stops claiming it is limited without anyone watching. It is the **only** producer of the fresh "allowed" verdict `resolve_rate_limited_now` needs — the ambient refresh is demand-driven, so an unwatched instance never re-asked at all, and no success path clears a failure row. Reuses the `max_tokens=1` Haiku probe (~a dozen tokens) and the Settings-surfaced `subscription_headroom_auto_refresh` toggle rather than adding a second knob, since that setting already answers *may Trinity probe on its own?*. Leader-locked (`subscription:recovery:leader`, fail-open — a duplicated probe is a dozen wasted tokens, failing closed would silently stop recovery detection); the probe itself stays fail-CLOSED on Redis (no readable cache ⇒ the result could not be stored for anyone to see). Cannot feed itself: `_probe` records a 429 into the snapshot only, never `subscription_rate_limit_events`. **ent#434 gives the sweep a second job** — headroom sampling + weekly-alert evaluation — deliberately in this loop rather than a new one, since a second loop means a second lease over the same probe budget and a 60s per-worker floor is not a coordination primitive. Per subscription: `recover_probe` **first**, then `ensure_reading` + `classify_headroom`, in **sibling** try/excepts — the ordering keeps #447 from being floored out by the sampler, and the split keeps an alert-path bug from killing the only mechanism that clears a stale `LIMIT` badge. The sweep gained bounded concurrency and a between-chunk lease refresh because it now touches every subscription, so `N × 15s` serial probes could outlive the TTL. The fail-open leadership rationale is re-priced in the module docstring: the "dozen wasted tokens" argument assumed a normally-empty probe set, and what keeps it safe now is that BOTH halves fail CLOSED on Redis one level down | | **Monitoring Service** | `monitoring_service.py` | Fleet-wide health checks on configurable interval (30s default); authoritative for aggregate status. **Lifespan-resumed (#1121):** boot reads the persisted `monitoring_config` (staggered +12s) and starts the loop only when `enabled` — the flag is the single source of truth, **defaults OFF**, persisted by `enable`/`disable`/`PUT /config` (which also reconcile the running loop) so the choice survives restarts; `*_check_interval` rejects non-positive values (422), loop clamps sleep ≥1s (MON-001). **Cross-worker leader lock (#1464):** the loop runs in every uvicorn worker but only the holder of the Redis `monitoring:leader` lease (SET NX, TTL 3×interval, own-lease-only refresh; fail-open to leader when Redis is down) performs each probe cycle, so `--workers 2` no longer double-probes the fleet or double-feeds the circuit breaker; leadership fails over automatically when the holder dies | diff --git a/docs/memory/architecture/database.md b/docs/memory/architecture/database.md index a7e780c07b..87521ef900 100644 --- a/docs/memory/architecture/database.md +++ b/docs/memory/architecture/database.md @@ -733,8 +733,8 @@ CREATE TABLE operator_queue ( expires_at TEXT, response TEXT, response_text TEXT, - responded_by_id TEXT, - responded_by_email TEXT, + responded_by_id TEXT, -- withheld from machine keys on get/list (ent#715) + responded_by_email TEXT, -- withheld from machine keys on get/list; never written into the agent's file (ent#715) responded_at TEXT, acknowledged_at TEXT, cleared_at TEXT, -- #1017: NULL = visible; set = hidden by Clear All (rows deleted by the #1142 retention sweep past operator_queue_retention_days) @@ -746,7 +746,7 @@ CREATE TABLE operator_queue ( delivery_detail TEXT, -- #2915: conflict|http_|unreachable|timeout|entry_missing|entry_changed|closed_by_filer|agent_not_running|platform_minted delivery_updated_at TEXT, -- #2915 divergence_acknowledged_at TEXT, -- #2915 (PR #2989 review): the human answered a changed/closed item knowingly; the write-back delivers into the entry as it is now - addressed_to_email TEXT, -- ent#364: the human this ask is for; NULL = operator ask. Validated at ingestion against the agent's roster, never trusted from the payload + addressed_to_email TEXT, -- ent#364: the human this ask is for; NULL = operator ask. Validated at ingestion against the agent's roster, never trusted from the payload; withheld from machine keys on get/list (ent#715) -- trinity-enterprise#611 (SQLite `operator_queue_ask_object` / Alembic `0076_operator_queue_ask_object`; nullable, no backfill): disposition TEXT, -- answered|cancelled|expired — written in the SAME CAS UPDATE that flips `status`; NULL = ended before the ledger disposed_at TEXT, @@ -789,10 +789,16 @@ CREATE TABLE agent_sync_state ( pack_count INTEGER, -- #1595: packs from `git count-objects -v` loose_objects INTEGER, -- #1595: loose objects (gc-health signal) maintenance_failures INTEGER DEFAULT 0, -- #1595: consecutive failed maintenance attempts + diverged_since TEXT, -- ent#706: divergence episode start (ISO-Z; set once, cleared at 0/0) + dirty_files INTEGER, -- ent#706: porcelain change count + dirty_since TEXT, -- ent#706: dirt episode start (ISO-Z; cleared at 0) + last_successful_push_at TEXT, -- ent#706: last push that landed (heartbeat or operator Push) last_check_at TEXT, updated_at TEXT NOT NULL, FOREIGN KEY (agent_name) REFERENCES agent_ownership(agent_name) ); +-- ent#706 columns: SQLite migration `agent_sync_state_divergence`, Alembic +-- `0082_agent_sync_state_divergence`; nullable, no backfill. CREATE INDEX idx_sync_state_status ON agent_sync_state(last_sync_status, consecutive_failures); diff --git a/docs/memory/architecture/execution.md b/docs/memory/architecture/execution.md index 2a36cfdfa9..1221d83548 100644 --- a/docs/memory/architecture/execution.md +++ b/docs/memory/architecture/execution.md @@ -136,7 +136,7 @@ schema change (all state is Redis). Details in [redelivery-governor.md](../featu Trigger-boundary dedup — policy in Architectural Invariant #18, table DDL under `idempotency_keys`, details in [idempotency-keys.md](../feature-flows/idempotency-keys.md). `services/idempotency_service.py` (`begin`/`complete`/`fail`) over `db/idempotency.py`. The `(scope, key)` PRIMARY KEY is the atomic claim: `claim()` INSERTs an `in_flight` row; a concurrent loser catches `IntegrityError` and reads the surviving row (cross-process safe over the shared SQLite file). Lifecycle: `claim` → (`attach_execution`) → `complete` (stores `response_snapshot` for replay) or `release` (deletes the in_flight row so retry is possible; never deletes a `completed` row). Rows >24h expire and re-claim (cleanup purges via `idempotency_purge_expired`). Duplicates within 24h short-circuit with the original result + `X-Idempotent-Replay: true`; an in-flight duplicate returns 409. Fail-open. -**Effect-scoped extension (#1084):** trigger-boundary dedup stops a re-POSTed `/chat`/webhook from creating a *second execution*; it does NOT reach an agent's individual outbound tool calls. So a re-delivered turn (the at-least-once semantics pull-mode / work-stealing will introduce, Epic #1045/#1081) re-emits the same side effect (re-sends a message, re-charges a payment). The same `idempotency_service` adds a per-sink guard — `effect_guard(effect_type, identifying_args, *, execution_id, agent_name, dedup_label, payment_request_id)` — enforced at the SINK, per resolved action identity. Scopes: `effect:{execution_id}` for messages/voip/share_file (after `resolve_and_validate_execution` confirms the execution belongs to the agent — generalizing MEM-001), `payment:{agent_request_id}` for Nevermined settles (a Nevermined observability id, **not** a provider exactly-once token — this local guard, not the provider, enforces at-most-once per id; residual at-least-once retry tracked by #1408). Key = `{effect_type}:sha256(execution_id ∥ effect_type ∥ resolved_identifying_args ∥ dedup_label)` on **resolved, immutable** identity only (recipient/channel/account) — **never the LLM-generated body** (non-deterministic across a re-run → would defeat dedup); `dedup_label` lets an agent intentionally repeat an effect to the same target in one turn. `in_flight ≠ completed`: a completed replay returns the stored sanitized snapshot (no re-emit); an in-flight replay raises `EffectInProgressError` (router → 409, never a silent skip-and-succeed). Reuses the 24h default TTL (already exceeds the lease window, so a completed row outlives a late re-delivery — no new TTL plumbing). Wired sinks: `proactive_message_service.send_message`, `voip_service.place_outbound_call`, `agent_shared_files_service.create_share`, `nevermined_payment_service.settle_payment_once`; agents pass `execution_id`+`dedup_label` as MCP tool args (`messages.ts`/`voip.ts`/`files.ts`), **fail-open when absent** (safe today — pull-mode re-delivery is OFF). Trusted runtime injection of `execution_id` + fail-closed-when-absent is a **BLOCKING prerequisite** on Epic #1045/#1081 before pull-mode default-ON for side-effect agents (git push is idempotent-by-construction and needs no key). **Target-direction note:** `TARGET_ARCHITECTURE.md` v2 reframes this from a **per-agent** gate to **per-effect** — read/analysis-only + reversible + capability-confined-irreversible effects default on, only irreversible-**un-confineable** effects gate via the async operator queue (#1402); `effect_guard` (this section) is the reversible/backend-sink slice and retry-with-prior-trace (#1401) is the general recovery. See [effect-idempotency.md](../feature-flows/effect-idempotency.md). +**Effect-scoped extension (#1084):** trigger-boundary dedup stops a re-POSTed `/chat`/webhook from creating a *second execution*; it does NOT reach an agent's individual outbound tool calls. So a re-delivered turn (the at-least-once semantics pull-mode / work-stealing will introduce, Epic #1045/#1081) re-emits the same side effect (re-sends a message, re-charges a payment). The same `idempotency_service` adds a per-sink guard — `effect_guard(effect_type, identifying_args, *, execution_id, agent_name, dedup_label, payment_request_id)` — enforced at the SINK, per resolved action identity. Scopes: `effect:{execution_id}` for messages/voip/share_file (after `resolve_and_validate_execution` confirms the execution belongs to the agent — generalizing MEM-001), `payment:{agent_request_id}` for Nevermined settles (a Nevermined observability id, **not** a provider exactly-once token — this local guard, not the provider, enforces at-most-once per id; residual at-least-once retry tracked by #1408). Key = `{effect_type}:sha256(execution_id ∥ effect_type ∥ resolved_identifying_args ∥ dedup_label)` on **resolved, immutable** identity only (recipient/channel/account) — **never the LLM-generated body** (non-deterministic across a re-run → would defeat dedup); `dedup_label` lets an agent intentionally repeat an effect to the same target in one turn. `in_flight ≠ completed`: a completed replay returns the stored sanitized snapshot (no re-emit); an in-flight replay raises `EffectInProgressError` (router → 409, never a silent skip-and-succeed). Reuses the 24h default TTL (already exceeds the lease window, so a completed row outlives a late re-delivery — no new TTL plumbing). Wired sinks: `proactive_message_service.send_message`, `voip_service.place_outbound_call`, `agent_shared_files_service.create_share`, `a2a_outbound_service.call_agent`, `voice_reply_service`, `channel_completion_report`, `nevermined_payment_service.settle_payment_once`; the platform supplies `execution_id` via the agent's MCP header `X-Trinity-Execution-Id` (the tool arg is the older-image fallback), and on a pull-mode agent an effect with no usable id is **refused** with an operator alarm, elsewhere sent and logged `effect_guard.degraded` (#2392; git push is idempotent-by-construction and needs no key). **Target-direction note:** `TARGET_ARCHITECTURE.md` v2 reframes this from a **per-agent** gate to **per-effect** — read/analysis-only + reversible + capability-confined-irreversible effects default on, only irreversible-**un-confineable** effects gate via the async operator queue (#1402); `effect_guard` (this section) is the reversible/backend-sink slice and retry-with-prior-trace (#1401) is the general recovery. See [effect-idempotency.md](../feature-flows/effect-idempotency.md). ### Sequential Agent Loops (#740, UI #1106) diff --git a/docs/memory/architecture/workspace.md b/docs/memory/architecture/workspace.md index d4dd8cd8a4..85e34a0e94 100644 --- a/docs/memory/architecture/workspace.md +++ b/docs/memory/architecture/workspace.md @@ -1142,6 +1142,9 @@ routed through `create_bounded_alert` (the volume is driven by a client clicking agent-influenceable side of the #1677 classification) with its own registered type and a reserved id prefix, one item per person per target. It fires on EVERY thumbs-down, not only commented ones: "this was not useful" is the report and the words are the elaboration. +The queue's own reads keep the redaction: since trinity-enterprise#715 they never return the +item to a machine key (`operator_queue_service.is_about_a_person`), which the rated agent's +key could otherwise list; the ent#308 inbox-collision alert is withheld the same way. Prerequisite, and a live bug: `operator_queue.type` is free TEXT and both queue cards hardcoded an `approval → question → alert` chain that rendered no control for anything else, so `skill_not_found` items have never been closeable and five of them would jam a budgeted diff --git a/docs/memory/feature-flows.md b/docs/memory/feature-flows.md index 3b4db90d9a..86d101f3e5 100644 --- a/docs/memory/feature-flows.md +++ b/docs/memory/feature-flows.md @@ -24,6 +24,7 @@ | Date | ID | Change | Flow | |------|-----|--------|------| | 2026-09-28 | #2996 | fix(auth): **human-only grant surfaces** — `PUT /autonomy` and the other agent-config writes are person-only (`require_person`); the sign-in email and personal GitHub PAT change only in a signed-in session (`require_interactive`, ent#711); a route census makes every new route choose a policy class | [autonomy-mode.md](feature-flows/autonomy-mode.md) | +| 2026-09-27 | ent#706 | feat(sync-health): **health measures whether the agent and its repository agree, and the freeze keys on divergence age for work agents**. `agent_sync_state` gains `diverged_since` / `dirty_files` / `dirty_since` / `last_successful_push_at` (both tracks: `agent_sync_state_divergence` / Alembic `0082`; no backfill, so the clocks start at the first post-upgrade poll — the 24 h soak). One stdlib policy module (`services/sync_freeze_policy.py`, vendored byte-identically to the scheduler) decides `state` / `reason` / `recommendation` / `freeze` for every surface; a work agent (`source_mode = 0 OR auto_sync_enabled = 1`) diverged > 24 h on a fresh (≤ 15 min) observation freezes when its owner opted in, a deployment never does. The scheduler's gate writes the reason into the skipped row; one `sync_diverged` operator-queue item per episode (deterministic, reserved id). `/api/agents/sync-health`, `/git/sync-state` (and MCP `get_git_sync_state`) carry the verdict; the dashboard dot renders it and holds no threshold | [git-sync-health.md](feature-flows/git-sync-health.md), [github-sync.md](feature-flows/github-sync.md) | | 2026-09-22 | #2958 | fix(chat): **a `/chat` turn resumes only the agent's own chat session** (`--resume `, never `--continue`, which took a scheduled run's persisted JSONL and paid its auto-compaction); a model change starts fresh; one cold retry on a reaped JSONL; the id rides a marker file into the reaper's keep set; chat rows now carry `compact_metadata` and `execution.compaction`, and `get_execution_result` returns it | [execution-queue.md](feature-flows/execution-queue.md), [parallel-headless-execution.md](feature-flows/parallel-headless-execution.md), [mcp-orchestration.md](feature-flows/mcp-orchestration.md), [authenticated-chat-tab.md](feature-flows/authenticated-chat-tab.md), [file-sharing-outbound.md](feature-flows/file-sharing-outbound.md), [system-wide-trinity-prompt.md](feature-flows/system-wide-trinity-prompt.md) | | 2026-09-16 | ent#625 | feat(workspace): **a theme switch (light / dark / system) at the top right of the central column**. The Workspace followed the global theme with no control (no NavBar), so a platform user could not change it from the Workspace and an external client could not change it at all. One switch, one store: `PortalThemeSwitch` drives the same `useThemeStore`/`trinity-theme` key the NavBar does, shows the RESOLVED state honestly under `system` ("System · dark"), sits in a `#header-end` slot of both column headers, and collapses to an icon below `sm`. The NavBar's inline picker is replaced by the shared `ThemeChoice` primitive (its raw-colour entry shrinks 35 → 23). First MOUNTED component test in the frontend suite (`@vue/test-utils` + per-file jsdom) — the click reaches `setTheme`. | [dark-mode-theme.md](feature-flows/dark-mode-theme.md) | | 2026-09-16 | ent#620 | feat(workspace): **the Work card says what the agent is doing — one fixed-height line that slides up, for the chat's own turn AND delegated/scheduled/room runs; the chat scrolls to the card on your own send**. The card's step slot was blank because its handler matched `evt.type === 'tool_use'`, a shape raw stream-json never carries. One vocabulary (`utils/workActivity.js`, shared with the Chat tab and Agent Detail), two feeds: the SSE frames for the own turn, the agent's heartbeat — now carrying a bounded per-execution `executions[]` from a per-execution activity slot — for everything else, folded onto live rostered rows by the Work read through the title sanitiser + roster mask, plus a Redis-only `/work/activity` poll at 2.5 s while live. ≥700 ms per line, bursts collapse, identical lines never re-animate, aged lines drop, terminal clears. Scroll only on the person's own send (#2624 holds). | [workspace-work.md](feature-flows/workspace-work.md) | @@ -231,7 +232,7 @@ | Agent Repo Binding | [agent-repo-binding.md](feature-flows/agent-repo-binding.md) | Bind a live agent to a GitHub repo the user owns (ent#109) | | GitHub Import Intents | [github-import-intents.md](feature-flows/github-import-intents.md) | fork / copy / clone create intents + inline compat check (ent#15) | | Persistent-State Allowlist | [persistent-state-allowlist.md](feature-flows/persistent-state-allowlist.md) | `.trinity/persistent-state.yaml` primitive for reset-preserve-state (S4, #383) | -| Git Sync Health | [git-sync-health.md](feature-flows/git-sync-health.md) | Auto-sync heartbeat, dual ahead/behind, dashboard dot, `/api/fleet/sync-audit`, creation-time canonical `.gitignore` seed (#2069, readiness-gated), lock-free single-flighted status read + observe-only stuck-lock report (#2742) | +| Git Sync Health | [git-sync-health.md](feature-flows/git-sync-health.md) | Auto-sync heartbeat, dual ahead/behind, dashboard dot, `/api/fleet/sync-audit`, creation-time canonical `.gitignore` seed (#2069, readiness-gated), lock-free single-flighted status read + observe-only stuck-lock report (#2742), divergence age + the divergence freeze + one backend-owned sync state (ent#706) | ### Skills Management diff --git a/docs/memory/feature-flows/effect-idempotency.md b/docs/memory/feature-flows/effect-idempotency.md index a47a116b43..6ffd37d169 100644 --- a/docs/memory/feature-flows/effect-idempotency.md +++ b/docs/memory/feature-flows/effect-idempotency.md @@ -87,10 +87,17 @@ side-effect-bearing agents. - Each sink stores a **sanitized, JSON-stable snapshot** (message_id / call_id+status / settle receipt / share URL), never a raw provider object. -4. **Agent passes `execution_id` as a tool arg** (MEM-001 template), **fail-open - when absent** — for this PR. Safe today because pull-mode re-delivery is OFF, - so fail-open cannot produce a duplicate. The MCP server can't currently see - `TRINITY_EXECUTION_ID`; trusted runtime injection is deferred (see Gate). +4. **The platform supplies `execution_id` (#2392).** The agent server sets + `TRINITY_EXECUTION_ID` in every spawned runtime process; the agent's Trinity + MCP entry sends it as `X-Trinity-Execution-Id` (Claude Code: + `${TRINITY_EXECUTION_ID:-manual}`; Codex `env_http_headers`; Gemini env + expansion). The MCP server reads it per request and it wins over the + agent-supplied tool arg, which stays as the fallback for older images. + `manual` marks a person's terminal/SSH session: no execution to re-deliver. + When no usable id arrives, a **pull-mode** agent's effect is **refused** + (`EffectUnguardedError` → 422 `effect_unguarded`, operator alarm + `effect_unguarded`); any other agent's effect is sent and logged as + `effect_guard.degraded` (see Pull-mode section). 5. **Guard at the service entry, keyed on resolved recipient+channel(+account).** A chunked message = one effect; a mid-chunk crash re-sends the whole message @@ -121,34 +128,52 @@ message/voip/share entries are exposed as MCP tool params on `messages.ts` / - **Primitive** — `services/idempotency_service.py`: `make_effect_scope`, `make_payment_scope`, `derive_effect_key`, `resolve_and_validate_execution` (generalizes the MEM-001 server-side resolution in `routers/public_memory.py` - — the agent supplies an execution_id, never its own identity; fail-open on - missing/mismatch), `EffectInProgressError`, and `effect_guard`. Reuses the + — the backend confirms the id belongs to the calling agent), `EffectInProgressError`, + `EffectUnguardedError`, `effect_dedup_required` (today: the pull pilots; Phase 5 + flips it), and `effect_guard`. Reuses the existing `begin`/`complete`/`fail` over `db/idempotency.py` — **no schema or migration change**. - **Sinks** wrap their actual emission in `effect_guard`; a success records the sanitized snapshot, an exception releases the claim for retry. - **Routers** surface a concurrent in-flight duplicate as **409** - (`messages.py`, `voip.py`, `agent_files.py`); `paid.py` returns a retryable + (`messages.py`, `voip.py`, `agent_files.py`, `a2a.py`) and a refused + unguarded effect as **422** `{"reason": "effect_unguarded"}` (same routers plus + `internal.py`'s `/agent-files/share`, which carries no id and is therefore + refused for a pull-mode agent); `paid.py` returns a retryable "settlement already in progress" result. The VoIP router keeps its boundary `Idempotency-Key` gate as the OUTER layer. ## MCP Layer (Invariant #13) -`messages.ts` / `voip.ts` / `files.ts` add optional `execution_id` + -`dedup_label` params (agent reads `execution_id` from the 'Execution Context' -block of its system prompt, same source as `write_user_memory`). `client.ts` -threads both into the request body. `chat_with_agent` already derives an +`server.ts` `authenticate` parses `X-Trinity-Execution-Id` into +`authContext.executionId` on every POST (fastmcp re-runs it per request). +`send_message`, `call_user`, `send_voice_reply`, `share_file` and +`call_a2a_agent` send `resolveExecutionId(authContext, params.execution_id)` +(`tools/execution_id.ts`): the header value, else the optional tool arg (older +agent images). `manual` is honoured only from the header; typed as the tool arg it is dropped. `client.ts` threads it and `dedup_label` into the request body. `chat_with_agent` already derives an `Idempotency-Key` (#525) — unchanged, regression-asserted in `messages.test.ts`. -## Pull-mode default-ON gate (BLOCKING prerequisite) +## Pull-mode: fail-closed without a usable id (#2392) -`dispatch_async_eligible()` stays `DISPATCH_ASYNC AND triggered_by in -ASYNC_DISPATCH_ELIGIBLE_TRIGGERS` (DISPATCH_ASYNC default-OFF). Enabling pull-mode -default-ON for **any** side-effect-bearing agent REQUIRES, first: -(a) **trusted runtime injection** of `execution_id` (so an agent can't forge or -omit it), and (b) **fail-closed-when-absent** (no execution_id → refuse the -effect, not fail-open). Both are tracked as a **blocking dependency on Epic -#1045/#1081**, not doc-only-and-forget. +Pull re-delivers the same `execution_id` after a lease expiry, so on a pull-mode +agent (`effect_dedup_required`) the guard refuses an effect it cannot de-duplicate: + +| id reaching the guard | pull-mode agent | other agent | +|---|---|---| +| resolves to the caller's own execution | dedup | dedup | +| `manual` (terminal/SSH session; Claude Code and Gemini) | send, `effect_guard.degraded` | same | +| absent (old image, raw API call, internal share route) | **refuse** + alarm | send, degraded log | +| unknown / another agent's execution | **refuse** + alarm | send, degraded log | +| lookup error (DB blip) | send, degraded log | same | + +Codex omits the header when `TRINITY_EXECUTION_ID` is unset, so a person running +Codex in a pull-mode agent's terminal reaches `absent` and is refused; so is a +user-scoped key acting for a pull-mode agent from outside a turn. + +The alarm goes through `create_bounded_alert` (type `effect_unguarded`, id prefix +`effect-unguarded-` reserved); at the budget cap the effect is still refused. +The header is not a trust boundary: an agent owns its `.mcp.json` and can cite +any of its own executions. The guard stops omission, not a hostile agent. > **Reframed (v2, 2026-07-01).** `TARGET_ARCHITECTURE.md` reframes the pull-mode side-effect rollout from a **per-agent** gate to **per-effect**: read/analysis-only + reversible + capability-confined-irreversible effects default on; only irreversible-**un-confineable** effects wait, via the **async operator queue** (#1402). `effect_guard` (this doc) is the reversible/backend-sink slice; general recovery is **retry-with-prior-trace** (#1401). The (a)/(b) trusted-injection requirement above still applies to the *confined-irreversible* tool-side gate. @@ -161,13 +186,13 @@ effect, not fail-open). Both are tracked as a **blocking dependency on Epic | `in_flight` replay | duplicate worker mid-flight | raise `EffectInProgressError` → 409 | clear retryable error | | chunked message crash | crash after chunk 3/5 | whole message re-sent on retry | duplicate chunks (at-least-once, documented) | | Nevermined settle | settle on terminal turn | terminal-turn guard preserved (no settle on failed execution) | no double-charge | -| absent execution_id | old image / agent omits arg | fail-open → send proceeds, no 5xx | no | +| no usable execution_id, pull-mode agent | old image / raw API call / foreign or unknown id | `EffectUnguardedError` → 422 `effect_unguarded` + operator alarm; nothing sent | agent sees a non-retryable refusal; operator sees the alarm | +| no usable execution_id, other agent | same | send proceeds, `effect_guard.degraded` warning | log only | ## NOT in Scope (deferred) - Pull-mode re-delivery itself (#1045/#1081) — this only makes it *safe* to enable. -- Trusted runtime/MCP injection of `execution_id` + fail-closed-when-absent — the - enforced capability gate (BLOCKING prerequisite above). +- A per-execution credential that would make the id unforgeable by the agent itself. - The "re-delivery preserves the same `execution_id`" dispatcher invariant + its integration test — owned by the pull-mode epic. - Per-chunk / per-provider-call guarding — single-target sinks use the entry @@ -194,6 +219,12 @@ effect, not fail-open). Both are tracked as a **blocking dependency on Epic (`tests/unit/test_ent549_file_audience.py`). - MCP: `send_message` forwards `execution_id` + `dedup_label` (undefined when omitted); `chat_with_agent` still sets a non-empty `mcp:` Idempotency-Key. +- #2392 (`tests/unit/test_2392_effect_guard_fail_closed.py`): the table above + for pull-mode and other agents, the alarm payload and budget-cap refusal, and a + re-delivered execution emitting each of message / voip_call / share_file / + a2a_call once. `tests/unit/test_2392_execution_id_header.py`: the MCP config + writers and validator. MCP: `tools/execution_id.test.ts`, + `execution-id-transport.test.ts` (two requests on one session carry distinct ids). ### Integration (sibling stack `-p trinity-1084-test`, remapped ports) - Re-deliver the same `execution_id` through proactive send → exactly one diff --git a/docs/memory/feature-flows/git-sync-health.md b/docs/memory/feature-flows/git-sync-health.md index bc7da47423..a3566b1fa5 100644 --- a/docs/memory/feature-flows/git-sync-health.md +++ b/docs/memory/feature-flows/git-sync-health.md @@ -28,6 +28,16 @@ This flow adds: Per-agent opt-outs are available via API for both the auto-sync heartbeat and schedule-freeze behaviour. +**trinity-enterprise#706 changed what "healthy" means.** Health used to +measure whether the last push *succeeded*; an agent 15 commits ahead with +auto-sync off never ran a heartbeat, never failed, and looked fine. It now +measures whether the agent and its repository *agree*: the poller keeps a +divergence clock (`diverged_since`), a dirt clock (`dirty_files` / +`dirty_since`) and `last_successful_push_at`, one backend policy module +(`services/sync_freeze_policy.py`) turns them into `state` / `reason` / +`recommendation`, and the schedule freeze now also fires for a **work agent** +diverged from origin for more than 24 h. See §2b and §5. + ## User Stories - **Operator**: "Show me which agents haven't synced successfully in the @@ -50,8 +60,10 @@ heartbeat and schedule-freeze behaviour. | **API** | `GET/PUT /api/agents/{name}/git/auto-sync` | Toggle the per-agent auto-sync flag | | **API** | `GET/PUT /api/agents/{name}/git/freeze-schedules-if-failing` | Toggle the freeze-schedules-on-sync-failure flag | | **API** | `GET /api/fleet/sync-audit` | Fleet-wide audit including `duplicate_binding` flag (admins see all; non-admins filtered) | -| **API** | `GET /api/internal/agents/{name}/sync-health-status` | Internal endpoint for the scheduler to check freeze-on-failure | +| **API** | `GET /api/internal/agents/{name}/sync-health-status` | The backend's read of the freeze decision (`should_freeze`, `freeze_reason`, `divergence_age_s`, `work_agent` — ent#706). The scheduler reads the DB directly with the vendored policy, not this endpoint | | **Operator Queue** | type=`sync_failing` | Inserted by `SyncHealthService` when `consecutive_failures` crosses 3 | +| **Operator Queue** | type=`sync_diverged` | ent#706: one per divergence episode, only while a divergence freeze is in force (id `sync-diverged-{agent}-{diverged_since}`) | +| **Scheduler** | `SchedulerDatabase.sync_freeze_reason` | ent#706: the cron gate's reason, written into the skipped row as `Git sync frozen: ` | ## Data Model @@ -73,11 +85,23 @@ git_dir_bytes BIGINT -- #1596: .git on-disk size (BIGINT sin pack_count INTEGER -- #1595: packs (count-objects -v) loose_objects INTEGER -- #1595: loose objects maintenance_failures INTEGER DEFAULT 0 -- #1595: failed maintenance streak +diverged_since TEXT -- ent#706: divergence episode start (set once, cleared at 0/0) +dirty_files INTEGER -- ent#706: porcelain change count (changes_count) +dirty_since TEXT -- ent#706: dirt episode start (cleared at 0) +last_successful_push_at TEXT -- ent#706: last push that landed last_check_at TEXT updated_at TEXT NOT NULL FOREIGN KEY (agent_name) REFERENCES agent_ownership(agent_name) ``` +The four ent#706 columns come from the SQLite migration +`agent_sync_state_divergence` and Alembic `0082_agent_sync_state_divergence`. +They are nullable with **no backfill**: the clocks cannot be known +retroactively, so they start at the first poll after upgrade — which is also +the 24 h soak before any divergence freeze can fire. The upsert takes a `KEEP` +sentinel for the two clocks (`db/sync_state.py`), because `_merged()` maps +None to "unchanged" and a clock needs "clear" too. + Index: `idx_sync_state_status` on `(last_sync_status, consecutive_failures)`. **New columns on `agent_git_config`**: @@ -438,6 +462,13 @@ SyncHealthService._poll_loop (SYNC_HEALTH_POLL_INTERVAL_SECONDS, default 60 s) │ │ pack_count / loose_objects / maintenance_failures → │ │ _coerce_nonneg_int (sync-state.json is agent-writable; │ │ reject strings/bools/objects/out-of-range at the boundary) + │ ├── ent#706 episode clocks from the RAW payload values + │ │ (_episode_clock): diverged_since set once on ahead/behind > 0, + │ │ cleared at ahead == 0 with behind 0 or None (no upstream), + │ │ KEPT when ahead is None; dirty_files = changes_count with + │ │ dirty_since the same way; last_successful_push_at = max(the + │ │ #3011 heartbeat field, agent_git_config.last_sync_at, stored) + │ │ — agent timestamps need an offset and must not be future │ ├── db.upsert_sync_state(...) │ ├── if consecutive_failures crossed 3: │ │ └── db.create_operator_queue_item( @@ -447,6 +478,11 @@ SyncHealthService._poll_loop (SYNC_HEALTH_POLL_INTERVAL_SECONDS, default 60 s) │ │ "Git token can't push", says it will not recover on │ │ its own, and carries context.cause='push_denied' + │ │ context.remediation (grant Contents: write / `repo`) + │ ├── ent#706: sync_view(updated, config) → if the freeze cause is + │ │ divergence: db.create_operator_queue_item(type='sync_diverged', + │ │ id='sync-diverged-{agent}-{diverged_since}') — the conflict + │ │ target makes it ONE row per episode across polls, restarts + │ │ and a fail-open double leader; counts + recommendation only │ ├── #1595 git_bloat alerts (same edge-trigger pattern): │ │ ├── git_dir_bytes crossed GIT_DIR_ALERT_BYTES (10 GiB) │ │ └── maintenance_failures crossed 3 @@ -454,6 +490,57 @@ SyncHealthService._poll_loop (SYNC_HEALTH_POLL_INTERVAL_SECONDS, default 60 s) │ (log line only — no operator-queue item, no DB column) ``` +### 2b. The policy and the freeze (trinity-enterprise#706) + +`services/sync_freeze_policy.py::classify(row, cfg, now, push_denied=)` is the +one rule. Stdlib only and never raises (the scheduler calls it on the fire +path over agent-written numbers); vendored **byte-identically** to +`src/scheduler/sync_freeze_policy.py` and pinned by +`test_ent706_sync_policy_parity.py`. `services/sync_health_view.py::sync_view` +is the backend's single call site; it resolves `git_service.is_push_denied` +so the leaf stays pure. + +``` +work agent = source_mode = 0 OR auto_sync_enabled = 1 (fork-to-own keeps + source_mode = 1) +unknown no row / never observed +red last sync failed (any binding) + work agent diverged > 24 h + work agent dirty > 24 h + auto-sync on, no heartbeat for 7 d (fresh observation) +yellow work agent diverged <= 24 h + deployment diverged (any age) or dirty > 24 h (never red on age) +green otherwise + +freeze = freeze_schedules_if_sync_failing AND ( + sync_failing (#1808, any binding) + OR (work agent AND diverged > 24 h AND last_check_at <= 15 min old)) +``` + +The freshness clause fails open: the poller writes nothing for an unreachable +agent (or a status call that 504s), so a stale `diverged_since` must never keep +an agent frozen that may already have pushed. The row stays red and the reason +says `(last observed 3h ago)`. `behind_main` (a push to `main` under a +working branch) is information in the reason, never divergence. + +Enforcement, both over the one rule: + +``` +scheduler _execute_schedule_with_lock (cron only) + └── db.sync_freeze_reason(agent) # direct DB read, fail-OPEN on any error + ├── None → fire + └── "diverged 0 behind / 7 ahead for 26h" + → _record_skipped_agent_schedule(skip_reason="Git sync frozen: ") + (one row per cron tick while frozen; retention prunes) + → _advance_next_run_only; un-freezes on the first tick after it clears +backend GET /api/internal/agents/{name}/sync-health-status + → pre-#706 keys unchanged + freeze_reason / divergence_age_s / work_agent +``` + +`sync_freeze_reason` returns a string, never a `(bool, reason)` tuple — a tuple +is truthy, and a caller left on the old `if db.should_freeze_...` shape would +freeze the whole fleet. `should_freeze_schedules` is its bool view. + ### 2a. Agent status handler (#2742) ``` @@ -515,6 +602,13 @@ remote holds, and `behind_working` is `null`. A count that can't be computed is `null`, never 0: for example, the main tuple on a repo with no `main`. The backend stores `null` as 0 (`sync_health_service._coerce_counter`). +**The working tuple is the divergence basis (ent#706).** `diverged_since` is +decided from the RAW `ahead_working` / `behind_working` payload values, before +that 0-coercion, so an uncomputable count never starts or clears the clock. +When the clock is kept that way, the stored `ahead_working` / `behind_working` +are kept too rather than coerced to 0, so a kept episode never reads +"diverged 0 behind / 0 ahead" in its reason or its operator-queue item. + Legacy `ahead` / `behind` in the response alias the main tuple so older clients keep working. @@ -551,13 +645,20 @@ the data-loss setup. - `src/frontend/src/utils/syncHealth.js::classifySyncHealth(entry)` → `'green' | 'yellow' | 'red' | 'gray'`. -- Rules: - - **gray**: `last_sync_status === 'never'` or no entry. - - **red**: `behind_working > 0`, OR `last_sync_status === 'failed'`, OR - last sync ≥ 7 days ago. - - **yellow**: 24 h ≤ last sync < 7 d (status success). - - **green**: last sync < 24 h AND status success AND - `behind_working === 0`. +- **The backend owns the state (ent#706).** `GET /api/agents/sync-health` + serves each entry's `state` / `reason` / `recommendation` / `binding` / + `freeze` from `sync_view` (plus `dirty_files`, `divergence_age_s`, + `last_successful_push_at`; every pre-#706 key is unchanged). The helper maps + `state` to a colour (`unknown` or missing → gray) and `syncHealthLabel` + shows `reason — recommendation` on hover. It holds **no threshold** — the + 24 h / 7 d / behind-is-red rules it used to compute are the backend's (§2b), + so the dot cannot disagree with the freeze. +- Visible change: an agent ahead of origin with auto-sync off used to render + **gray** (it had never synced); it now renders the backend's yellow or red. +- The batch reads through `db.list_sync_health_rows` — one query over + `agent_git_config` ⋈ live `agent_ownership` ⟕ `agent_sync_state`, the same + set the poller polls. An agent whose git sync is disabled (`sync_enabled = 0`) + now shows `unknown` rather than its last stale row. - `stores/agents.js::fetchSyncHealth()` calls `/api/agents/sync-health` on mount; `components/AgentListPanel.vue` (the Dashboard List mode — ent#260 retired the Agents page into it) renders the dot next to each @@ -585,7 +686,15 @@ the data-loss setup. | `routers/git.py` | `/git/auto-sync`, `/git/freeze-schedules-if-failing`, `/git/sync-state` | | `routers/agents.py` | `GET /api/agents/sync-health` (batch) | | `routers/fleet.py` | `GET /api/fleet/sync-audit` (new router) | -| `routers/internal.py` | `GET /api/internal/agents/{name}/sync-health-status` | +| `routers/internal.py` | `GET /api/internal/agents/{name}/sync-health-status` (ent#706: via `sync_view`) | +| `services/sync_freeze_policy.py` | ent#706: the one rule — `classify`, the constants, `format_age`; stdlib leaf | +| `services/sync_health_view.py` | ent#706: `sync_view(row, config)` — the backend's single call into the policy | +| `db/migrations.py`, `migrations/versions/0082_agent_sync_state_divergence.py`, `db/schema.py`, `db/tables.py` | ent#706: the four columns on both tracks | +| `db/sync_state.py` | ent#706: `KEEP` sentinel, the new upsert kwargs, `list_health_rows` (the shared reader) | +| `services/operator_queue_service.py` | ent#706: `sync-diverged-` in `_RESERVED_ID_PREFIXES` | +| `src/scheduler/sync_freeze_policy.py` | ent#706: byte-identical mirror of the policy | +| `src/scheduler/database.py` | ent#706: `sync_freeze_reason` (+ `should_freeze_schedules` as its bool view); the threshold is imported from the mirror | +| `src/scheduler/service.py` | ent#706: the gate writes `Git sync frozen: ` into the skipped row | | `main.py` | Starts `SyncHealthService` (staggered +5 s, PERF-269); registers `fleet_router` | ### Agent server @@ -602,7 +711,7 @@ the data-loss setup. | File | Purpose | |------|---------| | `stores/agents.js` | `syncHealth` state + `fetchSyncHealth()` action | -| `utils/syncHealth.js` | `classifySyncHealth`, `syncHealthColor`, `syncHealthLabel` | +| `utils/syncHealth.js` | `classifySyncHealth`, `syncHealthColor`, `syncHealthLabel` — render the backend's `state` / `reason` (ent#706), no thresholds | | `components/AgentListPanel.vue` | Renders the dot + imports helpers + fetches on mount + 60s visibility-aware refresh (ent#260 — replaces the retired `views/Agents.vue`) | ## Testing @@ -650,6 +759,29 @@ backend): Baseline: 75 passing tests added across the two PRs. +trinity-enterprise#706 (divergence age and the freeze): + +- `tests/unit/test_ent706_sync_policy.py` — `classify`, one case per rule, the + 24 h boundary at ± 1 s, fork-to-own vs deployment, the stale-observation + fail-open, aware / naive / offset timestamps agreeing, the recommendations. +- `tests/unit/test_ent706_sync_policy_parity.py` — backend ↔ scheduler byte + parity; the policy is a stdlib leaf; the scheduler imports the threshold. +- `tests/unit/test_ent706_sync_state_columns.py` — both migration tracks, the + SQLite migration run twice, and the upsert's set / keep / clear. +- `tests/unit/test_ent706_divergence_tracking.py` — the real `_poll_cycle`: + the clocks, `last_successful_push_at`, and the one-item-per-episode alert + (two pollers → one row; a new episode → a second; **control arms**: freeze + flag off or a deployment → zero items). +- `tests/unit/test_1808_sync_freeze_enforcement.py` — the scheduler gate on a + real SQLite file: divergence freezes at 24 h + 1 s, not at 24 h − 1 s; + **control arm**: the same row with `diverged_since` NULL fires; the skipped + row carries the reason. +- `tests/unit/test_ent706_internal_sync_health.py`, + `tests/unit/test_ent706_sync_health_surfaces.py` — the three read surfaces + keep their old keys and add the verdict. +- `src/frontend/tests/unit/syncHealth.spec.js` — the dot renders the backend + state and reason and holds no threshold. + The `.gitignore` half of §0 has its own real-git suites (no Docker either — they run the SHIPPED builder commands against throwaway repositories, because the defects live in git's own last-match-wins and dir-descent semantics): @@ -679,7 +811,8 @@ defects live in git's own last-match-wins and dir-descent semantics): | Auto-sync on/off per agent | `PUT /api/agents/{name}/git/auto-sync` body `{enabled: bool}` | `true` for non-source-mode GitHub-template agents | | Interval override | `GIT_SYNC_INTERVAL_SECONDS` env var in the agent container | 900 s (15 min) | | Fleet kill-switch | `GIT_SYNC_AUTO` env var (if missing/false the loop never starts) | `true` if the backend set it at creation **or** `auto_sync_enabled = 1` — re-derived as the OR of both on every container rebuild (ent#109) | -| Freeze schedules when sync failing | `PUT /api/agents/{name}/git/freeze-schedules-if-failing` | `false` (opt-in) | +| Freeze schedules when sync failing | `PUT /api/agents/{name}/git/freeze-schedules-if-failing` — since ent#706 it also freezes a **work agent** diverged from origin for more than 24 h (fresh observation). It is the only switch: there is no fleet kill switch and no env knob; the 24 h clock that starts at the first post-upgrade poll is the soak | `false` (opt-in) | +| Divergence / dirt / no-heartbeat / freshness thresholds (ent#706) | Constants in `services/sync_freeze_policy.py` (and its scheduler mirror) — deliberately not env knobs, so the dashboard and the freeze cannot be configured apart | 24 h / 24 h / 7 d / 15 min | | Alert threshold | Hardcoded in `SyncHealthService.ALERT_THRESHOLD` | 3 consecutive failures | | Sync-health poll cadence (#2742) | `SYNC_HEALTH_POLL_INTERVAL_SECONDS` env var on the backend — read at **call** time (a property, not an import-time copy), and wired into `docker-compose.yml`, `docker-compose.prod.yml` and `.env.example` as `${VAR:-60}`. Prod compose launches standalone (no base merge, no `env_file:`), so the explicit `environment:` list is the only route in; `/validate-pr` caught all three missing on this branch (the #1056 packaging class), and `test_2742_sync_health_leader_lock.py::TestPollIntervalReachesTheContainer` now pins the form so unset and empty both land on the default | 60 s (**unchanged** — the knob ships, the default does not move) | | Stuck-lock report sensitivity (#2742) | `_STUCK_LOCK_MIN_SIGHTINGS` / `_STUCK_LOCK_MIN_AGE_SECONDS` module constants in `agent_server/routers/git.py` — deliberately **not** an env var: the tunable is the number of stable sightings, not a wall-clock age, because age measures the in-flight operation | 3 sightings / 900 s | @@ -689,10 +822,28 @@ defects live in git's own last-match-wins and dir-descent semantics): - **`dirty_tree` in `/api/fleet/sync-audit` is always `false`** today. Populating it requires a live agent call per row; can land as a follow-up with `asyncio.gather`. -- **`freeze_schedules_if_sync_failing` is read-only from the scheduler - side**. The config flag, API, and internal lookup endpoint are wired - but actual enforcement belongs in the dedicated `trinity-scheduler` - container's pre-execution check — separate follow-up. +- ~~`freeze_schedules_if_sync_failing` is read-only from the scheduler + side~~ — enforced since #1808 (`SchedulerDatabase`), and since ent#706 it + also covers divergence. +- **A failed `git status` inside the agent reads as a clean tree (ent#706).** + The agent's status handler returns `changes = []` when `git status` fails, + so `dirty_files` goes to 0 and `dirty_since` clears. Rare (the read is + lock-free since #2742); fixing it needs a base-image change. +- **A fork-to-own or bound agent whose owner turned auto-sync off reads as a + deployment (ent#706)** and never divergence-freezes. `source_mode` stays 1 + for those agents and the binding kind is not persisted; closing the gap + needs a persisted binding (overlaps trinity-enterprise#704). +- **A starved pull cycles the freeze (ent#706).** An agent with auto-sync off + whose pull (trinity-enterprise#703) skips while executions run can sit + behind for > 24 h → freeze → cron stops → the pull lands → divergence clears + → the next tick fires: at most one short freeze window and one + `sync_diverged` item a day. It self-heals by design. +- **Raising `SYNC_HEALTH_POLL_INTERVAL_SECONDS` above 15 minutes disables the + divergence freeze (ent#706)**: every observation is then stale, and the + freshness guard fails open. `sync_failing` still freezes. +- **The agent writes every number the policy reads** (ahead / behind / + changes / push time), so a compromised agent can misreport them — the + existing #1595 trust model; the backend coerces types and ranges only. - **Auto-sync disabled for source-mode agents** by design. Source-mode tracks `main`, and auto-pushing to `main` would clobber protected branches. Source-mode agents still get sync-state tracking via the @@ -774,7 +925,7 @@ defects live in git's own last-match-wins and dir-descent semantics): - [github-repo-initialization.md](github-repo-initialization.md) — where instance IDs and working branches come from - [operating-room.md](operating-room.md) — the operator queue where - `sync_failing` entries surface + `sync_failing` and (ent#706) `sync_diverged` entries surface ## References diff --git a/docs/memory/feature-flows/github-sync.md b/docs/memory/feature-flows/github-sync.md index 37e3590a80..99f9a07954 100644 --- a/docs/memory/feature-flows/github-sync.md +++ b/docs/memory/feature-flows/github-sync.md @@ -1583,7 +1583,7 @@ Working - credential-free remotes + git credential helper (trinity-enterprise#61 |-----------|------|--------------| | **Upstream** | [Template Processing](template-processing.md) | GitHub templates trigger git sync setup | | **Upstream** | [Agent Lifecycle](agent-lifecycle.md) | Agent creation enables git sync | -| **Downstream** | [Git Sync Health](git-sync-health.md) | Auto-sync heartbeat + observability on top of this flow (#389, #390) | +| **Downstream** | [Git Sync Health](git-sync-health.md) | Auto-sync heartbeat + observability on top of this flow (#389, #390); since ent#706 "healthy" means the agent and its repository agree (divergence age), and the schedule freeze keys on it for work agents | | **Downstream** | Content generation | Large files go to `content/` folder | | **Related** | [Async Docker Operations](async-docker-operations.md) | Git commands use async docker exec (DOCKER-001) | @@ -1593,6 +1593,7 @@ Working - credential-free remotes + git credential helper (trinity-enterprise#61 | Date | Changes | |------|---------| +| 2026-09-27 | trinity-enterprise#706 (journey impact): the sync-health dot and the schedule freeze change their definition of "failing". A work agent diverged from origin for more than 24 h is now red and, when its owner opted in, has its schedules paused, not only one whose pushes failed 3 times; the dot renders a backend-owned state and reason. Details: [git-sync-health.md](git-sync-health.md) §2b, §5. | | 2026-09-22 | #2957: git status reads porcelain -z; first unstaged path no longer loses its first character; rename path = new path + additive orig_path; quoted paths returned raw; decoded with `errors="backslashreplace"` so a non-UTF-8 filename (raw under `-z`) cannot 500 the status. | | 2026-09-13 | ent#615: remote URLs carry no credential; the `trinity` git credential helper resolves it per operation; conditional restart rewrite + fleet remediation sweep. | | 2026-09-13 | **The status read stops taking the index lock** (#2742): `git --no-optional-locks status --porcelain` on the `/api/git/status` read path only — the auto-sync commit path and the `sync`/`pull` bodies keep the plain, index-refreshing form because they are lock-serialized and proceed to stage/commit. The three shared helpers `_compute_ahead_behind` / `_get_pull_branch` / `_persist_last_remote_sha` move to `run_registered`, which also sweep-registers their children on the **locked** `sync_to_github` / `pull_from_github` / 409 paths (intended, and pinned by a test). `remote_url` is now unconditionally redacted — the old shape special-cased `@github.com` and returned any other remote's URL verbatim, tokens included. Full reasoning: [git-sync-health.md](git-sync-health.md). | diff --git a/docs/memory/feature-flows/mcp-git-tools.md b/docs/memory/feature-flows/mcp-git-tools.md index 2a68db05a0..8d27df1ace 100644 --- a/docs/memory/feature-flows/mcp-git-tools.md +++ b/docs/memory/feature-flows/mcp-git-tools.md @@ -34,7 +34,7 @@ sync/recovery without an LLM turn and investigate exactly what a tool did. | **MCP tool** | `git_sync` | Stage + commit + push to working branch. Owner-only. `strategy: normal\|pull_first\|force_push`. | | **MCP tool** | `get_git_log` | Recent commits (limit clamped 1–100, default 10). Read-only. | | **MCP tool** | `git_pull` | Pull from GitHub. `strategy: clean\|stash_reapply\|force_reset`. Owner + shared accessors. | -| **MCP tool** | `get_git_sync_state` | Persisted `agent_sync_state` row (#389) — last status, consecutive failures, ahead/behind. Read-only. | +| **MCP tool** | `get_git_sync_state` | Persisted `agent_sync_state` row (#389) — last status, consecutive failures, ahead/behind, and (ent#706) `dirty_files` / `diverged_since` / `last_successful_push_at` plus the backend's `state` / `reason` / `recommendation` / `binding` / `freeze`. A pass-through of the backend JSON. Read-only. | | **MCP tool** | `reset_to_main_preserve_state` | ⚠️ DESTRUCTIVE recovery (#384) — adopt `origin/main`, force-with-lease. Owner-only. | | **API** (admin) | `GET /api/audit-log?request_id=` | Join an MCP `mcp_operation` row to the backend `git_operation` row it triggered. | diff --git a/docs/memory/feature-flows/operating-room.md b/docs/memory/feature-flows/operating-room.md index 1f45a5108d..2fbd28b9e1 100644 --- a/docs/memory/feature-flows/operating-room.md +++ b/docs/memory/feature-flows/operating-room.md @@ -1,6 +1,7 @@ # Operations (OPS-001) > **Status**: Implemented (Phases 1-4) +> **Updated 2026-09-29 (trinity-enterprise#715)**: machine keys read the queue without a person's identity — an allowlist projection on the get/list routes, the platform's heads-ups about a person hidden from them, no `responded_by` in the file write-back, no email on the clear-resolved trigger. See "Person fields withheld from machines" below. > **Updated 2026-09-25 (trinity-enterprise#611, PR B)**: agents RAISE asks natively — `POST /api/agents/{name}/operator-queue` and MCP `ask_operator` (self-acting) into `ask_service.raise_ask`: named 422s, replay of the first receipt, a 15-minute deadline floor, a re-ask linked to the agent's own expired ask (`supersedes_expired`, 422 `reask_requires_link` without it), role addressing, an atomic per-agent depth cap. The row is outside the file contract; the poller skips a file entry re-using a native id, counts only what it inserted, and logs the file channel's deprecation. The prompt leads with `ask_operator`. See [Raising an ask](#raising-an-ask-trinity-enterprise611-pr-b). > **Updated 2026-09-25 (trinity-enterprise#611, PR A)**: how an ask ENDS (OPS-001-ENDINGS). Every ending — the operator answer, the Workspace answer, single cancel, bulk cancel, expiry — goes through one sink, `services/ask_service.py` (compare-and-set → one audit row per transition → one thin WS trigger → ending observers). An endings ledger on the row (`disposition` answered|cancelled|expired, `disposed_at`, `disposed_by` person|timeout, `disposed_by_email`, `disposition_reason`, `batch_id`; migration `operator_queue_ask_object` / Alembic `0076`, which also adds the six agent-raised-ask columns PR B writes). Only a person ends an ask (403 `person_required` for agent/system/other keys). An answer after `expires_at` is 409 `expired`. The ent#329 wake fires on ANY ending (new trigger `operator_ending` for cancel/expiry). An agent reads its own ask back by `request_id` (`GET /api/agents/{name}/operator-queue/{request_id}`, MCP `get_my_ask`), and every composed turn lists the endings of the last 24 h. Surfaces show who and when. See "Endings" below. > **Updated 2026-09-23 (#2915)**: the file sync tells the truth (OPS-001-HONEST). Eight nullable columns record what the poller last established about the agent's file entry (`sync_state` / `sync_detail` / `sync_updated_at` / `last_confirmed_at`), whether an answer reached it (`delivery_state` / `delivery_detail` / `delivery_updated_at`) and whether the human answered a diverged item knowingly (`divergence_acknowledged_at`); the write-back delivers only into a matching, still-pending entry, re-reads before writing and writes with `if_match` (the agent server's `PUT /api/files` is atomic and answers 412); `POST …/respond` and the portal answer return **409 `item_diverged`** unless `acknowledge_divergence` is set; every list/get carries server-computed `aging` / `aged_since` and the list carries `undelivered_count` / `closed_by_filer_count`; one `operator_queue_sync` WS trigger per cycle; audit rows for the accountability transitions only. See "Sync honesty" below. @@ -316,12 +317,11 @@ class ClearResolvedRequest(BaseModel): # #1017 "type": "operator_queue_responded", "data": { "id": "", - "agent_name": "", - "responded_by_email": "", - "response": "" + "agent_name": "" } } ``` +Thin since trinity-enterprise#611 (listeners refetch); neither the answer nor who gave it rides on `/ws`. **`operator_queue_cleared` broadcast payload** (#1017, line 146-153 / 195-202 — one event per bulk operation, not per item): ```json @@ -329,11 +329,11 @@ class ClearResolvedRequest(BaseModel): # #1017 "type": "operator_queue_cleared", "data": { "scope": "pending|resolved", - "count": 7, - "cleared_by": "" + "count": 7 } } ``` +The count only: a fleet-level trigger reaches every `/ws` connection and any key can mint a ticket, so the acting operator's email is on neither copy (trinity-enterprise#715). ### Sync Service @@ -369,9 +369,9 @@ Accepts `terminal_items` (#1017 — cancelled/expired) and a `file_exists` param 1. Builds lookup maps of responded items and terminal items by ID 2. Iterates agent's JSON requests, updates matching pending items with response data 3. **Terminal-status propagation** (#1017, line 235-238): file entries still in `status=pending` whose ID is in the terminal map are flipped in place to the item's DB status (`req["status"] = terminal_map[req_id]["status"]` — `cancelled` or `expired`) so the agent stops waiting on them (and so a stale 'pending' file entry can't resurrect the item if its row is ever purged). Terminal entries are **never appended** if missing from the file (unlike responded items) — there is nothing to deliver. -4. **Reconstruction of missing items** (line 242-260): For any responded item in the DB whose ID is not found in the agent's `requests` array (tracked via `seen_ids` set), the service reconstructs a full request entry from DB data and appends it to the `requests` array. Reconstructed fields include: `id`, `type`, `status` (set to `"responded"`), `priority`, `title`, `question`, `options`, `context`, `created_at`, `response`, `response_text`, `responded_by`, `responded_at`. +4. **Reconstruction of missing items** (line 242-260): For any responded item in the DB whose ID is not found in the agent's `requests` array (tracked via `seen_ids` set), the service reconstructs a full request entry from DB data and appends it to the `requests` array. Reconstructed fields include: `id`, `type`, `status` (set to `"responded"`), `priority`, `title`, `question`, `options`, `context`, `created_at`, `response`, `response_text`, `responded_at` — never `responded_by` (trinity-enterprise#715: the agent's file carries the answer, not who gave it; an entry written before keeps its old value). Platform alarms (`is_platform_minted`) are never reconstructed. 5. Writes updated JSON back to agent via `client.write_file(QUEUE_FILE_PATH, content, timeout=10.0, platform=True)`; on success logs "Wrote N responses and M terminal-status flips back to {agent}" (line 277-280) -6. Sets `status=responded`, `response`, `response_text`, `responded_by`, `responded_at` fields in the agent's JSON +6. Sets `status=responded`, `response`, `response_text`, `responded_at` fields in the agent's JSON (no `responded_by` since trinity-enterprise#715) 7. The agent server's `write_file` endpoint (`docker/base-image/agent_server/routers/files.py:360`) automatically creates parent directories (`mkdir(parents=True, exist_ok=True)`), so `.trinity/` is created if it doesn't exist. **WebSocket broadcast payloads**: @@ -454,13 +454,13 @@ Writers are edge-triggered (`WHERE sync_state IS NULL OR != :v …`; the rowcoun **The agent's read path.** `GET /api/agents/{name}/operator-queue/{request_id}` + MCP `get_my_ask` (see Endpoints), and an `Ended asks (last 24 h)` line in every platform-composed turn's Execution Context (`platform_prompt_service._resolve_ended_asks`: up to 5 request ids + disposition + time, platform alarms excluded, fail-soft). The pull-worker path does not compose the platform prompt (#1629), so pull-claimed turns do not get the line. -**Person fields withheld from machines.** `disposed_by_email` and `resolved_to` are removed from `GET /api/operator-queue`, `/{id}` and `/agents/{name}` for any non-person principal; the pre-existing `responded_by_email` / `addressed_to_email` still pass (registered residual) — except `addressed_to_email` on a NATIVE ask (PR B), which holds the person the platform resolved a role to (the email `resolved_to` also carries) and is withheld as well. +**Person fields withheld from machines (trinity-enterprise#611, #715).** `GET /api/operator-queue`, `/{id}` and `/agents/{name}` give any non-person principal (agent, system, ops, connector, portal_delegate and whatever scope ships next) an ALLOWLIST projection, `routers/operator_queue.py::_MACHINE_ROW_FIELDS`: no `responded_by_id`, `responded_by_email`, `addressed_to_email` (a file entry's too: the agent wrote it, but the same key reads its siblings' rows), `disposed_by_email` or `resolved_to`. A column added later stays out until someone classifies it. The platform's heads-ups ABOUT a person (`operator_queue_service.is_about_a_person`: the ent#499 problem report and the ent#308 inbox collision, both reserved prefixes) are not returned to a machine at all: skipped on list, 404 on the item route. A person is `is_person_principal`: a JWT session or the person's own user-scoped key, so a machine that holds a user-scoped key (an ops dashboard) passes by design. That is the queue's rule. `routers/reports.py` withholds a report's audience from EVERY key, the person's own included (`is_interactive_principal`), because it governs what an MCP tool result may carry; the queue withholds a person's identity from machines, and a person's own key is a person reading their queue. **Surfaces.** ONE rule, `utils/operatorQueue.js::queueEnding(item) → {kind, label, who, when} | null` (+ `queueEndingText`, `queueEndingSortTime`, `recentlyEnded`): ResolvedCard shows who and when (relative, absolute on hover) and the cancel reason; the resolved feed and the list API order ended rows by ending time (#627 AC6); `/m` gets a "Recently ended" strip from the fetch it already made; the Workspace (`include_ended`, read past the operator's Clear All) lists asks that ended in the last 7 days on the agent page and in chat with a coarse who (`you` / `the operator` / `timeout`) — never an operator email or the reason — with no answer controls, while the sidebar count and the Work tab's "Waiting on you" (`pending-only`) stay pending-only; an answered ask stays listed as the server's `answered` projection. The wake opt-in copy (ReliabilityPanel) and the Clear-All confirmation say only what the platform delivers. **Guards.** `test_ent329_operator_resume.py` G2 re-pinned to the sink: both spawners' only call site is `services/ask_service.py` (separate floors), every function feeding `_ended(` must refuse on `_status_conflict` in an `if` or take its rows from a named set-CAS accessor, and only `_ended` iterates the observers. The #2376 writer guard names the sink as the one writer. The route-level test (pending A + already-cancelled B ⇒ the wake gets only A) and three call-site mutations are in `tests/unit/test_ent611_ask_endings.py`. -**Residuals.** Pre-existing person emails to agent keys on get/list; accepted wake loss on a crash; no native path for ephemeral agents; the pull path lacks the context line; the router's clear-resolved trigger still carries `cleared_by`. +**Residuals.** The answer and ending wakes dispatch with `source_user_email` = the person who ended the ask, so the turn's Execution Context names them and the execution row's `source_user_email` is readable by keys over the executions API (the platform-wide "who triggered this turn" attribution every user-triggered turn carries; registered, not a queue read); REST row breadth for an admin-owned agent key (MCP narrows to {self} ∪ permitted; trinity-enterprise#719); accepted wake loss on a crash; no native path for ephemeral agents; the pull path lacks the context line. ### Raising an ask (trinity-enterprise#611, PR B) @@ -480,7 +480,7 @@ Writers are edge-triggered (`WHERE sync_state IS NULL OR != :v …`; the rowcoun **The row.** `channel=mcp` and `raised_by=agent` (a gate's: `gate` and `gate`; a `gate-` row is platform-minted, so the ent#329 wakes skip it and the gate resumes its own call), `to_role`, `resolved_to`, `proposal`, `supersedes_expired` (the predecessor's uuid), `delivery_state=not_applicable` / `delivery_detail=mcp_raised`. It sits outside the file contract everywhere (`_file_contract()` = `channel` NULL or `file`): the sync index lists its id as `foreign`, the responded and terminal write-backs and both sweeps never touch it, and Clear All hides it once answered, since there is nothing left to deliver. -**The receipt.** `status`, `id`, `request_id`, `raised_by`, `channel`, `type`, `to_role`, `resolved`, `ask_status`, `disposition`, `disposed_at`, `expires_at`, `wakes_on_ending` (the owner's wake opt-in; always false on a platform-minted row, which the wakes skip), `supersedes_expired` (the predecessor's `request_id`; `get_my_ask` names it the same way), plus `differs` on a replay. It names the ROLE, never an email — and so do the reads: on a native row `addressed_to_email` is withheld from every non-person principal, like `resolved_to`. +**The receipt.** `status`, `id`, `request_id`, `raised_by`, `channel`, `type`, `to_role`, `resolved`, `ask_status`, `disposition`, `disposed_at`, `expires_at`, `wakes_on_ending` (the owner's wake opt-in; always false on a platform-minted row, which the wakes skip), `supersedes_expired` (the predecessor's `request_id`; `get_my_ask` names it the same way), plus `differs` on a replay. It names the ROLE, never an email — and so do the reads: `addressed_to_email` is withheld from every non-person principal, like `resolved_to` (on every row since trinity-enterprise#715). **The file channel.** Kept for two releases. The poller skips a file entry re-using a native id before any branch reads it (logged once per agent and id), creates through `create_operator_queue_item_with_outcome` and never counts, announces or audits a row it did not insert, and logs the file channel's deprecation (naming `ask_operator`) on the first ingest per agent per process. @@ -497,7 +497,7 @@ Two independent bounds plus field hygiene, applied in-order per new `pending` re 1. **DEPTH cap (primary, DB-measured ⇒ Redis-independent).** `pending = db.count_operator_queue_pending_for_agent(agent)` is computed once per cycle; a new item is admitted only while `pending + admitted < OPERATOR_QUEUE_MAX_PENDING_PER_AGENT` (25). At the cap the scan **stops** (`break`) — it does not keep dripping a growing file (that would re-scan on every 5s cycle → per-cycle DoS). Because it is DB-measured it holds through a Redis outage, which is why fail-open on the rate limiters is safe. Depth is the fatigue metric: a rate-aware drip that never trips the rate limit still can't accumulate more than 25 pending items. 2. **RATE cap (burst smoothing, Redis, fail-open).** Per-agent `rate_limiter.check("operator_queue_create:{agent}", 60, 60)` + fleet `check("operator_queue_create:_fleet", 300, 60)`, charged only at the real create point. Denied → the item is **held** this cycle and the scan `break`s (the window is monotonic within a cycle). The fleet cap bounds a colluding / shared-upstream-injected fleet in aggregate. 3. **Field hygiene** (`_clamp_ingested_item`, a total helper run **inside** the #1525 create try/except so any failure is quarantined, never hot-looped): `title`→300 / `question`→4000 chars **truncate-with-marker** (losing a real approval is worse than a clamped one); `context` serialized >8 KiB or non-serializable → `{"_truncated":true,"_original_bytes":N,"execution_id":}` (so a huge `execution_id` can't smuggle past the context cap); non-dict `context` → `{}`; `options` >4 KiB → a small dropped-marker list; `priority` validate-only (unknown → `medium`, legit `critical` untouched — the depth cap already bounds critical *volume*); **`created_at` normalized to ingest time** (an agent-supplied future date would pin the item atop the `created_at DESC` sort). `expires_at` is honored as authored. -4. **Reserved-id guard + malformed-id reject.** An agent item whose `id` starts with a platform-reserved prefix (`queue-flood-`, `poison-`, `cb-dormant-`, `sync-failing-`, `git-bloat-`, `skill-not-found-`, `val_`, `system-seed-`, `base-image-stale-`, `alert-budget-` (#1677), `sub-headroom-` (ent#434), `skills-legacy-adoption-` (#2744)) is **rejected** so an agent can't pre-create — and via `create_item`'s `on_conflict_do_nothing` silence — its own flood alarm or the #1402 poison alert. An `id` >256 chars or not matching `^[A-Za-z0-9._:-]+$` is rejected (a create PK can't be safely rewritten). +4. **Reserved-id guard + malformed-id reject.** An agent item whose `id` starts with a platform-reserved prefix (`queue-flood-`, `poison-`, `cb-dormant-`, `sync-failing-`, `git-bloat-`, `skill-not-found-`, `val_`, `system-seed-`, `base-image-stale-`, `alert-budget-` (#1677), `sub-headroom-` (ent#434), `skills-legacy-adoption-` (#2744), `portal-inbox-collision-` (trinity-enterprise#715)) is **rejected** so an agent can't pre-create — and via `create_item`'s `on_conflict_do_nothing` silence — its own flood alarm or the #1402 poison alert. An `id` >256 chars or not matching `^[A-Za-z0-9._:-]+$` is rejected (a create PK can't be safely rewritten). 5. **Flood alert.** A depth-held / rate-skipped / malformed-id / oversize-file episode emits **one** `type:"alert"` item per episode via a platform **direct-DB create** (exempt), id `queue-flood-{agent}-{utc_now_iso()}` (**un-guessable** — the agent can't pre-suppress it), priority `high`, softened wording, in-memory `OPERATOR_QUEUE_FLOOD_ALERT_COOLDOWN_SECONDS` (300) cooldown, wrapped so an emit failure never kills the sync. The single leader ⇒ no WS double-broadcast. **Deferred/held semantics.** Held items are **not dropped** — they stay `pending` in the agent's file and are re-evaluated on a later cycle once depth/rate frees up (or after the operator resolves existing items). This is a behavior change agents must understand: an agent-supplied `created_at` is ignored, and a burst beyond the caps is throttled rather than fully ingested at once. @@ -642,7 +642,7 @@ Agents write to `~/.trinity/operator-queue.json`: **Request ids are per-agent scoped (#1631, shipped)** — the item `id` is now a platform-minted `uuid.uuid4().hex` (the global row handle; REST/MCP/frontend lookups unchanged), and the agent-authored correlation string lives in a nullable `request_id` column with `UNIQUE(agent_name, request_id)`. `create_item` mints the uuid, stores `request_id`, and targets `on_conflict_do_nothing` at `(agent_name, request_id)` — a same-agent re-insert stays idempotent (re-reads and returns the surviving row's uuid) while two agents reusing the same string no longer collide (previously the second agent's item was silently swallowed by the global PK). The sync loop's `item_exists`, `mark_acknowledged`, the `_create_failures` quarantine map, and `_write_responses_to_agent` are all keyed on `(agent_name, request_id)`, closing the cross-agent write where agent B's acknowledgement flipped agent A's row. A guard rejects (log-once, bounded) agent-authored ids using reserved platform prefixes so an agent can't pre-claim the platform's own alert ids (the guarded prefix set is extended by #1632 — see the ingestion caps below). The #1402 contract still mandates execution-id-derived ids (`approval-{execution_id}-{slug}`), which make a re-park under pull-mode re-delivery idempotent. -**Ingestion caps (#1632, shipped).** Agent field values are bounded at ingestion: `title` ≤300 chars and `question` ≤4000 chars are truncated-with-marker; `context` >8 KiB and `options` >4 KiB collapse to a marker; an unknown `priority` becomes `medium`; **`created_at` is ignored and set to ingest time** (a future date can no longer pin the item atop the sort); the agent-authored `request_id` must be ≤256 chars and match `^[A-Za-z0-9._:-]+$`, and must **not** use a platform-reserved prefix (`queue-flood-`/`poison-`/`cb-dormant-`/`sync-failing-`/`git-bloat-`/`skill-not-found-`/`val_`/`system-seed-`/`base-image-stale-`/`alert-budget-`/`db-backup-`/`log-archive-`/`sub-headroom-`/`gitignore-untracked-`/`skills-legacy-adoption-`/`gate-`) or it is rejected. Per-agent pending depth is capped (default 25) and creation is rate-limited (60/60s per agent, 300/60s fleet); items over the cap are **held** (left `pending` in the file, re-evaluated on a later cycle), not dropped, and trigger one aggregated `queue-flood-*` operator alert. See "Ingestion caps (#1632)" under the Sync Service. These bounds apply **only** to the agent-authored file path — platform-created items (poison-park, validation alerts, internal breaker/skill/git alerts, the #1816 system-agent base-image staleness / start-failure alarms) bypass them; since #1677 the *agent-influenceable* platform emitters (`skill_not_found`, and `gitignore_untracked` since #2529) are additionally bounded by a per-(agent, type) pending budget via `create_bounded_alert`, with an `alert-budget-*` episode alert at the cap (see "Platform exemption & emitter budget (#1677)"). +**Ingestion caps (#1632, shipped).** Agent field values are bounded at ingestion: `title` ≤300 chars and `question` ≤4000 chars are truncated-with-marker; `context` >8 KiB and `options` >4 KiB collapse to a marker; an unknown `priority` becomes `medium`; **`created_at` is ignored and set to ingest time** (a future date can no longer pin the item atop the sort); the agent-authored `request_id` must be ≤256 chars and match `^[A-Za-z0-9._:-]+$`, and must **not** use a platform-reserved prefix (`queue-flood-`/`poison-`/`cb-dormant-`/`sync-failing-`/`git-bloat-`/`skill-not-found-`/`val_`/`system-seed-`/`base-image-stale-`/`alert-budget-`/`db-backup-`/`log-archive-`/`sub-headroom-`/`gitignore-untracked-`/`skills-legacy-adoption-`/`gate-`/`effect-unguarded-`) or it is rejected. Per-agent pending depth is capped (default 25) and creation is rate-limited (60/60s per agent, 300/60s fleet); items over the cap are **held** (left `pending` in the file, re-evaluated on a later cycle), not dropped, and trigger one aggregated `queue-flood-*` operator alert. See "Ingestion caps (#1632)" under the Sync Service. These bounds apply **only** to the agent-authored file path — platform-created items (poison-park, validation alerts, internal breaker/skill/git alerts, the #1816 system-agent base-image staleness / start-failure alarms) bypass them; since #1677 the *agent-influenceable* platform emitters (`skill_not_found`, and `gitignore_untracked` since #2529) are additionally bounded by a per-(agent, type) pending budget via `create_bounded_alert`, with an `alert-budget-*` episode alert at the cap (see "Platform exemption & emitter budget (#1677)"). ### Async contract — fire-and-park (#1402) @@ -996,6 +996,7 @@ Clear All on **Resolved** instead calls `clearResolved()` -> `POST /api/operator | Date | Change | |------|--------| +| 2026-09-29 | trinity-enterprise#715 — the reads stop handing a person's email to a machine key: the get/list routes give non-person principals the `_MACHINE_ROW_FIELDS` allowlist (no `responded_by_*`, `addressed_to_email` on any row, `disposed_by_email`, `resolved_to`); the platform's heads-ups about a person (ent#499 problem report, ent#308 inbox collision) are skipped for machines (404 on the item route) and `portal-inbox-collision-` is a reserved prefix, so the collision alert never reaches the agent's file or its resume turn; the file write-back no longer writes `responded_by` (both paths; the prompt now says `response`, `response_text` and `responded_at`); the clear-resolved `/ws` trigger is the count only. Tests: `tests/unit/test_ent715_queue_person_fields.py`. | | 2026-09-28 | trinity-enterprise#611 PR B, review round — the platform-reserved `gate-` prefix (required for a gate raise, refused for an agent's; gate rows never wake the agent), `raised_by` on the receipt, a provider's empty `primary` answer goes to the operators (ent#606), the rate check before any database read, a file entry racing the native insert replays instead of a 500, and `get_my_ask` names the predecessor by `request_id`. The `proposal` is rendered on every card that offers the decision (Operations, `/m`, the Workspace) and kept on the Resolved tab. Second pass: the gate row's receipt promises no wake, the channel goes with the raiser, a gate raise never replays a non-gate row (409), and the re-ask guard and link stay with their raiser. | | 2026-09-25 | trinity-enterprise#611 PR B — agents raise asks natively: `POST /api/agents/{name}/operator-queue` + MCP `ask_operator` (self-acting) into `ask_service.raise_ask` (named 422s, replay-first receipts with `differs`, the 15-minute deadline floor, `supersedes_expired` + `reask_requires_link`, role addressing through an optional provider, the #1632 rate buckets, an atomic per-agent depth cap); native rows outside the file contract; the poller skips re-used native ids, counts only inserted rows and logs the file's deprecation; the prompt leads with `ask_operator`; re-ask badges on the cards | | 2026-09-25 | trinity-enterprise#611 PR A — endings: one transition sink (`services/ask_service.py`) for every way an ask ends; the endings ledger (migration `operator_queue_ask_object` / Alembic `0076`, twelve nullable columns incl. PR B's); person-only endings (403 `person_required`); 409 `expired` for a late answer; cancel audited/raced/woken like respond with an optional reason; bulk returns its `batch_id` and audits the winners; expiry through the sink with the rider; the ent#329 wake on any ending (trigger `operator_ending`, one per agent per event, opted-in running agents only); the agent's own readback (REST + MCP `get_my_ask`) and the Execution Context "Ended asks" line; person fields withheld from machine keys; surfaces show who + when (ResolvedCard, `/m` strip, Workspace ended asks for 7 days); list ordered by ending time. Tests: `tests/unit/test_ent611_ask_endings.py`, re-pinned G2 / #2376 / ent#430 / ent#499 / #2048 guards, `operatorQueueEnding.spec.js`, `portalAskEndings.spec.js`, `mobileAdminRecentlyEnded.spec.js`, MCP `operator_queue.test.ts` + `access-wiring.test.ts`. | diff --git a/docs/memory/learnings/2026-09-27-a-kept-clock-keeps-its-counts.md b/docs/memory/learnings/2026-09-27-a-kept-clock-keeps-its-counts.md new file mode 100644 index 0000000000..e3cce5013f --- /dev/null +++ b/docs/memory/learnings/2026-09-27-a-kept-clock-keeps-its-counts.md @@ -0,0 +1,4 @@ +## 2026-09-27 — pitfall — a value kept on "unknown" must keep its companion values too + +**Context**: trinity-enterprise#706 review (#3035). `sync_health_service._sync_agent` decides `diverged_since` from the RAW payload, so an uncomputable working tuple (git timeout → `ahead_working: null`) correctly KEEPS the clock. The counts beside it still went through `_coerce_counter`, which reads null as 0, so the same row said "diverged for 25 h" and "0 behind / 0 ahead", and the freeze and its one operator-queue item carried that contradiction. +**Lesson**: When one column follows keep-on-unknown and its siblings follow coerce-unknown-to-0, the row is only coherent while every value is known. Any column that explains another (counts beside a clock, a reason beside a status) must take the same keep-on-unknown path in the same write, and the test must drive the unknown poll AFTER a known one and read both columns back. diff --git a/docs/memory/learnings/2026-09-29-guard-seam-swap-detaches-patches.md b/docs/memory/learnings/2026-09-29-guard-seam-swap-detaches-patches.md new file mode 100644 index 0000000000..d70abf0697 --- /dev/null +++ b/docs/memory/learnings/2026-09-29-guard-seam-swap-detaches-patches.md @@ -0,0 +1,3 @@ +## 2026-09-29 — pitfall — Rerouting a guard through a new private helper detaches every test patch on the old seam +**Context**: #2392 — `effect_guard` switched from `resolve_and_validate_execution` to a reason-returning private helper; seven test files monkeypatch the public name, and two `test_736` cases went red because their patch no longer reached the guard. +**Lesson**: When a widely-patched public function gains a sibling that returns more, keep the caller routed through the public name and derive the extra on the path that needs it. Before changing which function a hot path calls, `grep -rn "" tests/` — the patch count is the blast radius. diff --git a/docs/memory/learnings/2026-09-29-redacting-person-fields-misses-person-data-in-platform-text-and-ids.md b/docs/memory/learnings/2026-09-29-redacting-person-fields-misses-person-data-in-platform-text-and-ids.md new file mode 100644 index 0000000000..b45f397151 --- /dev/null +++ b/docs/memory/learnings/2026-09-29-redacting-person-fields-misses-person-data-in-platform-text-and-ids.md @@ -0,0 +1,3 @@ +## 2026-09-29 — pitfall — Withholding a row's person FIELDS misses the person data a platform alarm carries in its free text and in its id +**Context**: trinity-enterprise#715. The plan withheld `responded_by_*`, `addressed_to_email`, `disposed_by_email` and `resolved_to` from machine keys on the operator-queue reads, and its test scanned each row for `@example.com`. Independent review found two platform emitters that put a person in the TEXT of an ordinary row: the ent#499 problem report (`question` = the client's email plus their verbatim comment, which ent#366 deliberately withholds from the rated agent) and the ent#308 inbox-collision alert (`question` lists client addresses, and its `request_id` is built from an email slug). Every field-level rule, and the test's own fixture rows, passed straight over them. The collision id was also unreserved, so the file write-back, the resume wake and the Execution Context "Ended asks" line could each hand it to the agent. +**Lesson**: When a principal class must not learn who a person is, the census is the emitters, not the columns. List every writer of the table (the #1677 `_ALLOWED_CALLERS` inventory is that list for the operator queue) and ask of each: does its free text or its id carry a person? A row about a person is withheld whole, since per-field redaction cannot remove an email from prose or from an id. Its id prefix goes into `_RESERVED_ID_PREFIXES`, because that tuple is what every agent-facing sink (write-back, resume, sync exclusions, the ended-asks line) reads as "platform alarm". Drive the real emitters in the test, so a renamed id or type turns it red. diff --git a/docs/memory/requirements/core-agent.md b/docs/memory/requirements/core-agent.md index 6e040b6c80..8f750400f1 100644 --- a/docs/memory/requirements/core-agent.md +++ b/docs/memory/requirements/core-agent.md @@ -2419,7 +2419,8 @@ issue if it's ever wanted. Also deferred: `data.json` caching/streaming. the thing being criticised is a prompt-injection path into it — is why the operator's copy goes to the queue directly and the agent-facing redaction (`comment_withheld`) is untouched. The operator sees the comment; the agent - still does not. + still does not — including through its own key: the operator queue's reads + never return this item to a machine principal (trinity-enterprise#715). - **Routed through the budget, never allowlisted** (#1677). The volume here is driven by a *client* clicking, so this is an agent-influenceable emitter by the classification rule and goes through diff --git a/docs/memory/requirements/github.md b/docs/memory/requirements/github.md index da547eb074..8b98fcf265 100644 --- a/docs/memory/requirements/github.md +++ b/docs/memory/requirements/github.md @@ -76,6 +76,7 @@ - `SyncHealthService` emits `sync_failing` operator-queue entries at `consecutive_failures ≥ 3` (leader-leased across uvicorn workers, #2742 — `synchealth:leader`, fail-open. Note the counter is per *poll*, not per worker, so one leader instead of two **doubles** time-to-`sync_failing` from ~90 s to ~180 s; cadence knob `SYNC_HEALTH_POLL_INTERVAL_SECONDS`, default unchanged at 60 s) - `GET /api/agents/sync-health` (batch) + dashboard dot - `GET /api/fleet/sync-audit` with `duplicate_binding` flag (§P5 query) + - **Amended by §11.19 (trinity-enterprise#706):** health measures whether the agent and its repository *agree*, not only whether the last push succeeded. The state (green / yellow / red / unknown), its reason and the thresholds are computed by the backend from one policy module and served to every surface; `utils/syncHealth.js` no longer owns a threshold. The schedule freeze keys on divergence age for work agents as well as on `sync_failing`. - **Flow**: `docs/memory/feature-flows/git-sync-health.md` - **Upstream**: Epic #381 — sub-issues #389 (S1), #390 (S6) @@ -872,3 +873,30 @@ - **GitHub Issue**: abilityai/trinity-enterprise#705 --- + +### 11.19 Divergence Age and the Divergence Freeze (trinity-enterprise#706) +- **Status**: ✅ Implemented (2026-09-27) — the fleet-health / fleet-audit / MCP surfaces follow in trinity-enterprise#707 +- **Problem**: sync health measured whether the last push *succeeded*. An agent 15 commits ahead with auto-sync off never ran a heartbeat, so it never failed, and it rendered as healthy. The schedule freeze (#1808) fired only after three failed syncs, so a work agent whose repository had silently stopped receiving its work kept running on cron indefinitely. +- **Persisted on `agent_sync_state`** (all nullable, no backfill — the clocks start at the first poll after upgrade, which is also the 24 h soak before any divergence freeze can fire): + - `diverged_since` — the first poll at which the working tuple (origin on the agent's *own* branch, #2105) showed `ahead > 0 or behind > 0`. Set once, cleared when both are known 0 (a known `ahead == 0` with `behind` unknown means "no upstream" and also clears). An uncomputable `ahead` neither starts nor clears it. + - `dirty_files` — the porcelain change count the status call already computes (ignored paths excluded, untracked included). + - `dirty_since` — the first poll at which `dirty_files > 0`; cleared at 0. A separate episode from divergence (no carry-over when the dirt becomes a commit). + - `last_successful_push_at` — the later of the heartbeat's own record (#3011) and an operator Push (`agent_git_config.last_sync_at`); on an agent image older than #3011, the last `success` cycle. Monotonic; a future or unparseable agent value is rejected. +- **One policy, one module**: `services/sync_freeze_policy.py` (stdlib only), vendored byte-identically to `src/scheduler/sync_freeze_policy.py` with a parity test. Constants, not env knobs: 3 failed polls, 24 h divergence, 24 h dirt, 7-day no-heartbeat floor, 15-minute observation freshness. +- **Work agent** = `source_mode = 0 OR auto_sync_enabled = 1` (a working branch, or fork-to-own / bound-to-own that auto-syncs). Everything else is a *deployment*. Known gap: a fork-to-own or bound agent whose owner turned auto-sync off reads as a deployment. +- **State rules** (first red names the reason; the rest append): + - `unknown` — no row, or never observed. + - `red` — the last sync failed (any binding); a work agent diverged for more than 24 h; a work agent dirty for more than 24 h; auto-sync on with no heartbeat for 7 days (measured from the git binding's creation when no heartbeat was ever recorded). + - `yellow` — a work agent diverged for 24 h or less; a deployment diverged at any age, or dirty for more than 24 h. A deployment is never red on age and never freezes: a deployment being behind is normal. + - `green` — otherwise. + - The reason is operator-readable (`diverged 31 behind / 0 ahead for 26h`, `last sync failed (seen on 3 polls): …`, `dirty: 791 files uncommitted for 30h`, `auto-sync on, no heartbeat for 9d`), and a stale observation (last poll older than 15 minutes) is named in it. A recommendation names the fix (`credential is read-only`, `push via git_sync strategy=pull_first`, `deployment: turn auto-sync off`, `enable auto-sync`, `pull via git_pull`). +- **The freeze**: `should_freeze = freeze_schedules_if_sync_failing AND (sync_failing OR (work agent AND diverged > 24 h AND the observation is fresh))`. `sync_failing` is unchanged from #1808 and applies to any binding. The freshness guard fails open: the poller writes nothing for an unreachable agent, so a stale `diverged_since` must never keep an agent frozen after it may already have pushed. Enforced in two places that share the module: `GET /api/internal/agents/{name}/sync-health-status` (existing keys unchanged, plus `freeze_reason`, `divergence_age_s`, `work_agent`) and the scheduler's own DB gate, which stays fail-open and writes the reason into the skipped execution's `skip_reason`. Un-freezing is automatic: both points read live state, so the next cron tick after divergence clears fires. +- **One operator-queue item per episode**: when a divergence freeze is in force, the poller (inside its leader lease) raises one `sync_diverged` item with the deterministic id `sync-diverged-{agent}-{diverged_since}`; the `(agent_name, request_id)` conflict target collapses repeat polls, restarts and a fail-open double leader onto one row. The prefix is platform-reserved so an agent cannot pre-create the id and silence its own alarm. The item carries counts, the episode start and the recommendation — never raw error text. It is not auto-resolved; a new episode gets a new id. No item for red-but-not-frozen. +- **Surfaces (this slice)**: `GET /api/agents/sync-health` and `GET /api/agents/{name}/git/sync-state` (and therefore MCP `get_git_sync_state`, a pass-through) carry `state`, `reason`, `recommendation`, `binding`, `freeze`, the ages and the new columns; pre-existing keys are unchanged. The dashboard dot renders the backend's state and shows the reason on hover. The fleet-health and fleet-audit surfaces are trinity-enterprise#707. +- **Supersedes**: trinity-enterprise#142's dirty-tree advisory, by the persisted `dirty_files`. +- **Known limitations**: a failed `git status` inside the agent reads as a clean tree and clears `dirty_since` (agent-side; needs a base-image change). The agent writes every one of these numbers, so a compromised agent can misreport them (the existing #1595 trust model). With `SYNC_HEALTH_POLL_INTERVAL_SECONDS` raised above 15 minutes every observation is stale and the divergence freeze cannot fire (`sync_failing` still can). +- **Source of truth**: `services/sync_freeze_policy.py` (+ scheduler mirror), `services/sync_health_service.py`, `services/sync_health_view.py`, `db/sync_state.py`, `routers/internal.py`, `src/scheduler/database.py::sync_freeze_reason`. +- **Flow**: `docs/memory/feature-flows/git-sync-health.md` +- **GitHub Issue**: abilityai/trinity-enterprise#706 + +--- diff --git a/docs/memory/requirements/security.md b/docs/memory/requirements/security.md index 63283b393b..06833ee14d 100644 --- a/docs/memory/requirements/security.md +++ b/docs/memory/requirements/security.md @@ -583,7 +583,7 @@ - **No answer after the deadline:** the respond compare-and-set also requires `expires_at IS NULL OR expires_at > now`, so an answer that arrives after the deadline is **409 `expired`** even before the poller sweeps the row. `expires_at` is stored as ISO-8601 UTC with a `Z` suffix at file ingest (Invariant #16). - **Cancel is audited and raced like respond:** single cancel takes an optional `reason` (≤ 500 chars) and answers **409** when the row already left `pending`; bulk-cancel returns its sweep's `batch_id` and audits the ids it actually cancelled, not the ids requested. - **Wake on any ending (ent#329, same per-agent opt-in):** an answer keeps the existing `operator_response` resume. A cancel or an expiry wakes the agent that raised the ask with the new trigger `operator_ending` — one dispatch per agent per ending event, idempotency key over (agent, disposition, sorted item ids), the ending framed as data. An expiry says, verbatim: "Denied by timeout; do not re-ask the same action without new information." No wake for platform-minted rows, for rows the agent already closed on its side (`sync_state = closed_by_filer`), or for agents that are not running (audited `skipped_not_running`). A crash between the ending commit and the wake spawn loses that wake — accepted, because the readback and the Execution Context line still carry the ending. - - **Self-readback:** `GET /api/agents/{name}/operator-queue/{request_id}` and the MCP tool `get_my_ask` let an agent read one of its OWN asks by its own `request_id` — only as itself (the system key only as `trinity-system`), still readable after Clear All — through a redacted projection that never carries a person's email. The new person fields (`disposed_by_email`, `resolved_to`) are withheld from agent principals on the operator queue's get and list routes as well. + - **Self-readback:** `GET /api/agents/{name}/operator-queue/{request_id}` and the MCP tool `get_my_ask` let an agent read one of its OWN asks by its own `request_id` — only as itself (the system key only as `trinity-system`), still readable after Clear All — through a redacted projection that never carries a person's email. The operator queue's get and list routes give every non-person principal an allowlist projection with no person identity — no `responded_by_id`, `responded_by_email`, `addressed_to_email`, `disposed_by_email` or `resolved_to` — and never return the platform's heads-ups about a person (the ent#499 problem report, the ent#308 inbox collision) to one (trinity-enterprise#715). A person is a JWT session or the person's own user-scoped key (`is_person_principal`); report audiences keep their stricter every-key rule (`is_interactive_principal`), which governs what an MCP tool result may carry. The file write-back carries the answer, never who gave it (no `responded_by`), and the clear-resolved `/ws` trigger is the count only. Residual, registered: the answer and ending wakes still attribute the turn to the person (`source_user_email`), which the executions API returns to keys. - **Execution Context line:** every platform-composed turn lists the asks this agent raised that ended in the last 24 hours (ids, disposition, time; no human text). Bounded and fail-soft — a read failure omits the line, never the turn. - **Surfaces:** Operations (the resolved card shows who ended the ask and when; the resolved feed sorts by ending time), `/m` (a "Recently ended" strip), and the client portal (the in-chat card and the agent page list asks that ended in the last 7 days with a coarse who — you, the operator, or timeout — never an operator's email or the cancel reason; the sidebar count stays pending-only). The Clear-All copy states only what the platform delivers. The wake opt-in copy says it fires "when an ask it raised ends — answered, cancelled or expired". - **Paging the Workspace asks read (#3059):** `GET /api/enterprise/client-portal/asks` is paged, not capped. The body stays a **list** (no current caller changes); `limit` (1–200, default 200) and an opaque `cursor` are opt-in query parameters, and every response carries `X-Total-Count` (the viewer's whole visible set) and, when more remain, `X-Next-Cursor`. The total and the pages are honest because every filter the viewer's visibility depends on — the addressee, the visible kinds, the ended-ask window and the read-time roster re-check — is applied in SQL **before** the limit; the roster predicate itself is unchanged (`agent_on_roster`, asked once per agent). The order is stable across pages — pending first by priority then age, then ended by ending time, then `id` as the tie-break — so pending asks are never dropped while ended ones are shown. An unreadable cursor is a named **422 `invalid_cursor`**. @@ -606,7 +606,7 @@ - **The proposal is shown where the decision is made:** every card that offers the decision renders `proposal` read-only and expanded — the Operating Room card, the `/m` ops card and the Workspace ask — and the Resolved tab keeps it, so the record says what was decided (the `WorkspaceAsk` projection names it; `context` stays off) — as text, never markup. The prompt tells agents to put the exact action there so the person can verify what they are approving; before this, nothing rendered it. - **Addressed by role (`to`):** a registered assignment provider answers first, and an empty answer means nobody fills the role: for `primary` the ask then goes to the operators (the ent#606 fallback; the owner stands in only when no provider answers at all). With no provider, `primary` (the default for approval and question) is the agent's owner, `operator` (the default for alert) names no person, and `approver` / `viewer` are **422 `role_unassigned`** until someone fills them. An owner without an email turns a `primary` ask into an operator ask and the receipt says so (`resolved: false`); several people are recorded but none becomes the single Workspace addressee. - **Caps:** the file path's #1632 rate buckets (**429 `rate_limited`**), checked right after the replay and before every check that reads the database, so a refused ask spends a token and the re-ask scan cannot be repeated for free (a replay spends none); and the per-agent depth cap made atomic — replay check, count and insert under one per-agent lock — (**429 `queue_full`**). A file entry re-using the id that lands between the replay check and the insert (the poller does not take the lock) makes the insert a replay of that row, never an error. - - **Receipt:** `status`, `id`, `request_id`, `raised_by`, `channel`, `type`, `to_role`, `resolved`, `ask_status`, `disposition`, `disposed_at`, `expires_at`, `wakes_on_ending` (the owner's wake opt-in), `supersedes_expired` (the predecessor's `request_id`), `differs` on a replay — the role, never a person's email. The reads keep the rule: on a native row the operator queue's get and list routes withhold `addressed_to_email` (the resolved person, like `resolved_to`) from every non-person principal. + - **Receipt:** `status`, `id`, `request_id`, `raised_by`, `channel`, `type`, `to_role`, `resolved`, `ask_status`, `disposition`, `disposed_at`, `expires_at`, `wakes_on_ending` (the owner's wake opt-in), `supersedes_expired` (the predecessor's `request_id`), `differs` on a replay — the role, never a person's email. The reads keep the rule: the operator queue's get and list routes withhold `addressed_to_email` (the resolved person, like `resolved_to`) from every non-person principal — on every row since trinity-enterprise#715. - **Provenance and the file contract:** `raised_by` and `channel` (`mcp` for an agent's raise, `gate` for a gate's; the column also holds `file`) come from the call, never from content. A native row is outside the file contract everywhere: the poller skips a file entry re-using its id, and the write-backs, the stopped-agent sweeps and the delivery bookkeeping never touch it; Clear All hides it once answered. - **Audit** (`event_type=operator_queue`, ids and enums only): `raised` (`source` api or system), attributed to the agent — never to the owner its key resolves to — with the presented key and the owner's email as the join back to the human; plus one thin `operator_queue_new` trigger. - **File compatibility (two releases):** the poller keeps ingesting the file, creates through an outcome-reporting accessor and never counts, announces or audits a row it did not insert, and logs the file channel's deprecation — naming `ask_operator` — once per agent per process. diff --git a/docs/planning/PULL_MIGRATION_STATUS.md b/docs/planning/PULL_MIGRATION_STATUS.md index c1fe4ed56b..394cada974 100644 --- a/docs/planning/PULL_MIGRATION_STATUS.md +++ b/docs/planning/PULL_MIGRATION_STATUS.md @@ -51,16 +51,17 @@ The spec names the gates (`TARGET_ARCHITECTURE.md`, §Re-Delivery and Side-Effec > Default-on for effect-bearing agents is still gated on trace fidelity (#548/#333), `prior_trace` > injection (#1401), and **fail-closed `execution_id` injection**. -**Two of those three are shipped.** The full remaining list, in order: +**All three are built** (fail-closed injection by #2392). The full remaining list, in order: 1. **Land Phase 4** — [#2532](https://github.com/abilityai/trinity/pull/2532). Rebased on `dev`, migration renumbered to `0059`, full unit suite matched against unmodified `dev` (same single pre-existing failure, 21 net new tests). Blocked only on review. 2. **Fail-closed `execution_id` injection** — [#2392](https://github.com/abilityai/trinity/issues/2392). - **The policy is decided, not open**: the spec says fail-closed. What is missing is the build — - platform-side injection, reject + operator alarm when the id is still absent, and a regression test that - a re-delivered execution emits each effect once. *(Trace fidelity #548/#333 closed Aug/Jun; `prior_trace` - injection #1401 closed 2026-07-08; #1402 closed 2026-07-26.)* + The agent's MCP config sends each turn's id as `X-Trinity-Execution-Id`; on a pull-mode agent an effect + without a usable id is refused with an operator alarm, and a person's terminal session (`manual`) is sent + and logged. Pilots must run a base image with #2392, or every effect call whose model omits the id is + refused. *(Trace fidelity #548/#333 closed Aug/Jun; `prior_trace` injection #1401 closed 2026-07-08; + #1402 closed 2026-07-26.)* 3. **A soak on an agent that actually emits.** The current pilot (`cornelius-oracle` on eu2) emits no messages, calls or shares — measured 2026-09-02: 219 `idempotency_keys` rows, all `agent:*`, zero `effect:*`. It has therefore never entered the code path item 2 protects, so a clean window on it is not diff --git a/docs/planning/PULL_MIGRATION_TESTING.md b/docs/planning/PULL_MIGRATION_TESTING.md index 65b5e359e5..1e606f2359 100644 --- a/docs/planning/PULL_MIGRATION_TESTING.md +++ b/docs/planning/PULL_MIGRATION_TESTING.md @@ -189,7 +189,7 @@ Run on **both** engines (SQLite supported until EOS 2026-09-01), even though loc |----|------|----------| | T6.1 | Re-delivery preserves `execution_id` | Requeue and park both keep the row id. | | T6.2 | effect_guard dedup across re-delivery | Same `execution_id` re-run → `send_message`/`create_share`/`voip` de-duped. | -| T6.3 | **effect_guard fail-open when `execution_id` absent** | Without trusted injection a re-run **double-emits**. Must close before default-ON for side-effect agents. **Policy is decided — `TARGET_ARCHITECTURE.md` says fail-closed; #2392 is the build, not a decision.** | +| T6.3 | **effect_guard without a usable `execution_id`** | Platform-injected via the MCP header (#2392). A pull-mode agent's effect with no usable id is refused + alarmed; a `manual` terminal session is sent and logged. Unit: `test_2392_effect_guard_fail_closed.py`. Live positive control: the eu2 emitting arm (a pull-mode pilot agent) once it runs an image with #2392. | | T6.4 | Nevermined settle exactly-once | Duplicate re-delivery → single settle on native `agent_request_id` token. | ### TIER 7 — PostgreSQL (now the LOCAL backend) — P0 @@ -346,7 +346,7 @@ the moment the lease-reaper became eligible and stayed red until its next sweep. the reaper never touches still fires — that is M4's automated owner (§9). Canary lease-awareness is now complete: S-01 and E-05 exclude leased rows, E-01 grace-bounds them; E-02 deliberately does neither (a terminal→non-terminal reversal is corruption regardless of ownership, and pull is the more exposed path) · -Tier-6 `effect_guard` `execution_id` injection (**#2392** — build fail-closed injection + operator alarm; the policy itself is already decided in the spec, do not re-open it) · ~~G3 canary-on-PG (#1540)~~ ✅ closed · +~~Tier-6 `effect_guard` `execution_id` injection~~ ✅ **built by #2392** — the id reaches the sinks through the agent's MCP header, and a pull-mode agent's effect without a usable id is refused with an operator alarm (§3 T6.3) · ~~G3 canary-on-PG (#1540)~~ ✅ closed · B6 runtime-verify on the rebuilt image · the ≥2-week soak (#856 / #1766, measurement set in §9). **Also closed by #1766:** the pilot flag was purely additive, so a pilot ran push AND pull concurrently — diff --git a/docs/requirements/OPERATOR_QUEUE_OPERATING_ROOM.md b/docs/requirements/OPERATOR_QUEUE_OPERATING_ROOM.md index 7454295473..a99462c04e 100644 --- a/docs/requirements/OPERATOR_QUEUE_OPERATING_ROOM.md +++ b/docs/requirements/OPERATOR_QUEUE_OPERATING_ROOM.md @@ -48,7 +48,6 @@ Each agent maintains a file at `~/.trinity/operator-queue.json` in their workspa "expires_at": "2026-03-07T11:00:00Z", "response": null, "response_text": null, - "responded_by": null, "responded_at": null } ] @@ -71,7 +70,7 @@ Each agent maintains a file at `~/.trinity/operator-queue.json` in their workspa | `expires_at` | ISO 8601 | No | Optional deadline. Platform marks as expired if no response by this time. | | `response` | string | No | For `approval`: the selected option. For `question`: freeform text. For `alert`: `"acknowledged"`. Written by platform. | | `response_text` | string | No | Optional additional notes from the operator (for approvals that need explanation). Written by platform. | -| `responded_by` | string | No | Email of the operator who responded. Written by platform. | +| `responded_by` | string | No | No longer written (trinity-enterprise#715): the platform writes the answer, never who gave it. An entry written before keeps its old value. | | `responded_at` | ISO 8601 | No | When the operator responded. Written by platform. | ### 1.2 Request Types @@ -143,7 +142,7 @@ Each agent maintains a file at `~/.trinity/operator-queue.json` in their workspa 1. **Active items only** — The file should contain only `pending` and `responded` (not yet processed by agent) items, plus up to 3 most recent `acknowledged` items for immediate context. 2. **Agent writes** `pending` entries when it needs something. -3. **Platform writes** `responded` status with `response`, `responded_by`, `responded_at` fields. +3. **Platform writes** `responded` status with `response`, `response_text`, `responded_at` fields (never who answered — trinity-enterprise#715). 4. **Agent reads** responses, acts on them, then updates status to `acknowledged`. 5. **Agent cleans up** — removes `acknowledged` items older than the 3 most recent. The DB is the permanent record. @@ -288,9 +287,7 @@ New events broadcast via existing WebSocket infrastructure: "type": "operator_queue_responded", "data": { "id": "req-20260307-001", - "agent_name": "deploy-agent", - "responded_by_email": "eugene@ability.ai", - "response": "approve" + "agent_name": "deploy-agent" } } ``` diff --git a/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.json b/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.json new file mode 100644 index 0000000000..54e7ff1490 --- /dev/null +++ b/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.json @@ -0,0 +1,58 @@ +{ + "version": "1.1", + "date": "2026-09-29", + "mode": "daily", + "scope": "diff", + "confidence_gate": 8, + "branch": "feature/2392-inject-execution-id", + "issue": "abilityai/trinity#2392", + "phases_run": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 9, + 12, + 13, + 14 + ], + "attack_surface": { + "new_endpoints": 0, + "new_headers": [ + "X-Trinity-Execution-Id (MCP server inbound)" + ], + "validator_shapes_added": 1, + "new_error_paths": [ + "422 effect_unguarded" + ], + "new_alert_types": [ + "effect_unguarded" + ] + }, + "findings": [], + "fixed_during_audit": [], + "appendix": [ + "agent-typed execution_id 'manual' bypasses refusal (hostile-agent class, accepted)", + "Codex terminal / user-scoped key on pull agent refused (fail-closed, documented)" + ], + "supply_chain_summary": "no dependency changes", + "filter_stats": { + "candidates": 2, + "below_gate": 2, + "reported": 0 + }, + "totals": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0 + }, + "mutations": [ + "effect_dedup_required -> False: 5 refusal tests red" + ], + "trend": "n/a (diff mode)" +} \ No newline at end of file diff --git a/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.md b/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.md new file mode 100644 index 0000000000..abed4c6608 --- /dev/null +++ b/docs/security-reports/cso-diff-2026-09-29-2392-execution-id-injection.md @@ -0,0 +1,27 @@ +# /cso --diff — #2392 platform-injected execution id (2026-09-29) + +**Scope:** `feature/2392-inject-execution-id` vs `dev`. Daily gate: 8/10. +**Result:** 0 findings (0 critical, 0 high, 0 medium, 0 low). + +## Attack surface added +- **No new endpoint.** One new inbound header on the MCP server, `X-Trinity-Execution-Id`, read in `authenticate` on the validated-API-key path only. It is parsed against `^[A-Za-z0-9_.:-]{1,128}$`; anything else is dropped, never an error. The value is forwarded as the existing `execution_id` body field of five effect tools. +- **Agent MCP config shape.** The Trinity entry gains one header whose value is the literal `${TRINITY_EXECUTION_ID:-manual}` (Codex: `env_http_headers` naming the env var). The value carries no secret. +- **Validator.** `_is_canonical_trinity_entry` accepts one additional header set: exactly `{Authorization, X-Trinity-Execution-Id}` with that exact literal. The Bearer regex still applies. Both vendored copies are byte-identical (`cmp`). +- **Refusal path.** `EffectUnguardedError` → 422 `effect_unguarded` in five routers, plus a budgeted operator alarm. + +## Phases +- **Secrets:** no key patterns in the diff or the new files. +- **Enterprise docs guard:** 0 hits across changed docs. +- **Dependencies, CI, Docker:** unchanged. +- **Vendored parity (Invariant #5):** `mcp_validator.py` copies identical; parity test green. +- **Internal route:** `/api/internal/agent-files/share` stays on the `verify_internal_secret` router; the change only maps a new exception to 422. +- **Spoofing / A01:** the header cannot claim another agent's execution. `resolve_and_validate_execution` checks the execution's `agent_name` against the route's authenticated agent, and a mismatch resolves `foreign` (refused on a pull-mode agent, degraded log elsewhere). A user-scoped key that sends the header gains nothing the body field did not already allow. +- **Operator-visible text (G-04 class):** the alarm's `question` and `context` hold the agent name, a literal `effect_type` and a reason from a fixed set. No header or request text reaches it. The id prefix `effect-unguarded-` is reserved, so an agent cannot pre-create and suppress its own alarm. +- **Prompt surface:** tool descriptions changed wording only; no per-agent metadata added. + +## Mutations +- Pull-mode predicate forced to `False`: 5 refusal tests in `test_2392_effect_guard_fail_closed.py` go red. + +## Appendix (below the gate) +- **Accepted:** an agent holding its own key can call a sink directly with `execution_id: "manual"` and skip the refusal. The guard targets omission, not a hostile agent; closing this needs a per-execution credential (out of scope, recorded in the effect-idempotency flow). +- **Accepted:** Codex terminal sessions and user-scoped keys acting for a pull-mode agent carry no id and are refused. Fails closed; documented. diff --git a/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.json b/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.json new file mode 100644 index 0000000000..2614ad8cba --- /dev/null +++ b/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.json @@ -0,0 +1,122 @@ +{ + "version": "1.1", + "date": "2026-09-29", + "mode": "diff", + "scope": "all", + "confidence_gate": 8, + "branch": "feature/715-queue-person-emails", + "issue": "abilityai/trinity-enterprise#715", + "phases_run": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "attack_surface": { + "new_endpoints": 0, + "changed_endpoints": [ + "GET /api/operator-queue (machine principals: allowlist projection, rows about a person skipped)", + "GET /api/operator-queue/{item_id} (machine principals: allowlist projection; 404 for a row about a person)", + "GET /api/operator-queue/agents/{agent_name} (machine principals: allowlist projection, rows about a person skipped)" + ], + "changed_ws_payloads": [ + "operator_queue_cleared (router copy): drops cleared_by" + ], + "changed_agent_sinks": [ + "file write-back drops responded_by (delivery + reconstruction)", + "portal-inbox-collision- reserved: platform alarm for write-back, resume, sync exclusions, ended-asks line" + ], + "mcp_tools_changed": 0, + "migrations": 0 + }, + "findings": [], + "fixed_during_audit": [], + "appendix": [ + { + "status": "excluded", + "note": "get_queue_item pre-existing 404-then-403 split; ids are uuid4 hex, nothing enumerable; not introduced by the diff" + }, + { + "status": "registered", + "note": "old responded_by values in existing agent files: never scrubbed by ruling; cross-agent file reach is trinity-enterprise#719" + }, + { + "status": "registered", + "note": "execution rows' source_user_email readable by machine keys (debt:2026-09-29-executions-api-returns-person-emails-to-machine-keys)" + }, + { + "status": "registered", + "note": "notifications_cleared.cleared_by on /ws (debt:2026-09-29-notifications-cleared-trigger-carries-operator-email)" + }, + { + "status": "accepted", + "note": "operator-typed response_text / disposition_reason may name a person; by design the agent reads its answer" + } + ], + "supply_chain_summary": "no dependency changes", + "filter_stats": { + "candidates": 5, + "dropped_hard_exclusion": 1, + "registered_residual": 3, + "accepted_by_design": 1, + "reported": 0 + }, + "totals": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0 + }, + "mutations": [ + { + "mutation": "responded_by_email back on the machine allowlist", + "result": "42 red" + }, + { + "mutation": "about-a-person row filter removed", + "result": "3 red" + }, + { + "mutation": "item-route 404 for a hidden row removed", + "result": "3 red" + }, + { + "mutation": "responded_by restored in _deliver_into", + "result": "1 red" + }, + { + "mutation": "responded_by restored in the reconstruction", + "result": "1 red" + }, + { + "mutation": "portal-inbox-collision- un-reserved", + "result": "2 red" + }, + { + "mutation": "cleared_by restored on the /ws trigger", + "result": "1 red" + }, + { + "mutation": "responded_by restored in the prompt", + "result": "1 red" + }, + { + "mutation": "a new unclassified column on the row", + "result": "6 red (person-side classification test)" + } + ], + "live_verification": "operator eyeball on localhost at the branch commit: agent key list 0 alerts, no person-field keys, 0 example.com hits; item 404 for the agent key, 200 for JWT; /ws Clear All frame is scope + count only", + "trend": { + "note": "resolves the residual registered by cso-diff-2026-09-25-ent611-native-ask (file-row addressee to machine keys); no new findings" + } +} diff --git a/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.md b/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.md new file mode 100644 index 0000000000..78ef3e768a --- /dev/null +++ b/docs/security-reports/cso-diff-2026-09-29-ent715-queue-person-fields.md @@ -0,0 +1,60 @@ +# CSO Diff Audit — 2026-09-29 — abilityai/trinity-enterprise#715 (operator-queue reads without a person's identity) + +**Mode**: diff (all phases) · **Confidence gate**: 8/10 · **Branch**: `feature/715-queue-person-emails` → `dev` · **Skill**: cso v1.1 · **Audited**: the branch diff against merge-base `e87163125` (commit `3f13c6b26`). + +## Architecture (Phase 0) + +The change narrows what a MACHINE principal reads from the operator queue. A machine principal is any key that `dependencies.is_person_principal` refuses: agent, system, ops, connector and portal_delegate keys, a user-scoped key carrying an agent identity, and any future scope. + +- `routers/operator_queue.py`: the three read routes (`GET /api/operator-queue`, `/{id}`, `/agents/{name}`) give a machine an ALLOWLIST projection, `_MACHINE_ROW_FIELDS`, in place of a two-key pop list. The five person fields are outside it: `responded_by_id`, `responded_by_email`, `addressed_to_email`, `disposed_by_email` and `resolved_to`. +- `services/operator_queue_service.is_about_a_person`: the platform's heads-ups ABOUT a person are not returned to a machine at all (list skip, item 404). These are the ent#499 problem report (a client's email and verbatim comment in `question`) and the ent#308 inbox collision (client addresses in `question`, an email slug in the id). +- `portal-inbox-collision-` joins `_RESERVED_ID_PREFIXES`. `is_platform_minted` and every sink keyed on it now treat that alert as a platform alarm: the file write-back, the resume wake, the sync exclusions and the Execution Context "Ended asks" line. +- The file write-back no longer writes `responded_by`, on delivery or on reconstruction. +- The clear-resolved `/ws` trigger drops `cleared_by`. + +Trust boundary: persisted rows about people → keys held by machines (agents are prompt-injectable, and an admin-owned agent key reaches every agent's rows over REST). + +## Attack surface (Phase 1, diff-scoped) + +| Surface | Change | +|---|---| +| Endpoints | None new. Three existing reads return LESS to machine principals; the item route answers 404 for a hidden row (the same body as a missing id). | +| MCP tools | None changed. `list_operator_queue` / `get_operator_queue_item` pass the narrower backend response through; the item tool turns the 404 into a plain error. `types.ts` changes a comment only. | +| WebSocket | `operator_queue_cleared` (router copy) loses one key. | +| Agent-facing sinks | The file write-back writes one field fewer, and one more id prefix counts as a platform alarm. | +| Prompt | One sentence of the Operator Communication section (platform-authored, no caller input). | +| Migrations, dependencies, CI, Docker/compose, vendored policy files, skills | Unchanged. | + +## Findings (Phases 2–12) + +**None.** Every phase ran against the diff: + +- **Phase 2, secrets:** `git log -p e87163125..HEAD` added lines carry no known secret prefix, no private IP, and no email outside `@example.com`. The branch REMOVES one real internal address from a public requirements doc (an example payload in `docs/requirements/OPERATOR_QUEUE_OPERATING_ROOM.md`). +- **Phases 3–5, supply chain / CI / infra:** no dependency, lockfile, workflow, Dockerfile, compose or vendored-copy file is touched. +- **Phase 6, integrations:** no webhook or agent-server call path is touched. +- **Phase 7, LLM:** the change removes a prompt-injection path. The ent#499 problem report carried a stranger's verbatim words, which ent#366 deliberately withholds from the rated agent, and that agent's key could read them over the queue. No new model-facing input. +- **Phase 9, A01:** the machine rule is `is_person_principal`, an allowlist over `mcp_scope` that fails closed on a principal without one. Hidden rows answer 404 only after the access check, so they cannot reveal a row the caller could not already reach. A03: no SQL or command construction changed; the new predicate is a `startswith` over a normalised id. +- **Phase 10, STRIDE (operator-queue read surface):** information disclosure reduced; no change to spoofing, tampering, repudiation, DoS or elevation. +- **Phase 11, data:** person emails and client comments are CONFIDENTIAL (PII); they now stay on the person side of every queue read. + +### Candidates considered and not reported + +1. **The item route's pre-existing 404 (missing) → 403 (inaccessible) split** (Invariant #8 class). It predates this diff. Item ids are `uuid.uuid4().hex` (`db/operator_queue.py:420`), so the split leaks nothing enumerable. Excluded. +2. **Old `responded_by` values in existing agent files.** The write-back never scrubs history (a claim-time ruling). Reaching another agent's file is the REST breadth class owned by trinity-enterprise#719. Registered, not new. +3. **The execution row's `source_user_email`.** The answer and ending wakes still attribute the turn to the person, and the executions API returns that field to keys. Out of scope by ruling; registered in the debt inbox (`debt:2026-09-29-executions-api-returns-person-emails-to-machine-keys`). +4. **`notifications_cleared.cleared_by`**, the same `/ws` pattern in another router. Registered (`debt:2026-09-29-notifications-cleared-trigger-carries-operator-email`). +5. **Operator-typed free text** (`response_text`, `disposition_reason`) may name a person. By design: it is the answer the agent is meant to read. + +## Active verification (Phase 12) + +- **Unit, over the real routers and real SQLite:** `tests/unit/test_ent715_queue_person_fields.py` (63). It pins the machine key set as a literal across seven machine principals and one real minted agent key, and drives both platform emitters for real. Nine call-site mutations each turn it red. +- **Live, on localhost, by the operator** (main checkout detached at the branch commit, the ask answered through the UI): + - A person's JWT list shows `responded_by_email` and both alerts. + - The agent key's list shows 0 alerts, no person-field keys and 0 `example.com` hits. + - The item route answers the agent key 404 (the same body as a nonexistent id) and the JWT 200. + - The UI still names the answerer. + - The `/ws` Clear All frame carries `scope` and `count` only. + +## Trend + +This resolves the residual the ent#611 PR B diff audit registered (`cso-diff-2026-09-25-ent611-native-ask.md`: "A file entry's addressee is the agent's own input and stays"). No new findings. diff --git a/docs/testing/orchestration-invariant-catalog.md b/docs/testing/orchestration-invariant-catalog.md index a4bef363e3..97ce74d6f2 100644 --- a/docs/testing/orchestration-invariant-catalog.md +++ b/docs/testing/orchestration-invariant-catalog.md @@ -489,6 +489,7 @@ Signal: `SELECT count(*) FROM agent_sync_state WHERE (last_sync_status = 'failed **RD-03** A freeze freezes *(Tier A, 🟡)* — #389/#1808. When the owner opted in (`agent_git_config.freeze_schedules_if_sync_failing = 1`) and sync is failing (`agent_sync_state.last_sync_status = 'failed' AND consecutive_failures >= SYNC_FAILURE_FREEZE_THRESHOLD`, `src/scheduler/database.py`), the scheduler fires nothing for that agent: no `schedule_executions` row with `triggered_by = 'schedule'` is created after the failing state was recorded. The scheduler's own check is fail-OPEN (an error fires anyway — a freeze-on-error would stop the fleet), so this invariant is the only thing that notices a freeze that stopped freezing. Signal: `SELECT count(*) FROM schedule_executions e JOIN agent_git_config g ON g.agent_name = e.agent_name JOIN agent_sync_state s ON s.agent_name = e.agent_name WHERE g.freeze_schedules_if_sync_failing = 1 AND s.last_sync_status = 'failed' AND s.consecutive_failures >= AND e.triggered_by = 'schedule' AND e.started_at > s.updated_at` → must be 0. +Since trinity-enterprise#706 the freeze also fires for a **work agent** (`source_mode = 0 OR auto_sync_enabled = 1`) whose `agent_sync_state.diverged_since` is more than 24 h old, provided `last_check_at` is within 15 minutes (the freshness clause fails open). The signal above covers the `sync_failing` clause only; the divergence clause depends on the observation's age at fire time, so it is pinned by `tests/unit/test_1808_sync_freeze_enforcement.py` rather than by a standing SQL check. Both clauses write a `skipped` row whose `skip_reason` starts `Git sync frozen:`. --- diff --git a/docs/user-docs/faq/scheduling-and-automation.md b/docs/user-docs/faq/scheduling-and-automation.md index c454958dcb..b7a919b7af 100644 --- a/docs/user-docs/faq/scheduling-and-automation.md +++ b/docs/user-docs/faq/scheduling-and-automation.md @@ -20,7 +20,7 @@ Yes. Set `deliver_to_workspace_email` on the schedule over the API or with `crea ## Why didn't my schedule fire even though it's enabled? -The most common cause is the agent-level **autonomy toggle**: it's a master switch, and no schedules fire while it's off, regardless of their individual enabled state. Next, look at the schedule's cron chip in the list — a warning triangle with the tooltip **Invalid cron expression** means the stored expression is one the scheduler can't register, and that schedule never fires until you fix it. Also check that the agent still exists and isn't deleted (schedules stop firing immediately when an agent is deleted) and, if the agent has **freeze schedules if sync failing** enabled, that its git sync hasn't failed three times in a row. If the scheduler was restarted, missed runs are only caught up within a 1-hour grace window; anything older is skipped rather than fired late. See [Scheduling](../automation/scheduling.md). +The most common cause is the agent-level **autonomy toggle**: it's a master switch, and no schedules fire while it's off, regardless of their individual enabled state. Next, look at the schedule's cron chip in the list — a warning triangle with the tooltip **Invalid cron expression** means the stored expression is one the scheduler can't register, and that schedule never fires until you fix it. Also check that the agent still exists and isn't deleted (schedules stop firing immediately when an agent is deleted) and, if the agent has **freeze schedules if sync failing** enabled, that its git sync hasn't failed three times in a row and that it hasn't been out of step with its repository for more than 24 hours — the skipped run in the execution history names which. If the scheduler was restarted, missed runs are only caught up within a 1-hour grace window; anything older is skipped rather than fired late. See [Scheduling](../automation/scheduling.md). ## What's the difference between disabling a schedule and turning off autonomy? diff --git a/docs/user-docs/faq/troubleshooting.md b/docs/user-docs/faq/troubleshooting.md index 0c3fb6a6d4..8b91b72893 100644 --- a/docs/user-docs/faq/troubleshooting.md +++ b/docs/user-docs/faq/troubleshooting.md @@ -52,7 +52,7 @@ The microphone appears only where dictation can work. Trinity's own transcriptio ## My scheduled task didn't run -Work down this checklist. (1) **Autonomy** is the master switch — if the agent's autonomy toggle is off, no schedule fires. (2) The individual schedule may be disabled — check its toggle on the **Schedules** tab. (3) A warning triangle inside the schedule's cron chip (tooltip **Invalid cron expression**) means the scheduler could not register the expression, and the schedule never fires until you fix it. (4) The agent must be running; executions against a stopped container fail. (5) If the agent has the **freeze schedules if sync failing** flag on and its git sync has failed 3+ times in a row, the scheduler skips firing until sync recovers. (6) If the template ships a pre-check hook, an empty result records a `skipped` execution at zero cost — that's by design, and you'll see the row in the execution history. (7) Missed firings are only caught up within a 1-hour grace window after a scheduler restart; older ones are dropped. Also confirm the scheduler itself is healthy — its port is not published to the host, so probe it from inside the container: `docker exec trinity-scheduler curl -sf http://localhost:8001/health`. See [Scheduling](../automation/scheduling.md). +Work down this checklist. (1) **Autonomy** is the master switch — if the agent's autonomy toggle is off, no schedule fires. (2) The individual schedule may be disabled — check its toggle on the **Schedules** tab. (3) A warning triangle inside the schedule's cron chip (tooltip **Invalid cron expression**) means the scheduler could not register the expression, and the schedule never fires until you fix it. (4) The agent must be running; executions against a stopped container fail. (5) If the agent has the **freeze schedules if sync failing** flag on and its git sync has failed 3+ times in a row — or it owns its branch and has been ahead of or behind its repository for more than 24 hours — the scheduler skips firing until sync recovers; the skipped row in the execution history names the reason. (6) If the template ships a pre-check hook, an empty result records a `skipped` execution at zero cost — that's by design, and you'll see the row in the execution history. (7) Missed firings are only caught up within a 1-hour grace window after a scheduler restart; older ones are dropped. Also confirm the scheduler itself is healthy — its port is not published to the host, so probe it from inside the container: `docker exec trinity-scheduler curl -sf http://localhost:8001/health`. See [Scheduling](../automation/scheduling.md). ## `curl localhost:8001/health` fails, but the scheduler seems fine @@ -68,7 +68,7 @@ Every execution is bounded by the agent's execution timeout — default 3600 sec ## The git sync indicator is red / sync keeps failing -Trinity polls git-enabled agents every 60 seconds by default and tracks consecutive sync failures; at 3+ failures it raises an operator alert and the sync status chip on the dashboard and agent Overview turns unhealthy. Open the agent's **Git** tab to see the failure and its classification — the conflict modal explains each case in plain English. Auth failures mean the agent has no working GitHub token — it is missing, invalid, or expired: set or update it (the agent's **Git** tab, or the platform PAT in Settings) and sync again; git picks up the new token on its next operation, without a restart. For a deadlocked history (both sides diverged with no common ancestor), use the recovery reset: it adopts the upstream main branch while preserving the workspace state the agent flagged for persistence, available via the Git conflict flow or `POST /api/agents/{name}/git/reset-to-main-preserve-state`. See [GitHub Sync](../integrations/github-sync.md). +Trinity polls git-enabled agents every 60 seconds by default and tracks consecutive sync failures; at 3+ failures it raises an operator alert and the sync status chip on the dashboard and agent Overview turns unhealthy. The dot is also red when the agent has been ahead of or behind its own branch on GitHub for more than 24 hours, or has had uncommitted changes that long — hover it for the reason and the suggested fix (see [What the sync state means](../integrations/github-sync.md#what-the-sync-state-means)). Open the agent's **Git** tab to see the failure and its classification — the conflict modal explains each case in plain English. Auth failures mean the agent has no working GitHub token — it is missing, invalid, or expired: set or update it (the agent's **Git** tab, or the platform PAT in Settings) and sync again; git picks up the new token on its next operation, without a restart. For a deadlocked history (both sides diverged with no common ancestor), use the recovery reset: it adopts the upstream main branch while preserving the workspace state the agent flagged for persistence, available via the Git conflict flow or `POST /api/agents/{name}/git/reset-to-main-preserve-state`. See [GitHub Sync](../integrations/github-sync.md). ## Git in my agent says `index.lock` already exists diff --git a/docs/user-docs/integrations/github-sync.md b/docs/user-docs/integrations/github-sync.md index e033c69ef7..bf74e3b6ef 100644 --- a/docs/user-docs/integrations/github-sync.md +++ b/docs/user-docs/integrations/github-sync.md @@ -53,6 +53,32 @@ One limit, reported rather than fixed: many default patterns are directory-form Trinity reads the git status of every git-enabled agent once a minute. Each read runs a `git fetch` inside the agent, so on a large fleet you can poll less often by setting `SYNC_HEALTH_POLL_INTERVAL_SECONDS` in the backend's `.env` (default `60`; see [Single-Server Deployment → `.env` reference](../guides/deploying/single-server.md#observability)). When an agent's consecutive sync failures reach three, an alert lands in the [Operating Room](../operations/operating-room.md#sync-health-alerts). +### What the sync state means + +Sync health measures whether the agent and its repository **agree**, not only whether the last push worked. Trinity computes one state per agent and every surface shows the same one: the dashboard dot, `GET /api/agents/sync-health`, `GET /api/agents/{name}/git/sync-state` and the `get_git_sync_state` MCP tool. + +| State | When | +|-------|------| +| **Red** | The last sync failed; or the agent has been ahead of or behind its repository for more than 24 hours; or it has had uncommitted changes for more than 24 hours; or auto-sync is on but no sync has run for 7 days | +| **Yellow** | The agent has been ahead or behind for 24 hours or less | +| **Green** | The agent and its repository agree | +| **Gray** | Trinity has not observed the agent yet | + +Ahead and behind are counted against the agent's **own branch** on GitHub, not against `main`. Every entry carries a `reason` (for example `diverged 31 behind / 0 ahead for 26h`) and, when there is an obvious fix, a `recommendation` — `credential is read-only`, `push via git_sync strategy=pull_first`, `enable auto-sync`, `pull via git_pull`. Hover the dot to read both. + +An agent that tracks a shared branch with auto-sync off — a **deployment** of a codebase rather than an agent that owns its branch — is never red for being behind: being behind is normal for a deployment. It shows yellow instead. + +The ages start counting at the first check after the upgrade that introduced them, so an agent that was already out of step shows yellow for its first day and red after that. + +### Pausing schedules when sync is unhealthy + +With **freeze schedules if sync failing** on (`PUT /api/agents/{name}/git/freeze-schedules-if-failing`), the scheduler skips the agent's scheduled runs when either: + +- its git sync has failed on three consecutive checks, or +- it is an agent that owns its branch (a working branch, or auto-sync on) and has been ahead of or behind its repository for more than 24 hours. + +Each skipped run appears in the execution history with the reason, for example *Git sync frozen: diverged 0 behind / 7 ahead for 26h*. A divergence pause also files one **Agent diverged from GitHub — schedules paused** notice in the Operating Room per episode. Schedules resume by themselves on the first scheduled run after the agent is back in step — push its work, or pull what it is missing. The divergence pause needs a recent check (within 15 minutes): if Trinity cannot reach the agent, it does not keep it paused on old information. Manual runs are never paused. + The read is built to stay out of the agent's way: - **It takes no git lock.** The status read never takes the repository's `.git/index.lock`, so the agent's own `git add` or `git commit` cannot fail because Trinity was looking. diff --git a/docs/user-docs/operations/operating-room.md b/docs/user-docs/operations/operating-room.md index 771452deb1..273994f27f 100644 --- a/docs/user-docs/operations/operating-room.md +++ b/docs/user-docs/operations/operating-room.md @@ -83,7 +83,9 @@ All clear operations are scoped to agents you can access, affect all operators o For agents with GitHub sync enabled, the Sync Health Service polls every 60 seconds by default (`SYNC_HEALTH_POLL_INTERVAL_SECONDS`) and writes `sync_failing` queue entries when an agent's `consecutive_failures` hits 3. These appear in the Needs Response tab alongside agent-emitted items, so a broken git remote, expired PAT, or upstream divergence surfaces in the same place operators already watch. -Per-agent sync state (last sync at, last error, ahead/behind counts on `main` and the working branch) is also visible on the agent header dot and at `GET /api/agents/{name}/git/sync-state`. +When an agent's schedules are paused because it has been out of step with its repository for more than 24 hours, one **Agent diverged from GitHub — schedules paused** (`sync_diverged`) entry appears per episode, with the ahead/behind counts and a suggested fix. It stays until you clear it; if the agent drifts out of step again later, that is a new entry. See [GitHub Sync → Pausing schedules](../integrations/github-sync.md#pausing-schedules-when-sync-is-unhealthy). + +Per-agent sync state (last sync at, last error, ahead/behind counts on `main` and the working branch, uncommitted-file count, last successful push, and the computed state with its reason) is also visible on the agent header dot and at `GET /api/agents/{name}/git/sync-state`. ## For Agents diff --git a/src/backend/client_portal/models.py b/src/backend/client_portal/models.py index 84b08c2aaa..dbf32d3633 100644 --- a/src/backend/client_portal/models.py +++ b/src/backend/client_portal/models.py @@ -348,9 +348,13 @@ class PortalChatRequest(BaseModel): class PortalChatResponse(BaseModel): """The agent's reply to a portal chat turn. ``session_id`` echoes the thread - the turn landed in, so a client that sent none learns which session was used.""" + the turn landed in, so a client that sent none learns which session was used. + + No ``cost`` (#3063): external clients never see a turn's cost, and this + synchronous route is the streaming path's fallback. The service still + returns it for internal accounting; not declaring it is what keeps it off + the wire (``response_model`` drops undeclared keys).""" response: str - cost: Optional[float] = None session_id: Optional[str] = None # #2580: the persisted row's id, so the caller can rate the reply it was just # given instead of waiting for a reload to learn what to point at. The @@ -854,7 +858,11 @@ class PortalHistoryMessage(BaseModel): id: Optional[str] = None role: str # 'user' | 'assistant' content: str - cost: Optional[float] = None + # #3063: no `cost`. The row stores it, but a turn's cost is never a + # Workspace field — the viewer may be an external client and the contract + # excludes costs outright (the Work projection's rule). Undeclared here, the + # route's `response_model` strips it for every principal; nothing in the + # client read it. created_at: Optional[str] = None # The caller's OWN rating of this message, if any — never anyone else's. # Present so a reload shows the thumb the person already gave. diff --git a/src/backend/client_portal/service.py b/src/backend/client_portal/service.py index 9fec94666b..e074ca92db 100644 --- a/src/backend/client_portal/service.py +++ b/src/backend/client_portal/service.py @@ -4608,12 +4608,11 @@ def _alert_collided_inbox(agent_name: str, legacy_dir: str, claimants: list[str] repeat access does not re-alert. Never raises — an alert failure must not break a client's inbox read. - Residual, stated rather than hidden: this id carries none of the #1632 - reserved prefixes, so an agent writing its own `operator-queue.json` could - pre-create it and swallow the alert via that same ON CONFLICT. The ERROR log - in the caller is therefore the primary signal and fires on every access - regardless; the queue item is the convenience. Adding a reserved prefix would - mean changing the OSS guard list, which is out of scope here. + The id's prefix is reserved (`_RESERVED_ID_PREFIXES`, #715): the text lists + client addresses, so the item is a platform alarm the agent's own file and + resume turn never receive, the queue's reads never return it to a machine + key, and an agent cannot pre-create the id to swallow the alert through that + same ON CONFLICT. The ERROR log in the caller still fires on every access. """ try: from database import db as core_db diff --git a/src/backend/database.py b/src/backend/database.py index fe99e37f2c..ba50d1488f 100644 --- a/src/backend/database.py +++ b/src/backend/database.py @@ -2051,6 +2051,9 @@ def get_sync_state(self, agent_name: str): def list_sync_states(self): return self._sync_state_ops.list_all() + def list_sync_health_rows(self, agent_names=None): + return self._sync_state_ops.list_health_rows(agent_names) + def upsert_sync_state(self, agent_name: str, **fields): return self._sync_state_ops.upsert(agent_name, **fields) diff --git a/src/backend/db/migrations.py b/src/backend/db/migrations.py index 8a81d4e9f1..266d7109da 100644 --- a/src/backend/db/migrations.py +++ b/src/backend/db/migrations.py @@ -4876,6 +4876,31 @@ def _migrate_auto_sync_enabled_backfill(cursor, conn): conn.commit() +def _migrate_agent_sync_state_divergence(cursor, conn): + """Add the divergence / dirt episode columns to agent_sync_state (trinity-enterprise#706). + + `diverged_since` / `dirty_since` are ISO-Z episode clocks the + SyncHealthService sets once and clears on a return to 0; `dirty_files` is + the porcelain change count the agent's status call already computed and + threw away; `last_successful_push_at` is the last push that landed. All + nullable with NO backfill: the clocks cannot be known retroactively, and + starting them at the first post-upgrade poll is the 24 h soak before any + divergence freeze can fire. + """ + for column, sql_type in ( + ("diverged_since", "TEXT"), + ("dirty_files", "INTEGER"), + ("dirty_since", "TEXT"), + ("last_successful_push_at", "TEXT"), + ): + _safe_add_column( + cursor, + "agent_sync_state", + column, + f"ALTER TABLE agent_sync_state ADD COLUMN {column} {sql_type}", + ) + + MIGRATIONS = [ ("agent_sharing", _migrate_agent_sharing_table), ("schedule_executions_observability", _migrate_schedule_executions_observability), @@ -5027,4 +5052,5 @@ def _migrate_auto_sync_enabled_backfill(cursor, conn): ("telegram_group_context", _migrate_telegram_group_context), ("agent_skill_sets", _migrate_agent_skill_sets), ("portal_messages_unread_index", _migrate_portal_messages_unread_index), + ("agent_sync_state_divergence", _migrate_agent_sync_state_divergence), ] diff --git a/src/backend/db/schema.py b/src/backend/db/schema.py index 5302f53794..942c4d77e3 100644 --- a/src/backend/db/schema.py +++ b/src/backend/db/schema.py @@ -1194,6 +1194,10 @@ pack_count INTEGER, loose_objects INTEGER, maintenance_failures INTEGER DEFAULT 0, + diverged_since TEXT, + dirty_files INTEGER, + dirty_since TEXT, + last_successful_push_at TEXT, last_check_at TEXT, updated_at TEXT NOT NULL, FOREIGN KEY (agent_name) REFERENCES agent_ownership(agent_name) diff --git a/src/backend/db/sync_state.py b/src/backend/db/sync_state.py index 2d9d2089d5..f2a223abec 100644 --- a/src/backend/db/sync_state.py +++ b/src/backend/db/sync_state.py @@ -9,14 +9,14 @@ both SQLite and PostgreSQL. """ -from typing import Dict, List, Optional +from typing import Dict, Iterable, List, Optional -from sqlalchemy import select, delete +from sqlalchemy import and_, select, delete from utils.helpers import utc_now_iso from .engine import get_engine, make_insert -from .tables import agent_sync_state +from .tables import agent_git_config, agent_ownership, agent_sync_state # Keep in sync with agent_sync_state column order so row indexes match. _COLUMNS = ( @@ -35,6 +35,10 @@ "pack_count", # #1595: packs from `git count-objects -v` "loose_objects", # #1595: loose objects (gc-health signal) "maintenance_failures", # #1595: consecutive failed maintenance attempts + "diverged_since", # trinity-enterprise#706: divergence episode clock (ISO-Z) + "dirty_files", # trinity-enterprise#706: porcelain change count + "dirty_since", # trinity-enterprise#706: dirt episode clock (ISO-Z) + "last_successful_push_at", # trinity-enterprise#706: last push that landed "last_check_at", "updated_at", ) @@ -43,6 +47,22 @@ _UPSERT_SET_COLUMNS = tuple(c for c in _COLUMNS if c != "agent_name") +class _Keep: + """Sentinel: leave an episode clock as it is (trinity-enterprise#706). + + `diverged_since` / `dirty_since` need THREE caller intents — set, clear, + unchanged — and `_merged()` below maps None to "unchanged", so None alone + cannot mean "clear". For those two columns None means clear and KEEP means + unchanged. + """ + + def __repr__(self) -> str: + return "KEEP" + + +KEEP = _Keep() + + def _row_to_dict(row) -> Dict: return {col: row[col] for col in _COLUMNS} @@ -64,6 +84,74 @@ def list_all(self) -> List[Dict]: rows = conn.execute(stmt).mappings().all() return [_row_to_dict(r) for r in rows] + def list_health_rows(self, agent_names: Optional[Iterable[str]] = None) -> Dict[str, Dict]: + """Git config flags + sync-state row per live git-bound agent (trinity-enterprise#706). + + ONE query — `agent_git_config` joined to `agent_ownership` (soft-deleted + agents excluded, the #1561 filter of `list_git_enabled_agents`) and + LEFT-joined to `agent_sync_state` — so every sync-health surface reads + the same set the poller polls. Returns + `{name: {"config": {...}, "state": {...} | None}}`; `state` is None for + an agent the poller has not written yet. + + `agent_names` scopes the result (an empty iterable returns `{}`). The + scope is applied in Python rather than as an `IN (...)` list, which + needs no chunking under SQLite's host-parameter cap (#73); the table is + one row per git-bound agent, so the scan is fleet-sized. + """ + scope = None if agent_names is None else set(agent_names) + if scope is not None and not scope: + return {} + state_cols = [agent_sync_state.c[col].label(f"s_{col}") for col in _COLUMNS] + stmt = ( + select( + agent_git_config.c.agent_name, + agent_git_config.c.source_mode, + agent_git_config.c.auto_sync_enabled, + agent_git_config.c.freeze_schedules_if_sync_failing, + agent_git_config.c.created_at, + *state_cols, + ) + .select_from( + agent_git_config.join( + agent_ownership, + agent_ownership.c.agent_name == agent_git_config.c.agent_name, + ).outerjoin( + agent_sync_state, + agent_sync_state.c.agent_name == agent_git_config.c.agent_name, + ) + ) + .where( + and_( + agent_git_config.c.sync_enabled == 1, + agent_ownership.c.deleted_at.is_(None), + ) + ) + ) + result: Dict[str, Dict] = {} + with get_engine().connect() as conn: + for row in conn.execute(stmt).mappings(): + name = row["agent_name"] + if scope is not None and name not in scope: + continue + state = ( + {col: row[f"s_{col}"] for col in _COLUMNS} + if row["s_agent_name"] is not None + else None + ) + result[name] = { + "config": { + "source_mode": bool(row["source_mode"]), + "auto_sync_enabled": bool(row["auto_sync_enabled"]), + "freeze_schedules_if_sync_failing": bool( + row["freeze_schedules_if_sync_failing"] + ), + "created_at": row["created_at"], + }, + "state": state, + } + return result + def upsert( self, agent_name: str, @@ -81,11 +169,19 @@ def upsert( loose_objects: Optional[int] = None, maintenance_failures: Optional[int] = None, last_check_at: Optional[str] = None, + diverged_since=KEEP, + dirty_since=KEEP, + dirty_files: Optional[int] = None, + last_successful_push_at: Optional[str] = None, ) -> Dict: """Upsert a sync-state row. consecutive_failures is maintained internally: incremented on `failed`, reset on `success`, untouched on `never`. + + trinity-enterprise#706: `diverged_since` / `dirty_since` take KEEP + (the default — leave as is), None (clear) or an ISO string (set). + `dirty_files` / `last_successful_push_at` follow the `_merged()` rule. """ now = utc_now_iso() existing = self.get(agent_name) @@ -103,6 +199,11 @@ def _merged(field: str, new_value): return new_value return existing.get(field) if existing else None + def _clock(field: str, new_value): + if new_value is KEEP: + return existing.get(field) if existing else None + return new_value + row = { "agent_name": agent_name, "last_sync_at": _merged("last_sync_at", last_sync_at), @@ -124,6 +225,11 @@ def _merged(field: str, new_value): # 0 is a meaningful reset here (post-success), not "unset" — it # passes _merged as-is; only a true None falls back to prior. "maintenance_failures": _merged("maintenance_failures", maintenance_failures) or 0, + "diverged_since": _clock("diverged_since", diverged_since), # ent#706 + "dirty_files": _merged("dirty_files", dirty_files), # ent#706 + "dirty_since": _clock("dirty_since", dirty_since), # ent#706 + "last_successful_push_at": _merged( # ent#706 + "last_successful_push_at", last_successful_push_at), "last_check_at": last_check_at or now, "updated_at": now, } diff --git a/src/backend/db/tables.py b/src/backend/db/tables.py index 951acd7567..c525dbbe32 100644 --- a/src/backend/db/tables.py +++ b/src/backend/db/tables.py @@ -1017,6 +1017,13 @@ def process_bind_param(self, value, dialect): Column("pack_count", Integer), # #1595: packs from `git count-objects -v` Column("loose_objects", Integer), # #1595: loose objects (gc-health signal) Column("maintenance_failures", Integer), # #1595: consecutive failed maintenance + # trinity-enterprise#706: the divergence / dirt episode clocks (ISO-Z, set + # once, cleared on return to 0), the porcelain change count, and the last + # push that actually landed. All nullable, no backfill. + Column("diverged_since", Text), + Column("dirty_files", Integer), + Column("dirty_since", Text), + Column("last_successful_push_at", Text), Column("last_check_at", Text), Column("updated_at", Text), ) diff --git a/src/backend/migrations/versions/0082_agent_sync_state_divergence.py b/src/backend/migrations/versions/0082_agent_sync_state_divergence.py new file mode 100644 index 0000000000..73ed26c4ed --- /dev/null +++ b/src/backend/migrations/versions/0082_agent_sync_state_divergence.py @@ -0,0 +1,55 @@ +"""agent_sync_state divergence / dirt episode columns (trinity-enterprise#706) + +Adds four nullable columns on the PostgreSQL backend: + +- ``diverged_since`` — first poll at which the working tuple (origin on the + agent's own branch) showed ahead or behind; cleared when both return to 0. +- ``dirty_files`` — the porcelain change count from ``GET /api/git/status``. +- ``dirty_since`` — first poll at which ``dirty_files > 0``; cleared at 0. +- ``last_successful_push_at`` — the last push that landed (heartbeat or an + operator Push). + +No backfill: the clocks cannot be known retroactively, and starting them at the +first post-upgrade poll is the 24 h soak before any divergence freeze fires. +DDL-only, so ``downgrade()`` is a clean drop. + +Mirrors the SQLite ``agent_sync_state_divergence`` migration in +``db/migrations.py`` and the DDL in ``db/schema.py`` / MetaData in +``db/tables.py``. Fresh PG builds already get the columns via +``0001_baseline``'s reuse of the schema DDL; ``ADD COLUMN IF NOT EXISTS`` keeps +this a no-op there. + +Revision ID: 0082_agent_sync_state_divergence +Revises: 0081_portal_messages_unread_idx +Create Date: 2026-09-27 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0082_agent_sync_state_divergence" +down_revision = "0081_portal_messages_unread_idx" +branch_labels = None +depends_on = None + +def upgrade() -> None: + op.execute( + "ALTER TABLE agent_sync_state ADD COLUMN IF NOT EXISTS diverged_since TEXT" + ) + op.execute( + "ALTER TABLE agent_sync_state ADD COLUMN IF NOT EXISTS dirty_files INTEGER" + ) + op.execute( + "ALTER TABLE agent_sync_state ADD COLUMN IF NOT EXISTS dirty_since TEXT" + ) + op.execute( + "ALTER TABLE agent_sync_state " + "ADD COLUMN IF NOT EXISTS last_successful_push_at TEXT" + ) + +def downgrade() -> None: + op.execute("ALTER TABLE agent_sync_state DROP COLUMN IF EXISTS diverged_since") + op.execute("ALTER TABLE agent_sync_state DROP COLUMN IF EXISTS dirty_files") + op.execute("ALTER TABLE agent_sync_state DROP COLUMN IF EXISTS dirty_since") + op.execute( + "ALTER TABLE agent_sync_state DROP COLUMN IF EXISTS last_successful_push_at" + ) diff --git a/src/backend/routers/a2a.py b/src/backend/routers/a2a.py index 1c7bac3da5..d5f2afe0e4 100644 --- a/src/backend/routers/a2a.py +++ b/src/backend/routers/a2a.py @@ -57,7 +57,7 @@ A2AEndpointNotFound, A2AOutboundDisabled, ) -from services.idempotency_service import EffectInProgressError +from services.idempotency_service import EffectInProgressError, EffectUnguardedError from services.agent_auth import agent_httpx_client from services.docker_service import get_agent_container from services.platform_audit_service import AuditEventType, platform_audit_service @@ -760,6 +760,9 @@ async def call_a2a_agent( ), }, ) + except EffectUnguardedError as exc: + # #2392: pull-mode agent, no usable execution id — refused, not retryable. + raise HTTPException(status_code=422, detail={"reason": "effect_unguarded", "message": str(exc)}) except EffectInProgressError as exc: raise HTTPException(status_code=409, detail=str(exc)) except A2ACallError as exc: diff --git a/src/backend/routers/agent_files.py b/src/backend/routers/agent_files.py index 9963cbdd32..f069b33d6e 100644 --- a/src/backend/routers/agent_files.py +++ b/src/backend/routers/agent_files.py @@ -46,7 +46,7 @@ build_download_url, MAX_AGENT_QUOTA_BYTES, ) -from services.idempotency_service import EffectInProgressError +from services.idempotency_service import EffectInProgressError, EffectUnguardedError from services.platform_audit_service import platform_audit_service, AuditEventType logger = logging.getLogger(__name__) @@ -793,6 +793,9 @@ async def share_agent_file( # whatever execution id they cite, they are not an agent in a turn. actor_is_agent=bool(actor_agent), ) + except EffectUnguardedError as e: + # #2392: pull-mode agent, no usable execution id — refused, not retryable. + raise HTTPException(status_code=422, detail={"reason": "effect_unguarded", "message": str(e)}) except EffectInProgressError as e: # Concurrent duplicate share for the same (execution, file) is mid-flight # (#1084). Retryable — never a silent skip-and-succeed. diff --git a/src/backend/routers/agents.py b/src/backend/routers/agents.py index 866ce38fa9..d369ace3f9 100644 --- a/src/backend/routers/agents.py +++ b/src/backend/routers/agents.py @@ -282,31 +282,46 @@ async def get_all_sync_health( ): """Dashboard batch endpoint for sync-health dots (#389). - Returns one entry per accessible agent. Entries join `agent_sync_state` - with the per-agent auto-sync flag so the UI can colour dots and badge - agents that have auto-sync off. + Returns one entry per accessible agent. trinity-enterprise#706: each entry + carries the backend's verdict — `state` / `reason` / `recommendation` / + `binding` / `freeze` from `services/sync_health_view.py` — and the new + columns, so the dot renders a state the backend owns. The pre-#706 keys are + unchanged. One shared query (`db.list_sync_health_rows`) replaces the + per-table reads (still no N+1, #73). """ + from services.sync_health_view import sync_view + accessible = {a["name"] for a in get_accessible_agents(current_user)} - rows = db.list_sync_states() - by_name = {r["agent_name"]: r for r in rows if r["agent_name"] in accessible} - # #73: one scoped query instead of an N+1 per-agent lookup. - auto_sync_map = db.get_all_git_auto_sync_enabled(accessible) + rows = db.list_sync_health_rows(accessible) entries = [] for name in sorted(accessible): - row = by_name.get(name) + bound = rows.get(name) + row = (bound or {}).get("state") + config = (bound or {}).get("config") + view = sync_view(row, config) + row = row or {} entries.append({ "agent_name": name, - "auto_sync_enabled": auto_sync_map.get(name, False), - "last_sync_at": (row or {}).get("last_sync_at"), - "last_sync_status": (row or {}).get("last_sync_status") or "never", - "consecutive_failures": (row or {}).get("consecutive_failures") or 0, - "last_error_summary": (row or {}).get("last_error_summary"), - "behind_working": (row or {}).get("behind_working") or 0, - "behind_main": (row or {}).get("behind_main") or 0, - "ahead_working": (row or {}).get("ahead_working") or 0, - "ahead_main": (row or {}).get("ahead_main") or 0, - "git_dir_bytes": (row or {}).get("git_dir_bytes"), # #1596 bloat curve + "auto_sync_enabled": bool((config or {}).get("auto_sync_enabled")), + "last_sync_at": row.get("last_sync_at"), + "last_sync_status": row.get("last_sync_status") or "never", + "consecutive_failures": row.get("consecutive_failures") or 0, + "last_error_summary": row.get("last_error_summary"), + "behind_working": row.get("behind_working") or 0, + "behind_main": row.get("behind_main") or 0, + "ahead_working": row.get("ahead_working") or 0, + "ahead_main": row.get("ahead_main") or 0, + "git_dir_bytes": row.get("git_dir_bytes"), # #1596 bloat curve + # trinity-enterprise#706 + "dirty_files": row.get("dirty_files"), + "last_successful_push_at": row.get("last_successful_push_at"), + "state": view["state"], + "reason": view["reason"], + "recommendation": view["recommendation"], + "binding": view["binding"] if config else None, + "divergence_age_s": view["divergence_age_s"], + "freeze": view["freeze"], }) return {"agents": entries} diff --git a/src/backend/routers/git.py b/src/backend/routers/git.py index 97d93739bf..6a21b93b50 100644 --- a/src/backend/routers/git.py +++ b/src/backend/routers/git.py @@ -1216,12 +1216,21 @@ async def set_freeze_schedules_config( @router.get("/{agent_name}/git/sync-state") async def get_agent_sync_state(agent_name: AuthorizedAgentByName): - """Return the persisted sync-state row for this agent (#389).""" + """Return the persisted sync-state row for this agent (#389). + + trinity-enterprise#706: plus the backend's verdict (`state`, `reason`, + `recommendation`, `binding`, `freeze`, the ages — `services/sync_health_view.py`). + MCP `get_git_sync_state` passes this JSON through unchanged. + """ + from services.sync_health_view import sync_view + row = db.get_sync_state(agent_name) + view = sync_view(row, db.get_git_config(agent_name)) if row is None: return { "agent_name": agent_name, "last_sync_status": "never", "consecutive_failures": 0, + **view, } - return row + return {**row, **view} diff --git a/src/backend/routers/internal.py b/src/backend/routers/internal.py index cbfb376b5c..2119cb22cd 100644 --- a/src/backend/routers/internal.py +++ b/src/backend/routers/internal.py @@ -283,25 +283,34 @@ async def internal_agent_brief_readiness(agent_name: str): @router.get("/agents/{agent_name}/sync-health-status") async def internal_agent_sync_health(agent_name: str): - """#389: lightweight read used by the dedicated scheduler before dispatching. + """#389: the per-agent freeze decision, as the backend computes it. - Returns both the per-agent `freeze_schedules_if_sync_failing` flag and - whether the current sync state would trip it. The scheduler multiplies - the two to decide whether to skip the fire. + Returns the per-agent `freeze_schedules_if_sync_failing` flag, whether sync + is failing, and `should_freeze`. Since trinity-enterprise#706 the decision + is `sync_freeze_policy.classify` — the same module the scheduler vendors — + so `should_freeze` also covers a work agent diverged from origin for more + than 24 h, and `freeze_reason` / `divergence_age_s` / `work_agent` say why. + + The dedicated scheduler does NOT call this endpoint: it reads the database + directly (`SchedulerDatabase.sync_freeze_reason`) with the vendored copy of + the same policy. This is the backend's read of the same decision. """ from database import db as _db + from services.sync_health_view import sync_view + freeze_flag = _db.get_freeze_schedules_if_sync_failing(agent_name) + config = _db.get_git_config(agent_name) state = _db.get_sync_state(agent_name) or {} - failing = ( - state.get("last_sync_status") == "failed" - and (state.get("consecutive_failures") or 0) >= 3 - ) + view = sync_view(state or None, config) return { "agent_name": agent_name, "freeze_schedules_if_sync_failing": bool(freeze_flag), - "sync_failing": bool(failing), - "should_freeze": bool(freeze_flag and failing), + "sync_failing": bool(view["sync_failing"]), + "should_freeze": bool(freeze_flag and view["freeze"]), "consecutive_failures": state.get("consecutive_failures") or 0, + "freeze_reason": view["freeze_reason"] if freeze_flag else None, + "divergence_age_s": view["divergence_age_s"], + "work_agent": bool(config is not None and view["work_agent"]), } @@ -870,14 +879,20 @@ async def agent_files_share(payload: ShareFileRequest): /internal/execute-task (forging requires the internal secret). """ from services.agent_shared_files_service import create_share + from services.idempotency_service import EffectUnguardedError - result = await create_share( - agent_name=payload.agent_name, - filename=payload.filename, - display_name=payload.display_name, - expires_in=payload.expires_in, - created_by=payload.agent_name, - ) + try: + result = await create_share( + agent_name=payload.agent_name, + filename=payload.filename, + display_name=payload.display_name, + expires_in=payload.expires_in, + created_by=payload.agent_name, + ) + except EffectUnguardedError as e: + # #2392: this path carries no execution id, so a pull-mode agent's share + # is refused here; the MCP route (`/api/agents/{name}/shared-files`) is the one to use. + raise HTTPException(status_code=422, detail={"reason": "effect_unguarded", "message": str(e)}) return ShareFileResponse(**result) diff --git a/src/backend/routers/messages.py b/src/backend/routers/messages.py index 3d9b46e501..086f4cbcb0 100644 --- a/src/backend/routers/messages.py +++ b/src/backend/routers/messages.py @@ -18,7 +18,7 @@ from database import db from dependencies import get_current_user, AuthorizedAgentByName, assert_agent_owner from db_models import User -from services.idempotency_service import EffectInProgressError +from services.idempotency_service import EffectInProgressError, EffectUnguardedError from services.proactive_message_service import ( proactive_message_service, NotAuthorizedError, @@ -70,6 +70,10 @@ async def send_proactive_message( error=result.error, ) + except EffectUnguardedError as e: + # #2392: pull-mode agent, no usable execution id — refused, not retryable. + raise HTTPException(status_code=422, detail={"reason": "effect_unguarded", "message": str(e)}) + except EffectInProgressError as e: # A concurrent duplicate send for the same (execution, recipient, channel) # is mid-flight (#1084). Retryable — never a silent skip-and-succeed. diff --git a/src/backend/routers/operator_queue.py b/src/backend/routers/operator_queue.py index 2c9d4121a4..d9ea2fbb9a 100644 --- a/src/backend/routers/operator_queue.py +++ b/src/backend/routers/operator_queue.py @@ -74,27 +74,40 @@ def _assert_agent_accessible(agent_name: str, accessible: Optional[Set[str]]) -> raise HTTPException(status_code=403, detail="Access denied") -# trinity-enterprise#611: the person fields that arrived with the ask object. A -# machine principal (an agent-, system- or ops-scoped key) reads the queue for its -# own work, never to learn which person ended an ask or whom it was resolved to. -# The PRE-existing person fields on the row (`responded_by_email`, -# `addressed_to_email`) still pass here — a registered residual, not a decision. -_PERSON_FIELDS_WITHHELD_FROM_MACHINES = ("disposed_by_email", "resolved_to") +# What a MACHINE principal (an agent-, system-, ops-, connector- or +# portal_delegate key, and whatever scope ships next) reads of a queue row +# (trinity-enterprise#611, #715). It reads the queue for its own work, never to +# learn which person answered or ended an ask, or whom it was for. An ALLOWLIST, +# like `_READBACK_FIELDS` below, so a column added later stays out until someone +# decides a machine should read it. Withheld: `responded_by_id`, +# `responded_by_email`, `addressed_to_email` (a file entry's too — a sibling's +# rows are readable by the same key), `disposed_by_email`, `resolved_to`. +# A person — a JWT session or the person's own user-scoped key +# (`is_person_principal`) — reads the whole row. +_MACHINE_ROW_FIELDS = ( + "id", "agent_name", "request_id", "type", "status", "priority", + "title", "question", "options", "context", "execution_id", + "created_at", "expires_at", "response", "response_text", "responded_at", + "acknowledged_at", "cleared_at", + "sync_state", "sync_detail", "sync_updated_at", "last_confirmed_at", + "delivery_state", "delivery_detail", "delivery_updated_at", + "divergence_acknowledged_at", + "disposition", "disposed_at", "disposed_by", "disposition_reason", "batch_id", + "raised_by", "channel", "to_role", "proposal", "supersedes_expired", + "aging", "aged_since", +) def _for_principal(items: List[Dict[str, Any]], current_user: User) -> List[Dict[str, Any]]: if is_person_principal(current_user): return items - for item in items: - for key in _PERSON_FIELDS_WITHHELD_FROM_MACHINES: - item.pop(key, None) - # trinity-enterprise#611: a native ask's addressee is the person the - # PLATFORM resolved a role to — the email `resolved_to` also holds — so a - # machine does not get it back (the receipt's rule). A file entry's - # addressee is the agent's own input and stays (the registered residual). - if item.get("channel") not in (None, "file"): - item.pop("addressed_to_email", None) - return items + # #715: the platform's heads-ups ABOUT a person (a client's complaint, the + # client addresses behind a shared inbox) are the operator's, not a machine's. + return [ + {key: item[key] for key in _MACHINE_ROW_FIELDS if key in item} + for item in items + if not operator_queue_service.is_about_a_person(item) + ] # The agent's own readback (trinity-enterprise#611). An ALLOWLIST, so a column @@ -232,12 +245,13 @@ async def clear_resolved_queue_items( details={"cleared": cleared, "agent_name": body.agent_name}, ) if _websocket_manager: + # #715: the count only. A fleet-level trigger reaches every `/ws` + # connection, and any key can mint a ticket; listeners refetch. await _websocket_manager.broadcast(json.dumps({ "type": "operator_queue_cleared", "data": { "scope": "resolved", "count": cleared, - "cleared_by": current_user.email or current_user.username, } })) @@ -255,7 +269,11 @@ async def get_queue_item( raise HTTPException(status_code=404, detail="Queue item not found") accessible = _accessible_set(current_user) _assert_agent_accessible(item["agent_name"], accessible) - return _for_principal(operator_queue_service.annotate_aging([item]), current_user)[0] + visible = _for_principal(operator_queue_service.annotate_aging([item]), current_user) + if not visible: + # #715: a row about a person reads, to a machine, like one that is not there. + raise HTTPException(status_code=404, detail="Queue item not found") + return visible[0] @router.post("/{item_id}/respond") diff --git a/src/backend/routers/voip.py b/src/backend/routers/voip.py index c659be3cb2..8edde09603 100644 --- a/src/backend/routers/voip.py +++ b/src/backend/routers/voip.py @@ -198,6 +198,11 @@ async def place_voip_call( execution_id=request.execution_id, dedup_label=request.dedup_label, ) + except idempotency_service.EffectUnguardedError as e: + # #2392: pull-mode agent, no usable execution id — nothing dialed, release + # the outer claim; refused, not retryable. + idempotency_service.fail(idem) + raise HTTPException(status_code=422, detail={"reason": "effect_unguarded", "message": str(e)}) except idempotency_service.EffectInProgressError: # Concurrent duplicate dial for the same (execution, number) is mid-flight # (#1084). Release the outer trigger claim and surface a retryable 409. diff --git a/src/backend/services/agent_shared_files_service.py b/src/backend/services/agent_shared_files_service.py index 072f25aae1..0fcd3480fb 100644 --- a/src/backend/services/agent_shared_files_service.py +++ b/src/backend/services/agent_shared_files_service.py @@ -518,8 +518,9 @@ async def create_share( + the addressee (ent#549 — re-addressing a file is a second share), scoped to ``execution_id``. A re-run of the same turn sharing the same file replays the original signed URL instead of minting a second token; a changed - file (different content) under the same name produces a new share. Fail-open - when ``execution_id`` is absent/invalid (old image / the internal path). + file (different content) under the same name produces a new share. Without a + usable ``execution_id`` (old image / the internal path): refused on a + pull-mode agent, else shared and logged as degraded (#2392). """ # --- flag gate --- if not db.get_file_sharing_enabled(agent_name): diff --git a/src/backend/services/idempotency_service.py b/src/backend/services/idempotency_service.py index 8f655dc284..25d87ba466 100644 --- a/src/backend/services/idempotency_service.py +++ b/src/backend/services/idempotency_service.py @@ -52,6 +52,39 @@ class EffectInProgressError(Exception): """ +# Value the agent's MCP config sends in `X-Trinity-Execution-Id` when the +# spawning process carries no `TRINITY_EXECUTION_ID` — a person running +# `claude` in the web terminal or over SSH (#2392). There is no execution to +# re-deliver, so a send can go without dedup; it is logged, never rejected. +MANUAL_EXECUTION_ID = "manual" + +# Id prefix of the unguarded-effect alarm — reserved in operator_queue_service +# so an agent cannot pre-create (and so suppress) its own alarm. +EFFECT_UNGUARDED_ALERT_PREFIX = "effect-unguarded-" + + +class EffectUnguardedError(Exception): + """A side effect on a pull-mode agent arrived without a usable execution id (#2392). + + Pull re-delivers the SAME execution after a lease expiry, so the effect + guard is the only thing between a re-run and a duplicate message, call or + share. Without an id that resolves to the calling agent's own execution the + guard cannot de-duplicate, so the effect is refused rather than sent + (fail-closed, `TARGET_ARCHITECTURE.md` §Re-Delivery and Side-Effect + Recovery). Not retryable: a retry carries the same missing id. + """ + + def __init__(self, effect_type: str, reason: str) -> None: + self.effect_type = effect_type + self.reason = reason + super().__init__( + f"'{effect_type}' refused: this agent runs on the durable queue, where a " + f"turn can be re-delivered, and the request carried no usable execution id " + f"({reason}), so the send could not be de-duplicated. Do not retry — an " + f"operator has been alerted." + ) + + # --------------------------------------------------------------------------- # Scope + key derivation # --------------------------------------------------------------------------- @@ -248,39 +281,114 @@ def derive_effect_key( return f"{effect_type}:{h.hexdigest()}" -def resolve_and_validate_execution(execution_id: Optional[str], agent_name: str) -> Optional[Any]: - """Return the execution iff it exists AND belongs to `agent_name`; else None. +def _resolve_execution_with_reason( + execution_id: Optional[str], agent_name: str, *, log: bool = True +) -> "tuple[Optional[Any], str]": + """Resolve an agent's execution; return `(execution, reason)`. - Generalizes the MEM-001 server-side resolution (routers/public_memory.py): - the agent supplies an execution_id, never its own identity, and the backend - confirms ownership. FAIL-OPEN by design — a missing execution_id (old image / - absent tool arg), a lookup error, or an agent mismatch returns None so the - caller proceeds WITHOUT dedup rather than 5xx-ing a legitimate send. Safe - today because pull-mode re-delivery is off (fail-open cannot produce a - duplicate yet); enabling pull-mode default-ON for side-effect agents requires - trusted execution_id injection + fail-closed-when-absent first (a BLOCKING - prerequisite on Epic #1045/#1081 — see the contract doc). + `reason` is `ok` when the execution exists AND belongs to `agent_name`, + else one of `absent` (no id), `manual` (a person's terminal session, see + `MANUAL_EXECUTION_ID`), `unknown` (no such execution), `foreign` (another + agent's execution) or `lookup_error` (the lookup itself failed). """ if not execution_id: - return None + return None, "absent" + if execution_id == MANUAL_EXECUTION_ID: + return None, "manual" try: execution = db.get_execution(execution_id) - except Exception as e: # fail-open: a lookup hiccup must not block a real send - logger.warning( - "resolve_and_validate_execution: get_execution(%s) failed — fail-open: %s", - execution_id, e, - ) - return None + except Exception as e: + if log: + logger.warning( + "resolve_and_validate_execution: get_execution(%s) failed: %s", + execution_id, e, + ) + return None, "lookup_error" if not execution: - return None + return None, "unknown" if getattr(execution, "agent_name", None) != agent_name: - logger.warning( - "resolve_and_validate_execution: execution %s does not belong to agent %s " - "(owner=%s) — fail-open", - execution_id, agent_name, getattr(execution, "agent_name", None), + if log: + logger.warning( + "resolve_and_validate_execution: execution %s does not belong to agent %s " + "(owner=%s)", + execution_id, agent_name, getattr(execution, "agent_name", None), + ) + return None, "foreign" + return execution, "ok" + + +def resolve_and_validate_execution(execution_id: Optional[str], agent_name: str) -> Optional[Any]: + """Return the execution iff it exists AND belongs to `agent_name`; else None. + + Generalizes the MEM-001 server-side resolution (routers/public_memory.py): + the backend confirms ownership of the id it is handed. The id reaches the + effect sinks from the platform — the agent's MCP config carries it per + process (#2392) — with the agent-supplied tool argument as the fallback for + older images. What happens when this returns None is the caller's policy; + `effect_guard` refuses on a pull-mode agent (`_on_unguarded_effect`). + """ + return _resolve_execution_with_reason(execution_id, agent_name)[0] + + +def effect_dedup_required(agent_name: str) -> bool: + """True when this agent's turns can be re-delivered, so an effect must dedup. + + Today that is the pull pilots. Phase 5 (#429) puts every agent on the + durable queue; this is the one place that flips. + """ + from services.pull_pilot import is_pull_pilot_agent + return is_pull_pilot_agent(agent_name) + + +# Reasons that refuse the effect on a pull-mode agent. `manual` has no +# execution to re-deliver; `lookup_error` is a DB blip that must not stop every +# send (and `begin()` would fail open on the same outage anyway). +_REFUSED_REASONS = frozenset({"absent", "unknown", "foreign"}) + + +async def _on_unguarded_effect( + effect_type: str, agent_name: Optional[str], reason: str +) -> None: + """Refuse (pull-mode) or log (otherwise) an effect sent without dedup (#2392). + + Either way the degraded case is visible: an operator alarm for a refusal, + an `effect_guard.degraded` warning for a send that went out unguarded. + """ + if agent_name and reason in _REFUSED_REASONS and effect_dedup_required(agent_name): + logger.error( + "effect_guard.refused effect_type=%s agent=%s reason=%s — " + "pull-mode agent sent an effect without a usable execution id", + effect_type, agent_name, reason, ) - return None - return execution + from services.operator_queue_service import create_bounded_alert + from utils.helpers import utc_now_iso + now = utc_now_iso() + await create_bounded_alert(agent_name, { + "id": f"{EFFECT_UNGUARDED_ALERT_PREFIX}{agent_name}-{now}", + "agent_name": agent_name, + "type": "effect_unguarded", + "status": "pending", + "priority": "high", + "title": "Side effect refused: no execution id", + "question": ( + f"{agent_name} tried to send a '{effect_type}' without a usable execution " + f"id ({reason}). It runs on the durable queue, where a turn can be " + f"re-delivered, so the send was refused rather than risk a duplicate. " + f"Usual causes: the agent runs an image older than #2392 (rebuild the " + f"base image and restart it); the call came from outside a turn (a " + f"person running Codex in the agent's terminal, or a user-scoped key " + f"acting for this agent), which carries no id; or the API was called " + f"directly instead of through the agent's Trinity MCP tools." + ), + "context": {"effect_type": effect_type, "reason": reason}, + "created_at": now, + }) + raise EffectUnguardedError(effect_type, reason) + logger.warning( + "effect_guard.degraded effect_type=%s agent=%s reason=%s — " + "sent without de-duplication", + effect_type, agent_name, reason, + ) class _EffectGuardState: @@ -327,8 +435,12 @@ async def effect_guard( - **fresh claim** → yields `g.dedup_enabled=True`; on clean exit `complete()` stores `g.snapshot`, on exception `fail()` releases the claim so a failed attempt retries. - - **fail-open** → an absent/invalid execution_id (and no payment_request_id), - or a claim hiccup, yields a no-op state so the send proceeds without dedup. + - **no usable execution_id** (and no payment_request_id) → on a pull-mode + agent, raises `EffectUnguardedError` BEFORE yielding and raises an operator + alarm (#2392); otherwise yields a no-op state and logs + `effect_guard.degraded`. A `manual` id or a lookup error always proceeds. + - **fail-open** → a claim hiccup yields a no-op state so the send proceeds + without dedup. Two scopes: - Effect path (messages/voip/share): `effect:{execution_id}`, after @@ -352,7 +464,15 @@ async def effect_guard( else None ) if execution is None: - # Dedup disabled (absent/invalid execution_id) — run the body, no guard. + # Why it did not resolve — re-derived here, not returned above, so + # `resolve_and_validate_execution` stays the one patchable seam. + reason = ( + _resolve_execution_with_reason(execution_id, agent_name, log=False)[1] + if agent_name is not None + else "absent" + ) + # Raises on a pull-mode agent; otherwise the send runs unguarded, logged. + await _on_unguarded_effect(effect_type, agent_name, reason) yield state return scope = make_effect_scope(execution_id) # type: ignore[arg-type] diff --git a/src/backend/services/mcp_validator.py b/src/backend/services/mcp_validator.py index 3cfec4c458..37c43ce52a 100644 --- a/src/backend/services/mcp_validator.py +++ b/src/backend/services/mcp_validator.py @@ -87,6 +87,10 @@ class McpValidationError(ValueError): # documented default for fresh installs. _TRINITY_DEFAULT_URL = "http://mcp-server:8080/mcp" _TRINITY_BEARER_RE = re.compile(r"^Bearer\s+trinity_mcp_[A-Za-z0-9_-]{1,200}$") +# #2392: the injected entry also carries the spawning execution's id, expanded +# by Claude Code from its process env. Only this exact literal is canonical. +_TRINITY_EXEC_ID_HEADER = "X-Trinity-Execution-Id" +_TRINITY_EXEC_ID_VALUE = "${TRINITY_EXECUTION_ID:-manual}" def _is_canonical_trinity_entry(server: dict) -> bool: @@ -94,7 +98,8 @@ def _is_canonical_trinity_entry(server: dict) -> bool: entry: only ``type``, ``url``, ``headers`` keys; ``type=http``; ``url`` matches the configured Trinity MCP URL; ``headers`` contains only ``Authorization`` with a ``Bearer trinity_mcp_…`` token of the right - shape. + shape, optionally plus ``X-Trinity-Execution-Id`` whose value is exactly + ``${TRINITY_EXECUTION_ID:-manual}`` (#2392). Strict allowlist — any extra key, any wrong value, any extra header, any non-bearer auth scheme is rejected. Owners who want to redefine @@ -114,7 +119,12 @@ def _is_canonical_trinity_entry(server: dict) -> bool: return False headers = server.get("headers") - if not isinstance(headers, dict) or list(headers.keys()) != ["Authorization"]: + if not isinstance(headers, dict): + return False + if set(headers.keys()) == {"Authorization", _TRINITY_EXEC_ID_HEADER}: + if headers[_TRINITY_EXEC_ID_HEADER] != _TRINITY_EXEC_ID_VALUE: + return False + elif list(headers.keys()) != ["Authorization"]: return False auth = headers.get("Authorization") if not isinstance(auth, str) or not _TRINITY_BEARER_RE.match(auth): diff --git a/src/backend/services/operator_queue_service.py b/src/backend/services/operator_queue_service.py index 392572a899..d4720a48cc 100644 --- a/src/backend/services/operator_queue_service.py +++ b/src/backend/services/operator_queue_service.py @@ -140,6 +140,10 @@ # tool, which an agent-scoped key may call on itself — so an agent CAN drive # the volume, which is the whole test the #1677 classification applies. "gitignore_untracked", + # #2392: an effect refused on a pull-mode agent for lack of an execution id. + # Budgeted because the agent drives the volume — every send it attempts + # without an id is one refusal. + "effect_unguarded", }) # Shape guard for the episode alert's `last_triggered_by` triage field: a @@ -181,6 +185,9 @@ "cb-dormant-", # agent_client circuit-breaker-dormant alert "sync-failing-", # sync_health_service "git-bloat-", # sync_health_service + "sync-diverged-", # sync_health_service divergence-freeze episode item + # (trinity-enterprise#706) — deterministic id, so the + # reservation is what keeps it unsuppressible "skill-not-found-", # task_execution_service "val_", # validation_service "system-seed-", # system_seed_service first-run seed alerts (ent#124) @@ -188,6 +195,7 @@ "alert-budget-", # this service's #1677 budget episode alert (deterministic # bucketed id — reservation is what keeps the DB # on-conflict dedup from being agent-pre-suppressible) + "effect-unguarded-", # idempotency_service refused-effect alarm (#2392) "db-backup-", # db_backup_service failure/staleness alarms (#2216) "log-archive-", # archive_storage unwritable-directory alarm (#2205) "sub-headroom-", # subscription_headroom_alerts weekly-window alarm (ent#434) @@ -205,6 +213,11 @@ "workspace-problem-", # client_portal report-a-problem (ent#499) — reserved # so an agent cannot pre-create the id of a complaint # ABOUT ITSELF and silence it through ON CONFLICT + "portal-inbox-collision-", # client_portal shared legacy inbox (ent#308). + # Its text lists client addresses: reserved (#715) so + # `is_platform_minted` keeps it out of the agent's file + # and its resume turn, and so an agent cannot pre-create + # the id and swallow the alert through ON CONFLICT # git_service per-Push sweep alert (#2529). Reserved for the #1632 reason # above, and — since ent#499 keyed `is_platform_minted` on this very tuple — # this listing is ALSO what keeps the alert out of the agent's own @@ -279,6 +292,21 @@ def is_platform_minted(item) -> bool: return str(candidate).strip().lower().startswith(_RESERVED_ID_PREFIXES) +# #715: the platform alarms whose TEXT is about a person — a Workspace client's +# complaint (ent#499: their email and verbatim words, which ent#366 withholds from +# the rated agent) and the client addresses behind a shared legacy inbox (ent#308, +# whose id also carries an email slug). A subset of `_RESERVED_ID_PREFIXES`, so +# no agent can mint a row into it. +_ABOUT_A_PERSON_ID_PREFIXES = ("workspace-problem-", "portal-inbox-collision-") + + +def is_about_a_person(item: dict) -> bool: + """Is this row one of the platform's heads-ups ABOUT a person (#715)? They + are the operator's: the queue's reads never return one to a machine key.""" + candidate = item.get("request_id") or "" + return str(candidate).strip().lower().startswith(_ABOUT_A_PERSON_ID_PREFIXES) + + def _truncate_with_marker(text: str, max_len: int) -> str: """Truncate so the RESULT (content + marker) is ≤ max_len chars.""" if len(text) <= max_len: @@ -693,10 +721,11 @@ def _well_formed_queue(data) -> bool: def _deliver_into(req: dict, resp: dict) -> None: + # #715: never who answered — the agent's file carries the answer, not a + # person's email. req["status"] = "responded" req["response"] = resp["response"] req["response_text"] = resp.get("response_text") - req["responded_by"] = resp.get("responded_by_email") req["responded_at"] = resp.get("responded_at") @@ -1907,8 +1936,7 @@ async def _write_responses_to_agent( "created_at": resp.get("created_at", ""), "response": resp["response"], "response_text": resp.get("response_text"), - "responded_by": resp.get("responded_by_email"), - "responded_at": resp.get("responded_at"), + "responded_at": resp.get("responded_at"), # #715: never who }) updated = True delivered[resp["id"]] = resp diff --git a/src/backend/services/platform_prompt_service.py b/src/backend/services/platform_prompt_service.py index 6c7a9d2e2e..9ad1d5497e 100644 --- a/src/backend/services/platform_prompt_service.py +++ b/src/backend/services/platform_prompt_service.py @@ -185,7 +185,7 @@ } ``` -The operator's answer is written back into the entry: `status: "responded"` with `response`, `responded_by` and `responded_at`. An item that has waited past the operator's aging bound carries a `platform.aging_since` timestamp written by Trinity — read it, never write to `platform`. After processing a response, update the item's status to `"acknowledged"`. Keep only `pending` and `responded` items plus up to 3 recent `acknowledged` items. An ID you raised with `ask_operator` is never read from the file. +The operator's answer is written back into the entry: `status: "responded"` with `response`, `response_text` and `responded_at` — never who answered. An item that has waited past the operator's aging bound carries a `platform.aging_since` timestamp written by Trinity — read it, never write to `platform`. After processing a response, update the item's status to `"acknowledged"`. Keep only `pending` and `responded` items plus up to 3 recent `acknowledged` items. An ID you raised with `ask_operator` is never read from the file. #### When to Use diff --git a/src/backend/services/proactive_message_service.py b/src/backend/services/proactive_message_service.py index fcbf31b893..cc2263bb1c 100644 --- a/src/backend/services/proactive_message_service.py +++ b/src/backend/services/proactive_message_service.py @@ -193,7 +193,8 @@ async def send_message( turn it ran in is re-delivered (pull-mode at-least-once), the same resolved (recipient, channel) within the same execution dedupes to exactly one send — the LLM-generated body is NOT part of the key. - Fail-open when absent/invalid (old image): the send proceeds. + Without a usable id: refused on a pull-mode agent, else sent and + logged as degraded (#2392, `idempotency_service.effect_guard`). dedup_label: Optional agent-supplied discriminator (#1084) so an agent can intentionally send two distinct messages to the same recipient in one turn. Default "" → at-most-one send per (recipient, channel) diff --git a/src/backend/services/sync_freeze_policy.py b/src/backend/services/sync_freeze_policy.py new file mode 100644 index 0000000000..4388992479 --- /dev/null +++ b/src/backend/services/sync_freeze_policy.py @@ -0,0 +1,257 @@ +"""Sync-health policy: state, reason, recommendation and the freeze (trinity-enterprise#706). + +ONE place decides whether an agent and its repository agree. Every surface that +shows sync health (the dashboard dot, `GET /api/agents/sync-health`, +`GET /api/agents/{name}/git/sync-state`, the internal freeze endpoint) and the +scheduler's own freeze gate call :func:`classify` — the frontend owns no +threshold. + +CANONICAL SOURCE: src/backend/services/sync_freeze_policy.py. The scheduler is a +standalone image that cannot import the backend, so src/scheduler/sync_freeze_policy.py +is a BYTE-IDENTICAL vendored mirror (the failure_classifier.py precedent, #1088; +Invariant #5), pinned by tests/unit/test_ent706_sync_policy_parity.py. Edit this +file, then `cp` it over the mirror. + +Stdlib only, no I/O, never raises on a malformed row: the scheduler calls this +on its fire path, and every number in the row was written by the agent. + +Rules, evaluated together (the first red clause names the reason, the rest +append): + + unknown no row, or the poller never observed the agent + red the last sync failed (any binding) + a WORK agent diverged from origin for more than 24 h + a WORK agent's tree dirty for more than 24 h + auto-sync on and no heartbeat for 7 days (fresh observation only) + yellow a work agent diverged for 24 h or less + a DEPLOYMENT diverged at any age, or dirty for more than 24 h — + a deployment being behind is normal; it is never red on age + green otherwise + +A work agent is `source_mode = 0 OR auto_sync_enabled = 1`: a working branch, +or a fork-to-own / bound-to-own agent that auto-syncs to its own repository +(those keep `source_mode = 1` in the DB). Everything else is a deployment. + +freeze = freeze_schedules_if_sync_failing AND ( + sync_failing # #1808, unchanged, any binding + OR (work agent AND diverged > 24 h AND observation fresh)) + +The freshness guard fails OPEN on purpose: the poller writes nothing for an +unreachable agent, so a stale `diverged_since` must never keep an agent frozen +after it may already have pushed. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +SYNC_FAILURE_FREEZE_THRESHOLD = 3 # consecutive POLLS that saw `failed` (#1808) +DIVERGENCE_RED_SECONDS = 24 * 3600 +DIRTY_RED_SECONDS = 24 * 3600 +NO_HEARTBEAT_FLOOR_SECONDS = 7 * 24 * 3600 +OBSERVATION_FRESH_SECONDS = 15 * 60 +ERROR_EXCERPT_CHARS = 120 + +STATES = ("green", "yellow", "red", "unknown") + +# The agent's own refusal text for a source-mode agent on the shared default +# branch (#3011, `failed: "refused: source-mode on "`). +_SOURCE_MODE_REFUSAL = "refused: source-mode" + + +def parse_ts(value: Any) -> Optional[datetime]: + """An aware-UTC datetime from an ISO string or a datetime; None otherwise. + + A naive value is UTC — the convention of `parse_scheduler_ts` and of the + backend's `parse_iso_timestamp`. Every comparison happens here in Python, + never lexicographically in SQL (Invariant #16). + """ + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str) and 0 < len(value) <= 64: + text = value.strip() + if text.endswith(("Z", "z")): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + else: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def format_age(seconds: int) -> str: + """`45m`, `26h`, `9d` — the shape of the operator-facing reason strings.""" + seconds = max(int(seconds), 0) + if seconds < 3600: + return f"{seconds // 60}m" + if seconds < 48 * 3600: + return f"{seconds // 3600}h" + return f"{seconds // 86400}d" + + +def _count(value: Any) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int) and value > 0: + return value + return 0 + + +def _flag(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in ("1", "true") + return bool(value) + + +def _age(now: datetime, then: Optional[datetime]) -> Optional[int]: + if then is None: + return None + return max(int((now - then).total_seconds()), 0) + + +def is_work_agent(source_mode: Any, auto_sync_enabled: Any) -> bool: + """`source_mode = 0 OR auto_sync_enabled = 1` (see the module docstring).""" + return (not _flag(source_mode)) or _flag(auto_sync_enabled) + + +def classify( + row: Optional[Dict[str, Any]], + cfg: Optional[Dict[str, Any]], + now: Optional[datetime] = None, + *, + push_denied: bool = False, +) -> Dict[str, Any]: + """The verdict for one agent. + + `row` is its `agent_sync_state` row (or None). `cfg` carries the + `agent_git_config` flags: `source_mode`, `auto_sync_enabled`, + `freeze_schedules_if_sync_failing`, and optionally `created_at` (the + no-heartbeat floor's start when no heartbeat was ever recorded). + `push_denied` is `git_service.is_push_denied(last_error_summary)`, computed + by the backend caller so this module stays a leaf. + """ + now = parse_ts(now) or datetime.now(timezone.utc) + cfg = cfg or {} + auto_sync = _flag(cfg.get("auto_sync_enabled")) + work = is_work_agent(cfg.get("source_mode"), auto_sync) + freeze_flag = _flag(cfg.get("freeze_schedules_if_sync_failing")) + + verdict: Dict[str, Any] = { + "state": "unknown", + "reason": "no sync observation yet", + "recommendation": None, + "binding": "agent" if work else "deployment", + "work_agent": work, + "divergence_age_s": None, + "dirty_age_s": None, + "sync_failing": False, + "freeze": False, + "freeze_cause": None, + "freeze_reason": None, + "stale_observation": False, + } + if not row: + return verdict + observed_age = _age(now, parse_ts(row.get("last_check_at"))) + if observed_age is None: + return verdict + stale = observed_age > OBSERVATION_FRESH_SECONDS + + ahead = _count(row.get("ahead_working")) + behind = _count(row.get("behind_working")) + ahead_main = _count(row.get("ahead_main")) + behind_main = _count(row.get("behind_main")) + failures = _count(row.get("consecutive_failures")) + dirty = _count(row.get("dirty_files")) + error = row.get("last_error_summary") + error = error.strip() if isinstance(error, str) else "" + failed = row.get("last_sync_status") == "failed" + sync_failing = failed and failures >= SYNC_FAILURE_FREEZE_THRESHOLD + + divergence_age = _age(now, parse_ts(row.get("diverged_since"))) + dirty_age = _age(now, parse_ts(row.get("dirty_since"))) if dirty else None + + reds: List[str] = [] + yellows: List[str] = [] + + failed_clause = None + if failed: + plural = "" if failures == 1 else "s" + failed_clause = f"last sync failed (seen on {failures} poll{plural})" + if error: + failed_clause += f": {error[:ERROR_EXCERPT_CHARS]}" + reds.append(failed_clause) + + divergence_clause = None + divergence_red = False + if divergence_age is not None: + divergence_clause = ( + f"diverged {behind} behind / {ahead} ahead for {format_age(divergence_age)}" + ) + # D7: `main` moving under a working branch is information, not divergence. + if behind_main and (ahead_main, behind_main) != (ahead, behind): + divergence_clause += f" (main is {behind_main} ahead)" + divergence_red = work and divergence_age > DIVERGENCE_RED_SECONDS + (reds if divergence_red else yellows).append(divergence_clause) + + if dirty_age is not None and dirty_age > DIRTY_RED_SECONDS: + clause = f"dirty: {dirty} files uncommitted for {format_age(dirty_age)}" + (reds if work else yellows).append(clause) + + heartbeat_missing = False + if auto_sync and not stale: + heartbeat = parse_ts(row.get("last_sync_at")) or parse_ts(cfg.get("created_at")) + heartbeat_age = _age(now, heartbeat) + if heartbeat_age is not None and heartbeat_age > NO_HEARTBEAT_FLOOR_SECONDS: + heartbeat_missing = True + reds.append(f"auto-sync on, no heartbeat for {format_age(heartbeat_age)}") + + if reds: + state, reason = "red", "; ".join(reds + yellows) + elif yellows: + state, reason = "yellow", "; ".join(yellows) + else: + state, reason = "green", "in sync" + if stale: + reason += f" (last observed {format_age(observed_age)} ago)" + + freeze_cause = freeze_reason = None + if freeze_flag: + if sync_failing: + freeze_cause, freeze_reason = "sync_failing", failed_clause + elif divergence_red and not stale: + freeze_cause, freeze_reason = "divergence", divergence_clause + + if failed: + if push_denied: + recommendation = "credential is read-only" + elif _SOURCE_MODE_REFUSAL in error: + recommendation = "deployment: turn auto-sync off" + else: + recommendation = "push via git_sync strategy=pull_first" + elif work and ahead and not auto_sync: + recommendation = "enable auto-sync" + elif behind: + recommendation = "pull via git_pull" + elif heartbeat_missing: + recommendation = "restart the agent to resume the auto-sync heartbeat" + else: + recommendation = None + + verdict.update( + state=state, + reason=reason, + recommendation=recommendation, + divergence_age_s=divergence_age, + dirty_age_s=dirty_age, + sync_failing=sync_failing, + freeze=freeze_cause is not None, + freeze_cause=freeze_cause, + freeze_reason=freeze_reason, + stale_observation=stale, + ) + return verdict diff --git a/src/backend/services/sync_health_service.py b/src/backend/services/sync_health_service.py index 7cf091b5c8..779f91a220 100644 --- a/src/backend/services/sync_health_service.py +++ b/src/backend/services/sync_health_service.py @@ -51,8 +51,10 @@ from database import db from redis_breaker_util import get_breaker_redis from services import git_service +from services import sync_freeze_policy +from services.sync_health_view import sync_view from services.agent_client import AgentClient -from utils.helpers import parse_iso_timestamp, utc_now_iso +from utils.helpers import parse_iso_timestamp, to_utc_iso, utc_now_iso logger = logging.getLogger(__name__) @@ -97,6 +99,11 @@ def _poll_interval_seconds() -> int: GIT_DIR_ALERT_BYTES = int(os.getenv("GIT_DIR_ALERT_BYTES", str(10 * 1024**3))) MAINTENANCE_FAILURES_ALERT_THRESHOLD = 3 +# trinity-enterprise#706: id prefix of the one-per-episode divergence-freeze +# item. Platform-reserved in operator_queue_service._RESERVED_ID_PREFIXES, so an +# agent cannot pre-create the id and silence the alarm about itself. +SYNC_DIVERGED_ALERT_PREFIX = "sync-diverged-" + # PostgreSQL column ceilings. `Integer` is int4 there (SQLite's INTEGER is 64-bit # either way, which is how #2800 shipped): a value the boundary admits but the @@ -232,6 +239,80 @@ def _coerce_lock_stuck(value): return None return {k: v for k, v in coerced.items() if v is not None} +def _coerce_agent_timestamp(value) -> Optional[datetime]: + """An agent-authored ISO timestamp as aware UTC, or None (trinity-enterprise#706). + + The `_coerce_lock_recovery` posture for a value that becomes a column: a + bounded string with an EXPLICIT offset (our writer always stamps one, so a + naive value did not come from us), parseable, and not in the future beyond + the same grace. A far-future value would otherwise win every `max()` below + forever. Never raises. + """ + if not isinstance(value, str) or not (0 < len(value) <= 64): + return None + if not (value.endswith("Z") or _ISO_OFFSET_RE.search(value)): + return None + try: + parsed = parse_iso_timestamp(value) + now = datetime.now(timezone.utc) + if parsed > now + timedelta(seconds=_LOCK_RECOVERY_FUTURE_GRACE_SECONDS): + return None + except (ValueError, TypeError, OverflowError): + return None + return parsed + + +def _config_value(config, key: str): + """Read a git-config field from the model the poller gets, or a dict.""" + if isinstance(config, dict): + return config.get(key) + return getattr(config, key, None) + + +def _last_successful_push_at(sync_state: dict, config, prior: Optional[dict]) -> Optional[str]: + """The last push that landed, monotonic (trinity-enterprise#706). + + The later of: the heartbeat's own record (`last_successful_push_at`, #3011); + an operator Push (`agent_git_config.last_sync_at`, stamped only on success); + and the value already stored. An agent image older than #3011 has no such + key at all, so its last `success` cycle stands in. A #3011 image that has + never pushed carries the key as None and does NOT fall back — a + success-status cycle there is not evidence that anything was pushed. + """ + candidates = [] + if "last_successful_push_at" in sync_state: + candidates.append(_coerce_agent_timestamp(sync_state.get("last_successful_push_at"))) + elif sync_state.get("last_sync_status") == "success": + candidates.append(_coerce_agent_timestamp(sync_state.get("last_sync_at"))) + operator_push = _config_value(config, "last_sync_at") + if isinstance(operator_push, datetime): + candidates.append( + operator_push if operator_push.tzinfo else operator_push.replace(tzinfo=timezone.utc) + ) + stored = (prior or {}).get("last_successful_push_at") + if isinstance(stored, str): + try: + candidates.append(parse_iso_timestamp(stored)) + except ValueError: + pass + valid = [c for c in candidates if c is not None] + return to_utc_iso(max(valid)) if valid else None + + +def _episode_clock(prior_value, *, active: bool, known: bool, now: str): + """Set-once / clear / keep for `diverged_since` and `dirty_since` (#706). + + Returns the value to store: the prior start while the episode continues, a + fresh `now` when one begins, None when it ends, and the prior value + untouched when the observation could not be computed. + """ + if active: + return prior_value or now + if not known: + return prior_value + return None + + # WebSocket manager injected from main.py (optional, mirrors operator-queue pattern). _websocket_manager = None @@ -263,6 +344,11 @@ def __init__(self, poll_interval: Optional[int] = None): # far-future `at` would flood the log for the life of the process. self._last_lock_recovery: Dict[str, Dict] = {} self._lock_stuck_agents: set = set() + # trinity-enterprise#706: the last sync_diverged item id this process + # raised per agent — only to skip a redundant INSERT every poll while a + # freeze lasts. The DB's (agent_name, request_id) conflict target is + # what makes it one item per episode across restarts and workers. + self._diverged_alerted: Dict[str, str] = {} @property def poll_interval(self) -> int: @@ -438,6 +524,36 @@ async def _sync_agent(self, config) -> None: sync_state.get("maintenance_failures") ) + # trinity-enterprise#706: the divergence and dirt episode clocks. The + # decision reads the RAW values, so an uncomputable count (None) is + # never mistaken for 0. `behind is None` with a known `ahead == 0` is + # "no upstream" (#2105) — nothing on origin to be behind — and clears. + # The prior value is passed explicitly for "keep" (no sentinel), so a + # set-once start survives any number of polls. + now_iso = utc_now_iso() + ahead_raw = _coerce_nonneg_int(payload.get("ahead_working")) + behind_raw = _coerce_nonneg_int(payload.get("behind_working")) + diverged_since = _episode_clock( + (prior or {}).get("diverged_since"), + active=bool(ahead_raw) or bool(behind_raw), + known=ahead_raw is not None, + now=now_iso, + ) + # A clock KEPT because the tuple was uncomputable keeps the counts that + # started it: `_coerce_counter` reads the unknown as 0, which would sit + # next to a live `diverged_since` and freeze with "0 behind / 0 ahead". + # None is `upsert`'s "keep the prior value" (#706 /review). + keep_working_counts = ( + ahead_raw is None and not behind_raw and diverged_since is not None + ) + dirty_files = _coerce_nonneg_int(payload.get("changes_count")) + dirty_since = _episode_clock( + (prior or {}).get("dirty_since"), + active=bool(dirty_files), + known=dirty_files is not None, + now=now_iso, + ) + updated = db.upsert_sync_state( agent_name, last_sync_at=last_sync_at, @@ -448,13 +564,22 @@ async def _sync_agent(self, config) -> None: # #2827: coerced like their siblings — these four went in raw. ahead_main=_coerce_counter(payload, "ahead_main", "ahead"), behind_main=_coerce_counter(payload, "behind_main", "behind"), - ahead_working=_coerce_counter(payload, "ahead_working"), - behind_working=_coerce_counter(payload, "behind_working"), + ahead_working=( + None if keep_working_counts + else _coerce_counter(payload, "ahead_working")), + behind_working=( + None if keep_working_counts + else _coerce_counter(payload, "behind_working")), git_dir_bytes=git_dir_bytes, # #1596 bloat observability pack_count=pack_count, # #1595 loose_objects=loose_objects, # #1595 maintenance_failures=maintenance_failures, # #1595 - last_check_at=utc_now_iso(), + diverged_since=diverged_since, # trinity-enterprise#706 + dirty_files=dirty_files, # #706: None (garbage/absent) keeps the prior + dirty_since=dirty_since, # #706 + last_successful_push_at=_last_successful_push_at( # #706 + sync_state, config, prior), + last_check_at=now_iso, ) # Edge-triggered alert: only emit when we cross the threshold. @@ -462,6 +587,13 @@ async def _sync_agent(self, config) -> None: if prior_failures < ALERT_THRESHOLD <= new_failures: self._emit_sync_failing_alert(agent_name, updated) + # trinity-enterprise#706: a divergence freeze is in force → one + # sync_diverged item for the episode. Rule 2 (sync_failing) already + # has its own item above, so only the divergence cause raises this. + verdict = sync_view(updated, config) + if verdict["freeze_cause"] == "divergence": + self._emit_sync_diverged_alert(agent_name, updated, verdict) + # #2742: a self-healed wedge, announced once. No operator-queue item # and no DB column — the boot reap already fixed it, so this is a log # line an operator can find, not a decision anyone has to make. @@ -593,6 +725,51 @@ def _emit_sync_failing_alert(self, agent_name: str, state: Dict) -> None: except Exception: logger.exception("failed to emit sync_failing alert") + def _emit_sync_diverged_alert(self, agent_name: str, state: Dict, verdict: Dict) -> None: + """trinity-enterprise#706: one operator-queue item per divergence episode. + + The id is deterministic per episode (`diverged_since` is set once), so + every repeat poll, a restart and a fail-open double leader converge on + ONE row via create_item's `(agent_name, request_id)` conflict target — + the `alert-budget-` precedent, with no new column. Not auto-resolved + when the episode ends; a new episode has a new id. The context carries + counts and the recommendation, never agent-authored error text. + """ + diverged_since = state.get("diverged_since") + item_id = f"{SYNC_DIVERGED_ALERT_PREFIX}{agent_name}-{diverged_since}" + if self._diverged_alerted.get(agent_name) == item_id: + return + age = sync_freeze_policy.format_age(verdict["divergence_age_s"] or 0) + item = { + "id": item_id, + "agent_name": agent_name, + "type": "sync_diverged", + "status": "pending", + "priority": "high", + "title": f"Agent diverged from GitHub for {age} — schedules paused", + "question": ( + f"{agent_name}: {verdict['freeze_reason']}. Its scheduled runs are " + "paused until it is back in step with its repository." + ), + "context": { + "ahead": state.get("ahead_working") or 0, + "behind": state.get("behind_working") or 0, + "dirty_files": state.get("dirty_files"), + "diverged_since": diverged_since, + "divergence_age_s": verdict["divergence_age_s"], + "recommendation": verdict["recommendation"], + }, + "created_at": utc_now_iso(), + } + try: + db.create_operator_queue_item(agent_name, item) + self._diverged_alerted[agent_name] = item_id + logger.warning( + "sync_diverged raised for %s (%s)", agent_name, verdict["freeze_reason"] + ) + except Exception: + logger.exception("failed to emit sync_diverged alert") + def _emit_git_bloat_alert(self, agent_name: str, state: Dict, *, reason: str) -> None: """#1595: operator-queue entry for repo bloat / failing maintenance. diff --git a/src/backend/services/sync_health_view.py b/src/backend/services/sync_health_view.py new file mode 100644 index 0000000000..ab9eb2b9e9 --- /dev/null +++ b/src/backend/services/sync_health_view.py @@ -0,0 +1,70 @@ +"""One call, one answer: an agent's sync-health view (trinity-enterprise#706). + +Every backend surface that shows sync health goes through :func:`sync_view`, +which feeds the persisted `agent_sync_state` row and the `agent_git_config` +flags to `sync_freeze_policy.classify`. The policy module is a stdlib leaf +(the scheduler vendors it), so the two backend-only inputs are resolved here: + +- `push_denied` — `git_service.is_push_denied(last_error_summary)` (#2107), + which turns a refused push into the `credential is read-only` recommendation; +- the git-config model — read into the plain dict `classify` takes. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from services import git_service +from services.sync_freeze_policy import classify + +# The computed fields every surface adds next to the row it already returns. +VIEW_FIELDS = ( + "state", + "reason", + "recommendation", + "binding", + "work_agent", + "divergence_age_s", + "dirty_age_s", + "sync_failing", + "freeze", + "freeze_cause", + "freeze_reason", + "stale_observation", +) + +_CONFIG_FIELDS = ( + "source_mode", + "auto_sync_enabled", + "freeze_schedules_if_sync_failing", + "created_at", +) + + +def _config_dict(config: Any) -> Dict[str, Any]: + if config is None: + return {} + if isinstance(config, dict): + return {key: config.get(key) for key in _CONFIG_FIELDS} + return {key: getattr(config, key, None) for key in _CONFIG_FIELDS} + + +def sync_view( + row: Optional[Dict[str, Any]], + config: Any, + now: Optional[datetime] = None, +) -> Dict[str, Any]: + """The policy verdict for one agent, restricted to `VIEW_FIELDS`. + + `row` is its `agent_sync_state` row or None; `config` its + `AgentGitConfig` (or a dict with the same fields) or None when the agent + has no git binding. + """ + error = (row or {}).get("last_error_summary") or "" + verdict = classify( + row, + _config_dict(config), + now or datetime.now(timezone.utc), + push_denied=git_service.is_push_denied(error), + ) + return {key: verdict[key] for key in VIEW_FIELDS} diff --git a/src/backend/services/voip_service.py b/src/backend/services/voip_service.py index ee893b7338..30f2cdc51a 100644 --- a/src/backend/services/voip_service.py +++ b/src/backend/services/voip_service.py @@ -150,7 +150,8 @@ async def place_outbound_call( at-least-once), the same number within the same execution replays the original {call_id, status, ...} instead of placing a second PSTN call. The router's boundary Idempotency-Key gate stays the OUTER layer. - Fail-open when ``execution_id`` is absent/invalid (old image). + Without a usable ``execution_id``: refused on a pull-mode agent, else + placed and logged as degraded (#2392). """ if not self.is_available(): raise HTTPException(status_code=404, detail="VoIP is not enabled") diff --git a/src/frontend/src/components/portal/portalUtils.js b/src/frontend/src/components/portal/portalUtils.js index cdfdf21c71..2834275d53 100644 --- a/src/frontend/src/components/portal/portalUtils.js +++ b/src/frontend/src/components/portal/portalUtils.js @@ -2063,7 +2063,6 @@ export function replyFromHistory(messages, baseline) { if (!isNew) return null return { response: last.content, - cost: last.cost ?? null, id: last.id || null, myRating: last.my_rating || null, } diff --git a/src/frontend/src/utils/syncHealth.js b/src/frontend/src/utils/syncHealth.js index e47a64858e..705a1eef2b 100644 --- a/src/frontend/src/utils/syncHealth.js +++ b/src/frontend/src/utils/syncHealth.js @@ -1,32 +1,26 @@ /** - * Sync health indicator helpers (#389 S1). + * Sync health indicator helpers (#389 S1, trinity-enterprise#706). * - * Classifies a per-agent sync-health entry into one of four colors so the - * dashboard can render a small dot next to each agent: + * The BACKEND owns the sync state. `GET /api/agents/sync-health` serves each + * agent's `state` (green / yellow / red / unknown), an operator-readable + * `reason` and a `recommendation`, all computed by one policy module + * (src/backend/services/sync_freeze_policy.py — the same rule the scheduler's + * freeze uses). This file only maps that state to a colour and shows the + * reason; it holds no threshold, so the dot can never disagree with the + * freeze. * - * green — last sync succeeded, recent, working branch in sync - * yellow — last sync succeeded but stale (>24h, <7d) - * red — failed, very stale (>7d), OR behind_working > 0 (P6: peer wrote) - * gray — no sync attempt yet, or auto-sync disabled and never manual-synced + * green — the agent and its repository agree + * yellow — diverged for 24 h or less (any divergence for a deployment) + * red — a failed sync, diverged > 24 h, dirty > 24 h, or auto-sync on + * with no heartbeat for 7 days + * gray — no observation yet (`unknown`), or no entry */ -const DAY_MS = 24 * 60 * 60 * 1000 -const WEEK_MS = 7 * DAY_MS +const STATE_COLOURS = new Set(['green', 'yellow', 'red']) export function classifySyncHealth(entry) { - if (!entry) return 'gray' - const status = entry.last_sync_status || 'never' - if (status === 'never') return 'gray' - - if (entry.behind_working && entry.behind_working > 0) return 'red' - if (status === 'failed') return 'red' - - const lastAt = entry.last_sync_at ? new Date(entry.last_sync_at).getTime() : null - if (!lastAt) return 'gray' - const age = Date.now() - lastAt - if (age >= WEEK_MS) return 'red' - if (age >= DAY_MS) return 'yellow' - return 'green' + const state = entry && entry.state + return STATE_COLOURS.has(state) ? state : 'gray' } export function syncHealthColor(entry) { @@ -39,19 +33,8 @@ export function syncHealthColor(entry) { } export function syncHealthLabel(entry) { - if (!entry) return 'Sync status unknown' - const status = entry.last_sync_status || 'never' - if (status === 'never') return 'No sync attempts yet' - if (entry.behind_working && entry.behind_working > 0) { - return `Working branch has ${entry.behind_working} unseen commit(s) from a peer` - } - if (status === 'failed') { - const summary = entry.last_error_summary || 'unknown error' - const failures = entry.consecutive_failures || 0 - return `Last sync failed (${failures} in a row): ${summary}` - } - if (entry.last_sync_at) { - return `Last synced ${new Date(entry.last_sync_at).toLocaleString()}` - } - return 'Sync status: success' + if (!entry || !entry.reason) return 'Sync status unknown' + return entry.recommendation + ? `${entry.reason} — ${entry.recommendation}` + : entry.reason } diff --git a/src/frontend/tests/unit/portalReplyRateable.spec.js b/src/frontend/tests/unit/portalReplyRateable.spec.js index 40e52b8e51..b6c0acb2b6 100644 --- a/src/frontend/tests/unit/portalReplyRateable.spec.js +++ b/src/frontend/tests/unit/portalReplyRateable.spec.js @@ -71,7 +71,8 @@ describe('#2580 — replyFromHistory reads the row the server wrote', () => { it('returns the newest assistant row WITH its id once it differs from the baseline', () => { const before = replyBaseline(hist(1)) const reply = replyFromHistory([...hist(2), { role: 'user', content: 'q' }], before) - expect(reply).toEqual({ response: 'a1', cost: 1, id: 'm1', myRating: null }) + // #3063: no cost — history no longer carries it, and nothing read it off the reply. + expect(reply).toEqual({ response: 'a1', id: 'm1', myRating: null }) }) it('carries an existing rating through', () => { diff --git a/src/frontend/tests/unit/syncHealth.spec.js b/src/frontend/tests/unit/syncHealth.spec.js new file mode 100644 index 0000000000..0ea8d2f8d9 --- /dev/null +++ b/src/frontend/tests/unit/syncHealth.spec.js @@ -0,0 +1,88 @@ +/** + * trinity-enterprise#706 — the dashboard dot renders the BACKEND's sync state. + * + * `utils/syncHealth.js` used to own the thresholds (24 h yellow, 7 d red, + * behind_working > 0 red) and computed a colour from raw fields. The backend + * now serves `state` / `reason` / `recommendation` from one policy module + * (services/sync_freeze_policy.py), so this file maps state → colour and + * shows the reason — it must not second-guess the state from timestamps. + */ +import { describe, it, expect } from 'vitest' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { + classifySyncHealth, + syncHealthColor, + syncHealthLabel, +} from '../../src/utils/syncHealth.js' + +const MONTH_AGO = new Date(Date.now() - 30 * 24 * 3600 * 1000).toISOString() +const NOW = new Date().toISOString() + +describe('classifySyncHealth renders the backend state', () => { + it.each([ + ['green', 'green'], + ['yellow', 'yellow'], + ['red', 'red'], + ['unknown', 'gray'], + ])('state %s → %s', (state, colour) => { + expect(classifySyncHealth({ state })).toBe(colour) + }) + + it('a missing entry or a missing state is gray', () => { + expect(classifySyncHealth(null)).toBe('gray') + expect(classifySyncHealth({ last_sync_status: 'success', last_sync_at: NOW })).toBe('gray') + expect(classifySyncHealth({ state: 'purple' })).toBe('gray') + }) + + it('never re-derives the state from timestamps or counters', () => { + // A month-old heartbeat the backend calls green stays green… + expect(classifySyncHealth({ state: 'green', last_sync_at: MONTH_AGO })).toBe('green') + // …and the 15-ahead auto-sync-off agent the issue describes is what the + // backend says, not "never synced → gray". + expect(classifySyncHealth({ + state: 'red', last_sync_status: 'never', ahead_working: 15, behind_working: 0, + })).toBe('red') + // behind_working > 0 is no longer an automatic red. + expect(classifySyncHealth({ state: 'yellow', behind_working: 31 })).toBe('yellow') + }) +}) + +describe('syncHealthColor', () => { + it.each([ + ['green', 'bg-status-success-500'], + ['yellow', 'bg-status-warning-500'], + ['red', 'bg-status-danger-500'], + ['unknown', 'bg-gray-400'], + ])('state %s → %s', (state, cls) => { + expect(syncHealthColor({ state })).toBe(cls) + }) +}) + +describe('syncHealthLabel is the backend reason', () => { + it('shows the reason, and the recommendation when there is one', () => { + expect(syncHealthLabel({ + state: 'red', + reason: 'diverged 31 behind / 0 ahead for 26h', + recommendation: 'pull via git_pull', + })).toBe('diverged 31 behind / 0 ahead for 26h — pull via git_pull') + expect(syncHealthLabel({ state: 'green', reason: 'in sync', recommendation: null })) + .toBe('in sync') + }) + + it('has a fallback for no entry or no reason', () => { + expect(syncHealthLabel(null)).toBe('Sync status unknown') + expect(syncHealthLabel({ state: 'unknown' })).toBe('Sync status unknown') + }) +}) + +describe('no thresholds live in the frontend', () => { + // @source-text-pin: a util module, not an SFC — the pin is that the + // threshold constants and the clock are gone from the file entirely. + const src = readFileSync(resolve(__dirname, '../../src/utils/syncHealth.js'), 'utf8') + it('has no day/week constants and never reads the clock', () => { + expect(src).not.toMatch(/DAY_MS|WEEK_MS/) + expect(src).not.toMatch(/Date\.now\(\)/) + expect(src).not.toMatch(/behind_working/) + }) +}) diff --git a/src/mcp-server/src/execution-id-transport.test.ts b/src/mcp-server/src/execution-id-transport.test.ts new file mode 100644 index 0000000000..da53e3bc7f --- /dev/null +++ b/src/mcp-server/src/execution-id-transport.test.ts @@ -0,0 +1,93 @@ +/** + * #2392 — X-Trinity-Execution-Id is read per request, not per session. + * + * Real fastmcp server + real StreamableHTTP client, ONE session, two + * send_message calls whose POSTs carry different header values. A handler + * test cannot see this: freshness depends on fastmcp re-running + * `authenticate` on every POST (see inline-auth-transport.test.ts, #2035). + */ +import { strict as assert } from "node:assert"; +import { after, before, describe, it } from "node:test"; +import { createServer as createHttpServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; + +import { createServer } from "./server.js"; + +describe("#2392 execution id header is per-request (real transport)", () => { + let backend: Server; + let mcpServer: { stop: () => Promise }; + let mcpUrl: URL; + const sentBodies: any[] = []; + + before(async () => { + backend = createHttpServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", () => { + const send = (payload: unknown) => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(payload)); + }; + if (req.url === "/api/mcp/validate") { + return send({ valid: true, key_id: "k", user_id: "1", key_name: "agent-key", scope: "agent", agent_name: "atlas" }); + } + if (req.url === "/api/agents/atlas/messages") { + sentBodies.push(JSON.parse(body)); + return send({ success: true, channel: "auto", message_id: "m" }); + } + return send({}); + }); + }); + await new Promise((r) => backend.listen(0, "127.0.0.1", () => r())); + const backendPort = (backend.address() as AddressInfo).port; + + const probe = createHttpServer(); + await new Promise((r) => probe.listen(0, "127.0.0.1", () => r())); + const mcpPort = (probe.address() as AddressInfo).port; + await new Promise((r) => probe.close(() => r())); + + const { server } = await createServer({ + trinityApiUrl: `http://127.0.0.1:${backendPort}`, + requireApiKey: true, + port: mcpPort, + }); + await server.start({ transportType: "httpStream", httpStream: { port: mcpPort, host: "127.0.0.1" } }); + mcpServer = server; + mcpUrl = new URL(`http://127.0.0.1:${mcpPort}/mcp`); + }); + + after(async () => { + await mcpServer?.stop(); + await new Promise((r) => backend.close(() => r())); + }); + + it("two calls on one session carry their own header value", async () => { + let current = "exec-1"; + const transport = new StreamableHTTPClientTransport(mcpUrl, { + requestInit: { headers: { Authorization: "Bearer trinity_mcp_test" } }, + // Stamp the CURRENT value on every POST — the header changes mid-session. + fetch: (url, init) => { + const headers = new Headers(init?.headers); + headers.set("X-Trinity-Execution-Id", current); + return fetch(url, { ...init, headers }); + }, + }); + const client = new Client({ name: "t", version: "1.0.0" }); + await client.connect(transport); + try { + const call = () => + client.callTool({ + name: "send_message", + arguments: { recipient_email: "user@example.com", text: "hi", execution_id: "stale-param" }, + }); + await call(); + current = "manual"; + await call(); + } finally { + await client.close(); + } + assert.deepEqual(sentBodies.map((b) => b.execution_id), ["exec-1", "manual"]); + }); +}); diff --git a/src/mcp-server/src/server.ts b/src/mcp-server/src/server.ts index 4b0dacdc65..6d7bb57d30 100644 --- a/src/mcp-server/src/server.ts +++ b/src/mcp-server/src/server.ts @@ -48,6 +48,7 @@ import { createCredentialVaultTools } from "./tools/credential_vault.js"; import { configureAudit, withAudit } from "./audit.js"; import { installLogRedaction } from "./log-redaction.js"; import type { McpAuthContext } from "./types.js"; +import { EXECUTION_ID_HEADER, parseExecutionIdHeader } from "./tools/execution_id.js"; export interface ServerConfig { name?: string; @@ -543,6 +544,8 @@ export async function createServer(config: ServerConfig = {}) { // value the union does not name arrives here as a plain string. scope: scope as McpAuthContext["scope"], mcpApiKey: apiKey, // Store the actual API key for user-scoped requests + // #2392: per-request — fastmcp re-authenticates every POST. + executionId: parseExecutionIdHeader(readHeader(request.headers[EXECUTION_ID_HEADER])), }; return authContext; } diff --git a/src/mcp-server/src/tools/a2a_call.ts b/src/mcp-server/src/tools/a2a_call.ts index 1ea1e4271f..247d2e3f2b 100644 --- a/src/mcp-server/src/tools/a2a_call.ts +++ b/src/mcp-server/src/tools/a2a_call.ts @@ -40,6 +40,7 @@ import { z } from "zod"; import { TrinityClient, ApiError } from "../client.js"; import type { McpAuthContext } from "../types.js"; +import { resolveExecutionId } from "./execution_id.js"; import { accessDenied } from "../access.js"; export function createA2ACallTools(client: TrinityClient, requireApiKey: boolean) { @@ -154,7 +155,8 @@ export function createA2ACallTools(client: TrinityClient, requireApiKey: boolean "Continue a specific remote task (e.g. answering an input-required prompt).", ), execution_id: z.string().max(200).optional().describe( - "Your current execution id, if you have one. Enables at-most-once delivery on retry.", + "Your current execution id, if you have one. Enables at-most-once delivery on retry." + + " The platform normally supplies this automatically (#2392); this param is a fallback for older agent images.", ), }), execute: async ( @@ -183,7 +185,7 @@ export function createA2ACallTools(client: TrinityClient, requireApiKey: boolean dedup_label: params.dedup_label, context_id: params.context_id, task_id: params.task_id, - execution_id: params.execution_id, + execution_id: resolveExecutionId(context?.session, params.execution_id), }); return JSON.stringify({ success: true, ...(result as object) }, null, 2); } catch (e) { diff --git a/src/mcp-server/src/tools/execution_id.test.ts b/src/mcp-server/src/tools/execution_id.test.ts new file mode 100644 index 0000000000..2f8379e83d --- /dev/null +++ b/src/mcp-server/src/tools/execution_id.test.ts @@ -0,0 +1,114 @@ +/** + * #2392 — the platform-supplied execution id (X-Trinity-Execution-Id) wins over + * the agent-supplied `execution_id` param on every effect tool. + * + * Runner: built-in node:test → `node --import tsx --test src/tools/*.test.ts`. + */ +import { describe, it } from "node:test"; +import { strict as assert } from "node:assert"; + +import { parseExecutionIdHeader, resolveExecutionId } from "./execution_id.js"; +import { createMessageTools } from "./messages.js"; +import { createFileTools } from "./files.js"; +import { createVoiceReplyTools } from "./voice.js"; +import { readHeader } from "../server.js"; +import type { TrinityClient } from "../client.js"; +import type { McpAuthContext } from "../types.js"; + +const session = (executionId?: string) => + ({ session: { agentName: "atlas", scope: "agent", executionId } as unknown as McpAuthContext }); + +describe("#2392 resolveExecutionId", () => { + it("header wins over a conflicting param", () => { + assert.equal(resolveExecutionId(session("exec-hdr").session, "exec-param"), "exec-hdr"); + }); + it("falls back to the param when no header", () => { + assert.equal(resolveExecutionId(session().session, "exec-param"), "exec-param"); + assert.equal(resolveExecutionId(undefined, "exec-param"), "exec-param"); + }); + it("is undefined when neither is present", () => { + assert.equal(resolveExecutionId(session().session, undefined), undefined); + }); + it("forwards `manual` as-is", () => { + assert.equal(resolveExecutionId(session("manual").session, "exec-param"), "manual"); + }); + it("drops `manual` typed as the param — only the header may claim it", () => { + assert.equal(resolveExecutionId(session().session, "manual"), undefined); + assert.equal(resolveExecutionId(undefined, "manual"), undefined); + }); +}); + +describe("#2392 parseExecutionIdHeader", () => { + it("accepts and trims a well-formed id", () => { + assert.equal(parseExecutionIdHeader(" exec_1.a:b-2 "), "exec_1.a:b-2"); + assert.equal(parseExecutionIdHeader("manual"), "manual"); + }); + it("ignores absent, empty, over-long or malformed values", () => { + for (const bad of [undefined, "", " ", "a".repeat(129), "has space", "x;y", "ünï", "a/b"]) { + assert.equal(parseExecutionIdHeader(bad), undefined, String(bad)); + } + assert.equal(parseExecutionIdHeader("a".repeat(128)), "a".repeat(128)); + }); + it("takes the first of a repeated header (via readHeader)", () => { + assert.equal(parseExecutionIdHeader(readHeader(["exec-a", "exec-b"])), "exec-a"); + }); + it("two sequential requests with different headers yield different ids", () => { + const ctx = (h: string) => ({ executionId: parseExecutionIdHeader(readHeader(h)) }) as McpAuthContext; + const first = ctx("exec-1"); + const second = ctx("exec-2"); + assert.equal(resolveExecutionId(first, undefined), "exec-1"); + assert.equal(resolveExecutionId(second, undefined), "exec-2"); + }); +}); + +describe("#2392 effect tools forward the header value over a conflicting param", () => { + it("send_message", async () => { + const sent: any[] = []; + const fake: Partial = { + getBaseUrl: () => "http://localhost:8000", + sendUserMessage: async (_a: string, data: any) => { + sent.push(data); + return { success: true, channel: "auto", message_id: "m1" }; + }, + }; + await createMessageTools(fake as unknown as TrinityClient, false).sendMessage.execute( + { recipient_email: "user@example.com", text: "hi", execution_id: "exec-param" }, + session("exec-hdr") + ); + assert.equal(sent[0].execution_id, "exec-hdr"); + }); + + it("share_file", async () => { + const sent: any[] = []; + const fake: Partial = { + getBaseUrl: () => "http://localhost:8000", + shareAgentFile: async (_a: string, data: any) => { + sent.push(data); + return { file_id: "f", url: "u", expires_at: "2099-01-01T00:00:00Z", size_bytes: 1, mime_type: "text/plain" } as any; + }, + }; + await createFileTools(fake as unknown as TrinityClient, false).shareFile.execute( + { filename: "deck.pdf", execution_id: "exec-param" }, + session("manual") + ); + assert.equal(sent[0].execution_id, "manual"); + }); + + it("send_voice_reply works from the header alone and refuses with neither", async () => { + const sent: any[] = []; + const fake: Partial = { + getBaseUrl: () => "http://localhost:8000", + sendVoiceReply: async (_a: string, data: any) => { + sent.push(data); + return { delivered: true, channel: "telegram" } as any; + }, + }; + const tool = createVoiceReplyTools(fake as unknown as TrinityClient, false).sendVoiceReply; + await tool.execute({ text: "hi" }, session("exec-hdr")); + assert.equal(sent[0].execution_id, "exec-hdr"); + + const out = JSON.parse(await tool.execute({ text: "hi" }, session())); + assert.equal(out.delivered, false); + assert.equal(sent.length, 1); + }); +}); diff --git a/src/mcp-server/src/tools/execution_id.ts b/src/mcp-server/src/tools/execution_id.ts new file mode 100644 index 0000000000..852109949b --- /dev/null +++ b/src/mcp-server/src/tools/execution_id.ts @@ -0,0 +1,38 @@ +/** + * #2392 — the execution id an effect tool sends to the backend. + * + * The agent's MCP config sends `X-Trinity-Execution-Id` on every request: the + * current turn's execution id, or the literal `manual` for an interactive + * session with no execution (forwarded as-is; the backend interprets it). + * fastmcp re-runs `authenticate` on every POST, so the value on the auth + * context is per-request. The agent-supplied `execution_id` param remains a + * fallback for agent images that predate the header. + */ +import type { McpAuthContext } from "../types.js"; + +export const EXECUTION_ID_HEADER = "x-trinity-execution-id"; + +// 128 matches the backend's `execution_id` max_length, so an over-long value is +// dropped here instead of failing Pydantic validation downstream. +const EXECUTION_ID_RE = /^[A-Za-z0-9_.:-]{1,128}$/; + +/** Trimmed header value, or undefined when absent or malformed (ignored, never an error). */ +export function parseExecutionIdHeader(value: string | undefined): string | undefined { + const v = value?.trim(); + return v && EXECUTION_ID_RE.test(v) ? v : undefined; +} + +export const MANUAL_EXECUTION_ID = "manual"; + +/** + * Header value wins; the agent-supplied param is the fallback. `manual` is + * honoured only from the header (the platform-written config): a model typing + * it as the param would otherwise skip the pull-mode refusal. + */ +export function resolveExecutionId( + authContext: McpAuthContext | undefined, + paramValue: string | undefined +): string | undefined { + if (authContext?.executionId) return authContext.executionId; + return paramValue === MANUAL_EXECUTION_ID ? undefined : paramValue; +} diff --git a/src/mcp-server/src/tools/files.ts b/src/mcp-server/src/tools/files.ts index 6823f675d7..5c0a2ff158 100644 --- a/src/mcp-server/src/tools/files.ts +++ b/src/mcp-server/src/tools/files.ts @@ -10,6 +10,7 @@ import { z } from "zod"; import { TrinityClient } from "../client.js"; import type { McpAuthContext } from "../types.js"; +import { resolveExecutionId } from "./execution_id.js"; export function createFileTools( client: TrinityClient, @@ -82,7 +83,8 @@ export function createFileTools( "prompt. Pass it: it is how the platform knows which conversation this share belongs " + "to, and so whose Files tab lists the file — without it the file is listed for the " + "agent's owner only. It also lets a re-run of this turn replay the original signed " + - "URL instead of minting a new token (#1084)." + "URL instead of minting a new token (#1084)." + + " The platform normally supplies this automatically (#2392); this param is a fallback for older agent images." ), audience_email: z .string() @@ -142,7 +144,7 @@ export function createFileTools( filename: params.filename, display_name: params.display_name, expires_in: params.expires_in, - execution_id: params.execution_id, + execution_id: resolveExecutionId(authContext, params.execution_id), dedup_label: params.dedup_label, audience_email: params.audience_email, }); diff --git a/src/mcp-server/src/tools/git.ts b/src/mcp-server/src/tools/git.ts index 85c020fe5f..cbc3cdd822 100644 --- a/src/mcp-server/src/tools/git.ts +++ b/src/mcp-server/src/tools/git.ts @@ -285,7 +285,10 @@ export function createGitTools(client: TrinityClient, requireApiKey: boolean) { name: "get_git_sync_state", description: "Get the persisted git sync-health row for an agent (#389) — last sync " + - "status/time, consecutive failures, ahead/behind counts. Read-only. " + + "status/time, consecutive failures, ahead/behind counts, uncommitted-file " + + "count, last successful push — plus the backend's verdict: state " + + "(green/yellow/red/unknown), reason, recommendation, binding " + + "(agent/deployment) and whether its schedules are frozen. Read-only. " + "Access control: agent-scoped keys may only query themselves or permitted agents.", parameters: z.object({ agent_name: z.string().min(1).describe("Agent whose sync-state to read."), diff --git a/src/mcp-server/src/tools/messages.ts b/src/mcp-server/src/tools/messages.ts index 9d5ead0bb3..3aca666be7 100644 --- a/src/mcp-server/src/tools/messages.ts +++ b/src/mcp-server/src/tools/messages.ts @@ -8,6 +8,7 @@ import { z } from "zod"; import { TrinityClient } from "../client.js"; import type { McpAuthContext } from "../types.js"; +import { resolveExecutionId } from "./execution_id.js"; /** * Create message tools with the given client @@ -92,7 +93,7 @@ export function createMessageTools( "Your current execution_id — shown in the 'Execution Context' block of your system " + "prompt as '- **Execution ID**: '. Pass it so a re-delivery of this turn does NOT " + "send a duplicate message (effect-scoped idempotency, #1084). Optional: if omitted, " + - "the send proceeds without dedup." + "the send proceeds without dedup. The platform normally supplies this automatically (#2392); this param is a fallback for older agent images." ), dedup_label: z.string().optional() .describe( @@ -153,7 +154,7 @@ export function createMessageTools( text: params.text.trim(), channel: params.channel || "auto", reply_to_thread: params.reply_to_thread || false, - execution_id: params.execution_id, + execution_id: resolveExecutionId(authContext, params.execution_id), dedup_label: params.dedup_label, }); diff --git a/src/mcp-server/src/tools/voice.ts b/src/mcp-server/src/tools/voice.ts index 8fe2f81247..dbc7c7dd28 100644 --- a/src/mcp-server/src/tools/voice.ts +++ b/src/mcp-server/src/tools/voice.ts @@ -14,6 +14,7 @@ import { z } from "zod"; import { TrinityClient } from "../client.js"; import type { McpAuthContext } from "../types.js"; +import { resolveExecutionId } from "./execution_id.js"; export function createVoiceReplyTools( client: TrinityClient, @@ -68,11 +69,12 @@ export function createVoiceReplyTools( parameters: z.object({ text: z.string().min(1).max(4096) .describe("The text to speak as a voice note. Keep it short and natural to hear."), - execution_id: z.string().min(1) + execution_id: z.string().min(1).optional() .describe( "Your current execution_id — shown in the 'Execution Context' block of your " + - "system prompt as '- **Execution ID**: '. Required: the backend resolves the " + - "channel destination from it, and it dedupes a re-delivered turn (#1084)." + "system prompt as '- **Execution ID**: '. One of this or the platform-supplied id is required: the backend resolves the " + + "channel destination from it, and it dedupes a re-delivered turn (#1084)." + + " The platform normally supplies this automatically (#2392); this param is a fallback for older agent images." ), dedup_label: z.string().optional() .describe( @@ -88,7 +90,7 @@ export function createVoiceReplyTools( execute: async ( params: { text: string; - execution_id: string; + execution_id?: string; dedup_label?: string; agent_name?: string; }, @@ -101,11 +103,17 @@ export function createVoiceReplyTools( return JSON.stringify({ delivered: false, error: "Text cannot be empty" }, null, 2); } + // #2392: header first, param fallback; the backend needs one to find the channel. + const executionId = resolveExecutionId(authContext, params.execution_id); + if (!executionId) { + return JSON.stringify({ delivered: false, error: "execution_id is required", fallback_to_text: true }, null, 2); + } + try { const agentName = getAgentName(authContext, params.agent_name); const result = await apiClient.sendVoiceReply(agentName, { text: params.text.trim(), - execution_id: params.execution_id, + execution_id: executionId, dedup_label: params.dedup_label, }); return JSON.stringify( diff --git a/src/mcp-server/src/tools/voip.ts b/src/mcp-server/src/tools/voip.ts index 7f0eb19569..c3209ef4a7 100644 --- a/src/mcp-server/src/tools/voip.ts +++ b/src/mcp-server/src/tools/voip.ts @@ -10,6 +10,7 @@ import { z } from "zod"; import { TrinityClient } from "../client.js"; import type { McpAuthContext } from "../types.js"; +import { resolveExecutionId } from "./execution_id.js"; export function createVoipTools(client: TrinityClient, requireApiKey: boolean) { const getClient = (authContext?: McpAuthContext): TrinityClient => { @@ -63,7 +64,8 @@ export function createVoipTools(client: TrinityClient, requireApiKey: boolean) { .describe( "Your current execution_id — shown in the 'Execution Context' block of your system " + "prompt. Pass it so a re-delivery of this turn does NOT place a duplicate phone call " + - "(effect-scoped idempotency, #1084). Optional: if omitted, the call proceeds without dedup." + "(effect-scoped idempotency, #1084). Optional: if omitted, the call proceeds without dedup." + + " The platform normally supplies this automatically (#2392); this param is a fallback for older agent images." ), dedup_label: z.string().optional() .describe( @@ -90,7 +92,7 @@ export function createVoipTools(client: TrinityClient, requireApiKey: boolean) { to_number: params.to_number, context: params.context, process_transcript: params.process_transcript ?? true, - execution_id: params.execution_id, + execution_id: resolveExecutionId(authContext, params.execution_id), dedup_label: params.dedup_label, }); return JSON.stringify({ success: true, agent_name: agentName, ...result }, null, 2); diff --git a/src/mcp-server/src/types.ts b/src/mcp-server/src/types.ts index 8f8a3c327a..f144bfe20f 100644 --- a/src/mcp-server/src/types.ts +++ b/src/mcp-server/src/types.ts @@ -114,6 +114,8 @@ export interface McpAuthContext extends Record { // treated as least-privileged, never as `user`. scope: "user" | "agent" | "system" | "connector" | "portal_delegate" | "anonymous" | "ops"; mcpApiKey?: string; // The actual MCP API key (for user-scoped requests to Trinity backend) + // #2392: X-Trinity-Execution-Id of THIS request (turn id or "manual"); wins over an agent-supplied execution_id. + executionId?: string; // --- #848 inline email auth (anonymous scope only) --------------------- // Mutated IN PLACE by verify_login. FastMCP hands every tool the same auth @@ -493,12 +495,15 @@ export interface OperatorQueueItem { expires_at?: string | null; response?: string | null; response_text?: string | null; + // A person's identity — withheld from agent, system and other machine keys + // (trinity-enterprise#715), as are `addressed_to_email`, `disposed_by_email` + // and `resolved_to`. Only a person's JWT or user-scoped key reads them. responded_by_id?: string | null; responded_by_email?: string | null; responded_at?: string | null; acknowledged_at?: string | null; // trinity-enterprise#611 — how the ask ended (NULL on a row that ended before - // the ledger: read `status`). `disposed_by_email` is withheld from agent keys. + // the ledger: read `status`). disposition?: string | null; // answered | cancelled | expired disposed_at?: string | null; disposed_by?: string | null; // person | timeout diff --git a/src/scheduler/database.py b/src/scheduler/database.py index ab8f8483d3..d86dfed125 100644 --- a/src/scheduler/database.py +++ b/src/scheduler/database.py @@ -16,6 +16,10 @@ from .config import config from .models import Schedule, ScheduleExecution, ExecutionStatus, ProcessSchedule, ProcessScheduleExecution, Reminder +# #389/#1808: consecutive failed syncs before a freeze-enabled agent stops +# firing. ONE number: it lives in the sync policy, which this package vendors +# byte-identically from the backend (trinity-enterprise#706). +from .sync_freeze_policy import SYNC_FAILURE_FREEZE_THRESHOLD, classify # noqa: F401 (re-exported) from .utils import ( duration_ms_between, parse_scheduler_ts, @@ -25,11 +29,6 @@ logger = logging.getLogger(__name__) -# #389/#1808: consecutive failed syncs before a freeze-enabled agent stops -# firing. Mirrors the threshold in the backend's -# routers/internal.py::internal_agent_sync_health — keep the two in step. -SYNC_FAILURE_FREEZE_THRESHOLD = 3 - def _scheduler_pg_url() -> Optional[str]: """Return the PostgreSQL ``DATABASE_URL`` if configured, else None (#300). @@ -467,19 +466,34 @@ def create_execution( ) def should_freeze_schedules(self, agent_name: str) -> bool: - """Is this agent's cron firing frozen by a failing git sync? (#389/#1808) + """Is this agent's cron firing frozen by its git sync? (#389/#1808, ent#706) + + A bool view of :meth:`sync_freeze_reason` — kept for callers that only + need the yes/no. + """ + return self.sync_freeze_reason(agent_name) is not None + + def sync_freeze_reason(self, agent_name: str) -> Optional[str]: + """Why this agent's cron firing is frozen, or None when it may fire. True only when the owner opted in (`freeze_schedules_if_sync_failing`) - AND sync is actually failing. Mirrors the predicate the backend already - exposes at `/api/internal/agents/{name}/sync-health-status`; read - directly here because the scheduler talks to the same database for - every other gate (autonomy, enabled, reminders) and an extra HTTP hop - would add a failure mode to the fire path for no benefit. - - FAIL-OPEN: any error (missing table on an older DB, read failure) - returns False and the schedule fires. A freeze-on-error would silently - stop the whole fleet the moment this query broke — the opposite of the - bug this closes. + AND either sync is failing (#1808, any binding) or a WORK agent has + been diverged from origin for more than 24 h on a fresh observation + (trinity-enterprise#706). The rule is `sync_freeze_policy.classify`, + vendored byte-identically from the backend, so the dashboard and this + gate cannot disagree. Read directly here because the scheduler talks to + the same database for every other gate (autonomy, enabled, reminders) + and an extra HTTP hop would add a failure mode to the fire path for no + benefit. + + Returns a reason string rather than a tuple on purpose: a + `(False, None)` tuple is truthy, so a caller that kept the old + `if db.should_freeze_...(...)` shape would freeze the whole fleet. + + FAIL-OPEN: any error (missing table or column on an older DB, read + failure) returns None and the schedule fires. A freeze-on-error would + silently stop the whole fleet the moment this query broke — the + opposite of the bug this closes. """ try: with self.get_connection() as conn: @@ -487,8 +501,16 @@ def should_freeze_schedules(self, agent_name: str) -> bool: cursor.execute( """ SELECT gc.freeze_schedules_if_sync_failing, + gc.source_mode, + gc.auto_sync_enabled, ss.last_sync_status, - ss.consecutive_failures + ss.consecutive_failures, + ss.ahead_main, + ss.behind_main, + ss.ahead_working, + ss.behind_working, + ss.diverged_since, + ss.last_check_at FROM agent_git_config gc LEFT JOIN agent_sync_state ss ON ss.agent_name = gc.agent_name WHERE gc.agent_name = ? @@ -497,23 +519,40 @@ def should_freeze_schedules(self, agent_name: str) -> bool: ) row = cursor.fetchone() if not row: - return False + return None # Named access only: the PG path (#300) yields RealDictCursor # mapping rows with no positional indexing — row[0] would # KeyError and silently fail-open on every PostgreSQL deploy. - freeze_enabled = bool(row["freeze_schedules_if_sync_failing"]) - if not freeze_enabled: - return False - failing = ( - row["last_sync_status"] == "failed" - and (row["consecutive_failures"] or 0) >= SYNC_FAILURE_FREEZE_THRESHOLD + if not bool(row["freeze_schedules_if_sync_failing"]): + return None + verdict = classify( + { + key: row[key] + for key in ( + "last_sync_status", + "consecutive_failures", + "ahead_main", + "behind_main", + "ahead_working", + "behind_working", + "diverged_since", + "last_check_at", + ) + }, + { + "source_mode": row["source_mode"], + "auto_sync_enabled": row["auto_sync_enabled"], + "freeze_schedules_if_sync_failing": row[ + "freeze_schedules_if_sync_failing" + ], + }, ) - return bool(failing) + return verdict["freeze_reason"] if verdict["freeze"] else None except Exception as e: logger.warning( f"Sync-freeze check failed for {agent_name} ({e}); firing anyway" ) - return False + return None def create_skipped_execution( self, diff --git a/src/scheduler/service.py b/src/scheduler/service.py index 167b4866f5..4c64e98739 100644 --- a/src/scheduler/service.py +++ b/src/scheduler/service.py @@ -978,37 +978,41 @@ async def _execute_schedule_with_lock( # #1808: git-sync freeze gate. The owner opted in via # `freeze_schedules_if_sync_failing` (#389) so the agent stops doing - # autonomous work while its repo is broken — until now the flag was - # stored, reported back as enabled, and never enforced, and - # docs/user-docs/faq/troubleshooting.md told users it worked. + # autonomous work while its repo is broken — until #1808 the flag was + # stored, reported back as enabled, and never enforced. + # trinity-enterprise#706: the gate also fires when a WORK agent has + # been diverged from origin for more than 24 h (on a fresh + # observation), and the reason is carried into the skipped row. # # Cron only: a manual trigger is an operator explicitly asking, exactly # like the autonomy gate above. # - # Unlike the autonomy branch this DOES record a skipped execution row. - # Autonomy-off is a static config gate on a default-OFF fleet, so a row - # per tick would flood the table; a sync freeze needs BOTH an opt-in - # toggle AND 3+ consecutive real sync failures, so the row count is - # bounded and it is exactly the signal an operator needs to see. Uses - # the same audit path as the max_instances skip. - if triggered_by == "schedule" and self.db.should_freeze_schedules(schedule.agent_name): + # Unlike the autonomy branch this DOES record a skipped execution row: + # one per cron tick for as long as the freeze lasts. That is not + # "bounded" — a push-denied agent can fail for days, and a diverged one + # stays frozen until someone pushes — but each row IS the operator's + # signal, it needs an opt-in to exist at all, and execution retention + # prunes it. Uses the same audit path as the max_instances skip. + freeze_reason = ( + self.db.sync_freeze_reason(schedule.agent_name) + if triggered_by == "schedule" + else None + ) + if freeze_reason: logger.warning( - f"Schedule {schedule_id} skipped: agent {schedule.agent_name} git sync is " - f"failing and freeze_schedules_if_sync_failing is enabled" + f"Schedule {schedule_id} skipped: agent {schedule.agent_name} " + f"git sync frozen ({freeze_reason})" ) self._record_skipped_agent_schedule( schedule_id, - skip_reason=( - "Git sync is failing and freeze_schedules_if_sync_failing is " - "enabled for this agent" - ), - event_reason="Git sync failing (schedules frozen)", + skip_reason=f"Git sync frozen: {freeze_reason}", + event_reason="Git sync frozen (schedules paused)", ) # Same projection advance as the autonomy branch (#1472) so the # schedule never renders a receding "Next: Nd ago" while frozen. self._advance_next_run_only(schedule) self._abandon_precreated_execution( - execution, "Git sync started failing before the run started" + execution, f"Git sync frozen before the run started: {freeze_reason}" ) return diff --git a/src/scheduler/sync_freeze_policy.py b/src/scheduler/sync_freeze_policy.py new file mode 100644 index 0000000000..4388992479 --- /dev/null +++ b/src/scheduler/sync_freeze_policy.py @@ -0,0 +1,257 @@ +"""Sync-health policy: state, reason, recommendation and the freeze (trinity-enterprise#706). + +ONE place decides whether an agent and its repository agree. Every surface that +shows sync health (the dashboard dot, `GET /api/agents/sync-health`, +`GET /api/agents/{name}/git/sync-state`, the internal freeze endpoint) and the +scheduler's own freeze gate call :func:`classify` — the frontend owns no +threshold. + +CANONICAL SOURCE: src/backend/services/sync_freeze_policy.py. The scheduler is a +standalone image that cannot import the backend, so src/scheduler/sync_freeze_policy.py +is a BYTE-IDENTICAL vendored mirror (the failure_classifier.py precedent, #1088; +Invariant #5), pinned by tests/unit/test_ent706_sync_policy_parity.py. Edit this +file, then `cp` it over the mirror. + +Stdlib only, no I/O, never raises on a malformed row: the scheduler calls this +on its fire path, and every number in the row was written by the agent. + +Rules, evaluated together (the first red clause names the reason, the rest +append): + + unknown no row, or the poller never observed the agent + red the last sync failed (any binding) + a WORK agent diverged from origin for more than 24 h + a WORK agent's tree dirty for more than 24 h + auto-sync on and no heartbeat for 7 days (fresh observation only) + yellow a work agent diverged for 24 h or less + a DEPLOYMENT diverged at any age, or dirty for more than 24 h — + a deployment being behind is normal; it is never red on age + green otherwise + +A work agent is `source_mode = 0 OR auto_sync_enabled = 1`: a working branch, +or a fork-to-own / bound-to-own agent that auto-syncs to its own repository +(those keep `source_mode = 1` in the DB). Everything else is a deployment. + +freeze = freeze_schedules_if_sync_failing AND ( + sync_failing # #1808, unchanged, any binding + OR (work agent AND diverged > 24 h AND observation fresh)) + +The freshness guard fails OPEN on purpose: the poller writes nothing for an +unreachable agent, so a stale `diverged_since` must never keep an agent frozen +after it may already have pushed. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +SYNC_FAILURE_FREEZE_THRESHOLD = 3 # consecutive POLLS that saw `failed` (#1808) +DIVERGENCE_RED_SECONDS = 24 * 3600 +DIRTY_RED_SECONDS = 24 * 3600 +NO_HEARTBEAT_FLOOR_SECONDS = 7 * 24 * 3600 +OBSERVATION_FRESH_SECONDS = 15 * 60 +ERROR_EXCERPT_CHARS = 120 + +STATES = ("green", "yellow", "red", "unknown") + +# The agent's own refusal text for a source-mode agent on the shared default +# branch (#3011, `failed: "refused: source-mode on "`). +_SOURCE_MODE_REFUSAL = "refused: source-mode" + + +def parse_ts(value: Any) -> Optional[datetime]: + """An aware-UTC datetime from an ISO string or a datetime; None otherwise. + + A naive value is UTC — the convention of `parse_scheduler_ts` and of the + backend's `parse_iso_timestamp`. Every comparison happens here in Python, + never lexicographically in SQL (Invariant #16). + """ + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str) and 0 < len(value) <= 64: + text = value.strip() + if text.endswith(("Z", "z")): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + else: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def format_age(seconds: int) -> str: + """`45m`, `26h`, `9d` — the shape of the operator-facing reason strings.""" + seconds = max(int(seconds), 0) + if seconds < 3600: + return f"{seconds // 60}m" + if seconds < 48 * 3600: + return f"{seconds // 3600}h" + return f"{seconds // 86400}d" + + +def _count(value: Any) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int) and value > 0: + return value + return 0 + + +def _flag(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in ("1", "true") + return bool(value) + + +def _age(now: datetime, then: Optional[datetime]) -> Optional[int]: + if then is None: + return None + return max(int((now - then).total_seconds()), 0) + + +def is_work_agent(source_mode: Any, auto_sync_enabled: Any) -> bool: + """`source_mode = 0 OR auto_sync_enabled = 1` (see the module docstring).""" + return (not _flag(source_mode)) or _flag(auto_sync_enabled) + + +def classify( + row: Optional[Dict[str, Any]], + cfg: Optional[Dict[str, Any]], + now: Optional[datetime] = None, + *, + push_denied: bool = False, +) -> Dict[str, Any]: + """The verdict for one agent. + + `row` is its `agent_sync_state` row (or None). `cfg` carries the + `agent_git_config` flags: `source_mode`, `auto_sync_enabled`, + `freeze_schedules_if_sync_failing`, and optionally `created_at` (the + no-heartbeat floor's start when no heartbeat was ever recorded). + `push_denied` is `git_service.is_push_denied(last_error_summary)`, computed + by the backend caller so this module stays a leaf. + """ + now = parse_ts(now) or datetime.now(timezone.utc) + cfg = cfg or {} + auto_sync = _flag(cfg.get("auto_sync_enabled")) + work = is_work_agent(cfg.get("source_mode"), auto_sync) + freeze_flag = _flag(cfg.get("freeze_schedules_if_sync_failing")) + + verdict: Dict[str, Any] = { + "state": "unknown", + "reason": "no sync observation yet", + "recommendation": None, + "binding": "agent" if work else "deployment", + "work_agent": work, + "divergence_age_s": None, + "dirty_age_s": None, + "sync_failing": False, + "freeze": False, + "freeze_cause": None, + "freeze_reason": None, + "stale_observation": False, + } + if not row: + return verdict + observed_age = _age(now, parse_ts(row.get("last_check_at"))) + if observed_age is None: + return verdict + stale = observed_age > OBSERVATION_FRESH_SECONDS + + ahead = _count(row.get("ahead_working")) + behind = _count(row.get("behind_working")) + ahead_main = _count(row.get("ahead_main")) + behind_main = _count(row.get("behind_main")) + failures = _count(row.get("consecutive_failures")) + dirty = _count(row.get("dirty_files")) + error = row.get("last_error_summary") + error = error.strip() if isinstance(error, str) else "" + failed = row.get("last_sync_status") == "failed" + sync_failing = failed and failures >= SYNC_FAILURE_FREEZE_THRESHOLD + + divergence_age = _age(now, parse_ts(row.get("diverged_since"))) + dirty_age = _age(now, parse_ts(row.get("dirty_since"))) if dirty else None + + reds: List[str] = [] + yellows: List[str] = [] + + failed_clause = None + if failed: + plural = "" if failures == 1 else "s" + failed_clause = f"last sync failed (seen on {failures} poll{plural})" + if error: + failed_clause += f": {error[:ERROR_EXCERPT_CHARS]}" + reds.append(failed_clause) + + divergence_clause = None + divergence_red = False + if divergence_age is not None: + divergence_clause = ( + f"diverged {behind} behind / {ahead} ahead for {format_age(divergence_age)}" + ) + # D7: `main` moving under a working branch is information, not divergence. + if behind_main and (ahead_main, behind_main) != (ahead, behind): + divergence_clause += f" (main is {behind_main} ahead)" + divergence_red = work and divergence_age > DIVERGENCE_RED_SECONDS + (reds if divergence_red else yellows).append(divergence_clause) + + if dirty_age is not None and dirty_age > DIRTY_RED_SECONDS: + clause = f"dirty: {dirty} files uncommitted for {format_age(dirty_age)}" + (reds if work else yellows).append(clause) + + heartbeat_missing = False + if auto_sync and not stale: + heartbeat = parse_ts(row.get("last_sync_at")) or parse_ts(cfg.get("created_at")) + heartbeat_age = _age(now, heartbeat) + if heartbeat_age is not None and heartbeat_age > NO_HEARTBEAT_FLOOR_SECONDS: + heartbeat_missing = True + reds.append(f"auto-sync on, no heartbeat for {format_age(heartbeat_age)}") + + if reds: + state, reason = "red", "; ".join(reds + yellows) + elif yellows: + state, reason = "yellow", "; ".join(yellows) + else: + state, reason = "green", "in sync" + if stale: + reason += f" (last observed {format_age(observed_age)} ago)" + + freeze_cause = freeze_reason = None + if freeze_flag: + if sync_failing: + freeze_cause, freeze_reason = "sync_failing", failed_clause + elif divergence_red and not stale: + freeze_cause, freeze_reason = "divergence", divergence_clause + + if failed: + if push_denied: + recommendation = "credential is read-only" + elif _SOURCE_MODE_REFUSAL in error: + recommendation = "deployment: turn auto-sync off" + else: + recommendation = "push via git_sync strategy=pull_first" + elif work and ahead and not auto_sync: + recommendation = "enable auto-sync" + elif behind: + recommendation = "pull via git_pull" + elif heartbeat_missing: + recommendation = "restart the agent to resume the auto-sync heartbeat" + else: + recommendation = None + + verdict.update( + state=state, + reason=reason, + recommendation=recommendation, + divergence_age_s=divergence_age, + dirty_age_s=dirty_age, + sync_failing=sync_failing, + freeze=freeze_cause is not None, + freeze_cause=freeze_cause, + freeze_reason=freeze_reason, + stale_observation=stale, + ) + return verdict diff --git a/tests/registry.json b/tests/registry.json index b389a664e6..8106d90b48 100644 --- a/tests/registry.json +++ b/tests/registry.json @@ -1,5 +1,90 @@ { "test_files": [ + { + "file": "unit/test_ent706_sync_health_surfaces.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "unit", + "git-sync", + "api" + ], + "description": "The sync-health read surfaces carry the backend's verdict (trinity-enterprise#706); real DB via db_harness, handlers called directly with the Docker-backed accessibility lookup stubbed. Shared reader db.list_sync_health_rows: config flags + sync-state row per live git-bound agent in one query, soft-deleted and sync_enabled=0 agents excluded, state None before the first poll, scoped by name, empty scope -> {}. GET /api/agents/sync-health: every pre-#706 key kept with its meaning plus state / reason / recommendation / binding / dirty_files / divergence_age_s / last_successful_push_at / freeze; a work agent diverged 26 h is red + frozen with the issue's reason string, a deployment the same age is yellow and not frozen, an agent with no git binding is unknown with the old defaults; scoped to accessible agents. GET /api/agents/{name}/git/sync-state: the raw row (new columns included) plus the verdict; no row -> the old minimal dict plus state unknown." + }, + { + "file": "unit/test_ent706_internal_sync_health.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "unit", + "git-sync", + "scheduling" + ], + "description": "GET /api/internal/agents/{name}/sync-health-status after trinity-enterprise#706 (real DB via db_harness, handler called directly): the pre-#706 keys (agent_name, freeze_schedules_if_sync_failing, sync_failing, should_freeze, consecutive_failures) keep their names and meaning; a work agent diverged 7 ahead for 26 h with a fresh observation reports should_freeze with freeze_reason 'diverged 0 behind / 7 ahead for 26h', divergence_age_s ~26 h and work_agent; three failed polls still freeze as sync_failing with the failed-sync reason; control arm: an in-sync row does not freeze and carries no reason; an agent with no git binding is not frozen and is not a work agent." + }, + { + "file": "unit/test_1808_sync_freeze_enforcement.py", + "feature": "#1808, ent#706", + "added": "2026-09-27", + "categories": [ + "scheduler", + "unit", + "git-sync", + "reliability" + ], + "description": "The scheduler's own freeze gate, on a real temp SQLite through SchedulerDatabase (src/scheduler cannot import the backend). #1808: opted-in + 3+ failed polls freezes; below threshold, not opted in, no sync row and an unknown agent fire; a broken/missing table FAILS OPEN; mapping-only (PG RealDictCursor-shaped) rows still freeze; the threshold is the one policy module's (the backend endpoint no longer restates the rule). trinity-enterprise#706: sync_freeze_reason — a work agent diverged 24 h + 1 s on a fresh observation freezes with 'diverged 0 behind / 7 ahead for 24h' (should_freeze True), 24 h - 1 s fires, control arm diverged_since NULL on the same row fires, a deployment never divergence-freezes, fork-to-own (source_mode=1, auto_sync=1) does, a 16-minute-stale observation fails open, the owner opt-in is required, the sync_failing reason names its polls, and the mapping-only row shape freezes on divergence too. The SchedulerService gate (fake DB, real _execute_schedule_with_lock) writes 'Git sync frozen: ' as the skipped row's skip_reason; control arm: no reason, no skip row; a manual trigger is never frozen." + }, + { + "file": "unit/test_ent706_divergence_tracking.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "unit", + "git-sync", + "reliability" + ], + "description": "The SyncHealthService poller keeps the trinity-enterprise#706 columns, driven through the REAL _poll_cycle on a real DB (db_harness) with only /api/git/status faked. diverged_since: set once on the first ahead/behind > 0 poll and unchanged across later polls with different counts; cleared at (0, 0) AND at (0, None) (no upstream, #2105); an uncomputable ahead neither starts nor clears it; a known behind with unknown ahead starts it. dirty_files/dirty_since from changes_count: set-once / clear at 0; garbage (str, negative, bool, > int4) keeps both prior values; dirt never starts the divergence clock. last_successful_push_at: the later of the heartbeat's own #3011 record and an operator Push (agent_git_config.last_sync_at), either direction; an image older than #3011 (key absent) falls back to a success cycle but not a failed one; a #3011 image that never pushed (key present, None) does not; the agent value is validated (garbage, naive, future, non-str rejected); monotonic. An unreachable agent changes nothing. The sync_diverged operator-queue item: exactly one across repeated polls of an aged episode (id sync-diverged-{agent}-{diverged_since}, high priority, reason in the question, counts + recommendation in context, NO last_error_summary); two SyncHealthService instances (fail-open double leader) converge on one; a new episode after a clear gets a second; control arms: the same aged seed with the freeze flag off, or as a deployment, raises ZERO; the prefix is platform-reserved." + }, + { + "file": "unit/test_ent706_sync_state_columns.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "unit", + "database", + "migrations", + "git-sync" + ], + "description": "The four trinity-enterprise#706 columns on agent_sync_state (diverged_since, dirty_files, dirty_since, last_successful_push_at), both tracks (Invariant #9): the SQLite migration adds them to the pre-#706 table, runs twice as a no-op and backfills nothing; it is registered once in MIGRATIONS; schema.py DDL and the tables.py Core table carry them; the Alembic revision's id equals its filename, its down_revision is a real revision, and it ADD COLUMN IF NOT EXISTS / DROP COLUMN IF EXISTS each of the four. The upsert's three-way clock semantics (db_harness, SQLite + PG when TEST_POSTGRES_URL): the episode clocks default to KEEP across a partial upsert (and KEEP is the explicit spelling), None CLEARS them, and dirty_files = 0 is stored rather than dropped. KEEP is read from the live ops object's upsert.__globals__ (module-identity gotcha under sibling db.* eviction)." + }, + { + "file": "unit/test_ent706_sync_policy.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "unit", + "git-sync", + "reliability" + ], + "description": "The one sync-health policy (trinity-enterprise#706, services/sync_freeze_policy.py::classify), table-driven: unknown (no row / never observed); green in sync; one failed poll red not frozen; sync_failing (3 polls) freezes ANY binding and ignores observation freshness (#1808 unchanged); the failed reason carries at most 120 chars of error; the 24 h boundary at +/-1 s (yellow/runs vs red/frozen, divergence_age_s = 86401, freeze_reason names the divergence clause); freeze flag off -> red not frozen; a deployment behind 31 for 26 h is yellow and never freezes; fork-to-own (source_mode=1, auto_sync=1) freezes, source_mode=1 auto_sync=0 never does; a stale observation (16 min) stays red but does not freeze and names its age, 14 min freezes; aware Z, naive string, naive datetime and +03:00 offset give the same verdict; garbage timestamps degrade; behind_main is information only; dirt 24 h +/- 1 s (red not frozen for a work agent, yellow for a deployment, not a state under 24 h); the 7-day no-heartbeat floor (9 d red, 6 d not, never-heartbeat measured from the binding's creation, auto-sync off exempt); first red names the reason and the rest append; the recommendation mapping incl. push_denied / source-mode refusal; the constants; hostile types never raise. Non-ambient pins throughout." + }, + { + "file": "unit/test_ent706_sync_policy_parity.py", + "feature": "ent#706", + "added": "2026-09-27", + "categories": [ + "backend", + "scheduler", + "unit", + "git-sync" + ], + "description": "Vendored-mirror guard (trinity-enterprise#706, the #1088 precedent): src/scheduler/sync_freeze_policy.py is byte-identical to src/backend/services/sync_freeze_policy.py; the policy is a stdlib-only leaf with no relative imports (it must import in the scheduler image); the scheduler's SYNC_FAILURE_FREEZE_THRESHOLD is imported from the mirror, never re-assigned (one number)." + }, { "file": "unit/test_3080_detect_mime_zip.py", "feature": "#3080", @@ -3509,6 +3594,17 @@ ], "description": "The native ask create is atomic per agent on BOTH backends (trinity-enterprise#611, PR B): `requires_postgres` + the db_backend harness, so schema-parity CI runs it on real PostgreSQL. Each create is held inside its critical section (after the count, before the insert) so racers overlap deterministically: six creates at the cap admit exactly one, six retries of one id make one row, one agent's full queue never holds back another's, and the file create reports whether its ON CONFLICT DO NOTHING insert happened. Removing the pg_advisory_xact_lock / BEGIN IMMEDIATE turns the matching backend red on every run." }, + { + "file": "unit/test_ent715_queue_person_fields.py", + "feature": "trinity-enterprise#715", + "added": "2026-09-29", + "categories": [ + "backend", + "unit", + "security" + ], + "description": "The operator queue stops handing a person's email to a machine key (trinity-enterprise#715). GET /api/operator-queue, /{id} and /agents/{name} give every non-person principal (agent, system, ops, connector, portal_delegate, a user scope carrying an agent identity, an unknown scope) exactly the allowlisted machine key set, pinned as a literal on file and native rows whose person fields are all non-null; a person (JWT, user key) reads every field, which is also the classification guard (a new column fails until classified). One arm presents a real minted agent key with no overrides. The platform's heads-ups about a person (the ent#499 problem report, the ent#308 inbox collision), raised through their real emitters, are never listed to a machine and 404 on the item route, while a person still reads them; both are platform alarms, and every prefix the reads treat as about a person is a reserved prefix (so the file write-back and the resume wake treat it as a platform alarm too). The file write-back names no person on delivery or reconstruction, and an answered collision alert is never written into the agent's file. The clear-resolved /ws trigger is the count only. The composed prompt no longer promises responded_by. Call-site mutations red: the allowlist, the row filter, the item 404, both write-back lines, the reserved prefix, the trigger, the prompt, and a new unclassified column." + }, { "file": "unit/test_2845_retry_trigger_sets.py", "feature": "#2845", diff --git a/tests/unit/test_1677_operator_alert_emitters.py b/tests/unit/test_1677_operator_alert_emitters.py index 079b43cf05..cd353161aa 100644 --- a/tests/unit/test_1677_operator_alert_emitters.py +++ b/tests/unit/test_1677_operator_alert_emitters.py @@ -132,6 +132,10 @@ "platform-only: edge-triggered once per failure series (2→3 crossing, #389)", ("services/sync_health_service.py", "SyncHealthService._emit_git_bloat_alert"): "platform-only: edge-triggered per threshold crossing (#1595)", + ("services/sync_health_service.py", "SyncHealthService._emit_sync_diverged_alert"): + "platform-only: deterministic per-episode id (sync-diverged-{agent}-" + "{diverged_since}), and an episode must last > 24 h before it can raise " + "one, so at most one per agent per day (trinity-enterprise#706)", ("services/system_agent_service.py", "SystemAgentService._emit_base_image_stale_alert"): "platform-only: per-process cooldown, hosted on trinity-system (#1816)", diff --git a/tests/unit/test_1808_sync_freeze_enforcement.py b/tests/unit/test_1808_sync_freeze_enforcement.py index a4ba6f7406..7e18bf2ab7 100644 --- a/tests/unit/test_1808_sync_freeze_enforcement.py +++ b/tests/unit/test_1808_sync_freeze_enforcement.py @@ -18,10 +18,13 @@ from __future__ import annotations +import asyncio import os import sqlite3 import sys +from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace import pytest @@ -57,23 +60,55 @@ def _scheduler_database_module(): return scheduler_database -def _seed(db_path: Path, *, freeze: int, status: str | None, failures: int | None) -> None: - """Minimal schema + one agent's git config and sync state.""" +def _iso(dt: datetime) -> str: + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _seed( + db_path: Path, + *, + freeze: int, + status: str | None, + failures: int | None, + source_mode: int = 0, + auto_sync: int = 1, + ahead: int = 0, + behind: int = 0, + diverged_since: str | None = None, + last_check_at: str | None = "now", +) -> None: + """Minimal schema + one agent's git config and sync state. + + The columns are the ones the gate reads, named as in `db/schema.py` + (trinity-enterprise#706 added the divergence inputs). `last_check_at` + defaults to "now": every real row carries it, because the poller's upsert + stamps it on every write. + """ + if last_check_at == "now": + last_check_at = _iso(datetime.now(timezone.utc) - timedelta(seconds=30)) conn = sqlite3.connect(db_path) conn.execute( "CREATE TABLE agent_git_config (agent_name TEXT PRIMARY KEY, " - "freeze_schedules_if_sync_failing INTEGER DEFAULT 0)" + "freeze_schedules_if_sync_failing INTEGER DEFAULT 0, " + "source_mode INTEGER DEFAULT 0, auto_sync_enabled INTEGER DEFAULT 0)" ) conn.execute( "CREATE TABLE agent_sync_state (agent_name TEXT PRIMARY KEY, " - "last_sync_status TEXT, consecutive_failures INTEGER)" + "last_sync_status TEXT, consecutive_failures INTEGER, " + "ahead_main INTEGER DEFAULT 0, behind_main INTEGER DEFAULT 0, " + "ahead_working INTEGER DEFAULT 0, behind_working INTEGER DEFAULT 0, " + "diverged_since TEXT, last_check_at TEXT)" ) conn.execute( - "INSERT INTO agent_git_config VALUES (?, ?)", ("a1", freeze) + "INSERT INTO agent_git_config VALUES (?, ?, ?, ?)", + ("a1", freeze, source_mode, auto_sync), ) if status is not None: conn.execute( - "INSERT INTO agent_sync_state VALUES (?, ?, ?)", ("a1", status, failures) + "INSERT INTO agent_sync_state (agent_name, last_sync_status, " + "consecutive_failures, ahead_working, behind_working, diverged_since, " + "last_check_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ("a1", status, failures, ahead, behind, diverged_since, last_check_at), ) conn.commit() conn.close() @@ -124,11 +159,17 @@ def test_fails_open_when_the_query_breaks(tmp_path): def test_threshold_matches_the_backend(tmp_path): - """The scheduler's threshold must stay in step with the backend endpoint.""" + """The scheduler's threshold must stay in step with the backend endpoint. + + trinity-enterprise#706: both now read it from ONE policy module (vendored + byte-identically; parity in test_ent706_sync_policy_parity.py), and the + backend endpoint delegates to that module instead of restating the rule. + """ threshold = _scheduler_database_module().SYNC_FAILURE_FREEZE_THRESHOLD + assert threshold == 3 backend = (_REPO / "src" / "backend" / "routers" / "internal.py").read_text() - # internal.py expresses the same rule inline. - assert f">= {threshold}" in backend + assert "sync_health_view" in backend or "sync_freeze_policy" in backend + assert ">= 3" not in backend # no second copy of the rule def test_mapping_only_rows_still_freeze(tmp_path, monkeypatch): @@ -169,3 +210,169 @@ def dict_row_connection(): monkeypatch.setattr(db, "get_connection", dict_row_connection) assert db.should_freeze_schedules("a1") is True + + +# --------------------------------------------------------------------------- +# trinity-enterprise#706: the freeze also keys on divergence age (work agents) +# --------------------------------------------------------------------------- + +def _diverged_seed(db_path, *, age: timedelta, **kw): + kw.setdefault("freeze", 1) + _seed( + db_path, + status="success", + failures=0, + ahead=kw.pop("ahead", 7), + diverged_since=_iso(datetime.now(timezone.utc) - age) if age is not None else None, + **kw, + ) + + +def test_divergence_past_24h_freezes_a_work_agent(tmp_path): + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=24, seconds=1)) + db = _db(db_path) + reason = db.sync_freeze_reason("a1") + assert reason is not None and reason.startswith("diverged 0 behind / 7 ahead for 24h") + assert db.should_freeze_schedules("a1") is True + + +def test_divergence_under_24h_fires(tmp_path): + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=24) - timedelta(seconds=1)) + assert _db(db_path).sync_freeze_reason("a1") is None + + +def test_control_arm_no_divergence_clock_fires(tmp_path): + """The SAME row with diverged_since NULL fires — the freeze above comes + from divergence, not from anything ambient in the seed.""" + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=None) + assert _db(db_path).sync_freeze_reason("a1") is None + + +def test_a_deployment_never_divergence_freezes(tmp_path): + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=300), source_mode=1, auto_sync=0) + assert _db(db_path).sync_freeze_reason("a1") is None + + +def test_fork_to_own_divergence_freezes(tmp_path): + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=30), source_mode=1, auto_sync=1) + assert _db(db_path).should_freeze_schedules("a1") is True + + +def test_a_stale_observation_does_not_divergence_freeze(tmp_path): + """D6: fail open when the poller has not seen the agent for 15+ minutes.""" + db_path = tmp_path / "t.db" + _diverged_seed( + db_path, age=timedelta(hours=30), + last_check_at=_iso(datetime.now(timezone.utc) - timedelta(minutes=16)), + ) + assert _db(db_path).sync_freeze_reason("a1") is None + + +def test_divergence_needs_the_owner_opt_in(tmp_path): + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=30), freeze=0) + assert _db(db_path).sync_freeze_reason("a1") is None + + +def test_sync_failing_reason_names_the_failures(tmp_path): + db_path = tmp_path / "t.db" + _seed(db_path, freeze=1, status="failed", failures=4) + assert _db(db_path).sync_freeze_reason("a1").startswith( + "last sync failed (seen on 4 polls)") + + +def test_mapping_only_rows_divergence_freeze(tmp_path, monkeypatch): + """The PG RealDictCursor shape (#300) for the new divergence columns.""" + from contextlib import contextmanager + + db_path = tmp_path / "t.db" + _diverged_seed(db_path, age=timedelta(hours=30)) + db = _db(db_path) + + class _DictRowCursor: + def __init__(self, cur): + self._cur = cur + + def execute(self, sql, params=()): + self._cur.execute(sql, params) + return self + + def fetchone(self): + row = self._cur.fetchone() + return None if row is None else dict(row) + + @contextmanager + def dict_row_connection(): + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + yield type( + "Conn", (), {"cursor": lambda self: _DictRowCursor(conn.cursor())} + )() + finally: + conn.close() + + monkeypatch.setattr(db, "get_connection", dict_row_connection) + assert db.should_freeze_schedules("a1") is True + + +# --------------------------------------------------------------------------- +# The service gate writes the reason into the skipped row +# --------------------------------------------------------------------------- + +class _GateDB: + """Just enough of SchedulerDatabase to reach the #1808 gate.""" + + def __init__(self, reason): + self._reason = reason + + def get_schedule(self, schedule_id): + return SimpleNamespace(id=schedule_id, agent_name="a1", enabled=True) + + def get_autonomy_enabled(self, agent_name): + return True + + def sync_freeze_reason(self, agent_name): + return self._reason + + +def _gate_service(reason): + import src.scheduler.service as scheduler_service + + svc = scheduler_service.SchedulerService(database=_GateDB(reason), lock_manager=object()) + calls = [] + svc._record_skipped_agent_schedule = lambda sid, skip_reason=None, event_reason=None: \ + calls.append({"skip_reason": skip_reason, "event_reason": event_reason}) + svc._advance_next_run_only = lambda schedule: None + + async def _hold(schedule, triggered_by): + return False # stop right after the freeze gate + + svc._apply_readiness_gate = _hold + return svc, calls + + +def test_the_skipped_row_carries_the_freeze_reason(): + svc, calls = _gate_service("diverged 0 behind / 7 ahead for 26h") + asyncio.run(svc._execute_schedule_with_lock("s1", triggered_by="schedule")) + assert calls == [{ + "skip_reason": "Git sync frozen: diverged 0 behind / 7 ahead for 26h", + "event_reason": "Git sync frozen (schedules paused)", + }] + + +def test_control_arm_no_reason_no_skip_row(): + svc, calls = _gate_service(None) + asyncio.run(svc._execute_schedule_with_lock("s1", triggered_by="schedule")) + assert calls == [] + + +def test_a_manual_trigger_is_never_frozen(): + svc, calls = _gate_service("diverged 0 behind / 7 ahead for 26h") + asyncio.run(svc._execute_schedule_with_lock("s1", triggered_by="manual")) + assert calls == [] diff --git a/tests/unit/test_2392_effect_guard_fail_closed.py b/tests/unit/test_2392_effect_guard_fail_closed.py new file mode 100644 index 0000000000..d78517aa5f --- /dev/null +++ b/tests/unit/test_2392_effect_guard_fail_closed.py @@ -0,0 +1,188 @@ +""" +Fail-closed effect guard on pull-mode agents (#2392). + +Pull re-delivers the SAME execution after a lease expiry, so an outbound side +effect (message, call, share, A2A call) without a usable execution id cannot be +de-duplicated. On a pull-mode agent `effect_guard` refuses it and raises an +operator alarm; elsewhere the send proceeds and is logged as degraded. A +`manual` id (a person's terminal session) and a lookup error always proceed. + +Reuses the temp-DB `effect_service` fixture from test_idempotency.py. +Related flow: docs/memory/feature-flows/effect-idempotency.md +""" + +import logging +import sys +import types +from pathlib import Path + +import pytest + +# Sibling import (the unit dir is not implicitly importable) — reuse the +# temp-DB effect_service harness rather than fork it. +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_idempotency import ( # noqa: E402,F401 — fixtures + _register_execution, + effect_service, + idem_ops, +) + + +PILOT = "pilot-agent" +PUSH = "push-agent" +ARGS = {"recipient": "u@example.com", "channel": "telegram"} + + +@pytest.fixture +def alarms(monkeypatch): + """Stub the operator-queue seam the guard imports lazily; record alarms.""" + calls = [] + stub = types.ModuleType("services.operator_queue_service") + + async def create_bounded_alert(agent_name, item): + calls.append((agent_name, item)) + return True + + stub.create_bounded_alert = create_bounded_alert + monkeypatch.setitem(sys.modules, "services.operator_queue_service", stub) + return calls + + +@pytest.fixture +def pilot_env(monkeypatch): + monkeypatch.setenv("PULL_MODE_PILOT_AGENTS", PILOT) + + +async def _send(svc, *, execution_id, agent_name, sends, effect_type="message", args=ARGS): + async with svc.effect_guard( + effect_type, args, execution_id=execution_id, agent_name=agent_name, + ) as g: + if g.replay: + return "replay" + sends.append(execution_id) + g.snapshot = {"ok": True} + return "sent" + + +pytestmark = pytest.mark.asyncio + + +class TestPullAgentRefuses: + + @pytest.mark.parametrize("execution_id,reason", [ + (None, "absent"), + ("", "absent"), + ("ghost-exec", "unknown"), + ]) + async def test_unusable_id_refused_with_alarm( + self, effect_service, alarms, pilot_env, execution_id, reason + ): + sends = [] + with pytest.raises(effect_service.EffectUnguardedError) as exc: + await _send(effect_service, execution_id=execution_id, agent_name=PILOT, sends=sends) + assert sends == [] + assert exc.value.reason == reason + assert "Do not retry" in str(exc.value) + assert len(alarms) == 1 + agent, item = alarms[0] + assert agent == PILOT + assert item["type"] == "effect_unguarded" + assert item["id"].startswith(effect_service.EFFECT_UNGUARDED_ALERT_PREFIX + PILOT) + assert item["context"] == {"effect_type": "message", "reason": reason} + + async def test_foreign_execution_refused(self, effect_service, alarms, pilot_env): + _register_execution(effect_service, "exec-other", "someone-else") + sends = [] + with pytest.raises(effect_service.EffectUnguardedError) as exc: + await _send(effect_service, execution_id="exec-other", agent_name=PILOT, sends=sends) + assert exc.value.reason == "foreign" + assert sends == [] + assert len(alarms) == 1 + + async def test_refused_even_when_alarm_budget_exhausted( + self, effect_service, alarms, pilot_env, monkeypatch + ): + async def at_cap(agent_name, item): + return False # budget full — no queue item, but the send still must not go + + monkeypatch.setattr(sys.modules["services.operator_queue_service"], "create_bounded_alert", at_cap) + sends = [] + with pytest.raises(effect_service.EffectUnguardedError): + await _send(effect_service, execution_id=None, agent_name=PILOT, sends=sends) + assert sends == [] + + async def test_manual_session_proceeds_without_alarm(self, effect_service, alarms, pilot_env, caplog): + sends = [] + with caplog.at_level(logging.WARNING): + assert await _send(effect_service, execution_id="manual", agent_name=PILOT, sends=sends) == "sent" + assert sends == ["manual"] + assert alarms == [] + assert "effect_guard.degraded" in caplog.text and "reason=manual" in caplog.text + + async def test_lookup_error_proceeds(self, effect_service, alarms, pilot_env, monkeypatch, caplog): + def boom(eid): + raise RuntimeError("db down") + + monkeypatch.setattr(sys.modules["database"].db, "get_execution", boom) + sends = [] + with caplog.at_level(logging.WARNING): + assert await _send(effect_service, execution_id="exec-1", agent_name=PILOT, sends=sends) == "sent" + assert alarms == [] + assert "reason=lookup_error" in caplog.text + + +class TestRedeliveryEmitsOnce: + """AC: a re-delivered execution emits each effect exactly once.""" + + @pytest.mark.parametrize("effect_type,args", [ + ("message", {"recipient": "u@example.com", "channel": "telegram"}), + ("voip_call", {"to": "+15550100"}), + ("share_file", {"filename": "report.pdf"}), + ("a2a_call", {"endpoint": "peer", "message_sha": "abc"}), + ]) + async def test_same_execution_twice_one_send( + self, effect_service, alarms, pilot_env, effect_type, args + ): + _register_execution(effect_service, "exec-1", PILOT) + sends = [] + first = await _send(effect_service, execution_id="exec-1", agent_name=PILOT, + sends=sends, effect_type=effect_type, args=args) + # Lease expired → reaper re-queues the SAME execution id → the turn runs again. + second = await _send(effect_service, execution_id="exec-1", agent_name=PILOT, + sends=sends, effect_type=effect_type, args=args) + assert (first, second) == ("sent", "replay") + assert sends == ["exec-1"] + assert alarms == [] + + +class TestPushAgentUnchanged: + + @pytest.mark.parametrize("execution_id,reason", [ + (None, "absent"), ("ghost-exec", "unknown"), ("manual", "manual"), + ]) + async def test_send_proceeds_and_is_logged( + self, effect_service, alarms, pilot_env, caplog, execution_id, reason + ): + sends = [] + with caplog.at_level(logging.WARNING): + assert await _send(effect_service, execution_id=execution_id, agent_name=PUSH, sends=sends) == "sent" + assert len(sends) == 1 + assert alarms == [] + assert "effect_guard.degraded" in caplog.text and f"reason={reason}" in caplog.text + + async def test_no_pilots_configured_never_refuses(self, effect_service, alarms, monkeypatch): + monkeypatch.delenv("PULL_MODE_PILOT_AGENTS", raising=False) + sends = [] + assert await _send(effect_service, execution_id=None, agent_name=PILOT, sends=sends) == "sent" + assert alarms == [] + + +def test_alarm_type_registered_and_prefix_reserved(): + """The alarm must pass create_bounded_alert's closed type set, and its id + prefix must be reserved so an agent cannot pre-create and suppress it.""" + import services.operator_queue_service as oqs + from services.idempotency_service import EFFECT_UNGUARDED_ALERT_PREFIX + + assert "effect_unguarded" in oqs._BUDGETED_ALERT_TYPES + assert EFFECT_UNGUARDED_ALERT_PREFIX in oqs._RESERVED_ID_PREFIXES diff --git a/tests/unit/test_2392_execution_id_header.py b/tests/unit/test_2392_execution_id_header.py new file mode 100644 index 0000000000..10bfc44274 --- /dev/null +++ b/tests/unit/test_2392_execution_id_header.py @@ -0,0 +1,87 @@ +"""#2392: the Trinity MCP entry carries X-Trinity-Execution-Id. + +Writers (agent server) emit the runtime-native form; the validator's +canonical-trinity allowlist accepts the old and the new shape and nothing else. +Byte parity of the two validator copies is pinned by +test_2007_mcp_template_render.py. +""" + +from __future__ import annotations + +import json +import tomllib + +import pytest + +from agent_server.services import trinity_mcp # noqa: E402 +from services.mcp_validator import McpValidationError, validate_mcp_config # noqa: E402 + +pytestmark = pytest.mark.unit + +URL = "http://mcp-server:8080/mcp" +KEY = "trinity_mcp_abc123" +LITERAL = "${TRINITY_EXECUTION_ID:-manual}" + + +@pytest.fixture +def home(tmp_path, monkeypatch): + # Both writers hard-code Path("/home/developer"). + monkeypatch.setattr(trinity_mcp, "Path", lambda _p: tmp_path) + return tmp_path + + +def test_claude_writer_emits_both_headers(home): + assert trinity_mcp._inject_claude_mcp(URL, KEY) is True + entry = json.loads((home / ".mcp.json").read_text())["mcpServers"]["trinity"] + assert entry["headers"] == { + "Authorization": f"Bearer {KEY}", + "X-Trinity-Execution-Id": LITERAL, + } + # The writer's own output must pass the reserved-name gate. + validate_mcp_config(json.dumps({"mcpServers": {"trinity": entry}})) + + +def test_gemini_writer_emits_expansion_literal(home): + assert trinity_mcp._inject_gemini_mcp(URL, KEY) is True + entry = json.loads((home / ".gemini" / "settings.json").read_text())["mcpServers"]["trinity"] + assert entry["headers"]["X-Trinity-Execution-Id"] == LITERAL + + +def test_codex_writer_uses_env_http_headers(tmp_path, monkeypatch): + monkeypatch.setenv("CODEX_HOME", str(tmp_path)) + assert trinity_mcp._inject_codex_mcp(URL, KEY) is True + trinity = tomllib.loads((tmp_path / "config.toml").read_text())["mcp_servers"]["trinity"] + assert trinity["env_http_headers"] == {"X-Trinity-Execution-Id": "TRINITY_EXECUTION_ID"} + assert trinity["bearer_token_env_var"] == "TRINITY_MCP_API_KEY" + + +def _cfg(headers: dict) -> str: + return json.dumps({"mcpServers": {"trinity": {"type": "http", "url": URL, "headers": headers}}}) + + +AUTH = {"Authorization": f"Bearer {KEY}"} + + +def test_validator_accepts_old_shape(): + validate_mcp_config(_cfg(AUTH)) + + +def test_validator_accepts_new_shape(): + validate_mcp_config(_cfg({**AUTH, "X-Trinity-Execution-Id": LITERAL})) + + +@pytest.mark.parametrize( + "headers", + [ + {**AUTH, "X-Trinity-Execution-Id": "${TRINITY_EXECUTION_ID}"}, + {**AUTH, "X-Trinity-Execution-Id": "manual"}, + {**AUTH, "X-Trinity-Execution-Id": "${HOME:-manual}"}, + {**AUTH, "x-trinity-execution-id": LITERAL}, + {**AUTH, "X-Trinity-Execution-Id": LITERAL, "X-Extra": "1"}, + {"X-Trinity-Execution-Id": LITERAL}, + ], + ids=["no-default", "static", "other-var", "lowercase", "extra-header", "no-auth"], +) +def test_validator_rejects_variants(headers): + with pytest.raises(McpValidationError, match="reserved"): + validate_mcp_config(_cfg(headers)) diff --git a/tests/unit/test_3063_portal_history_no_cost.py b/tests/unit/test_3063_portal_history_no_cost.py new file mode 100644 index 0000000000..f932b9129f --- /dev/null +++ b/tests/unit/test_3063_portal_history_no_cost.py @@ -0,0 +1,140 @@ +"""#3063 — Workspace history never carries a turn's `cost`. + +`GET /client-portal/agents/{name}/history` serialised `cost` on every message to +every principal, a portal token (an external client) included. The Workspace +contract is that external clients never see costs, and every other projection +already honours it (the Work projection: "No message, response, cost or +model"). Nothing in the frontend reads a history message's cost, so it is +projected out for EVERY principal rather than split by principal. + +Driven through the REAL route (`response_model=PortalHistory` is the projection +under test) against a throwaway sqlite carrying the real portal tables, with a +row written by the real writer carrying a real cost. + +Execution ids are deliberately NOT asserted absent: `in_flight_execution_id` +and `last_turn_outcome.execution_id` are the client's own turn handles — the +202 dispatch response issues the same id, and the client streams, cancels and +reattaches by it. That decision is stated in the PR. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_BACKEND = Path(__file__).resolve().parent.parent.parent / "src" / "backend" +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +CLIENT = "client@example.com" +AGENT = "scribe" +THREAD = "thread-3063" + + +@pytest.fixture() +def client_for(tmp_path, monkeypatch): + db_file = tmp_path / "trinity-3063.db" + monkeypatch.setenv("TRINITY_DB_PATH", str(db_file)) + import db.connection as conn_mod + monkeypatch.setattr(conn_mod, "DB_PATH", str(db_file)) + from db.engine import get_engine + from db.tables import metadata, enterprise_portal_messages, enterprise_portal_sessions + metadata.create_all(get_engine(), tables=[enterprise_portal_messages, enterprise_portal_sessions]) + + from client_portal import db as pdb, service as svc + from utils.helpers import utc_now_iso + pdb.create_portal_session(THREAD, AGENT, CLIENT, utc_now_iso()) + pdb.add_portal_message("m-user", AGENT, CLIENT, "user", "what did it cost?", None, + utc_now_iso(), session_id=THREAD) + pdb.add_portal_message("m-reply", AGENT, CLIENT, "assistant", "here you go", 0.4242, + utc_now_iso(), session_id=THREAD) + monkeypatch.setattr(svc, "agent_on_roster", lambda a, e, include_owned=False: True) + monkeypatch.setattr(svc, "_attach_own_ratings", lambda *a, **kw: None) + monkeypatch.setattr(svc, "get_turn_inflight", lambda sid: None) + + from fastapi import FastAPI + from fastapi.testclient import TestClient + from client_portal import router + from client_portal.portal_auth import PortalPrincipal, get_portal_principal + + def make(is_platform: bool): + app = FastAPI() + app.include_router(router.router) + app.dependency_overrides[get_portal_principal] = lambda: PortalPrincipal( + email=CLIENT, is_platform=is_platform) + return TestClient(app) + return make + + +def _keys(obj): + """Every key anywhere in a JSON document.""" + if isinstance(obj, dict): + for k, v in obj.items(): + yield k + yield from _keys(v) + elif isinstance(obj, list): + for v in obj: + yield from _keys(v) + + +@pytest.mark.parametrize("is_platform", [False, True], ids=["portal-token", "platform"]) +def test_history_carries_no_cost_anywhere(client_for, is_platform): + r = client_for(is_platform).get(f"/api/enterprise/client-portal/agents/{AGENT}/history", + params={"session_id": THREAD}) + assert r.status_code == 200, r.text + body = r.json() + # Positive control: the costed row IS in the response, so an absent key means + # "projected out", not "no rows came back". + assert [m["content"] for m in body["messages"]] == ["what did it cost?", "here you go"] + assert "cost" not in set(_keys(body)), f"cost reached the wire: {json.dumps(body)[:400]}" + assert "0.4242" not in r.text + + +def test_the_narrow_poll_read_carries_no_cost_either(client_for): + """The reply poll (`?limit=`) is the read that runs every 700 ms mid-turn.""" + r = client_for(False).get(f"/api/enterprise/client-portal/agents/{AGENT}/history", + params={"session_id": THREAD, "limit": 2}) + assert r.status_code == 200, r.text + assert "cost" not in set(_keys(r.json())) + + +def test_the_history_message_model_declares_no_cost(): + """The projection itself, so a later row-to-model path cannot reintroduce it.""" + from client_portal.models import PortalHistoryMessage + assert "cost" not in PortalHistoryMessage.model_fields + + +# --------------------------------------------------------------------------- +# The synchronous chat route (the streaming fallback) is the same contract. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("is_platform", [False, True], ids=["portal-token", "platform"]) +def test_the_sync_chat_reply_carries_no_cost(client_for, monkeypatch, is_platform): + from client_portal import router as prouter, service as svc + from services import rate_limiter + monkeypatch.setattr(rate_limiter, "enforce", lambda *a, **kw: None) + monkeypatch.setattr(prouter, "_require_roster", lambda *a, **kw: None) + monkeypatch.setattr(svc, "validate_requested_model", lambda m, is_platform: None) + monkeypatch.setattr(svc, "validated_open_canvas", lambda *a, **kw: None) + + async def fake_chat(*a, **kw): + # The real service return shape: it still carries the cost for accounting. + return {"response": "done", "cost": 0.4242, "session_id": THREAD, "message_id": "m-reply"} + monkeypatch.setattr(svc, "portal_chat", fake_chat) + + r = client_for(is_platform).post(f"/api/enterprise/client-portal/agents/{AGENT}/chat", + json={"message": "go", "session_id": THREAD}) + assert r.status_code == 200, r.text + body = r.json() + # Positive control: the reply and its handles are on the wire. + assert body["response"] == "done" and body["message_id"] == "m-reply" + assert "cost" not in body, f"cost reached the wire: {json.dumps(body)}" + + +def test_the_chat_response_model_declares_no_cost(): + from client_portal.models import PortalChatResponse + assert "cost" not in PortalChatResponse.model_fields diff --git a/tests/unit/test_736_a2a_outbound_call.py b/tests/unit/test_736_a2a_outbound_call.py index cf5813bd15..3b9135e29d 100644 --- a/tests/unit/test_736_a2a_outbound_call.py +++ b/tests/unit/test_736_a2a_outbound_call.py @@ -509,6 +509,19 @@ async def _boom(**kwargs): assert r.status_code == 409 +def test_an_unguarded_effect_on_a_pull_agent_is_a_422(client, monkeypatch): + """#2392: a refused effect surfaces as a named, non-retryable 422.""" + from services.idempotency_service import EffectUnguardedError + + async def _refuse(**kwargs): + raise EffectUnguardedError("a2a_call", "absent") + + monkeypatch.setattr(a2a_outbound_service, "call_agent", _refuse) + r = client.http.post("/api/agents/bot/a2a/call", json=_body()) + assert r.status_code == 422 + assert r.json()["detail"]["reason"] == "effect_unguarded" + + def test_a_failed_call_releases_the_claim_so_a_retry_is_possible(monkeypatch, endpoint): """`effect_guard` releases on exception. Without it a transient peer failure would wedge that (endpoint, label) for the whole 24h TTL.""" diff --git a/tests/unit/test_ent467_ws_agent_scope.py b/tests/unit/test_ent467_ws_agent_scope.py index b16578fe87..92e258a8a6 100644 --- a/tests/unit/test_ent467_ws_agent_scope.py +++ b/tests/unit/test_ent467_ws_agent_scope.py @@ -442,9 +442,9 @@ def test_connect_defaults_are_fail_closed(): ("routers/operator_queue.py", "operator_queue_cleared"), # trinity-enterprise#611: the ask sink's bulk-cancel trigger — ONE per sweep, # and a sweep can span agents, so it names none; it carries only the count - # (the acting operator's email, which the router's copy still carries, is not - # on it). Listeners refetch the access-controlled list, the same footing as - # the entry above. + # (the acting operator's email is on neither copy — the router's dropped it + # in trinity-enterprise#715). Listeners refetch the access-controlled list, + # the same footing as the entry above. ("services/ask_service.py", "operator_queue_cleared"), # #2915: ONE thin trigger per poll cycle when any row's sync or delivery # state changed. Carries NO payload (#918); listeners refetch diff --git a/tests/unit/test_ent611_native_ask.py b/tests/unit/test_ent611_native_ask.py index 92f38ddb80..ec19485595 100644 --- a/tests/unit/test_ent611_native_ask.py +++ b/tests/unit/test_ent611_native_ask.py @@ -1008,8 +1008,9 @@ def test_a_native_asks_addressee_is_withheld_from_machine_keys(self, route, path does a read: `resolved_to` and `addressed_to_email` hold the same email, so both are withheld from machines. Today that person is the owner (whose email an agent key can read anyway); once an assignment provider - answers `people_for` it can be someone else. A file entry's addressee is - the agent's own input and stays.""" + answers `people_for` it can be someone else. Since trinity-enterprise#715 + a file entry's addressee is withheld too: the agent wrote it, but the + same key reads its siblings' rows.""" uid = self._post(route, _body(f"rt-wh-{path}")).json()["id"] filed = route.ask.db.create_operator_queue_item( self.AGENT, {"id": f"rt-file-{path}", "type": "question", "title": "t", "question": "q", @@ -1031,7 +1032,7 @@ def rows(principal): assert native["addressed_to_email"] == OWNER native, file_row = rows(lambda: route.as_(mcp_scope="agent", agent_name=self.AGENT)) assert "addressed_to_email" not in native and "resolved_to" not in native - assert file_row["addressed_to_email"] == "client@example.com" + assert "addressed_to_email" not in file_row def test_a_null_context_or_proposal_reads_as_absent(self, route): """The MCP tool publishes both as object-or-null (so the published diff --git a/tests/unit/test_ent706_divergence_tracking.py b/tests/unit/test_ent706_divergence_tracking.py new file mode 100644 index 0000000000..675691dc85 --- /dev/null +++ b/tests/unit/test_ent706_divergence_tracking.py @@ -0,0 +1,383 @@ +"""The poller keeps the trinity-enterprise#706 columns (divergence tracking). + +Driven through the REAL `SyncHealthService._poll_cycle` against a real DB +(db_harness, #300); only the agent's `/api/git/status` response is faked. +Pinned values are non-ambient (7 ahead, 31 behind, 791 dirty). +""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import pytest + +from db_harness import db_backend, run as _hrun # noqa: F401 + +pytestmark = pytest.mark.unit + +_UNSET = object() + + +def _payload(*, ahead=0, behind=0, changes=0, status="success", + last_sync_at="2026-09-27T09:00:00+00:00", last_push=_UNSET, + error=None): + sync_state = { + "last_sync_status": status, + "last_sync_at": last_sync_at, + "last_error_summary": error, + "consecutive_failures": 0, + } + if last_push is not _UNSET: # absent on an agent image older than #3011 + sync_state["last_successful_push_at"] = last_push + return { + "git_enabled": True, + "branch": "trinity/alpha/abc123", + "last_commit": {"sha": "deadbeef"}, + "changes_count": changes, + "ahead_main": 0, + "behind_main": 0, + "ahead_working": ahead, + "behind_working": behind, + "sync_state": sync_state, + } + + +@pytest.fixture +def db(db_backend): # noqa: F811 + from database import db as database + + return database + + +@pytest.fixture +def service(db, monkeypatch): + import services.sync_health_service as shs + + # Fail-open lease: no Redis, so every test polls (see test_sync_health_service). + monkeypatch.setattr(shs, "get_breaker_redis", lambda: None) + # Bind the poller to the SAME db object the test reads — sibling suites + # evict and re-import `database`, so the module-level name can be stale. + monkeypatch.setattr(shs, "db", db) + return shs.SyncHealthService(poll_interval=0) + + +@pytest.fixture +def seed_agent(db): + def _seed(name="alpha", *, source_mode=0, auto_sync=1, freeze=1, + operator_push_at=None): + _hrun( + "INSERT INTO agent_ownership (agent_name, owner_id, created_at) " + "VALUES (:n, 1, '2026-01-01T00:00:00Z')", + n=name, + ) + _hrun( + "INSERT INTO agent_git_config " + "(id, agent_name, github_repo, working_branch, instance_id, created_at, " + " sync_enabled, source_mode, auto_sync_enabled, " + " freeze_schedules_if_sync_failing, last_sync_at) " + "VALUES (:gid, :n, :repo, :wb, 'abc123', '2026-01-01T00:00:00Z', 1, " + " :sm, :asy, :fz, :ops)", + gid=name + "-git", n=name, repo=f"org/{name}", wb=f"trinity/{name}/abc123", + sm=source_mode, asy=auto_sync, fz=freeze, ops=operator_push_at, + ) + return name + return _seed + + +async def _poll(service, payload): + with patch.object(service, "_fetch_git_status", AsyncMock(return_value=payload)): + await service._poll_cycle() + + +# -------------------------------------------------------------------------- +# diverged_since +# -------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_diverged_since_is_set_once(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=7)) + first = db.get_sync_state("alpha")["diverged_since"] + assert first is not None and first.endswith("Z") + await _poll(service, _payload(ahead=9, behind=31)) + assert db.get_sync_state("alpha")["diverged_since"] == first + + +@pytest.mark.asyncio +@pytest.mark.parametrize("behind_after", [0, None]) +async def test_diverged_since_clears_when_back_in_sync(service, seed_agent, db, behind_after): + """`behind is None` with a known `ahead == 0` is "no upstream" (#2105): + nothing on origin to be behind, so it clears too.""" + seed_agent() + await _poll(service, _payload(ahead=7)) + assert db.get_sync_state("alpha")["diverged_since"] is not None + await _poll(service, _payload(ahead=0, behind=behind_after)) + assert db.get_sync_state("alpha")["diverged_since"] is None + + +@pytest.mark.asyncio +async def test_unknown_ahead_neither_starts_nor_clears(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=None, behind=None)) + assert db.get_sync_state("alpha")["diverged_since"] is None # never starts + await _poll(service, _payload(ahead=7)) + first = db.get_sync_state("alpha")["diverged_since"] + await _poll(service, _payload(ahead=None, behind=None)) + assert db.get_sync_state("alpha")["diverged_since"] == first # never clears + + +@pytest.mark.asyncio +async def test_known_behind_with_unknown_ahead_starts_the_clock(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=None, behind=31)) + assert db.get_sync_state("alpha")["diverged_since"] is not None + + +# -------------------------------------------------------------------------- +# dirty_files / dirty_since +# -------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_dirty_count_and_clock(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(changes=791)) + row = db.get_sync_state("alpha") + assert row["dirty_files"] == 791 + first = row["dirty_since"] + assert first is not None + await _poll(service, _payload(changes=12)) + row = db.get_sync_state("alpha") + assert (row["dirty_files"], row["dirty_since"]) == (12, first) + await _poll(service, _payload(changes=0)) + row = db.get_sync_state("alpha") + assert (row["dirty_files"], row["dirty_since"]) == (0, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("garbage", ["791", -1, True, 2**40]) +async def test_garbage_dirty_count_keeps_the_prior_values(service, seed_agent, db, garbage): + seed_agent() + await _poll(service, _payload(changes=791)) + before = db.get_sync_state("alpha") + await _poll(service, _payload(changes=garbage)) + after = db.get_sync_state("alpha") + assert (after["dirty_files"], after["dirty_since"]) == (791, before["dirty_since"]) + + +@pytest.mark.asyncio +async def test_dirt_does_not_start_the_divergence_clock(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(changes=791)) + assert db.get_sync_state("alpha")["diverged_since"] is None + + +# -------------------------------------------------------------------------- +# last_successful_push_at +# -------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_last_push_is_the_later_of_heartbeat_and_operator_push(service, seed_agent, db): + seed_agent(operator_push_at="2026-09-26T08:00:00.000000Z") + await _poll(service, _payload(last_push="2026-09-26T09:30:00+00:00")) + assert db.get_sync_state("alpha")["last_successful_push_at"] == "2026-09-26T09:30:00.000000Z" + + +@pytest.mark.asyncio +async def test_operator_push_wins_when_it_is_later(service, seed_agent, db): + seed_agent(operator_push_at="2026-09-26T11:15:00.000000Z") + await _poll(service, _payload(last_push="2026-09-26T09:30:00+00:00")) + assert db.get_sync_state("alpha")["last_successful_push_at"] == "2026-09-26T11:15:00.000000Z" + + +@pytest.mark.asyncio +async def test_old_image_falls_back_to_a_successful_cycle(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(status="success", last_sync_at="2026-09-26T07:45:00+00:00")) + assert db.get_sync_state("alpha")["last_successful_push_at"] == "2026-09-26T07:45:00.000000Z" + + +@pytest.mark.asyncio +async def test_old_image_failed_cycle_is_not_a_push(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(status="failed", last_sync_at="2026-09-26T07:45:00+00:00", + error="rejected")) + assert db.get_sync_state("alpha")["last_successful_push_at"] is None + + +@pytest.mark.asyncio +async def test_new_image_that_never_pushed_does_not_fall_back(service, seed_agent, db): + """The key is PRESENT (None) on a #3011 image: no push has ever landed, and + `last_sync_at` of a success-status cycle is not evidence of one.""" + seed_agent() + await _poll(service, _payload(status="success", last_push=None)) + assert db.get_sync_state("alpha")["last_successful_push_at"] is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad", [ + "yesterday", + "2026-09-26T09:30:00", # naive: our writer always stamps an offset + "FUTURE", # resolved below: a collection-time timestamp would give each + # xdist worker a different test id ("Different tests were collected") + 12345, +]) +async def test_agent_push_time_is_validated(service, seed_agent, db, bad): + if bad == "FUTURE": + bad = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + seed_agent() + await _poll(service, _payload(last_push=bad)) + assert db.get_sync_state("alpha")["last_successful_push_at"] is None + + +@pytest.mark.asyncio +async def test_last_push_is_monotonic(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(last_push="2026-09-26T09:30:00+00:00")) + await _poll(service, _payload(last_push="2026-09-20T09:30:00+00:00")) + assert db.get_sync_state("alpha")["last_successful_push_at"] == "2026-09-26T09:30:00.000000Z" + + +# -------------------------------------------------------------------------- +# unreachable +# -------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_unreachable_agent_changes_nothing(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=7, changes=791)) + before = db.get_sync_state("alpha") + await _poll(service, None) + assert db.get_sync_state("alpha") == before + + +# -------------------------------------------------------------------------- +# The sync_diverged operator-queue item: one per episode, only when frozen +# -------------------------------------------------------------------------- + +def _age_the_episode(name="alpha", hours=24, extra_seconds=60): + start = datetime.now(timezone.utc) - timedelta(hours=hours, seconds=extra_seconds) + stamp = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + _hrun("UPDATE agent_sync_state SET diverged_since = :s WHERE agent_name = :n", + s=stamp, n=name) + return stamp + + +def _diverged_items(db, name="alpha"): + return [i for i in db.list_operator_queue_items(agent_name=name) + if i["type"] == "sync_diverged"] + + +@pytest.mark.asyncio +async def test_one_item_per_episode_across_polls(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=7)) + assert _diverged_items(db) == [] # just started: yellow, no item + stamp = _age_the_episode() + await _poll(service, _payload(ahead=7)) + await _poll(service, _payload(ahead=7)) + items = _diverged_items(db) + assert len(items) == 1 + item = items[0] + assert item["priority"] == "high" + assert "schedules paused" in item["title"] + assert "diverged 0 behind / 7 ahead for 24h" in item["question"] + ctx = item.get("context") or {} + assert ctx["ahead"] == 7 and ctx["behind"] == 0 + assert ctx["diverged_since"] == stamp + # Auto-sync is on, so "enable auto-sync" is not the fix; nothing is behind. + assert "recommendation" in ctx + assert "last_error_summary" not in ctx + + +@pytest.mark.asyncio +async def test_two_pollers_converge_on_one_item(service, seed_agent, db, monkeypatch): + """A fail-open lease (Redis down) makes every worker a leader.""" + import services.sync_health_service as shs + + seed_agent() + await _poll(service, _payload(ahead=7)) + _age_the_episode() + other = shs.SyncHealthService(poll_interval=0) + await _poll(service, _payload(ahead=7)) + await _poll(other, _payload(ahead=7)) + assert len(_diverged_items(db)) == 1 + + +@pytest.mark.asyncio +async def test_a_new_episode_gets_a_new_item(service, seed_agent, db): + seed_agent() + await _poll(service, _payload(ahead=7)) + _age_the_episode(hours=30) + await _poll(service, _payload(ahead=7)) + await _poll(service, _payload(ahead=0, behind=0)) # episode ends + assert db.get_sync_state("alpha")["diverged_since"] is None + await _poll(service, _payload(ahead=3)) # a new one begins + _age_the_episode(hours=25) + await _poll(service, _payload(ahead=3)) + assert len(_diverged_items(db)) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "seed_kwargs", + [ + {"freeze": 0}, # control arm: flag off + {"source_mode": 1, "auto_sync": 0}, # a deployment never freezes + ], + ids=["freeze-flag-off", "deployment"], +) +async def test_no_item_unless_a_freeze_is_in_force(service, seed_agent, db, seed_kwargs): + """Control arm: the same seed and the same aged episode, with the freeze + not in force, produce ZERO items — so the item above comes from the freeze + and not from anything ambient.""" + seed_agent(**seed_kwargs) + await _poll(service, _payload(ahead=7)) + _age_the_episode(hours=30) + await _poll(service, _payload(ahead=7)) + assert _diverged_items(db) == [] + + +def test_the_id_prefix_is_platform_reserved(): + from services.operator_queue_service import _RESERVED_ID_PREFIXES + + assert "sync-diverged-" in _RESERVED_ID_PREFIXES + + +# -------------------------------------------------------------------------- +# /review follow-up: a KEPT clock keeps the counts that started it +# -------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_unknown_tuple_keeps_the_counts_with_the_clock(service, seed_agent, db): + """The working tuple could not be computed (a git timeout: the agent sends + `ahead_working: None, behind_working: None`), so the episode clock is kept. + The counts must be kept WITH it. `_coerce_counter` turns the unknown into + 0, so before this fix the row read `diverged_since` = 25 h ago next to 0/0: + the freeze fired with the self-contradicting reason "diverged 0 behind / + 0 ahead" and the episode's one operator-queue item carried ahead 0 / + behind 0. Non-ambient values (7 ahead, 31 behind).""" + seed_agent() + await _poll(service, _payload(ahead=7, behind=31)) + _age_the_episode(hours=25) + await _poll(service, _payload(ahead=None, behind=None)) + + row = db.get_sync_state("alpha") + assert row["diverged_since"] is not None + assert (row["ahead_working"], row["behind_working"]) == (7, 31) + + items = _diverged_items(db) + assert len(items) == 1 + assert "diverged 31 behind / 7 ahead" in items[0]["question"] + ctx = items[0].get("context") or {} + assert (ctx["ahead"], ctx["behind"]) == (7, 31) + + +@pytest.mark.asyncio +async def test_unknown_tuple_without_an_episode_still_reads_zero(service, seed_agent, db): + """Control arm: with no episode to keep, an unknown tuple is stored as + before (#2827's default 0) — the keep is scoped to a kept clock.""" + seed_agent() + await _poll(service, _payload(ahead=None, behind=None)) + row = db.get_sync_state("alpha") + assert row["diverged_since"] is None + assert (row["ahead_working"], row["behind_working"]) == (0, 0) diff --git a/tests/unit/test_ent706_internal_sync_health.py b/tests/unit/test_ent706_internal_sync_health.py new file mode 100644 index 0000000000..2dcd6b4bdc --- /dev/null +++ b/tests/unit/test_ent706_internal_sync_health.py @@ -0,0 +1,89 @@ +"""`GET /api/internal/agents/{name}/sync-health-status` (trinity-enterprise#706). + +The pre-#706 keys keep their names and meaning; `should_freeze` now also +covers the divergence clause, and the response names why +(`freeze_reason`, `divergence_age_s`, `work_agent`). Real DB via db_harness. +""" +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone + +import pytest + +from db_harness import db_backend # noqa: F401 + +pytestmark = pytest.mark.unit + +_OLD_KEYS = { + "agent_name", + "freeze_schedules_if_sync_failing", + "sync_failing", + "should_freeze", + "consecutive_failures", +} + + +def _iso(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +@pytest.fixture +def db(db_backend): # noqa: F811 + from database import db as database + + database.create_git_config( + agent_name="ent706-i", github_repo="owner/ent706-i", + working_branch="trinity/ent706-i/x", instance_id="inst-i", + ) + database.set_freeze_schedules_if_sync_failing("ent706-i", True) + return database + + +def _call(name="ent706-i"): + from routers.internal import internal_agent_sync_health + + return asyncio.run(internal_agent_sync_health(name)) + + +def test_divergence_freeze_is_reported_with_its_reason(db): + now = datetime.now(timezone.utc) + db.upsert_sync_state( + "ent706-i", last_sync_status="success", ahead_working=7, + diverged_since=_iso(now - timedelta(hours=26)), + last_check_at=_iso(now - timedelta(seconds=30)), + ) + body = _call() + assert _OLD_KEYS <= set(body) + assert body["freeze_schedules_if_sync_failing"] is True + assert body["sync_failing"] is False + assert body["should_freeze"] is True + assert body["consecutive_failures"] == 0 + assert body["freeze_reason"] == "diverged 0 behind / 7 ahead for 26h" + assert 26 * 3600 <= body["divergence_age_s"] < 26 * 3600 + 60 + assert body["work_agent"] is True + + +def test_sync_failing_is_unchanged(db): + for _ in range(3): + db.upsert_sync_state("ent706-i", last_sync_status="failed", + last_error_summary="rejected") + body = _call() + assert (body["sync_failing"], body["should_freeze"], body["consecutive_failures"]) == ( + True, True, 3) + assert body["freeze_reason"].startswith("last sync failed (seen on 3 polls)") + + +def test_control_arm_in_sync_does_not_freeze(db): + db.upsert_sync_state("ent706-i", last_sync_status="success") + body = _call() + assert body["should_freeze"] is False + assert body["freeze_reason"] is None + assert body["divergence_age_s"] is None + + +def test_unknown_agent_is_not_frozen(db): + body = _call("no-such-agent") + assert _OLD_KEYS <= set(body) + assert body["should_freeze"] is False + assert body["work_agent"] is False diff --git a/tests/unit/test_ent706_sync_health_surfaces.py b/tests/unit/test_ent706_sync_health_surfaces.py new file mode 100644 index 0000000000..208359da13 --- /dev/null +++ b/tests/unit/test_ent706_sync_health_surfaces.py @@ -0,0 +1,172 @@ +"""The sync-health read surfaces carry the backend's verdict (trinity-enterprise#706). + +`GET /api/agents/sync-health` (the dashboard dot's batch feed) and +`GET /api/agents/{name}/git/sync-state` (and MCP `get_git_sync_state`, a +pass-through) add the policy's `state` / `reason` / `recommendation` / `binding` +/ `freeze` and the new columns, and keep every pre-existing key unchanged. +Also the shared reader `db.list_sync_health_rows`. Real DB via db_harness; the +handlers are called directly with the Docker-backed accessibility lookup +stubbed. +""" +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest + +from db_harness import db_backend, run as _hrun # noqa: F401 + +pytestmark = pytest.mark.unit + +_PRE_706_BATCH_KEYS = { + "agent_name", "auto_sync_enabled", "last_sync_at", "last_sync_status", + "consecutive_failures", "last_error_summary", "behind_working", "behind_main", + "ahead_working", "ahead_main", "git_dir_bytes", +} +_NEW_KEYS = { + "state", "reason", "recommendation", "binding", "dirty_files", + "divergence_age_s", "last_successful_push_at", "freeze", +} + + +def _iso(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +@pytest.fixture +def db(db_backend): # noqa: F811 + from database import db as database + + return database + + +def _bind(db, name, *, source_mode=0, auto_sync=True, freeze=True, sync_enabled=1, + deleted=False): + _hrun( + "INSERT INTO agent_ownership (agent_name, owner_id, created_at, deleted_at) " + "VALUES (:n, 1, '2026-01-01T00:00:00Z', :d)", + n=name, d="2026-09-01T00:00:00Z" if deleted else None, + ) + db.create_git_config( + agent_name=name, github_repo=f"owner/{name}", + working_branch=f"trinity/{name}/x", instance_id=f"inst-{name}", + source_mode=bool(source_mode), + ) + db.set_git_auto_sync_enabled(name, auto_sync) + db.set_freeze_schedules_if_sync_failing(name, freeze) + if not sync_enabled: + _hrun("UPDATE agent_git_config SET sync_enabled = 0 WHERE agent_name = :n", n=name) + + +def _diverge(db, name, *, hours, ahead=7): + now = datetime.now(timezone.utc) + db.upsert_sync_state( + name, last_sync_status="success", ahead_working=ahead, + diverged_since=_iso(now - timedelta(hours=hours)), + dirty_files=791, last_successful_push_at="2026-09-26T09:30:00.000000Z", + last_check_at=_iso(now - timedelta(seconds=30)), + ) + + +# -------------------------------------------------------------------------- +# Shared reader +# -------------------------------------------------------------------------- + +def test_reader_returns_config_and_state_for_live_git_agents(db): + _bind(db, "r-live") + _diverge(db, "r-live", hours=2) + _bind(db, "r-nostate") + _bind(db, "r-deleted", deleted=True) + _bind(db, "r-off", sync_enabled=0) + rows = db.list_sync_health_rows() + assert set(rows) >= {"r-live", "r-nostate"} + assert not {"r-deleted", "r-off"} & set(rows) + live = rows["r-live"] + assert live["config"]["auto_sync_enabled"] is True + assert live["config"]["source_mode"] is False + assert live["state"]["ahead_working"] == 7 + assert live["state"]["dirty_files"] == 791 + assert rows["r-nostate"]["state"] is None + assert set(db.list_sync_health_rows(["r-live", "someone-else"])) == {"r-live"} + assert db.list_sync_health_rows([]) == {} + + +# -------------------------------------------------------------------------- +# GET /api/agents/sync-health +# -------------------------------------------------------------------------- + +def _batch(monkeypatch, names): + import routers.agents as agents_router + + monkeypatch.setattr(agents_router, "get_accessible_agents", + lambda user: [{"name": n} for n in names]) + body = asyncio.run(agents_router.get_all_sync_health(current_user=SimpleNamespace())) + return {e["agent_name"]: e for e in body["agents"]} + + +def test_batch_carries_the_verdict_and_keeps_every_old_key(db, monkeypatch): + _bind(db, "b-red") + _diverge(db, "b-red", hours=26) + _bind(db, "b-yellow", source_mode=1, auto_sync=False) + _diverge(db, "b-yellow", hours=26) + entries = _batch(monkeypatch, ["b-red", "b-yellow", "b-nogit"]) + + red = entries["b-red"] + assert _PRE_706_BATCH_KEYS | _NEW_KEYS <= set(red) + assert red["state"] == "red" + assert red["reason"].startswith("diverged 0 behind / 7 ahead for 26h") + assert red["binding"] == "agent" + assert red["freeze"] is True + assert red["dirty_files"] == 791 + assert red["last_successful_push_at"] == "2026-09-26T09:30:00.000000Z" + assert 26 * 3600 <= red["divergence_age_s"] < 26 * 3600 + 60 + # Pre-existing keys keep their meaning. + assert (red["auto_sync_enabled"], red["ahead_working"], red["last_sync_status"]) == ( + True, 7, "success") + + yellow = entries["b-yellow"] + assert (yellow["state"], yellow["binding"], yellow["freeze"]) == ( + "yellow", "deployment", False) + + nogit = entries["b-nogit"] + assert _PRE_706_BATCH_KEYS | _NEW_KEYS <= set(nogit) + assert nogit["state"] == "unknown" + assert (nogit["last_sync_status"], nogit["auto_sync_enabled"]) == ("never", False) + + +def test_batch_is_scoped_to_accessible_agents(db, monkeypatch): + _bind(db, "b-mine") + _bind(db, "b-theirs") + assert set(_batch(monkeypatch, ["b-mine"])) == {"b-mine"} + + +# -------------------------------------------------------------------------- +# GET /api/agents/{name}/git/sync-state +# -------------------------------------------------------------------------- + +def _single(name): + from routers.git import get_agent_sync_state + + return asyncio.run(get_agent_sync_state(name)) + + +def test_sync_state_is_the_row_plus_the_verdict(db): + _bind(db, "s-red") + _diverge(db, "s-red", hours=26) + body = _single("s-red") + assert body["ahead_working"] == 7 + assert body["diverged_since"] is not None + assert body["dirty_files"] == 791 + assert body["state"] == "red" + assert body["freeze"] is True + assert body["reason"].startswith("diverged 0 behind / 7 ahead for 26h") + + +def test_sync_state_without_a_row_is_unknown(db): + _bind(db, "s-none") + body = _single("s-none") + assert (body["agent_name"], body["last_sync_status"], body["consecutive_failures"]) == ( + "s-none", "never", 0) + assert body["state"] == "unknown" diff --git a/tests/unit/test_ent706_sync_policy.py b/tests/unit/test_ent706_sync_policy.py new file mode 100644 index 0000000000..dbf4b20796 --- /dev/null +++ b/tests/unit/test_ent706_sync_policy.py @@ -0,0 +1,467 @@ +"""The sync-health policy (trinity-enterprise#706). + +`services/sync_freeze_policy.py::classify` is the one place that decides an +agent's sync state, its reason, its recommendation and whether its schedules +freeze. The backend surfaces and the scheduler's gate both call it (the +scheduler through a byte-identical mirror, pinned in +`test_ent706_sync_policy_parity.py`). + +Every pinned value is NON-ambient (24 h ± 1 s, 31 behind, 7 ahead, 791 dirty), +so a default that happened to equal the pin cannot make a case pass. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from services import sync_freeze_policy as policy + +pytestmark = pytest.mark.unit + +NOW = datetime(2026, 9, 27, 12, 0, 0, tzinfo=timezone.utc) +DAY = timedelta(hours=24) + + +def _iso(dt: datetime) -> str: + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _row(**over): + row = { + "last_check_at": _iso(NOW - timedelta(seconds=30)), + "last_sync_at": _iso(NOW - timedelta(minutes=10)), + "last_sync_status": "success", + "consecutive_failures": 0, + "last_error_summary": None, + "ahead_main": 0, + "behind_main": 0, + "ahead_working": 0, + "behind_working": 0, + "diverged_since": None, + "dirty_files": 0, + "dirty_since": None, + } + row.update(over) + return row + + +def _cfg(source_mode=0, auto_sync=1, freeze=1, created_at=None): + return { + "source_mode": source_mode, + "auto_sync_enabled": auto_sync, + "freeze_schedules_if_sync_failing": freeze, + "created_at": created_at, + } + + +def _classify(row, cfg, **kw): + return policy.classify(row, cfg, NOW, **kw) + + +# -------------------------------------------------------------------------- +# Rule 0 — unknown +# -------------------------------------------------------------------------- + + +def test_no_row_is_unknown_and_never_freezes(): + v = _classify(None, _cfg()) + assert v["state"] == "unknown" + assert v["freeze"] is False + + +def test_never_observed_row_is_unknown(): + v = _classify(_row(last_check_at=None), _cfg()) + assert v["state"] == "unknown" + assert v["freeze"] is False + + +def test_green_when_in_sync(): + v = _classify(_row(), _cfg()) + assert v["state"] == "green" + assert v["freeze"] is False + assert v["divergence_age_s"] is None + + +# -------------------------------------------------------------------------- +# Rules 1 & 2 — failed syncs +# -------------------------------------------------------------------------- + + +def test_one_failed_poll_is_red_but_not_frozen(): + v = _classify( + _row( + last_sync_status="failed", + consecutive_failures=1, + last_error_summary="rejected: fetch first", + ), + _cfg(), + ) + assert v["state"] == "red" + assert v["sync_failing"] is False + assert v["freeze"] is False + assert "rejected: fetch first" in v["reason"] + + +def test_sync_failing_freezes_any_binding(): + """#1808 unchanged: a deployment that is failing still freezes.""" + v = _classify( + _row( + last_sync_status="failed", consecutive_failures=3, last_error_summary="boom" + ), + _cfg(source_mode=1, auto_sync=0), + ) + assert v["state"] == "red" + assert v["sync_failing"] is True + assert v["freeze"] is True + assert v["freeze_cause"] == "sync_failing" + assert "3 polls" in v["freeze_reason"] + + +def test_sync_failing_freeze_ignores_observation_freshness(): + """Rule 2 is the #1808 rule as it shipped — only the NEW divergence clause + is freshness-guarded.""" + v = _classify( + _row( + last_sync_status="failed", + consecutive_failures=5, + last_check_at=_iso(NOW - timedelta(hours=3)), + ), + _cfg(), + ) + assert v["freeze"] is True + + +def test_the_failed_reason_is_bounded_to_120_chars_of_error(): + err = "x" * 500 + v = _classify( + _row(last_sync_status="failed", consecutive_failures=1, last_error_summary=err), + _cfg(), + ) + assert "x" * 120 in v["reason"] + assert "x" * 121 not in v["reason"] + + +# -------------------------------------------------------------------------- +# Rule 3 — divergence age, the 24 h boundary +# -------------------------------------------------------------------------- + + +def test_work_agent_diverged_one_second_under_24h_is_yellow_and_runs(): + v = _classify( + _row(ahead_working=7, diverged_since=_iso(NOW - DAY + timedelta(seconds=1))), + _cfg(), + ) + assert v["state"] == "yellow" + assert v["freeze"] is False + + +def test_work_agent_diverged_one_second_over_24h_is_red_and_frozen(): + v = _classify( + _row(ahead_working=7, diverged_since=_iso(NOW - DAY - timedelta(seconds=1))), + _cfg(), + ) + assert v["state"] == "red" + assert v["freeze"] is True + assert v["freeze_cause"] == "divergence" + assert v["divergence_age_s"] == 86401 + assert v["freeze_reason"] == "diverged 0 behind / 7 ahead for 24h" + + +def test_reason_matches_the_issue_example(): + v = _classify( + _row(behind_working=31, diverged_since=_iso(NOW - timedelta(hours=26))), + _cfg(), + ) + assert v["reason"].startswith("diverged 31 behind / 0 ahead for 26h") + + +def test_freeze_flag_off_is_red_but_never_frozen(): + v = _classify( + _row(ahead_working=7, diverged_since=_iso(NOW - timedelta(hours=30))), + _cfg(freeze=0), + ) + assert v["state"] == "red" + assert v["freeze"] is False + + +def test_deployment_behind_31_for_26h_is_yellow_and_runs(): + """AC 3: a deployment being behind is normal.""" + v = _classify( + _row(behind_working=31, diverged_since=_iso(NOW - timedelta(hours=26))), + _cfg(source_mode=1, auto_sync=0), + ) + assert v["state"] == "yellow" + assert v["binding"] == "deployment" + assert v["work_agent"] is False + assert v["freeze"] is False + + +def test_fork_to_own_freezes(): + """source_mode=1 in the DB but auto-syncing to its own fork: a work agent.""" + v = _classify( + _row(ahead_working=7, diverged_since=_iso(NOW - timedelta(hours=30))), + _cfg(source_mode=1, auto_sync=1), + ) + assert v["work_agent"] is True + assert v["binding"] == "agent" + assert v["freeze"] is True + + +def test_source_mode_with_auto_sync_off_never_freezes_on_divergence(): + v = _classify( + _row(ahead_working=7, diverged_since=_iso(NOW - timedelta(hours=300))), + _cfg(source_mode=1, auto_sync=0), + ) + assert v["freeze"] is False + assert v["state"] == "yellow" + + +def test_stale_observation_stays_red_but_does_not_freeze(): + """D6: the poller writes nothing for an unreachable agent, so the last row + can be arbitrarily old. A stale divergence must not keep an agent frozen.""" + v = _classify( + _row( + ahead_working=7, + diverged_since=_iso(NOW - timedelta(hours=30)), + last_check_at=_iso(NOW - timedelta(minutes=16)), + ), + _cfg(), + ) + assert v["state"] == "red" + assert v["stale_observation"] is True + assert v["freeze"] is False + assert "last observed 16m ago" in v["reason"] + + +def test_fresh_observation_within_15_minutes_freezes(): + v = _classify( + _row( + ahead_working=7, + diverged_since=_iso(NOW - timedelta(hours=30)), + last_check_at=_iso(NOW - timedelta(minutes=14)), + ), + _cfg(), + ) + assert v["stale_observation"] is False + assert v["freeze"] is True + + +def test_aware_and_naive_timestamps_give_the_same_verdict(): + """The scheduler reads TEXT from SQLite and the backend may hold datetimes; + a naive value is UTC (the `parse_scheduler_ts` convention).""" + diverged = NOW - DAY - timedelta(seconds=1) + aware = _classify(_row(ahead_working=7, diverged_since=_iso(diverged)), _cfg()) + naive_str = _classify( + _row(ahead_working=7, diverged_since=diverged.replace(tzinfo=None).isoformat()), + _cfg(), + ) + naive_dt = _classify( + _row( + ahead_working=7, + diverged_since=diverged.replace(tzinfo=None), + last_check_at=(NOW - timedelta(seconds=30)).replace(tzinfo=None), + ), + _cfg(), + ) + offset = _classify( + _row( + ahead_working=7, + diverged_since=diverged.astimezone( + timezone(timedelta(hours=3)) + ).isoformat(), + ), + _cfg(), + ) + for v in (naive_str, naive_dt, offset): + assert (v["state"], v["freeze"], v["divergence_age_s"]) == ( + aware["state"], + aware["freeze"], + aware["divergence_age_s"], + ) + assert aware["freeze"] is True + + +def test_garbage_timestamp_is_ignored_not_raised(): + v = _classify(_row(ahead_working=7, diverged_since="not-a-date"), _cfg()) + assert v["freeze"] is False + assert v["divergence_age_s"] is None + + +def test_behind_main_is_information_only(): + """D7/AC 6: a human push to `main` is not divergence of a working branch.""" + v = _classify(_row(behind_main=4), _cfg()) + assert v["state"] == "green" + v2 = _classify( + _row( + ahead_working=7, + behind_main=4, + diverged_since=_iso(NOW - timedelta(hours=2)), + ), + _cfg(), + ) + assert "(main is 4 ahead)" in v2["reason"] + + +# -------------------------------------------------------------------------- +# Rule 4 — dirt +# -------------------------------------------------------------------------- + + +def test_work_agent_dirty_one_second_over_24h_is_red_but_not_frozen(): + v = _classify( + _row(dirty_files=791, dirty_since=_iso(NOW - DAY - timedelta(seconds=1))), + _cfg(), + ) + assert v["state"] == "red" + assert v["freeze"] is False + assert v["dirty_age_s"] == 86401 + assert "dirty: 791 files uncommitted for 24h" in v["reason"] + + +def test_dirt_under_24h_is_not_a_state(): + v = _classify( + _row(dirty_files=791, dirty_since=_iso(NOW - DAY + timedelta(seconds=1))), + _cfg(), + ) + assert v["state"] == "green" + + +def test_deployment_dirty_over_24h_is_yellow(): + v = _classify( + _row(dirty_files=791, dirty_since=_iso(NOW - timedelta(hours=30))), + _cfg(source_mode=1, auto_sync=0), + ) + assert v["state"] == "yellow" + + +# -------------------------------------------------------------------------- +# Rule 5 — the 7-day no-heartbeat floor +# -------------------------------------------------------------------------- + + +def test_auto_sync_on_without_a_heartbeat_for_9_days_is_red(): + v = _classify( + _row(last_sync_at=_iso(NOW - timedelta(days=9)), last_sync_status="success"), + _cfg(), + ) + assert v["state"] == "red" + assert "auto-sync on, no heartbeat for 9d" in v["reason"] + assert v["freeze"] is False + + +def test_heartbeat_six_days_old_is_not_the_floor(): + v = _classify(_row(last_sync_at=_iso(NOW - timedelta(days=6))), _cfg()) + assert v["state"] == "green" + + +def test_never_heartbeat_is_measured_from_the_binding_creation(): + fresh = _classify( + _row(last_sync_at=None, last_sync_status="never"), + _cfg(created_at=_iso(NOW - timedelta(minutes=5))), + ) + assert fresh["state"] == "green" + old = _classify( + _row(last_sync_at=None, last_sync_status="never"), + _cfg(created_at=_iso(NOW - timedelta(days=8))), + ) + assert old["state"] == "red" + assert "no heartbeat for 8d" in old["reason"] + + +def test_no_heartbeat_floor_does_not_apply_with_auto_sync_off(): + v = _classify( + _row(last_sync_at=_iso(NOW - timedelta(days=30))), + _cfg(source_mode=0, auto_sync=0), + ) + assert v["state"] == "green" + + +# -------------------------------------------------------------------------- +# Multiple clauses and recommendations +# -------------------------------------------------------------------------- + + +def test_first_red_names_the_reason_and_the_rest_append(): + v = _classify( + _row( + last_sync_status="failed", + consecutive_failures=1, + last_error_summary="boom", + ahead_working=7, + diverged_since=_iso(NOW - timedelta(hours=30)), + ), + _cfg(), + ) + assert v["reason"].startswith("last sync failed") + assert "diverged 0 behind / 7 ahead for 30h" in v["reason"] + # The freeze reason names ONLY the clause that froze. + assert v["freeze_reason"] == "diverged 0 behind / 7 ahead for 30h" + + +@pytest.mark.parametrize( + ("row_over", "cfg_over", "kw", "expected"), + [ + ( + { + "last_sync_status": "failed", + "consecutive_failures": 1, + "last_error_summary": "remote: Write access to repository not granted.", + }, + {}, + {"push_denied": True}, + "credential is read-only", + ), + ( + { + "last_sync_status": "failed", + "consecutive_failures": 1, + "last_error_summary": "refused: source-mode on main", + }, + {}, + {}, + "deployment: turn auto-sync off", + ), + ( + { + "last_sync_status": "failed", + "consecutive_failures": 1, + "last_error_summary": "rejected (fetch first)", + }, + {}, + {}, + "push via git_sync strategy=pull_first", + ), + ({"ahead_working": 7}, {"auto_sync": 0}, {}, "enable auto-sync"), + ({"behind_working": 31}, {}, {}, "pull via git_pull"), + ({}, {}, {}, None), + ], +) +def test_recommendation_mapping(row_over, cfg_over, kw, expected): + v = _classify(_row(**row_over), _cfg(**cfg_over), **kw) + assert v["recommendation"] == expected + + +def test_constants_are_the_ruling(): + assert policy.SYNC_FAILURE_FREEZE_THRESHOLD == 3 + assert policy.DIVERGENCE_RED_SECONDS == 86400 + assert policy.DIRTY_RED_SECONDS == 86400 + assert policy.NO_HEARTBEAT_FLOOR_SECONDS == 7 * 86400 + assert policy.OBSERVATION_FRESH_SECONDS == 900 + + +def test_classify_never_raises_on_hostile_input(): + """Agent-authored numbers reach this function via the DB; a wrong type + must degrade, never raise into the scheduler's fire path.""" + v = _classify( + _row( + ahead_working="7", + behind_working=None, + dirty_files=[1], + consecutive_failures="x", + diverged_since=12345, + ), + {"source_mode": None, "auto_sync_enabled": "1"}, + ) + assert v["state"] in {"green", "yellow", "red", "unknown"} diff --git a/tests/unit/test_ent706_sync_policy_parity.py b/tests/unit/test_ent706_sync_policy_parity.py new file mode 100644 index 0000000000..2ae791046e --- /dev/null +++ b/tests/unit/test_ent706_sync_policy_parity.py @@ -0,0 +1,68 @@ +"""The scheduler's copy of the sync policy is the backend's, byte for byte +(trinity-enterprise#706). + +`src/scheduler` is a standalone image that cannot import `src/backend`, so the +policy is vendored rather than imported — the `failure_classifier.py` precedent +(#1088, Invariant #5). A copy that drifts is two policies: the dashboard would +say one thing and the scheduler would freeze on another. +""" +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[2] +_BACKEND_POLICY = _REPO / "src" / "backend" / "services" / "sync_freeze_policy.py" +_SCHEDULER_POLICY = _REPO / "src" / "scheduler" / "sync_freeze_policy.py" +_SCHEDULER_DB = _REPO / "src" / "scheduler" / "database.py" + + +def test_policy_files_are_byte_identical(): + assert _BACKEND_POLICY.read_bytes() == _SCHEDULER_POLICY.read_bytes(), ( + "src/scheduler/sync_freeze_policy.py has drifted from the canonical " + "src/backend/services/sync_freeze_policy.py. Re-sync the mirror:\n" + " cp src/backend/services/sync_freeze_policy.py " + "src/scheduler/sync_freeze_policy.py" + ) + + +def test_policy_is_a_stdlib_leaf(): + """The mirror must import in the scheduler image, which has neither the + backend's packages nor its modules on the path.""" + tree = ast.parse(_BACKEND_POLICY.read_text()) + allowed = {"__future__", "datetime", "typing"} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [a.name.split(".")[0] for a in node.names] + elif isinstance(node, ast.ImportFrom): + assert node.level == 0, "no relative imports in a vendored leaf" + names = [(node.module or "").split(".")[0]] + else: + continue + for name in names: + assert name in allowed, f"sync_freeze_policy imports {name!r}" + + +def test_scheduler_threshold_is_imported_not_redefined(): + """One number: the scheduler's freeze threshold comes from the policy.""" + tree = ast.parse(_SCHEDULER_DB.read_text()) + assigned = { + t.id + for node in ast.walk(tree) + if isinstance(node, ast.Assign) + for t in node.targets + if isinstance(t, ast.Name) + } + assert "SYNC_FAILURE_FREEZE_THRESHOLD" not in assigned + imported = { + alias.asname or alias.name + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module == "sync_freeze_policy" + for alias in node.names + } + assert "SYNC_FAILURE_FREEZE_THRESHOLD" in imported + assert "classify" in imported # the gate decides with the same function diff --git a/tests/unit/test_ent706_sync_state_columns.py b/tests/unit/test_ent706_sync_state_columns.py new file mode 100644 index 0000000000..6d086d6432 --- /dev/null +++ b/tests/unit/test_ent706_sync_state_columns.py @@ -0,0 +1,183 @@ +"""The four divergence columns on `agent_sync_state` (trinity-enterprise#706). + +Both migration tracks (Invariant #9) and the upsert's set / keep / clear +semantics. `diverged_since` and `dirty_since` are episode clocks: the poller +must be able to say "clear" (None) as distinct from "unchanged", which the +upsert's `_merged()` fallback cannot express — hence the `KEEP` sentinel. + +Backend-agnostic via ``db_harness`` (#300) for the upsert cases. +""" +from __future__ import annotations + +import ast +import sqlite3 +from pathlib import Path + +import pytest + +from db_harness import db_backend # noqa: F401 (fixture) + +pytestmark = pytest.mark.unit + +_REPO = Path(__file__).resolve().parents[2] +_VERSIONS = _REPO / "src" / "backend" / "migrations" / "versions" +NEW_COLUMNS = ("diverged_since", "dirty_files", "dirty_since", "last_successful_push_at") + +# agent_sync_state exactly as it stood before #706 — what an upgrading +# install's SQLite file holds when the new migration runs. +_PRE_706_DDL = """ +CREATE TABLE agent_sync_state ( + agent_name TEXT PRIMARY KEY, + last_sync_at TEXT, + last_sync_status TEXT, + consecutive_failures INTEGER DEFAULT 0, + last_error_summary TEXT, + last_remote_sha_main TEXT, + last_remote_sha_working TEXT, + ahead_main INTEGER DEFAULT 0, + behind_main INTEGER DEFAULT 0, + ahead_working INTEGER DEFAULT 0, + behind_working INTEGER DEFAULT 0, + git_dir_bytes BIGINT, + pack_count INTEGER, + loose_objects INTEGER, + maintenance_failures INTEGER DEFAULT 0, + last_check_at TEXT, + updated_at TEXT NOT NULL +) +""" + + +def _cols(conn) -> set: + return {r[1] for r in conn.execute("PRAGMA table_info(agent_sync_state)")} + + +# -------------------------------------------------------------------------- +# SQLite track +# -------------------------------------------------------------------------- + +def test_sqlite_migration_adds_the_columns_and_is_idempotent(): + from db.migrations import _migrate_agent_sync_state_divergence + + conn = sqlite3.connect(":memory:") + conn.execute(_PRE_706_DDL) + conn.execute( + "INSERT INTO agent_sync_state (agent_name, ahead_working, updated_at) " + "VALUES ('a1', 7, '2026-09-27T00:00:00Z')" + ) + assert not set(NEW_COLUMNS) & _cols(conn) + _migrate_agent_sync_state_divergence(conn.cursor(), conn) + _migrate_agent_sync_state_divergence(conn.cursor(), conn) # second run: no-op + assert set(NEW_COLUMNS) <= _cols(conn) + # No backfill: the clocks start at the first post-upgrade poll. + row = conn.execute( + "SELECT ahead_working, diverged_since, dirty_files, dirty_since, " + "last_successful_push_at FROM agent_sync_state" + ).fetchone() + assert row == (7, None, None, None, None) + + +def test_sqlite_migration_is_registered(): + from db.migrations import MIGRATIONS + + names = [name for name, _ in MIGRATIONS] + assert names.count("agent_sync_state_divergence") == 1 + + +def test_fresh_schema_carries_the_columns(): + from db.schema import TABLES + + conn = sqlite3.connect(":memory:") + conn.execute(TABLES["agent_sync_state"]) + assert set(NEW_COLUMNS) <= _cols(conn) + + +def test_core_table_carries_the_columns(): + from db.tables import agent_sync_state + + assert set(NEW_COLUMNS) <= set(agent_sync_state.c.keys()) + + +# -------------------------------------------------------------------------- +# PostgreSQL track (the revision's shape; the live upgrade is proved on a real +# PostgreSQL in the PR evidence — CI runs no PG pytest) +# -------------------------------------------------------------------------- + +def _revision_file() -> Path: + matches = sorted(_VERSIONS.glob("*_agent_sync_state_divergence.py")) + assert len(matches) == 1, matches + return matches[0] + + +def test_alembic_revision_adds_and_drops_all_four(): + path = _revision_file() + tree = ast.parse(path.read_text()) + assigns = { + t.id: node.value.value + for node in tree.body + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant) + for t in node.targets + if isinstance(t, ast.Name) + } + assert assigns["revision"] == path.stem + parent = _VERSIONS / f"{assigns['down_revision']}.py" + assert parent.exists(), f"down_revision {assigns['down_revision']} is not a revision" + text = path.read_text() + for col in NEW_COLUMNS: + assert f"ADD COLUMN IF NOT EXISTS {col} " in text + assert f"DROP COLUMN IF EXISTS {col}" in text + + +# -------------------------------------------------------------------------- +# Upsert: set / keep / clear +# -------------------------------------------------------------------------- + +@pytest.fixture +def sync_db(db_backend): # noqa: F811 + from database import db + + db.create_git_config( + agent_name="ent706-a", github_repo="owner/ent706-a", + working_branch="main", instance_id="inst-ent706-a", + ) + return db + + +def test_clocks_default_to_keep(sync_db): + # The sentinel from the module the LIVE ops object was built from — sibling + # suites evict and re-import `db.*`, so `from db.sync_state import KEEP` + # can name a different module object than the one `db` is holding. + KEEP = type(sync_db._sync_state_ops).upsert.__globals__["KEEP"] + + sync_db.upsert_sync_state( + "ent706-a", diverged_since="2026-09-26T10:00:00Z", + dirty_since="2026-09-26T11:00:00Z", dirty_files=791, + last_successful_push_at="2026-09-26T09:00:00Z", + ) + # A later partial upsert that says nothing about the clocks keeps them. + sync_db.upsert_sync_state("ent706-a", last_sync_status="success") + row = sync_db.get_sync_state("ent706-a") + assert row["diverged_since"] == "2026-09-26T10:00:00Z" + assert row["dirty_since"] == "2026-09-26T11:00:00Z" + assert row["dirty_files"] == 791 + assert row["last_successful_push_at"] == "2026-09-26T09:00:00Z" + # KEEP is also the explicit spelling. + sync_db.upsert_sync_state("ent706-a", diverged_since=KEEP, dirty_since=KEEP) + assert sync_db.get_sync_state("ent706-a")["diverged_since"] == "2026-09-26T10:00:00Z" + + +def test_none_clears_a_clock(sync_db): + sync_db.upsert_sync_state( + "ent706-a", diverged_since="2026-09-26T10:00:00Z", + dirty_since="2026-09-26T11:00:00Z", + ) + sync_db.upsert_sync_state("ent706-a", diverged_since=None, dirty_since=None) + row = sync_db.get_sync_state("ent706-a") + assert row["diverged_since"] is None + assert row["dirty_since"] is None + + +def test_zero_dirty_files_is_stored_not_dropped(sync_db): + sync_db.upsert_sync_state("ent706-a", dirty_files=791) + sync_db.upsert_sync_state("ent706-a", dirty_files=0) + assert sync_db.get_sync_state("ent706-a")["dirty_files"] == 0 diff --git a/tests/unit/test_ent715_queue_person_fields.py b/tests/unit/test_ent715_queue_person_fields.py new file mode 100644 index 0000000000..56d8d3d97a --- /dev/null +++ b/tests/unit/test_ent715_queue_person_fields.py @@ -0,0 +1,455 @@ +"""trinity-enterprise#715 — the operator queue stops handing a person's email to +a machine key. + +A machine principal (an agent-, system-, ops-, connector- or portal_delegate +key, a user-scoped key carrying an agent identity, and whatever scope ships +next) reads the queue for its own work. It never learns who answered, whom an +ask was for, or who a Workspace client is. Four doors: + + 1. `GET /api/operator-queue`, `/{id}` and `/agents/{name}` give a machine the + ALLOWLIST projection. Its key set is pinned below as a literal, and so is + the withheld set: a column added tomorrow turns this file red until + someone decides which side of the line it is on (learning 2026-09-23 — a + "whole row minus the withheld fields" diff cannot catch a new leak). + 2. The platform's heads-ups ABOUT a person (the ent#499 problem report, the + ent#308 inbox collision) are not returned to a machine at all. + 3. The file write-back no longer writes `responded_by`, on either path, and + the collision alert is a platform alarm, never written into the agent's + own file. + 4. The clear-resolved `/ws` trigger no longer carries the operator's email. + +A person — a JWT session or the person's own user-scoped key — still reads +every field. + +Related flow: docs/memory/feature-flows/operating-room.md +""" + +import asyncio +import json +import os +import sys +import uuid +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +pytest.importorskip("sqlalchemy") + +os.environ.setdefault("REDIS_URL", "redis://u:p@localhost:6379") +os.environ.setdefault("SECRET_KEY", "test-secret") + +_BACKEND = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "src", "backend")) +if _BACKEND not in sys.path: + sys.path.insert(0, _BACKEND) + +pytestmark = pytest.mark.unit + +# Test-local literals, never imported from the router: importing the code's own +# tuple would make the classification check circular. +WITHHELD = frozenset({ + "responded_by_id", "responded_by_email", "addressed_to_email", + "disposed_by_email", "resolved_to", +}) +MACHINE_ROW_KEYS = frozenset({ + "id", "agent_name", "request_id", "type", "status", "priority", "title", + "question", "options", "context", "execution_id", "created_at", "expires_at", + "response", "response_text", "responded_at", "acknowledged_at", "cleared_at", + "sync_state", "sync_detail", "sync_updated_at", "last_confirmed_at", + "delivery_state", "delivery_detail", "delivery_updated_at", + "divergence_acknowledged_at", + "disposition", "disposed_at", "disposed_by", "disposition_reason", "batch_id", + "raised_by", "channel", "to_role", "proposal", "supersedes_expired", + "aging", "aged_since", +}) + +AGENT = "agent-715-reads" +OP_EMAIL = "op-715@example.com" +FOR_EMAIL = "client-715@example.com" +RESOLVED_EMAIL = "ceo-715@example.com" + +PERSON_PRINCIPALS = [ + pytest.param({}, id="jwt"), + pytest.param({"mcp_scope": "user"}, id="user-key"), +] +MACHINE_PRINCIPALS = [ + pytest.param({"mcp_scope": "agent", "agent_name": AGENT}, id="agent-key"), + pytest.param({"mcp_scope": "system"}, id="system-key"), + pytest.param({"mcp_scope": "ops"}, id="ops-key"), + pytest.param({"mcp_scope": "connector", "connector_agent": AGENT}, id="connector-key"), + pytest.param({"mcp_scope": "portal_delegate", "portal_delegate": True}, id="portal-delegate-key"), + pytest.param({"mcp_scope": "user", "agent_name": AGENT}, id="user-scope-with-agent-identity"), + pytest.param({"mcp_scope": "a-scope-that-ships-tomorrow"}, id="unknown-scope"), +] +ROUTES = ["list", "item", "agent_list"] + + +def _rid(prefix="r715"): + return f"{prefix}-{uuid.uuid4().hex[:12]}" + + +@pytest.fixture +def real_db(): + from database import db + return db + + +# --------------------------------------------------------------------------- +# One app over the real router; `get_current_user` overridden by walking the +# routes' own dependant trees (learning 2026-09-23: override the dependency the +# route captured, never a fresh import). +# --------------------------------------------------------------------------- + +_APP = None +_PRINCIPAL = {"user": None} + + +def _client(): + global _APP + from fastapi.testclient import TestClient + if _APP is None: + from fastapi import FastAPI + from routers import operator_queue as r + app = FastAPI() + app.include_router(r.router) + found = set() + + def walk(dependant): + for sub in dependant.dependencies: + if getattr(sub.call, "__name__", "") == "get_current_user": + found.add(sub.call) + walk(sub) + + for route in r.router.routes: + if getattr(route, "dependant", None) is not None: + walk(route.dependant) + assert found, "no get_current_user dependency on the operator-queue routes" + for call in found: + app.dependency_overrides[call] = lambda: _PRINCIPAL["user"] + _APP = app + return TestClient(_APP, raise_server_exceptions=True) + + +def _as(**principal): + from models import User + base = {"id": 7, "username": "op-715", "email": OP_EMAIL, "role": "admin"} + base.update(principal) + _PRINCIPAL["user"] = User(**base) + + +def _seed(real_db, *, channel, agent=AGENT): + """An answered row carrying EVERY person field, each asserted non-null — the + redaction below is proven on real values, never on NULLs.""" + from sqlalchemy import update + from db.engine import get_engine + from db.tables import operator_queue + + uid = real_db.create_operator_queue_item(agent, { + "id": _rid(), "type": "approval", "status": "pending", "priority": "high", + "title": "Approve payout", "question": "Release 500 USDC?", + "options": ["approve", "reject"], "context": {}, + "created_at": "2026-09-01T10:00:00Z", "addressed_to_email": FOR_EMAIL, + }, channel=channel, raised_by="agent") + real_db.respond_to_operator_queue_item(uid, "approve", "fine", "7", OP_EMAIL) + with get_engine().begin() as conn: + conn.execute(update(operator_queue).where(operator_queue.c.id == uid).values( + addressed_to_email=FOR_EMAIL, resolved_to=json.dumps([RESOLVED_EMAIL]))) + row = real_db.get_operator_queue_item(uid) + for key in WITHHELD: + assert row[key], key + return uid + + +def _read(client, route, uid, agent=AGENT, headers=None): + if route == "item": + res = client.get(f"/api/operator-queue/{uid}", headers=headers) + assert res.status_code == 200, res.text + return res.json() + url = (f"/api/operator-queue?agent_name={agent}&limit=500" if route == "list" + else f"/api/operator-queue/agents/{agent}?limit=500") + res = client.get(url, headers=headers) + assert res.status_code == 200, res.text + rows = [i for i in res.json()["items"] if i["id"] == uid] + assert len(rows) == 1, f"row {uid} not in the {route} response" + return rows[0] + + +# =========================================================================== +# 1. The machine view is an allowlist; a person reads everything +# =========================================================================== + +class TestTheMachineView: + @pytest.mark.parametrize("channel", ["file", "mcp"]) + @pytest.mark.parametrize("route", ROUTES) + @pytest.mark.parametrize("principal", MACHINE_PRINCIPALS) + def test_a_machine_reads_exactly_the_machine_keys(self, real_db, principal, route, channel): + """File rows included: the agent wrote their addressee, but a sibling's + rows are readable by the same key, so it is a person's email all the same.""" + uid = _seed(real_db, channel=channel) + _as(**principal) + row = _read(_client(), route, uid) + assert set(row) == MACHINE_ROW_KEYS + assert "example.com" not in json.dumps(row) + + @pytest.mark.parametrize("route", ROUTES) + @pytest.mark.parametrize("principal", PERSON_PRINCIPALS) + def test_a_person_reads_every_person_field(self, real_db, principal, route): + """Also the classification guard: a person reads the whole row, so a new + column not yet placed on either side of the line fails here.""" + uid = _seed(real_db, channel="file") + _as(**principal) + row = _read(_client(), route, uid) + assert set(row) == MACHINE_ROW_KEYS | WITHHELD + assert (row["responded_by_email"], row["responded_by_id"], row["addressed_to_email"], + row["disposed_by_email"], row["resolved_to"]) == ( + OP_EMAIL, "7", FOR_EMAIL, OP_EMAIL, [RESOLVED_EMAIL]) + + def test_the_two_sides_do_not_overlap_and_the_readback_sits_inside_the_machine_view(self): + from routers import operator_queue as r + assert not MACHINE_ROW_KEYS & WITHHELD + assert set(r._READBACK_FIELDS) <= MACHINE_ROW_KEYS + + +class TestARealAgentKey: + """The override above stands in for key resolution. This presents a real + minted agent key to an app with NO overrides, so `get_current_user` resolves + it to the owner — carrying `mcp_scope='agent'` — as in production.""" + OWNER = "owner-715-keys" + AGENT = "agent-715-keys" + + @pytest.mark.parametrize("route", ROUTES) + def test_a_minted_agent_key_reads_the_machine_view(self, real_db, route, monkeypatch): + from db_models import UserCreate + from fastapi import FastAPI + from fastapi.testclient import TestClient + from routers import operator_queue as r + + monkeypatch.setattr(r, "_websocket_manager", None) + if not real_db.get_user_by_username(self.OWNER): + real_db.create_user(UserCreate(username=self.OWNER, role="user", email="owner-715@example.com")) + if not real_db.get_agent_owner(self.AGENT): + real_db.register_agent_owner(self.AGENT, self.OWNER) + uid = _seed(real_db, channel="file", agent=self.AGENT) + app = FastAPI() + app.include_router(r.router) + key = real_db.create_agent_mcp_api_key(self.AGENT, self.OWNER) + row = _read(TestClient(app), route, uid, agent=self.AGENT, + headers={"Authorization": f"Bearer {key.api_key}"}) + assert set(row) == MACHINE_ROW_KEYS + assert "example.com" not in json.dumps(row) + + +# =========================================================================== +# 2. The platform's heads-ups about a person never reach a machine +# =========================================================================== + +class TestPlatformRowsAboutAPerson: + """Both driven through their REAL emitters, so a renamed id or type turns + this red instead of leaving the registry pointing at nothing.""" + + @pytest.fixture + def heads_ups(self, real_db, monkeypatch): + from types import SimpleNamespace + from client_portal import service as cps + from routers import operator_queue as r + + monkeypatch.setattr(r, "_websocket_manager", None) + tag = uuid.uuid4().hex[:10] + agent = f"agent-715-hu-{tag}" # its own agent: the #1677 budget counts per agent + cps._alert_collided_inbox(agent, f"legacy_{tag}", + [f"a-{tag}@example.com", f"b-{tag}@example.com"]) + assert asyncio.run(cps.raise_problem_report( + agent, f"client-{tag}@example.com", target_kind="message", + target_id=f"m-{tag}", comment="the words they typed")) is True + rows = real_db.list_operator_queue_items(agent_name=agent, limit=50) + collision = next(i for i in rows if i["request_id"].startswith("portal-inbox-collision-")) + report = next(i for i in rows if i["type"] == "workspace_problem_report") + assert f"a-{tag}@example.com" in collision["question"] + assert f"client-{tag}@example.com" in report["question"] + return SimpleNamespace(agent=agent, ids={collision["id"], report["id"]}, + collision=collision, report=report) + + @pytest.mark.parametrize("principal", MACHINE_PRINCIPALS[:3]) + def test_a_machine_never_reads_them(self, heads_ups, principal): + _as(**principal) + client = _client() + for url in (f"/api/operator-queue?agent_name={heads_ups.agent}", + f"/api/operator-queue/agents/{heads_ups.agent}"): + res = client.get(url) + assert res.status_code == 200, res.text + assert not {i["id"] for i in res.json()["items"]} & heads_ups.ids + assert "example.com" not in res.text + for item_id in heads_ups.ids: + res = client.get(f"/api/operator-queue/{item_id}") + assert res.status_code == 404 and res.json() == {"detail": "Queue item not found"} + + @pytest.mark.parametrize("principal", PERSON_PRINCIPALS) + def test_a_person_still_reads_them(self, heads_ups, principal): + _as(**principal) + res = _client().get(f"/api/operator-queue?agent_name={heads_ups.agent}") + assert {i["id"] for i in res.json()["items"]} >= heads_ups.ids + + def test_both_are_platform_alarms_never_written_into_the_agents_file(self, heads_ups): + from services.operator_queue_service import is_platform_minted + assert is_platform_minted(heads_ups.collision) + assert is_platform_minted(heads_ups.report) + + def test_every_prefix_about_a_person_is_reserved(self): + """The machine reads key on `_ABOUT_A_PERSON_ID_PREFIXES`; the file + write-back, the resume wake and the ended-asks line key on + `_RESERVED_ID_PREFIXES`. A heads-up prefix outside the reserved set would + be hidden from a machine's read yet still written into the agent's file, + and an agent could pre-create its id. The tests above drive only today's + two emitters; this holds for the next one.""" + import services.operator_queue_service as oqs + assert set(oqs._ABOUT_A_PERSON_ID_PREFIXES) <= set(oqs._RESERVED_ID_PREFIXES) + + +# =========================================================================== +# 3. The file write-back no longer writes who answered +# (the #2915 harness: explicit returns, never MagicMock defaults) +# =========================================================================== + +def _recent(): + return (datetime.now(timezone.utc) - timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _row(rid="req-1", **over): + r = { + "id": f"uuid-{rid}", "request_id": rid, "agent_name": "a", "type": "approval", + "status": "responded", "priority": "high", "title": "Approve payout", + "question": "Release 500 USDC?", "options": ["approve", "reject"], "context": {}, + "execution_id": None, "created_at": _recent(), "expires_at": None, + "response": "approve", "response_text": "fine", + "responded_by_id": "7", "responded_by_email": OP_EMAIL, + "responded_at": "2026-09-02T10:00:00Z", "acknowledged_at": None, "cleared_at": None, + "addressed_to_email": None, "sync_state": "confirmed", "sync_detail": None, + "sync_updated_at": None, "last_confirmed_at": None, "delivery_state": None, + "delivery_detail": None, "delivery_updated_at": None, + } + r.update(over) + return r + + +def _entry(rid="req-1"): + return {"id": rid, "type": "approval", "status": "pending", "priority": "high", + "title": "Approve payout", "question": "Release 500 USDC?", + "options": ["approve", "reject"], "created_at": _recent()} + + +def _fake_db(responded): + db = MagicMock() + db.count_operator_queue_pending_for_agent.return_value = 0 + db.get_operator_queue_sync_index_for_agent.return_value = { + "open": [dict(r) for r in responded], "terminal": {}, "foreign": []} + db.set_operator_queue_sync_state.return_value = True + db.set_operator_queue_delivery_state.return_value = True + db.mark_operator_queue_unconfirmed.return_value = 1 + db.refresh_operator_queue_last_confirmed.return_value = 0 + db.mark_operator_queue_acknowledged.return_value = None + db.get_operator_queue_responded_for_agent.return_value = [dict(r) for r in responded] + db.get_operator_queue_terminal_for_agent.return_value = [] + db.get_setting_value.return_value = "24" + db.create_operator_queue_item_with_outcome.side_effect = AssertionError("no create expected") + db.mark_operator_queue_expired.return_value = [] + db.mark_operator_queue_undelivered_for_stopped_agents.return_value = [] + return db + + +def _sync(monkeypatch, responded, *entries): + import services.operator_queue_service as oqs + from services.rate_limiter import RateLimitResult + + db = _fake_db(responded) + content = json.dumps({"$schema": "operator-queue-v1", "requests": list(entries)}) + client = MagicMock() + client.read_file = AsyncMock(return_value={"success": True, "content": content}) + client.write_file = AsyncMock(return_value={"success": True}) + monkeypatch.setattr(oqs, "db", db) + monkeypatch.setattr(oqs, "AgentClient", lambda name: client) + monkeypatch.setattr(oqs.rate_limiter, "check", lambda *a, **k: RateLimitResult(True, 10, 0, 60)) + monkeypatch.setattr(oqs, "_audit_sync", AsyncMock()) + asyncio.run(oqs.OperatorQueueSyncService()._sync_agent("a")) + return db, client + + +def _delivery_calls(db): + return [(c.args[0], c.args[1]) for c in db.set_operator_queue_delivery_state.call_args_list] + + +class TestTheFileWriteBack: + def test_an_answer_delivered_into_the_pending_entry_names_no_person(self, monkeypatch): + db, client = _sync(monkeypatch, [_row()], _entry()) + client.write_file.assert_awaited_once() + (written,) = json.loads(client.write_file.call_args.args[1])["requests"] + assert (written["status"], written["response"], written["response_text"], + written["responded_at"]) == ("responded", "approve", "fine", "2026-09-02T10:00:00Z") + assert "responded_by" not in written + assert "example.com" not in client.write_file.call_args.args[1] + + def test_a_reconstructed_entry_names_no_person(self, monkeypatch): + db, client = _sync(monkeypatch, [_row()]) # the file lost the entry + client.write_file.assert_awaited_once() + (written,) = json.loads(client.write_file.call_args.args[1])["requests"] + assert (written["id"], written["status"], written["response"]) == ("req-1", "responded", "approve") + assert "responded_by" not in written + assert "example.com" not in client.write_file.call_args.args[1] + + def test_an_answered_inbox_collision_alert_is_never_written_into_the_agents_file(self, monkeypatch): + """Its text lists client addresses. A platform alarm, like ent#499's: + not reconstructed, not delivered, `not_applicable`.""" + alarm = _row("portal-inbox-collision-alice_example_com", + type="alert", question="written to by 2 client addresses: a@example.com, b@example.com") + db, client = _sync(monkeypatch, [alarm]) + for call in client.write_file.call_args_list: + assert "portal-inbox-collision-" not in call.args[1] + assert ("uuid-portal-inbox-collision-alice_example_com", "not_applicable") in _delivery_calls(db) + + +# =========================================================================== +# 4. The clear-resolved trigger carries no one's email +# =========================================================================== + +class TestTheClearResolvedTrigger: + def test_the_trigger_is_the_count_only(self, real_db, monkeypatch): + """`/ws` hands a fleet-level trigger to every connection, and any key can + mint a ticket. The frontend only refetches on it; the email was dead + weight that leaked.""" + from routers import operator_queue as r + + sent = [] + + class _WS: + async def broadcast(self, message): + sent.append(json.loads(message)) + + class _Audit: + async def log(self, **kw): + return "evt" + + monkeypatch.setattr(r, "_websocket_manager", _WS()) + monkeypatch.setattr(r, "platform_audit_service", _Audit()) + agent = f"agent-715-clear-{uuid.uuid4().hex[:10]}" + uid = real_db.create_operator_queue_item(agent, { + "id": _rid(), "type": "question", "status": "pending", "priority": "low", + "title": "t", "question": "q", "context": {}, "created_at": "2026-09-01T10:00:00Z"}) + real_db.cancel_operator_queue_item(uid, disposed_by_email=OP_EMAIL) + _as() + res = _client().post("/api/operator-queue/clear-resolved", json={"agent_name": agent}) + assert res.status_code == 200 and res.json() == {"cleared": 1} + assert sent == [{"type": "operator_queue_cleared", "data": {"scope": "resolved", "count": 1}}] + + +# =========================================================================== +# 5. The agent contract says what the platform writes +# =========================================================================== + +def test_the_composed_prompt_no_longer_promises_who_answered(): + from services.platform_prompt_service import ExecutionContext, compose_system_prompt + composed = compose_system_prompt( + ExecutionContext(agent_name="a1", mode="task", triggered_by="schedule", + collaborators=[], platform_url=""), + runtime="claude-code", + ) + assert "with `response`, `response_text` and `responded_at`" in composed + assert "responded_by" not in composed diff --git a/tests/unit/test_idempotency.py b/tests/unit/test_idempotency.py index b1e73456c7..43a806184a 100644 --- a/tests/unit/test_idempotency.py +++ b/tests/unit/test_idempotency.py @@ -312,6 +312,15 @@ def effect_service(idem_ops, monkeypatch): fake_database = types.ModuleType("database") fake_database.db = fake_db monkeypatch.setitem(sys.modules, "database", fake_database) + # #2392: the guard asks the stdlib-only pull_pilot leaf whether the agent + # is pull-mode. Register it directly so sibling fixtures that stub the + # `services` package still resolve it. + pp_spec = importlib.util.spec_from_file_location( + "services.pull_pilot", os.path.join(_backend_path, "services", "pull_pilot.py") + ) + pull_pilot = importlib.util.module_from_spec(pp_spec) + pp_spec.loader.exec_module(pull_pilot) + monkeypatch.setitem(sys.modules, "services.pull_pilot", pull_pilot) path = os.path.join(_backend_path, "services", "idempotency_service.py") spec = importlib.util.spec_from_file_location("_idem_service_effect", path)