diff --git a/.env.example b/.env.example index 75d0226f..7edf86c5 100644 --- a/.env.example +++ b/.env.example @@ -435,6 +435,18 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 # Maximum number of concurrent WebSocket connections accepted by the gateway. # Connections beyond this limit are rejected with close code 1013 (try again later). diff --git a/.env.mainnet.example b/.env.mainnet.example index f284d45f..f2692132 100644 --- a/.env.mainnet.example +++ b/.env.mainnet.example @@ -310,6 +310,18 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 # ─── Killswitch ────────────────────────────────────────────────────────────── # Keep this well under 5000 so worst-case propagation stays inside the 5 s requirement. diff --git a/.env.staging.example b/.env.staging.example index 0606d33b..dc1bdb6d 100644 --- a/.env.staging.example +++ b/.env.staging.example @@ -206,3 +206,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/.env.testnet.example b/.env.testnet.example index cd115145..b772bc38 100644 --- a/.env.testnet.example +++ b/.env.testnet.example @@ -265,3 +265,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f6a18d6..65f6b4d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,17 @@ Commit message format is enforced via [commitlint](https://commitlint.js.org/) s ## [Unreleased] ### Added +- Transactional outbox for on-chain writes: `onchain_outbox` table, intent change + outbox + row committed in one Prisma transaction, `OutboxRelayService` (SKIP LOCKED claims, per-intent + ordering, envelope hash persisted before submit, dead-lettering with alert), + `TxConfirmationService`, live `StellarTxService.invokeContract` submit path, and + `POST /api/v1/admin/outbox/:id/requeue` (Closes #396) +- Durable solver slashing saga: `pending_slashes` table (exactly-once per intent), + configurable challenge window, on-chain re-verification with clock-skew tolerance, + solver fill-proof and admin cancellation endpoints, compensation via `rollbackPenalty`, + metrics, alert rules and `docs/runbooks/slash-cancellation.md` (Closes #397) +- Admin slash cancellation and outbox requeue use the shared `AdminGuard` RBAC (`x-admin-key`) + and are recorded in `admin_audit_log` - `scripts/generate-client.ts` — generates a typed TypeScript API client from the live OpenAPI spec using `openapi-typescript` v7; output committed to `src/generated/` (Closes #134) @@ -72,6 +83,10 @@ Commit message format is enforced via [commitlint](https://commitlint.js.org/) s `npm run test:scripts` (see `prisma/migrations/README.md`) ### Fixed +- `SorobanModule` referenced `forwardRef`/`IntentsModule` without importing them; it now + imports `SolversModule` (what `EventIngestionService` actually needs) +- e2e `@stellar/stellar-sdk` mock now re-exports the real SDK and stubs only + `SorobanRpc.Server` (it previously lacked `Networks`, `Keypair`, … so no e2e suite could load) - `IntentsService.create()` idempotency-key handling is now race-safe — concurrent requests carrying the same key synchronously claim an in-flight slot before any `await`, so exactly one intent is created and the losers replay its result diff --git a/docs/architecture/onchain-settlement.md b/docs/architecture/onchain-settlement.md index f26507a0..b1330f01 100644 --- a/docs/architecture/onchain-settlement.md +++ b/docs/architecture/onchain-settlement.md @@ -1,5 +1,10 @@ # Architecture: On-Chain Settlement (Target Design) +> **Update (issues #396 / #397):** the write-side plumbing below now exists — +> see [Transactional outbox](#transactional-outbox) and +> [Slashing saga](#slashing-saga). Contract method names remain provisional +> until the ADR (issue #19) fixes the interface. +> > **Status: target architecture, not yet implemented.** As of this writing, > `IntentsService` and `SolversService` are in-memory `Map`s > (`src/intents/intents.service.ts`, `src/solvers/solvers.service.ts`), and @@ -166,6 +171,84 @@ transactions. above on a narrower surface (sweeper-triggered only, no user-facing HTTP write path). +## Transactional outbox + +*Implemented — issue #396.* Stage 1 above ("HTTP request → Soroban tx") no +longer submits inside the request. Submitting a transaction and writing +Postgres as two separate steps is a dual write: a crash in between leaves the +database saying "accepted" while the transaction never went out, or the +reverse. Instead: + +```mermaid +sequenceDiagram + participant API as IntentsService + participant DB as Postgres + participant Relay as OutboxRelayService + participant Chain as Soroban + + API->>DB: BEGIN; UPDATE intents …; INSERT onchain_outbox (pending); COMMIT + loop every OUTBOX_RELAY_INTERVAL_MS + Relay->>DB: claim due head-of-intent rows (FOR UPDATE SKIP LOCKED) → processing + Relay->>Relay: build + simulate + sign + Relay->>DB: store envelope_hash (fenced on attempts) + Relay->>Chain: sendTransaction + Relay->>DB: status = submitted, tx_hash + Relay->>Chain: getTransaction(tx_hash) (TxConfirmationService) + Relay->>DB: status = confirmed + end +``` + +- **Atomicity.** `IntentsService` runs `create` / `acceptIfOpen` / + `fillIfAccepted` / `cancelIfOpen` through `IIntentsUnitOfWork` + (`src/intents/intents.unit-of-work.ts`). The Prisma adapter wraps the + intent write and the `onchain_outbox` insert in one `$transaction`. A + transition whose guard fails (lost race) enqueues nothing. With + `ONCHAIN_INTENTS_ENABLED=false` the outbox is bypassed entirely. +- **Fail fast.** The payload is encoded to contract arguments at enqueue time, + so malformed input (bad address, non-integer amount) fails the HTTP request + rather than becoming a poison row. +- **Ordering.** `onchain_outbox.id` is a sequence. A row is only claimable when + every earlier row for the same `intent_id` is `confirmed` or `simulated`, + so one intent's operations apply in order while different intents run in + parallel. `SKIP LOCKED` lets several relay instances share the work. +- **Crash idempotency.** The signed envelope hash is persisted *before* + broadcast. A worker that dies mid-submit leaves the row `processing`; after + `OUTBOX_LEASE_SECONDS` it is reclaimed, and the relay first looks the stored + hash up: `SUCCESS` → confirm without resubmitting; `FAILED` → retry; + `NOT_FOUND` → rebuild. `NOT_FOUND` is conclusive only because the lease + (120 s) outlives the transaction's 30 s time bound + (`INVOKE_TX_TIMEOUT_SECONDS`). Every post-claim write is fenced on + `attempts`, so a worker whose lease expired cannot clobber a reclaimed row. +- **Retries and poison rows.** Failures back off exponentially (1 s doubling, + capped at 5 min). After `OUTBOX_MAX_ATTEMPTS` claims a row becomes `dead`, + `vortex_outbox_dead_total` increments (alerted), and it blocks its intent + until requeued via `POST /api/v1/admin/outbox/:id/requeue`. +- **Dry run.** Under `ONCHAIN_DRY_RUN=true` rows end as `simulated` (terminal). + They are not replayed when dry-run is later switched off. +- **Out of scope:** cross-service delivery (Kafka etc.). + +Row lifecycle: `pending → processing → submitted → confirmed`, with +`processing → simulated` (dry run), back to `pending` on retry, and `dead` +past the attempt limit. + +## Slashing saga + +*Implemented — issue #397.* The `accepted → slashed` row of the mapping table +runs as a durable saga (`src/intents/slashing-pipeline.service.ts`, table +`pending_slashes`): + +`detected → challenge_window → submitted → confirmed | cancelled` + +The sweeper only *detects*. The slash is broadcast after a configurable +challenge window, and only after the chain has been re-checked for a fill that +landed by `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` (by ledger close +time, so server clock skew can't cause a wrong slash). A unique constraint on +`intent_id` makes it exactly-once. Cancellation (solver fill-proof, admin, or +giving up after `SLASH_MAX_SUBMIT_ATTEMPTS`) runs the compensation +(`SolversService.rollbackPenalty`, intent leaves `slashed`) exactly once. The +operator procedure is in +[`docs/runbooks/slash-cancellation.md`](../runbooks/slash-cancellation.md). + ## Persistence layer Both `IntentsService` and `SolversService` delegate all storage to an diff --git a/docs/runbooks/alerts/onchain-writes.rules.yml b/docs/runbooks/alerts/onchain-writes.rules.yml new file mode 100644 index 00000000..9c0affa1 --- /dev/null +++ b/docs/runbooks/alerts/onchain-writes.rules.yml @@ -0,0 +1,33 @@ +# Prometheus alerting rules for the on-chain write path. +# Metric names are defined in src/metrics/metrics.service.ts — keep in sync. +groups: + - name: vortex-onchain-writes + rules: + # Issue #396 — a poison outbox row was dead-lettered. Later operations for + # the same intent are blocked until it is requeued. + - alert: VortexOutboxRowDead + expr: increase(vortex_outbox_dead_total[5m]) > 0 + labels: + severity: page + annotations: + summary: "Outbox row moved to dead after exhausting OUTBOX_MAX_ATTEMPTS" + runbook: docs/runbooks/on-call.md#scenario-g--outbox-rows-dead-or-backlogged + + # Issue #396 — the relay is not draining (RPC outage, relay disabled, signer broken). + - alert: VortexOutboxBacklog + expr: sum(vortex_outbox_rows{status=~"pending|processing|submitted"}) > 100 + for: 15m + labels: + severity: warn + annotations: + summary: "More than 100 on-chain writes waiting in the outbox for 15m" + runbook: docs/runbooks/on-call.md#scenario-g--outbox-rows-dead-or-backlogged + + # Issue #397 — the slashing saga gave up on a slash (submit kept failing). + - alert: VortexSlashSubmitFailed + expr: increase(vortex_slash_pipeline_transitions_total{to_state="cancelled",reason="submit_failed"}[15m]) > 0 + labels: + severity: page + annotations: + summary: "A detected solver slash was cancelled after repeated submission failures" + runbook: docs/runbooks/slash-cancellation.md#submit-failed-slashes diff --git a/docs/runbooks/on-call.md b/docs/runbooks/on-call.md index 0ff57aae..2d3e716f 100644 --- a/docs/runbooks/on-call.md +++ b/docs/runbooks/on-call.md @@ -19,8 +19,9 @@ 8. [Scenario E — Synthetic canary failing](#scenario-e--synthetic-canary-failing) 9. [Health probes](#health-probes) 10. [Scenario F — WebSocket backplane and slow consumers](#scenario-f--websocket-backplane-and-slow-consumers) -11. [Key configuration](#key-configuration) -12. [Escalation path](#escalation-path) +11. [Scenario G — Outbox rows dead or backlogged](#scenario-g--outbox-rows-dead-or-backlogged) +12. [Key configuration](#key-configuration) +13. [Escalation path](#escalation-path) --- @@ -426,6 +427,105 @@ Alerts `VortexCanaryConsecutiveFailures`, `VortexCanaryFundsLow`, Canary intents are excluded from public stats and leaderboards via `CANARY_ADDRESSES`; if they show up there, that variable is missing on the API. +## Scenario F — WebSocket backplane and slow consumers + +**Backplane (issue #454).** With `WS_BACKPLANE=redis` every replica publishes +events into a Redis stream (`vortex:ws:events`) with a global sequence number +(`vortex:ws:seq`) and delivers from that stream, so clients on any replica +see the same events, in the same order, with the same `seq`, and replay works +against any replica. Publishing is queued in the background — request +handlers never wait for Redis. + +- **Redis down:** `/health/ready` on WS-role pods goes 503 (`ws_backplane` + down) and `vortex_ws_backplane_connected` drops to 0. Publishes queue + (bounded) and are retried in order; on recovery each replica resumes the + stream from the last event it delivered — no loss, duplicates or + reordering. Watch `vortex_ws_backplane_dropped_total{reason="queue_full"}` + for events dropped during a long outage. +- **Latency:** `vortex_ws_backplane_publish_duration_seconds`. + +**Connection limits and slow consumers (issue #455).** + +- Connections over `WS_MAX_CONNECTIONS` or `WS_MAX_CONNECTIONS_PER_IP` are + closed with 1013 (`vortex_ws_connections_rejected_total{reason}`). Behind a + load balancer set `WS_TRUST_PROXY_HOPS` to the number of proxies, or every + client shares the proxy's IP. +- Clients over the inbound token bucket get `rate_limited` frames and are + closed with 1008 after `WS_RATE_LIMIT_MAX_VIOLATIONS` + (`vortex_ws_rate_limited_total{action}`). Frames over + `WS_MAX_PAYLOAD_BYTES` close the socket with 1009. +- Slow consumers: once a socket's buffer passes `WS_OUTBOUND_BUFFER_BYTES`, + messages queue (at most `WS_OUTBOUND_QUEUE_MAX`); beyond that the oldest are + dropped (`vortex_ws_outbound_dropped_total`) or, with + `WS_SLOW_CONSUMER_POLICY=disconnect`, the client is closed + (`vortex_ws_slow_consumer_disconnects_total`). Clients recover dropped + events with `replay` — the solver SDK does this automatically. +- Solvers can authenticate with a SEP-10 JWT (`?token=`, `Authorization: + Bearer`, or `{ "type": "auth", "token" }`) when `AUTH_JWT_SECRET` is set, + in addition to signed `auth` frames. Anonymous connections still receive + the public feed. + +--- + +## Scenario G — Outbox rows dead or backlogged + +On-chain writes (intent create/accept/fill/cancel) are committed to the +`onchain_outbox` table in the same transaction as the intent change, and +`OutboxRelayService` submits them afterwards (issue #396). Alert rules live in +[`alerts/onchain-writes.rules.yml`](alerts/onchain-writes.rules.yml). + +### Symptoms + +- `VortexOutboxRowDead`: `vortex_outbox_dead_total` increased. Logs contain + `[outbox] ALERT row (...) moved to dead after N attempts: `. +- `VortexOutboxBacklog`: `vortex_outbox_rows{status="pending"}` keeps growing. + +### Impact + +A dead row **blocks every later row for the same intent** (per-intent +ordering), so that intent's on-chain state stops advancing. Other intents +are unaffected. The API keeps serving from the database. + +### Diagnosis + +```sql +SELECT id, intent_id, operation, attempts, last_error, updated_at +FROM onchain_outbox WHERE status = 'dead' ORDER BY id; + +SELECT status, count(*) FROM onchain_outbox GROUP BY status; +``` + +| `last_error` | Likely cause | +|---|---| +| `SETTLEMENT_CONTRACT_ID is not configured` | Env misconfiguration | +| `signer is not configured` | `SOROBAN_SIGNING_KEY` missing | +| `sendTransaction returned ERROR` / `simulation error` | Contract rejected the call — inspect the payload | +| `RPC ...` / timeouts | Scenario A (RPC downtime) | + +Backlog with no dead rows usually means the relay is off +(`OUTBOX_RELAY_ENABLED=false` — look for `[outbox] relay disabled` at boot) +or Scenario A. + +### Remediation + +Fix the root cause first, then requeue (resets attempts; the intent unblocks): + +```bash +curl -s -X POST -H "x-admin-key: $ADMIN_KEY" \ + "$API/api/v1/admin/outbox//requeue" +``` + +Never delete outbox rows or edit `status` by hand while the relay is running; +the relay's writes are fenced on `attempts` and a manual edit can be +overwritten. A row whose operation must be abandoned (e.g. the contract will +never accept it) needs a code/ADR decision — escalate. + +### Crash safety (why duplicates don't happen) + +The relay stores the signed envelope hash before broadcasting. After a crash +the row is reclaimed once `OUTBOX_LEASE_SECONDS` (default 120 s, longer than +the 30 s transaction time bound) has passed; the relay looks that hash up and +confirms the row if it landed, instead of resubmitting. --- @@ -475,46 +575,6 @@ readinessProbe: --- -## Scenario F — WebSocket backplane and slow consumers - -**Backplane (issue #454).** With `WS_BACKPLANE=redis` every replica publishes -events into a Redis stream (`vortex:ws:events`) with a global sequence number -(`vortex:ws:seq`) and delivers from that stream, so clients on any replica -see the same events, in the same order, with the same `seq`, and replay works -against any replica. Publishing is queued in the background — request -handlers never wait for Redis. - -- **Redis down:** `/health/ready` on WS-role pods goes 503 (`ws_backplane` - down) and `vortex_ws_backplane_connected` drops to 0. Publishes queue - (bounded) and are retried in order; on recovery each replica resumes the - stream from the last event it delivered — no loss, duplicates or - reordering. Watch `vortex_ws_backplane_dropped_total{reason="queue_full"}` - for events dropped during a long outage. -- **Latency:** `vortex_ws_backplane_publish_duration_seconds`. - -**Connection limits and slow consumers (issue #455).** - -- Connections over `WS_MAX_CONNECTIONS` or `WS_MAX_CONNECTIONS_PER_IP` are - closed with 1013 (`vortex_ws_connections_rejected_total{reason}`). Behind a - load balancer set `WS_TRUST_PROXY_HOPS` to the number of proxies, or every - client shares the proxy's IP. -- Clients over the inbound token bucket get `rate_limited` frames and are - closed with 1008 after `WS_RATE_LIMIT_MAX_VIOLATIONS` - (`vortex_ws_rate_limited_total{action}`). Frames over - `WS_MAX_PAYLOAD_BYTES` close the socket with 1009. -- Slow consumers: once a socket's buffer passes `WS_OUTBOUND_BUFFER_BYTES`, - messages queue (at most `WS_OUTBOUND_QUEUE_MAX`); beyond that the oldest are - dropped (`vortex_ws_outbound_dropped_total`) or, with - `WS_SLOW_CONSUMER_POLICY=disconnect`, the client is closed - (`vortex_ws_slow_consumer_disconnects_total`). Clients recover dropped - events with `replay` — the solver SDK does this automatically. -- Solvers can authenticate with a SEP-10 JWT (`?token=`, `Authorization: - Bearer`, or `{ "type": "auth", "token" }`) when `AUTH_JWT_SECRET` is set, - in addition to signed `auth` frames. Anonymous connections still receive - the public feed. - ---- - ## Key configuration | Variable | Default | Effect | @@ -532,6 +592,10 @@ handlers never wait for Redis. | `ADMIN_API_KEYS` | empty (admin APIs disabled) | `id:role:secret` entries for admin / superadmin endpoints | | `PROCESS_ROLE` / `JOBS_DRIVER` | `all` / `memory` | Where job workers run; `bullmq` for multi-instance | | `CANARY_ADDRESSES` | empty | Canary accounts excluded from public stats | +| `OUTBOX_RELAY_ENABLED` | `true` | Kill switch for the outbox relay; rows accumulate while off | +| `OUTBOX_MAX_ATTEMPTS` | `8` | Claims before an outbox row is dead-lettered | +| `OUTBOX_LEASE_SECONDS` | `120` | Crash-reclaim delay; must exceed the 30 s tx time bound | +| `SLASH_CHALLENGE_WINDOW_SECONDS` | `600` | Delay before a detected slash may be broadcast — see [slash-cancellation.md](slash-cancellation.md) | --- diff --git a/docs/runbooks/slash-cancellation.md b/docs/runbooks/slash-cancellation.md new file mode 100644 index 00000000..19fb908f --- /dev/null +++ b/docs/runbooks/slash-cancellation.md @@ -0,0 +1,112 @@ +# Runbook — Solver Slash Review and Manual Cancellation + +Issue #397. Applies to the slashing saga in +`src/intents/slashing-pipeline.service.ts`. + +## How a slash flows + +``` +detected ──▶ challenge_window ──(window over + re-verified)──▶ submitted ──▶ confirmed + │ │ + └──── fill proof / admin / give-up ──▶ cancelled ◀┘ (tx failed past retries) +``` + +1. The sweeper finds an `accepted` intent past its fill deadline, marks it + `slashed`, bumps the solver's `fillsFailed` optimistically, and records a + row in `pending_slashes` (unique per `intent_id`, so exactly-once). +2. The row sits in `challenge_window` for `SLASH_CHALLENGE_WINDOW_SECONDS` + (default 600 s). **Nothing is sent on-chain during the window.** +3. When the window ends the pipeline re-checks the settlement contract's + `intent_filled` events. A fill whose **ledger close time** is at or before + `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` cancels the slash. If + the check itself fails (RPC down) the slash is retried later, never + submitted blind. +4. Otherwise `SolverRegistryService.slashSolver` is called and the row + becomes `submitted`, then `confirmed` once the transaction lands. + +Every cancellation runs the compensation exactly once: `fillsFailed` is +reverted (`SolversService.rollbackPenalty`) and the intent leaves `slashed` +(`filled` if a fill was proven, otherwise `expired`). An +`intent_slash_cancelled` WebSocket event and an audit-log entry are emitted. + +## Inspecting slashes + +```bash +# One intent (public) +curl -s $API/api/v1/slashes/ | jq + +# Everything waiting in the window (admin) +curl -s -H "x-admin-key: $ADMIN_KEY" \ + "$API/api/v1/admin/slashes?state=challenge_window" | jq +``` + +Admin routes use the shared admin RBAC (`AdminGuard`): send your key in the +`x-admin-key` header. Keys come from `ADMIN_API_KEYS` (`id:role:secret`); when +it is empty every admin route returns 401. The key's `id` is recorded as the +actor in the audit log. + +## Cancelling a slash manually + +Use when the slash is wrong — e.g. the fill landed on-chain but the event was +late, the solver's fill was blocked by a protocol incident, or the intent's +deadline was misconfigured. + +```bash +curl -s -X POST -H "x-admin-key: $ADMIN_KEY" \ + -H "Content-Type: application/json" \ + -d '{"note":""}' \ + "$API/api/v1/admin/slashes//cancel" | jq +``` + +Responses: + +| Status | Meaning | Action | +|---|---|---| +| 200 | Cancelled and compensated | Verify below | +| 404 | No slash for that intent | Check the intent id | +| 409 `submission in progress` | A worker holds the lease and may be broadcasting right now | Retry in ~2 minutes | +| 409 `state=submitted` / `confirmed` | Already broadcast — the backend can no longer stop it | Escalate: reversal must happen on-chain / via governance (out of scope here) | + +Verify: + +```bash +curl -s $API/api/v1/slashes/ | jq '.state, .cancelReason, .cancelledBy' +curl -s $API/api/v1/intents//audit | jq '.[-1]' +curl -s $API/api/v1/solvers/ | jq '.fillsFailed' +``` + +**Buying time:** if you need longer than the window to investigate a batch of +slashes, raise `SLASH_CHALLENGE_WINDOW_SECONDS` and restart. The new window +only applies to newly detected slashes; cancel existing ones individually. + +## Solver-initiated cancellation (fill proof) + +A solver whose fill landed in time can cancel during the window without +operator involvement: + +``` +POST /api/v1/slashes//fill-proof +{ "solver": "G...", "txHash": "<64 hex>", "signature": "" } +``` + +`signature` is the solver's Ed25519 signature of +`fill-proof:::`. The tx must be +successful, have closed by `fillDeadline + tolerance`, and emit +`intent_filled` for the intent from the settlement contract. + +## Submit-failed slashes + +Alert `VortexSlashSubmitFailed` fires when a slash was cancelled with reason +`submit_failed` after `SLASH_MAX_SUBMIT_ATTEMPTS` failures (RPC outage, +simulation errors). The solver was **not** penalised and the compensation has +already run. Check logs for `[slashing] ALERT giving up`, fix the root cause +(RPC, signing key, registry contract id), and decide with the service owner +whether the miss warrants a governance-level penalty. + +## Known limitations + +- While `ONCHAIN_DRY_RUN=true`, or until the registry submit path is un-gated + (issue #23), slashes stop at `submitted` with `simulated=true` and are never + confirmed. That is expected. +- `solver deregistered mid-window` does **not** cancel the slash — + deregistration must not be an escape hatch. diff --git a/prisma/migrations/20260928000001_onchain_outbox/down.sql b/prisma/migrations/20260928000001_onchain_outbox/down.sql new file mode 100644 index 00000000..9c812856 --- /dev/null +++ b/prisma/migrations/20260928000001_onchain_outbox/down.sql @@ -0,0 +1,5 @@ +-- Rollback for 20260928000001_onchain_outbox. +-- WARNING: drops any unsent outbox rows. Drain the relay (no pending/processing/ +-- submitted rows) before rolling back, or those on-chain writes are lost. +DROP TABLE IF EXISTS "onchain_outbox"; +DROP TYPE IF EXISTS "OutboxStatus"; diff --git a/prisma/migrations/20260928000001_onchain_outbox/migration.sql b/prisma/migrations/20260928000001_onchain_outbox/migration.sql new file mode 100644 index 00000000..e06cd0a2 --- /dev/null +++ b/prisma/migrations/20260928000001_onchain_outbox/migration.sql @@ -0,0 +1,26 @@ +-- Migration: transactional outbox for on-chain writes (issue #396). +-- No FK to intents: outbox rows must outlive in-memory/retention-evicted intents +-- and the migration must apply standalone in the rollback CI job. + +CREATE TYPE "OutboxStatus" AS ENUM ('pending', 'processing', 'submitted', 'confirmed', 'simulated', 'dead'); + +CREATE TABLE "onchain_outbox" ( + "id" BIGSERIAL PRIMARY KEY, + "intent_id" TEXT NOT NULL, + "operation" TEXT NOT NULL, + "payload" JSONB NOT NULL, + "status" "OutboxStatus" NOT NULL DEFAULT 'pending', + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "locked_until" TIMESTAMPTZ, + "envelope_hash" TEXT, + "tx_hash" TEXT, + "last_error" TEXT, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Relay claim scan: due rows by status. +CREATE INDEX "onchain_outbox_claim_idx" ON "onchain_outbox" ("status", "next_attempt_at"); +-- Per-intent ordering check ("is there an earlier unfinished row for this intent?"). +CREATE INDEX "onchain_outbox_intent_order_idx" ON "onchain_outbox" ("intent_id", "id"); diff --git a/prisma/migrations/20260928000002_pending_slashes/down.sql b/prisma/migrations/20260928000002_pending_slashes/down.sql new file mode 100644 index 00000000..78d2abd4 --- /dev/null +++ b/prisma/migrations/20260928000002_pending_slashes/down.sql @@ -0,0 +1,5 @@ +-- Rollback for 20260928000002_pending_slashes. +-- WARNING: drops saga state. Any slash still in challenge_window/submitted is +-- forgotten; reconcile those manually (docs/runbooks/slash-cancellation.md). +DROP TABLE IF EXISTS "pending_slashes"; +DROP TYPE IF EXISTS "PendingSlashState"; diff --git a/prisma/migrations/20260928000002_pending_slashes/migration.sql b/prisma/migrations/20260928000002_pending_slashes/migration.sql new file mode 100644 index 00000000..a8a236c7 --- /dev/null +++ b/prisma/migrations/20260928000002_pending_slashes/migration.sql @@ -0,0 +1,33 @@ +-- Migration: durable slashing saga state (issue #397). + +CREATE TYPE "PendingSlashState" AS ENUM ('detected', 'challenge_window', 'submitted', 'confirmed', 'cancelled'); + +CREATE TABLE "pending_slashes" ( + "id" TEXT PRIMARY KEY, + "intent_id" TEXT NOT NULL, + "solver_address" TEXT NOT NULL, + "reason" TEXT NOT NULL, + "state" "PendingSlashState" NOT NULL DEFAULT 'detected', + "fill_deadline" INTEGER NOT NULL, + "detected_at" TIMESTAMPTZ NOT NULL, + "challenge_ends_at" TIMESTAMPTZ NOT NULL, + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "locked_until" TIMESTAMPTZ, + "tx_hash" TEXT, + "simulated" BOOLEAN NOT NULL DEFAULT FALSE, + "submitted_at" TIMESTAMPTZ, + "confirmed_at" TIMESTAMPTZ, + "cancelled_at" TIMESTAMPTZ, + "cancel_reason" TEXT, + "cancelled_by" TEXT, + "fill_tx_hash" TEXT, + "last_error" TEXT, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Exactly-once slash per intent. +CREATE UNIQUE INDEX "pending_slashes_intent_id_key" ON "pending_slashes" ("intent_id"); +CREATE INDEX "pending_slashes_due_idx" ON "pending_slashes" ("state", "challenge_ends_at"); +CREATE INDEX "pending_slashes_solver_idx" ON "pending_slashes" ("solver_address"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e69de29b..c7b2e8c1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -0,0 +1,546 @@ +// ─── Prisma Schema ──────────────────────────────────────────────────────────── +// Database: PostgreSQL (swap to sqlite for local dev / CI without a real DB) +// Run `npm run db:generate` after editing this file. +// ───────────────────────────────────────────────────────────────────────────── + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +// ─── Enums ─────────────────────────────────────────────────────────────────── + +enum IntentState { + open + accepted + filled + cancelled + expired + slashed +} + +enum SupportedChain { + stellar + ethereum + base + polygon + arbitrum + optimism + avalanche +} + +// ─── Kill-switch scopes (issue #477) ────────────────────────────────────────── +// A switch is addressed by exactly one of four mutually-exclusive scopes. +// `global` has no chain/token; `chain` sets chain only; `token` sets chain + +// token; `operation` narrows further to a single operation name. +// Evaluation walks GLOBAL → CHAIN → TOKEN → OPERATION and the most specific +// matching switch wins, so an operation-level resume can never re-open a +// scope that a broader switch is still holding closed. + +enum KillSwitchScope { + global + chain + token + operation +} + +/// Operations a switch can gate. Constrained in the application layer (DTO + +/// normalisation) before it reaches the database, so the enum here is a +/// backstop rather than the only gate. +/// Changing this list requires a migration, since Prisma maps it to a real +/// Postgres enum type. +enum KillSwitchOperation { + create + accept + fill + slash + onchain +} + +// ─── Intent ────────────────────────────────────────────────────────────────── +// Represents a cross-chain swap request submitted by a user. +// The nested token objects (srcToken / dstToken) are stored as JSONB so the +// schema remains flexible while the intent's own scalar fields stay queryable. + +model Intent { + id String @id @default(uuid()) @map("id") + /// Externally-visible intent identifier (UUID). + intentId String @unique @map("intent_id") + /// Stellar / EVM address of the user that created the intent. + user String @map("user") + srcChain SupportedChain @map("src_chain") + /// ERC-20 / native token info on the source chain (stored as JSON). + srcToken Json @map("src_token") + /// Raw amount in the token's base unit (stored as string to preserve bigint precision). + srcAmount String @map("src_amount") + /// Stellar destination token info (stored as JSON). + dstToken Json @map("dst_token") + /// Minimum acceptable destination amount (base unit string). + minDstAmount String @map("min_dst_amount") + /// Best quote from solvers, populated after quoting. + quotedDstAmount String? @map("quoted_dst_amount") + /// Solver address that accepted / filled this intent. + solver String? @map("solver") + state IntentState @default(open) @map("state") + /// Unix epoch seconds. + createdAt Int @map("created_at") + /// Unix epoch seconds – intent expires after this. + deadline Int @map("deadline") + /// Unix epoch seconds – set when state becomes `filled`. + filledAt Int? @map("filled_at") + /// Actual amount received by the user (base unit string). + fillAmount String? @map("fill_amount") + /// Realized protocol fee charged on this fill (destination-token base units). + feeAmount String? @map("fee_amount") + /// On-chain transaction hash of the Stellar fill transaction. + txHash String? @map("tx_hash") + + @@index([user]) + @@index([state]) + @@index([solver]) + @@map("intents") +} + +// ─── Solver ────────────────────────────────────────────────────────────────── +// Registered solver nodes that fulfil cross-chain intents. + +model Solver { + id String @id @default(uuid()) @map("id") + /// Public key / address that uniquely identifies the solver. + address String @unique @map("address") + name String @map("name") + /// Bond amount locked in the registry contract (base-unit string). + bondAmount String @map("bond_amount") + fillsCompleted Int @default(0) @map("fills_completed") + fillsFailed Int @default(0) @map("fills_failed") + /// Cumulative filled volume (base-unit string). + totalVolume String @default("0") @map("total_volume") + /// Rolling average fill time in seconds. + avgFillTime Float @default(0) @map("avg_fill_time") + isActive Boolean @default(true) @map("is_active") + /// Unix epoch seconds. + registeredAt Int @map("registered_at") + /// Unix epoch seconds of the solver's most recent activity (registration, fill, or status change). + lastActiveAt Int @map("last_active_at") + /// Chains this solver supports (stored as JSON array of SupportedChain values). + supportedChains Json @map("supported_chains") + /// Token symbols this solver can handle. + supportedTokens Json @map("supported_tokens") + + // ── On-chain projection fields (issue #399) ─────────────────────────────── + /// Indicates the authoritative data source: "api" (REST registration) or + /// "chain" (projected from solver-registry contract events). + source String @default("api") @map("source") + /// Ledger sequence of the most recent on-chain event that updated this row. + /// NULL when source="api" (no on-chain event has been observed yet). + chainUpdatedLedger Int? @map("chain_updated_ledger") + + @@index([isActive]) + @@map("solvers") +} + +// ─── IntentAuditLog ────────────────────────────────────────────────────────── +// Append-only record of every state transition for an intent (issue #217 / #62). +// Queried by intentId to reconstruct the full history of a swap. + +model IntentAuditLog { + id BigInt @id @default(autoincrement()) @map("id") + /// FK to intents.intent_id (the user-visible UUID, not the surrogate PK). + intentId String @map("intent_id") + /// ISO-8601 / TIMESTAMPTZ of when the transition was recorded. + timestamp DateTime @default(now()) @map("timestamp") @db.Timestamptz + /// State the intent moved INTO (e.g. "cancelled", "expired", "slashed"). + toState String @map("to_state") + /// Actor who triggered the transition: a user address, solver address, or "system". + actor String @map("actor") + /// Human-readable explanation. + reason String @map("reason") + /// Optional extra data (fill amount, tx hash, deadline, …). + metadata Json? @map("metadata") + + @@index([intentId, timestamp(sort: Asc)], name: "audit_log_intent_idx") + @@map("intent_audit_log") +} + +// ─── Token ─────────────────────────────────────────────────────────────────── +// Static registry of tokens the protocol supports. +// Kept separate so it can be updated without migrations when the token list changes. + +model Token { + id String @id @default(uuid()) @map("id") + /// Contract address (EVM hex address or Stellar contract ID). + address String @map("address") + symbol String @map("symbol") + name String @map("name") + decimals Int @map("decimals") + chain SupportedChain @map("chain") + logoUri String? @map("logo_uri") + /// Latest known USD price (nullable – updated by a price-feed worker). + priceUsd Float? @map("price_usd") + /// Whether this is a destination-side Stellar token. + isStellar Boolean @default(false) @map("is_stellar") + + @@unique([address, chain]) + @@index([chain]) + @@index([symbol]) + @@map("tokens") +} + +// ─── KillSwitch (issue #477) ───────────────────────────────────────────────── +// Hierarchical emergency pause. One row per (scope, chain, token, operation) +// tuple; activating the row closes that scope for the gated operation. +// +// Because the unique key is the full scope tuple, "resume" is an idempotent +// state flip on a single row rather than a delete/insert race between replicas. +// +// `scope` is stored alongside the nullable chain/token/operation columns so the +// row is self-describing; the application maintains that invariant in one place +// (KillSwitchService.normaliseScope) rather than trusting callers. + +model KillSwitch { + id String @id @default(uuid()) @map("id") + scope KillSwitchScope @map("scope") + /// NULL for global scope. + chain String? @map("chain") + /// NULL for global and chain scopes. + token String? @map("token") + /// NULL unless scope = operation. + operation KillSwitchOperation? @map("operation") + + /// Canonical encoding of (scope, chain, token, operation), e.g. + /// "operation|stellar|USDC|fill". Postgres will not let Prisma build a + /// findUnique over nullable columns, so this non-null surrogate carries the + /// uniqueness guarantee while the typed columns above stay queryable. + scopeKey String @unique @map("scope_key") + + /// true = writes are blocked for this scope, false = explicitly resumed. + active Boolean @map("active") + /// Machine-readable reason surfaced to clients as `reason` in the 503 body. + reasonCode String @map("reason_code") + /// Free-text operator explanation (incident ticket, etc). + reason String @map("reason") + /// Operator identity that activated or resumed the switch. + activatedBy String @map("activated_by") + + /// Unix epoch ms of the last state change. Used as the cheap change-detection + /// key for the polling fallback (monotonic: re-pausing bumps it). + updatedAt Int @map("updated_at") + + createdAt Int @map("created_at") + /// Unix epoch ms of the last time this switch transitioned active -> inactive. + /// Unset while never resumed, so a brand-new row cannot inherit a stale + /// cooldown from a previous pause of the same scope. + lastResumedAt Int? @map("last_resumed_at") + + // Resume requires two distinct approvals; see KillSwitchApproval. + approvals KillSwitchApproval[] + /// Number of distinct approvals still required before the switch may resume. + approvalsRequired Int @default(2) @map("approvals_required") + + @@index([active]) + @@index([scope, chain, token, operation]) + /// Backs the polling fallback's "max(updated_at) since last snapshot?" probe. + @@index([updatedAt]) + @@map("kill_switches") +} +// ─── KillSwitchApproval (issue #477) ────────────────────────────────────────── +// One row per operator approval to resume a switch. The resume guard is a +// count of DISTINCT approver over this table, so a single operator cannot +// approve twice, and two different operators can never be one person unless the +// caller lies about identity (the API ties identity to the operator token). + +model KillSwitchApproval { + id String @id @default(uuid()) @map("id") + killSwitchId String @map("kill_switch_id") + killSwitch KillSwitch @relation(fields: [killSwitchId], references: [id], onDelete: Cascade) + /// Operator identity that granted this approval. Unique per switch so one + /// operator cannot satisfy the two-approval rule alone. + approver String @map("approver") + /// Unix epoch ms. + approvedAt Int @map("approved_at") + /// Optional note explaining the approval. + note String? @map("note") + + @@unique([killSwitchId, approver]) + @@index([killSwitchId]) + @@map("kill_switch_approvals") +} + +// ─── TreasurySnapshot ──────────────────────────────────────────────────────── +// Daily snapshots of expected vs actual treasury balances per asset. +// Used for reconciliation and historical reporting. + +model TreasurySnapshot { + id BigInt @id @default(autoincrement()) @map("id") + /// Date of the snapshot (YYYY-MM-DD). + snapshotDate String @map("snapshot_date") + /// Asset identifier (contract address or asset code). + asset String @map("asset") + /// Expected balance from fee ledger + slashes - refunds (base-unit string). + expectedBalance String @map("expected_balance") + /// Actual on-chain balance (base-unit string). + actualBalance String @map("actual_balance") + /// Difference (actualBalance - expectedBalance, base-unit string). + discrepancy String @map("discrepancy") + /// Tolerance threshold used for this check (base-unit string). + toleranceThreshold String @map("tolerance_threshold") + /// Whether abs(discrepancy) > toleranceThreshold. + hasUnexplainedDiscrepancy Boolean @map("has_unexplained_discrepancy") + /// Human-readable explanation of differences (in-flight settlements, refunds, etc). + explanation String? @map("explanation") + /// ISO-8601 / TIMESTAMPTZ of when the snapshot was taken. + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + /// Breakdown of expected balance sources (fees, slashes, refunds). + breakdown Json? @map("breakdown") + + @@unique([snapshotDate, asset], name: "snapshot_date_asset_unique") + @@index([snapshotDate]) + @@index([hasUnexplainedDiscrepancy]) + @@map("treasury_snapshots") +} + +// ─── FeeLedger ─────────────────────────────────────────────────────────────── +// Append-only ledger of all fee accruals from filled intents. + +model FeeLedger { + id BigInt @id @default(autoincrement()) @map("id") + /// FK to intents.intent_id. + intentId String @map("intent_id") + /// Asset identifier (contract address or asset code). + asset String @map("asset") + /// Fee amount in base units (string). + amount String @map("amount") + /// ISO-8601 / TIMESTAMPTZ of when the fee was accrued. + accrualAt DateTime @default(now()) @map("accrual_at") @db.Timestamptz + /// Optional transaction hash. + txHash String? @map("tx_hash") + + @@index([intentId]) + @@index([asset, accrualAt]) + @@map("fee_ledger") +} + +// ─── SlashLedger ───────────────────────────────────────────────────────────── +// Append-only ledger of all slash proceeds from solver penalties. + +model SlashLedger { + id BigInt @id @default(autoincrement()) @map("id") + /// Solver address that was slashed. + solverAddress String @map("solver_address") + /// Asset identifier (contract address or asset code). + asset String @map("asset") + /// Slashed amount in base units (string). + amount String @map("amount") + /// ISO-8601 / TIMESTAMPTZ of when the slash occurred. + slashedAt DateTime @default(now()) @map("slashed_at") @db.Timestamptz + /// Reason for the slash. + reason String @map("reason") + /// Optional transaction hash. + txHash String? @map("tx_hash") + + @@index([solverAddress]) + @@index([asset, slashedAt]) + @@map("slash_ledger") +} + +// ─── RefundLedger ──────────────────────────────────────────────────────────── +// Append-only ledger of all refunds issued to users. + +model RefundLedger { + id BigInt @id @default(autoincrement()) @map("id") + /// FK to intents.intent_id. + intentId String @map("intent_id") + /// User address that received the refund. + userAddress String @map("user_address") + /// Asset identifier (contract address or asset code). + asset String @map("asset") + /// Refund amount in base units (string). + amount String @map("amount") + /// ISO-8601 / TIMESTAMPTZ of when the refund was issued. + issuedAt DateTime @default(now()) @map("issued_at") @db.Timestamptz + /// Reason for the refund. + reason String @map("reason") + /// Optional transaction hash. + txHash String? @map("tx_hash") + + @@index([intentId]) + @@index([userAddress]) + @@index([asset, issuedAt]) + @@map("refund_ledger") +} + +// ─── AdminAuditLog ─────────────────────────────────────────────────────────── +// Append-only record of privileged operator and governance actions: feature +// flag changes (issue #495), kill-switch toggles and guardian overrides (#507). + +model AdminAuditLog { + id BigInt @id @default(autoincrement()) @map("id") + /// Admin principal id, or "guardian" / "system" for automated actions. + actor String @map("actor") + /// Dotted action name, e.g. "flag.update", "guardian.override". + action String @map("action") + /// Target of the action, e.g. "flag:onchain-dry-run". + target String @map("target") + before Json? @map("before") + after Json? @map("after") + reason String? @map("reason") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + + @@index([target, createdAt(sort: Desc)]) + @@map("admin_audit_log") +} + +// ─── FeatureFlag ───────────────────────────────────────────────────────────── +// Runtime feature flags (issue #495). `rules` is an ordered JSON array of +// { value, percentage?, solvers?, chains? }; the first matching rule wins. + +model FeatureFlag { + key String @id @map("key") + defaultValue Boolean @map("default_value") + rules Json @map("rules") + version Int @default(1) @map("version") + updatedBy String @map("updated_by") + updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz + + @@map("feature_flags") +} + +// Pending flag changes that need a second approver (e.g. dry-run off in production). +model FlagChangeRequest { + id String @id @default(uuid()) @map("id") + flagKey String @map("flag_key") + /// Proposed { defaultValue, rules }. + proposed Json @map("proposed") + proposedBy String @map("proposed_by") + /// Admin ids that approved, proposer first. + approvals String[] @map("approvals") + /// pending | applied + status String @default("pending") @map("status") + reason String? @map("reason") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + appliedAt DateTime? @map("applied_at") @db.Timestamptz + + @@index([flagKey, status]) + @@map("flag_change_requests") +} + +// ─── GuardianAction ────────────────────────────────────────────────────────── +// Emergency actions ingested from the on-chain guardian contract (issue #507). +// One row per activating event; cleared by the matching guardian event or a +// superadmin override. + +model GuardianAction { + /// Soroban event id of the activating event. + id String @id @map("id") + /// pause | freeze | blacklist + kind String @map("kind") + /// Frozen parameter key or blacklisted solver address; "" for pause. + target String @map("target") + active Boolean @map("active") + txHash String @map("tx_hash") + ledger Int @map("ledger") + activatedAt DateTime @map("activated_at") @db.Timestamptz + clearedAt DateTime? @map("cleared_at") @db.Timestamptz + clearedTxHash String? @map("cleared_tx_hash") + overriddenBy String? @map("overridden_by") + + @@index([active]) + @@map("guardian_actions") +} + +// ─── OnchainOutbox ─────────────────────────────────────────────────────────── +// Transactional outbox for on-chain writes (issue #396). A row is inserted in +// the same database transaction as the intent mutation it mirrors, and +// OutboxRelayService submits it to Soroban afterwards — so the DB can never say +// "accepted" while the corresponding transaction silently never went out. +// +// `id` is a monotonically increasing sequence and doubles as the per-intent +// ordering key: a row is only claimable once every earlier row for the same +// intent has reached a terminal success state. + +enum OutboxStatus { + pending + processing + submitted + confirmed + simulated + dead +} + +model OnchainOutbox { + id BigInt @id @default(autoincrement()) @map("id") + /// intents.intent_id this operation belongs to (ordering partition key). + intentId String @map("intent_id") + /// Contract operation, e.g. "create_intent", "accept_intent". + operation String @map("operation") + /// Operation arguments (JSON-safe, bigint amounts as strings). + payload Json @map("payload") + status OutboxStatus @default(pending) @map("status") + /// Number of times this row has been claimed by a relay worker. + attempts Int @default(0) @map("attempts") + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") @db.Timestamptz + /// Lease expiry while status = processing; a crashed worker's row is reclaimed after this. + lockedUntil DateTime? @map("locked_until") @db.Timestamptz + /// Hash of the signed envelope, persisted BEFORE submission (crash idempotency). + envelopeHash String? @map("envelope_hash") + /// Hash of the transaction that was submitted. + txHash String? @map("tx_hash") + lastError String? @map("last_error") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz + + @@index([status, nextAttemptAt], name: "onchain_outbox_claim_idx") + @@index([intentId, id], name: "onchain_outbox_intent_order_idx") + @@map("onchain_outbox") +} + +// ─── PendingSlash ──────────────────────────────────────────────────────────── +// Durable saga state for a solver slash (issue #397): +// detected → challenge_window → submitted → confirmed | cancelled +// The unique constraint on intent_id enforces exactly-once slashing per intent. + +enum PendingSlashState { + detected + challenge_window + submitted + confirmed + cancelled +} + +model PendingSlash { + id String @id @default(uuid()) @map("id") + intentId String @unique @map("intent_id") + solverAddress String @map("solver_address") + reason String @map("reason") + state PendingSlashState @default(detected) @map("state") + /// The intent's fill deadline (unix seconds) that the solver missed. + fillDeadline Int @map("fill_deadline") + detectedAt DateTime @map("detected_at") @db.Timestamptz + /// Slash may not be submitted before this instant. + challengeEndsAt DateTime @map("challenge_ends_at") @db.Timestamptz + attempts Int @default(0) @map("attempts") + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") @db.Timestamptz + /// Lease held by the pipeline worker that is currently verifying/submitting. + lockedUntil DateTime? @map("locked_until") @db.Timestamptz + txHash String? @map("tx_hash") + /// true when the registry client simulated but did not broadcast (dry-run / gated submit). + simulated Boolean @default(false) @map("simulated") + submittedAt DateTime? @map("submitted_at") @db.Timestamptz + confirmedAt DateTime? @map("confirmed_at") @db.Timestamptz + cancelledAt DateTime? @map("cancelled_at") @db.Timestamptz + cancelReason String? @map("cancel_reason") + cancelledBy String? @map("cancelled_by") + /// Fill transaction that cancelled the slash, when cancelled by a fill proof. + fillTxHash String? @map("fill_tx_hash") + lastError String? @map("last_error") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz + + @@index([state, challengeEndsAt], name: "pending_slashes_due_idx") + @@index([solverAddress], name: "pending_slashes_solver_idx") + @@map("pending_slashes") +} diff --git a/src/common/stellar-signature.ts b/src/common/stellar-signature.ts index f51e7a80..20fc23be 100644 --- a/src/common/stellar-signature.ts +++ b/src/common/stellar-signature.ts @@ -169,3 +169,11 @@ export function buildDisputeDecisionMessage(disputeId: string, resolution: strin export function buildUpdateSolverMessage(address: string): string { return `update-solver:${address}`; } + +/** + * Canonical message a solver signs to prove a fill landed in time and cancel + * a pending slash during its challenge window (issue #397). + */ +export function buildFillProofMessage(intentId: string, solver: string, txHash: string): string { + return `fill-proof:${intentId}:${solver}:${txHash}`; +} diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index bab0459b..e69de29b 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -1,489 +0,0 @@ -import * as Joi from "joi"; - -// Stellar secret seeds ("S..." strkeys) are 56-char base32: prefix + 32-byte -// payload + checksum. This rejects placeholders like "changeme" outright — -// it does not by itself prove the key is a *real, funded* signer. -const STELLAR_SECRET_KEY_PATTERN = /^S[A-Z2-7]{55}$/; - -// One message for both "absent" and "empty". Joi's .required() alone accepts an -// empty string, which for the kill-switch would be a silently disabled control -// plane — the exact condition this rule exists to prevent, so both cases must -// produce the same actionable error. -const KILLSWITCH_TOKEN_REQUIRED_MESSAGE = - "KILLSWITCH_OPERATOR_TOKEN must be a non-empty secret in production so the " + - "emergency pause control plane (/api/v1/ops/killswitch) is usable. Generate " + - "one with `openssl rand -hex 32`. See docs/runbooks/killswitch.md."; - -export const envValidationSchema = Joi.object({ - NODE_ENV: Joi.string().valid("development", "production", "test").default("development"), - PORT: Joi.number().port().default(4000), - - // Prisma requires DATABASE_URL in production; optional (with a default) in - // development/test so the app can boot without a live database for unit tests. - DATABASE_URL: Joi.string() - .uri({ scheme: ["postgresql", "postgres"] }) - .default("postgresql://vortex:vortex@localhost:5432/vortex?schema=public"), - - STELLAR_NETWORK: Joi.string().valid("testnet", "futurenet", "mainnet").default("testnet"), - SOROBAN_RPC_URL: Joi.string().uri().default("https://soroban-testnet.stellar.org"), - // Horizon base URL, used for account/balance reads (treasury, canary tooling). - HORIZON_URL: Joi.string().uri().default("https://horizon-testnet.stellar.org"), - SETTLEMENT_CONTRACT_ID: Joi.string().allow("").default(""), - SOLVER_REGISTRY_CONTRACT_ID: Joi.string().allow("").default(""), - STELLAR_SIGNER_SECRET_KEY: Joi.string().allow("").default(""), - - // Secret key for the backend's own Soroban signer (submits on-chain writes - // such as settlement and slashing calls). No default is provided anywhere - // in this schema — an unset value fails closed (empty string) rather than - // ever falling back to a placeholder that could be mistaken for a real key. - SOROBAN_SIGNING_KEY: Joi.string() - .pattern(STELLAR_SECRET_KEY_PATTERN) - .messages({ - "string.pattern.base": - 'SOROBAN_SIGNING_KEY must be a valid Stellar secret seed (starts with "S", 56 base32 characters). ' + - "Generate a throwaway testnet key for local dev — see README's Signing Key section — never commit a real one.", - }) - .when("NODE_ENV", { - is: "production", - then: Joi.required(), - otherwise: Joi.string().allow("").default(""), - }), - - ONCHAIN_INTENTS_ENABLED: Joi.boolean().default(false), - // Stellar public key of the treasury account (fee/slash/refund accumulator). - TREASURY_ADDRESS: Joi.string().allow("").default(""), - - // Stellar public key of the treasury account (fee accumulator). - TREASURY_ADDRESS: Joi.string().allow("").default(""), - - ALLOW_LEGACY_STELLAR_SIGNATURES: Joi.boolean().default(false), - ETHEREUM_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - ETHEREUM_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - BASE_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - BASE_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - POLYGON_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - POLYGON_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - ARBITRUM_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - ARBITRUM_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - OPTIMISM_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - OPTIMISM_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - AVALANCHE_RPC_URL: Joi.string().uri({ scheme: ["http", "https"] }).allow("").default(""), - AVALANCHE_ESCROW_ADDRESS: Joi.string().pattern(/^$|^0x[a-fA-F0-9]{40}$/).default(""), - EVM_RPC_ALLOWLIST: Joi.string().allow("").default(""), - CORS_ORIGIN: Joi.string().default("*"), - WS_MAX_CONNECTIONS: Joi.number().integer().min(0).default(1000), - SOROBAN_FEE_PERCENTILE: Joi.string() - .valid( - "min", - "mode", - "p10", - "p20", - "p30", - "p40", - "p50", - "p60", - "p70", - "p80", - "p90", - "p95", - "p99", - "max", - ) - .default("p50"), - - WS_BACKPLANE: Joi.string().valid("memory", "redis").default("memory"), - REDIS_URL: Joi.string().uri({ scheme: ["redis", "rediss"] }).default("redis://localhost:6379"), - - // ── Persistence adapter selection ───────────────────────────────────────── - // Controls which repository adapter is used for intents and solvers. - // "memory" (default) keeps everything in-process — no database required. - // "prisma" writes to PostgreSQL via Prisma — requires DATABASE_URL to point - // to a live database. Intended for production / staging. - INTENTS_PERSISTENCE: Joi.string().valid("memory", "prisma").default("memory"), - SOLVERS_PERSISTENCE: Joi.string().valid("memory", "prisma").default("memory"), - - // ── Intent retention (in-memory store hygiene) ───────────────────────────── - // How long terminal intents are kept in the in-memory adapter, and how often - // the eviction sweep runs. Both are read by IntentsService. - INTENT_RETENTION_DAYS: Joi.number().integer().min(0).default(30), - INTENT_RETENTION_SWEEP_MS: Joi.number().integer().min(0).default(60000), - - // ── Reference solver bot (scripts/solver-bot.ts) ─────────────────────────── - // Read by the standalone bot process rather than by the server, but declared - // here so `npm run check:env-drift` sees one consistent variable set across - // env.validation.ts, configuration.ts and the .env*.example files. - SOLVER_SECRET: Joi.string().allow("").default(""), - SOLVER_ADDRESS: Joi.string().allow("").default(""), - SOLVER_CHAINS: Joi.string().allow("").default(""), - - // ── Observability ───────────────────────────────────────────────────────── - // Sentry DSN for error alerting. Omit (or leave blank) to disable Sentry. - SENTRY_DSN: Joi.string().uri().allow("").default(""), - - // Bearer token that guards GET /metrics (issue #298). - // When set, Prometheus scrape jobs must supply: - // Authorization: Bearer - // When empty (the default): - // - non-production: unauthenticated scraping allowed (local dev Prometheus) - // - production: endpoint returns 401 (fail-closed — set the token before deploying) - // Generate with: openssl rand -hex 32 - METRICS_TOKEN: Joi.string().allow("").default(""), - - // Winston log level. Defaults to "debug" in dev/test and "info" in production. - LOG_LEVEL: Joi.string() - .valid("error", "warn", "info", "http", "verbose", "debug", "silly") - .default( - // Joi.ref doesn't evaluate lazily here, so we rely on the logger's own - // resolveLogLevel() for the runtime default — this schema default acts - // as a documentation hint and config validation guard only. - "debug", - ), - - // Log shipping — off by default so local dev/CI remain stdout-only. When - // enabled, structured logs are also shipped to LOG_SHIPPING_HOST:PORT. - LOG_SHIPPING_ENABLED: Joi.boolean().default(false), - LOG_SHIPPING_HOST: Joi.string().when("LOG_SHIPPING_ENABLED", { - is: true, - then: Joi.required(), - otherwise: Joi.string().allow("").default(""), - }), - LOG_SHIPPING_PORT: Joi.number().port().when("LOG_SHIPPING_ENABLED", { - is: true, - then: Joi.required(), - otherwise: Joi.number().optional(), - }), - LOG_SHIPPING_PATH: Joi.string().default("/"), - LOG_SHIPPING_SSL: Joi.boolean().default(false), - LOG_SERVICE_NAME: Joi.string().default("vortex-backend"), - - // ── Pluggable signer backend (issue #400) ──────────────────────────────── - // SIGNER_BACKEND selects which signing implementation is used: - // "local" (default) — LocalKeypairSigner: key loaded from SOROBAN_SIGNING_KEY / file. - // Refused in production unless ALLOW_LOCAL_SIGNER_IN_PROD=true. - // "vault" — VaultTransitSigner: signs via HashiCorp Vault Transit (ed25519). - // Requires VAULT_ADDR + VAULT_TOKEN. Key never enters RAM. - SIGNER_BACKEND: Joi.string().valid("local", "vault").default("local"), - - // Required when SIGNER_BACKEND=vault. - VAULT_ADDR: Joi.string().uri({ scheme: ["http", "https"] }).when("SIGNER_BACKEND", { - is: "vault", - then: Joi.required(), - otherwise: Joi.string().allow("").default(""), - }), - VAULT_TOKEN: Joi.string().when("SIGNER_BACKEND", { - is: "vault", - then: Joi.required(), - otherwise: Joi.string().allow("").default(""), - }), - // Name of the Vault Transit key (default: "vortex-signer"). - VAULT_TRANSIT_KEY_NAME: Joi.string().default("vortex-signer"), - - // Escape hatch: allow LocalKeypairSigner in production. - // Must be explicitly set to "true" — any other value is treated as false. - // A startup warning is emitted when this is enabled in production. - ALLOW_LOCAL_SIGNER_IN_PROD: Joi.boolean().default(false), - - // ── Resource-exhaustion limits (issue #476) ─────────────────────────────── - // These values are consumed by src/config/limits.config.ts at startup and - // override the compile-time defaults when set. All have safe defaults so - // the service can boot without them. - - /** Max JSON nesting depth before the body is rejected (default 10). */ - JSON_MAX_DEPTH: Joi.number().integer().min(1).max(100).default(10), - - /** Max WS chain-filter values per subscribe message (default 20). */ - WS_MAX_FILTER_CHAINS: Joi.number().integer().min(1).max(100).default(20), - - /** Max active subscriptions per WS connection (default 10). */ - WS_MAX_SUBSCRIPTIONS: Joi.number().integer().min(1).max(100).default(10), - - /** Default Postgres statement_timeout in ms for standard route queries (default 5000). */ - DB_QUERY_TIMEOUT_MS: Joi.number().integer().min(100).max(60000).default(5000), - - /** Postgres statement_timeout in ms for batch-lookup queries (default 10000). */ - DB_BATCH_QUERY_TIMEOUT_MS: Joi.number().integer().min(100).max(60000).default(10000), - - /** Postgres statement_timeout in ms for stats/aggregate queries (default 15000). */ - DB_STATS_QUERY_TIMEOUT_MS: Joi.number().integer().min(100).max(60000).default(15000), - - // ── Emergency kill-switch (issue #477) ───────────────────────────────────── - // Shared secret for the operator control plane. Empty (the default) leaves - // /api/v1/ops/killswitch disabled — fail closed, never open. - // - // The kill-switch is the only way to stop writes at runtime, so a production - // deploy without a token ships a protocol that cannot be paused. Requiring it - // in production fails validation rather than silently running with the - // control plane disabled. - KILLSWITCH_OPERATOR_TOKEN: Joi.string() - .when("NODE_ENV", { - is: Joi.valid("production"), - then: Joi.string() - .required() - .invalid("") - .messages({ - "any.required": KILLSWITCH_TOKEN_REQUIRED_MESSAGE, - "string.empty": KILLSWITCH_TOKEN_REQUIRED_MESSAGE, - "any.invalid": KILLSWITCH_TOKEN_REQUIRED_MESSAGE, - }), - otherwise: Joi.string().allow("").default(""), - }), - - /** - * Redis URL for cross-replica pause propagation. Empty means "polling only", - * which still meets the 5 s budget. Defaults to reusing REDIS_URL when - * WS_BACKPLANE=redis, so existing deployments propagate without new config. - */ - KILLSWITCH_REDIS_URL: Joi.string().allow("").optional(), - - /** - * DB change-probe interval (ms) that backstops Redis pub/sub. Capped at 5000 - * so the worst-case propagation delay cannot exceed the requirement, however - * misconfigured. - */ - KILLSWITCH_POLL_MS: Joi.number().integer().min(100).max(5000).default(2000), - - // Same adapter-selection convention as the other repositories. - KILLSWITCH_PERSISTENCE: Joi.string().valid("memory", "prisma").default("memory"), - - // ── On-chain write safety flag (issue #35 / issue #260) ────────────────── - // When true, every on-chain-write code path (invokeContract, slashSolver) - // builds and simulates the transaction, logs what it *would* submit, and - // returns without broadcasting — safe by construction. - // - // Default behaviour: - // - Outside production: defaults to true (simulate-only, fail closed - // toward safety — no real funds moved without an explicit opt-out). - // - In production: *required* to be explicitly set. Omitting it in a - // production deploy fails validation so the operator must consciously - // decide between dry-run and live mode before traffic reaches - // on-chain write paths. This matches the fail-closed pattern used - // for SOROBAN_SIGNING_KEY. - // - // This is the env default for the `onchain-dry-run` runtime feature flag - // (issue #495); the flag can override it without a restart, and turning - // dry-run off in production through the flag requires two approvals. - // Set ONCHAIN_DRY_RUN=false only after completing the dry-run soak - // described in docs/runbooks/onchain-cutover.md. - ONCHAIN_DRY_RUN: Joi.boolean() - .when("NODE_ENV", { - is: "production", - then: Joi.required().messages({ - "any.required": - "ONCHAIN_DRY_RUN must be explicitly set in production. " + - "Set to true to remain in simulate-only mode, or false to enable live on-chain writes. " + - "See docs/runbooks/onchain-cutover.md for the staged rollout procedure.", - }), - otherwise: Joi.boolean().default(true), - }), - - // ── Shadow-mode divergence monitor (issue #401) ─────────────────────────── - // Runs read-only on-chain simulations of every intent state transition in - // parallel with the authoritative off-chain path and reports where the two - // disagree. Never submits a transaction; see src/soroban/shadow.service.ts. - // - // Off by default: a sampled simulation is a real RPC call with a real - // rate-limit footprint, so it is an explicit per-environment opt-in. - SHADOW_MODE_ENABLED: Joi.boolean().default(false), - - // Fraction of transitions to simulate, as a probability in [0, 1]. - // 1 (the default) compares every transition; 0 disables sampling entirely - // while leaving the monitor "enabled" — useful for a canary that only wants - // the queue/metric plumbing live. - SHADOW_SAMPLE_RATE: Joi.number().min(0).max(1).default(1), - - // Hard cap on queued observations. Beyond this, observations are dropped and - // counted (`vortex_shadow_dropped_total`) rather than queued, so a slow or - // unreachable RPC degrades the monitor instead of the service. - SHADOW_QUEUE_MAX: Joi.number().integer().min(1).default(256), - - // How many queued observations the background drain simulates concurrently. - SHADOW_CONCURRENCY: Joi.number().integer().min(1).max(32).default(4), - - // Public key used as the transaction source for shadow simulations. A Stellar - // public key (strkey G...). It is never signed, never submitted and never - // charged a fee — it only has to be a valid address for the envelope. - // Optional: when empty the monitor reports `contract_unconfigured` rather - // than silently recording zero divergence. - SHADOW_SOURCE_ACCOUNT: Joi.string().allow("").default(""), - - // ── Governance parameters contract ──────────────────────────────────────── - // When set, ProtocolParamsService reads current + scheduled protocol - // parameters (fee bps, fill windows, deadlines, exposure ratio, slash - // amount) from this Soroban contract address. Leave blank to use code - // and env defaults. - PARAMS_CONTRACT_ID: Joi.string().allow("").default(""), - - // How often (ms) to poll the parameters contract. 30 s is the default; - // lower values increase RPC load; raise in production if rate-limited. - PARAMS_POLL_INTERVAL_MS: Joi.number().integer().min(5_000).default(30_000), - - // ── Leader election (issue #493) ────────────────────────────────────────── - // Controls whether Postgres advisory-lock based leader election is enabled - // for singleton workers (sweeper, event-ingestion). - // - // Set LEADER_ELECTION_ENABLED=false in single-instance dev deployments or - // when no database is available. When disabled, every worker considers - // itself leader unconditionally — the pre-election behaviour. - // - // IMPORTANT: Do NOT route the leader election connection through PgBouncer - // in transaction-pooling mode. Advisory locks are session-scoped; they are - // released when the connection is returned to the pool. Use a direct - // connection or PgBouncer in session mode. - LEADER_ELECTION_ENABLED: Joi.boolean().default(false), - - // Heartbeat interval in milliseconds — how often non-leaders attempt to - // acquire the lock and leaders renew it. Lower values reduce failover time - // but increase DB load. Default 5 s gives ≤ 15 s failover. - LEADER_ELECTION_HEARTBEAT_MS: Joi.number().integer().min(1000).max(60000).default(5000), - - // ── Background jobs (issue #494) ────────────────────────────────────────── - // PROCESS_ROLE: "api" serves HTTP/WS only, "worker" runs queue workers, - // "all" does both (single-process dev default). Producers work in any role. - PROCESS_ROLE: Joi.string().valid("api", "worker", "all").default("all"), - // JOBS_DRIVER: "memory" is single-process and non-durable (dev/test); - // "bullmq" uses REDIS_URL and is required for multi-instance deploys. - JOBS_DRIVER: Joi.string().valid("memory", "bullmq").default("memory"), - JOBS_SHUTDOWN_TIMEOUT_MS: Joi.number().integer().min(0).default(25000), - - // ── Runtime feature flags (issue #495) ──────────────────────────────────── - FLAGS_PUBSUB: Joi.string().valid("memory", "redis").default("memory"), - FLAGS_REFRESH_MS: Joi.number().integer().min(1000).default(30000), - // Comma-separated "key=true|false" pins that win over DB state (break-glass). - FLAG_OVERRIDES: Joi.string() - .allow("") - .pattern(/^([a-z0-9-]+=(true|false))(,[a-z0-9-]+=(true|false))*$/) - .default(""), - - // ── Admin RBAC ──────────────────────────────────────────────────────────── - // Comma-separated "id:role:secret" entries; role is "admin" or "superadmin". - // Empty disables every admin endpoint (401). - ADMIN_API_KEYS: Joi.string() - .allow("") - .pattern(/^([A-Za-z0-9_.-]+:(admin|superadmin):[^,:]{16,})(,[A-Za-z0-9_.-]+:(admin|superadmin):[^,:]{16,})*$/) - .default(""), - - // ── Public anonymised datasets ──────────────────────────────────────────── - DATASETS_ENABLED: Joi.boolean().default(false), - DATASETS_ANONYMIZE: Joi.boolean().default(true), - DATASETS_SALT: Joi.string().allow("").default(""), - DATASETS_SALT_ROTATION_HOURS: Joi.number().integer().min(1).max(720).default(24), - DATASETS_SALT_RETENTION_WINDOWS: Joi.number().integer().min(0).max(30).default(2), - DATASETS_PUBLIC_BUCKET: Joi.string().allow("").default(""), - DATASETS_STORAGE_KIND: Joi.string().valid("local", "memory").default("memory"), - DATASETS_LOCAL_DIR: Joi.string().allow("").default(""), - - // ── Guardian emergency ingestion (issue #507) ───────────────────────────── - GUARDIAN_CONTRACT_ID: Joi.string().allow("").default(""), - - // ── Synthetic canary (issue #496) ───────────────────────────────────────── - // Comma-separated canary user/solver addresses, excluded from public stats - // and leaderboards. - CANARY_ADDRESSES: Joi.string().allow("").default(""), - - // ── Public anonymised datasets (docs/rfcs/0001) ─────────────────────────── - DATASETS_ENABLED: Joi.boolean().default(false), - DATASETS_ANONYMIZE: Joi.boolean().default(true), - // Required only when datasets are enabled AND anonymisation is on — an - // empty/weak salt would collapse pseudonymisation to a fixed, reversible - // transform. It stays optional (default "") otherwise so existing dev/test - // configs are unaffected. - DATASETS_SALT: Joi.string() - .when("DATASETS_ENABLED", { - is: true, - then: Joi.string().when("DATASETS_ANONYMIZE", { - is: true, - then: Joi.string() - .min(32) - .required() - .messages({ - "any.required": - "DATASETS_SALT must be set when DATASETS_ENABLED=true and DATASETS_ANONYMIZE=true. " + - "Generate a strong random secret (e.g. `openssl rand -hex 32`).", - "string.min": "DATASETS_SALT must be at least 32 characters.", - }), - otherwise: Joi.string().allow("").default(""), - }), - otherwise: Joi.string().allow("").default(""), - }), - DATASETS_SALT_ROTATION_HOURS: Joi.number().integer().min(1).default(24), - DATASETS_SALT_RETENTION_WINDOWS: Joi.number().integer().min(0).default(2), - DATASETS_PUBLIC_BUCKET: Joi.string().default("vortex-public-datasets"), - DATASETS_STORAGE: Joi.string().valid("local", "memory").default("local"), - DATASETS_LOCAL_DIR: Joi.string().default(".datasets"), - - // ── Secrets Manager (issue #465) ──────────────────────────────────────────── - SECRETS_PROVIDER: Joi.string().valid("env", "aws-secrets-manager", "vault-kv").default("env"), - SECRETS_REFRESH_INTERVAL_MS: Joi.number().integer().min(5000).default(60000), - SECRETS_EXTRA: Joi.string().allow("").default(""), - - // AWS Secrets Manager - AWS_SECRETS_MANAGER_PREFIX: Joi.string().allow("").default(""), - AWS_SECRETS_MANAGER_POLL_INTERVAL_MS: Joi.number().integer().min(5000).default(60000), - - // Vault KV - VAULT_KV_MOUNT: Joi.string().default("secret"), - VAULT_KV_PREFIX: Joi.string().default("vortex/"), - VAULT_KV_POLL_INTERVAL_MS: Joi.number().integer().min(5000).default(60000), - - // Extra secret env vars referenced by the default SecretConfig - JWT_SIGNING_KEY: Joi.string().allow("").default(""), - WEBHOOK_SECRET: Joi.string().allow("").default(""), - CHANNEL_KEY: Joi.string().allow("").default(""), - // ── Egress / SSRF Protection (issue #468) ───────────────────────────────── - // Controls the centralized HttpEgressService used for all outbound HTTP requests - // (RPC, Horizon, oracles, webhooks) to prevent SSRF attacks. - EGRESS_TIMEOUT_MS: Joi.number().integer().min(1000).max(60000).default(10000), - EGRESS_MAX_REDIRECTS: Joi.number().integer().min(0).max(5).default(3), - EGRESS_MAX_BODY_SIZE_BYTES: Joi.number().integer().min(1024).default(10485760), // 10MB - SOROBAN_RPC_ALLOWLIST: Joi.string().allow("").default(""), - WEBHOOK_ALLOWLIST: Joi.string().allow("").default(""), - ORACLE_ALLOWLIST: Joi.string().allow("").default(""), - - // ── WS gateway hardening (issue #455) ───────────────────────────────────── - WS_MAX_PAYLOAD_BYTES: Joi.number().integer().min(1024).default(16384), - WS_MAX_CONNECTIONS_PER_IP: Joi.number().integer().min(0).default(20), - // Hops of trusted reverse proxies in front of the service. 0 ignores - // X-Forwarded-For entirely so clients cannot spoof their IP. - WS_TRUST_PROXY_HOPS: Joi.number().integer().min(0).default(0), - WS_RATE_LIMIT_PER_SEC: Joi.number().positive().default(10), - WS_RATE_LIMIT_BURST: Joi.number().integer().min(1).default(20), - WS_RATE_LIMIT_MAX_VIOLATIONS: Joi.number().integer().min(1).default(5), - WS_OUTBOUND_QUEUE_MAX: Joi.number().integer().min(1).default(1000), - WS_OUTBOUND_BUFFER_BYTES: Joi.number().integer().min(1024).default(1048576), - WS_SLOW_CONSUMER_POLICY: Joi.string().valid("drop_oldest", "disconnect").default("drop_oldest"), - // HS256 secret shared with the SEP-10 auth endpoint (#442). Empty disables - // JWT auth; signature auth keeps working. - AUTH_JWT_SECRET: Joi.string().allow("").min(32).default(""), - - // ── Distributed rate limiting (issue #441) ───────────────────────────────── - // How often the local rate-limiter fallback prunes expired window entries. - // Only relevant during a Redis outage; keeps the fallback map bounded. - RATE_LIMIT_LOCAL_PRUNE_MS: Joi.number().integer().min(1000).max(60000).default(60000), - - // Redis URL for the distributed rate limiter (#441). Leave empty to force the - // bounded local limiter (the limit is still enforced, per process). - RATE_LIMIT_REDIS_URL: Joi.string().allow("").optional(), - - // ── Scoped solver credentials (issue #443) ───────────────────────────────── - // Cross-replica transport for credential revocation invalidation. - CREDENTIAL_REVOCATION_PUBSUB: Joi.string().valid("memory", "redis").default("memory"), - - // ── SSE intent feed (issue #433) ─────────────────────────────────────────── - // Heartbeat comment interval and per-client backpressure limit for the - // Server-Sent Events intent stream. - SSE_HEARTBEAT_MS: Joi.number().integer().min(1000).max(60000).default(15000), - SSE_MAX_BUFFER_BYTES: Joi.number().integer().min(1024).default(1048576), - - // ── Health probes (issue #492) ──────────────────────────────────────────── - // Comma-separated roles this process serves: api, ws, worker. - SERVICE_ROLES: Joi.string() - .pattern(/^(api|ws|worker)(,(api|ws|worker))*$/) - .default("api,ws,worker"), - HEALTH_CHECK_INTERVAL_MS: Joi.number().integer().min(500).default(5000), - HEALTH_READY_FAILURE_THRESHOLD: Joi.number().integer().min(1).default(3), - HEALTH_READY_SUCCESS_THRESHOLD: Joi.number().integer().min(1).default(2), - HEALTH_EVENT_LOOP_MAX_LAG_MS: Joi.number().integer().min(50).default(1000), - // Comma-separated Soroban RPC URLs for the RPC-quorum readiness check. - // Defaults to SOROBAN_RPC_URL. - SOROBAN_RPC_HEALTH_URLS: Joi.string().allow("").default(""), -}); diff --git a/src/intents/dto/slash.dto.ts b/src/intents/dto/slash.dto.ts new file mode 100644 index 00000000..ebd62dad --- /dev/null +++ b/src/intents/dto/slash.dto.ts @@ -0,0 +1,52 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; +import { Type } from "class-transformer"; +import { IsIn, IsInt, IsOptional, IsString, Matches, Max, MaxLength, Min, MinLength } from "class-validator"; +import { PENDING_SLASH_STATES, PendingSlashState } from "../../solvers/pending-slashes.repository"; + +const ED25519_SIGNATURE_MAX_LENGTH = 88; + +export class ListSlashesDto { + @ApiPropertyOptional({ enum: PENDING_SLASH_STATES }) + @IsOptional() + @IsIn(PENDING_SLASH_STATES) + state?: PendingSlashState; + + @ApiPropertyOptional({ default: 50, maximum: 200 }) + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(200) + limit?: number; +} + +export class AdminCancelSlashDto { + @ApiProperty({ description: "Why the slash is being cancelled (audit log)", maxLength: 500 }) + @IsString() + @MinLength(5) + @MaxLength(500) + note!: string; +} + +export class FillProofDto { + @ApiProperty({ description: "Stellar address of the slashed solver", maxLength: 56 }) + @IsString() + @MinLength(10) + @MaxLength(56) + solver!: string; + + @ApiProperty({ description: "Hash of the Stellar transaction that filled the intent (64 hex chars)" }) + @Matches(/^[0-9a-f]{64}$/i) + txHash!: string; + + @ApiProperty({ + description: + 'Base64 Ed25519 signature by `solver` of "fill-proof:::" ' + + "(txHash lowercased)", + maxLength: ED25519_SIGNATURE_MAX_LENGTH, + }) + @IsString() + @MinLength(10) + @MaxLength(ED25519_SIGNATURE_MAX_LENGTH) + signature!: string; +} diff --git a/src/intents/intents-sweeper.manual-trigger.spec.ts b/src/intents/intents-sweeper.manual-trigger.spec.ts index bf811ca9..1f0a0a97 100644 --- a/src/intents/intents-sweeper.manual-trigger.spec.ts +++ b/src/intents/intents-sweeper.manual-trigger.spec.ts @@ -3,7 +3,7 @@ import { IntentsSweeperService } from "./intents-sweeper.service"; import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { MetricsService } from "../metrics/metrics.service"; import { KillSwitchService } from "../killswitch/killswitch.service"; import { LeaderElectionService } from "../common/leader-election"; @@ -37,9 +37,7 @@ describe("IntentsSweeperService — manual sweep trigger (#269)", () => { } as unknown as IntentsService; const gateway = { broadcast: jest.fn() } as unknown as IntentsGateway; const solversService = { recordFailedFill: jest.fn() } as unknown as SolversService; - const solverRegistry = { - slashSolver: jest.fn().mockResolvedValue({ detail: "no-op" }), - } as unknown as SolverRegistryService; + const slashingPipeline = { detect: jest.fn() } as unknown as SlashingPipelineService; const metricsService = { recordSweep: jest.fn() } as unknown as MetricsService; const killSwitch = { evaluateTarget: jest.fn().mockReturnValue({ paused: false, matched: null, matchedChain: [] }), @@ -49,7 +47,7 @@ describe("IntentsSweeperService — manual sweep trigger (#269)", () => { intentsService, gateway, solversService, - solverRegistry, + slashingPipeline, metricsService, killSwitch, noopLeaderElection(), diff --git a/src/intents/intents-sweeper.service.spec.ts b/src/intents/intents-sweeper.service.spec.ts index 804265aa..9c98b02b 100644 --- a/src/intents/intents-sweeper.service.spec.ts +++ b/src/intents/intents-sweeper.service.spec.ts @@ -6,7 +6,7 @@ import { KillSwitchService } from "../killswitch/killswitch.service"; import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { MetricsService } from "../metrics/metrics.service"; import { InMemorySolversRepository } from "../solvers/in-memory-solvers.repository"; import { SOLVERS_REPOSITORY } from "../solvers/solvers.repository"; @@ -51,7 +51,7 @@ function buildIntentsService(): IntentsService { const protocolParams = { snapshotForChain: jest.fn().mockReturnValue({ version: 0, feeBps: 30, deadlineSeconds: 1800, fillWindowSeconds: 600, capturedAt: new Date().toISOString() }), } as unknown as ProtocolParamsService; - return new IntentsService(repo, configService, stellarTxService, prismaService, protocolParams); + return new IntentsService(repo, configService, stellarTxService, prismaService, undefined, undefined, protocolParams); } async function buildSolversService(): Promise { @@ -68,7 +68,7 @@ describe("IntentsSweeperService", () => { let intentsService: IntentsService; let gateway: IntentsGateway; let solversService: SolversService; - let solverRegistryService: jest.Mocked; + let slashingPipeline: jest.Mocked>; let metricsService: jest.Mocked>; let killSwitch: jest.Mocked>; let sweeper: IntentsSweeperService; @@ -77,13 +77,13 @@ describe("IntentsSweeperService", () => { intentsService = buildIntentsService(); gateway = { broadcast: jest.fn().mockResolvedValue(undefined) } as unknown as IntentsGateway; solversService = await buildSolversService(); - solverRegistryService = { - slashSolver: jest.fn().mockResolvedValue({ - submitted: false, - simulated: false, - detail: "not configured — no-op", - }), - } as unknown as jest.Mocked; + slashingPipeline = { + detect: jest.fn().mockImplementation(async (input) => ({ + ...input, + state: "challenge_window", + challengeEndsAt: new Date((input.detectedAt + 600) * 1000), + })), + } as unknown as jest.Mocked>; metricsService = { recordSweep: jest.fn() } as unknown as jest.Mocked>; // Default: no pause active, so existing sweeper expectations are unchanged. killSwitch = { @@ -94,7 +94,7 @@ describe("IntentsSweeperService", () => { intentsService, gateway, solversService, - solverRegistryService, + slashingPipeline as unknown as SlashingPipelineService, metricsService as unknown as MetricsService, killSwitch as unknown as KillSwitchService, noopLeaderElection(), @@ -149,8 +149,9 @@ describe("IntentsSweeperService", () => { expect(gateway.broadcast).toHaveBeenCalledWith( expect.objectContaining({ type: "intent_slashed", intentId, solver: ALPHA_ADDR }), ); - expect(solverRegistryService.slashSolver).toHaveBeenCalledWith( - expect.objectContaining({ solverAddress: ALPHA_ADDR, intentId }), + // Issue #397: the sweeper only detects — the saga owns the on-chain slash. + expect(slashingPipeline.detect).toHaveBeenCalledWith( + expect.objectContaining({ solverAddress: ALPHA_ADDR, intentId, fillDeadline: past }), ); }); @@ -184,7 +185,7 @@ describe("IntentsSweeperService", () => { await sweeper.sweep(); expect((await intentsService.get(intentId))?.state).toBe("accepted"); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); }); it("does not throw if an accepted intent somehow has no solver on record", async () => { @@ -202,7 +203,7 @@ describe("IntentsSweeperService", () => { await expect(sweeper.sweep()).resolves.not.toThrow(); expect((await intentsService.get(intent.intentId))?.state).toBe("slashed"); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); }); // ── #259: MetricsService integration ──────────────────────────────────── @@ -271,7 +272,7 @@ describe("IntentsSweeperService", () => { // Not slashed — the pause, not the solver, caused the missed fill. expect(result.slashedCount).toBe(0); expect(result.extendedDeadlines).toBe(1); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); const updated = await intentsService.get(intentId); expect(updated?.state).toBe("accepted"); diff --git a/src/intents/intents-sweeper.service.ts b/src/intents/intents-sweeper.service.ts index 98f355eb..0c9d6777 100644 --- a/src/intents/intents-sweeper.service.ts +++ b/src/intents/intents-sweeper.service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/commo import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { logger } from "../common/logger"; import { MetricsService } from "../metrics/metrics.service"; import { KillSwitchService } from "../killswitch/killswitch.service"; @@ -34,7 +34,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { private readonly intentsService: IntentsService, private readonly intentsGateway: IntentsGateway, private readonly solversService: SolversService, - private readonly solverRegistryService: SolverRegistryService, + private readonly slashingPipeline: SlashingPipelineService, private readonly metricsService: MetricsService, private readonly killSwitch: KillSwitchService, private readonly leaderElection: LeaderElectionService, @@ -136,7 +136,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { continue; } - const slashed = await this.slashMissedFill(intent.intentId, intent.solver, now); + const slashed = await this.slashMissedFill(intent.intentId, intent.solver, intent.deadline, now); if (slashed) slashedCount++; } @@ -197,9 +197,16 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { } } + /** + * Detection half of the slashing saga (issue #397). Marks the intent + * `slashed`, applies the optimistic local penalty, and hands off to + * SlashingPipelineService, which holds the slash in a challenge window, + * re-verifies, and only then broadcasts — nothing is sent on-chain here. + */ private async slashMissedFill( intentId: string, solver: string | undefined, + fillDeadline: number, now: number, ): Promise { const reason = "accepted intent not filled before deadline"; @@ -215,7 +222,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { solver, slashedAt: now, }); - await this.intentsGateway.broadcast({ type: "intent_slashed", intentId, solver, reason }); + await this.intentsGateway.broadcast({ type: "intent_slashed", intentId, solver, reason, pending: true }); if (!solver) { // Shouldn't happen in practice — an "accepted" intent always has a @@ -224,16 +231,21 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { return true; } + // Optimistic local penalty; the saga compensates via rollbackPenalty if + // the slash is cancelled. await this.solversService.recordFailedFill(solver, intentId); const slashRecord = await this.solversService.recordSlash(solver, intentId, reason, now); - const result = await this.solverRegistryService.slashSolver({ - solverAddress: solver, + const pending = await this.slashingPipeline.detect({ intentId, + solverAddress: solver, reason, + fillDeadline, + detectedAt: now, }); - console.log( - `[sweeper] slashed solver=${solver} for intent=${intentId}: ${result.detail} slashId=${slashRecord?.slashId ?? "unknown"}`, + this.logger.log( + `[sweeper] slash detected solver=${solver} intent=${intentId} state=${pending.state} ` + + `challengeEndsAt=${pending.challengeEndsAt.toISOString()} slashId=${slashRecord?.slashId ?? "unknown"}`, ); return true; } diff --git a/src/intents/intents.gateway.ts b/src/intents/intents.gateway.ts index c1eb2709..e69de29b 100644 --- a/src/intents/intents.gateway.ts +++ b/src/intents/intents.gateway.ts @@ -1,1108 +0,0 @@ -import { Inject, OnModuleDestroy, Optional } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; -import { OnGatewayConnection, OnGatewayDisconnect, WebSocketGateway } from "@nestjs/websockets"; -import type { IncomingMessage } from "node:http"; -import { WebSocket } from "ws"; -import { IntentsService } from "./intents.service"; -import { SolversService } from "../solvers/solvers.service"; -import { MetricsService } from "../metrics/metrics.service"; -import { logger } from "../common/logger"; -import { SUPPORTED_CHAINS, SupportedChain } from "./intents.types"; -import { verifyStellarSignature, buildWsAuthMessage } from "../common/stellar-signature"; -import { buildMatchPredicate, IntentCapabilityIndex, SolverMatchPredicate } from "./solver-intent-matcher"; -import { - WS_MAX_FILTER_CHAINS, - WS_MAX_SUBSCRIPTIONS_PER_CONNECTION, -} from "../config/limits.config"; -import configuration, { AppConfig } from "../config/configuration"; -import { verifyHs256Jwt } from "../common/jwt"; -import { Backplane, SequencedEvent, WS_BACKPLANE } from "./backplane/backplane.types"; -import { MemoryBackplane } from "./backplane/memory.backplane"; -import { ConnectionState, resolveClientIp, EncodingFormat } from "./ws/connection-state"; -import { EncodingCache } from "./ws/encoding-cache"; -import { ConnectionState, resolveClientIp } from "./ws/connection-state"; -import { IntentFeedService } from "./feed/intent-feed.service"; -import { FeedClient, FeedFilter } from "./feed/feed.types"; -import { randomUUID } from "node:crypto"; - -export type { SequencedEvent } from "./backplane/backplane.types"; -export { EventRingBuffer } from "./event-ring-buffer"; - -/** Read WS_HEARTBEAT_INTERVAL_MS from the environment, falling back to 30 s. */ -function resolveHeartbeatIntervalMs(): number { - const parsed = Number(process.env.WS_HEARTBEAT_INTERVAL_MS); - return Number.isFinite(parsed) && parsed > 0 ? parsed : HEARTBEAT_INTERVAL_MS; -} - -/** - * WebSocket adapter over the transport-agnostic {@link IntentFeedService} - * (issue #433). - * - * The feed service owns sequencing, replay, filtering, delivery and connection - * accounting. This class owns only the WebSocket protocol: the upgrade - * handshake, the `subscribe` / `replay` / `auth` message frames, the - * solver-JWT handshake, and the ping/pong heartbeat. - */ -// maxPayload is enforced by `ws` itself (close 1009). Read from the -// environment because decorator options are evaluated at import time; -// handleMessage re-checks against the validated config. -@WebSocketGateway({ path: "/ws", maxPayload: configuration().ws.maxPayloadBytes }) -export class IntentsGateway - implements OnGatewayConnection, OnGatewayDisconnect, OnModuleDestroy -{ - /** Per-connection identity, rate-limit and outbound-queue state (issue #455). */ - private readonly connections = new Map(); - /** Maps a live WebSocket to its feed-service client wrapper. */ - private readonly feedClients = new Map(); - private readonly authenticatedSolver = new WeakMap(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private heartbeatTimer: any; - private readonly wsConfig: AppConfig["ws"]; - private readonly jwtSecret: string; - - /** Fan-out + global sequencing (issue #454): memory or Redis Streams. */ - private readonly backplane: Backplane; - /** Serialises local delivery so events reach clients in `seq` order. */ - private deliveryChain: Promise = Promise.resolve(); - - private nextSeq = 1; - - /** Configured heartbeat interval in milliseconds (default 30 000). */ - public readonly heartbeatIntervalMs: number; - - /** Number of connections terminated in the most recent heartbeat cycle. */ - private lastHeartbeatTerminatedCount = 0; - - /** Ring buffer storing the last REPLAY_BUFFER_SIZE broadcast events. */ - private readonly ringBuffer = new EventRingBuffer(REPLAY_BUFFER_SIZE); - - /** Encoding cache: serialize once per format, not per client (Activity 1). */ - private readonly encodingCache = new EncodingCache(REPLAY_BUFFER_SIZE); - - /** Graceful shutdown state (Activity 2). */ - private draining = false; - private drainStartedAt = 0; - - constructor( - private readonly intentsService: IntentsService, - private readonly solversService: SolversService, - private readonly intentIndex: IntentCapabilityIndex, - private readonly feed: IntentFeedService, - @Optional() private readonly metricsService?: MetricsService, - @Optional() config?: ConfigService, - @Optional() @Inject(WS_BACKPLANE) backplane?: Backplane, - ) { - const defaults = configuration(); - this.wsConfig = config?.get("ws", { infer: true }) ?? defaults.ws; - this.jwtSecret = config?.get("authJwtSecret", { infer: true }) ?? defaults.authJwtSecret; - this.heartbeatIntervalMs = resolveHeartbeatIntervalMs(); - this.heartbeatTimer = setInterval(() => this.heartbeat(), this.heartbeatIntervalMs); - this.backplane = this.createBackplane(); - if (this.backplane) { - this.backplane.subscribe((event) => { - const type = typeof event.type === "string" ? event.type : ""; - if (!type) return; - this.dispatchRemoteEvent(event as Record); - }); - } - logger.info(`ws heartbeat started (backplane=${this.feed.backplaneHealth().mode})`); - } - - /** Backplane health for /health (issue #454). */ - backplaneHealth() { - return this.feed.backplaneHealth(); - } - - /** - * Broadcast an event to every connected client (WS and SSE) on every replica. - * - * Activity 1: Uses encoding cache — serializes once per format, not per client. - * - * Delivery rules (evaluated in order): - * 1. Client is not OPEN → skip. - * 2. Client set wantAll=true → always deliver. - * 3. Client has a solver capability predicate: - * a. Event carries an inlined intent → apply predicate to that intent. - * b. Event is a state-transition (only intentId available) → deliver - * (we cannot efficiently look up the intent here; the solver would - * already have received the intent_created event through the filter). - * 4. Client has a plain chain filter (`chains != null`) → apply chain match. - * 5. No filter → full unfiltered feed (backward-compatible default). - */ - private deliverToMatchingSubscribers( - seq: number, - chain: SupportedChain | null, - event: { type: string; [key: string]: unknown }, - ) { - for (const [client, filter] of this.subscribers) { - if (client.readyState !== WebSocket.OPEN) continue; - - // Opt-out: solver requested full feed. - if (filter.wantAll) { - this.sendEncoded(client, seq, event); - continue; - } - - // Authenticated solver — apply capability predicate. - if (filter.solver !== null) { - const solverPredicate = filter.solver; - const inlinedIntent = (event as { intent?: unknown }).intent; - - // intent_created carries a full intent object we can test directly. - if (event.type === "intent_created" && inlinedIntent && typeof inlinedIntent === "object") { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const matches = solverPredicate.matches(inlinedIntent as any); - if (matches) { - this.sendEncoded(client, seq, event); - try { this.metricsService?.incWsDelivered(solverPredicate.solverAddress); } catch { /* noop */ } - } else { - try { this.metricsService?.incWsFiltered(solverPredicate.solverAddress); } catch { /* noop */ } - } - continue; - } - - // State-transition events: the solver already filtered on intent_created, - // so we pass them through to keep the feed self-consistent. - this.sendEncoded(client, seq, event); - try { this.metricsService?.incWsDelivered(solverPredicate.solverAddress); } catch { /* noop */ } - continue; - } - - // No filter set → full unfiltered feed (backward-compatible default). - if (filter.chains === null) { - this.sendEncoded(client, seq, event); - continue; - } - - // Chain couldn't be resolved → deliver to everyone (safe default). - if (chain === null) { - this.sendEncoded(client, seq, event); - continue; - } - - // Only send if the event's chain is in this subscriber's filter. - if (filter.chains.has(chain)) { - this.sendEncoded(client, seq, event); - } - } - } - - handleConnection(client: WebSocket) { - this.subscribers.set(client, { - chains: null, - solver: null, - wantAll: false, - subscriptionCount: 0, - }); - /** - * Send an event to a client using its negotiated encoding format (Activity 1). - * - * Retrieves pre-serialized payload from encoding cache, avoiding redundant - * serialization work. At 10k connections with msgpack, this saves 9,999 - * msgpackEncode() calls per broadcast. - */ - private sendEncoded(client: WebSocket, seq: number, event: Record): void { - const state = this.connections.get(client); - if (!state) { - // Fallback for connections without state (shouldn't happen) - if (client.readyState === WebSocket.OPEN) { - client.send(JSON.stringify({ seq, ...event })); - } - return; - } - - const payload = this.encodingCache.get(seq, event, state.encoding); - const result = state.send(payload); - - if (result === "dropped_oldest") { - this.metricsService?.wsOutboundDropped.inc(); - } else if (result === "disconnected") { - this.metricsService?.wsSlowConsumerDisconnects.inc(); - logger.warn(`ws slow consumer disconnected (ip=${state.ip}, queue full)`); - this.removeSubscriber(client); - } - } - - /** - * Admits a connection (issue #455): enforces WS_MAX_CONNECTIONS and the - * per-IP limit (IP resolved through WS_TRUST_PROXY_HOPS), creates the - * per-connection state, and accepts an optional solver JWT from - * `?token=` or `Authorization: Bearer` (anonymous connections stay allowed). - * - * Activity 1: Negotiates encoding format via Sec-WebSocket-Protocol header. - * Activity 2: Rejects new connections when draining. - * Delegates to the feed service, which owns sequencing and delivery. This - * method exists so existing callers (controller, sweeper) keep working - * against the gateway's public surface. - */ - broadcast(event: { type: string; [key: string]: unknown }): Promise { - return this.feed.broadcast(event); - } - - private static isSupportedChain(value: unknown): value is SupportedChain { - return typeof value === "string" && (SUPPORTED_CHAINS as readonly string[]).includes(value); - } - - /** - * Admit a WebSocket connection (issue #455): enforces the shared connection - * limits through the feed service, creates the per-connection state, and - * accepts an optional solver JWT from `?token=` or `Authorization: Bearer`. - */ - handleConnection(client: WebSocket, request?: IncomingMessage) { - // Activity 2: Reject new connections during graceful shutdown - if (this.draining) { - client.close(1001, "Server draining"); - this.metricsService?.wsConnectionsRejected.inc({ reason: "draining" }); - return; - } - - const ip = resolveClientIp( - request?.socket?.remoteAddress, - request?.headers?.["x-forwarded-for"], - this.wsConfig.trustProxyHops, - ); - - const filter: FeedFilter = { chains: null, solver: null, wantAll: false, users: null, states: null, subscriptionCount: 0 }; - const state = new ConnectionState( - client, - ip, - { perSec: this.wsConfig.rateLimitPerSec, burst: this.wsConfig.rateLimitBurst }, - { - queueMax: this.wsConfig.outboundQueueMax, - bufferBytes: this.wsConfig.outboundBufferBytes, - policy: this.wsConfig.slowConsumerPolicy, - }, - ); - - // Activity 1: Negotiate encoding format from Sec-WebSocket-Protocol header - const protocols = request?.headers?.["sec-websocket-protocol"]; - const encoding = this.negotiateEncoding(protocols); - state.encoding = encoding; - - this.connections.set(client, state); - this.connectionsPerIp.set(ip, perIp + 1); - this.subscribers.set(client, { chains: null, solver: null, wantAll: false, subscriptionCount: 0 }); - this.alive.set(client, true); - const feedClient = new WsFeedClient(client, ip, state, this.metricsService); - const admission = this.feed.addClient(feedClient, filter); - if (!admission.ok) { - state.close(); - this.metricsService?.wsConnectionsRejected.inc({ reason: admission.reason ?? "max_connections" }); - client.close(1013, admission.reason === "per_ip" ? "Too many connections from this IP" : "Server at capacity"); - return; - } - - this.connections.set(client, state); - this.feedClients.set(client, feedClient); - this.metricsService?.incWsConnection(); - - client.on("message", (raw) => { - void this.handleMessage(client, raw); - }); - - client.on("pong", () => { - feedClient.alive = true; - }); - - client.on("error", () => { - this.removeSubscriber(client); - logger.debug( - `ws client error/drop — active subscribers: ${this.feed.connectionCount}`, - ); - }); - - const currentSeq = this.feed.currentSeq; - - this.send( - client, - JSON.stringify({ - type: "connected", - message: "Vortex intent stream", - seq: currentSeq, - encoding, - }), - ); - - // Send the initial snapshot asynchronously — the client receives it - // immediately after the "connected" message. - Promise.resolve(this.intentsService.getByState("open")) - .then((open) => { - this.send(client, JSON.stringify({ type: "snapshot", intents: open.slice(0, 20), seq: currentSeq })); - }) - .catch(() => { - /* snapshot failure is non-fatal — client can re-fetch via REST */ - }); - - const token = IntentsGateway.bearerToken(request); - if (token) void this.authenticateJwt(client, token); - - logger.info(`ws client connected (subscribers=${this.subscribers.size}, encoding=${encoding})`); - } - - /** - * Negotiate encoding format from Sec-WebSocket-Protocol header (Activity 1). - * - * Clients send: `Sec-WebSocket-Protocol: vortex.v1+msgpack` - * Server responds with the negotiated protocol in upgrade response. - * - * @returns "msgpack" if client requests it, otherwise "json" (default) - */ - private negotiateEncoding(protocols: string | string[] | undefined): EncodingFormat { - if (!protocols) return "json"; - const requested = Array.isArray(protocols) ? protocols : protocols.split(",").map(p => p.trim()); - if (requested.includes("vortex.v1+msgpack")) { - return "msgpack"; - } - return "json"; - logger.info(`ws client connected (subscribers=${this.feed.connectionCount})`); - } - - /** JWT from `?token=` or `Authorization: Bearer` on the upgrade request. */ - private static bearerToken(request?: IncomingMessage): string | null { - const auth = request?.headers?.authorization; - if (auth?.startsWith("Bearer ")) return auth.slice(7).trim(); - try { - return new URL(request?.url ?? "", "http://localhost").searchParams.get("token"); - } catch { - return null; - } - } - - /** - * Queues `payload` for `client` through its backpressure-aware state - * (issue #455), counting slow-consumer drops and disconnects. - */ - private send(client: WebSocket, payload: string): void { - const state = this.connections.get(client); - if (!state) { - if (client.readyState === WebSocket.OPEN) client.send(payload); - return; - } - const result = state.send(payload); - if (result === "dropped_oldest") { - this.metricsService?.wsOutboundDropped.inc(); - } else if (result === "disconnected") { - this.metricsService?.wsSlowConsumerDisconnects.inc(); - logger.warn(`ws slow consumer disconnected (ip=${state.ip}, queue full)`); - this.removeSubscriber(client); - } - } - - handleDisconnect(client: WebSocket) { - this.removeSubscriber(client); - logger.info(`ws client disconnected (subscribers=${this.feed.connectionCount})`); - } - - /** Drop a client from the feed service and keep the connection gauge honest. */ - private removeSubscriber(client: WebSocket): void { - const state = this.connections.get(client); - if (state) { - state.close(); - this.connections.delete(client); - } - const feedClient = this.feedClients.get(client); - if (feedClient) { - this.feed.removeClient(feedClient); - this.feedClients.delete(client); - } - this.authenticatedSolver.delete(client); - } - - /** - * Handle a single incoming WebSocket message from a client. - * - * Supported message types: - * - `{ type: "subscribe", chains?: string[], all?: boolean }` — set a - * per-connection filter or opt out of capability filtering with `all: true`. - * - `{ type: "replay", fromSeq: number }` — replay buffered events. - * - `{ type: "auth", solver, timestamp, signature }` — authenticate as a - * registered solver; installs a capability predicate and sends an - * auto-scoped snapshot of currently-eligible open intents. - */ - private async handleMessage(client: WebSocket, raw: import("ws").RawData): Promise { - if (client.readyState !== WebSocket.OPEN) return; - const size = Array.isArray(raw) - ? raw.reduce((n, b) => n + b.length, 0) - : (raw as Buffer | ArrayBuffer).byteLength; - if (size > this.wsConfig.maxPayloadBytes) { - client.close(1009, "Message too big"); - return; - } - - const state = this.connections.get(client); - if (state && !state.bucket.take()) { - state.violations += 1; - if (state.violations >= this.wsConfig.rateLimitMaxViolations) { - this.metricsService?.wsRateLimited.inc({ action: "disconnected" }); - client.close(1008, "Rate limit exceeded"); - return; - } - this.metricsService?.wsRateLimited.inc({ action: "rejected" }); - this.send(client, JSON.stringify({ type: "rate_limited", retryAfterMs: Math.ceil(1000 / this.wsConfig.rateLimitPerSec) })); - return; - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw.toString()); - } catch { - return; - } - - if (typeof parsed !== "object" || parsed === null) return; - - const msg = parsed as Record; - - switch (msg.type) { - case "subscribe": - this.handleSubscribe(client, msg); - break; - case "replay": - this.handleReplay(client, msg); - break; - case "auth": - await this.handleAuth(client, msg); - break; - default: - break; - } - } - - /** - * Process a `{ type: "subscribe", chains?: string[], all?: boolean }` message. - * - * When `all: true` is present, the connection opts out of capability filtering - * and receives the complete unfiltered feed regardless of solver auth status. - * - * When `chains` is present, a per-connection chain filter is installed (this - * clears any existing solver capability predicate on the connection). - * Validates each chain value against `SUPPORTED_CHAINS` and stores only - * the valid subset. A subscribe message with no valid chains is treated as - * "subscribe to nothing" (the client will receive only chainless events). - * An entirely missing or non-array `chains` field is rejected silently - * without updating the existing filter. - * - * Issue #476: enforces two per-connection limits. - */ - private handleSubscribe(client: WebSocket, msg: Record): void { - const feedClient = this.feedClients.get(client); - if (!feedClient) return; - const current = this.feed.getFilter(feedClient); - if (!current) return; - - // all=true: opt out of capability filtering. - if (msg.all === true) { - const existing = this.subscribers.get(client) ?? { - chains: null, - solver: null, - wantAll: false, - subscriptionCount: 0, - }; - const existing = this.subscribers.get(client) ?? { chains: null, solver: null, wantAll: false, subscriptionCount: 0 }; - this.subscribers.set(client, { ...existing, wantAll: true }); - this.feed.updateClientFilter(feedClient, { ...current, wantAll: true }); - logger.debug("ws client opted out of capability filtering (all=true)"); - if (client.readyState === WebSocket.OPEN) { - this.send(client, JSON.stringify({ type: "subscribed", filter: { all: true } })); - } - return; - } - - if (!Array.isArray(msg.chains)) { - logger.debug("ws subscribe ignored: chains field missing or not an array"); - return; - } - - // ── Limit 1: max subscriptions per connection (issue #476) ─────────────── - const maxSubs = parseInt( - process.env.WS_MAX_SUBSCRIPTIONS ?? String(WS_MAX_SUBSCRIPTIONS_PER_CONNECTION), - 10, - ); - if (current.subscriptionCount >= maxSubs) { - logger.warn( - `ws subscribe_rejected: connection has reached the max subscription limit (${maxSubs})`, - ); - if (client.readyState === WebSocket.OPEN) { - this.send(client, - JSON.stringify({ - type: "subscribe_rejected", - reason: `Maximum subscription limit of ${maxSubs} reached for this connection`, - }), - ); - } - return; - } - - // ── Limit 2: max chain-filter values per subscribe message (issue #476) ── - const maxChains = parseInt( - process.env.WS_MAX_FILTER_CHAINS ?? String(WS_MAX_FILTER_CHAINS), - 10, - ); - const rawChains = msg.chains as unknown[]; - if (rawChains.length > maxChains) { - logger.warn( - `ws subscribe_rejected: chains array length ${rawChains.length} exceeds max ${maxChains}`, - ); - if (client.readyState === WebSocket.OPEN) { - this.send(client, - JSON.stringify({ - type: "subscribe_rejected", - reason: `chains array may contain at most ${maxChains} values`, - }), - ); - } - return; - } - - const validChains = rawChains.filter( - (c): c is SupportedChain => - typeof c === "string" && (SUPPORTED_CHAINS as readonly string[]).includes(c), - ); - - filter.chains = new Set(validChains); - filter.subscriptionCount += 1; - // An explicit chain filter replaces any solver capability predicate. - this.feed.updateClientFilter(feedClient, { - ...current, - chains: new Set(validChains), - solver: null, - wantAll: false, - subscriptionCount: current.subscriptionCount + 1, - }); - - logger.debug(`ws client subscribed to chains: ${validChains.join(", ") || "(none)"}`); - - if (client.readyState === WebSocket.OPEN) { - this.send(client, - JSON.stringify({ - type: "subscribed", - filter: { chains: validChains }, - }), - ); - } - } - - /** - * Process a `{ type: "replay", fromSeq: number }` message. - */ - private handleReplay(client: WebSocket, msg: Record): void { - const fromSeq = typeof msg.fromSeq === "number" ? msg.fromSeq : null; - if (fromSeq === null || !Number.isInteger(fromSeq) || fromSeq < 0) { - logger.debug("ws replay ignored: fromSeq missing or invalid"); - return; - } - - if (client.readyState !== WebSocket.OPEN) return; - - const feedClient = this.feedClients.get(client); - if (!feedClient) return; - - const result = this.feed.replaySince(fromSeq, feedClient); - - if (result.tooOld) { - this.send(client, - JSON.stringify({ - type: "replay_too_old", - fromSeq, - oldestAvailableSeq: result.oldestSeq, - }), - ); - logger.debug(`ws replay_too_old: fromSeq=${fromSeq} oldestAvailable=${result.oldestSeq}`); - return; - } - - this.send(client, - JSON.stringify({ - type: "replay_start", - fromSeq, - count: result.events.length, - }), - ); - - for (const event of result.events) { - if (client.readyState !== WebSocket.OPEN) break; - this.send(client, JSON.stringify(event)); - } - - if (client.readyState === WebSocket.OPEN) { - this.send(client, - JSON.stringify({ - type: "replay_end", - count: result.events.length, - }), - ); - } - - logger.debug(`ws replay complete: fromSeq=${fromSeq} count=${result.events.length}`); - } - - /** - * Authenticate a solver connection and install a capability predicate. - */ - private async handleAuth(client: WebSocket, payload: Record) { - if (typeof payload.token === "string") { - await this.authenticateJwt(client, payload.token); - return; - } - const solver = typeof payload.solver === "string" ? payload.solver : ""; - const timestamp = payload.timestamp; - const signature = typeof payload.signature === "string" ? payload.signature : ""; - - if (!solver || !signature || typeof timestamp !== "number") { - this.send(client, JSON.stringify({ type: "auth_error", reason: "auth payload requires solver, timestamp, and signature" })); - return; - } - - const now = Math.floor(Date.now() / 1000); - const skew = Math.abs(now - timestamp); - if (skew > 300) { - this.send(client, JSON.stringify({ type: "auth_error", reason: "stale or future auth timestamp" })); - return; - } - - const solverRecord = await this.solversService.get(solver); - if (!solverRecord || !solverRecord.isActive) { - this.send(client, JSON.stringify({ type: "auth_error", reason: "solver not registered or inactive" })); - return; - } - - try { - verifyStellarSignature(solver, buildWsAuthMessage(solver, timestamp), signature); - } catch { - this.send(client, JSON.stringify({ type: "auth_error", reason: "invalid solver signature" })); - return; - } - - await this.installSolver(client, solverRecord, "signature"); - } - - /** - * Authenticates with a solver JWT from the SEP-10 flow (issue #455 / #442). - */ - private async authenticateJwt(client: WebSocket, token: string): Promise { - const claims = verifyHs256Jwt(token, this.jwtSecret); - if (!claims) { - this.send(client, JSON.stringify({ type: "auth_error", reason: "invalid or expired token" })); - return; - } - const solverRecord = await this.solversService.get(claims.sub); - if (!solverRecord || !solverRecord.isActive) { - this.send(client, JSON.stringify({ type: "auth_error", reason: "solver not registered or inactive" })); - return; - } - await this.installSolver(client, solverRecord, "jwt"); - } - - /** Binds a verified solver identity to the connection and sends its scoped snapshot. */ - private async installSolver( - client: WebSocket, - solverRecord: NonNullable>>, - method: "signature" | "jwt", - ): Promise { - const solver = solverRecord.address; - const predicate = buildMatchPredicate(solverRecord); - this.authenticatedSolver.set(client, solver); - const authFilter = this.subscribers.get(client); - this.subscribers.set(client, { - chains: authFilter?.chains ?? null, - solver: predicate, - wantAll: authFilter?.wantAll ?? false, - subscriptionCount: authFilter?.subscriptionCount ?? 0, - }); - const state = this.connections.get(client); - if (state) state.identity = solver; - const feedClient = this.feedClients.get(client); - if (feedClient) { - const current = this.feed.getFilter(feedClient); - if (current) { - this.feed.updateClientFilter(feedClient, { ...current, solver: predicate }); - } - } - - this.send(client, JSON.stringify({ type: "auth_ok", method })); - - try { - const eligible = this.intentIndex.getEligibleFor(solverRecord); - this.send(client, JSON.stringify({ - type: "eligible_snapshot", - intents: eligible, - count: eligible.length, - })); - } catch { - // Non-fatal — solver can fall back to GET /solvers/:address/eligible-intents. - } - - // Use the predicate's normalized capability lists: a partially-populated - // solver record must not throw here and take down the auth path. - logger.info( - `ws solver auth ok: address=${solver} chains=${predicate.supportedChains.join(",")} tokens=${predicate.supportedTokens.join(",")}`, - ); - } - - /** - * Update the capability predicate for all live connections authenticated as - * the given solver address. - */ - async updateSolverPredicate(solverAddress: string): Promise { - await this.feed.updateSolverPredicate(solverAddress); - const solverRecord = await this.solversService.get(solverAddress); - if (!solverRecord) return; - - const predicate = buildMatchPredicate(solverRecord); - for (const [client, filter] of this.subscribers) { - if (this.authenticatedSolver.get(client) === solverAddress && filter.solver !== null) { - this.subscribers.set(client, { ...filter, solver: predicate }); - } - } - - logger.debug(`ws solver predicate updated for ${solverAddress}`); - } - - /** - * Resolve the source chain for an event payload. - */ - private async getEventChain( - event: { type: string; [key: string]: unknown }, - ): Promise { - if (event.type === "intent_created") { - const intent = event.intent as { srcChain?: string } | undefined; - const chain = intent?.srcChain; - if (chain && (SUPPORTED_CHAINS as readonly string[]).includes(chain)) { - return chain as SupportedChain; - } - return null; - } - - const lookupTypes = new Set([ - "intent_accepted", - "intent_filled", - "intent_cancelled", - "intent_expired", - "intent_slashed", - "auction_price", - ]); - - if (lookupTypes.has(event.type)) { - const intentId = typeof event.intentId === "string" ? event.intentId : null; - if (!intentId) return null; - - try { - const intent = await this.intentsService.get(intentId); - if (intent && (SUPPORTED_CHAINS as readonly string[]).includes(intent.srcChain)) { - return intent.srcChain as SupportedChain; - } - } catch { - // Lookup failure is non-fatal — deliver to all subscribers. - } - return null; - } - - return null; - } - - /** - * Broadcast an event to every client on every replica (issue #454). - * - * The backplane assigns the global sequence number and hands the event - * back to each replica's {@link deliver}. In memory mode this resolves after - * local delivery (unchanged behaviour); in redis mode it resolves once the - * event is queued, so request handlers never wait on Redis. - */ - async broadcast(event: { type: string; [key: string]: unknown }): Promise { - await this.backplane.publish(event); - } - - /** Chains deliveries so async chain lookups cannot reorder events. */ - private enqueueDelivery(event: SequencedEvent): Promise { - const run = this.deliveryChain.then(() => this.deliver(event)); - this.deliveryChain = run.catch((err: Error) => { - logger.error(`ws delivery failed: ${err.message}`); - }); - return this.deliveryChain; - } - - /** - * Push a sequenced event into the replay buffer, then deliver it to every - * subscriber whose filter matches. - * - * For authenticated solvers without `all=true`, only intents matching their - * capability predicate are delivered. State-transition events (no inlined - * intent) are always delivered to authenticated subscribers. - * - * Side-effects: - * - Updates the intent index for `intent_created` (add) and terminal-state - * events (remove), keeping the capability index fresh without a rebuild. - */ - private async deliver(sequencedEvent: SequencedEvent): Promise { - const enqueuedAt = Date.now(); - const { seq, ...event } = sequencedEvent; - - // Update the capability index before delivery so a racing replay or - // eligible-intents call sees fresh state. - this.updateIndexForEvent(sequencedEvent); - - // Push into replay buffer before sending. - this.ringBuffer.push(sequencedEvent); - - logger.debug(`ws broadcast type=${event.type} seq=${seq} subscribers=${this.subscribers.size}`); - - // Resolve the chain once — shared across all subscriber checks. - const eventChain = await this.getEventChain(sequencedEvent); - - // Activity 1: Pass seq + event separately so encoding cache can serialize - this.deliverToMatchingSubscribers(seq, eventChain, event); - - try { - this.metricsService?.observeWsDelivery((Date.now() - enqueuedAt) / 1000); - } catch { - // Metrics must never break broadcasts. - } - } - - /** Keep the IntentCapabilityIndex in sync with broadcast events. */ - private updateIndexForEvent(event: { type: string; [key: string]: unknown }): void { - try { - if (event.type === "intent_created") { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const intent = (event as any).intent; - if (intent) this.intentIndex.addIntent(intent); - } else if ( - event.type === "intent_accepted" || - event.type === "intent_filled" || - event.type === "intent_cancelled" || - event.type === "intent_expired" || - event.type === "intent_slashed" - ) { - const intentId = typeof event.intentId === "string" ? event.intentId : null; - if (intentId) this.intentIndex.removeIntent(intentId); - } - } catch { - // Index update is best-effort — never break broadcasts. - } - } - - getAliveCount(): number { - let count = 0; - for (const [client, feedClient] of this.feedClients) { - if (feedClient.alive) count++; - } - return count; - } - - getSubscriberCount(): number { - return this.feed.connectionCount; - } - - /** Returns the current number of active WebSocket subscribers. */ - get subscriberCount(): number { - return this.feed.connectionCount; - } - - /** - * Check if the gateway is draining connections (Activity 2). - * Used by health indicators to flip readiness during shutdown. - */ - isDraining(): boolean { - return this.draining; - } - - private heartbeat() { - let terminated = 0; - for (const [client, feedClient] of this.feedClients) { - if (!feedClient.alive) { - client.terminate(); - this.removeSubscriber(client); - terminated++; - logger.debug( - `ws heartbeat terminated dead client (subscribers=${this.feed.connectionCount})`, - ); - continue; - } - - feedClient.alive = false; - if (client.readyState === WebSocket.OPEN) { - client.ping(); - } - } - this.lastHeartbeatTerminatedCount = terminated; - if (terminated > 0) { - logger.debug( - `ws heartbeat terminated ${terminated} dead client(s) (subscribers=${this.subscribers.size})`, - ); - } - continue; - } - - feedClient.alive = false; - if (client.readyState === WebSocket.OPEN) { - client.ping(); - } - } - this.lastHeartbeatTerminatedCount = terminated; - if (terminated > 0) { - logger.debug( - `ws heartbeat terminated ${terminated} dead client(s) (subscribers=${this.subscribers.size})`, - ); - } - } - - /** - * Returns the number of connections terminated in the most recent - * heartbeat cycle. Useful for presence stats and observability. - */ - getLastTerminatedCount(): number { - return this.lastHeartbeatTerminatedCount; - } - - /** - * Returns the number of connections that missed the last ping and are - * waiting to be terminated in the next heartbeat cycle ("zombies"). - */ - getZombieCount(): number { - let count = 0; - for (const client of this.subscribers.keys()) { - if (this.alive.get(client) === false) count++; - } - return count; - } - - async onModuleDestroy() { - // Activity 2: Graceful shutdown with connection draining - await this.startDraining(); - - if (this.heartbeatTimer) clearInterval(this.heartbeatTimer); - for (const [client, feedClient] of this.feedClients) { - this.removeSubscriber(client); - client.close(1001, "Server shutting down"); - } - } -} - -const HEARTBEAT_INTERVAL_MS = 30_000; - -/** - * WebSocket adapter that bridges a `ws` socket to the feed service's - * {@link FeedClient} interface (issue #433). - * - * `send` applies the per-connection backpressure policy (issue #455) and - * returns `false` only when the connection should be terminated. - */ -class WsFeedClient implements FeedClient { - alive = true; - readonly id: string; - readonly ip: string; - /** Number of `subscribe` messages this connection has sent (issue #476). */ - subscriptionCount = 0; - - constructor( - private readonly socket: WebSocket, - ip: string, - private readonly state: ConnectionState, - private readonly metricsService?: MetricsService, - ) { - this.id = randomUUID(); - this.ip = ip; - } - - send(payload: string): boolean { - const result = this.state.send(payload); - if (result === "dropped_oldest") { - this.metricsService?.wsOutboundDropped.inc(); - return true; - } - if (result === "disconnected") { - this.metricsService?.wsSlowConsumerDisconnects.inc(); - return false; - } - return true; - } - - close(): void { - this.state.close(); - if (this.socket.readyState === WebSocket.OPEN) { - this.socket.close(1001, "Server shutting down"); - } - } - - /** - * Begin graceful shutdown (Activity 2): notify all clients and close - * connections in batches within the termination grace period. - * - * Flow: - * 1. Set draining flag (rejects new connections) - * 2. Send server_draining event to all clients with resumeFrom seq - * 3. Close connections in batches with jittered delays - * 4. Background workers finish current batch (handled by IntentsSweeperService) - * - * Kubernetes terminationGracePeriodSeconds should be at least DRAIN_TIMEOUT_MS + 5s. - */ - async startDraining(): Promise { - if (this.draining) return; - - this.draining = true; - this.drainStartedAt = Date.now(); - - const drainTimeoutMs = parseInt(process.env.WS_DRAIN_TIMEOUT_MS ?? "25000", 10); - const currentSeq = this.backplane.health().lastSeq; - const clientCount = this.subscribers.size; - - logger.warn(`ws draining started: ${clientCount} clients, timeout=${drainTimeoutMs}ms`); - - // Notify all clients to reconnect with resume - const drainMessage = JSON.stringify({ - type: "server_draining", - resumeFrom: currentSeq, - reconnectAfterMs: this.jitter(1000, 5000), // Stagger reconnects - reason: "graceful_shutdown", - }); - - for (const [client] of this.subscribers) { - if (client.readyState === WebSocket.OPEN) { - try { - client.send(drainMessage); - } catch { - // Best effort notification - } - } - } - - // Close connections in batches to avoid thundering herd - const batchSize = Math.max(10, Math.ceil(clientCount / 10)); - const clients = Array.from(this.subscribers.keys()); - const batchDelayMs = Math.floor(drainTimeoutMs / Math.ceil(clientCount / batchSize)); - - for (let i = 0; i < clients.length; i += batchSize) { - const batch = clients.slice(i, i + batchSize); - - // Wait between batches - if (i > 0) { - await this.sleep(batchDelayMs); - } - - for (const client of batch) { - if (client.readyState === WebSocket.OPEN) { - client.close(1001, "Server draining"); - } - this.removeSubscriber(client); - } - - logger.info(`ws drain progress: ${Math.min(i + batchSize, clientCount)}/${clientCount} closed`); - } - - logger.warn(`ws draining complete: all clients closed`); - } - - /** - * Generate a jittered delay in [min, max] ms to stagger reconnects. - */ - private jitter(min: number, max: number): number { - return Math.floor(Math.random() * (max - min + 1)) + min; - } - - /** - * Promise-based sleep helper. - */ - private sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); - } -} diff --git a/src/intents/intents.module.ts b/src/intents/intents.module.ts index d31542a3..e69de29b 100644 --- a/src/intents/intents.module.ts +++ b/src/intents/intents.module.ts @@ -1,100 +0,0 @@ -import { Module, forwardRef } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; -import { IntentsService } from "./intents.service"; -import { IntentsController } from "./intents.controller"; -import { IntentsSseController } from "./intents-sse.controller"; -import { IntentsGateway } from "./intents.gateway"; -import { IntentsSweeperService } from "./intents-sweeper.service"; -import { IntentsMaintenanceJobs } from "./intents-maintenance.jobs"; -import { INTENTS_REPOSITORY, InMemoryIntentsRepository } from "./intents.repository"; -import { PrismaIntentsRepository } from "./prisma-intents.repository"; -import { IntentCapabilityIndex } from "./solver-intent-matcher"; -import { backplaneProvider } from "./backplane/backplane.factory"; -import { backplaneHealthIndicator } from "./backplane/backplane-health.provider"; -import { IntentFeedService } from "./feed/intent-feed.service"; -import { Backplane, WS_BACKPLANE } from "./backplane/backplane.types"; -import { SolversModule } from "../solvers/solvers.module"; -import { SolversService } from "../solvers/solvers.service"; -import { MetricsService } from "../metrics/metrics.service"; -import { RoutingModule } from "../routing/routing.module"; -import { TokensModule } from "../tokens/tokens.module"; -import { SorobanModule } from "../soroban/soroban.module"; -import { AppConfig } from "../config/configuration"; -import { PrismaService } from "../prisma/prisma.service"; -import { GovernanceModule } from "../governance/governance.module"; -import { AbuseModule } from "../abuse/abuse.module"; -import { SignatureNonceService } from "../common/signature-nonce.service"; -import { EvmSignatureVerifier } from "../common/evm-signature"; -import { AuctionTickerService } from "../auctions/auction-ticker.service"; -import { FillVerifierService } from "../soroban/fill-verifier.service"; - -@Module({ - // Both SolversModule and SorobanModule import IntentsModule back, so both - // edges of each cycle must be deferred — a bare import resolves to `undefined` - // when the peer module is still mid-initialization (AppModule reaches - // SorobanModule through HealthModule before IntentsModule has finished). - // `forwardRef` on the SorobanModule import mirrors the one in SorobanModule: - // the two modules need each other (ShadowService here, IntentsService there). - imports: [ - forwardRef(() => SolversModule), - RoutingModule, - TokensModule, - forwardRef(() => SorobanModule), - GovernanceModule, - ], - controllers: [IntentsController, IntentsSseController], - controllers: [IntentsController], - providers: [ - // Select the persistence adapter based on INTENTS_PERSISTENCE env var. - // INTENTS_PERSISTENCE=prisma → PrismaIntentsRepository (production/staging) - // INTENTS_PERSISTENCE=memory → InMemoryIntentsRepository (default, dev/test) - { - provide: INTENTS_REPOSITORY, - inject: [ConfigService, PrismaService], - useFactory: (config: ConfigService, prisma: PrismaService) => { - const adapter = process.env.INTENTS_PERSISTENCE ?? "memory"; - if (adapter === "prisma") { - return new PrismaIntentsRepository(prisma); - } - return new InMemoryIntentsRepository(); - }, - }, - IntentsService, - SignatureNonceService, - EvmSignatureVerifier, - AuctionTickerService, - FillVerifierService, - IntentCapabilityIndex, - backplaneProvider, - // IntentFeedService is provided via a factory so its optional constructor - // parameters are not resolved positionally by Nest's injector. - { - provide: IntentFeedService, - inject: [ - IntentsService, - SolversService, - IntentCapabilityIndex, - { token: MetricsService, optional: true }, - ConfigService, - { token: WS_BACKPLANE, optional: true }, - ], - useFactory: ( - intentsService: IntentsService, - solversService: SolversService, - intentIndex: IntentCapabilityIndex, - metricsService: MetricsService | undefined, - config: ConfigService, - backplane: Backplane | undefined, - ) => - new IntentFeedService(intentsService, solversService, intentIndex, metricsService, config, backplane), - }, - IntentsGateway, - backplaneHealthIndicator, - IntentsSweeperService, - IntentsMaintenanceJobs, - // Note: EventIngestionService is provided by SorobanModule (imported above) - // and exported from there — no re-declaration needed here. - ], - exports: [IntentsService, IntentsGateway, IntentCapabilityIndex, IntentFeedService], -}) -export class IntentsModule {} diff --git a/src/intents/intents.service.spec.ts b/src/intents/intents.service.spec.ts index 83d3470c..e69de29b 100644 --- a/src/intents/intents.service.spec.ts +++ b/src/intents/intents.service.spec.ts @@ -1,547 +0,0 @@ -import { Test, TestingModule } from "@nestjs/testing"; -import { ConfigService } from "@nestjs/config"; -import { Keypair } from "@stellar/stellar-sdk"; -import { AppConfig, CHAIN_FILL_WINDOW_DEFAULTS, DEFAULT_FILL_WINDOW_SECONDS } from "../config/configuration"; -import { StellarTxService } from "../soroban/stellar-tx.service"; -import { IntentsService } from "./intents.service"; -import { INTENTS_REPOSITORY, InMemoryIntentsRepository } from "./intents.repository"; -import { PrismaService } from "../prisma/prisma.service"; -import { ProtocolParamsService } from "../governance/params.service"; - -const VALID_CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; - -function fakeConfig(overrides: { onchainIntentsEnabled?: boolean; settlementContractId?: string } = {}) { - const values: Record = { - onchainIntentsEnabled: overrides.onchainIntentsEnabled ?? false, - "stellar.settlementContractId": overrides.settlementContractId ?? "", - }; - return { get: (path: string) => values[path] } as ConfigService; -} - -function fakeStellarTxService() { - return { invokeContract: jest.fn() } as unknown as jest.Mocked; -} - -function fakePrismaService(): PrismaService { - return { - intentAuditLog: { - create: jest.fn().mockResolvedValue({}), - findMany: jest.fn().mockResolvedValue([]), - }, - } as unknown as PrismaService; -} - -function fakeProtocolParamsService(): ProtocolParamsService { - return { - snapshotForChain: jest.fn().mockReturnValue({ - version: 0, - feeBps: 30, - deadlineSeconds: 1800, - fillWindowSeconds: 600, - capturedAt: new Date().toISOString(), - }), - getCurrent: jest.fn().mockReturnValue({ version: 0, feeBps: 30, chains: {}, maxExposureRatio: 0.05, slashAmount: "100000000", activeSinceLedger: 0, adoptedAt: new Date().toISOString() }), - getPending: jest.fn().mockReturnValue(null), - getHistory: jest.fn().mockReturnValue([]), - } as unknown as ProtocolParamsService; -} - -function makeService( - configOverrides: { onchainIntentsEnabled?: boolean; settlementContractId?: string } = {}, - stellarTx?: jest.Mocked, -) { - return new IntentsService( - new InMemoryIntentsRepository(), - fakeConfig(configOverrides), - stellarTx ?? fakeStellarTxService(), - fakePrismaService(), - fakeProtocolParamsService(), - ); -} - -function validCreateData() { - return { - user: Keypair.random().publicKey(), - srcChain: "ethereum" as const, - srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" as const }, - srcAmount: "1000000", - dstToken: { contract: VALID_CONTRACT_ID, symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: Math.floor(Date.now() / 1000) + 1800, - }; -} - -async function buildService( - configOverrides: { onchainIntentsEnabled?: boolean; settlementContractId?: string } = {}, - stellarTxService?: jest.Mocked, -): Promise { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - { - provide: INTENTS_REPOSITORY, - useClass: InMemoryIntentsRepository, - }, - { - provide: ConfigService, - useValue: fakeConfig(configOverrides), - }, - { - provide: StellarTxService, - useValue: stellarTxService ?? fakeStellarTxService(), - }, - { - provide: PrismaService, - useValue: fakePrismaService(), - }, - { - provide: ProtocolParamsService, - useValue: fakeProtocolParamsService(), - }, - IntentsService, - ], - }).compile(); - - return module.get(IntentsService); -} - -describe("IntentsService", () => { - let service: IntentsService; - - beforeEach(() => { - service = makeService(); - }); - - it("seeds 5 intents on construction", async () => { - expect(await service.getAll()).toHaveLength(5); - }); - - it("getAll returns intents sorted by createdAt descending", async () => { - const all = await service.getAll(); - for (let i = 1; i < all.length; i++) { - expect(all[i - 1].createdAt).toBeGreaterThanOrEqual(all[i].createdAt); - } - }); - - it("create adds an open intent with a generated id", async () => { - const before = (await service.getAll()).length; - const deadline = Math.floor(Date.now() / 1000) + 1800; - const intent = await service.create({ - user: "GTEST...0000", - srcChain: "ethereum", - srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline, - }); - - expect(intent.state).toBe("open"); - expect(intent.intentId).toBeTruthy(); - expect(intent.deadline).toBe(deadline); - expect(await service.getAll()).toHaveLength(before + 1); - }); - - it("create defaults deadline to now + 1800 when omitted", async () => { - const before = Math.floor(Date.now() / 1000); - const intent = await service.create({ - user: "GTEST...0000", - srcChain: "ethereum", - srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: undefined as unknown as number, - }); - - expect(intent.deadline).toBeGreaterThanOrEqual(before + 1800); - }); - - it("get returns undefined for an unknown id", async () => { - expect(await service.get("does-not-exist")).toBeUndefined(); - }); - - it("update mutates and returns the patched intent", async () => { - const [existing] = await service.getByState("open"); - const updated = await service.update(existing.intentId, { state: "accepted", solver: "SOLVER_X" }); - - expect(updated?.state).toBe("accepted"); - expect(updated?.solver).toBe("SOLVER_X"); - expect((await service.get(existing.intentId))?.state).toBe("accepted"); - }); - - it("update returns null for an unknown id", async () => { - expect(await service.update("does-not-exist", { state: "cancelled" })).toBeNull(); - }); - - it("getByUser is case-insensitive", async () => { - const [existing] = await service.getAll(); - const found = await service.getByUser(existing.user.toLowerCase()); - expect(found.some((i) => i.intentId === existing.intentId)).toBe(true); - }); - - it("getByState only returns intents in that state", async () => { - for (const intent of await service.getByState("filled")) { - expect(intent.state).toBe("filled"); - } - }); - - describe("acceptIfOpen", () => { - it("transitions an open intent to accepted and returns it", async () => { - const [open] = await service.getByState("open"); - const result = await service.acceptIfOpen(open.intentId, "SOLVER_X"); - - expect(result).not.toBeNull(); - expect(result!.state).toBe("accepted"); - expect(result!.solver).toBe("SOLVER_X"); - expect((await service.get(open.intentId))!.state).toBe("accepted"); - }); - - it("returns null for a non-existent intent", async () => { - expect(await service.acceptIfOpen("does-not-exist", "SOLVER_X")).toBeNull(); - }); - - it("returns null when the intent is already accepted", async () => { - const [accepted] = await service.getByState("accepted"); - expect(await service.acceptIfOpen(accepted.intentId, "SOLVER_X")).toBeNull(); - }); - - it("only the first caller wins under simulated concurrency", async () => { - const [open] = await service.getByState("open"); - const results = await Promise.all( - Array.from({ length: 10 }, (_, i) => - service.acceptIfOpen(open.intentId, `SOLVER_${i}`), - ), - ); - - const successes = results.filter((r) => r !== null); - expect(successes).toHaveLength(1); - expect(successes[0]!.state).toBe("accepted"); - }); - - // ----------------------------------------------------------------------- - // Per-chain fill-window tests (issue: chain-aware fill window) - // ----------------------------------------------------------------------- - - it("sets deadline to now + stellar fill window (120 s) for a stellar intent", async () => { - const now = Math.floor(Date.now() / 1000); - const intent = await service.create({ - user: "GTEST_STELLAR_CHAIN1", - srcChain: "stellar", - srcToken: { address: "native", symbol: "XLM", name: "Stellar Lumens", decimals: 7, chain: "stellar" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: now + 900, - }); - - const result = await service.acceptIfOpen(intent.intentId, "SOLVER_X"); - - expect(result).not.toBeNull(); - const expectedWindow = CHAIN_FILL_WINDOW_DEFAULTS["stellar"] ?? DEFAULT_FILL_WINDOW_SECONDS; - // Allow a 2-second tolerance for test execution time - expect(result!.deadline).toBeGreaterThanOrEqual(now + expectedWindow - 2); - expect(result!.deadline).toBeLessThanOrEqual(now + expectedWindow + 2); - }); - - it("sets deadline to now + ethereum fill window (1800 s) for an ethereum intent", async () => { - const now = Math.floor(Date.now() / 1000); - const intent = await service.create({ - user: "GTEST_ETHEREUM_CHAIN1", - srcChain: "ethereum", - srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: now + 3600, - }); - - const result = await service.acceptIfOpen(intent.intentId, "SOLVER_X"); - - expect(result).not.toBeNull(); - const expectedWindow = CHAIN_FILL_WINDOW_DEFAULTS["ethereum"] ?? DEFAULT_FILL_WINDOW_SECONDS; - // Allow a 2-second tolerance for test execution time - expect(result!.deadline).toBeGreaterThanOrEqual(now + expectedWindow - 2); - expect(result!.deadline).toBeLessThanOrEqual(now + expectedWindow + 2); - }); - - it("stellar and ethereum accepted intents get distinct (non-equal) fill deadlines", async () => { - const now = Math.floor(Date.now() / 1000); - - const stellarIntent = await service.create({ - user: "GTEST_STELLAR_DIFF1", - srcChain: "stellar", - srcToken: { address: "native", symbol: "XLM", name: "Stellar Lumens", decimals: 7, chain: "stellar" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: now + 900, - }); - const ethIntent = await service.create({ - user: "GTEST_ETHEREUM_DIFF1", - srcChain: "ethereum", - srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: now + 3600, - }); - - const stellarResult = await service.acceptIfOpen(stellarIntent.intentId, "SOLVER_STELLAR"); - const ethResult = await service.acceptIfOpen(ethIntent.intentId, "SOLVER_ETH"); - - expect(stellarResult).not.toBeNull(); - expect(ethResult).not.toBeNull(); - - // Ethereum solver gets a materially larger fill window than Stellar - expect(ethResult!.deadline).toBeGreaterThan(stellarResult!.deadline); - - // Confirm the windows match the config constants exactly (allowing 2 s clock drift) - const stellarWindow = CHAIN_FILL_WINDOW_DEFAULTS["stellar"] ?? DEFAULT_FILL_WINDOW_SECONDS; - const ethWindow = CHAIN_FILL_WINDOW_DEFAULTS["ethereum"] ?? DEFAULT_FILL_WINDOW_SECONDS; - expect(ethWindow).toBeGreaterThan(stellarWindow); // sanity-check on config - }); - - it("falls back to DEFAULT_FILL_WINDOW_SECONDS for an unknown chain", async () => { - const now = Math.floor(Date.now() / 1000); - const intent = await service.create({ - user: "GTEST_UNKNOWN_CHAIN01", - srcChain: "stellar", // create as valid chain, then patch for test - srcToken: { address: "native", symbol: "XLM", name: "Stellar Lumens", decimals: 7, chain: "stellar" }, - srcAmount: "1000000", - dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, - minDstAmount: "990000", - deadline: now + 3600, - }); - // Manually patch to an unknown chain to exercise the fallback - await service.update(intent.intentId, { srcChain: "unknown_chain" as never }); - - const result = await service.acceptIfOpen(intent.intentId, "SOLVER_X"); - - expect(result).not.toBeNull(); - expect(result!.deadline).toBeGreaterThanOrEqual(now + DEFAULT_FILL_WINDOW_SECONDS - 2); - expect(result!.deadline).toBeLessThanOrEqual(now + DEFAULT_FILL_WINDOW_SECONDS + 2); - }); - }); // end describe("acceptIfOpen") - - describe("fillIfAccepted", () => { - it("transitions an accepted intent to filled when solver matches", async () => { - const [accepted] = await service.getByState("accepted"); - const result = await service.fillIfAccepted(accepted.intentId, accepted.solver!, { - fillAmount: "100", - txHash: "test-hash", - filledAt: Math.floor(Date.now() / 1000), - }); - - expect(result).not.toBeNull(); - expect(result!.state).toBe("filled"); - expect(result!.fillAmount).toBe("100"); - }); - - it("returns null when solver does not match", async () => { - const [accepted] = await service.getByState("accepted"); - const result = await service.fillIfAccepted(accepted.intentId, "WRONG_SOLVER", { - fillAmount: "100", - }); - expect(result).toBeNull(); - }); - - it("returns null for a non-existent intent", async () => { - expect(await service.fillIfAccepted("nope", "SOLVER_X", {})).toBeNull(); - }); - - it("only the first caller wins under simulated concurrency", async () => { - const [accepted] = await service.getByState("accepted"); - const results = await Promise.all( - Array.from({ length: 10 }, () => - service.fillIfAccepted(accepted.intentId, accepted.solver!, { - fillAmount: "100", - txHash: "race-hash", - filledAt: Math.floor(Date.now() / 1000), - }), - ), - ); - - const successes = results.filter((r) => r !== null); - expect(successes).toHaveLength(1); - expect(successes[0]!.state).toBe("filled"); - }); - }); - - describe("on-chain registration (ONCHAIN_INTENTS_ENABLED)", () => { - it("stays fully in the repository when the flag is off, never touching StellarTxService", async () => { - const stellarTxService = fakeStellarTxService(); - const svc = makeService({ onchainIntentsEnabled: false }, stellarTxService); - - const intent = await svc.create(validCreateData()); - - expect(stellarTxService.invokeContract).not.toHaveBeenCalled(); - expect(await svc.get(intent.intentId)).toEqual(intent); - }); - - it("invokes the settlement contract and preserves the Intent shape when the flag is on", async () => { - const stellarTxService = fakeStellarTxService(); - stellarTxService.invokeContract.mockResolvedValue({ hash: "deadbeef", status: "SUCCESS" } as never); - const svc = makeService( - { onchainIntentsEnabled: true, settlementContractId: VALID_CONTRACT_ID }, - stellarTxService, - ); - - const data = validCreateData(); - const intent = await svc.create(data); - - expect(stellarTxService.invokeContract).toHaveBeenCalledTimes(1); - const call = stellarTxService.invokeContract.mock.calls[0][0]; - expect(call.contractId).toBe(VALID_CONTRACT_ID); - expect(call.method).toBe("create_intent"); - - // Response shape: the in-memory and on-chain paths return the same keys. - // usdValueAtCreate (#440) and paramsVersion (#500) are both stamped by - // persistNewIntent, so they appear on every newly created intent. - expect(Object.keys(intent).sort()).toEqual( - Object.keys({ - intentId: "", - user: "", - srcChain: "", - srcToken: "", - srcAmount: "", - dstToken: "", - minDstAmount: "", - state: "", - createdAt: 0, - deadline: 0, - paramsVersion: 0, - usdValueAtCreate: 0, - }).sort(), - ); - expect(await svc.get(intent.intentId)).toBeDefined(); - }); - - it("rejects with a clear error and does not create the intent when SETTLEMENT_CONTRACT_ID is unset", async () => { - const stellarTxService = fakeStellarTxService(); - const service = makeService({ onchainIntentsEnabled: true }, stellarTxService); - const before = (await service.getAll()).length; - - await expect(service.create(validCreateData())).rejects.toMatchObject({ - message: expect.stringContaining("SETTLEMENT_CONTRACT_ID"), - }); - expect(stellarTxService.invokeContract).not.toHaveBeenCalled(); - expect(await service.getAll()).toHaveLength(before); - }); - - it("rejects and does not create the intent when the on-chain call fails", async () => { - const stellarTxService = fakeStellarTxService(); - stellarTxService.invokeContract.mockRejectedValue(new Error("submission failed after 5 attempts")); - const svc = makeService( - { onchainIntentsEnabled: true, settlementContractId: VALID_CONTRACT_ID }, - stellarTxService, - ); - const before = (await svc.getAll()).length; - - await expect(svc.create(validCreateData())).rejects.toThrow(/settlement contract/i); - expect(await svc.getAll()).toHaveLength(before); - }); - }); - - // --------------------------------------------------------------------------- - // Audit trail (issue #217 / #62) - // --------------------------------------------------------------------------- - - describe("appendAuditEntry / getAuditLog", () => { - it("returns an empty array for an intent with no audit entries", () => { - expect(service.getAuditLog("no-such-intent")).toEqual([]); - }); - - it("appends a single entry and getAuditLog returns it", () => { - service.appendAuditEntry("intent-1", "cancelled", "USER_ADDR", "user cancelled"); - const log = service.getAuditLog("intent-1"); - expect(log).toHaveLength(1); - expect(log[0]).toMatchObject({ - toState: "cancelled", - actor: "USER_ADDR", - reason: "user cancelled", - }); - expect(log[0].timestamp).toBeTruthy(); // ISO timestamp - }); - - it("appends multiple entries in order and getAuditLog returns oldest-first", async () => { - service.appendAuditEntry("intent-2", "accepted", "SOLVER_A", "solver accepted"); - await new Promise((r) => setTimeout(r, 5)); // small gap so timestamps differ - service.appendAuditEntry("intent-2", "filled", "SOLVER_A", "solver filled"); - - const log = service.getAuditLog("intent-2"); - expect(log).toHaveLength(2); - expect(log[0].toState).toBe("accepted"); - expect(log[1].toState).toBe("filled"); - }); - - it("stores optional metadata in the entry", () => { - service.appendAuditEntry("intent-3", "expired", "system", "deadline passed", { - deadline: 1234567890, - sweepedAt: 1234567900, - }); - const log = service.getAuditLog("intent-3"); - expect(log[0].metadata).toEqual({ deadline: 1234567890, sweepedAt: 1234567900 }); - }); - - it("does not mix entries across different intentIds", () => { - service.appendAuditEntry("intent-A", "cancelled", "USER_A", "cancel A"); - service.appendAuditEntry("intent-B", "expired", "system", "expire B"); - - expect(service.getAuditLog("intent-A")).toHaveLength(1); - expect(service.getAuditLog("intent-B")).toHaveLength(1); - expect(service.getAuditLog("intent-A")[0].toState).toBe("cancelled"); - expect(service.getAuditLog("intent-B")[0].toState).toBe("expired"); - }); - - it("fires a DB write via PrismaService on each append (non-blocking)", async () => { - const prismaService = { - intentAuditLog: { - create: jest.fn().mockResolvedValue({}), - findMany: jest.fn().mockResolvedValue([]), - }, - } as unknown as PrismaService; - const svc = new IntentsService(new InMemoryIntentsRepository(), fakeConfig(), fakeStellarTxService(), prismaService, fakeProtocolParamsService()); - - svc.appendAuditEntry("intent-db", "slashed", "system", "missed fill", { foo: "bar" }); - - // The DB write is fire-and-forget — wait one tick for the promise chain - await new Promise((r) => setImmediate(r)); - - const mockPrisma = prismaService as unknown as { - intentAuditLog: { create: jest.Mock }; - }; - expect(mockPrisma.intentAuditLog.create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - intentId: "intent-db", - toState: "slashed", - actor: "system", - reason: "missed fill", - }), - }), - ); - }); - - it("does NOT throw when the DB write fails — logs an error but returns normally", async () => { - const prismaService = { - intentAuditLog: { - create: jest.fn().mockRejectedValue(new Error("DB is down")), - findMany: jest.fn().mockResolvedValue([]), - }, - } as unknown as PrismaService; - const svc = new IntentsService(new InMemoryIntentsRepository(), fakeConfig(), fakeStellarTxService(), prismaService, fakeProtocolParamsService()); - - // Should not throw synchronously - expect(() => - svc.appendAuditEntry("intent-fail", "expired", "system", "deadline"), - ).not.toThrow(); - - // In-memory log still has the entry - expect(svc.getAuditLog("intent-fail")).toHaveLength(1); - - // Wait for the rejected promise — should not propagate - await new Promise((r) => setImmediate(r)); - // No unhandled rejection here (jest would fail the test if one occurred) - }); - }); -}); diff --git a/src/intents/intents.unit-of-work.spec.ts b/src/intents/intents.unit-of-work.spec.ts new file mode 100644 index 00000000..bc72b12f --- /dev/null +++ b/src/intents/intents.unit-of-work.spec.ts @@ -0,0 +1,57 @@ +import { PrismaService } from "../prisma/prisma.service"; +import { InMemoryOutboxRepository } from "../soroban/outbox.repository"; +import { PrismaOutboxRepository } from "../soroban/prisma-outbox.repository"; +import { InMemoryIntentsRepository } from "./intents.repository"; +import { InMemoryIntentsUnitOfWork, PrismaIntentsUnitOfWork } from "./intents.unit-of-work"; +import { PrismaIntentsRepository } from "./prisma-intents.repository"; + +describe("IntentsUnitOfWork (#396)", () => { + it("Prisma: runs the work in one $transaction with repositories bound to the tx client", async () => { + const txClient = { intent: {}, onchainOutbox: {} }; + const prisma = { + $transaction: jest.fn((fn: (tx: unknown) => Promise) => fn(txClient)), + } as unknown as PrismaService; + + const result = await new PrismaIntentsUnitOfWork(prisma).run(async ({ intents, outbox }) => { + expect(intents).toBeInstanceOf(PrismaIntentsRepository); + expect(outbox).toBeInstanceOf(PrismaOutboxRepository); + expect((intents as unknown as { prisma: unknown }).prisma).toBe(txClient); + expect((outbox as unknown as { prisma: unknown }).prisma).toBe(txClient); + return "ok"; + }); + + expect(result).toBe("ok"); + expect(prisma.$transaction).toHaveBeenCalledTimes(1); + }); + + it("Prisma: a throw inside the work propagates (so $transaction rolls back)", async () => { + const prisma = { + $transaction: jest.fn((fn: (tx: unknown) => Promise) => fn({})), + } as unknown as PrismaService; + await expect( + new PrismaIntentsUnitOfWork(prisma).run(async () => { + throw new Error("rollback"); + }), + ).rejects.toThrow("rollback"); + }); + + it("in-memory: commits buffered outbox rows only when the work succeeds", async () => { + const outbox = new InMemoryOutboxRepository(); + const uow = new InMemoryIntentsUnitOfWork(new InMemoryIntentsRepository(), outbox); + const entry = { intentId: "i1", operation: "create_intent" as const, payload: {} }; + + await expect( + uow.run(async (tx) => { + await tx.outbox.enqueue(entry); + throw new Error("intent write failed"); + }), + ).rejects.toThrow(); + expect(await outbox.findByIntent("i1")).toEqual([]); + + await uow.run(async (tx) => { + await tx.outbox.enqueue(entry); + expect(await outbox.findByIntent("i1")).toEqual([]); // not visible until commit + }); + expect(await outbox.findByIntent("i1")).toHaveLength(1); + }); +}); diff --git a/src/intents/intents.unit-of-work.ts b/src/intents/intents.unit-of-work.ts new file mode 100644 index 00000000..7e9fd02b --- /dev/null +++ b/src/intents/intents.unit-of-work.ts @@ -0,0 +1,73 @@ +import { PrismaService } from "../prisma/prisma.service"; +import { IIntentsRepository } from "./intents.repository"; +import { PrismaIntentsRepository } from "./prisma-intents.repository"; +import { + InMemoryOutboxRepository, + IOutboxWriter, + NewOutboxEntry, + OutboxEntry, +} from "../soroban/outbox.repository"; +import { PrismaOutboxRepository } from "../soroban/prisma-outbox.repository"; + +/** Injection token for {@link IIntentsUnitOfWork}. */ +export const INTENTS_UNIT_OF_WORK = Symbol("INTENTS_UNIT_OF_WORK"); + +/** Repositories scoped to one unit of work. */ +export interface IntentsTransaction { + intents: IIntentsRepository; + outbox: IOutboxWriter; +} + +/** + * Runs an intent mutation and its outbox rows as one atomic unit (issue #396), + * so the database can never record a state change whose on-chain write was + * silently dropped, or vice versa. + */ +export interface IIntentsUnitOfWork { + run(work: (tx: IntentsTransaction) => Promise): Promise; +} + +/** + * Prisma adapter: one interactive `$transaction` — the intent write and the + * `onchain_outbox` insert commit or roll back together. + */ +export class PrismaIntentsUnitOfWork implements IIntentsUnitOfWork { + constructor(private readonly prisma: PrismaService) {} + + run(work: (tx: IntentsTransaction) => Promise): Promise { + return this.prisma.$transaction((client) => + work({ + intents: new PrismaIntentsRepository(client), + outbox: new PrismaOutboxRepository(client), + }), + ); + } +} + +/** + * In-memory adapter (dev/test). Outbox rows are buffered and only committed + * when `work` resolves, so a throw after the enqueue never leaves an orphan + * row. Intent writes are not rolled back — callers order their work so the + * intent write is the last step that can fail (see IntentsService). + */ +export class InMemoryIntentsUnitOfWork implements IIntentsUnitOfWork { + constructor( + private readonly intents: IIntentsRepository, + private readonly outbox: InMemoryOutboxRepository, + ) {} + + async run(work: (tx: IntentsTransaction) => Promise): Promise { + const buffered: NewOutboxEntry[] = []; + const writer: IOutboxWriter = { + enqueue: async (entry) => { + buffered.push(entry); + // The row id is assigned on commit; callers inside the unit of work + // must not depend on it. + return { ...entry, id: "uncommitted" } as OutboxEntry; + }, + }; + const result = await work({ intents: this.intents, outbox: writer }); + for (const entry of buffered) await this.outbox.enqueue(entry); + return result; + } +} diff --git a/src/intents/prisma-intents.repository.ts b/src/intents/prisma-intents.repository.ts index 080d8b05..25a9e3be 100644 --- a/src/intents/prisma-intents.repository.ts +++ b/src/intents/prisma-intents.repository.ts @@ -4,6 +4,9 @@ import { IIntentsRepository, IntentSearchQuery, IntentSearchResult } from "./int import { Intent, IntentState, StellarToken, TokenInfo } from "./intents.types"; import { IntentState as PrismaIntentState, Prisma } from "@prisma/client"; +/** PrismaService, or the client handed to a `$transaction` callback (issue #396). */ +export type IntentsPrismaClient = PrismaService | Prisma.TransactionClient; + /** * Prisma-backed implementation of IIntentsRepository. * @@ -18,7 +21,7 @@ import { IntentState as PrismaIntentState, Prisma } from "@prisma/client"; */ @Injectable() export class PrismaIntentsRepository implements IIntentsRepository { - constructor(private readonly prisma: PrismaService) {} + constructor(private readonly prisma: IntentsPrismaClient) {} async save(intent: Intent): Promise { const data = this.toDbData(intent); diff --git a/src/intents/slashes.controller.spec.ts b/src/intents/slashes.controller.spec.ts new file mode 100644 index 00000000..2ecdcdc3 --- /dev/null +++ b/src/intents/slashes.controller.spec.ts @@ -0,0 +1,87 @@ +import { NotFoundException, UnauthorizedException } from "@nestjs/common"; +import { Keypair } from "@stellar/stellar-sdk"; +import { buildFillProofMessage } from "../common/stellar-signature"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { AdminSlashesController, SlashesController } from "./slashes.controller"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +describe("SlashesController (#397)", () => { + const pipeline = { + getByIntent: jest.fn(), + cancelByFillProof: jest.fn().mockResolvedValue({ state: "cancelled" }), + cancelByAdmin: jest.fn().mockResolvedValue({ state: "cancelled" }), + list: jest.fn().mockResolvedValue([{ intentId: "i1" }]), + }; + const controller = new SlashesController(pipeline as unknown as SlashingPipelineService); + const solver = Keypair.random(); + const txHash = "AB".repeat(32); + + it("returns a slash or 404", async () => { + pipeline.getByIntent.mockResolvedValueOnce({ intentId: "i1" }); + await expect(controller.get("i1")).resolves.toEqual({ intentId: "i1" }); + pipeline.getByIntent.mockResolvedValueOnce(undefined); + await expect(controller.get("i2")).rejects.toBeInstanceOf(NotFoundException); + }); + + it("verifies the solver's signature over the lowercased tx hash before cancelling", async () => { + const message = buildFillProofMessage("i1", solver.publicKey(), txHash.toLowerCase()); + const signature = solver.sign(Buffer.from(message)).toString("base64"); + + await controller.fillProof("i1", { solver: solver.publicKey(), txHash, signature }); + expect(pipeline.cancelByFillProof).toHaveBeenCalledWith("i1", solver.publicKey(), txHash.toLowerCase()); + }); + + it("rejects a fill proof signed by someone else", async () => { + pipeline.cancelByFillProof.mockClear(); + const signature = Keypair.random() + .sign(Buffer.from(buildFillProofMessage("i1", solver.publicKey(), txHash.toLowerCase()))) + .toString("base64"); + await expect(controller.fillProof("i1", { solver: solver.publicKey(), txHash, signature })).rejects.toBeInstanceOf( + UnauthorizedException, + ); + expect(pipeline.cancelByFillProof).not.toHaveBeenCalled(); + }); + + describe("AdminSlashesController", () => { + const audit = { record: jest.fn().mockResolvedValue(undefined) }; + const admin = new AdminSlashesController( + pipeline as unknown as SlashingPipelineService, + audit as unknown as AdminAuditService, + ); + const principal = { id: "ops-1", role: "admin" as const }; + + it("lists slashes", async () => { + await expect(admin.list({})).resolves.toEqual({ slashes: [{ intentId: "i1" }], count: 1 }); + expect(pipeline.list).toHaveBeenCalledWith(undefined, 50); + await admin.list({ state: "submitted", limit: 5 }); + expect(pipeline.list).toHaveBeenLastCalledWith("submitted", 5); + }); + + it("audits a cancel before applying it, attributing it to the authenticated admin", async () => { + const order: string[] = []; + audit.record.mockImplementationOnce(async () => void order.push("audit")); + pipeline.cancelByAdmin.mockImplementationOnce(async () => { + order.push("cancel"); + return { state: "cancelled" }; + }); + + await admin.cancel("i1", { note: "manual review" }, principal); + + expect(order).toEqual(["audit", "cancel"]); + expect(audit.record).toHaveBeenCalledWith({ + actor: "ops-1", + action: "slash.cancel", + target: "slash:i1", + reason: "manual review", + }); + expect(pipeline.cancelByAdmin).toHaveBeenCalledWith("i1", "ops-1", "manual review"); + }); + + it("does not cancel when the audit write fails", async () => { + pipeline.cancelByAdmin.mockClear(); + audit.record.mockRejectedValueOnce(new Error("audit down")); + await expect(admin.cancel("i1", { note: "manual review" }, principal)).rejects.toThrow("audit down"); + expect(pipeline.cancelByAdmin).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/intents/slashes.controller.ts b/src/intents/slashes.controller.ts new file mode 100644 index 00000000..2c86d22f --- /dev/null +++ b/src/intents/slashes.controller.ts @@ -0,0 +1,86 @@ +import { Body, Controller, Get, NotFoundException, Param, Post, Query, UseGuards } from "@nestjs/common"; +import { + ApiConflictResponse, + ApiForbiddenResponse, + ApiHeader, + ApiNotFoundResponse, + ApiTags, + ApiUnauthorizedResponse, + ApiUnprocessableEntityResponse, +} from "@nestjs/swagger"; +import { AdminGuard, CurrentAdmin, RequireAdminRole } from "../admin/admin.guard"; +import { AdminPrincipal } from "../admin/admin-auth"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { buildFillProofMessage, verifyStellarSignature } from "../common/stellar-signature"; +import { AdminCancelSlashDto, FillProofDto, ListSlashesDto } from "./dto/slash.dto"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +/** + * Public surface of the slashing saga (issue #397): status lookup and the + * solver's fill-proof challenge. + */ +@ApiTags("slashes") +@Controller("api/v1/slashes") +export class SlashesController { + constructor(private readonly pipeline: SlashingPipelineService) {} + + @Get(":intentId") + @ApiNotFoundResponse({ description: "No slash recorded for this intent" }) + async get(@Param("intentId") intentId: string) { + const slash = await this.pipeline.getByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + return slash; + } + + @Post(":intentId/fill-proof") + @ApiUnauthorizedResponse({ description: "Signature invalid" }) + @ApiForbiddenResponse({ description: "Caller is not the slashed solver" }) + @ApiConflictResponse({ description: "Challenge window is over" }) + @ApiUnprocessableEntityResponse({ description: "Fill proof did not verify on-chain" }) + async fillProof(@Param("intentId") intentId: string, @Body() dto: FillProofDto) { + const txHash = dto.txHash.toLowerCase(); + verifyStellarSignature(dto.solver, buildFillProofMessage(intentId, dto.solver, txHash), dto.signature); + return this.pipeline.cancelByFillProof(intentId, dto.solver, txHash); + } +} + +/** + * Operator surface of the slashing saga (issue #397). Runbook: + * docs/runbooks/slash-cancellation.md. + */ +@ApiTags("admin") +@ApiHeader({ name: "x-admin-key", required: true }) +@Controller("api/v1/admin/slashes") +@UseGuards(AdminGuard) +@RequireAdminRole("admin") +export class AdminSlashesController { + constructor( + private readonly pipeline: SlashingPipelineService, + private readonly audit: AdminAuditService, + ) {} + + @Get() + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + async list(@Query() dto: ListSlashesDto) { + const slashes = await this.pipeline.list(dto.state, dto.limit ?? 50); + return { slashes, count: slashes.length }; + } + + /** Cancels a slash that has not been broadcast yet. Audited before it is applied. */ + @Post(":intentId/cancel") + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + @ApiConflictResponse({ description: "Slash already submitted, or submission in progress" }) + async cancel( + @Param("intentId") intentId: string, + @Body() dto: AdminCancelSlashDto, + @CurrentAdmin() admin: AdminPrincipal, + ) { + await this.audit.record({ + actor: admin.id, + action: "slash.cancel", + target: `slash:${intentId}`, + reason: dto.note, + }); + return this.pipeline.cancelByAdmin(intentId, admin.id, dto.note); + } +} diff --git a/src/intents/slashing-pipeline.service.spec.ts b/src/intents/slashing-pipeline.service.spec.ts new file mode 100644 index 00000000..ee9abac8 --- /dev/null +++ b/src/intents/slashing-pipeline.service.spec.ts @@ -0,0 +1,546 @@ +import { ConflictException, ForbiddenException, NotFoundException, UnprocessableEntityException } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { PrismaService } from "../prisma/prisma.service"; +import { ProtocolParamsService } from "../governance/params.service"; +import { InMemorySolversRepository } from "../solvers/in-memory-solvers.repository"; +import { InMemoryPendingSlashesRepository } from "../solvers/pending-slashes.repository"; +import { SolversService } from "../solvers/solvers.service"; +import { FillVerifierService } from "../soroban/fill-verifier.service"; +import { SlashResult, SolverRegistryService } from "../soroban/solver-registry.service"; +import { StellarTxService } from "../soroban/stellar-tx.service"; +import { TxConfirmationService } from "../soroban/tx-confirmation.service"; +import { IntentsGateway } from "./intents.gateway"; +import { InMemoryIntentsRepository } from "./intents.repository"; +import { IntentsService } from "./intents.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +const WINDOW = 600; +const SKEW = 30; +const MAX_ATTEMPTS = 3; +const T0 = 1_900_000_000; // detection time, unix seconds +const at = (seconds: number) => new Date((T0 + seconds) * 1000); + +const ok = (overrides: Partial = {}): SlashResult => ({ + submitted: true, + simulated: true, + dryRun: false, + failed: false, + txHash: "slash-tx", + detail: "submitted", + ...overrides, +}); + +describe("SlashingPipelineService (#397)", () => { + let slashes: InMemoryPendingSlashesRepository; + let intents: IntentsService; + let solvers: SolversService; + let gateway: { broadcast: jest.Mock }; + let registry: { slashSolver: jest.Mock> }; + let verifier: { findLandedFill: jest.Mock; verifyFillProof: jest.Mock }; + let confirmation: { check: jest.Mock }; + let metrics: { recordSlashTransition: jest.Mock }; + let pipeline: SlashingPipelineService; + let solver: string; + let intentId: string; + let fillDeadline: number; + + async function acceptedThenSlashedIntent(): Promise { + const created = await intents.create({ + user: Keypair.random().publicKey(), + srcChain: "stellar", + srcToken: { address: "native", symbol: "XLM", name: "Stellar Lumens", decimals: 7, chain: "stellar" }, + srcAmount: "1000000", + dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + deadline: T0 + 10_000, + }); + await intents.update(created.intentId, { state: "accepted", solver, deadline: fillDeadline }); + await intents.slashIfAccepted(created.intentId, { slashedAt: T0, slashReason: "missed" }); + await solvers.recordFailedFill(solver, created.intentId); + return created.intentId; + } + + const detect = () => + pipeline.detect({ intentId, solverAddress: solver, reason: "missed fill", fillDeadline, detectedAt: T0 }); + const state = async () => (await slashes.findByIntent(intentId))?.state; + const fillsFailed = async () => (await solvers.get(solver))?.fillsFailed; + + beforeEach(async () => { + slashes = new InMemoryPendingSlashesRepository(); + const repo = new InMemoryIntentsRepository(); + (repo as unknown as { store: Map }).store.clear(); + intents = new IntentsService( + repo, + { get: jest.fn().mockReturnValue(false) } as unknown as ConfigService, + {} as StellarTxService, + { intentAuditLog: { create: jest.fn().mockResolvedValue({}) } } as unknown as PrismaService, + undefined, + undefined, + { + snapshotForChain: jest.fn().mockReturnValue({ version: 0, feeBps: 30, deadlineSeconds: 1800, fillWindowSeconds: 600 }), + } as unknown as ProtocolParamsService, + ); + solvers = new SolversService(new InMemorySolversRepository()); + solver = Keypair.random().publicKey(); + await solvers.register({ + address: solver, + name: "Alpha", + bondAmount: "1000000", + isActive: true, + supportedChains: ["stellar"], + supportedTokens: ["XLM"], + avgFillTime: 30, + }); + gateway = { broadcast: jest.fn().mockResolvedValue(undefined) }; + registry = { slashSolver: jest.fn().mockResolvedValue(ok()) }; + verifier = { findLandedFill: jest.fn().mockResolvedValue(null), verifyFillProof: jest.fn() }; + confirmation = { check: jest.fn().mockResolvedValue({ status: "not_found" }) }; + metrics = { recordSlashTransition: jest.fn() }; + const slashing: AppConfig["slashing"] = { + challengeWindowSeconds: WINDOW, + clockSkewToleranceSeconds: SKEW, + maxSubmitAttempts: MAX_ATTEMPTS, + }; + pipeline = new SlashingPipelineService( + slashes, + intents, + gateway as unknown as IntentsGateway, + solvers, + registry as unknown as SolverRegistryService, + verifier as unknown as FillVerifierService, + confirmation as unknown as TxConfirmationService, + metrics as unknown as MetricsService, + { get: () => slashing } as unknown as ConfigService, + ); + fillDeadline = T0 - 5; + intentId = await acceptedThenSlashedIntent(); + }); + + afterEach(() => { + pipeline.onModuleDestroy(); + }); + + describe("detection", () => { + it("records the slash and opens a challenge window ending detectedAt + window", async () => { + const slash = await detect(); + expect(slash).toMatchObject({ state: "challenge_window", solverAddress: solver, fillDeadline }); + expect(slash.challengeEndsAt).toEqual(at(WINDOW)); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("detected"); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("challenge_window"); + expect(intents.getAuditLog(intentId).at(-1)?.reason).toMatch(/challenge window open/); + }); + + it("is exactly-once per intent: re-detection is a no-op", async () => { + await detect(); + const again = await detect(); + expect(again.state).toBe("challenge_window"); + expect(metrics.recordSlashTransition.mock.calls.filter(([s]) => s === "detected")).toHaveLength(1); + + await pipeline.processDue(at(WINDOW + 1)); + await detect(); + await pipeline.processDue(at(WINDOW + 2)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + }); + + it("resumes a row left in `detected` by a crash", async () => { + await slashes.createIfAbsent({ + intentId, + solverAddress: solver, + reason: "r", + fillDeadline, + detectedAt: at(0), + challengeEndsAt: at(WINDOW), + }); + await pipeline.processDue(at(1)); + expect(await state()).toBe("challenge_window"); + }); + }); + + describe("challenge window", () => { + it("does not submit before the window ends", async () => { + await detect(); + await pipeline.processDue(at(WINDOW - 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await state()).toBe("challenge_window"); + }); + + it("re-verifies, then submits once the window is over", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(verifier.findLandedFill).toHaveBeenCalledWith(intentId, fillDeadline, fillDeadline + SKEW, T0 + WINDOW); + expect(registry.slashSolver).toHaveBeenCalledWith({ solverAddress: solver, intentId, reason: "missed fill" }); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "submitted", + txHash: "slash-tx", + simulated: false, + }); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("submitted", "broadcast"); + }); + + it("dry-run / gated submit is recorded as simulated and not polled for confirmation", async () => { + registry.slashSolver.mockResolvedValueOnce(ok({ submitted: false, dryRun: true, txHash: undefined })); + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "submitted", simulated: true }); + await pipeline.processDue(at(WINDOW + 3600)); + expect(confirmation.check).not.toHaveBeenCalled(); + }); + }); + + describe("scenario: late fill", () => { + it("cancels when re-verification finds a fill that landed in time, and compensates", async () => { + await detect(); + verifier.findLandedFill.mockResolvedValueOnce({ txHash: "fill-tx", ledger: 9, closedAt: fillDeadline + SKEW }); + + await pipeline.processDue(at(WINDOW)); + + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "cancelled", + cancelReason: "fill_landed", + cancelledBy: "system", + fillTxHash: "fill-tx", + }); + expect(await fillsFailed()).toBe(0); + expect((await intents.get(intentId))?.state).toBe("filled"); + expect((await intents.get(intentId))?.txHash).toBe("fill-tx"); + expect(gateway.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ type: "intent_slash_cancelled", intentId, reason: "fill_landed" }), + ); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("cancelled", "fill_landed"); + }); + + it("solver fill-proof during the window cancels the slash", async () => { + await detect(); + verifier.verifyFillProof.mockResolvedValueOnce({ valid: true, fill: { txHash: "f", ledger: 1, closedAt: 1 } }); + + const cancelled = await pipeline.cancelByFillProof(intentId, solver, "f"); + + expect(verifier.verifyFillProof).toHaveBeenCalledWith("f", intentId, fillDeadline + SKEW); + expect(cancelled).toMatchObject({ state: "cancelled", cancelledBy: solver, fillTxHash: "f" }); + expect(await fillsFailed()).toBe(0); + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + }); + + it("rejects fill-proofs that do not verify, come from another solver, or arrive after the window", async () => { + await expect(pipeline.cancelByFillProof("nope", solver, "f")).rejects.toBeInstanceOf(NotFoundException); + await detect(); + await expect(pipeline.cancelByFillProof(intentId, Keypair.random().publicKey(), "f")).rejects.toBeInstanceOf( + ForbiddenException, + ); + verifier.verifyFillProof.mockResolvedValueOnce({ valid: false, reason: "closed too late" }); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf( + UnprocessableEntityException, + ); + + await pipeline.processDue(at(WINDOW)); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf(ConflictException); + }); + + it("reports a conflict if the slash moved on while the proof was being verified", async () => { + await detect(); + verifier.verifyFillProof.mockImplementationOnce(async () => { + await slashes.transition(intentId, ["challenge_window"], { state: "submitted" }); + return { valid: true, fill: { txHash: "f", ledger: 1, closedAt: 1 } }; + }); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf(ConflictException); + }); + }); + + describe("scenario: admin cancel", () => { + it("cancels during the window, compensates, and expires the intent", async () => { + await detect(); + const cancelled = await pipeline.cancelByAdmin(intentId, "ops@vortex", "event delayed, fill confirmed manually"); + expect(cancelled).toMatchObject({ + state: "cancelled", + cancelledBy: "ops@vortex", + cancelReason: "admin: event delayed, fill confirmed manually", + }); + expect(await fillsFailed()).toBe(0); + expect((await intents.get(intentId))?.state).toBe("expired"); + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + }); + + it("compensates at most once", async () => { + await detect(); + await pipeline.cancelByAdmin(intentId, "ops", "first cancel"); + await expect(pipeline.cancelByAdmin(intentId, "ops", "second cancel")).rejects.toBeInstanceOf(ConflictException); + expect(await fillsFailed()).toBe(0); + }); + + it("refuses once submitted, while a worker holds the lease, or for an unknown intent", async () => { + await expect(pipeline.cancelByAdmin("unknown", "ops", "why not")).rejects.toBeInstanceOf(NotFoundException); + + await detect(); + await slashes.claim(intentId, new Date(), new Date(Date.now() + 60_000)); + await expect(pipeline.cancelByAdmin(intentId, "ops", "mid submit")).rejects.toThrow(/submission in progress/); + + await slashes.transition(intentId, ["challenge_window"], { state: "submitted" }); + await expect(pipeline.cancelByAdmin(intentId, "ops", "too late")).rejects.toThrow(/state=submitted/); + }); + }); + + describe("scenario: RPC failure during submit", () => { + it("retries with backoff and does not submit blind when re-verification fails", async () => { + await detect(); + verifier.findLandedFill.mockRejectedValueOnce(new Error("RPC timeout")); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "challenge_window", attempts: 1, lastError: expect.stringContaining("RPC timeout") }); + expect(row!.nextAttemptAt).toEqual(new Date(at(WINDOW).getTime() + 5_000)); + + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); // still backing off + await pipeline.processDue(at(WINDOW + 5)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + expect(await state()).toBe("submitted"); + }); + + it("gives up after max attempts: cancels, compensates, and alerts", async () => { + await detect(); + registry.slashSolver.mockResolvedValue(ok({ submitted: false, failed: true, detail: "simulation failed" })); + const errorLog = jest.spyOn((pipeline as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + + let t = WINDOW; + for (let i = 0; i < MAX_ATTEMPTS; i++) { + await pipeline.processDue(at(t)); + t += 3600; + } + + expect(registry.slashSolver).toHaveBeenCalledTimes(MAX_ATTEMPTS); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "cancelled", + cancelReason: "submit_failed", + attempts: MAX_ATTEMPTS, + }); + expect(await fillsFailed()).toBe(0); + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("ALERT")); + }); + + it("an unexpected throw is retried, not fatal to the batch", async () => { + await detect(); + registry.slashSolver.mockRejectedValueOnce(new Error("kaboom")); + await pipeline.processDue(at(WINDOW)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "challenge_window", + lastError: expect.stringContaining("kaboom"), + }); + }); + }); + + describe("scenario: kill-switch pause (issue #477)", () => { + it("defers the slash without consuming attempts, then submits after resume", async () => { + await detect(); + registry.slashSolver.mockRejectedValue( + new KillSwitchActiveException({ + reasonCode: "INCIDENT", + reason: "paused", + scope: "operation", + chain: "stellar", + token: null, + operation: "slash", + } as never), + ); + + let t = WINDOW; + for (let i = 0; i < MAX_ATTEMPTS + 2; i++) { + await pipeline.processDue(at(t)); + t += 61; + } + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "challenge_window", attempts: 0, lastError: expect.stringContaining("kill-switch") }); + expect(await fillsFailed()).toBe(1); // not compensated: the slash is only paused + + registry.slashSolver.mockReset(); + registry.slashSolver.mockResolvedValue(ok()); + await pipeline.processDue(at(t + 61)); + expect(await state()).toBe("submitted"); + }); + }); + + describe("confirmation", () => { + beforeEach(async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + }); + + it("confirms on success", async () => { + confirmation.check.mockResolvedValueOnce({ status: "success", ledger: 1 }); + await pipeline.processDue(at(WINDOW + 10)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "confirmed", confirmedAt: at(WINDOW + 10) }); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("confirmed"); + }); + + it("waits while not found within the grace period", async () => { + await pipeline.processDue(at(WINDOW + 10)); + expect(await state()).toBe("submitted"); + }); + + it("re-opens for re-verification and resubmission when the tx failed or vanished", async () => { + confirmation.check.mockResolvedValueOnce({ status: "failed", ledger: 1 }); + await pipeline.processDue(at(WINDOW + 10)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "challenge_window", attempts: 1 }); + + await pipeline.processDue(at(WINDOW + 100)); + expect(registry.slashSolver).toHaveBeenCalledTimes(2); + expect(verifier.findLandedFill).toHaveBeenCalledTimes(2); + + await pipeline.processDue(at(WINDOW + 100 + 121)); // not found past grace + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "challenge_window", + lastError: expect.stringContaining("not found"), + }); + }); + + it("backs off when the confirmation lookup itself fails", async () => { + confirmation.check.mockRejectedValueOnce(new Error("RPC down")); + await pipeline.processDue(at(WINDOW + 10)); + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "submitted", lastError: expect.stringContaining("RPC down") }); + expect(row!.nextAttemptAt.getTime()).toBeGreaterThan(at(WINDOW + 10).getTime()); + }); + }); + + describe("edge cases", () => { + it("clock skew: judges timeliness against fillDeadline + tolerance, on chain time", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(verifier.findLandedFill.mock.calls[0][2]).toBe(fillDeadline + SKEW); + }); + + it("solver deregistered mid-window is still slashed (deregistration is not an escape hatch)", async () => { + await detect(); + await solvers.deregister(solver); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + expect(intents.getAuditLog(intentId).at(-1)?.metadata).toMatchObject({ solverActive: false }); + }); + + it("solver with no record is cancelled and compensated rather than submitted", async () => { + await detect(); + jest.spyOn(solvers, "get").mockResolvedValueOnce(undefined); + const rollback = jest.spyOn(solvers, "rollbackPenalty"); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "cancelled", cancelReason: "solver_not_found" }); + expect(rollback).toHaveBeenCalledWith(intentId, solver); + }); + + it("compensation survives a restart that lost the in-memory penalty record", async () => { + await detect(); + (solvers as unknown as { pendingPenalties: Map }).pendingPenalties.clear(); + await pipeline.cancelByAdmin(intentId, "ops", "after restart"); + expect(await fillsFailed()).toBe(0); + }); + + it("leaves the intent alone if it is no longer `slashed`", async () => { + await detect(); + await intents.update(intentId, { state: "filled" }); + await pipeline.cancelByAdmin(intentId, "ops", "already filled"); + expect((await intents.get(intentId))?.state).toBe("filled"); + }); + + it("skips rows another worker has leased, and overlapping passes", async () => { + await detect(); + await slashes.claim(intentId, at(0), at(WINDOW + 60)); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + + const claim = jest.spyOn(slashes, "claim").mockResolvedValueOnce(false); + await pipeline.processDue(at(WINDOW + 61)); + expect(claim).toHaveBeenCalled(); + expect(registry.slashSolver).not.toHaveBeenCalled(); + + let release!: () => void; + jest.spyOn(slashes, "findDue").mockImplementationOnce( + () => new Promise((resolve) => (release = () => resolve([]))), + ); + const first = pipeline.processDue(at(WINDOW + 62)); + await pipeline.processDue(at(WINDOW + 62)); // overlapping → no-op + release(); + await first; + }); + + it("exposes lookup and listing", async () => { + await detect(); + expect((await pipeline.getByIntent(intentId))?.intentId).toBe(intentId); + expect(await pipeline.list("challenge_window", 10)).toHaveLength(1); + expect(await pipeline.list("confirmed", 10)).toHaveLength(0); + }); + + it("runs on an interval and logs failures", () => { + jest.useFakeTimers(); + try { + const process = jest.spyOn(pipeline, "processDue").mockRejectedValue(new Error("tick failed")); + const errorLog = jest.spyOn((pipeline as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + pipeline.onModuleInit(); + jest.advanceTimersByTime(15_000); + expect(process).toHaveBeenCalledTimes(1); + return Promise.resolve().then(() => expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("tick failed"))); + } finally { + jest.useRealTimers(); + } + }); + }); + + describe("lost races (another worker or an admin won the conditional transition)", () => { + const loseNextTransition = () => jest.spyOn(slashes, "transition").mockResolvedValueOnce(null); + + it("detect returns the stored row if the window was already opened elsewhere", async () => { + loseNextTransition(); + const slash = await detect(); + expect(slash.state).toBe("detected"); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("challenge_window"); + }); + + it("does not record a submission it lost", async () => { + await detect(); + loseNextTransition(); + await pipeline.processDue(at(WINDOW)); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("submitted", expect.anything()); + }); + + it("does not record a confirmation or reopen it lost", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + + confirmation.check.mockResolvedValueOnce({ status: "success", ledger: 1 }); + loseNextTransition(); + await pipeline.processDue(at(WINDOW + 10)); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("confirmed"); + + confirmation.check.mockResolvedValueOnce({ status: "failed", ledger: 1 }); + loseNextTransition(); + await pipeline.processDue(at(WINDOW + 20)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + }); + + it("releases a broadcast row with no tx hash, and treats a missing submittedAt as past grace", async () => { + await detect(); + await slashes.transition(intentId, ["challenge_window"], { state: "submitted", simulated: false }); + await pipeline.processDue(at(WINDOW)); // no txHash → released, nothing checked + expect(confirmation.check).not.toHaveBeenCalled(); + + await slashes.transition(intentId, ["submitted"], { txHash: "h" }); + await pipeline.processDue(at(WINDOW + 1)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "challenge_window" }); + }); + + it("stringifies non-Error failures and defaults processDue's clock", async () => { + await slashes.createIfAbsent({ + intentId, solverAddress: solver, reason: "r", fillDeadline, detectedAt: new Date(0), challengeEndsAt: new Date(0), + }); + await slashes.transition(intentId, ["detected"], { state: "challenge_window" }); + verifier.findLandedFill.mockRejectedValueOnce("plain string"); + await pipeline.processDue(); + expect((await slashes.findByIntent(intentId))?.lastError).toContain("plain string"); + }); + }); +}); diff --git a/src/intents/slashing-pipeline.service.ts b/src/intents/slashing-pipeline.service.ts new file mode 100644 index 00000000..7bb8ba84 --- /dev/null +++ b/src/intents/slashing-pipeline.service.ts @@ -0,0 +1,459 @@ +import { + ConflictException, + ForbiddenException, + Inject, + Injectable, + Logger, + NotFoundException, + OnModuleDestroy, + OnModuleInit, + UnprocessableEntityException, +} from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { SolversService } from "../solvers/solvers.service"; +import { + IPendingSlashesRepository, + PENDING_SLASHES_REPOSITORY, + PendingSlash, + PendingSlashState, +} from "../solvers/pending-slashes.repository"; +import { FillVerifierService } from "../soroban/fill-verifier.service"; +import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { TxConfirmationService } from "../soroban/tx-confirmation.service"; +import { IntentsGateway } from "./intents.gateway"; +import { IntentsService } from "./intents.service"; + +const PROCESS_INTERVAL_MS = 15_000; +const PROCESS_BATCH_SIZE = 25; +/** Lease while one worker verifies/submits a slash; also blocks admin cancel mid-submit. */ +const LEASE_SECONDS = 120; +const BASE_BACKOFF_MS = 5_000; +const MAX_BACKOFF_MS = 10 * 60_000; +/** Re-check interval while a kill-switch pause blocks slashing. */ +const PAUSED_RECHECK_MS = 60_000; +/** How long a submitted slash may stay unseen by RPC before it is treated as dropped. */ +const SUBMIT_NOT_FOUND_GRACE_SECONDS = 120; + +/** Why a slash was cancelled — also the `reason` label on the metric. */ +export type SlashCancelReason = + | "fill_landed" + | "admin" + | "solver_not_found" + | "submit_failed"; + +/** Cancellable before broadcast only; after that the chain decides. */ +const CANCELLABLE: PendingSlashState[] = ["detected", "challenge_window"]; + +export interface DetectMissedFillInput { + intentId: string; + solverAddress: string; + reason: string; + /** The accepted intent's fill deadline, unix seconds. */ + fillDeadline: number; + /** Server time at detection, unix seconds. */ + detectedAt: number; +} + +/** + * Saga connecting sweeper detection to the on-chain slash (issue #397): + * + * detected → challenge_window → submitted → confirmed | cancelled + * + * - **Durable + exactly-once**: one `pending_slashes` row per intent (unique + * constraint). Re-detection is a no-op; every step is a conditional state + * transition, so a crash anywhere resumes from the stored state. + * - **Challenge window** (SLASH_CHALLENGE_WINDOW_SECONDS, default 10 min): the + * slash is not broadcast before it ends. During it a solver fill-proof or an + * admin cancels the slash. + * - **Re-verification**: when the window ends the chain is checked again for a + * fill that landed by `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` + * (ledger close time). If one landed, the slash is cancelled. If the check + * itself fails, the slash is retried later — never submitted blind. + * - **Compensation**: the sweeper's optimistic `recordFailedFill` is reverted + * via `SolversService.rollbackPenalty` on every cancellation, and the intent + * leaves `slashed` (→ `filled` for a proven fill, → `expired` otherwise). + * Compensation runs only on the winning `→ cancelled` transition, so it + * happens at most once. + */ +@Injectable() +export class SlashingPipelineService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(SlashingPipelineService.name); + private readonly settings: AppConfig["slashing"]; + private interval?: NodeJS.Timeout; + private running = false; + + constructor( + @Inject(PENDING_SLASHES_REPOSITORY) private readonly slashes: IPendingSlashesRepository, + private readonly intentsService: IntentsService, + private readonly intentsGateway: IntentsGateway, + private readonly solversService: SolversService, + private readonly solverRegistry: SolverRegistryService, + private readonly fillVerifier: FillVerifierService, + private readonly confirmation: TxConfirmationService, + private readonly metrics: MetricsService, + configService: ConfigService, + ) { + this.settings = configService.get("slashing", { infer: true }); + } + + onModuleInit() { + this.interval = setInterval(() => { + this.processDue().catch((err) => + this.logger.error(`[slashing] processing failed: ${errorMessage(err)}`), + ); + }, PROCESS_INTERVAL_MS); + this.interval.unref?.(); + } + + onModuleDestroy() { + if (this.interval) clearInterval(this.interval); + } + + /** + * Records a missed fill and opens its challenge window. Idempotent per + * intent: a second detection returns the existing slash unchanged. + */ + async detect(input: DetectMissedFillInput): Promise { + const detectedAt = new Date(input.detectedAt * 1000); + const { slash, created } = await this.slashes.createIfAbsent({ + intentId: input.intentId, + solverAddress: input.solverAddress, + reason: input.reason, + fillDeadline: input.fillDeadline, + detectedAt, + challengeEndsAt: new Date(detectedAt.getTime() + this.settings.challengeWindowSeconds * 1000), + }); + if (!created) { + this.logger.warn(`[slashing] intent=${input.intentId} already has a slash (state=${slash.state}); ignoring`); + return slash; + } + this.metrics.recordSlashTransition("detected"); + return (await this.openChallengeWindow(slash)) ?? slash; + } + + /** Runs one pass over every slash that needs action now. */ + async processDue(now: Date = new Date()): Promise { + if (this.running) return; + this.running = true; + try { + const due = await this.slashes.findDue(now, PROCESS_BATCH_SIZE); + for (const slash of due) { + const leaseUntil = new Date(now.getTime() + LEASE_SECONDS * 1000); + if (!(await this.slashes.claim(slash.intentId, now, leaseUntil))) continue; + try { + await this.step(slash, now); + } catch (err) { + await this.retryOrGiveUp(slash, `unexpected error: ${errorMessage(err)}`, now); + } + } + } finally { + this.running = false; + } + } + + /** + * Cancels a slash that has not been broadcast yet (admin action). + * @throws NotFoundException / ConflictException when there is nothing to cancel. + */ + async cancelByAdmin(intentId: string, actor: string, note: string): Promise { + const slash = await this.slashes.findByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + const cancelled = await this.cancel(slash, "admin", actor, { note }, new Date()); + if (!cancelled) { + throw new ConflictException( + `Slash for intent ${intentId} cannot be cancelled (state=${slash.state}` + + `${slash.lockedUntil ? ", submission in progress — retry shortly" : ""})`, + ); + } + return cancelled; + } + + /** + * Cancels a slash on a solver-supplied proof that its fill landed in time. + * @throws when the slash is not cancellable, the solver is not the slashed + * one, or the proof does not verify on-chain. + */ + async cancelByFillProof(intentId: string, solver: string, txHash: string): Promise { + const slash = await this.slashes.findByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + if (slash.solverAddress !== solver) { + throw new ForbiddenException("Only the slashed solver may submit a fill proof"); + } + if (!CANCELLABLE.includes(slash.state)) { + throw new ConflictException(`Slash for intent ${intentId} is ${slash.state}; the challenge window is over`); + } + + const proof = await this.fillVerifier.verifyFillProof(txHash, intentId, this.latestAcceptable(slash)); + if (!proof.valid) throw new UnprocessableEntityException(`Fill proof rejected: ${proof.reason}`); + + const cancelled = await this.cancel(slash, "fill_landed", solver, { fillTxHash: txHash }, new Date()); + if (!cancelled) { + throw new ConflictException(`Slash for intent ${intentId} changed state while verifying; retry`); + } + return cancelled; + } + + getByIntent(intentId: string): Promise { + return this.slashes.findByIntent(intentId); + } + + list(state: PendingSlashState | undefined, limit: number): Promise { + return this.slashes.list({ state, limit }); + } + + // ── saga steps ──────────────────────────────────────────────────────────── + + private async step(slash: PendingSlash, now: Date): Promise { + switch (slash.state) { + case "detected": + await this.openChallengeWindow(slash); + return; + case "challenge_window": + await this.verifyAndSubmit(slash, now); + return; + case "submitted": + await this.confirm(slash, now); + return; + // confirmed / cancelled are terminal and never returned by findDue. + } + } + + private async openChallengeWindow(slash: PendingSlash): Promise { + const opened = await this.slashes.transition(slash.intentId, ["detected"], { state: "challenge_window" }); + if (!opened) return null; + this.metrics.recordSlashTransition("challenge_window"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash pending: challenge window open", { + solver: slash.solverAddress, + challengeEndsAt: opened.challengeEndsAt.toISOString(), + }); + this.logger.log( + `[slashing] intent=${slash.intentId} solver=${slash.solverAddress} challenge window open until ` + + opened.challengeEndsAt.toISOString(), + ); + return opened; + } + + private async verifyAndSubmit(slash: PendingSlash, now: Date): Promise { + // 1. Re-verify: a fill that landed in time (late event, missed API call) + // must never be slashed. Failure to check is not evidence of absence. + let fill; + try { + fill = await this.fillVerifier.findLandedFill( + slash.intentId, + slash.fillDeadline, + this.latestAcceptable(slash), + Math.floor(now.getTime() / 1000), + ); + } catch (err) { + await this.retryOrGiveUp(slash, `fill re-verification failed: ${errorMessage(err)}`, now); + return; + } + if (fill) { + await this.cancel(slash, "fill_landed", "system", { fillTxHash: fill.txHash }, undefined); + return; + } + + // 2. Solver deregistered mid-window: deregistration must not be an escape + // hatch, so the slash proceeds. A solver with no record at all cannot + // be penalised by the registry — cancel and compensate. + const solver = await this.solversService.get(slash.solverAddress); + if (!solver) { + await this.cancel(slash, "solver_not_found", "system", {}, undefined); + return; + } + if (!solver.isActive) { + this.logger.warn( + `[slashing] solver=${slash.solverAddress} deregistered during the challenge window; slashing intent=${slash.intentId} anyway`, + ); + } + + // 3. Submit. The registry contract keys slashes by intent id, so a + // resubmission after a crash cannot double-slash on-chain. + let result; + try { + result = await this.solverRegistry.slashSolver({ + solverAddress: slash.solverAddress, + intentId: slash.intentId, + reason: slash.reason, + }); + } catch (err) { + if (!(err instanceof KillSwitchActiveException)) throw err; + // Issue #477 — a pause on `slash`/`onchain` defers the slash without + // consuming an attempt, so a long pause can never make the saga give up. + await this.slashes.transition(slash.intentId, ["challenge_window"], { + nextAttemptAt: new Date(now.getTime() + PAUSED_RECHECK_MS), + lastError: `paused by kill-switch: ${err.message}`, + }); + return; + } + if (result.failed) { + await this.retryOrGiveUp(slash, result.detail, now); + return; + } + + const submitted = await this.slashes.transition(slash.intentId, ["challenge_window"], { + state: "submitted", + txHash: result.txHash, + simulated: !result.submitted, + submittedAt: now, + nextAttemptAt: now, + lastError: undefined, + }); + if (!submitted) return; + this.metrics.recordSlashTransition("submitted", result.submitted ? "broadcast" : "simulated"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash submitted", { + solver: slash.solverAddress, + txHash: result.txHash, + simulated: !result.submitted, + solverActive: solver.isActive, + detail: result.detail, + }); + this.logger.log( + `[slashing] intent=${slash.intentId} solver=${slash.solverAddress} submitted ` + + `(${result.submitted ? `tx ${result.txHash}` : "simulated only"}): ${result.detail}`, + ); + } + + private async confirm(slash: PendingSlash, now: Date): Promise { + if (!slash.txHash) { + await this.slashes.transition(slash.intentId, ["submitted"], {}); + return; + } + let status; + try { + status = (await this.confirmation.check(slash.txHash)).status; + } catch (err) { + await this.slashes.transition(slash.intentId, ["submitted"], { + nextAttemptAt: this.backoff(slash.attempts, now), + lastError: `confirmation lookup failed: ${errorMessage(err)}`, + }); + return; + } + + if (status === "success") { + const confirmed = await this.slashes.transition(slash.intentId, ["submitted"], { + state: "confirmed", + confirmedAt: now, + }); + if (confirmed) { + this.metrics.recordSlashTransition("confirmed"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash confirmed on-chain", { + txHash: slash.txHash, + }); + } + return; + } + + const age = slash.submittedAt ? now.getTime() - slash.submittedAt.getTime() : Infinity; + if (status === "failed" || age > SUBMIT_NOT_FOUND_GRACE_SECONDS * 1000) { + // Back into the window state so the next pass re-verifies and resubmits. + const reopened = await this.slashes.transition(slash.intentId, ["submitted"], { + state: "challenge_window", + txHash: undefined, + }); + if (reopened) { + await this.retryOrGiveUp(reopened, `slash tx ${slash.txHash} ${status === "failed" ? "failed on-chain" : "not found"}`, now); + } + return; + } + + await this.slashes.transition(slash.intentId, ["submitted"], { + nextAttemptAt: new Date(now.getTime() + BASE_BACKOFF_MS), + }); + } + + private async retryOrGiveUp(slash: PendingSlash, error: string, now: Date): Promise { + const attempts = slash.attempts + 1; + if (attempts >= this.settings.maxSubmitAttempts) { + this.logger.error( + `[slashing] ALERT giving up on slash for intent=${slash.intentId} solver=${slash.solverAddress} ` + + `after ${attempts} attempts: ${error}. Cancelling and compensating — see docs/runbooks/slash-cancellation.md`, + ); + await this.cancel({ ...slash, attempts }, "submit_failed", "system", { lastError: error }, undefined); + return; + } + await this.slashes.transition(slash.intentId, [slash.state], { + attempts, + nextAttemptAt: this.backoff(attempts, now), + lastError: error, + }); + this.logger.warn( + `[slashing] intent=${slash.intentId} attempt ${attempts}/${this.settings.maxSubmitAttempts} failed: ${error}`, + ); + } + + /** + * Compensating transaction. Runs only if this call wins the `→ cancelled` + * transition, so the rollback happens at most once per slash. + * + * @param now pass for externally-triggered cancels so an in-flight submission + * (active lease) is never cancelled underneath the worker. + */ + private async cancel( + slash: PendingSlash, + reason: SlashCancelReason, + actor: string, + extra: { fillTxHash?: string; note?: string; lastError?: string }, + now: Date | undefined, + ): Promise { + const cancelled = await this.slashes.transition( + slash.intentId, + reason === "submit_failed" ? ["challenge_window", "submitted"] : CANCELLABLE, + { + state: "cancelled", + cancelledAt: new Date(), + cancelReason: extra.note ? `${reason}: ${extra.note}` : reason, + cancelledBy: actor, + fillTxHash: extra.fillTxHash, + lastError: extra.lastError, + attempts: slash.attempts, + }, + now, + ); + if (!cancelled) return null; + + await this.solversService.rollbackPenalty(slash.intentId, slash.solverAddress); + + const intent = await this.intentsService.get(slash.intentId); + if (intent?.state === "slashed") { + if (reason === "fill_landed") { + await this.intentsService.update(slash.intentId, { state: "filled", txHash: extra.fillTxHash }); + } else { + await this.intentsService.update(slash.intentId, { state: "expired" }); + } + } + const toState = reason === "fill_landed" ? "filled" : "expired"; + this.intentsService.appendAuditEntry(slash.intentId, toState, actor, `slash cancelled: ${reason}`, { + solver: slash.solverAddress, + fillTxHash: extra.fillTxHash, + note: extra.note, + }); + await this.intentsGateway.broadcast({ + type: "intent_slash_cancelled", + intentId: slash.intentId, + solver: slash.solverAddress, + reason, + }); + this.metrics.recordSlashTransition("cancelled", reason); + this.logger.warn( + `[slashing] slash for intent=${slash.intentId} solver=${slash.solverAddress} cancelled by ${actor}: ${reason}`, + ); + return cancelled; + } + + private latestAcceptable(slash: PendingSlash): number { + return slash.fillDeadline + this.settings.clockSkewToleranceSeconds; + } + + private backoff(attempts: number, now: Date): Date { + const delay = Math.min(BASE_BACKOFF_MS * 2 ** Math.max(0, attempts - 1), MAX_BACKOFF_MS); + return new Date(now.getTime() + delay); + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/solvers/pending-slashes.repository.spec.ts b/src/solvers/pending-slashes.repository.spec.ts new file mode 100644 index 00000000..995dbe71 --- /dev/null +++ b/src/solvers/pending-slashes.repository.spec.ts @@ -0,0 +1,127 @@ +import { InMemoryPendingSlashesRepository, NewPendingSlash } from "./pending-slashes.repository"; +import { PrismaPendingSlashesRepository } from "./prisma-pending-slashes.repository"; +import { PrismaService } from "../prisma/prisma.service"; + +const t = (s: number) => new Date(1_900_000_000_000 + s * 1000); +const input = (intentId = "i1"): NewPendingSlash => ({ + intentId, + solverAddress: "GSOLVER", + reason: "missed", + fillDeadline: 1_899_999_990, + detectedAt: t(0), + challengeEndsAt: t(600), +}); + +describe("InMemoryPendingSlashesRepository (#397)", () => { + let repo: InMemoryPendingSlashesRepository; + beforeEach(() => (repo = new InMemoryPendingSlashesRepository())); + + it("enforces one slash per intent", async () => { + const first = await repo.createIfAbsent(input()); + const second = await repo.createIfAbsent({ ...input(), reason: "dup" }); + expect(first.created).toBe(true); + expect(second).toMatchObject({ created: false, slash: { id: first.slash.id, reason: "missed" } }); + expect(await repo.findByIntent("nope")).toBeUndefined(); + }); + + it("finds due rows by state, window, backoff and lease", async () => { + await repo.createIfAbsent(input("detected")); + await repo.createIfAbsent(input("window")); + await repo.transition("window", ["detected"], { state: "challenge_window" }); + await repo.createIfAbsent(input("sim")); + await repo.transition("sim", ["detected"], { state: "submitted", simulated: true }); + await repo.createIfAbsent(input("live")); + await repo.transition("live", ["detected"], { state: "submitted", simulated: false, nextAttemptAt: t(0) }); + + expect((await repo.findDue(t(599), 10)).map((r) => r.intentId).sort()).toEqual(["detected", "live"]); + expect((await repo.findDue(t(600), 10)).map((r) => r.intentId).sort()).toEqual(["detected", "live", "window"]); + expect(await repo.findDue(t(600), 1)).toHaveLength(1); + + expect(await repo.claim("window", t(600), t(700))).toBe(true); + expect(await repo.claim("window", t(650), t(750))).toBe(false); + expect(await repo.claim("missing", t(650), t(750))).toBe(false); + expect((await repo.findDue(t(650), 10)).map((r) => r.intentId)).not.toContain("window"); + expect((await repo.findDue(t(701), 10)).map((r) => r.intentId)).toContain("window"); + }); + + it("transition is guarded by state and, when `now` is given, by the lease", async () => { + await repo.createIfAbsent(input()); + expect(await repo.transition("i1", ["submitted"], { state: "confirmed" })).toBeNull(); + expect(await repo.transition("missing", ["detected"], {})).toBeNull(); + + await repo.claim("i1", t(0), t(100)); + expect(await repo.transition("i1", ["detected"], { state: "cancelled" }, t(50))).toBeNull(); + const moved = await repo.transition("i1", ["detected"], { state: "challenge_window" }); + expect(moved).toMatchObject({ state: "challenge_window", lockedUntil: undefined }); + }); + + it("lists newest first with an optional state filter", async () => { + await repo.createIfAbsent(input("a")); + await repo.createIfAbsent({ ...input("b"), detectedAt: t(10) }); + expect((await repo.list({ limit: 10 })).map((r) => r.intentId)).toEqual(["b", "a"]); + expect(await repo.list({ state: "confirmed", limit: 10 })).toEqual([]); + }); +}); + +describe("PrismaPendingSlashesRepository (#397)", () => { + const row = { ...input(), id: "s1", state: "detected", attempts: 0, nextAttemptAt: t(0), lockedUntil: null, txHash: null, + simulated: false, submittedAt: null, confirmedAt: null, cancelledAt: null, cancelReason: null, cancelledBy: null, + fillTxHash: null, lastError: null, createdAt: t(0), updatedAt: t(0) }; + let pendingSlash: Record; + let repo: PrismaPendingSlashesRepository; + + beforeEach(() => { + pendingSlash = { + create: jest.fn().mockResolvedValue(row), + findUnique: jest.fn().mockResolvedValue(row), + findUniqueOrThrow: jest.fn().mockResolvedValue(row), + findMany: jest.fn().mockResolvedValue([row]), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }; + repo = new PrismaPendingSlashesRepository({ pendingSlash } as unknown as PrismaService); + }); + + it("maps a unique violation (P2002) to created=false and rethrows anything else", async () => { + expect(await repo.createIfAbsent(input())).toMatchObject({ created: true, slash: { id: "s1", lockedUntil: undefined } }); + pendingSlash.create.mockRejectedValueOnce(Object.assign(new Error("dup"), { code: "P2002" })); + expect(await repo.createIfAbsent(input())).toMatchObject({ created: false }); + pendingSlash.create.mockRejectedValueOnce(Object.assign(new Error("down"), { code: "P1001" })); + await expect(repo.createIfAbsent(input())).rejects.toThrow("down"); + }); + + it("reads by intent and lists with filters", async () => { + expect(await repo.findByIntent("i1")).toMatchObject({ intentId: "i1" }); + pendingSlash.findUnique.mockResolvedValueOnce(null); + expect(await repo.findByIntent("x")).toBeUndefined(); + await repo.list({ state: "submitted", limit: 5 }); + expect(pendingSlash.findMany).toHaveBeenLastCalledWith({ where: { state: "submitted" }, orderBy: { detectedAt: "desc" }, take: 5 }); + await repo.list({ limit: 5 }); + expect(pendingSlash.findMany).toHaveBeenLastCalledWith(expect.objectContaining({ where: undefined })); + }); + + it("queries due, unleased rows", async () => { + await repo.findDue(t(1), 3); + const args = pendingSlash.findMany.mock.calls[0][0]; + expect(JSON.stringify(args.where)).toContain("challenge_window"); + expect(args.take).toBe(3); + }); + + it("claims and transitions with conditional updates", async () => { + expect(await repo.claim("i1", t(0), t(100))).toBe(true); + expect(pendingSlash.updateMany).toHaveBeenLastCalledWith({ + where: { intentId: "i1", OR: [{ lockedUntil: null }, { lockedUntil: { lt: t(0) } }] }, + data: { lockedUntil: t(100) }, + }); + + await repo.transition("i1", ["detected"], { state: "challenge_window", txHash: undefined }, t(5)); + expect(pendingSlash.updateMany).toHaveBeenLastCalledWith({ + where: { intentId: "i1", state: { in: ["detected"] }, OR: [{ lockedUntil: null }, { lockedUntil: { lt: t(5) } }] }, + data: { state: "challenge_window", lockedUntil: null }, + }); + + pendingSlash.updateMany.mockResolvedValueOnce({ count: 0 }); + expect(await repo.transition("i1", ["detected"], {})).toBeNull(); + pendingSlash.findUnique.mockResolvedValueOnce(null); + expect(await repo.transition("i1", ["detected"], {})).toBeNull(); + }); +}); diff --git a/src/solvers/pending-slashes.repository.ts b/src/solvers/pending-slashes.repository.ts new file mode 100644 index 00000000..f7584214 --- /dev/null +++ b/src/solvers/pending-slashes.repository.ts @@ -0,0 +1,167 @@ +import { Injectable } from "@nestjs/common"; +import { v4 as uuidv4 } from "uuid"; + +/** Injection token for {@link IPendingSlashesRepository} (issue #397). */ +export const PENDING_SLASHES_REPOSITORY = Symbol("PENDING_SLASHES_REPOSITORY"); + +/** + * Slash saga states: + * + * detected ──▶ challenge_window ──(window over, re-verified)──▶ submitted ──▶ confirmed + * │ │ │ + * └───────────────┴──(fill proof / admin / give-up)──▶ cancelled ◀┘ (tx failed past retries) + */ +export const PENDING_SLASH_STATES = [ + "detected", + "challenge_window", + "submitted", + "confirmed", + "cancelled", +] as const; +export type PendingSlashState = (typeof PENDING_SLASH_STATES)[number]; + +export interface PendingSlash { + id: string; + intentId: string; + solverAddress: string; + reason: string; + state: PendingSlashState; + /** The fill deadline (unix seconds) the solver missed. */ + fillDeadline: number; + detectedAt: Date; + challengeEndsAt: Date; + attempts: number; + nextAttemptAt: Date; + lockedUntil?: Date; + txHash?: string; + /** Registry client simulated but did not broadcast (dry-run / gated submit). */ + simulated: boolean; + submittedAt?: Date; + confirmedAt?: Date; + cancelledAt?: Date; + cancelReason?: string; + cancelledBy?: string; + fillTxHash?: string; + lastError?: string; + createdAt: Date; + updatedAt: Date; +} + +export interface NewPendingSlash { + intentId: string; + solverAddress: string; + reason: string; + fillDeadline: number; + detectedAt: Date; + challengeEndsAt: Date; +} + +export type PendingSlashPatch = Partial< + Omit +>; + +export interface IPendingSlashesRepository { + /** + * Inserts a `detected` row unless one already exists for the intent — the + * unique constraint on intent_id is what makes slashing exactly-once. + */ + createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }>; + + findByIntent(intentId: string): Promise; + + list(filter: { state?: PendingSlashState; limit: number }): Promise; + + /** + * Rows the pipeline should act on now and that are not leased: + * `detected`; `challenge_window` past both challengeEndsAt and nextAttemptAt; + * non-simulated `submitted` past nextAttemptAt. + */ + findDue(now: Date, limit: number): Promise; + + /** Takes the processing lease if it is free or expired. */ + claim(intentId: string, now: Date, leaseUntil: Date): Promise; + + /** + * Conditional update: applies `patch` only if the row is in one of `from`, + * and — when `now` is given — only if no *other* holder's lease is active. + * Always clears the lease. Returns the updated row, or null if the guard failed. + */ + transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise; +} + +/** In-memory adapter (dev/test). Each method is atomic — none of them await. */ +@Injectable() +export class InMemoryPendingSlashesRepository implements IPendingSlashesRepository { + private readonly rows = new Map(); + + async createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }> { + const existing = this.rows.get(input.intentId); + if (existing) return { slash: { ...existing }, created: false }; + const now = new Date(); + const row: PendingSlash = { + ...input, + id: uuidv4(), + state: "detected", + attempts: 0, + nextAttemptAt: now, + simulated: false, + createdAt: now, + updatedAt: now, + }; + this.rows.set(row.intentId, row); + return { slash: { ...row }, created: true }; + } + + async findByIntent(intentId: string): Promise { + const row = this.rows.get(intentId); + return row ? { ...row } : undefined; + } + + async list(filter: { state?: PendingSlashState; limit: number }): Promise { + return [...this.rows.values()] + .filter((r) => !filter.state || r.state === filter.state) + .sort((a, b) => b.detectedAt.getTime() - a.detectedAt.getTime()) + .slice(0, filter.limit) + .map((r) => ({ ...r })); + } + + async findDue(now: Date, limit: number): Promise { + return [...this.rows.values()] + .filter((r) => !r.lockedUntil || r.lockedUntil < now) + .filter( + (r) => + r.state === "detected" || + (r.state === "challenge_window" && r.challengeEndsAt <= now && r.nextAttemptAt <= now) || + (r.state === "submitted" && !r.simulated && r.nextAttemptAt <= now), + ) + .sort((a, b) => a.detectedAt.getTime() - b.detectedAt.getTime()) + .slice(0, limit) + .map((r) => ({ ...r })); + } + + async claim(intentId: string, now: Date, leaseUntil: Date): Promise { + const row = this.rows.get(intentId); + if (!row || (row.lockedUntil && row.lockedUntil >= now)) return false; + row.lockedUntil = leaseUntil; + row.updatedAt = now; + return true; + } + + async transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise { + const row = this.rows.get(intentId); + if (!row || !from.includes(row.state)) return null; + if (now && row.lockedUntil && row.lockedUntil >= now) return null; + Object.assign(row, patch, { lockedUntil: undefined, updatedAt: new Date() }); + return { ...row }; + } +} diff --git a/src/solvers/prisma-pending-slashes.repository.ts b/src/solvers/prisma-pending-slashes.repository.ts new file mode 100644 index 00000000..a618c8f7 --- /dev/null +++ b/src/solvers/prisma-pending-slashes.repository.ts @@ -0,0 +1,127 @@ +import { Prisma, PendingSlash as PendingSlashRow } from "@prisma/client"; +import { PrismaService } from "../prisma/prisma.service"; +import { + IPendingSlashesRepository, + NewPendingSlash, + PendingSlash, + PendingSlashPatch, + PendingSlashState, +} from "./pending-slashes.repository"; + +/** + * Prisma adapter for the slashing saga (issue #397). Every state change is a + * single conditional `updateMany`, so concurrent pipeline workers and admin + * requests are arbitrated by the database. + */ +export class PrismaPendingSlashesRepository implements IPendingSlashesRepository { + constructor(private readonly prisma: PrismaService) {} + + async createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }> { + try { + const row = await this.prisma.pendingSlash.create({ data: { ...input, state: "detected" } }); + return { slash: fromRow(row), created: true }; + } catch (err) { + // P2002 = unique constraint violation on intent_id: already detected. + if ((err as Prisma.PrismaClientKnownRequestError).code !== "P2002") throw err; + const existing = await this.prisma.pendingSlash.findUniqueOrThrow({ + where: { intentId: input.intentId }, + }); + return { slash: fromRow(existing), created: false }; + } + } + + async findByIntent(intentId: string): Promise { + const row = await this.prisma.pendingSlash.findUnique({ where: { intentId } }); + return row ? fromRow(row) : undefined; + } + + async list(filter: { state?: PendingSlashState; limit: number }): Promise { + const rows = await this.prisma.pendingSlash.findMany({ + where: filter.state ? { state: filter.state } : undefined, + orderBy: { detectedAt: "desc" }, + take: filter.limit, + }); + return rows.map(fromRow); + } + + async findDue(now: Date, limit: number): Promise { + const rows = await this.prisma.pendingSlash.findMany({ + where: { + AND: [ + { OR: [{ lockedUntil: null }, { lockedUntil: { lt: now } }] }, + { + OR: [ + { state: "detected" }, + { state: "challenge_window", challengeEndsAt: { lte: now }, nextAttemptAt: { lte: now } }, + { state: "submitted", simulated: false, nextAttemptAt: { lte: now } }, + ], + }, + ], + }, + orderBy: { detectedAt: "asc" }, + take: limit, + }); + return rows.map(fromRow); + } + + async claim(intentId: string, now: Date, leaseUntil: Date): Promise { + const result = await this.prisma.pendingSlash.updateMany({ + where: { intentId, OR: [{ lockedUntil: null }, { lockedUntil: { lt: now } }] }, + data: { lockedUntil: leaseUntil }, + }); + return result.count > 0; + } + + async transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise { + const where: Prisma.PendingSlashWhereInput = { intentId, state: { in: from } }; + if (now) where.OR = [{ lockedUntil: null }, { lockedUntil: { lt: now } }]; + const result = await this.prisma.pendingSlash.updateMany({ + where, + data: { ...toData(patch), lockedUntil: null }, + }); + if (result.count === 0) return null; + const row = await this.prisma.pendingSlash.findUnique({ where: { intentId } }); + return row ? fromRow(row) : null; + } +} + +function toData(patch: PendingSlashPatch): Prisma.PendingSlashUpdateManyMutationInput { + const data: Prisma.PendingSlashUpdateManyMutationInput = {}; + for (const [key, value] of Object.entries(patch)) { + // undefined means "leave as is"; clearing a column is not needed by the saga. + if (value !== undefined) (data as Record)[key] = value; + } + return data; +} + +function fromRow(row: PendingSlashRow): PendingSlash { + return { + id: row.id, + intentId: row.intentId, + solverAddress: row.solverAddress, + reason: row.reason, + state: row.state as PendingSlashState, + fillDeadline: row.fillDeadline, + detectedAt: row.detectedAt, + challengeEndsAt: row.challengeEndsAt, + attempts: row.attempts, + nextAttemptAt: row.nextAttemptAt, + lockedUntil: row.lockedUntil ?? undefined, + txHash: row.txHash ?? undefined, + simulated: row.simulated, + submittedAt: row.submittedAt ?? undefined, + confirmedAt: row.confirmedAt ?? undefined, + cancelledAt: row.cancelledAt ?? undefined, + cancelReason: row.cancelReason ?? undefined, + cancelledBy: row.cancelledBy ?? undefined, + fillTxHash: row.fillTxHash ?? undefined, + lastError: row.lastError ?? undefined, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} diff --git a/src/solvers/solvers.service.spec.ts b/src/solvers/solvers.service.spec.ts index 9841c5ea..99383a34 100644 --- a/src/solvers/solvers.service.spec.ts +++ b/src/solvers/solvers.service.spec.ts @@ -98,4 +98,29 @@ describe("SolversService", () => { expect(solver?.isActive).toBe(false); expect((await service.get(ALPHA_ADDR))?.isActive).toBe(false); }); + + describe("rollbackPenalty durable fallback (#397)", () => { + it("reverts fillsFailed from the solver address when the in-memory penalty is gone", async () => { + await service.recordFailedFill(ALPHA_ADDR, "intent-x"); + const bumped = (await service.get(ALPHA_ADDR))!.fillsFailed; + (service as unknown as { pendingPenalties: Map }).pendingPenalties.clear(); + + const result = await service.rollbackPenalty("intent-x", ALPHA_ADDR); + expect(result?.fillsFailed).toBe(bumped - 1); + }); + + it("returns null for an unknown solver and keeps the legacy no-record behaviour without an address", async () => { + expect(await service.rollbackPenalty("intent-y", "GUNKNOWN")).toBeNull(); + expect(await service.rollbackPenalty("intent-y")).toBeNull(); + }); + + it("uses the in-memory record when present", async () => { + await service.recordFailedFill(ALPHA_ADDR, "intent-z"); + const bumped = (await service.get(ALPHA_ADDR))!.fillsFailed; + await service.rollbackPenalty("intent-z", ALPHA_ADDR); + expect((await service.get(ALPHA_ADDR))!.fillsFailed).toBe(bumped - 1); + // Second rollback is a no-op: the record is now "failed", not missing. + expect(await service.rollbackPenalty("intent-z", ALPHA_ADDR)).toBeNull(); + }); + }); }); diff --git a/src/solvers/solvers.service.ts b/src/solvers/solvers.service.ts index fc0a4db9..a940b1cc 100644 --- a/src/solvers/solvers.service.ts +++ b/src/solvers/solvers.service.ts @@ -356,9 +356,22 @@ export class SolversService { * investigate the discrepancy. * * @param intentId The intent whose slash submission failed. + * @param solverAddress Optional fallback for callers that track the + * penalty durably (the slashing saga, issue #397): when the in-memory + * pending entry is gone — e.g. lost in a restart — the fillsFailed + * increment is still reverted for this solver. Callers passing it must + * guarantee they compensate at most once per intent. */ - async rollbackPenalty(intentId: string): Promise { + async rollbackPenalty(intentId: string, solverAddress?: string): Promise { const penalty = this.pendingPenalties.get(intentId); + if (!penalty && solverAddress) { + const solver = await this.repo.findByAddress(solverAddress); + if (!solver) return null; + this.logger.warn( + `[penalty] rolled back without in-memory record: solver=${solverAddress} intent=${intentId}`, + ); + return this.repo.save({ ...solver, fillsFailed: Math.max(0, solver.fillsFailed - 1) }); + } if (!penalty || penalty.state !== "pending") { this.logger.warn( `rollbackPenalty called for intentId=${intentId} but no pending penalty found (state=${penalty?.state ?? "none"})`, diff --git a/src/soroban/fill-verifier.service.spec.ts b/src/soroban/fill-verifier.service.spec.ts index 0bd42e76..e69de29b 100644 --- a/src/soroban/fill-verifier.service.spec.ts +++ b/src/soroban/fill-verifier.service.spec.ts @@ -1,63 +0,0 @@ -import { Asset } from "@stellar/stellar-sdk"; -import { FillVerifierService } from "./fill-verifier.service"; -import { NETWORK_PASSPHRASES } from "../config/configuration"; -import { Intent } from "../intents/intents.types"; - -describe("FillVerifierService", () => { - const intent = { - intentId: "intent-123", - user: "GDESTINATION", - dstToken: { contract: Asset.native().contractId(NETWORK_PASSPHRASES.testnet), decimals: 7 }, - minDstAmount: "12000000", - } as Intent; - const config = { - get: (key: string) => key === "stellar.network" ? "testnet" : "https://horizon.test", - } as never; - const service = new FillVerifierService(config); - const originalFetch = global.fetch; - - afterEach(() => { global.fetch = originalFetch; }); - - it("uses the delivered amount for path payments before verifying the minimum", async () => { - global.fetch = jest.fn() - .mockResolvedValueOnce(new Response(JSON.stringify({ - successful: true, - memo_type: "text", - memo: intent.intentId, - _links: { operations: { href: "ignored" } }, - }), { status: 200 })) - .mockResolvedValueOnce(new Response(JSON.stringify({ _embedded: { records: [{ - type: "path_payment_strict_send", - to: intent.user, - asset_type: "native", - destination_amount: "1.3", - amount: "2.0", - }] } }), { status: 200 })); - - await expect(service.verify("a".repeat(64), intent)).resolves.toEqual({ - status: "verified", - deliveredAmount: "13000000", - operation: "path_payment_strict_send", - }); - }); - - it("keeps a transaction not yet indexed by Horizon retryable", async () => { - global.fetch = jest.fn().mockResolvedValueOnce(new Response("{}", { status: 404 })); - await expect(service.verify("b".repeat(64), intent)).resolves.toEqual({ - status: "pending", - reason: "not_indexed", - }); - }); - - it("rejects a successful transaction without the intent binding memo", async () => { - global.fetch = jest.fn().mockResolvedValueOnce(new Response(JSON.stringify({ - successful: true, - memo_type: "text", - memo: "another-intent", - }), { status: 200 })); - await expect(service.verify("c".repeat(64), intent)).resolves.toEqual({ - status: "rejected", - reason: "intent_memo_mismatch", - }); - }); -}); diff --git a/src/soroban/fill-verifier.service.ts b/src/soroban/fill-verifier.service.ts index c29041ca..e69de29b 100644 --- a/src/soroban/fill-verifier.service.ts +++ b/src/soroban/fill-verifier.service.ts @@ -1,89 +0,0 @@ -import { Injectable } from "@nestjs/common"; -import { ConfigService } from "@nestjs/config"; -import { Asset } from "@stellar/stellar-sdk"; -import { AppConfig, NETWORK_PASSPHRASES } from "../config/configuration"; -import { Intent } from "../intents/intents.types"; - -/** A structured independent verdict for a submitted Stellar fill transaction. */ -export type FillVerificationVerdict = - | { status: "verified"; deliveredAmount: string; operation: string } - | { status: "pending"; reason: "not_indexed" | "horizon_unavailable" } - | { status: "rejected"; reason: string }; - -type HorizonTransaction = { - successful?: boolean; - memo_type?: string; - memo?: string; - _links?: { operations?: { href?: string } }; -}; - -/** Independently checks Horizon's indexed Stellar transaction and payment operations. */ -@Injectable() -export class FillVerifierService { - constructor(private readonly config: ConfigService) {} - - /** - * Verify the transaction against the persisted intent. Unknown/indexing errors - * remain retryable; malformed or mismatched transactions are definitive. - */ - async verify(txHash: string, intent: Intent): Promise { - const base = this.config.get("stellar.horizonUrl", { infer: true }).replace(/\/$/, ""); - let transaction: HorizonTransaction; - try { - const response = await fetch(`${base}/transactions/${encodeURIComponent(txHash)}`); - if (response.status === 404) return { status: "pending", reason: "not_indexed" }; - if (!response.ok) return { status: "pending", reason: "horizon_unavailable" }; - transaction = (await response.json()) as HorizonTransaction; - } catch { - return { status: "pending", reason: "horizon_unavailable" }; - } - - if (transaction.successful !== true) return { status: "rejected", reason: "transaction_failed" }; - if (transaction.memo_type !== "text" || transaction.memo !== intent.intentId) { - return { status: "rejected", reason: "intent_memo_mismatch" }; - } - if (!transaction._links?.operations?.href) return { status: "rejected", reason: "operations_missing" }; - try { - const response = await fetch(`${base}/transactions/${encodeURIComponent(txHash)}/operations?limit=200&order=asc`); - if (!response.ok) return { status: "pending", reason: "horizon_unavailable" }; - const body = (await response.json()) as { _embedded?: { records?: Array> } }; - const operations = body._embedded?.records ?? []; - for (const operation of operations) { - const type = operation.type as string; - if (!["payment", "path_payment_strict_send", "path_payment_strict_receive"].includes(type)) continue; - if (operation.source_account && operation.source_account !== intent.solver) continue; - const destination = String(operation.to ?? ""); - const network = this.config.get("stellar.network", { infer: true }); - let assetContractId: string; - try { - assetContractId = operation.asset_type === "native" - ? Asset.native().contractId(NETWORK_PASSPHRASES[network]) - : new Asset(String(operation.asset_code ?? ""), String(operation.asset_issuer ?? "")) - .contractId(NETWORK_PASSPHRASES[network]); - } catch { - continue; - } - if (destination !== intent.user) continue; - // Horizon exposes classic assets by code/issuer. Soroban contract IDs - // require Soroban RPC event verification, and are never credited here. - if (assetContractId !== intent.dstToken.contract) continue; - const raw = type.startsWith("path_payment_") ? operation.destination_amount : operation.amount; - if (typeof raw !== "string" || !/^\d+(\.\d+)?$/.test(raw)) continue; - const delivered = decimalToBaseUnits(raw, intent.dstToken.decimals); - if (BigInt(delivered) < BigInt(intent.minDstAmount)) { - return { status: "rejected", reason: "insufficient_delivered_amount" }; - } - return { status: "verified", deliveredAmount: delivered, operation: type }; - } - return { status: "rejected", reason: "matching_payment_missing" }; - } catch { - return { status: "pending", reason: "horizon_unavailable" }; - } - } -} - -function decimalToBaseUnits(value: string, decimals: number): string { - const [whole, fraction = ""] = value.split("."); - if (fraction.length > decimals) throw new Error("Horizon amount precision exceeds token decimals"); - return (BigInt(whole) * 10n ** BigInt(decimals) + BigInt((fraction + "0".repeat(decimals)).slice(0, decimals) || "0")).toString(); -} diff --git a/src/soroban/outbox-admin.controller.spec.ts b/src/soroban/outbox-admin.controller.spec.ts new file mode 100644 index 00000000..26f32f85 --- /dev/null +++ b/src/soroban/outbox-admin.controller.spec.ts @@ -0,0 +1,34 @@ +import { NotFoundException } from "@nestjs/common"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { InMemoryOutboxRepository } from "./outbox.repository"; +import { OutboxAdminController } from "./outbox-admin.controller"; + +describe("OutboxAdminController (#396)", () => { + const principal = { id: "ops-1", role: "admin" as const }; + + it("audits and requeues dead rows, 404s everything else", async () => { + const outbox = new InMemoryOutboxRepository(); + const audit = { record: jest.fn().mockResolvedValue(undefined) }; + const controller = new OutboxAdminController(outbox, audit as unknown as AdminAuditService); + const row = await outbox.enqueue({ intentId: "i1", operation: "create_intent", payload: {} }); + const [claimed] = await outbox.claimDue(new Date(Date.now() + 1000), 1, new Date(Date.now() + 60_000)); + await outbox.markDead(claimed, "poison"); + + await expect(controller.requeue(row.id, principal)).resolves.toEqual({ id: row.id, status: "pending" }); + expect(audit.record).toHaveBeenCalledWith({ actor: "ops-1", action: "outbox.requeue", target: `outbox:${row.id}` }); + + await expect(controller.requeue(row.id, principal)).rejects.toBeInstanceOf(NotFoundException); + audit.record.mockClear(); + await expect(controller.requeue("not-a-number", principal)).rejects.toBeInstanceOf(NotFoundException); + expect(audit.record).not.toHaveBeenCalled(); + }); + + it("does not requeue when the audit write fails", async () => { + const outbox = new InMemoryOutboxRepository(); + const requeue = jest.spyOn(outbox, "requeueDead"); + const audit = { record: jest.fn().mockRejectedValue(new Error("audit down")) }; + const controller = new OutboxAdminController(outbox, audit as unknown as AdminAuditService); + await expect(controller.requeue("1", principal)).rejects.toThrow("audit down"); + expect(requeue).not.toHaveBeenCalled(); + }); +}); diff --git a/src/soroban/outbox-admin.controller.ts b/src/soroban/outbox-admin.controller.ts new file mode 100644 index 00000000..a3da1a62 --- /dev/null +++ b/src/soroban/outbox-admin.controller.ts @@ -0,0 +1,35 @@ +import { Controller, Inject, NotFoundException, Param, Post, UseGuards } from "@nestjs/common"; +import { ApiHeader, ApiNotFoundResponse, ApiTags, ApiUnauthorizedResponse } from "@nestjs/swagger"; +import { AdminGuard, CurrentAdmin, RequireAdminRole } from "../admin/admin.guard"; +import { AdminPrincipal } from "../admin/admin-auth"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { IOutboxRepository, OUTBOX_REPOSITORY } from "./outbox.repository"; + +/** + * Operator actions on the transactional outbox (issue #396). Registered in + * IntentsModule, which owns the OUTBOX_REPOSITORY binding. + */ +@ApiTags("admin") +@ApiHeader({ name: "x-admin-key", required: true }) +@Controller("api/v1/admin/outbox") +@UseGuards(AdminGuard) +@RequireAdminRole("admin") +export class OutboxAdminController { + constructor( + @Inject(OUTBOX_REPOSITORY) private readonly outbox: IOutboxRepository, + private readonly audit: AdminAuditService, + ) {} + + /** Moves a `dead` row back to `pending` with attempts reset, unblocking its intent. */ + @Post(":id/requeue") + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + @ApiNotFoundResponse({ description: "No dead outbox row with this id" }) + async requeue(@Param("id") id: string, @CurrentAdmin() admin: AdminPrincipal) { + if (!/^\d+$/.test(id)) throw new NotFoundException(`No dead outbox row with id ${id}`); + await this.audit.record({ actor: admin.id, action: "outbox.requeue", target: `outbox:${id}` }); + if (!(await this.outbox.requeueDead(id))) { + throw new NotFoundException(`No dead outbox row with id ${id}`); + } + return { id, status: "pending" }; + } +} diff --git a/src/soroban/outbox-operations.spec.ts b/src/soroban/outbox-operations.spec.ts new file mode 100644 index 00000000..53b2d847 --- /dev/null +++ b/src/soroban/outbox-operations.spec.ts @@ -0,0 +1,65 @@ +import { Keypair, scValToNative } from "@stellar/stellar-sdk"; +import { Intent } from "../intents/intents.types"; +import { + acceptIntentEntry, + buildOutboxInvocation, + cancelIntentEntry, + createIntentEntry, + fillIntentEntry, +} from "./outbox-operations"; +import { OutboxOperation } from "./outbox.repository"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; +const USER = Keypair.random().publicKey(); +const SOLVER = Keypair.random().publicKey(); + +const intent: Intent = { + intentId: "11111111-2222-3333-4444-555555555555", + user: USER, + srcChain: "ethereum", + srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, + srcAmount: "1000000", + dstToken: { contract: CONTRACT_ID, symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + state: "filled", + createdAt: 1, + deadline: 1_900_000_000, + solver: SOLVER, + fillAmount: "995000", + txHash: "ab".repeat(32), +}; + +const decode = (entry: { operation: OutboxOperation; payload: Record }) => + buildOutboxInvocation(entry, CONTRACT_ID).args.map((a) => scValToNative(a)); + +describe("outbox operations (#396)", () => { + it("encodes create_intent with bigint amounts and the deadline", () => { + const entry = createIntentEntry(intent); + expect(entry).toMatchObject({ intentId: intent.intentId, operation: "create_intent" }); + // Payload survives a JSON round-trip (JSONB column). + const roundTripped = { ...entry, payload: JSON.parse(JSON.stringify(entry.payload)) }; + const params = buildOutboxInvocation(roundTripped, CONTRACT_ID); + expect(params).toMatchObject({ contractId: CONTRACT_ID, method: "create_intent" }); + expect(decode(roundTripped)).toEqual([ + intent.intentId, USER, "ethereum", "0xabc", 1_000_000n, CONTRACT_ID, 990_000n, 1_900_000_000n, + ]); + }); + + it("encodes accept_intent, fill_intent and cancel_intent", () => { + expect(decode(acceptIntentEntry(intent))).toEqual([intent.intentId, SOLVER, 1_900_000_000n]); + expect(decode(fillIntentEntry(intent))).toEqual([intent.intentId, SOLVER, 995_000n, "ab".repeat(32)]); + expect(decode(fillIntentEntry({ ...intent, txHash: undefined }))[3]).toBe(""); + expect(decode(cancelIntentEntry(intent))).toEqual([intent.intentId, USER]); + }); + + it("rejects malformed payloads so they fail at enqueue time", () => { + expect(() => buildOutboxInvocation(acceptIntentEntry({ ...intent, solver: undefined }), CONTRACT_ID)).toThrow( + /missing a required address/, + ); + expect(() => buildOutboxInvocation(createIntentEntry({ ...intent, user: "nope" }), CONTRACT_ID)).toThrow(); + expect(() => buildOutboxInvocation(createIntentEntry({ ...intent, srcAmount: "1.5" }), CONTRACT_ID)).toThrow(); + expect(() => + buildOutboxInvocation({ operation: "bogus" as OutboxOperation, payload: {} }, CONTRACT_ID), + ).toThrow(/unknown outbox operation/); + }); +}); diff --git a/src/soroban/outbox-operations.ts b/src/soroban/outbox-operations.ts new file mode 100644 index 00000000..937f860d --- /dev/null +++ b/src/soroban/outbox-operations.ts @@ -0,0 +1,118 @@ +import { Address, nativeToScVal, xdr } from "@stellar/stellar-sdk"; +import type { Intent } from "../intents/intents.types"; +import type { NewOutboxEntry, OutboxOperation } from "./outbox.repository"; +import type { InvokeContractParams } from "./stellar-tx.service"; + +/** + * Payload builders and ScVal encoders for settlement-contract operations that + * flow through the outbox (issue #396). + * + * Payloads are plain JSON (bigint amounts as strings) so they survive the + * `payload JSONB` column; ScVal encoding happens at relay time. The contract + * method names follow docs/architecture/onchain-settlement.md and stay + * provisional until the settlement ADR (issue #19) fixes the interface. + */ + +export function createIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "create_intent", + payload: { + intentId: intent.intentId, + user: intent.user, + srcChain: intent.srcChain, + srcTokenAddress: intent.srcToken.address, + srcAmount: intent.srcAmount, + dstTokenContract: intent.dstToken.contract, + minDstAmount: intent.minDstAmount, + deadline: intent.deadline, + }, + }; +} + +export function acceptIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "accept_intent", + payload: { intentId: intent.intentId, solver: intent.solver, fillDeadline: intent.deadline }, + }; +} + +export function fillIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "fill_intent", + payload: { + intentId: intent.intentId, + solver: intent.solver, + fillAmount: intent.fillAmount, + fillTxHash: intent.txHash ?? "", + }, + }; +} + +export function cancelIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "cancel_intent", + payload: { intentId: intent.intentId, user: intent.user }, + }; +} + +/** + * Encodes an outbox payload as a contract invocation. + * + * @throws on a malformed payload (bad address, non-integer amount). Callers + * run this at enqueue time too, so bad input fails the HTTP request + * instead of becoming a poison row. + */ +export function buildOutboxInvocation( + entry: { operation: OutboxOperation; payload: Record }, + settlementContractId: string, +): InvokeContractParams { + const p = entry.payload; + let args: xdr.ScVal[]; + switch (entry.operation) { + case "create_intent": + args = [ + str(p.intentId), + address(p.user), + nativeToScVal(String(p.srcChain), { type: "symbol" }), + str(p.srcTokenAddress), + i128(p.srcAmount), + address(p.dstTokenContract), + i128(p.minDstAmount), + nativeToScVal(Number(p.deadline), { type: "u64" }), + ]; + break; + case "accept_intent": + args = [str(p.intentId), address(p.solver), nativeToScVal(Number(p.fillDeadline), { type: "u64" })]; + break; + case "fill_intent": + args = [str(p.intentId), address(p.solver), i128(p.fillAmount), str(p.fillTxHash)]; + break; + case "cancel_intent": + args = [str(p.intentId), address(p.user)]; + break; + default: { + const unknown: never = entry.operation; + throw new Error(`unknown outbox operation: ${String(unknown)}`); + } + } + return { contractId: settlementContractId, method: entry.operation, args }; +} + +function str(value: unknown): xdr.ScVal { + return nativeToScVal(String(value ?? ""), { type: "string" }); +} + +function address(value: unknown): xdr.ScVal { + if (typeof value !== "string" || value.length === 0) { + throw new Error("outbox payload is missing a required address"); + } + return new Address(value).toScVal(); +} + +function i128(value: unknown): xdr.ScVal { + return nativeToScVal(BigInt(String(value)), { type: "i128" }); +} diff --git a/src/soroban/outbox-relay.service.spec.ts b/src/soroban/outbox-relay.service.spec.ts new file mode 100644 index 00000000..9f3f874d --- /dev/null +++ b/src/soroban/outbox-relay.service.spec.ts @@ -0,0 +1,475 @@ +import { ConfigService } from "@nestjs/config"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { InMemoryIntentsRepository } from "../intents/intents.repository"; +import { InMemoryIntentsUnitOfWork } from "../intents/intents.unit-of-work"; +import { IntentsService } from "../intents/intents.service"; +import { PrismaService } from "../prisma/prisma.service"; +import { ProtocolParamsService } from "../governance/params.service"; +import { InMemoryOutboxRepository } from "./outbox.repository"; +import { createIntentEntry } from "./outbox-operations"; +import { OutboxRelayService } from "./outbox-relay.service"; +import { InvokeContractOptions, InvokeContractParams, StellarTxService } from "./stellar-tx.service"; +import { TxConfirmation, TxConfirmationService } from "./tx-confirmation.service"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; +const LEASE_SECONDS = 120; +const MAX_ATTEMPTS = 3; + +function intentFor(intentId: string) { + return { + intentId, + user: Keypair.random().publicKey(), + srcChain: "ethereum" as const, + srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" as const }, + srcAmount: "1000000", + dstToken: { contract: CONTRACT_ID, symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + state: "open" as const, + createdAt: 1, + deadline: 2_000_000_000, + }; +} + +function config(overrides: Partial = {}, settlementContractId = CONTRACT_ID) { + const outbox: AppConfig["outbox"] = { + relayEnabled: true, + relayIntervalMs: 1000, + batchSize: 10, + maxAttempts: MAX_ATTEMPTS, + leaseSeconds: LEASE_SECONDS, + ...overrides, + }; + return { + get: (key: string) => + key === "outbox" ? outbox : key === "stellar.settlementContractId" ? settlementContractId : undefined, + } as unknown as ConfigService; +} + +describe("OutboxRelayService (#396)", () => { + let outbox: InMemoryOutboxRepository; + let invokeContract: jest.Mock, [InvokeContractParams, InvokeContractOptions?]>; + let check: jest.Mock, [string]>; + let metrics: { recordOutboxOutcome: jest.Mock; setOutboxBacklog: jest.Mock }; + let relay: OutboxRelayService; + let now: Date; + + /** Default fake network: sign → beforeSubmit(hash) → accepted as PENDING. */ + function submitsAs(hash: string) { + invokeContract.mockImplementationOnce(async (_params, options) => { + await options?.beforeSubmit?.(hash); + return { hash, status: "PENDING", dryRun: false }; + }); + } + + function build(cfg = config()) { + relay = new OutboxRelayService( + outbox, + { invokeContract } as unknown as StellarTxService, + { check } as unknown as TxConfirmationService, + metrics as unknown as MetricsService, + cfg, + ); + } + + const later = (seconds: number) => new Date(now.getTime() + seconds * 1000); + + beforeEach(() => { + outbox = new InMemoryOutboxRepository(); + invokeContract = jest.fn(); + check = jest.fn().mockResolvedValue({ status: "not_found" }); + metrics = { recordOutboxOutcome: jest.fn(), setOutboxBacklog: jest.fn() }; + now = new Date(Date.now() + 1000); + build(); + }); + + afterEach(() => relay.onModuleDestroy()); + + it("submits a pending row, records the envelope before broadcast, then confirms it", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + let hashAtSubmit: string | undefined; + invokeContract.mockImplementationOnce(async (params, options) => { + expect(params).toMatchObject({ contractId: CONTRACT_ID, method: "create_intent" }); + await options?.beforeSubmit?.("h1"); + hashAtSubmit = (await outbox.findByIntent("i1"))[0].envelopeHash; + return { hash: "h1", status: "PENDING", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ claimed: 1, submitted: 1 }); + expect(hashAtSubmit).toBe("h1"); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "submitted", txHash: "h1" }); + + check.mockResolvedValueOnce({ status: "success", ledger: 10 }); + expect(await relay.tick(later(5))).toMatchObject({ confirmed: 1 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("confirmed"); + expect(metrics.recordOutboxOutcome).toHaveBeenCalledWith("confirmed"); + expect(metrics.setOutboxBacklog).toHaveBeenCalled(); + }); + + it("confirms immediately when the live path already waited for confirmation (SUCCESS)", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-sync"); + return { hash: "h-sync", status: "SUCCESS", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ submitted: 1, confirmed: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "confirmed", txHash: "h-sync" }); + expect(check).not.toHaveBeenCalled(); + + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockResolvedValueOnce({ hash: "h2", status: "SUCCESS", dryRun: false }); + expect(await relay.tick(later(1))).toMatchObject({ confirmed: 0 }); + }); + + it("marks rows simulated under ONCHAIN_DRY_RUN and lets the next row for the intent proceed", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await outbox.enqueue({ intentId: "i1", operation: "cancel_intent", payload: { intentId: "i1", user: Keypair.random().publicKey() } }); + invokeContract.mockResolvedValue({ hash: "dry-run-no-hash", status: "DRY_RUN", dryRun: true }); + + expect(await relay.tick(now)).toMatchObject({ simulated: 1 }); + expect(await relay.tick(later(1))).toMatchObject({ simulated: 1 }); + expect((await outbox.findByIntent("i1")).map((r) => r.status)).toEqual(["simulated", "simulated"]); + }); + + describe("crash injection", () => { + it("crash between DB commit and submit: the committed row is picked up by the next relay", async () => { + // The unit of work commits intent + outbox row, then the process dies + // before any relay tick — nothing was submitted. + const repo = new InMemoryIntentsRepository(); + const intents = new IntentsService( + repo, + { get: (k: string) => (k === "onchainIntentsEnabled" ? true : k === "stellar.settlementContractId" ? CONTRACT_ID : undefined) } as unknown as ConfigService, + {} as StellarTxService, + { intentAuditLog: { create: jest.fn().mockResolvedValue({}) } } as unknown as PrismaService, + undefined, + undefined, + { snapshotForChain: jest.fn().mockReturnValue({ version: 0, deadlineSeconds: 1800, fillWindowSeconds: 600 }) } as unknown as ProtocolParamsService, + undefined, + new InMemoryIntentsUnitOfWork(repo, outbox), + ); + const { intentId: _ignored, state: _state, createdAt: _createdAt, ...data } = intentFor("ignored"); + const created = await intents.create(data); + expect(invokeContract).not.toHaveBeenCalled(); + + // "Restart": a fresh relay over the same durable outbox. + build(); + submitsAs("h-after-restart"); + expect(await relay.tick(now)).toMatchObject({ submitted: 1 }); + expect((await outbox.findByIntent(created.intentId))[0].txHash).toBe("h-after-restart"); + }); + + it("crash after broadcast, before markSubmitted: detects the landed tx and does not resubmit", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-landed"); + throw new Error("process killed"); + }); + // Simulate the kill: the row is left `processing` with the envelope hash + // (the catch path's retry write is lost along with the process). + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "processing", envelopeHash: "h-landed" }); + + // Before the lease expires nobody touches it. + expect(await relay.tick(later(LEASE_SECONDS - 1))).toMatchObject({ claimed: 0 }); + + check.mockResolvedValueOnce({ status: "success", ledger: 7 }); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ claimed: 1, confirmed: 1 }); + expect(invokeContract).toHaveBeenCalledTimes(1); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "confirmed", txHash: "h-landed" }); + }); + + it("crash after signing, envelope never landed: rebuilds and resubmits once the lease expires", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-lost"); + throw new Error("process killed"); + }); + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + + check.mockResolvedValueOnce({ status: "not_found" }); + submitsAs("h-rebuilt"); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ submitted: 1 }); + expect(check).toHaveBeenCalledWith("h-lost"); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "submitted", txHash: "h-rebuilt", attempts: 2 }); + }); + + it("a reclaimed row whose earlier envelope failed on-chain is retried", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-failed"); + throw new Error("process killed"); + }); + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + + check.mockResolvedValueOnce({ status: "failed", ledger: 9 }); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ + status: "pending", + lastError: expect.stringContaining("h-failed"), + }); + }); + }); + + it("a kill-switch pause puts the row back without consuming an attempt (never dead-letters)", async () => { + build(config({ maxAttempts: 1 })); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + const paused = new KillSwitchActiveException({ + reasonCode: "INCIDENT", + reason: "paused", + scope: "operation", + chain: "stellar", + token: null, + operation: "onchain", + } as never); + invokeContract.mockRejectedValue(paused); + + for (let i = 0; i < 5; i++) { + const t = i === 0 ? now : (await outbox.findByIntent("i1"))[0].nextAttemptAt; + expect(await relay.tick(t)).toMatchObject({ claimed: 1, paused: 1, dead: 0, retried: 0 }); + } + const [row] = await outbox.findByIntent("i1"); + expect(row).toMatchObject({ status: "pending", attempts: 0, lastError: expect.stringContaining("kill-switch") }); + + // Resume: the next attempt submits normally. + invokeContract.mockReset(); + submitsAs("h-after-resume"); + expect(await relay.tick(row.nextAttemptAt)).toMatchObject({ submitted: 1 }); + + // A lost fence on release is not counted. + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockRejectedValueOnce(paused); + jest.spyOn(outbox, "release").mockResolvedValueOnce(false); + expect(await relay.tick(later(3600))).toMatchObject({ paused: 0 }); + }); + + it("does not submit when the lease was lost before broadcast", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + jest.spyOn(outbox, "recordEnvelope").mockResolvedValueOnce(false); + const sent = jest.fn(); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h"); + sent(); + return { hash: "h", status: "PENDING", dryRun: false }; + }); + + await relay.tick(now); + expect(sent).not.toHaveBeenCalled(); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "pending", lastError: expect.stringContaining("lost lease") }); + }); + + it("keeps per-intent order across ticks while other intents proceed in parallel", async () => { + await outbox.enqueue(createIntentEntry(intentFor("a"))); + await outbox.enqueue({ intentId: "a", operation: "cancel_intent", payload: { intentId: "a", user: Keypair.random().publicKey() } }); + await outbox.enqueue(createIntentEntry(intentFor("b"))); + const order: string[] = []; + invokeContract.mockImplementation(async (params, options) => { + order.push(params.method); + const hash = `h${order.length}`; + await options?.beforeSubmit?.(hash); + return { hash, status: "PENDING", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ claimed: 2, submitted: 2 }); + expect(order).toEqual(["create_intent", "create_intent"]); + + // a's cancel must wait for a's create to confirm. + expect(await relay.tick(later(1))).toMatchObject({ claimed: 0 }); + + check.mockImplementation(async (hash) => ({ status: hash === "h1" ? "success" : "not_found" })); + expect(await relay.tick(later(2))).toMatchObject({ confirmed: 1, claimed: 1, submitted: 1 }); + expect(order).toEqual(["create_intent", "create_intent", "cancel_intent"]); + }); + + it("retries with exponential backoff and moves a poison row to dead with an alert", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await outbox.enqueue({ intentId: "i1", operation: "cancel_intent", payload: { intentId: "i1", user: Keypair.random().publicKey() } }); + invokeContract.mockRejectedValue(new Error("RPC 503")); + const errorLog = jest.spyOn((relay as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + + expect(await relay.tick(now)).toMatchObject({ retried: 1 }); + let [row] = await outbox.findByIntent("i1"); + expect(row.nextAttemptAt.getTime() - now.getTime()).toBe(1000); + + expect(await relay.tick(new Date(row.nextAttemptAt.getTime()))).toMatchObject({ retried: 1 }); + const t2 = row.nextAttemptAt.getTime(); + [row] = await outbox.findByIntent("i1"); + expect(row.nextAttemptAt.getTime() - t2).toBe(2000); + + expect(await relay.tick(new Date(row.nextAttemptAt.getTime()))).toMatchObject({ dead: 1 }); + [row] = await outbox.findByIntent("i1"); + expect(row).toMatchObject({ status: "dead", attempts: MAX_ATTEMPTS, lastError: "RPC 503" }); + expect(metrics.recordOutboxOutcome).toHaveBeenCalledWith("dead"); + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("ALERT")); + + // The dead row blocks the intent's later operations. + expect(await relay.tick(later(3600))).toMatchObject({ claimed: 0 }); + }); + + it("caps the backoff", async () => { + build(config({ maxAttempts: 100 })); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockRejectedValue(new Error("nope")); + let t = now; + for (let i = 0; i < 12; i++) { + await relay.tick(t); + t = (await outbox.findByIntent("i1"))[0].nextAttemptAt; + } + const [row] = await outbox.findByIntent("i1"); + await relay.tick(row.nextAttemptAt); + const [after] = await outbox.findByIntent("i1"); + expect(after.nextAttemptAt.getTime() - row.nextAttemptAt.getTime()).toBe(5 * 60_000); + }); + + it("retries when SETTLEMENT_CONTRACT_ID is missing at relay time", async () => { + build(config({}, "")); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await relay.tick(now); + expect(invokeContract).not.toHaveBeenCalled(); + expect((await outbox.findByIntent("i1"))[0].lastError).toContain("SETTLEMENT_CONTRACT_ID"); + }); + + describe("confirmation of submitted rows", () => { + beforeEach(async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + await relay.tick(now); + }); + + it("rebuilds when the transaction failed on-chain", async () => { + check.mockResolvedValueOnce({ status: "failed", ledger: 3 }); + expect(await relay.tick(later(5))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "pending", txHash: undefined }); + }); + + it("waits while not found within the lease, rebuilds after it", async () => { + expect(await relay.tick(later(LEASE_SECONDS - 10))).toMatchObject({ retried: 0, confirmed: 0 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("submitted"); + + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0].lastError).toContain("not found after lease"); + }); + + it("treats a lookup failure as no evidence and looks again next tick", async () => { + check.mockRejectedValueOnce(new Error("RPC down")); + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ retried: 0, confirmed: 0 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("submitted"); + }); + }); + + it("skips overlapping ticks", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + let release!: () => void; + invokeContract.mockImplementationOnce( + (_p, options) => + new Promise((resolve) => { + release = () => { + void options?.beforeSubmit?.("h").then(() => resolve({ hash: "h", status: "PENDING", dryRun: false })); + }; + }), + ); + const first = relay.tick(now); + await new Promise((r) => setImmediate(r)); + expect(await relay.tick(now)).toMatchObject({ claimed: 0 }); + release(); + expect(await first).toMatchObject({ submitted: 1 }); + }); + + it("survives a backlog gauge failure", async () => { + jest.spyOn(outbox, "countByStatus").mockRejectedValueOnce(new Error("db")); + await expect(relay.tick(now)).resolves.toMatchObject({ claimed: 0 }); + }); + + it("starts an interval on init only when enabled", () => { + jest.useFakeTimers(); + try { + const tick = jest.spyOn(relay, "tick").mockResolvedValue({} as never); + relay.onModuleInit(); + jest.advanceTimersByTime(1000); + expect(tick).toHaveBeenCalledTimes(1); + relay.onModuleDestroy(); + + build(config({ relayEnabled: false })); + const disabledTick = jest.spyOn(relay, "tick"); + relay.onModuleInit(); + jest.advanceTimersByTime(5000); + expect(disabledTick).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + } + }); + + it("logs, but does not crash, when an interval tick throws", () => { + jest.useFakeTimers(); + try { + jest.spyOn(relay, "tick").mockRejectedValue(new Error("boom")); + const errorLog = jest.spyOn((relay as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + relay.onModuleInit(); + jest.advanceTimersByTime(1000); + return Promise.resolve().then(() => { + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("boom")); + }); + } finally { + jest.useRealTimers(); + } + }); + + describe("lost fences (another worker reclaimed the row)", () => { + it("does not count outcomes whose fenced write lost", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + jest.spyOn(outbox, "markSubmitted").mockResolvedValueOnce(false); + expect(await relay.tick(now)).toMatchObject({ claimed: 1, submitted: 0 }); + + invokeContract.mockResolvedValueOnce({ hash: "x", status: "DRY_RUN", dryRun: true }); + jest.spyOn(outbox, "markSimulated").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ simulated: 0 }); + }); + + it("does not count a lost confirmation, dead-letter, or retry", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + await relay.tick(now); + check.mockResolvedValue({ status: "success", ledger: 1 }); + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + expect(await relay.tick(later(1))).toMatchObject({ confirmed: 0 }); + + // Reclaimed row whose envelope landed, but the confirm write loses. + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h2"); + throw new Error("killed"); + }); + const retry = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + expect(await relay.tick(later(2))).toMatchObject({ retried: 0 }); + retry.mockRestore(); + jest.spyOn(outbox, "findSubmitted").mockResolvedValueOnce([]); + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 5))).toMatchObject({ claimed: 1, confirmed: 0 }); + + build(config({ maxAttempts: 1 })); + await outbox.enqueue(createIntentEntry(intentFor("i3"))); + invokeContract.mockRejectedValueOnce("not an Error"); + jest.spyOn(outbox, "markDead").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ dead: 0 }); + }); + + it("skips submitted rows without a tx hash and defaults the clock", async () => { + jest.spyOn(outbox, "findSubmitted").mockResolvedValueOnce([ + { id: "1", intentId: "i", operation: "create_intent", payload: {}, status: "submitted", attempts: 1, + nextAttemptAt: now, createdAt: now, updatedAt: now }, + ]); + await expect(relay.tick()).resolves.toMatchObject({ confirmed: 0, retried: 0 }); + expect(check).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/soroban/outbox-relay.service.ts b/src/soroban/outbox-relay.service.ts new file mode 100644 index 00000000..9482d4ce --- /dev/null +++ b/src/soroban/outbox-relay.service.ts @@ -0,0 +1,265 @@ +import { Inject, Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { IOutboxRepository, OUTBOX_REPOSITORY, OutboxEntry } from "./outbox.repository"; +import { buildOutboxInvocation } from "./outbox-operations"; +import { StellarTxService } from "./stellar-tx.service"; +import { TxConfirmationService } from "./tx-confirmation.service"; + +/** Upper bound for the exponential retry backoff. */ +const MAX_BACKOFF_MS = 5 * 60_000; +const BASE_BACKOFF_MS = 1_000; + +/** Re-check interval for rows blocked by a kill-switch pause. */ +const PAUSED_RECHECK_MS = 30_000; + +/** Counts from one relay tick — returned for tests and manual triggers. */ +export interface RelayTickResult { + claimed: number; + /** Rows put back untouched because a kill-switch pause blocked the write. */ + paused: number; + submitted: number; + simulated: number; + confirmed: number; + retried: number; + dead: number; +} + +/** + * Relay worker for the transactional outbox (issue #396). + * + * Each tick: + * 1. **Confirm** — polls `submitted` rows through TxConfirmationService and + * marks them `confirmed`, or schedules a rebuild when the transaction + * failed on-chain or expired unseen. + * 2. **Relay** — claims due head-of-intent rows (SKIP LOCKED in Postgres, so + * several instances can run this safely) and submits them via + * StellarTxService. + * + * StellarTxService.invokeContract blocks until the transaction confirms (or + * throws on FAILED / confirmation TIMEOUT). A TIMEOUT retry is safe: the wait + * (120 s) outlasts the envelope's 30 s time bound, so the timed-out envelope + * can no longer land. + * + * Crash idempotency: the signed envelope's hash is persisted *before* + * broadcast (`beforeSubmit`). If the process dies between broadcast and + * `markSubmitted`, the lease expires, the row is reclaimed with its + * `envelopeHash` set, and the relay looks that hash up first — SUCCESS means + * the earlier submission landed and the row is confirmed without + * resubmitting. NOT_FOUND is only trusted because the lease + * (OUTBOX_LEASE_SECONDS) is longer than the envelope's time bound, so by + * reclaim time an unseen envelope can no longer be included. + * + * Poison rows: after OUTBOX_MAX_ATTEMPTS claims a row moves to `dead`, + * increments `vortex_outbox_dead_total` (alerted on) and blocks later rows + * for the same intent until an operator requeues it — see + * docs/runbooks/on-call.md. + */ +@Injectable() +export class OutboxRelayService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(OutboxRelayService.name); + private readonly settings: AppConfig["outbox"]; + private readonly settlementContractId: string; + private interval?: NodeJS.Timeout; + private running = false; + + constructor( + @Inject(OUTBOX_REPOSITORY) private readonly outbox: IOutboxRepository, + private readonly stellarTxService: StellarTxService, + private readonly confirmation: TxConfirmationService, + private readonly metrics: MetricsService, + configService: ConfigService, + ) { + this.settings = configService.get("outbox", { infer: true }); + this.settlementContractId = configService.get("stellar.settlementContractId", { infer: true }); + } + + onModuleInit() { + if (!this.settings.relayEnabled) { + this.logger.warn("[outbox] relay disabled (OUTBOX_RELAY_ENABLED=false) — rows will accumulate"); + return; + } + this.interval = setInterval(() => { + this.tick().catch((err) => + this.logger.error(`[outbox] relay tick failed: ${errorMessage(err)}`), + ); + }, this.settings.relayIntervalMs); + this.interval.unref?.(); + } + + onModuleDestroy() { + if (this.interval) clearInterval(this.interval); + } + + /** Runs one confirm + relay cycle. Overlapping calls are skipped. */ + async tick(now: Date = new Date()): Promise { + const result: RelayTickResult = { + claimed: 0, + paused: 0, + submitted: 0, + simulated: 0, + confirmed: 0, + retried: 0, + dead: 0, + }; + if (this.running) return result; + this.running = true; + try { + await this.confirmSubmitted(now, result); + await this.relayDue(now, result); + await this.refreshBacklogGauge(); + return result; + } finally { + this.running = false; + } + } + + private async confirmSubmitted(now: Date, result: RelayTickResult): Promise { + const submitted = await this.outbox.findSubmitted(this.settings.batchSize); + for (const entry of submitted) { + if (!entry.txHash) continue; + let status; + try { + status = (await this.confirmation.check(entry.txHash)).status; + } catch (err) { + // RPC outage is not evidence either way — look again next tick. + this.logger.warn(`[outbox] confirmation lookup failed for row ${entry.id}: ${errorMessage(err)}`); + continue; + } + + if (status === "success") { + if (await this.outbox.markConfirmed(entry, entry.txHash)) { + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + } else if (status === "failed") { + await this.fail(entry, `transaction ${entry.txHash} failed on-chain`, now, result); + } else if (now.getTime() - entry.updatedAt.getTime() > this.settings.leaseSeconds * 1000) { + await this.fail(entry, `transaction ${entry.txHash} not found after lease; rebuilding`, now, result); + } + } + } + + private async relayDue(now: Date, result: RelayTickResult): Promise { + const leaseUntil = new Date(now.getTime() + this.settings.leaseSeconds * 1000); + const claimed = await this.outbox.claimDue(now, this.settings.batchSize, leaseUntil); + result.claimed += claimed.length; + // Sequential on purpose: every submission draws the next sequence number + // from the single signing account (SignerService serializes anyway). + for (const entry of claimed) { + await this.process(entry, now, result); + } + } + + private async process(entry: OutboxEntry, now: Date, result: RelayTickResult): Promise { + try { + if (entry.envelopeHash) { + const previous = await this.confirmation.check(entry.envelopeHash); + if (previous.status === "success") { + this.logger.warn( + `[outbox] row ${entry.id} (${entry.operation} intent=${entry.intentId}) was already ` + + `submitted before a crash (tx ${entry.envelopeHash}); confirming without resubmitting`, + ); + if (await this.outbox.markConfirmed(entry, entry.envelopeHash)) { + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + return; + } + if (previous.status === "failed") { + throw new Error(`previous envelope ${entry.envelopeHash} failed on-chain`); + } + // not_found: the lease outlived the envelope's time bound — rebuild. + } + + if (!this.settlementContractId) { + throw new Error("SETTLEMENT_CONTRACT_ID is not configured"); + } + const invocation = buildOutboxInvocation(entry, this.settlementContractId); + const sent = await this.stellarTxService.invokeContract(invocation, { + beforeSubmit: async (hash) => { + if (!(await this.outbox.recordEnvelope(entry, hash))) { + throw new Error(`lost lease on row ${entry.id} before submit`); + } + }, + }); + + if (sent.dryRun) { + if (await this.outbox.markSimulated(entry)) { + result.simulated++; + this.metrics.recordOutboxOutcome("simulated"); + } + return; + } + + // StellarTxService's live path waits for confirmation and reports + // SUCCESS; anything else (e.g. PENDING) is confirmed by a later tick. + if (sent.status === "SUCCESS") { + if (await this.outbox.markConfirmed(entry, sent.hash)) { + result.submitted++; + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + return; + } + if (await this.outbox.markSubmitted(entry, sent.hash)) { + result.submitted++; + this.metrics.recordOutboxOutcome("submitted"); + } + } catch (err) { + if (err instanceof KillSwitchActiveException) { + // Issue #477 — a pause is not a failure: put the row back without + // consuming an attempt so a long pause cannot dead-letter it. + if (await this.outbox.release(entry, `paused by kill-switch: ${err.message}`, new Date(now.getTime() + PAUSED_RECHECK_MS))) { + result.paused++; + } + return; + } + await this.fail(entry, errorMessage(err), now, result); + } + } + + private async fail( + entry: OutboxEntry, + error: string, + now: Date, + result: RelayTickResult, + ): Promise { + if (entry.attempts >= this.settings.maxAttempts) { + if (await this.outbox.markDead(entry, error)) { + result.dead++; + this.metrics.recordOutboxOutcome("dead"); + this.logger.error( + `[outbox] ALERT row ${entry.id} (${entry.operation} intent=${entry.intentId}) moved to dead ` + + `after ${entry.attempts} attempts: ${error}. Later operations for this intent are blocked ` + + `until it is requeued — see docs/runbooks/on-call.md`, + ); + } + return; + } + + const delay = Math.min(BASE_BACKOFF_MS * 2 ** Math.max(0, entry.attempts - 1), MAX_BACKOFF_MS); + if (await this.outbox.scheduleRetry(entry, error, new Date(now.getTime() + delay))) { + result.retried++; + this.metrics.recordOutboxOutcome("retry"); + this.logger.warn( + `[outbox] row ${entry.id} (${entry.operation} intent=${entry.intentId}) attempt ` + + `${entry.attempts}/${this.settings.maxAttempts} failed: ${error}; retrying in ${delay}ms`, + ); + } + } + + private async refreshBacklogGauge(): Promise { + try { + this.metrics.setOutboxBacklog(await this.outbox.countByStatus()); + } catch (err) { + this.logger.warn(`[outbox] backlog gauge refresh failed: ${errorMessage(err)}`); + } + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/soroban/outbox.repository.spec.ts b/src/soroban/outbox.repository.spec.ts new file mode 100644 index 00000000..429031b6 --- /dev/null +++ b/src/soroban/outbox.repository.spec.ts @@ -0,0 +1,145 @@ +import { InMemoryOutboxRepository, NewOutboxEntry } from "./outbox.repository"; + +const at = (ms: number) => new Date(1_700_000_000_000 + ms); +const entry = (intentId: string, operation: NewOutboxEntry["operation"] = "create_intent"): NewOutboxEntry => ({ + intentId, + operation, + payload: { intentId }, +}); + +describe("InMemoryOutboxRepository (#396)", () => { + let repo: InMemoryOutboxRepository; + + beforeEach(() => { + repo = new InMemoryOutboxRepository(); + }); + + it("assigns monotonic ids and starts rows as pending with zero attempts", async () => { + const a = await repo.enqueue(entry("i1")); + const b = await repo.enqueue(entry("i1", "accept_intent")); + expect(BigInt(b.id)).toBeGreaterThan(BigInt(a.id)); + expect(a).toMatchObject({ status: "pending", attempts: 0 }); + }); + + it("claims only the head row per intent, but runs different intents in parallel", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + await repo.enqueue(entry("i2")); + + const claimed = await repo.claimDue(new Date(Date.now() + 1000), 10, at(60_000)); + + expect(claimed.map((r) => [r.intentId, r.operation])).toEqual([ + ["i1", "create_intent"], + ["i2", "create_intent"], + ]); + expect(claimed.every((r) => r.status === "processing" && r.attempts === 1)).toBe(true); + }); + + it("releases the next row for an intent only once the previous one is confirmed or simulated", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + const now = new Date(Date.now() + 1000); + + const [first] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markSubmitted(first, "tx1"); + expect(await repo.claimDue(now, 10, new Date(now.getTime() + 60_000))).toEqual([]); + + await repo.markConfirmed({ ...first }, "tx1"); + const [second] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(second.operation).toBe("accept_intent"); + + await repo.markSimulated(second); + await repo.enqueue(entry("i1", "fill_intent")); + const [third] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(third.operation).toBe("fill_intent"); + }); + + it("respects the batch limit and nextAttemptAt", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i2")); + const now = new Date(Date.now() + 1000); + const [claimed] = await repo.claimDue(now, 1, new Date(now.getTime() + 60_000)); + expect(claimed.intentId).toBe("i1"); + + await repo.scheduleRetry(claimed, "boom", new Date(now.getTime() + 10_000)); + const next = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(next.map((r) => r.intentId)).toEqual(["i2"]); + const later = await repo.claimDue(new Date(now.getTime() + 10_001), 10, new Date(now.getTime() + 70_000)); + expect(later.map((r) => r.intentId)).toEqual(["i1"]); + expect(later[0].attempts).toBe(2); + }); + + it("reclaims a processing row only after its lease expires (crashed worker)", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const leaseUntil = new Date(now.getTime() + 60_000); + await repo.claimDue(now, 10, leaseUntil); + + expect(await repo.claimDue(new Date(leaseUntil.getTime() - 1), 10, at(0))).toEqual([]); + const [reclaimed] = await repo.claimDue(new Date(leaseUntil.getTime() + 1), 10, at(0)); + expect(reclaimed.attempts).toBe(2); + }); + + it("fences writes on attempts so a stale worker cannot overwrite a reclaimed row", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const [stale] = await repo.claimDue(now, 10, new Date(now.getTime() + 1)); + const [fresh] = await repo.claimDue(new Date(now.getTime() + 2), 10, new Date(now.getTime() + 60_000)); + + expect(await repo.recordEnvelope(stale, "old")).toBe(false); + expect(await repo.markSubmitted(stale, "old")).toBe(false); + expect(await repo.recordEnvelope(fresh, "new")).toBe(true); + expect((await repo.findByIntent("i1"))[0].envelopeHash).toBe("new"); + }); + + it("clears envelope and tx hash on retry so the relay rebuilds", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const [row] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.recordEnvelope(row, "h"); + await repo.markSubmitted(row, "h"); + await repo.scheduleRetry(row, "failed on-chain", now); + + const [stored] = await repo.findByIntent("i1"); + expect(stored).toMatchObject({ status: "pending", lastError: "failed on-chain" }); + expect(stored.envelopeHash).toBeUndefined(); + expect(stored.txHash).toBeUndefined(); + }); + + it("dead rows block later rows for the same intent until requeued", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + const now = new Date(Date.now() + 1000); + const [row] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markDead(row, "poison"); + + expect(await repo.claimDue(now, 10, new Date(now.getTime() + 60_000))).toEqual([]); + expect((await repo.countByStatus()).dead).toBe(1); + + expect(await repo.requeueDead("999")).toBe(false); + expect(await repo.requeueDead(row.id)).toBe(true); + expect(await repo.requeueDead(row.id)).toBe(false); + const [again] = await repo.claimDue(new Date(Date.now() + 1000), 10, at(0)); + expect(again).toMatchObject({ id: row.id, attempts: 1, operation: "create_intent" }); + }); + + it("lists submitted rows oldest first and counts by status", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i2")); + const now = new Date(Date.now() + 1000); + const [a, b] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markSubmitted(b, "tb"); + await repo.markSubmitted(a, "ta"); + + expect((await repo.findSubmitted(10)).map((r) => r.txHash)).toEqual(["ta", "tb"]); + expect((await repo.findSubmitted(1)).map((r) => r.txHash)).toEqual(["ta"]); + expect(await repo.countByStatus()).toEqual({ + pending: 0, + processing: 0, + submitted: 2, + confirmed: 0, + simulated: 0, + dead: 0, + }); + }); +}); diff --git a/src/soroban/outbox.repository.ts b/src/soroban/outbox.repository.ts new file mode 100644 index 00000000..ef13c845 --- /dev/null +++ b/src/soroban/outbox.repository.ts @@ -0,0 +1,269 @@ +import { Injectable } from "@nestjs/common"; + +/** + * NestJS injection token for the on-chain outbox repository (issue #396). + * Bound in IntentsModule to the in-memory or Prisma adapter, following + * INTENTS_PERSISTENCE. + */ +export const OUTBOX_REPOSITORY = Symbol("OUTBOX_REPOSITORY"); + +/** Settlement-contract operations that flow through the outbox. */ +export const OUTBOX_OPERATIONS = [ + "create_intent", + "accept_intent", + "fill_intent", + "cancel_intent", +] as const; +export type OutboxOperation = (typeof OUTBOX_OPERATIONS)[number]; + +/** + * Row lifecycle: + * + * pending ──claim──▶ processing ──submit──▶ submitted ──confirm──▶ confirmed + * ▲ │ │ │ + * └──── retry ────────┘ └─dry-run─▶ simulated│ + * ▲ │ + * └──────────── tx failed / expired ─────────┘ + * any retry past OUTBOX_MAX_ATTEMPTS ──▶ dead (alerted; blocks later rows + * for the same intent) + */ +export type OutboxStatus = + | "pending" + | "processing" + | "submitted" + | "confirmed" + | "simulated" + | "dead"; + +/** Statuses after which the next row for the same intent may proceed. */ +export const OUTBOX_DONE_STATUSES: readonly OutboxStatus[] = ["confirmed", "simulated"]; + +export interface OutboxEntry { + /** Monotonic sequence (bigint serialized as string); defines per-intent order. */ + id: string; + intentId: string; + operation: OutboxOperation; + payload: Record; + status: OutboxStatus; + /** + * Number of claims so far. Also the fencing token: every state change after + * a claim is conditional on `attempts` still matching, so a worker whose + * lease expired cannot overwrite the row after another worker reclaimed it. + */ + attempts: number; + nextAttemptAt: Date; + lockedUntil?: Date; + envelopeHash?: string; + txHash?: string; + lastError?: string; + createdAt: Date; + updatedAt: Date; +} + +export interface NewOutboxEntry { + intentId: string; + operation: OutboxOperation; + payload: Record; +} + +/** The only outbox capability intent-mutating code needs inside a transaction. */ +export interface IOutboxWriter { + enqueue(entry: NewOutboxEntry): Promise; +} + +export interface IOutboxRepository extends IOutboxWriter { + /** + * Atomically claims up to `limit` due rows and moves them to `processing` + * (attempts + 1, lockedUntil = `leaseUntil`). A row is due when it is + * `pending` with nextAttemptAt <= now, or `processing` with an expired lease + * (its worker crashed). Only the head row per intent is eligible — every + * earlier row for that intent must be in {@link OUTBOX_DONE_STATUSES} — + * which gives per-intent ordering while different intents run in parallel. + * The Prisma adapter uses `FOR UPDATE SKIP LOCKED`. + */ + claimDue(now: Date, limit: number, leaseUntil: Date): Promise; + + /** Rows waiting on confirmation, oldest first. */ + findSubmitted(limit: number): Promise; + + findByIntent(intentId: string): Promise; + + countByStatus(): Promise>; + + /** Persists the signed envelope hash before broadcast. Fenced on `attempts`. */ + recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise; + + markSubmitted(entry: OutboxEntry, txHash: string): Promise; + + markConfirmed(entry: OutboxEntry, txHash: string): Promise; + + markSimulated(entry: OutboxEntry): Promise; + + /** Back to `pending` at `nextAttemptAt`; clears the envelope so it is rebuilt. */ + scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise; + + markDead(entry: OutboxEntry, error: string): Promise; + + /** + * Back to `pending` at `nextAttemptAt` *without* consuming the claim's + * attempt (attempts - 1). Used when a kill-switch pause blocked the write, + * so an emergency pause can never dead-letter rows. + */ + release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise; + + /** Operator action: `dead` → `pending` with attempts reset. */ + requeueDead(id: string): Promise; +} + +export function emptyStatusCounts(): Record { + return { pending: 0, processing: 0, submitted: 0, confirmed: 0, simulated: 0, dead: 0 }; +} + +/** + * In-memory adapter — development and tests. Single-process only; the Node + * event loop makes each method atomic because none of them await. + */ +@Injectable() +export class InMemoryOutboxRepository implements IOutboxRepository { + private readonly rows = new Map(); + private sequence = 0n; + + async enqueue(entry: NewOutboxEntry): Promise { + const now = new Date(); + const row: OutboxEntry = { + ...entry, + id: (++this.sequence).toString(), + status: "pending", + attempts: 0, + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }; + this.rows.set(row.id, row); + return { ...row }; + } + + async claimDue(now: Date, limit: number, leaseUntil: Date): Promise { + const claimed: OutboxEntry[] = []; + const headSeen = new Set(); + for (const row of this.ordered()) { + if (claimed.length >= limit) break; + if (OUTBOX_DONE_STATUSES.includes(row.status)) continue; + // First unfinished row for this intent is its head; anything after it waits. + if (headSeen.has(row.intentId)) continue; + headSeen.add(row.intentId); + + const due = + (row.status === "pending" && row.nextAttemptAt <= now) || + (row.status === "processing" && row.lockedUntil !== undefined && row.lockedUntil < now); + if (!due) continue; + + Object.assign(row, { + status: "processing", + attempts: row.attempts + 1, + lockedUntil: leaseUntil, + updatedAt: now, + }); + claimed.push({ ...row }); + } + return claimed; + } + + async findSubmitted(limit: number): Promise { + return this.ordered() + .filter((r) => r.status === "submitted") + .slice(0, limit) + .map((r) => ({ ...r })); + } + + async findByIntent(intentId: string): Promise { + return this.ordered() + .filter((r) => r.intentId === intentId) + .map((r) => ({ ...r })); + } + + async countByStatus(): Promise> { + const counts = emptyStatusCounts(); + for (const row of this.rows.values()) counts[row.status]++; + return counts; + } + + async recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise { + return this.fenced(entry, ["processing"], { envelopeHash }); + } + + async markSubmitted(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing"], { status: "submitted", txHash, lockedUntil: undefined }); + } + + async markConfirmed(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "confirmed", + txHash, + lockedUntil: undefined, + }); + } + + async markSimulated(entry: OutboxEntry): Promise { + return this.fenced(entry, ["processing"], { status: "simulated", lockedUntil: undefined }); + } + + async scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "pending", + nextAttemptAt, + lastError: error, + lockedUntil: undefined, + envelopeHash: undefined, + txHash: undefined, + }); + } + + async markDead(entry: OutboxEntry, error: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "dead", + lastError: error, + lockedUntil: undefined, + }); + } + + async release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing"], { + status: "pending", + attempts: Math.max(0, entry.attempts - 1), + nextAttemptAt, + lastError: note, + lockedUntil: undefined, + envelopeHash: undefined, + }); + } + + async requeueDead(id: string): Promise { + const row = this.rows.get(id); + if (!row || row.status !== "dead") return false; + Object.assign(row, { + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + envelopeHash: undefined, + txHash: undefined, + updatedAt: new Date(), + }); + return true; + } + + private ordered(): OutboxEntry[] { + return [...this.rows.values()].sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1)); + } + + private fenced( + entry: OutboxEntry, + from: OutboxStatus[], + patch: Partial, + ): boolean { + const row = this.rows.get(entry.id); + if (!row || row.attempts !== entry.attempts || !from.includes(row.status)) return false; + Object.assign(row, patch, { updatedAt: new Date() }); + return true; + } +} diff --git a/src/soroban/prisma-outbox.repository.spec.ts b/src/soroban/prisma-outbox.repository.spec.ts new file mode 100644 index 00000000..09135966 --- /dev/null +++ b/src/soroban/prisma-outbox.repository.spec.ts @@ -0,0 +1,129 @@ +import { Prisma } from "@prisma/client"; +import { OutboxEntry } from "./outbox.repository"; +import { OutboxPrismaClient, PrismaOutboxRepository } from "./prisma-outbox.repository"; + +const created = new Date("2026-09-27T00:00:00Z"); +const modelRow = (overrides: Record = {}) => ({ + id: 7n, + intentId: "i1", + operation: "create_intent", + payload: { intentId: "i1" }, + status: "pending", + attempts: 0, + nextAttemptAt: created, + lockedUntil: null, + envelopeHash: null, + txHash: null, + lastError: null, + createdAt: created, + updatedAt: created, + ...overrides, +}); + +const entry = { id: "7", attempts: 2 } as OutboxEntry; + +function fakeClient() { + return { + onchainOutbox: { + create: jest.fn().mockResolvedValue(modelRow()), + findMany: jest.fn().mockResolvedValue([modelRow({ status: "submitted", txHash: "h" })]), + groupBy: jest.fn().mockResolvedValue([ + { status: "pending", _count: { _all: 3 } }, + { status: "dead", _count: { _all: 1 } }, + ]), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, + $queryRaw: jest.fn(), + }; +} + +describe("PrismaOutboxRepository (#396)", () => { + let client: ReturnType; + let repo: PrismaOutboxRepository; + + beforeEach(() => { + client = fakeClient(); + repo = new PrismaOutboxRepository(client as unknown as OutboxPrismaClient); + }); + + it("enqueues through the (transaction) client and maps bigint ids to strings", async () => { + const row = await repo.enqueue({ intentId: "i1", operation: "create_intent", payload: { intentId: "i1" } }); + expect(client.onchainOutbox.create).toHaveBeenCalledWith({ + data: { intentId: "i1", operation: "create_intent", payload: { intentId: "i1" } }, + }); + expect(row).toMatchObject({ id: "7", status: "pending", lockedUntil: undefined, envelopeHash: undefined }); + }); + + it("claims with a single SKIP LOCKED statement that enforces per-intent ordering", async () => { + client.$queryRaw.mockResolvedValue([ + { + id: 9n, intent_id: "b", operation: "create_intent", payload: {}, status: "processing", attempts: 1, + next_attempt_at: created, locked_until: created, envelope_hash: null, tx_hash: null, last_error: null, + created_at: created, updated_at: created, + }, + { + id: 3n, intent_id: "a", operation: "accept_intent", payload: null, status: "processing", attempts: 1, + next_attempt_at: created, locked_until: created, envelope_hash: "e", tx_hash: null, last_error: "x", + created_at: created, updated_at: created, + }, + ]); + const now = new Date("2026-09-27T01:00:00Z"); + const lease = new Date("2026-09-27T01:02:00Z"); + + const rows = await repo.claimDue(now, 5, lease); + + const sql = client.$queryRaw.mock.calls[0][0] as Prisma.Sql; + expect(sql.sql).toMatch(/FOR UPDATE SKIP LOCKED/); + expect(sql.sql).toMatch(/NOT EXISTS/); + expect(sql.sql).toMatch(/p\.id < c\.id/); + expect(sql.sql).toMatch(/attempts = o\.attempts \+ 1/); + expect(sql.values).toEqual(expect.arrayContaining([now, 5, lease, "confirmed", "simulated"])); + expect(rows.map((r) => r.id)).toEqual(["3", "9"]); + expect(rows[0]).toMatchObject({ intentId: "a", envelopeHash: "e", lastError: "x", payload: {} }); + }); + + it("reads submitted rows, rows by intent, and status counts", async () => { + expect(await repo.findSubmitted(4)).toHaveLength(1); + expect(client.onchainOutbox.findMany).toHaveBeenLastCalledWith({ + where: { status: "submitted" }, + orderBy: { id: "asc" }, + take: 4, + }); + await repo.findByIntent("i1"); + expect(client.onchainOutbox.findMany).toHaveBeenLastCalledWith({ where: { intentId: "i1" }, orderBy: { id: "asc" } }); + expect(await repo.countByStatus()).toEqual({ + pending: 3, processing: 0, submitted: 0, confirmed: 0, simulated: 0, dead: 1, + }); + }); + + it.each([ + ["recordEnvelope", () => repo.recordEnvelope(entry, "h"), ["processing"], { envelopeHash: "h" }], + ["markSubmitted", () => repo.markSubmitted(entry, "h"), ["processing"], { status: "submitted", txHash: "h", lockedUntil: null }], + ["markConfirmed", () => repo.markConfirmed(entry, "h"), ["processing", "submitted"], { status: "confirmed", txHash: "h", lockedUntil: null }], + ["markSimulated", () => repo.markSimulated(entry), ["processing"], { status: "simulated", lockedUntil: null }], + ["markDead", () => repo.markDead(entry, "e"), ["processing", "submitted"], { status: "dead", lastError: "e", lockedUntil: null }], + ])("%s is fenced on id + attempts + status", async (_name, call, from, data) => { + expect(await (call as () => Promise)()).toBe(true); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith({ + where: { id: 7n, attempts: 2, status: { in: from } }, + data, + }); + }); + + it("scheduleRetry clears the envelope so the relay rebuilds, and reports a lost fence", async () => { + client.onchainOutbox.updateMany.mockResolvedValueOnce({ count: 0 }); + const next = new Date(); + expect(await repo.scheduleRetry(entry, "boom", next)).toBe(false); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith({ + where: { id: 7n, attempts: 2, status: { in: ["processing", "submitted"] } }, + data: { status: "pending", nextAttemptAt: next, lastError: "boom", lockedUntil: null, envelopeHash: null, txHash: null }, + }); + }); + + it("requeues only dead rows", async () => { + expect(await repo.requeueDead("7")).toBe(true); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ where: { id: 7n, status: "dead" } }), + ); + }); +}); diff --git a/src/soroban/prisma-outbox.repository.ts b/src/soroban/prisma-outbox.repository.ts new file mode 100644 index 00000000..1172b27b --- /dev/null +++ b/src/soroban/prisma-outbox.repository.ts @@ -0,0 +1,229 @@ +import { Prisma, OnchainOutbox, OutboxStatus as PrismaOutboxStatus } from "@prisma/client"; +import { + IOutboxRepository, + NewOutboxEntry, + OUTBOX_DONE_STATUSES, + OutboxEntry, + OutboxOperation, + OutboxStatus, + emptyStatusCounts, +} from "./outbox.repository"; + +/** PrismaService or the client handed to a `$transaction` callback. */ +export type OutboxPrismaClient = Prisma.TransactionClient; + +/** Raw row shape returned by the claim query (snake_case columns). */ +interface RawOutboxRow { + id: bigint; + intent_id: string; + operation: string; + payload: Prisma.JsonValue; + status: PrismaOutboxStatus; + attempts: number; + next_attempt_at: Date; + locked_until: Date | null; + envelope_hash: string | null; + tx_hash: string | null; + last_error: string | null; + created_at: Date; + updated_at: Date; +} + +/** + * Prisma-backed outbox (issue #396). + * + * Constructed either with PrismaService (relay worker) or with a transaction + * client (inside IntentsUnitOfWork) so `enqueue` commits atomically with the + * intent mutation it mirrors. + */ +export class PrismaOutboxRepository implements IOutboxRepository { + constructor(private readonly prisma: OutboxPrismaClient) {} + + async enqueue(entry: NewOutboxEntry): Promise { + const row = await this.prisma.onchainOutbox.create({ + data: { + intentId: entry.intentId, + operation: entry.operation, + payload: entry.payload as Prisma.InputJsonObject, + }, + }); + return fromModel(row); + } + + /** + * Single statement: pick the per-intent head rows that are due, lock them + * with SKIP LOCKED so concurrent relays partition the work, and flip them to + * `processing`. The NOT EXISTS clause treats every non-done earlier row + * (including `dead`) as blocking, which is what guarantees per-intent order. + */ + async claimDue(now: Date, limit: number, leaseUntil: Date): Promise { + const done = Prisma.join(OUTBOX_DONE_STATUSES.map((s) => Prisma.sql`${s}::"OutboxStatus"`)); + const rows = await this.prisma.$queryRaw(Prisma.sql` + WITH candidates AS ( + SELECT c.id + FROM onchain_outbox c + WHERE ( + (c.status = 'pending' AND c.next_attempt_at <= ${now}) + OR (c.status = 'processing' AND c.locked_until < ${now}) + ) + AND NOT EXISTS ( + SELECT 1 FROM onchain_outbox p + WHERE p.intent_id = c.intent_id + AND p.id < c.id + AND p.status NOT IN (${done}) + ) + ORDER BY c.id + LIMIT ${limit} + FOR UPDATE SKIP LOCKED + ) + UPDATE onchain_outbox o + SET status = 'processing', + attempts = o.attempts + 1, + locked_until = ${leaseUntil}, + updated_at = NOW() + FROM candidates + WHERE o.id = candidates.id + RETURNING o.* + `); + return rows.map(fromRaw).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1)); + } + + async findSubmitted(limit: number): Promise { + const rows = await this.prisma.onchainOutbox.findMany({ + where: { status: "submitted" }, + orderBy: { id: "asc" }, + take: limit, + }); + return rows.map(fromModel); + } + + async findByIntent(intentId: string): Promise { + const rows = await this.prisma.onchainOutbox.findMany({ + where: { intentId }, + orderBy: { id: "asc" }, + }); + return rows.map(fromModel); + } + + async countByStatus(): Promise> { + const groups = await this.prisma.onchainOutbox.groupBy({ + by: ["status"], + _count: { _all: true }, + }); + const counts = emptyStatusCounts(); + for (const g of groups) counts[g.status as OutboxStatus] = g._count._all; + return counts; + } + + recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise { + return this.fenced(entry, ["processing"], { envelopeHash }); + } + + markSubmitted(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing"], { status: "submitted", txHash, lockedUntil: null }); + } + + markConfirmed(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "confirmed", + txHash, + lockedUntil: null, + }); + } + + markSimulated(entry: OutboxEntry): Promise { + return this.fenced(entry, ["processing"], { status: "simulated", lockedUntil: null }); + } + + scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "pending", + nextAttemptAt, + lastError: error, + lockedUntil: null, + envelopeHash: null, + txHash: null, + }); + } + + markDead(entry: OutboxEntry, error: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "dead", + lastError: error, + lockedUntil: null, + }); + } + + release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing"], { + status: "pending", + attempts: Math.max(0, entry.attempts - 1), + nextAttemptAt, + lastError: note, + lockedUntil: null, + envelopeHash: null, + }); + } + + async requeueDead(id: string): Promise { + const result = await this.prisma.onchainOutbox.updateMany({ + where: { id: BigInt(id), status: "dead" }, + data: { + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + envelopeHash: null, + txHash: null, + }, + }); + return result.count > 0; + } + + private async fenced( + entry: OutboxEntry, + from: OutboxStatus[], + data: Prisma.OnchainOutboxUpdateManyMutationInput, + ): Promise { + const result = await this.prisma.onchainOutbox.updateMany({ + where: { id: BigInt(entry.id), attempts: entry.attempts, status: { in: from } }, + data, + }); + return result.count > 0; + } +} + +function fromModel(row: OnchainOutbox): OutboxEntry { + return { + id: row.id.toString(), + intentId: row.intentId, + operation: row.operation as OutboxOperation, + payload: (row.payload ?? {}) as Record, + status: row.status as OutboxStatus, + attempts: row.attempts, + nextAttemptAt: row.nextAttemptAt, + lockedUntil: row.lockedUntil ?? undefined, + envelopeHash: row.envelopeHash ?? undefined, + txHash: row.txHash ?? undefined, + lastError: row.lastError ?? undefined, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function fromRaw(row: RawOutboxRow): OutboxEntry { + return fromModel({ + id: row.id, + intentId: row.intent_id, + operation: row.operation, + payload: row.payload, + status: row.status, + attempts: row.attempts, + nextAttemptAt: row.next_attempt_at, + lockedUntil: row.locked_until, + envelopeHash: row.envelope_hash, + txHash: row.tx_hash, + lastError: row.last_error, + createdAt: row.created_at, + updatedAt: row.updated_at, + }); +} diff --git a/src/soroban/solver-registry.service.ts b/src/soroban/solver-registry.service.ts index 50d513fb..59a35a49 100644 --- a/src/soroban/solver-registry.service.ts +++ b/src/soroban/solver-registry.service.ts @@ -45,6 +45,12 @@ export interface SlashResult { * depending on whether the contract is configured. */ dryRun: boolean; + /** + * true when the call errored (RPC failure, simulation error) and should be + * retried by the caller. false for successful, dry-run, and unconfigured + * (no-op) outcomes (issue #397). + */ + failed: boolean; } /** @@ -117,6 +123,7 @@ export class SolverRegistryService { submitted: false, simulated: false, dryRun: true, + failed: false, detail: "ONCHAIN_DRY_RUN=true — simulated log only, no transaction submitted", }; } @@ -128,7 +135,7 @@ export class SolverRegistryService { this.logger.log( `[solver-registry] would slash solver=${params.solverAddress} intent=${params.intentId} reason="${params.reason}" (${detail})`, ); - return { submitted: false, simulated: false, dryRun: false, detail }; + return { submitted: false, simulated: false, dryRun: false, failed: false, detail }; } try { @@ -158,7 +165,7 @@ export class SolverRegistryService { this.logger.error( `[solver-registry] slash simulation errored for solver=${params.solverAddress} intent=${params.intentId}: ${detail}`, ); - return { submitted: false, simulated: true, dryRun: false, detail }; + return { submitted: false, simulated: true, dryRun: false, failed: true, detail }; } // TODO: Once issue #23 confirms the real contract interface, replace @@ -172,7 +179,7 @@ export class SolverRegistryService { this.logger.log( `[solver-registry] simulated slash tx for solver=${params.solverAddress} intent=${params.intentId} (${detail})`, ); - return { submitted: false, simulated: true, dryRun: false, detail }; + return { submitted: false, simulated: true, dryRun: false, failed: false, detail }; } catch (err) { // Issue #300 — the SDK may include serialized transaction/XDR details in // thrown errors; do not log the signing key or any raw secret here. @@ -180,7 +187,7 @@ export class SolverRegistryService { this.logger.error( `[solver-registry] slash call errored for solver=${params.solverAddress} intent=${params.intentId}: ${detail}`, ); - return { submitted: false, simulated: false, dryRun: false, detail }; + return { submitted: false, simulated: false, dryRun: false, failed: true, detail }; } } diff --git a/src/soroban/stellar-tx.before-submit.spec.ts b/src/soroban/stellar-tx.before-submit.spec.ts new file mode 100644 index 00000000..d9f0b3b9 --- /dev/null +++ b/src/soroban/stellar-tx.before-submit.spec.ts @@ -0,0 +1,96 @@ +import { ConfigService } from "@nestjs/config"; +import { Keypair, Networks, Transaction } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchService } from "../killswitch/killswitch.service"; +import { SignerService } from "./signer.service"; +import { SorobanService } from "./soroban.service"; +import { INVOKE_TX_TIMEOUT_SECONDS, StellarTxService } from "./stellar-tx.service"; +import { TxConfirmationService } from "./tx-confirmation.service"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; + +/** + * Issue #396 — the outbox relay persists the signed envelope hash through + * `invokeContract`'s `beforeSubmit` hook, which must run after signing and + * strictly before broadcast. + */ +describe("StellarTxService.invokeContract beforeSubmit (#396)", () => { + const signerKeypair = Keypair.random(); + let events: string[]; + let submitTransaction: jest.Mock; + let service: StellarTxService; + + beforeEach(() => { + events = []; + submitTransaction = jest.fn(async (tx: Transaction) => { + events.push("submit"); + return { status: "PENDING", hash: tx.hash().toString("hex") }; + }); + const soroban = { + getFeeStats: jest.fn().mockRejectedValue(new Error("no stats")), + simulateTransaction: jest.fn().mockResolvedValue({ minResourceFee: "0", latestLedger: 1 }), + prepareTransaction: jest.fn(async (tx: Transaction) => tx), + submitTransaction, + }; + const signer = { + withNextSequence: (fn: (seq: string) => Promise) => fn("100"), + getPublicKey: () => signerKeypair.publicKey(), + getNetworkPassphrase: () => Networks.TESTNET, + sign: async (tx: Transaction) => { + tx.sign(signerKeypair); + return tx; + }, + }; + const confirmation = { + waitForConfirmation: jest.fn(async (hash: string) => ({ hash, status: "SUCCESS", durationMs: 1 })), + }; + const config = { + get: (key: string) => + ({ onchainDryRun: false, "stellar.network": "testnet", "stellar.feePercentile": "p50" })[key], + }; + service = new StellarTxService( + soroban as unknown as SorobanService, + signer as unknown as SignerService, + confirmation as unknown as TxConfirmationService, + config as unknown as ConfigService, + undefined, + { evaluateTarget: () => ({ paused: false }) } as unknown as KillSwitchService, + ); + }); + + it("hands the signed envelope hash to beforeSubmit before broadcasting", async () => { + let hookHash = ""; + const result = await service.invokeContract( + { contractId: CONTRACT_ID, method: "create_intent", args: [] }, + { + beforeSubmit: async (hash) => { + events.push("beforeSubmit"); + hookHash = hash; + }, + }, + ); + + expect(events).toEqual(["beforeSubmit", "submit"]); + const submitted = submitTransaction.mock.calls[0][0] as Transaction; + expect(submitted.signatures).toHaveLength(1); + expect(hookHash).toBe(submitted.hash().toString("hex")); + expect(result).toMatchObject({ hash: hookHash, status: "SUCCESS", dryRun: false }); + // The envelope's time bound is what makes NOT_FOUND-after-lease conclusive. + const maxTime = Number(submitted.timeBounds?.maxTime); + expect(maxTime - Math.floor(Date.now() / 1000)).toBeLessThanOrEqual(INVOKE_TX_TIMEOUT_SECONDS); + }); + + it("never broadcasts when beforeSubmit throws", async () => { + await expect( + service.invokeContract( + { contractId: CONTRACT_ID, method: "create_intent", args: [] }, + { + beforeSubmit: async () => { + throw new Error("lost lease"); + }, + }, + ), + ).rejects.toThrow("lost lease"); + expect(submitTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/src/soroban/stellar-tx.service.ts b/src/soroban/stellar-tx.service.ts index 1f5a4263..36eaadbc 100644 --- a/src/soroban/stellar-tx.service.ts +++ b/src/soroban/stellar-tx.service.ts @@ -87,6 +87,24 @@ export interface InvokeContractParams { args: xdr.ScVal[]; } +/** + * Time bound (seconds) on every transaction built by {@link StellarTxService.invokeContract}. + * After this the network rejects the envelope, which is what lets the outbox + * relay treat a NOT_FOUND envelope hash as "never landed, safe to rebuild" + * once its processing lease (OUTBOX_LEASE_SECONDS) has expired (issue #396). + */ +export const INVOKE_TX_TIMEOUT_SECONDS = 30; + +export interface InvokeContractOptions { + /** + * Called with the signed envelope's hash after signing and *before* + * broadcast (issue #396). If it throws, nothing is submitted. The outbox + * relay uses this to durably record the hash so a crash mid-submit can be + * detected on retry instead of double-submitting. + */ + beforeSubmit?: (envelopeHash: string) => Promise; +} + export interface InvokeContractResult { hash: string; status: string; @@ -263,7 +281,10 @@ export class StellarTxService { * 3. Sign and submit the (now-prepared) original transaction. * 4. Confirm and return the result. */ - async invokeContract(params: InvokeContractParams): Promise { + async invokeContract( + params: InvokeContractParams, + options: InvokeContractOptions = {}, + ): Promise { // Issue #477 — the last gate before anything touches the chain. Checking // here rather than only in controllers also covers background callers (the // sweeper, event ingestion) that never pass through an HTTP guard. @@ -300,7 +321,7 @@ export class StellarTxService { .addOperation( new Contract(params.contractId).call(params.method, ...params.args), ) - .setTimeout(30) + .setTimeout(INVOKE_TX_TIMEOUT_SECONDS) .build(); let simulation = await this.sorobanService.simulateTransaction(rawTx); @@ -331,6 +352,8 @@ export class StellarTxService { const prepared = await this.sorobanService.prepareTransaction(rawTx); const signed = await this.signerService.sign(prepared as Transaction); + await options.beforeSubmit?.(signed.hash().toString("hex")); + const submittedAt = Date.now(); const sendResponse = await this.sorobanService.submitTransaction(signed); diff --git a/src/soroban/tx-confirmation.service.spec.ts b/src/soroban/tx-confirmation.service.spec.ts index f4814bfb..e69de29b 100644 --- a/src/soroban/tx-confirmation.service.spec.ts +++ b/src/soroban/tx-confirmation.service.spec.ts @@ -1,203 +0,0 @@ -import { SorobanRpc } from "@stellar/stellar-sdk"; -import { TxConfirmationService, TrackTxOptions } from "./tx-confirmation.service"; -import { PrismaService } from "../prisma/prisma.service"; -import { MetricsService } from "../metrics/metrics.service"; -import { SorobanService } from "./soroban.service"; -import { SignerService } from "./signer.service"; -import { FeeEscalationPolicy } from "./fee-escalation-policy"; -import { TxConfirmed, TxFailed, TxExpired } from "./tx-events"; - -// EventEmitter2 is ESM-only; mock it for Jest's CommonJS environment -jest.mock("@nestjs/event-emitter", () => ({ - EventEmitter2: jest.fn().mockImplementation(() => ({ emit: jest.fn() })), -})); - -function makePrisma(overrides: Partial<{ - queryRaw: jest.Mock; - update: jest.Mock; - upsert: jest.Mock; -}> = {}): PrismaService { - return { - pendingTransaction: { - upsert: overrides.upsert ?? jest.fn().mockResolvedValue({}), - update: overrides.update ?? jest.fn().mockResolvedValue({}), - }, - $queryRaw: overrides.queryRaw ?? jest.fn().mockResolvedValue([]), - } as unknown as PrismaService; -} - -function makeSoroban( - getTransactionResult: SorobanRpc.Api.GetTransactionResponse, -): SorobanService { - return { - getTransaction: jest.fn().mockResolvedValue(getTransactionResult), - submitTransaction: jest.fn().mockResolvedValue({ status: "PENDING" }), - } as unknown as SorobanService; -} - -function makeSigner(configured = false): SignerService { - return { - isConfigured: jest.fn().mockReturnValue(configured), - getNetworkPassphrase: jest.fn().mockReturnValue("Test SDF Network ; September 2015"), - getSecretKey: jest.fn().mockReturnValue(""), - } as unknown as SignerService; -} - -function makeMetrics(): MetricsService { - return { - txConfirmationOutcomes: { inc: jest.fn() }, - txConfirmationLatency: { observe: jest.fn() }, - } as unknown as MetricsService; -} - -function makeFeePolicy(shouldEscalate = false): FeeEscalationPolicy { - return { - shouldEscalate: jest.fn().mockReturnValue(shouldEscalate), - buildFeeBump: jest.fn().mockResolvedValue(null), - } as unknown as FeeEscalationPolicy; -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function makeEvents(): any { - return { emit: jest.fn() }; -} - -const BASE_ROW = { - id: 1n, - tx_hash: "abc123", - tx_xdr: "AAAA", - intent_id: "intent-1", - channel_key: null, - status: "pending", - max_track_until: Math.floor(Date.now() / 1000) + 300, - attempts: 0, - next_poll_at: Math.floor(Date.now() / 1000) - 1, - last_fee_stroops: null, - fee_bump_count: 0, - created_at: new Date(), -}; - -describe("TxConfirmationService", () => { - describe("track()", () => { - it("upserts a pending transaction record", async () => { - const upsert = jest.fn().mockResolvedValue({}); - const prisma = makePrisma({ upsert }); - const svc = new TxConfirmationService( - prisma, - makeSoroban({ status: SorobanRpc.Api.GetTransactionStatus.NOT_FOUND } as SorobanRpc.Api.GetTransactionResponse), - makeSigner(), - makeFeePolicy(), - makeMetrics(), - makeEvents(), - ); - - const opts: TrackTxOptions = { - txHash: "abc123", - txXdr: "AAAA", - intentId: "intent-1", - maxTrackUntil: Math.floor(Date.now() / 1000) + 300, - }; - await svc.track(opts); - - expect(upsert).toHaveBeenCalledWith( - expect.objectContaining({ - where: { txHash: "abc123" }, - create: expect.objectContaining({ txHash: "abc123", status: "pending" }), - }), - ); - }); - }); - - describe("processRow() via pollBatch()", () => { - it("marks confirmed and emits TxConfirmed on SUCCESS", async () => { - const update = jest.fn().mockResolvedValue({}); - const prisma = makePrisma({ queryRaw: jest.fn().mockResolvedValue([BASE_ROW]), update }); - const soroban = makeSoroban({ - status: SorobanRpc.Api.GetTransactionStatus.SUCCESS, - ledger: 42, - } as SorobanRpc.Api.GetSuccessfulTransactionResponse); - const events = makeEvents(); - const metrics = makeMetrics(); - - const svc = new TxConfirmationService( - prisma, soroban, makeSigner(), makeFeePolicy(), metrics, events, - ); - - // Call the private pollBatch via onModuleInit's interval—instead directly - // trigger it by casting to any - await (svc as unknown as { pollBatch: () => Promise }).pollBatch(); - - expect(update).toHaveBeenCalledWith( - expect.objectContaining({ data: { status: "confirmed" } }), - ); - expect(events.emit).toHaveBeenCalledWith(TxConfirmed.EVENT, expect.any(TxConfirmed)); - expect(metrics.txConfirmationOutcomes.inc).toHaveBeenCalledWith({ status: "confirmed" }); - }); - - it("marks failed and emits TxFailed on FAILED (no fee bump)", async () => { - const update = jest.fn().mockResolvedValue({}); - const prisma = makePrisma({ queryRaw: jest.fn().mockResolvedValue([BASE_ROW]), update }); - const soroban = makeSoroban({ - status: SorobanRpc.Api.GetTransactionStatus.FAILED, - } as SorobanRpc.Api.GetFailedTransactionResponse); - const events = makeEvents(); - const metrics = makeMetrics(); - - const svc = new TxConfirmationService( - prisma, soroban, makeSigner(false), makeFeePolicy(false), metrics, events, - ); - await (svc as unknown as { pollBatch: () => Promise }).pollBatch(); - - expect(update).toHaveBeenCalledWith( - expect.objectContaining({ data: { status: "failed" } }), - ); - expect(events.emit).toHaveBeenCalledWith(TxFailed.EVENT, expect.any(TxFailed)); - }); - - it("schedules retry with backoff on NOT_FOUND", async () => { - const update = jest.fn().mockResolvedValue({}); - const prisma = makePrisma({ queryRaw: jest.fn().mockResolvedValue([BASE_ROW]), update }); - const soroban = makeSoroban({ - status: SorobanRpc.Api.GetTransactionStatus.NOT_FOUND, - } as SorobanRpc.Api.GetTransactionResponse); - - const svc = new TxConfirmationService( - prisma, soroban, makeSigner(), makeFeePolicy(false), makeMetrics(), makeEvents(), - ); - await (svc as unknown as { pollBatch: () => Promise }).pollBatch(); - - expect(update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ attempts: 1 }), - }), - ); - }); - - it("marks expired and emits TxExpired when past maxTrackUntil", async () => { - const expiredRow = { - ...BASE_ROW, - max_track_until: Math.floor(Date.now() / 1000) - 10, // already expired - }; - const update = jest.fn().mockResolvedValue({}); - const prisma = makePrisma({ queryRaw: jest.fn().mockResolvedValue([expiredRow]), update }); - const events = makeEvents(); - const metrics = makeMetrics(); - - const svc = new TxConfirmationService( - prisma, - makeSoroban({ status: SorobanRpc.Api.GetTransactionStatus.NOT_FOUND } as SorobanRpc.Api.GetTransactionResponse), - makeSigner(), - makeFeePolicy(), - metrics, - events, - ); - await (svc as unknown as { pollBatch: () => Promise }).pollBatch(); - - expect(update).toHaveBeenCalledWith( - expect.objectContaining({ data: { status: "expired" } }), - ); - expect(events.emit).toHaveBeenCalledWith(TxExpired.EVENT, expect.any(TxExpired)); - expect(metrics.txConfirmationOutcomes.inc).toHaveBeenCalledWith({ status: "expired" }); - }); - }); -});