From e88dfb56349a0e8dc4a02b8f024ef742148479b3 Mon Sep 17 00:00:00 2001 From: frienzy Date: Mon, 28 Sep 2026 17:49:33 +0100 Subject: [PATCH] feat(onchain): transactional outbox and durable slashing saga (#396 #397) Outbox (#396): - onchain_outbox table; intent create/accept/fill/cancel and the mirroring outbox row commit in one Prisma transaction (IIntentsUnitOfWork) - OutboxRelayService: FOR UPDATE SKIP LOCKED claims, per-intent ordering, signed envelope hash persisted before submit, attempts-fenced writes, exponential backoff, dead-lettering with metric + alert, dry-run aware - TxConfirmationService, live StellarTxService.invokeContract submit path - POST /api/v1/admin/outbox/:id/requeue Slashing saga (#397): - pending_slashes table, detected -> challenge_window -> submitted -> confirmed | cancelled, exactly-once via unique intent_id - challenge window, on-chain re-verification with clock-skew tolerance, solver fill-proof and admin cancellation, compensation via rollbackPenalty - sweeper now only detects; runbook docs/runbooks/slash-cancellation.md Also fixes SorobanModule's missing imports and the e2e stellar-sdk mock. Closes #396 Closes #397 --- .env.example | 12 + .env.mainnet.example | 12 + .env.staging.example | 12 + .env.testnet.example | 12 + CHANGELOG.md | 15 + docs/architecture/onchain-settlement.md | 83 +++ docs/runbooks/alerts/onchain-writes.rules.yml | 33 ++ docs/runbooks/on-call.md | 148 +++-- docs/runbooks/slash-cancellation.md | 112 ++++ .../20260928000001_onchain_outbox/down.sql | 5 + .../migration.sql | 26 + .../20260928000002_pending_slashes/down.sql | 5 + .../migration.sql | 33 ++ prisma/schema.prisma | 93 +++ src/common/stellar-signature.ts | 8 + src/config/configuration.ts | 35 ++ src/config/env.validation.ts | 14 + src/intents/dto/slash.dto.ts | 52 ++ .../intents-sweeper.manual-trigger.spec.ts | 10 +- src/intents/intents-sweeper.service.spec.ts | 33 +- src/intents/intents-sweeper.service.ts | 28 +- src/intents/intents.gateway.ts | 1 + src/intents/intents.module.ts | 50 +- src/intents/intents.service.spec.ts | 154 +++-- src/intents/intents.service.ts | 137 +++-- src/intents/intents.unit-of-work.spec.ts | 57 ++ src/intents/intents.unit-of-work.ts | 73 +++ src/intents/prisma-intents.repository.ts | 5 +- src/intents/slashes.controller.spec.ts | 87 +++ src/intents/slashes.controller.ts | 86 +++ src/intents/slashing-pipeline.service.spec.ts | 546 ++++++++++++++++++ src/intents/slashing-pipeline.service.ts | 459 +++++++++++++++ src/metrics/metrics.service.ts | 51 ++ .../pending-slashes.repository.spec.ts | 127 ++++ src/solvers/pending-slashes.repository.ts | 167 ++++++ .../prisma-pending-slashes.repository.ts | 127 ++++ src/solvers/solvers.service.spec.ts | 25 + src/solvers/solvers.service.ts | 15 +- src/soroban/fill-verifier.service.spec.ts | 136 +++++ src/soroban/fill-verifier.service.ts | 151 +++++ src/soroban/outbox-admin.controller.spec.ts | 34 ++ src/soroban/outbox-admin.controller.ts | 35 ++ src/soroban/outbox-operations.spec.ts | 65 +++ src/soroban/outbox-operations.ts | 118 ++++ src/soroban/outbox-relay.service.spec.ts | 475 +++++++++++++++ src/soroban/outbox-relay.service.ts | 265 +++++++++ src/soroban/outbox.repository.spec.ts | 145 +++++ src/soroban/outbox.repository.ts | 269 +++++++++ src/soroban/prisma-outbox.repository.spec.ts | 129 +++++ src/soroban/prisma-outbox.repository.ts | 229 ++++++++ src/soroban/solver-registry.service.spec.ts | 2 + src/soroban/solver-registry.service.ts | 15 +- src/soroban/soroban.module.ts | 4 + src/soroban/stellar-tx.before-submit.spec.ts | 96 +++ src/soroban/stellar-tx.service.ts | 27 +- src/soroban/tx-confirmation.service.spec.ts | 23 + src/soroban/tx-confirmation.service.ts | 42 ++ 57 files changed, 5032 insertions(+), 176 deletions(-) create mode 100644 docs/runbooks/alerts/onchain-writes.rules.yml create mode 100644 docs/runbooks/slash-cancellation.md create mode 100644 prisma/migrations/20260928000001_onchain_outbox/down.sql create mode 100644 prisma/migrations/20260928000001_onchain_outbox/migration.sql create mode 100644 prisma/migrations/20260928000002_pending_slashes/down.sql create mode 100644 prisma/migrations/20260928000002_pending_slashes/migration.sql create mode 100644 src/intents/dto/slash.dto.ts create mode 100644 src/intents/intents.unit-of-work.spec.ts create mode 100644 src/intents/intents.unit-of-work.ts create mode 100644 src/intents/slashes.controller.spec.ts create mode 100644 src/intents/slashes.controller.ts create mode 100644 src/intents/slashing-pipeline.service.spec.ts create mode 100644 src/intents/slashing-pipeline.service.ts create mode 100644 src/solvers/pending-slashes.repository.spec.ts create mode 100644 src/solvers/pending-slashes.repository.ts create mode 100644 src/solvers/prisma-pending-slashes.repository.ts create mode 100644 src/soroban/fill-verifier.service.spec.ts create mode 100644 src/soroban/fill-verifier.service.ts create mode 100644 src/soroban/outbox-admin.controller.spec.ts create mode 100644 src/soroban/outbox-admin.controller.ts create mode 100644 src/soroban/outbox-operations.spec.ts create mode 100644 src/soroban/outbox-operations.ts create mode 100644 src/soroban/outbox-relay.service.spec.ts create mode 100644 src/soroban/outbox-relay.service.ts create mode 100644 src/soroban/outbox.repository.spec.ts create mode 100644 src/soroban/outbox.repository.ts create mode 100644 src/soroban/prisma-outbox.repository.spec.ts create mode 100644 src/soroban/prisma-outbox.repository.ts create mode 100644 src/soroban/stellar-tx.before-submit.spec.ts create mode 100644 src/soroban/tx-confirmation.service.spec.ts diff --git a/.env.example b/.env.example index 6ede42f5..c85752f1 100644 --- a/.env.example +++ b/.env.example @@ -311,3 +311,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/.env.mainnet.example b/.env.mainnet.example index 8b625085..03a36d9b 100644 --- a/.env.mainnet.example +++ b/.env.mainnet.example @@ -200,3 +200,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/.env.staging.example b/.env.staging.example index 91ba2b28..b532e10d 100644 --- a/.env.staging.example +++ b/.env.staging.example @@ -120,3 +120,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/.env.testnet.example b/.env.testnet.example index e12d80fd..3cb946b9 100644 --- a/.env.testnet.example +++ b/.env.testnet.example @@ -181,3 +181,15 @@ HEALTH_READY_SUCCESS_THRESHOLD=2 HEALTH_EVENT_LOOP_MAX_LAG_MS=1000 # Soroban RPC endpoints for the quorum check (default: SOROBAN_RPC_URL). SOROBAN_RPC_HEALTH_URLS= +# ── Transactional outbox relay (issue #396) ────────────────────────────────── +OUTBOX_RELAY_ENABLED=true +OUTBOX_RELAY_INTERVAL_MS=2000 +OUTBOX_RELAY_BATCH_SIZE=10 +OUTBOX_MAX_ATTEMPTS=8 +# Must exceed the signed transaction's 30 s time bound. +OUTBOX_LEASE_SECONDS=120 + +# ── Slashing saga (issue #397) ─────────────────────────────────────────────── +SLASH_CHALLENGE_WINDOW_SECONDS=600 +SLASH_CLOCK_SKEW_TOLERANCE_SECONDS=30 +SLASH_MAX_SUBMIT_ATTEMPTS=5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f6a18d6..65f6b4d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,17 @@ Commit message format is enforced via [commitlint](https://commitlint.js.org/) s ## [Unreleased] ### Added +- Transactional outbox for on-chain writes: `onchain_outbox` table, intent change + outbox + row committed in one Prisma transaction, `OutboxRelayService` (SKIP LOCKED claims, per-intent + ordering, envelope hash persisted before submit, dead-lettering with alert), + `TxConfirmationService`, live `StellarTxService.invokeContract` submit path, and + `POST /api/v1/admin/outbox/:id/requeue` (Closes #396) +- Durable solver slashing saga: `pending_slashes` table (exactly-once per intent), + configurable challenge window, on-chain re-verification with clock-skew tolerance, + solver fill-proof and admin cancellation endpoints, compensation via `rollbackPenalty`, + metrics, alert rules and `docs/runbooks/slash-cancellation.md` (Closes #397) +- Admin slash cancellation and outbox requeue use the shared `AdminGuard` RBAC (`x-admin-key`) + and are recorded in `admin_audit_log` - `scripts/generate-client.ts` — generates a typed TypeScript API client from the live OpenAPI spec using `openapi-typescript` v7; output committed to `src/generated/` (Closes #134) @@ -72,6 +83,10 @@ Commit message format is enforced via [commitlint](https://commitlint.js.org/) s `npm run test:scripts` (see `prisma/migrations/README.md`) ### Fixed +- `SorobanModule` referenced `forwardRef`/`IntentsModule` without importing them; it now + imports `SolversModule` (what `EventIngestionService` actually needs) +- e2e `@stellar/stellar-sdk` mock now re-exports the real SDK and stubs only + `SorobanRpc.Server` (it previously lacked `Networks`, `Keypair`, … so no e2e suite could load) - `IntentsService.create()` idempotency-key handling is now race-safe — concurrent requests carrying the same key synchronously claim an in-flight slot before any `await`, so exactly one intent is created and the losers replay its result diff --git a/docs/architecture/onchain-settlement.md b/docs/architecture/onchain-settlement.md index f26507a0..b1330f01 100644 --- a/docs/architecture/onchain-settlement.md +++ b/docs/architecture/onchain-settlement.md @@ -1,5 +1,10 @@ # Architecture: On-Chain Settlement (Target Design) +> **Update (issues #396 / #397):** the write-side plumbing below now exists — +> see [Transactional outbox](#transactional-outbox) and +> [Slashing saga](#slashing-saga). Contract method names remain provisional +> until the ADR (issue #19) fixes the interface. +> > **Status: target architecture, not yet implemented.** As of this writing, > `IntentsService` and `SolversService` are in-memory `Map`s > (`src/intents/intents.service.ts`, `src/solvers/solvers.service.ts`), and @@ -166,6 +171,84 @@ transactions. above on a narrower surface (sweeper-triggered only, no user-facing HTTP write path). +## Transactional outbox + +*Implemented — issue #396.* Stage 1 above ("HTTP request → Soroban tx") no +longer submits inside the request. Submitting a transaction and writing +Postgres as two separate steps is a dual write: a crash in between leaves the +database saying "accepted" while the transaction never went out, or the +reverse. Instead: + +```mermaid +sequenceDiagram + participant API as IntentsService + participant DB as Postgres + participant Relay as OutboxRelayService + participant Chain as Soroban + + API->>DB: BEGIN; UPDATE intents …; INSERT onchain_outbox (pending); COMMIT + loop every OUTBOX_RELAY_INTERVAL_MS + Relay->>DB: claim due head-of-intent rows (FOR UPDATE SKIP LOCKED) → processing + Relay->>Relay: build + simulate + sign + Relay->>DB: store envelope_hash (fenced on attempts) + Relay->>Chain: sendTransaction + Relay->>DB: status = submitted, tx_hash + Relay->>Chain: getTransaction(tx_hash) (TxConfirmationService) + Relay->>DB: status = confirmed + end +``` + +- **Atomicity.** `IntentsService` runs `create` / `acceptIfOpen` / + `fillIfAccepted` / `cancelIfOpen` through `IIntentsUnitOfWork` + (`src/intents/intents.unit-of-work.ts`). The Prisma adapter wraps the + intent write and the `onchain_outbox` insert in one `$transaction`. A + transition whose guard fails (lost race) enqueues nothing. With + `ONCHAIN_INTENTS_ENABLED=false` the outbox is bypassed entirely. +- **Fail fast.** The payload is encoded to contract arguments at enqueue time, + so malformed input (bad address, non-integer amount) fails the HTTP request + rather than becoming a poison row. +- **Ordering.** `onchain_outbox.id` is a sequence. A row is only claimable when + every earlier row for the same `intent_id` is `confirmed` or `simulated`, + so one intent's operations apply in order while different intents run in + parallel. `SKIP LOCKED` lets several relay instances share the work. +- **Crash idempotency.** The signed envelope hash is persisted *before* + broadcast. A worker that dies mid-submit leaves the row `processing`; after + `OUTBOX_LEASE_SECONDS` it is reclaimed, and the relay first looks the stored + hash up: `SUCCESS` → confirm without resubmitting; `FAILED` → retry; + `NOT_FOUND` → rebuild. `NOT_FOUND` is conclusive only because the lease + (120 s) outlives the transaction's 30 s time bound + (`INVOKE_TX_TIMEOUT_SECONDS`). Every post-claim write is fenced on + `attempts`, so a worker whose lease expired cannot clobber a reclaimed row. +- **Retries and poison rows.** Failures back off exponentially (1 s doubling, + capped at 5 min). After `OUTBOX_MAX_ATTEMPTS` claims a row becomes `dead`, + `vortex_outbox_dead_total` increments (alerted), and it blocks its intent + until requeued via `POST /api/v1/admin/outbox/:id/requeue`. +- **Dry run.** Under `ONCHAIN_DRY_RUN=true` rows end as `simulated` (terminal). + They are not replayed when dry-run is later switched off. +- **Out of scope:** cross-service delivery (Kafka etc.). + +Row lifecycle: `pending → processing → submitted → confirmed`, with +`processing → simulated` (dry run), back to `pending` on retry, and `dead` +past the attempt limit. + +## Slashing saga + +*Implemented — issue #397.* The `accepted → slashed` row of the mapping table +runs as a durable saga (`src/intents/slashing-pipeline.service.ts`, table +`pending_slashes`): + +`detected → challenge_window → submitted → confirmed | cancelled` + +The sweeper only *detects*. The slash is broadcast after a configurable +challenge window, and only after the chain has been re-checked for a fill that +landed by `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` (by ledger close +time, so server clock skew can't cause a wrong slash). A unique constraint on +`intent_id` makes it exactly-once. Cancellation (solver fill-proof, admin, or +giving up after `SLASH_MAX_SUBMIT_ATTEMPTS`) runs the compensation +(`SolversService.rollbackPenalty`, intent leaves `slashed`) exactly once. The +operator procedure is in +[`docs/runbooks/slash-cancellation.md`](../runbooks/slash-cancellation.md). + ## Persistence layer Both `IntentsService` and `SolversService` delegate all storage to an diff --git a/docs/runbooks/alerts/onchain-writes.rules.yml b/docs/runbooks/alerts/onchain-writes.rules.yml new file mode 100644 index 00000000..9c0affa1 --- /dev/null +++ b/docs/runbooks/alerts/onchain-writes.rules.yml @@ -0,0 +1,33 @@ +# Prometheus alerting rules for the on-chain write path. +# Metric names are defined in src/metrics/metrics.service.ts — keep in sync. +groups: + - name: vortex-onchain-writes + rules: + # Issue #396 — a poison outbox row was dead-lettered. Later operations for + # the same intent are blocked until it is requeued. + - alert: VortexOutboxRowDead + expr: increase(vortex_outbox_dead_total[5m]) > 0 + labels: + severity: page + annotations: + summary: "Outbox row moved to dead after exhausting OUTBOX_MAX_ATTEMPTS" + runbook: docs/runbooks/on-call.md#scenario-g--outbox-rows-dead-or-backlogged + + # Issue #396 — the relay is not draining (RPC outage, relay disabled, signer broken). + - alert: VortexOutboxBacklog + expr: sum(vortex_outbox_rows{status=~"pending|processing|submitted"}) > 100 + for: 15m + labels: + severity: warn + annotations: + summary: "More than 100 on-chain writes waiting in the outbox for 15m" + runbook: docs/runbooks/on-call.md#scenario-g--outbox-rows-dead-or-backlogged + + # Issue #397 — the slashing saga gave up on a slash (submit kept failing). + - alert: VortexSlashSubmitFailed + expr: increase(vortex_slash_pipeline_transitions_total{to_state="cancelled",reason="submit_failed"}[15m]) > 0 + labels: + severity: page + annotations: + summary: "A detected solver slash was cancelled after repeated submission failures" + runbook: docs/runbooks/slash-cancellation.md#submit-failed-slashes diff --git a/docs/runbooks/on-call.md b/docs/runbooks/on-call.md index 276f33bb..0ebc3a63 100644 --- a/docs/runbooks/on-call.md +++ b/docs/runbooks/on-call.md @@ -19,8 +19,9 @@ 8. [Scenario E — Synthetic canary failing](#scenario-e--synthetic-canary-failing) 9. [Health probes](#health-probes) 10. [Scenario F — WebSocket backplane and slow consumers](#scenario-f--websocket-backplane-and-slow-consumers) -11. [Key configuration](#key-configuration) -12. [Escalation path](#escalation-path) +11. [Scenario G — Outbox rows dead or backlogged](#scenario-g--outbox-rows-dead-or-backlogged) +12. [Key configuration](#key-configuration) +13. [Escalation path](#escalation-path) --- @@ -426,6 +427,105 @@ Alerts `VortexCanaryConsecutiveFailures`, `VortexCanaryFundsLow`, Canary intents are excluded from public stats and leaderboards via `CANARY_ADDRESSES`; if they show up there, that variable is missing on the API. +## Scenario F — WebSocket backplane and slow consumers + +**Backplane (issue #454).** With `WS_BACKPLANE=redis` every replica publishes +events into a Redis stream (`vortex:ws:events`) with a global sequence number +(`vortex:ws:seq`) and delivers from that stream, so clients on any replica +see the same events, in the same order, with the same `seq`, and replay works +against any replica. Publishing is queued in the background — request +handlers never wait for Redis. + +- **Redis down:** `/health/ready` on WS-role pods goes 503 (`ws_backplane` + down) and `vortex_ws_backplane_connected` drops to 0. Publishes queue + (bounded) and are retried in order; on recovery each replica resumes the + stream from the last event it delivered — no loss, duplicates or + reordering. Watch `vortex_ws_backplane_dropped_total{reason="queue_full"}` + for events dropped during a long outage. +- **Latency:** `vortex_ws_backplane_publish_duration_seconds`. + +**Connection limits and slow consumers (issue #455).** + +- Connections over `WS_MAX_CONNECTIONS` or `WS_MAX_CONNECTIONS_PER_IP` are + closed with 1013 (`vortex_ws_connections_rejected_total{reason}`). Behind a + load balancer set `WS_TRUST_PROXY_HOPS` to the number of proxies, or every + client shares the proxy's IP. +- Clients over the inbound token bucket get `rate_limited` frames and are + closed with 1008 after `WS_RATE_LIMIT_MAX_VIOLATIONS` + (`vortex_ws_rate_limited_total{action}`). Frames over + `WS_MAX_PAYLOAD_BYTES` close the socket with 1009. +- Slow consumers: once a socket's buffer passes `WS_OUTBOUND_BUFFER_BYTES`, + messages queue (at most `WS_OUTBOUND_QUEUE_MAX`); beyond that the oldest are + dropped (`vortex_ws_outbound_dropped_total`) or, with + `WS_SLOW_CONSUMER_POLICY=disconnect`, the client is closed + (`vortex_ws_slow_consumer_disconnects_total`). Clients recover dropped + events with `replay` — the solver SDK does this automatically. +- Solvers can authenticate with a SEP-10 JWT (`?token=`, `Authorization: + Bearer`, or `{ "type": "auth", "token" }`) when `AUTH_JWT_SECRET` is set, + in addition to signed `auth` frames. Anonymous connections still receive + the public feed. + +--- + +## Scenario G — Outbox rows dead or backlogged + +On-chain writes (intent create/accept/fill/cancel) are committed to the +`onchain_outbox` table in the same transaction as the intent change, and +`OutboxRelayService` submits them afterwards (issue #396). Alert rules live in +[`alerts/onchain-writes.rules.yml`](alerts/onchain-writes.rules.yml). + +### Symptoms + +- `VortexOutboxRowDead`: `vortex_outbox_dead_total` increased. Logs contain + `[outbox] ALERT row (...) moved to dead after N attempts: `. +- `VortexOutboxBacklog`: `vortex_outbox_rows{status="pending"}` keeps growing. + +### Impact + +A dead row **blocks every later row for the same intent** (per-intent +ordering), so that intent's on-chain state stops advancing. Other intents +are unaffected. The API keeps serving from the database. + +### Diagnosis + +```sql +SELECT id, intent_id, operation, attempts, last_error, updated_at +FROM onchain_outbox WHERE status = 'dead' ORDER BY id; + +SELECT status, count(*) FROM onchain_outbox GROUP BY status; +``` + +| `last_error` | Likely cause | +|---|---| +| `SETTLEMENT_CONTRACT_ID is not configured` | Env misconfiguration | +| `signer is not configured` | `SOROBAN_SIGNING_KEY` missing | +| `sendTransaction returned ERROR` / `simulation error` | Contract rejected the call — inspect the payload | +| `RPC ...` / timeouts | Scenario A (RPC downtime) | + +Backlog with no dead rows usually means the relay is off +(`OUTBOX_RELAY_ENABLED=false` — look for `[outbox] relay disabled` at boot) +or Scenario A. + +### Remediation + +Fix the root cause first, then requeue (resets attempts; the intent unblocks): + +```bash +curl -s -X POST -H "x-admin-key: $ADMIN_KEY" \ + "$API/api/v1/admin/outbox//requeue" +``` + +Never delete outbox rows or edit `status` by hand while the relay is running; +the relay's writes are fenced on `attempts` and a manual edit can be +overwritten. A row whose operation must be abandoned (e.g. the contract will +never accept it) needs a code/ADR decision — escalate. + +### Crash safety (why duplicates don't happen) + +The relay stores the signed envelope hash before broadcasting. After a crash +the row is reclaimed once `OUTBOX_LEASE_SECONDS` (default 120 s, longer than +the 30 s transaction time bound) has passed; the relay looks that hash up and +confirms the row if it landed, instead of resubmitting. --- @@ -475,46 +575,6 @@ readinessProbe: --- -## Scenario F — WebSocket backplane and slow consumers - -**Backplane (issue #454).** With `WS_BACKPLANE=redis` every replica publishes -events into a Redis stream (`vortex:ws:events`) with a global sequence number -(`vortex:ws:seq`) and delivers from that stream, so clients on any replica -see the same events, in the same order, with the same `seq`, and replay works -against any replica. Publishing is queued in the background — request -handlers never wait for Redis. - -- **Redis down:** `/health/ready` on WS-role pods goes 503 (`ws_backplane` - down) and `vortex_ws_backplane_connected` drops to 0. Publishes queue - (bounded) and are retried in order; on recovery each replica resumes the - stream from the last event it delivered — no loss, duplicates or - reordering. Watch `vortex_ws_backplane_dropped_total{reason="queue_full"}` - for events dropped during a long outage. -- **Latency:** `vortex_ws_backplane_publish_duration_seconds`. - -**Connection limits and slow consumers (issue #455).** - -- Connections over `WS_MAX_CONNECTIONS` or `WS_MAX_CONNECTIONS_PER_IP` are - closed with 1013 (`vortex_ws_connections_rejected_total{reason}`). Behind a - load balancer set `WS_TRUST_PROXY_HOPS` to the number of proxies, or every - client shares the proxy's IP. -- Clients over the inbound token bucket get `rate_limited` frames and are - closed with 1008 after `WS_RATE_LIMIT_MAX_VIOLATIONS` - (`vortex_ws_rate_limited_total{action}`). Frames over - `WS_MAX_PAYLOAD_BYTES` close the socket with 1009. -- Slow consumers: once a socket's buffer passes `WS_OUTBOUND_BUFFER_BYTES`, - messages queue (at most `WS_OUTBOUND_QUEUE_MAX`); beyond that the oldest are - dropped (`vortex_ws_outbound_dropped_total`) or, with - `WS_SLOW_CONSUMER_POLICY=disconnect`, the client is closed - (`vortex_ws_slow_consumer_disconnects_total`). Clients recover dropped - events with `replay` — the solver SDK does this automatically. -- Solvers can authenticate with a SEP-10 JWT (`?token=`, `Authorization: - Bearer`, or `{ "type": "auth", "token" }`) when `AUTH_JWT_SECRET` is set, - in addition to signed `auth` frames. Anonymous connections still receive - the public feed. - ---- - ## Key configuration | Variable | Default | Effect | @@ -532,6 +592,10 @@ handlers never wait for Redis. | `ADMIN_API_KEYS` | empty (admin APIs disabled) | `id:role:secret` entries for admin / superadmin endpoints | | `PROCESS_ROLE` / `JOBS_DRIVER` | `all` / `memory` | Where job workers run; `bullmq` for multi-instance | | `CANARY_ADDRESSES` | empty | Canary accounts excluded from public stats | +| `OUTBOX_RELAY_ENABLED` | `true` | Kill switch for the outbox relay; rows accumulate while off | +| `OUTBOX_MAX_ATTEMPTS` | `8` | Claims before an outbox row is dead-lettered | +| `OUTBOX_LEASE_SECONDS` | `120` | Crash-reclaim delay; must exceed the 30 s tx time bound | +| `SLASH_CHALLENGE_WINDOW_SECONDS` | `600` | Delay before a detected slash may be broadcast — see [slash-cancellation.md](slash-cancellation.md) | --- diff --git a/docs/runbooks/slash-cancellation.md b/docs/runbooks/slash-cancellation.md new file mode 100644 index 00000000..19fb908f --- /dev/null +++ b/docs/runbooks/slash-cancellation.md @@ -0,0 +1,112 @@ +# Runbook — Solver Slash Review and Manual Cancellation + +Issue #397. Applies to the slashing saga in +`src/intents/slashing-pipeline.service.ts`. + +## How a slash flows + +``` +detected ──▶ challenge_window ──(window over + re-verified)──▶ submitted ──▶ confirmed + │ │ + └──── fill proof / admin / give-up ──▶ cancelled ◀┘ (tx failed past retries) +``` + +1. The sweeper finds an `accepted` intent past its fill deadline, marks it + `slashed`, bumps the solver's `fillsFailed` optimistically, and records a + row in `pending_slashes` (unique per `intent_id`, so exactly-once). +2. The row sits in `challenge_window` for `SLASH_CHALLENGE_WINDOW_SECONDS` + (default 600 s). **Nothing is sent on-chain during the window.** +3. When the window ends the pipeline re-checks the settlement contract's + `intent_filled` events. A fill whose **ledger close time** is at or before + `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` cancels the slash. If + the check itself fails (RPC down) the slash is retried later, never + submitted blind. +4. Otherwise `SolverRegistryService.slashSolver` is called and the row + becomes `submitted`, then `confirmed` once the transaction lands. + +Every cancellation runs the compensation exactly once: `fillsFailed` is +reverted (`SolversService.rollbackPenalty`) and the intent leaves `slashed` +(`filled` if a fill was proven, otherwise `expired`). An +`intent_slash_cancelled` WebSocket event and an audit-log entry are emitted. + +## Inspecting slashes + +```bash +# One intent (public) +curl -s $API/api/v1/slashes/ | jq + +# Everything waiting in the window (admin) +curl -s -H "x-admin-key: $ADMIN_KEY" \ + "$API/api/v1/admin/slashes?state=challenge_window" | jq +``` + +Admin routes use the shared admin RBAC (`AdminGuard`): send your key in the +`x-admin-key` header. Keys come from `ADMIN_API_KEYS` (`id:role:secret`); when +it is empty every admin route returns 401. The key's `id` is recorded as the +actor in the audit log. + +## Cancelling a slash manually + +Use when the slash is wrong — e.g. the fill landed on-chain but the event was +late, the solver's fill was blocked by a protocol incident, or the intent's +deadline was misconfigured. + +```bash +curl -s -X POST -H "x-admin-key: $ADMIN_KEY" \ + -H "Content-Type: application/json" \ + -d '{"note":""}' \ + "$API/api/v1/admin/slashes//cancel" | jq +``` + +Responses: + +| Status | Meaning | Action | +|---|---|---| +| 200 | Cancelled and compensated | Verify below | +| 404 | No slash for that intent | Check the intent id | +| 409 `submission in progress` | A worker holds the lease and may be broadcasting right now | Retry in ~2 minutes | +| 409 `state=submitted` / `confirmed` | Already broadcast — the backend can no longer stop it | Escalate: reversal must happen on-chain / via governance (out of scope here) | + +Verify: + +```bash +curl -s $API/api/v1/slashes/ | jq '.state, .cancelReason, .cancelledBy' +curl -s $API/api/v1/intents//audit | jq '.[-1]' +curl -s $API/api/v1/solvers/ | jq '.fillsFailed' +``` + +**Buying time:** if you need longer than the window to investigate a batch of +slashes, raise `SLASH_CHALLENGE_WINDOW_SECONDS` and restart. The new window +only applies to newly detected slashes; cancel existing ones individually. + +## Solver-initiated cancellation (fill proof) + +A solver whose fill landed in time can cancel during the window without +operator involvement: + +``` +POST /api/v1/slashes//fill-proof +{ "solver": "G...", "txHash": "<64 hex>", "signature": "" } +``` + +`signature` is the solver's Ed25519 signature of +`fill-proof:::`. The tx must be +successful, have closed by `fillDeadline + tolerance`, and emit +`intent_filled` for the intent from the settlement contract. + +## Submit-failed slashes + +Alert `VortexSlashSubmitFailed` fires when a slash was cancelled with reason +`submit_failed` after `SLASH_MAX_SUBMIT_ATTEMPTS` failures (RPC outage, +simulation errors). The solver was **not** penalised and the compensation has +already run. Check logs for `[slashing] ALERT giving up`, fix the root cause +(RPC, signing key, registry contract id), and decide with the service owner +whether the miss warrants a governance-level penalty. + +## Known limitations + +- While `ONCHAIN_DRY_RUN=true`, or until the registry submit path is un-gated + (issue #23), slashes stop at `submitted` with `simulated=true` and are never + confirmed. That is expected. +- `solver deregistered mid-window` does **not** cancel the slash — + deregistration must not be an escape hatch. diff --git a/prisma/migrations/20260928000001_onchain_outbox/down.sql b/prisma/migrations/20260928000001_onchain_outbox/down.sql new file mode 100644 index 00000000..9c812856 --- /dev/null +++ b/prisma/migrations/20260928000001_onchain_outbox/down.sql @@ -0,0 +1,5 @@ +-- Rollback for 20260928000001_onchain_outbox. +-- WARNING: drops any unsent outbox rows. Drain the relay (no pending/processing/ +-- submitted rows) before rolling back, or those on-chain writes are lost. +DROP TABLE IF EXISTS "onchain_outbox"; +DROP TYPE IF EXISTS "OutboxStatus"; diff --git a/prisma/migrations/20260928000001_onchain_outbox/migration.sql b/prisma/migrations/20260928000001_onchain_outbox/migration.sql new file mode 100644 index 00000000..e06cd0a2 --- /dev/null +++ b/prisma/migrations/20260928000001_onchain_outbox/migration.sql @@ -0,0 +1,26 @@ +-- Migration: transactional outbox for on-chain writes (issue #396). +-- No FK to intents: outbox rows must outlive in-memory/retention-evicted intents +-- and the migration must apply standalone in the rollback CI job. + +CREATE TYPE "OutboxStatus" AS ENUM ('pending', 'processing', 'submitted', 'confirmed', 'simulated', 'dead'); + +CREATE TABLE "onchain_outbox" ( + "id" BIGSERIAL PRIMARY KEY, + "intent_id" TEXT NOT NULL, + "operation" TEXT NOT NULL, + "payload" JSONB NOT NULL, + "status" "OutboxStatus" NOT NULL DEFAULT 'pending', + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "locked_until" TIMESTAMPTZ, + "envelope_hash" TEXT, + "tx_hash" TEXT, + "last_error" TEXT, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Relay claim scan: due rows by status. +CREATE INDEX "onchain_outbox_claim_idx" ON "onchain_outbox" ("status", "next_attempt_at"); +-- Per-intent ordering check ("is there an earlier unfinished row for this intent?"). +CREATE INDEX "onchain_outbox_intent_order_idx" ON "onchain_outbox" ("intent_id", "id"); diff --git a/prisma/migrations/20260928000002_pending_slashes/down.sql b/prisma/migrations/20260928000002_pending_slashes/down.sql new file mode 100644 index 00000000..78d2abd4 --- /dev/null +++ b/prisma/migrations/20260928000002_pending_slashes/down.sql @@ -0,0 +1,5 @@ +-- Rollback for 20260928000002_pending_slashes. +-- WARNING: drops saga state. Any slash still in challenge_window/submitted is +-- forgotten; reconcile those manually (docs/runbooks/slash-cancellation.md). +DROP TABLE IF EXISTS "pending_slashes"; +DROP TYPE IF EXISTS "PendingSlashState"; diff --git a/prisma/migrations/20260928000002_pending_slashes/migration.sql b/prisma/migrations/20260928000002_pending_slashes/migration.sql new file mode 100644 index 00000000..a8a236c7 --- /dev/null +++ b/prisma/migrations/20260928000002_pending_slashes/migration.sql @@ -0,0 +1,33 @@ +-- Migration: durable slashing saga state (issue #397). + +CREATE TYPE "PendingSlashState" AS ENUM ('detected', 'challenge_window', 'submitted', 'confirmed', 'cancelled'); + +CREATE TABLE "pending_slashes" ( + "id" TEXT PRIMARY KEY, + "intent_id" TEXT NOT NULL, + "solver_address" TEXT NOT NULL, + "reason" TEXT NOT NULL, + "state" "PendingSlashState" NOT NULL DEFAULT 'detected', + "fill_deadline" INTEGER NOT NULL, + "detected_at" TIMESTAMPTZ NOT NULL, + "challenge_ends_at" TIMESTAMPTZ NOT NULL, + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "locked_until" TIMESTAMPTZ, + "tx_hash" TEXT, + "simulated" BOOLEAN NOT NULL DEFAULT FALSE, + "submitted_at" TIMESTAMPTZ, + "confirmed_at" TIMESTAMPTZ, + "cancelled_at" TIMESTAMPTZ, + "cancel_reason" TEXT, + "cancelled_by" TEXT, + "fill_tx_hash" TEXT, + "last_error" TEXT, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Exactly-once slash per intent. +CREATE UNIQUE INDEX "pending_slashes_intent_id_key" ON "pending_slashes" ("intent_id"); +CREATE INDEX "pending_slashes_due_idx" ON "pending_slashes" ("state", "challenge_ends_at"); +CREATE INDEX "pending_slashes_solver_idx" ON "pending_slashes" ("solver_address"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7a048e53..c7b2e8c1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -451,3 +451,96 @@ model GuardianAction { @@index([active]) @@map("guardian_actions") } + +// ─── OnchainOutbox ─────────────────────────────────────────────────────────── +// Transactional outbox for on-chain writes (issue #396). A row is inserted in +// the same database transaction as the intent mutation it mirrors, and +// OutboxRelayService submits it to Soroban afterwards — so the DB can never say +// "accepted" while the corresponding transaction silently never went out. +// +// `id` is a monotonically increasing sequence and doubles as the per-intent +// ordering key: a row is only claimable once every earlier row for the same +// intent has reached a terminal success state. + +enum OutboxStatus { + pending + processing + submitted + confirmed + simulated + dead +} + +model OnchainOutbox { + id BigInt @id @default(autoincrement()) @map("id") + /// intents.intent_id this operation belongs to (ordering partition key). + intentId String @map("intent_id") + /// Contract operation, e.g. "create_intent", "accept_intent". + operation String @map("operation") + /// Operation arguments (JSON-safe, bigint amounts as strings). + payload Json @map("payload") + status OutboxStatus @default(pending) @map("status") + /// Number of times this row has been claimed by a relay worker. + attempts Int @default(0) @map("attempts") + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") @db.Timestamptz + /// Lease expiry while status = processing; a crashed worker's row is reclaimed after this. + lockedUntil DateTime? @map("locked_until") @db.Timestamptz + /// Hash of the signed envelope, persisted BEFORE submission (crash idempotency). + envelopeHash String? @map("envelope_hash") + /// Hash of the transaction that was submitted. + txHash String? @map("tx_hash") + lastError String? @map("last_error") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz + + @@index([status, nextAttemptAt], name: "onchain_outbox_claim_idx") + @@index([intentId, id], name: "onchain_outbox_intent_order_idx") + @@map("onchain_outbox") +} + +// ─── PendingSlash ──────────────────────────────────────────────────────────── +// Durable saga state for a solver slash (issue #397): +// detected → challenge_window → submitted → confirmed | cancelled +// The unique constraint on intent_id enforces exactly-once slashing per intent. + +enum PendingSlashState { + detected + challenge_window + submitted + confirmed + cancelled +} + +model PendingSlash { + id String @id @default(uuid()) @map("id") + intentId String @unique @map("intent_id") + solverAddress String @map("solver_address") + reason String @map("reason") + state PendingSlashState @default(detected) @map("state") + /// The intent's fill deadline (unix seconds) that the solver missed. + fillDeadline Int @map("fill_deadline") + detectedAt DateTime @map("detected_at") @db.Timestamptz + /// Slash may not be submitted before this instant. + challengeEndsAt DateTime @map("challenge_ends_at") @db.Timestamptz + attempts Int @default(0) @map("attempts") + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") @db.Timestamptz + /// Lease held by the pipeline worker that is currently verifying/submitting. + lockedUntil DateTime? @map("locked_until") @db.Timestamptz + txHash String? @map("tx_hash") + /// true when the registry client simulated but did not broadcast (dry-run / gated submit). + simulated Boolean @default(false) @map("simulated") + submittedAt DateTime? @map("submitted_at") @db.Timestamptz + confirmedAt DateTime? @map("confirmed_at") @db.Timestamptz + cancelledAt DateTime? @map("cancelled_at") @db.Timestamptz + cancelReason String? @map("cancel_reason") + cancelledBy String? @map("cancelled_by") + /// Fill transaction that cancelled the slash, when cancelled by a fill proof. + fillTxHash String? @map("fill_tx_hash") + lastError String? @map("last_error") + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz + + @@index([state, challengeEndsAt], name: "pending_slashes_due_idx") + @@index([solverAddress], name: "pending_slashes_solver_idx") + @@map("pending_slashes") +} diff --git a/src/common/stellar-signature.ts b/src/common/stellar-signature.ts index 7724cc88..5de5706b 100644 --- a/src/common/stellar-signature.ts +++ b/src/common/stellar-signature.ts @@ -110,3 +110,11 @@ export function buildDisputeDecisionMessage(disputeId: string, resolution: strin export function buildUpdateSolverMessage(address: string): string { return `update-solver:${address}`; } + +/** + * Canonical message a solver signs to prove a fill landed in time and cancel + * a pending slash during its challenge window (issue #397). + */ +export function buildFillProofMessage(intentId: string, solver: string, txHash: string): string { + return `fill-proof:${intentId}:${solver}:${txHash}`; +} diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 710004e9..c78bcbd4 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -281,6 +281,29 @@ export interface AppConfig { /** Soroban RPC endpoints probed for quorum (majority must be healthy). */ rpcHealthUrls: string[]; }; + /** Transactional outbox relay for on-chain writes (issue #396). */ + outbox: { + /** Kill switch for the relay worker. Rows keep accumulating while false. */ + relayEnabled: boolean; + relayIntervalMs: number; + batchSize: number; + /** Claims after which a row is moved to `dead` and alerted on. */ + maxAttempts: number; + /** + * Processing lease. Must exceed the signed transaction's time bound so a + * reclaimed row whose envelope is NOT_FOUND can be safely resubmitted. + */ + leaseSeconds: number; + }; + /** On-chain slashing saga (issue #397). */ + slashing: { + /** Delay between detection and broadcast during which a slash can be cancelled. */ + challengeWindowSeconds: number; + /** Grace added to the fill deadline when judging whether a fill landed in time. */ + clockSkewToleranceSeconds: number; + /** Failed submissions after which the slash is cancelled and compensated. */ + maxSubmitAttempts: number; + }; } export default (): AppConfig => ({ @@ -395,6 +418,18 @@ export default (): AppConfig => ({ .map((u) => u.trim()) .filter(Boolean), }, + outbox: { + relayEnabled: (process.env.OUTBOX_RELAY_ENABLED ?? "true") === "true", + relayIntervalMs: parseInt(process.env.OUTBOX_RELAY_INTERVAL_MS ?? "2000", 10), + batchSize: parseInt(process.env.OUTBOX_RELAY_BATCH_SIZE ?? "10", 10), + maxAttempts: parseInt(process.env.OUTBOX_MAX_ATTEMPTS ?? "8", 10), + leaseSeconds: parseInt(process.env.OUTBOX_LEASE_SECONDS ?? "120", 10), + }, + slashing: { + challengeWindowSeconds: parseInt(process.env.SLASH_CHALLENGE_WINDOW_SECONDS ?? "600", 10), + clockSkewToleranceSeconds: parseInt(process.env.SLASH_CLOCK_SKEW_TOLERANCE_SECONDS ?? "30", 10), + maxSubmitAttempts: parseInt(process.env.SLASH_MAX_SUBMIT_ATTEMPTS ?? "5", 10), + }, }); /** Parse `SHADOW_SAMPLE_RATE` into a probability, defaulting to full sampling. */ diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 5f9a5d2d..0bfa5545 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -348,6 +348,20 @@ export const envValidationSchema = Joi.object({ SOROBAN_RPC_ALLOWLIST: Joi.string().allow("").default(""), WEBHOOK_ALLOWLIST: Joi.string().allow("").default(""), ORACLE_ALLOWLIST: Joi.string().allow("").default(""), + + // ── Transactional outbox relay (issue #396) ────────────────────────────── + // OUTBOX_LEASE_SECONDS must exceed the signed tx time bound (30 s) — see + // docs/architecture/onchain-settlement.md#transactional-outbox. + OUTBOX_RELAY_ENABLED: Joi.boolean().default(true), + OUTBOX_RELAY_INTERVAL_MS: Joi.number().integer().min(100).default(2000), + OUTBOX_RELAY_BATCH_SIZE: Joi.number().integer().min(1).max(100).default(10), + OUTBOX_MAX_ATTEMPTS: Joi.number().integer().min(1).default(8), + OUTBOX_LEASE_SECONDS: Joi.number().integer().min(60).default(120), + + // ── Slashing saga (issue #397) ─────────────────────────────────────────── + SLASH_CHALLENGE_WINDOW_SECONDS: Joi.number().integer().min(0).default(600), + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS: Joi.number().integer().min(0).default(30), + SLASH_MAX_SUBMIT_ATTEMPTS: Joi.number().integer().min(1).default(5), }); // ── WS gateway hardening (issue #455) ───────────────────────────────────── WS_MAX_PAYLOAD_BYTES: Joi.number().integer().min(1024).default(16384), diff --git a/src/intents/dto/slash.dto.ts b/src/intents/dto/slash.dto.ts new file mode 100644 index 00000000..ebd62dad --- /dev/null +++ b/src/intents/dto/slash.dto.ts @@ -0,0 +1,52 @@ +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; +import { Type } from "class-transformer"; +import { IsIn, IsInt, IsOptional, IsString, Matches, Max, MaxLength, Min, MinLength } from "class-validator"; +import { PENDING_SLASH_STATES, PendingSlashState } from "../../solvers/pending-slashes.repository"; + +const ED25519_SIGNATURE_MAX_LENGTH = 88; + +export class ListSlashesDto { + @ApiPropertyOptional({ enum: PENDING_SLASH_STATES }) + @IsOptional() + @IsIn(PENDING_SLASH_STATES) + state?: PendingSlashState; + + @ApiPropertyOptional({ default: 50, maximum: 200 }) + @IsOptional() + @Type(() => Number) + @IsInt() + @Min(1) + @Max(200) + limit?: number; +} + +export class AdminCancelSlashDto { + @ApiProperty({ description: "Why the slash is being cancelled (audit log)", maxLength: 500 }) + @IsString() + @MinLength(5) + @MaxLength(500) + note!: string; +} + +export class FillProofDto { + @ApiProperty({ description: "Stellar address of the slashed solver", maxLength: 56 }) + @IsString() + @MinLength(10) + @MaxLength(56) + solver!: string; + + @ApiProperty({ description: "Hash of the Stellar transaction that filled the intent (64 hex chars)" }) + @Matches(/^[0-9a-f]{64}$/i) + txHash!: string; + + @ApiProperty({ + description: + 'Base64 Ed25519 signature by `solver` of "fill-proof:::" ' + + "(txHash lowercased)", + maxLength: ED25519_SIGNATURE_MAX_LENGTH, + }) + @IsString() + @MinLength(10) + @MaxLength(ED25519_SIGNATURE_MAX_LENGTH) + signature!: string; +} diff --git a/src/intents/intents-sweeper.manual-trigger.spec.ts b/src/intents/intents-sweeper.manual-trigger.spec.ts index 7ebbb735..1f0a0a97 100644 --- a/src/intents/intents-sweeper.manual-trigger.spec.ts +++ b/src/intents/intents-sweeper.manual-trigger.spec.ts @@ -3,7 +3,7 @@ import { IntentsSweeperService } from "./intents-sweeper.service"; import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { MetricsService } from "../metrics/metrics.service"; import { KillSwitchService } from "../killswitch/killswitch.service"; import { LeaderElectionService } from "../common/leader-election"; @@ -37,9 +37,7 @@ describe("IntentsSweeperService — manual sweep trigger (#269)", () => { } as unknown as IntentsService; const gateway = { broadcast: jest.fn() } as unknown as IntentsGateway; const solversService = { recordFailedFill: jest.fn() } as unknown as SolversService; - const solverRegistry = { - slashSolver: jest.fn().mockResolvedValue({ detail: "no-op" }), - } as unknown as SolverRegistryService; + const slashingPipeline = { detect: jest.fn() } as unknown as SlashingPipelineService; const metricsService = { recordSweep: jest.fn() } as unknown as MetricsService; const killSwitch = { evaluateTarget: jest.fn().mockReturnValue({ paused: false, matched: null, matchedChain: [] }), @@ -49,11 +47,11 @@ describe("IntentsSweeperService — manual sweep trigger (#269)", () => { intentsService, gateway, solversService, - solverRegistry, + slashingPipeline, metricsService, killSwitch, + noopLeaderElection(), ); - return new IntentsSweeperService(intentsService, gateway, solversService, solverRegistry, metricsService, noopLeaderElection()); } afterEach(() => jest.restoreAllMocks()); diff --git a/src/intents/intents-sweeper.service.spec.ts b/src/intents/intents-sweeper.service.spec.ts index 804265aa..9c98b02b 100644 --- a/src/intents/intents-sweeper.service.spec.ts +++ b/src/intents/intents-sweeper.service.spec.ts @@ -6,7 +6,7 @@ import { KillSwitchService } from "../killswitch/killswitch.service"; import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { MetricsService } from "../metrics/metrics.service"; import { InMemorySolversRepository } from "../solvers/in-memory-solvers.repository"; import { SOLVERS_REPOSITORY } from "../solvers/solvers.repository"; @@ -51,7 +51,7 @@ function buildIntentsService(): IntentsService { const protocolParams = { snapshotForChain: jest.fn().mockReturnValue({ version: 0, feeBps: 30, deadlineSeconds: 1800, fillWindowSeconds: 600, capturedAt: new Date().toISOString() }), } as unknown as ProtocolParamsService; - return new IntentsService(repo, configService, stellarTxService, prismaService, protocolParams); + return new IntentsService(repo, configService, stellarTxService, prismaService, undefined, undefined, protocolParams); } async function buildSolversService(): Promise { @@ -68,7 +68,7 @@ describe("IntentsSweeperService", () => { let intentsService: IntentsService; let gateway: IntentsGateway; let solversService: SolversService; - let solverRegistryService: jest.Mocked; + let slashingPipeline: jest.Mocked>; let metricsService: jest.Mocked>; let killSwitch: jest.Mocked>; let sweeper: IntentsSweeperService; @@ -77,13 +77,13 @@ describe("IntentsSweeperService", () => { intentsService = buildIntentsService(); gateway = { broadcast: jest.fn().mockResolvedValue(undefined) } as unknown as IntentsGateway; solversService = await buildSolversService(); - solverRegistryService = { - slashSolver: jest.fn().mockResolvedValue({ - submitted: false, - simulated: false, - detail: "not configured — no-op", - }), - } as unknown as jest.Mocked; + slashingPipeline = { + detect: jest.fn().mockImplementation(async (input) => ({ + ...input, + state: "challenge_window", + challengeEndsAt: new Date((input.detectedAt + 600) * 1000), + })), + } as unknown as jest.Mocked>; metricsService = { recordSweep: jest.fn() } as unknown as jest.Mocked>; // Default: no pause active, so existing sweeper expectations are unchanged. killSwitch = { @@ -94,7 +94,7 @@ describe("IntentsSweeperService", () => { intentsService, gateway, solversService, - solverRegistryService, + slashingPipeline as unknown as SlashingPipelineService, metricsService as unknown as MetricsService, killSwitch as unknown as KillSwitchService, noopLeaderElection(), @@ -149,8 +149,9 @@ describe("IntentsSweeperService", () => { expect(gateway.broadcast).toHaveBeenCalledWith( expect.objectContaining({ type: "intent_slashed", intentId, solver: ALPHA_ADDR }), ); - expect(solverRegistryService.slashSolver).toHaveBeenCalledWith( - expect.objectContaining({ solverAddress: ALPHA_ADDR, intentId }), + // Issue #397: the sweeper only detects — the saga owns the on-chain slash. + expect(slashingPipeline.detect).toHaveBeenCalledWith( + expect.objectContaining({ solverAddress: ALPHA_ADDR, intentId, fillDeadline: past }), ); }); @@ -184,7 +185,7 @@ describe("IntentsSweeperService", () => { await sweeper.sweep(); expect((await intentsService.get(intentId))?.state).toBe("accepted"); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); }); it("does not throw if an accepted intent somehow has no solver on record", async () => { @@ -202,7 +203,7 @@ describe("IntentsSweeperService", () => { await expect(sweeper.sweep()).resolves.not.toThrow(); expect((await intentsService.get(intent.intentId))?.state).toBe("slashed"); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); }); // ── #259: MetricsService integration ──────────────────────────────────── @@ -271,7 +272,7 @@ describe("IntentsSweeperService", () => { // Not slashed — the pause, not the solver, caused the missed fill. expect(result.slashedCount).toBe(0); expect(result.extendedDeadlines).toBe(1); - expect(solverRegistryService.slashSolver).not.toHaveBeenCalled(); + expect(slashingPipeline.detect).not.toHaveBeenCalled(); const updated = await intentsService.get(intentId); expect(updated?.state).toBe("accepted"); diff --git a/src/intents/intents-sweeper.service.ts b/src/intents/intents-sweeper.service.ts index 98f355eb..0c9d6777 100644 --- a/src/intents/intents-sweeper.service.ts +++ b/src/intents/intents-sweeper.service.ts @@ -2,7 +2,7 @@ import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/commo import { IntentsService } from "./intents.service"; import { IntentsGateway } from "./intents.gateway"; import { SolversService } from "../solvers/solvers.service"; -import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; import { logger } from "../common/logger"; import { MetricsService } from "../metrics/metrics.service"; import { KillSwitchService } from "../killswitch/killswitch.service"; @@ -34,7 +34,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { private readonly intentsService: IntentsService, private readonly intentsGateway: IntentsGateway, private readonly solversService: SolversService, - private readonly solverRegistryService: SolverRegistryService, + private readonly slashingPipeline: SlashingPipelineService, private readonly metricsService: MetricsService, private readonly killSwitch: KillSwitchService, private readonly leaderElection: LeaderElectionService, @@ -136,7 +136,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { continue; } - const slashed = await this.slashMissedFill(intent.intentId, intent.solver, now); + const slashed = await this.slashMissedFill(intent.intentId, intent.solver, intent.deadline, now); if (slashed) slashedCount++; } @@ -197,9 +197,16 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { } } + /** + * Detection half of the slashing saga (issue #397). Marks the intent + * `slashed`, applies the optimistic local penalty, and hands off to + * SlashingPipelineService, which holds the slash in a challenge window, + * re-verifies, and only then broadcasts — nothing is sent on-chain here. + */ private async slashMissedFill( intentId: string, solver: string | undefined, + fillDeadline: number, now: number, ): Promise { const reason = "accepted intent not filled before deadline"; @@ -215,7 +222,7 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { solver, slashedAt: now, }); - await this.intentsGateway.broadcast({ type: "intent_slashed", intentId, solver, reason }); + await this.intentsGateway.broadcast({ type: "intent_slashed", intentId, solver, reason, pending: true }); if (!solver) { // Shouldn't happen in practice — an "accepted" intent always has a @@ -224,16 +231,21 @@ export class IntentsSweeperService implements OnModuleInit, OnModuleDestroy { return true; } + // Optimistic local penalty; the saga compensates via rollbackPenalty if + // the slash is cancelled. await this.solversService.recordFailedFill(solver, intentId); const slashRecord = await this.solversService.recordSlash(solver, intentId, reason, now); - const result = await this.solverRegistryService.slashSolver({ - solverAddress: solver, + const pending = await this.slashingPipeline.detect({ intentId, + solverAddress: solver, reason, + fillDeadline, + detectedAt: now, }); - console.log( - `[sweeper] slashed solver=${solver} for intent=${intentId}: ${result.detail} slashId=${slashRecord?.slashId ?? "unknown"}`, + this.logger.log( + `[sweeper] slash detected solver=${solver} intent=${intentId} state=${pending.state} ` + + `challengeEndsAt=${pending.challengeEndsAt.toISOString()} slashId=${slashRecord?.slashId ?? "unknown"}`, ); return true; } diff --git a/src/intents/intents.gateway.ts b/src/intents/intents.gateway.ts index 0f7a29a1..13d747be 100644 --- a/src/intents/intents.gateway.ts +++ b/src/intents/intents.gateway.ts @@ -769,6 +769,7 @@ export class IntentsGateway "intent_cancelled", "intent_expired", "intent_slashed", + "intent_slash_cancelled", ]); if (lookupTypes.has(event.type)) { diff --git a/src/intents/intents.module.ts b/src/intents/intents.module.ts index 378b82f6..14ab8b90 100644 --- a/src/intents/intents.module.ts +++ b/src/intents/intents.module.ts @@ -17,6 +17,26 @@ import { SorobanModule } from "../soroban/soroban.module"; import { AppConfig } from "../config/configuration"; import { PrismaService } from "../prisma/prisma.service"; import { GovernanceModule } from "../governance/governance.module"; +import { + INTENTS_UNIT_OF_WORK, + InMemoryIntentsUnitOfWork, + PrismaIntentsUnitOfWork, +} from "./intents.unit-of-work"; +import { IIntentsRepository } from "./intents.repository"; +import { InMemoryOutboxRepository, OUTBOX_REPOSITORY } from "../soroban/outbox.repository"; +import { PrismaOutboxRepository } from "../soroban/prisma-outbox.repository"; +import { OutboxRelayService } from "../soroban/outbox-relay.service"; +import { + InMemoryPendingSlashesRepository, + PENDING_SLASHES_REPOSITORY, +} from "../solvers/pending-slashes.repository"; +import { PrismaPendingSlashesRepository } from "../solvers/prisma-pending-slashes.repository"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; +import { AdminSlashesController, SlashesController } from "./slashes.controller"; +import { OutboxAdminController } from "../soroban/outbox-admin.controller"; + +/** The outbox, unit of work, and slash saga share INTENTS_PERSISTENCE with the intents store. */ +const usePrisma = () => (process.env.INTENTS_PERSISTENCE ?? "memory") === "prisma"; @Module({ // Both SolversModule and SorobanModule import IntentsModule back, so both @@ -32,7 +52,7 @@ import { GovernanceModule } from "../governance/governance.module"; forwardRef(() => SorobanModule), ], imports: [forwardRef(() => SolversModule), RoutingModule, TokensModule, SorobanModule, GovernanceModule], - controllers: [IntentsController], + controllers: [IntentsController, SlashesController, AdminSlashesController, OutboxAdminController], providers: [ // Select the persistence adapter based on INTENTS_PERSISTENCE env var. // INTENTS_PERSISTENCE=prisma → PrismaIntentsRepository (production/staging) @@ -48,6 +68,34 @@ import { GovernanceModule } from "../governance/governance.module"; return new InMemoryIntentsRepository(); }, }, + // Transactional outbox (issue #396). + { + provide: OUTBOX_REPOSITORY, + inject: [PrismaService], + useFactory: (prisma: PrismaService) => + usePrisma() ? new PrismaOutboxRepository(prisma) : new InMemoryOutboxRepository(), + }, + { + provide: INTENTS_UNIT_OF_WORK, + inject: [PrismaService, INTENTS_REPOSITORY, OUTBOX_REPOSITORY], + useFactory: ( + prisma: PrismaService, + intents: IIntentsRepository, + outbox: InMemoryOutboxRepository | PrismaOutboxRepository, + ) => + outbox instanceof InMemoryOutboxRepository + ? new InMemoryIntentsUnitOfWork(intents, outbox) + : new PrismaIntentsUnitOfWork(prisma), + }, + OutboxRelayService, + // Slashing saga (issue #397). + { + provide: PENDING_SLASHES_REPOSITORY, + inject: [PrismaService], + useFactory: (prisma: PrismaService) => + usePrisma() ? new PrismaPendingSlashesRepository(prisma) : new InMemoryPendingSlashesRepository(), + }, + SlashingPipelineService, IntentsService, IntentCapabilityIndex, backplaneProvider, diff --git a/src/intents/intents.service.spec.ts b/src/intents/intents.service.spec.ts index a9867fe0..07d02759 100644 --- a/src/intents/intents.service.spec.ts +++ b/src/intents/intents.service.spec.ts @@ -7,6 +7,8 @@ import { IntentsService } from "./intents.service"; import { INTENTS_REPOSITORY, InMemoryIntentsRepository } from "./intents.repository"; import { PrismaService } from "../prisma/prisma.service"; import { ProtocolParamsService } from "../governance/params.service"; +import { InMemoryOutboxRepository } from "../soroban/outbox.repository"; +import { InMemoryIntentsUnitOfWork } from "./intents.unit-of-work"; const VALID_CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; @@ -367,76 +369,126 @@ describe("IntentsService", () => { }); }); - describe("on-chain registration (ONCHAIN_INTENTS_ENABLED)", () => { - it("stays fully in the repository when the flag is off, never touching StellarTxService", async () => { + describe("on-chain writes via the transactional outbox (ONCHAIN_INTENTS_ENABLED, #396)", () => { + function makeOutboxService(onchain: boolean, settlementContractId = VALID_CONTRACT_ID) { const stellarTxService = fakeStellarTxService(); - const svc = makeService({ onchainIntentsEnabled: false }, stellarTxService); + const repo = new InMemoryIntentsRepository(); + const outbox = new InMemoryOutboxRepository(); + const service = new IntentsService( + repo, + fakeConfig({ onchainIntentsEnabled: onchain, settlementContractId }), + stellarTxService, + fakePrismaService(), + undefined, // shadowService + undefined, // metricsService + fakeProtocolParamsService(), + undefined, // flags + new InMemoryIntentsUnitOfWork(repo, outbox), + ); + return { service, outbox, stellarTxService, repo }; + } + + it("stays fully off-chain when the flag is off: no outbox rows, no StellarTxService", async () => { + const { service, outbox, stellarTxService } = makeOutboxService(false); const intent = await service.create(validCreateData()); expect(stellarTxService.invokeContract).not.toHaveBeenCalled(); + expect(await outbox.findByIntent(intent.intentId)).toEqual([]); expect(await service.get(intent.intentId)).toEqual(intent); }); - it("invokes the settlement contract and preserves the Intent shape when the flag is on", async () => { - const stellarTxService = fakeStellarTxService(); - stellarTxService.invokeContract.mockResolvedValue({ hash: "deadbeef", status: "SUCCESS" } as never); - const svc = makeService( - { onchainIntentsEnabled: true, settlementContractId: VALID_CONTRACT_ID }, - stellarTxService, - ); + it("commits the intent and a create_intent outbox row together, without submitting inline", async () => { + const { service, outbox, stellarTxService } = makeOutboxService(true); - const data = validCreateData(); - const intent = await service.create(data); - - expect(stellarTxService.invokeContract).toHaveBeenCalledTimes(1); - const call = stellarTxService.invokeContract.mock.calls[0][0]; - expect(call.contractId).toBe(VALID_CONTRACT_ID); - expect(call.method).toBe("create_intent"); - - // response shape is unchanged relative to the in-memory path - expect(Object.keys(intent).sort()).toEqual( - Object.keys({ - intentId: "", - user: "", - srcChain: "", - srcToken: "", - srcAmount: "", - dstToken: "", - minDstAmount: "", - state: "", - createdAt: 0, - deadline: 0, - }).sort(), - ); - expect(await service.get(intent.intentId)).toBeDefined(); + const intent = await service.create(validCreateData()); + + expect(stellarTxService.invokeContract).not.toHaveBeenCalled(); + const rows = await outbox.findByIntent(intent.intentId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + operation: "create_intent", + status: "pending", + payload: expect.objectContaining({ intentId: intent.intentId, srcAmount: "1000000" }), + }); + expect(await service.get(intent.intentId)).toEqual(intent); }); - it("rejects with a clear error and does not create the intent when SETTLEMENT_CONTRACT_ID is unset", async () => { - const stellarTxService = fakeStellarTxService(); - const service = makeService({ onchainIntentsEnabled: true }, stellarTxService); - const before = (await service.getAll()).length; - const svc = makeService({ onchainIntentsEnabled: true }, stellarTxService); - const before = (await svc.getAll()).length; + it("rejects with a clear error and writes nothing when SETTLEMENT_CONTRACT_ID is unset", async () => { + const { service, outbox, repo } = makeOutboxService(true, ""); + const before = repo.findAll().length; await expect(service.create(validCreateData())).rejects.toMatchObject({ message: expect.stringContaining("SETTLEMENT_CONTRACT_ID"), }); - expect(stellarTxService.invokeContract).not.toHaveBeenCalled(); - expect(await service.getAll()).toHaveLength(before); + expect(repo.findAll()).toHaveLength(before); + expect((await outbox.countByStatus()).pending).toBe(0); }); - it("rejects and does not create the intent when the on-chain call fails", async () => { - const stellarTxService = fakeStellarTxService(); - stellarTxService.invokeContract.mockRejectedValue(new Error("submission failed after 5 attempts")); - const svc = makeService( - { onchainIntentsEnabled: true, settlementContractId: VALID_CONTRACT_ID }, - stellarTxService, + it("rejects a payload that cannot be encoded instead of creating a poison row", async () => { + const { service, outbox } = makeOutboxService(true); + + await expect(service.create({ ...validCreateData(), user: "not-a-stellar-address" })).rejects.toThrow(); + expect((await outbox.countByStatus()).pending).toBe(0); + }); + + it("drops the outbox row when the intent write fails (atomic unit of work)", async () => { + const { service, outbox, repo } = makeOutboxService(true); + jest.spyOn(repo, "save").mockImplementationOnce(() => { + throw new Error("db down"); + }); + + await expect(service.create(validCreateData())).rejects.toThrow("db down"); + expect((await outbox.countByStatus()).pending).toBe(0); + }); + + it("enqueues accept, fill and cancel transitions in order, and nothing for a lost race", async () => { + const { service, outbox } = makeOutboxService(true); + const solver = Keypair.random().publicKey(); + + const a = await service.create(validCreateData()); + expect(await service.acceptIfOpen(a.intentId, solver)).not.toBeNull(); + expect(await service.acceptIfOpen(a.intentId, solver)).toBeNull(); // lost race → no row + expect( + await service.fillIfAccepted(a.intentId, solver, { fillAmount: "995000", txHash: "ab".repeat(32) }), + ).not.toBeNull(); + + const b = await service.create(validCreateData()); + expect(await service.cancelIfOpen(b.intentId)).not.toBeNull(); + + expect((await outbox.findByIntent(a.intentId)).map((r) => r.operation)).toEqual([ + "create_intent", + "accept_intent", + "fill_intent", + ]); + expect((await outbox.findByIntent(b.intentId)).map((r) => r.operation)).toEqual([ + "create_intent", + "cancel_intent", + ]); + }); + + it("follows the onchain-intents-enabled runtime flag per intent when flags are wired", async () => { + const repo = new InMemoryIntentsRepository(); + const outbox = new InMemoryOutboxRepository(); + const flags = { getBooleanValue: jest.fn().mockResolvedValue(true) }; + const service = new IntentsService( + repo, + fakeConfig({ onchainIntentsEnabled: false, settlementContractId: VALID_CONTRACT_ID }), + fakeStellarTxService(), + fakePrismaService(), + undefined, + undefined, + fakeProtocolParamsService(), + flags as never, + new InMemoryIntentsUnitOfWork(repo, outbox), ); - const before = (await service.getAll()).length; - await expect(service.create(validCreateData())).rejects.toThrow(/settlement contract/i); - expect(await service.getAll()).toHaveLength(before); + const intent = await service.create(validCreateData()); + expect(flags.getBooleanValue).toHaveBeenCalledWith("onchain-intents-enabled", { + targetingKey: intent.intentId, + chain: "ethereum", + }); + expect(await outbox.findByIntent(intent.intentId)).toHaveLength(1); }); }); diff --git a/src/intents/intents.service.ts b/src/intents/intents.service.ts index c8222ed3..627515cc 100644 --- a/src/intents/intents.service.ts +++ b/src/intents/intents.service.ts @@ -24,6 +24,19 @@ import { MetricsService } from "../metrics/metrics.service"; import { PrismaService } from "../prisma/prisma.service"; import { ProtocolParamsService } from "../governance/params.service"; import { FeatureFlagService } from "../flags/feature-flag.service"; +import { + IIntentsUnitOfWork, + INTENTS_UNIT_OF_WORK, + InMemoryIntentsUnitOfWork, +} from "./intents.unit-of-work"; +import { InMemoryOutboxRepository, IOutboxWriter, NewOutboxEntry } from "../soroban/outbox.repository"; +import { + acceptIntentEntry, + buildOutboxInvocation, + cancelIntentEntry, + createIntentEntry, + fillIntentEntry, +} from "../soroban/outbox-operations"; const TERMINAL_STATES: IntentState[] = ["filled", "cancelled", "expired", "slashed"]; @@ -103,6 +116,13 @@ export class IntentsService { */ private readonly auditLog = new Map(); + /** + * Commits intent mutations together with their outbox rows (issue #396). + * Falls back to an in-memory unit of work over `repo` when not injected + * (unit tests that construct the service directly). + */ + private readonly unitOfWork: IIntentsUnitOfWork; + constructor( @Inject(INTENTS_REPOSITORY) private readonly repo: IIntentsRepository, @@ -130,7 +150,10 @@ export class IntentsService { @Optional() private readonly metricsService?: MetricsService, private readonly protocolParamsService: ProtocolParamsService, @Optional() private readonly flags?: FeatureFlagService, - ) {} + @Optional() @Inject(INTENTS_UNIT_OF_WORK) unitOfWork?: IIntentsUnitOfWork, + ) { + this.unitOfWork = unitOfWork ?? new InMemoryIntentsUnitOfWork(repo, new InMemoryOutboxRepository()); + } /** * Logs the store size and evicts stale terminal intents from the in-memory @@ -256,17 +279,16 @@ export class IntentsService { // ONCHAIN_INTENTS_ENABLED is the default; the `onchain-intents-enabled` // runtime flag (issue #495) can roll it out per chain / percentage. - const onchain = this.flags - ? await this.flags.getBooleanValue("onchain-intents-enabled", { - targetingKey: intent.intentId, - chain: intent.srcChain, - }) - : this.configService.get("onchainIntentsEnabled", { infer: true }); - if (onchain) { - await this.registerOnChain(intent); + if (await this.onchainEnabledFor(intent)) { + // Issue #396: intent row + create_intent outbox row commit atomically; + // OutboxRelayService submits afterwards, never inside this request. + await this.unitOfWork.run(async ({ intents, outbox }) => { + await this.registerOnChain(intent, outbox); + await intents.save(intent); + }); + } else { + await this.repo.save(intent); } - - await this.repo.save(intent); // Creation is the entry edge of the funnel: the `vortex:intent:*` recording // rules count transitions *into* each state, so without this the intent // dashboard would start every conversion ratio from zero. `from_state` is @@ -276,46 +298,64 @@ export class IntentsService { } /** - * Registers `intent` with the settlement contract. Only called when - * ONCHAIN_INTENTS_ENABLED is on; while that flag is off, create() stays - * fully in-memory (the rollout fallback). + * Queues the settlement contract's `create_intent` call for `intent` on the + * outbox (issue #396). Only called when on-chain intents are enabled for + * this intent; otherwise create() stays fully off-chain (the rollout + * fallback). Nothing is broadcast here — the request never waits on Soroban, + * and a Soroban outage can no longer fail intent creation. */ - private async registerOnChain(intent: Intent): Promise { + private async registerOnChain(intent: Intent, outbox: IOutboxWriter): Promise { + await this.enqueueOnchain(outbox, createIntentEntry(intent)); + this.logger.log(`Queued on-chain registration for intent ${intent.intentId}`); + } + + /** + * Validates that `entry` encodes to a contract call (so malformed input is + * rejected in the request instead of becoming a poison row) and enqueues it. + */ + private async enqueueOnchain(outbox: IOutboxWriter, entry: NewOutboxEntry): Promise { const contractId = this.configService.get("stellar.settlementContractId", { infer: true }); if (!contractId) { throw new ServiceUnavailableException( "On-chain intent registration is enabled but SETTLEMENT_CONTRACT_ID is not configured", ); } + buildOutboxInvocation(entry, contractId); + await outbox.enqueue(entry); + } - try { - const result = await this.stellarTxService.invokeContract({ - contractId, - method: "create_intent", - args: this.buildCreateIntentArgs(intent), - }); - this.logger.log(`Registered intent ${intent.intentId} on-chain (tx ${result.hash})`); - } catch (err) { - this.logger.error( - `Failed to register intent ${intent.intentId} on-chain: ${(err as Error).message}`, - ); - throw new ServiceUnavailableException( - "Failed to register intent with the settlement contract", - ); - } + /** + * Whether `intent`'s state changes are mirrored on-chain. Evaluated with the + * same targeting (intent id + source chain) for every transition, so a + * percentage rollout never splits one intent's lifecycle across paths. + */ + private async onchainEnabledFor(intent: Pick): Promise { + return Boolean( + this.flags + ? await this.flags.getBooleanValue("onchain-intents-enabled", { + targetingKey: intent.intentId, + chain: intent.srcChain, + }) + : this.configService.get("onchainIntentsEnabled", { infer: true }), + ); } - private buildCreateIntentArgs(intent: Intent): xdr.ScVal[] { - return [ - nativeToScVal(intent.intentId, { type: "string" }), - new Address(intent.user).toScVal(), - nativeToScVal(intent.srcChain, { type: "symbol" }), - nativeToScVal(intent.srcToken.address, { type: "string" }), - nativeToScVal(BigInt(intent.srcAmount), { type: "i128" }), - new Address(intent.dstToken.contract).toScVal(), - nativeToScVal(BigInt(intent.minDstAmount), { type: "i128" }), - nativeToScVal(intent.deadline, { type: "u64" }), - ]; + /** + * Applies a guarded state transition and, when on-chain writes are enabled + * for the intent, enqueues the mirroring contract call in the same unit of + * work (issue #396). A `null` transition (guard failed) enqueues nothing. + */ + private transitionWithOutbox( + mutate: (intents: IIntentsRepository) => Promise | Intent | null, + toEntry: (updated: Intent) => NewOutboxEntry, + ): Promise { + return this.unitOfWork.run(async ({ intents, outbox }) => { + const updated = await mutate(intents); + if (updated && (await this.onchainEnabledFor(updated))) { + await this.enqueueOnchain(outbox, toEntry(updated)); + } + return updated; + }); } // --------------------------------------------------------------------------- @@ -512,7 +552,10 @@ export class IntentsService { const nowSec = now ?? Math.floor(Date.now() / 1000); const fillWindow = CHAIN_FILL_WINDOW_DEFAULTS[intent.srcChain] ?? DEFAULT_FILL_WINDOW_SECONDS; - const updated = await this.repo.acceptIfOpen(id, solver, nowSec + fillWindow, nowSec); + const updated = await this.transitionWithOutbox( + (intents) => intents.acceptIfOpen(id, solver, nowSec + fillWindow, nowSec), + acceptIntentEntry, + ); if (updated !== null) this.countTransition("open", "accepted"); if (this.beginShadowObservation()) { this.observeAccept(updated ?? intent, solver, updated !== null); @@ -551,7 +594,10 @@ export class IntentsService { now?: number, ): Promise { const nowSec = now ?? Math.floor(Date.now() / 1000); - const updated = await this.repo.fillIfAccepted(id, solver, patch, nowSec); + const updated = await this.transitionWithOutbox( + (intents) => intents.fillIfAccepted(id, solver, patch, nowSec), + fillIntentEntry, + ); if (updated !== null) this.countTransition("accepted", "filled"); if (this.beginShadowObservation()) { // Report from `patch` rather than re-reading: on a lost race the stored @@ -592,7 +638,10 @@ export class IntentsService { * (e.g. a concurrent accept() or sweeper expiry already transitioned it). */ async cancelIfOpen(id: string): Promise { - const updated = await this.repo.cancelIfOpen(id); + const updated = await this.transitionWithOutbox( + (intents) => intents.cancelIfOpen(id), + cancelIntentEntry, + ); if (updated !== null) this.countTransition("open", "cancelled"); if (this.beginShadowObservation()) { const subject = updated ?? (await this.repo.findById(id)); diff --git a/src/intents/intents.unit-of-work.spec.ts b/src/intents/intents.unit-of-work.spec.ts new file mode 100644 index 00000000..bc72b12f --- /dev/null +++ b/src/intents/intents.unit-of-work.spec.ts @@ -0,0 +1,57 @@ +import { PrismaService } from "../prisma/prisma.service"; +import { InMemoryOutboxRepository } from "../soroban/outbox.repository"; +import { PrismaOutboxRepository } from "../soroban/prisma-outbox.repository"; +import { InMemoryIntentsRepository } from "./intents.repository"; +import { InMemoryIntentsUnitOfWork, PrismaIntentsUnitOfWork } from "./intents.unit-of-work"; +import { PrismaIntentsRepository } from "./prisma-intents.repository"; + +describe("IntentsUnitOfWork (#396)", () => { + it("Prisma: runs the work in one $transaction with repositories bound to the tx client", async () => { + const txClient = { intent: {}, onchainOutbox: {} }; + const prisma = { + $transaction: jest.fn((fn: (tx: unknown) => Promise) => fn(txClient)), + } as unknown as PrismaService; + + const result = await new PrismaIntentsUnitOfWork(prisma).run(async ({ intents, outbox }) => { + expect(intents).toBeInstanceOf(PrismaIntentsRepository); + expect(outbox).toBeInstanceOf(PrismaOutboxRepository); + expect((intents as unknown as { prisma: unknown }).prisma).toBe(txClient); + expect((outbox as unknown as { prisma: unknown }).prisma).toBe(txClient); + return "ok"; + }); + + expect(result).toBe("ok"); + expect(prisma.$transaction).toHaveBeenCalledTimes(1); + }); + + it("Prisma: a throw inside the work propagates (so $transaction rolls back)", async () => { + const prisma = { + $transaction: jest.fn((fn: (tx: unknown) => Promise) => fn({})), + } as unknown as PrismaService; + await expect( + new PrismaIntentsUnitOfWork(prisma).run(async () => { + throw new Error("rollback"); + }), + ).rejects.toThrow("rollback"); + }); + + it("in-memory: commits buffered outbox rows only when the work succeeds", async () => { + const outbox = new InMemoryOutboxRepository(); + const uow = new InMemoryIntentsUnitOfWork(new InMemoryIntentsRepository(), outbox); + const entry = { intentId: "i1", operation: "create_intent" as const, payload: {} }; + + await expect( + uow.run(async (tx) => { + await tx.outbox.enqueue(entry); + throw new Error("intent write failed"); + }), + ).rejects.toThrow(); + expect(await outbox.findByIntent("i1")).toEqual([]); + + await uow.run(async (tx) => { + await tx.outbox.enqueue(entry); + expect(await outbox.findByIntent("i1")).toEqual([]); // not visible until commit + }); + expect(await outbox.findByIntent("i1")).toHaveLength(1); + }); +}); diff --git a/src/intents/intents.unit-of-work.ts b/src/intents/intents.unit-of-work.ts new file mode 100644 index 00000000..7e9fd02b --- /dev/null +++ b/src/intents/intents.unit-of-work.ts @@ -0,0 +1,73 @@ +import { PrismaService } from "../prisma/prisma.service"; +import { IIntentsRepository } from "./intents.repository"; +import { PrismaIntentsRepository } from "./prisma-intents.repository"; +import { + InMemoryOutboxRepository, + IOutboxWriter, + NewOutboxEntry, + OutboxEntry, +} from "../soroban/outbox.repository"; +import { PrismaOutboxRepository } from "../soroban/prisma-outbox.repository"; + +/** Injection token for {@link IIntentsUnitOfWork}. */ +export const INTENTS_UNIT_OF_WORK = Symbol("INTENTS_UNIT_OF_WORK"); + +/** Repositories scoped to one unit of work. */ +export interface IntentsTransaction { + intents: IIntentsRepository; + outbox: IOutboxWriter; +} + +/** + * Runs an intent mutation and its outbox rows as one atomic unit (issue #396), + * so the database can never record a state change whose on-chain write was + * silently dropped, or vice versa. + */ +export interface IIntentsUnitOfWork { + run(work: (tx: IntentsTransaction) => Promise): Promise; +} + +/** + * Prisma adapter: one interactive `$transaction` — the intent write and the + * `onchain_outbox` insert commit or roll back together. + */ +export class PrismaIntentsUnitOfWork implements IIntentsUnitOfWork { + constructor(private readonly prisma: PrismaService) {} + + run(work: (tx: IntentsTransaction) => Promise): Promise { + return this.prisma.$transaction((client) => + work({ + intents: new PrismaIntentsRepository(client), + outbox: new PrismaOutboxRepository(client), + }), + ); + } +} + +/** + * In-memory adapter (dev/test). Outbox rows are buffered and only committed + * when `work` resolves, so a throw after the enqueue never leaves an orphan + * row. Intent writes are not rolled back — callers order their work so the + * intent write is the last step that can fail (see IntentsService). + */ +export class InMemoryIntentsUnitOfWork implements IIntentsUnitOfWork { + constructor( + private readonly intents: IIntentsRepository, + private readonly outbox: InMemoryOutboxRepository, + ) {} + + async run(work: (tx: IntentsTransaction) => Promise): Promise { + const buffered: NewOutboxEntry[] = []; + const writer: IOutboxWriter = { + enqueue: async (entry) => { + buffered.push(entry); + // The row id is assigned on commit; callers inside the unit of work + // must not depend on it. + return { ...entry, id: "uncommitted" } as OutboxEntry; + }, + }; + const result = await work({ intents: this.intents, outbox: writer }); + for (const entry of buffered) await this.outbox.enqueue(entry); + return result; + } +} diff --git a/src/intents/prisma-intents.repository.ts b/src/intents/prisma-intents.repository.ts index 4a99461e..8364eed8 100644 --- a/src/intents/prisma-intents.repository.ts +++ b/src/intents/prisma-intents.repository.ts @@ -4,6 +4,9 @@ import { IIntentsRepository } from "./intents.repository"; import { Intent, IntentState, StellarToken, TokenInfo } from "./intents.types"; import { IntentState as PrismaIntentState, Prisma } from "@prisma/client"; +/** PrismaService, or the client handed to a `$transaction` callback (issue #396). */ +export type IntentsPrismaClient = PrismaService | Prisma.TransactionClient; + /** * Prisma-backed implementation of IIntentsRepository. * @@ -18,7 +21,7 @@ import { IntentState as PrismaIntentState, Prisma } from "@prisma/client"; */ @Injectable() export class PrismaIntentsRepository implements IIntentsRepository { - constructor(private readonly prisma: PrismaService) {} + constructor(private readonly prisma: IntentsPrismaClient) {} async save(intent: Intent): Promise { const data = this.toDbData(intent); diff --git a/src/intents/slashes.controller.spec.ts b/src/intents/slashes.controller.spec.ts new file mode 100644 index 00000000..2ecdcdc3 --- /dev/null +++ b/src/intents/slashes.controller.spec.ts @@ -0,0 +1,87 @@ +import { NotFoundException, UnauthorizedException } from "@nestjs/common"; +import { Keypair } from "@stellar/stellar-sdk"; +import { buildFillProofMessage } from "../common/stellar-signature"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { AdminSlashesController, SlashesController } from "./slashes.controller"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +describe("SlashesController (#397)", () => { + const pipeline = { + getByIntent: jest.fn(), + cancelByFillProof: jest.fn().mockResolvedValue({ state: "cancelled" }), + cancelByAdmin: jest.fn().mockResolvedValue({ state: "cancelled" }), + list: jest.fn().mockResolvedValue([{ intentId: "i1" }]), + }; + const controller = new SlashesController(pipeline as unknown as SlashingPipelineService); + const solver = Keypair.random(); + const txHash = "AB".repeat(32); + + it("returns a slash or 404", async () => { + pipeline.getByIntent.mockResolvedValueOnce({ intentId: "i1" }); + await expect(controller.get("i1")).resolves.toEqual({ intentId: "i1" }); + pipeline.getByIntent.mockResolvedValueOnce(undefined); + await expect(controller.get("i2")).rejects.toBeInstanceOf(NotFoundException); + }); + + it("verifies the solver's signature over the lowercased tx hash before cancelling", async () => { + const message = buildFillProofMessage("i1", solver.publicKey(), txHash.toLowerCase()); + const signature = solver.sign(Buffer.from(message)).toString("base64"); + + await controller.fillProof("i1", { solver: solver.publicKey(), txHash, signature }); + expect(pipeline.cancelByFillProof).toHaveBeenCalledWith("i1", solver.publicKey(), txHash.toLowerCase()); + }); + + it("rejects a fill proof signed by someone else", async () => { + pipeline.cancelByFillProof.mockClear(); + const signature = Keypair.random() + .sign(Buffer.from(buildFillProofMessage("i1", solver.publicKey(), txHash.toLowerCase()))) + .toString("base64"); + await expect(controller.fillProof("i1", { solver: solver.publicKey(), txHash, signature })).rejects.toBeInstanceOf( + UnauthorizedException, + ); + expect(pipeline.cancelByFillProof).not.toHaveBeenCalled(); + }); + + describe("AdminSlashesController", () => { + const audit = { record: jest.fn().mockResolvedValue(undefined) }; + const admin = new AdminSlashesController( + pipeline as unknown as SlashingPipelineService, + audit as unknown as AdminAuditService, + ); + const principal = { id: "ops-1", role: "admin" as const }; + + it("lists slashes", async () => { + await expect(admin.list({})).resolves.toEqual({ slashes: [{ intentId: "i1" }], count: 1 }); + expect(pipeline.list).toHaveBeenCalledWith(undefined, 50); + await admin.list({ state: "submitted", limit: 5 }); + expect(pipeline.list).toHaveBeenLastCalledWith("submitted", 5); + }); + + it("audits a cancel before applying it, attributing it to the authenticated admin", async () => { + const order: string[] = []; + audit.record.mockImplementationOnce(async () => void order.push("audit")); + pipeline.cancelByAdmin.mockImplementationOnce(async () => { + order.push("cancel"); + return { state: "cancelled" }; + }); + + await admin.cancel("i1", { note: "manual review" }, principal); + + expect(order).toEqual(["audit", "cancel"]); + expect(audit.record).toHaveBeenCalledWith({ + actor: "ops-1", + action: "slash.cancel", + target: "slash:i1", + reason: "manual review", + }); + expect(pipeline.cancelByAdmin).toHaveBeenCalledWith("i1", "ops-1", "manual review"); + }); + + it("does not cancel when the audit write fails", async () => { + pipeline.cancelByAdmin.mockClear(); + audit.record.mockRejectedValueOnce(new Error("audit down")); + await expect(admin.cancel("i1", { note: "manual review" }, principal)).rejects.toThrow("audit down"); + expect(pipeline.cancelByAdmin).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/intents/slashes.controller.ts b/src/intents/slashes.controller.ts new file mode 100644 index 00000000..2c86d22f --- /dev/null +++ b/src/intents/slashes.controller.ts @@ -0,0 +1,86 @@ +import { Body, Controller, Get, NotFoundException, Param, Post, Query, UseGuards } from "@nestjs/common"; +import { + ApiConflictResponse, + ApiForbiddenResponse, + ApiHeader, + ApiNotFoundResponse, + ApiTags, + ApiUnauthorizedResponse, + ApiUnprocessableEntityResponse, +} from "@nestjs/swagger"; +import { AdminGuard, CurrentAdmin, RequireAdminRole } from "../admin/admin.guard"; +import { AdminPrincipal } from "../admin/admin-auth"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { buildFillProofMessage, verifyStellarSignature } from "../common/stellar-signature"; +import { AdminCancelSlashDto, FillProofDto, ListSlashesDto } from "./dto/slash.dto"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +/** + * Public surface of the slashing saga (issue #397): status lookup and the + * solver's fill-proof challenge. + */ +@ApiTags("slashes") +@Controller("api/v1/slashes") +export class SlashesController { + constructor(private readonly pipeline: SlashingPipelineService) {} + + @Get(":intentId") + @ApiNotFoundResponse({ description: "No slash recorded for this intent" }) + async get(@Param("intentId") intentId: string) { + const slash = await this.pipeline.getByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + return slash; + } + + @Post(":intentId/fill-proof") + @ApiUnauthorizedResponse({ description: "Signature invalid" }) + @ApiForbiddenResponse({ description: "Caller is not the slashed solver" }) + @ApiConflictResponse({ description: "Challenge window is over" }) + @ApiUnprocessableEntityResponse({ description: "Fill proof did not verify on-chain" }) + async fillProof(@Param("intentId") intentId: string, @Body() dto: FillProofDto) { + const txHash = dto.txHash.toLowerCase(); + verifyStellarSignature(dto.solver, buildFillProofMessage(intentId, dto.solver, txHash), dto.signature); + return this.pipeline.cancelByFillProof(intentId, dto.solver, txHash); + } +} + +/** + * Operator surface of the slashing saga (issue #397). Runbook: + * docs/runbooks/slash-cancellation.md. + */ +@ApiTags("admin") +@ApiHeader({ name: "x-admin-key", required: true }) +@Controller("api/v1/admin/slashes") +@UseGuards(AdminGuard) +@RequireAdminRole("admin") +export class AdminSlashesController { + constructor( + private readonly pipeline: SlashingPipelineService, + private readonly audit: AdminAuditService, + ) {} + + @Get() + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + async list(@Query() dto: ListSlashesDto) { + const slashes = await this.pipeline.list(dto.state, dto.limit ?? 50); + return { slashes, count: slashes.length }; + } + + /** Cancels a slash that has not been broadcast yet. Audited before it is applied. */ + @Post(":intentId/cancel") + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + @ApiConflictResponse({ description: "Slash already submitted, or submission in progress" }) + async cancel( + @Param("intentId") intentId: string, + @Body() dto: AdminCancelSlashDto, + @CurrentAdmin() admin: AdminPrincipal, + ) { + await this.audit.record({ + actor: admin.id, + action: "slash.cancel", + target: `slash:${intentId}`, + reason: dto.note, + }); + return this.pipeline.cancelByAdmin(intentId, admin.id, dto.note); + } +} diff --git a/src/intents/slashing-pipeline.service.spec.ts b/src/intents/slashing-pipeline.service.spec.ts new file mode 100644 index 00000000..ee9abac8 --- /dev/null +++ b/src/intents/slashing-pipeline.service.spec.ts @@ -0,0 +1,546 @@ +import { ConflictException, ForbiddenException, NotFoundException, UnprocessableEntityException } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { PrismaService } from "../prisma/prisma.service"; +import { ProtocolParamsService } from "../governance/params.service"; +import { InMemorySolversRepository } from "../solvers/in-memory-solvers.repository"; +import { InMemoryPendingSlashesRepository } from "../solvers/pending-slashes.repository"; +import { SolversService } from "../solvers/solvers.service"; +import { FillVerifierService } from "../soroban/fill-verifier.service"; +import { SlashResult, SolverRegistryService } from "../soroban/solver-registry.service"; +import { StellarTxService } from "../soroban/stellar-tx.service"; +import { TxConfirmationService } from "../soroban/tx-confirmation.service"; +import { IntentsGateway } from "./intents.gateway"; +import { InMemoryIntentsRepository } from "./intents.repository"; +import { IntentsService } from "./intents.service"; +import { SlashingPipelineService } from "./slashing-pipeline.service"; + +const WINDOW = 600; +const SKEW = 30; +const MAX_ATTEMPTS = 3; +const T0 = 1_900_000_000; // detection time, unix seconds +const at = (seconds: number) => new Date((T0 + seconds) * 1000); + +const ok = (overrides: Partial = {}): SlashResult => ({ + submitted: true, + simulated: true, + dryRun: false, + failed: false, + txHash: "slash-tx", + detail: "submitted", + ...overrides, +}); + +describe("SlashingPipelineService (#397)", () => { + let slashes: InMemoryPendingSlashesRepository; + let intents: IntentsService; + let solvers: SolversService; + let gateway: { broadcast: jest.Mock }; + let registry: { slashSolver: jest.Mock> }; + let verifier: { findLandedFill: jest.Mock; verifyFillProof: jest.Mock }; + let confirmation: { check: jest.Mock }; + let metrics: { recordSlashTransition: jest.Mock }; + let pipeline: SlashingPipelineService; + let solver: string; + let intentId: string; + let fillDeadline: number; + + async function acceptedThenSlashedIntent(): Promise { + const created = await intents.create({ + user: Keypair.random().publicKey(), + srcChain: "stellar", + srcToken: { address: "native", symbol: "XLM", name: "Stellar Lumens", decimals: 7, chain: "stellar" }, + srcAmount: "1000000", + dstToken: { contract: "CTEST", symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + deadline: T0 + 10_000, + }); + await intents.update(created.intentId, { state: "accepted", solver, deadline: fillDeadline }); + await intents.slashIfAccepted(created.intentId, { slashedAt: T0, slashReason: "missed" }); + await solvers.recordFailedFill(solver, created.intentId); + return created.intentId; + } + + const detect = () => + pipeline.detect({ intentId, solverAddress: solver, reason: "missed fill", fillDeadline, detectedAt: T0 }); + const state = async () => (await slashes.findByIntent(intentId))?.state; + const fillsFailed = async () => (await solvers.get(solver))?.fillsFailed; + + beforeEach(async () => { + slashes = new InMemoryPendingSlashesRepository(); + const repo = new InMemoryIntentsRepository(); + (repo as unknown as { store: Map }).store.clear(); + intents = new IntentsService( + repo, + { get: jest.fn().mockReturnValue(false) } as unknown as ConfigService, + {} as StellarTxService, + { intentAuditLog: { create: jest.fn().mockResolvedValue({}) } } as unknown as PrismaService, + undefined, + undefined, + { + snapshotForChain: jest.fn().mockReturnValue({ version: 0, feeBps: 30, deadlineSeconds: 1800, fillWindowSeconds: 600 }), + } as unknown as ProtocolParamsService, + ); + solvers = new SolversService(new InMemorySolversRepository()); + solver = Keypair.random().publicKey(); + await solvers.register({ + address: solver, + name: "Alpha", + bondAmount: "1000000", + isActive: true, + supportedChains: ["stellar"], + supportedTokens: ["XLM"], + avgFillTime: 30, + }); + gateway = { broadcast: jest.fn().mockResolvedValue(undefined) }; + registry = { slashSolver: jest.fn().mockResolvedValue(ok()) }; + verifier = { findLandedFill: jest.fn().mockResolvedValue(null), verifyFillProof: jest.fn() }; + confirmation = { check: jest.fn().mockResolvedValue({ status: "not_found" }) }; + metrics = { recordSlashTransition: jest.fn() }; + const slashing: AppConfig["slashing"] = { + challengeWindowSeconds: WINDOW, + clockSkewToleranceSeconds: SKEW, + maxSubmitAttempts: MAX_ATTEMPTS, + }; + pipeline = new SlashingPipelineService( + slashes, + intents, + gateway as unknown as IntentsGateway, + solvers, + registry as unknown as SolverRegistryService, + verifier as unknown as FillVerifierService, + confirmation as unknown as TxConfirmationService, + metrics as unknown as MetricsService, + { get: () => slashing } as unknown as ConfigService, + ); + fillDeadline = T0 - 5; + intentId = await acceptedThenSlashedIntent(); + }); + + afterEach(() => { + pipeline.onModuleDestroy(); + }); + + describe("detection", () => { + it("records the slash and opens a challenge window ending detectedAt + window", async () => { + const slash = await detect(); + expect(slash).toMatchObject({ state: "challenge_window", solverAddress: solver, fillDeadline }); + expect(slash.challengeEndsAt).toEqual(at(WINDOW)); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("detected"); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("challenge_window"); + expect(intents.getAuditLog(intentId).at(-1)?.reason).toMatch(/challenge window open/); + }); + + it("is exactly-once per intent: re-detection is a no-op", async () => { + await detect(); + const again = await detect(); + expect(again.state).toBe("challenge_window"); + expect(metrics.recordSlashTransition.mock.calls.filter(([s]) => s === "detected")).toHaveLength(1); + + await pipeline.processDue(at(WINDOW + 1)); + await detect(); + await pipeline.processDue(at(WINDOW + 2)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + }); + + it("resumes a row left in `detected` by a crash", async () => { + await slashes.createIfAbsent({ + intentId, + solverAddress: solver, + reason: "r", + fillDeadline, + detectedAt: at(0), + challengeEndsAt: at(WINDOW), + }); + await pipeline.processDue(at(1)); + expect(await state()).toBe("challenge_window"); + }); + }); + + describe("challenge window", () => { + it("does not submit before the window ends", async () => { + await detect(); + await pipeline.processDue(at(WINDOW - 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await state()).toBe("challenge_window"); + }); + + it("re-verifies, then submits once the window is over", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(verifier.findLandedFill).toHaveBeenCalledWith(intentId, fillDeadline, fillDeadline + SKEW, T0 + WINDOW); + expect(registry.slashSolver).toHaveBeenCalledWith({ solverAddress: solver, intentId, reason: "missed fill" }); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "submitted", + txHash: "slash-tx", + simulated: false, + }); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("submitted", "broadcast"); + }); + + it("dry-run / gated submit is recorded as simulated and not polled for confirmation", async () => { + registry.slashSolver.mockResolvedValueOnce(ok({ submitted: false, dryRun: true, txHash: undefined })); + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "submitted", simulated: true }); + await pipeline.processDue(at(WINDOW + 3600)); + expect(confirmation.check).not.toHaveBeenCalled(); + }); + }); + + describe("scenario: late fill", () => { + it("cancels when re-verification finds a fill that landed in time, and compensates", async () => { + await detect(); + verifier.findLandedFill.mockResolvedValueOnce({ txHash: "fill-tx", ledger: 9, closedAt: fillDeadline + SKEW }); + + await pipeline.processDue(at(WINDOW)); + + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "cancelled", + cancelReason: "fill_landed", + cancelledBy: "system", + fillTxHash: "fill-tx", + }); + expect(await fillsFailed()).toBe(0); + expect((await intents.get(intentId))?.state).toBe("filled"); + expect((await intents.get(intentId))?.txHash).toBe("fill-tx"); + expect(gateway.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ type: "intent_slash_cancelled", intentId, reason: "fill_landed" }), + ); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("cancelled", "fill_landed"); + }); + + it("solver fill-proof during the window cancels the slash", async () => { + await detect(); + verifier.verifyFillProof.mockResolvedValueOnce({ valid: true, fill: { txHash: "f", ledger: 1, closedAt: 1 } }); + + const cancelled = await pipeline.cancelByFillProof(intentId, solver, "f"); + + expect(verifier.verifyFillProof).toHaveBeenCalledWith("f", intentId, fillDeadline + SKEW); + expect(cancelled).toMatchObject({ state: "cancelled", cancelledBy: solver, fillTxHash: "f" }); + expect(await fillsFailed()).toBe(0); + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + }); + + it("rejects fill-proofs that do not verify, come from another solver, or arrive after the window", async () => { + await expect(pipeline.cancelByFillProof("nope", solver, "f")).rejects.toBeInstanceOf(NotFoundException); + await detect(); + await expect(pipeline.cancelByFillProof(intentId, Keypair.random().publicKey(), "f")).rejects.toBeInstanceOf( + ForbiddenException, + ); + verifier.verifyFillProof.mockResolvedValueOnce({ valid: false, reason: "closed too late" }); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf( + UnprocessableEntityException, + ); + + await pipeline.processDue(at(WINDOW)); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf(ConflictException); + }); + + it("reports a conflict if the slash moved on while the proof was being verified", async () => { + await detect(); + verifier.verifyFillProof.mockImplementationOnce(async () => { + await slashes.transition(intentId, ["challenge_window"], { state: "submitted" }); + return { valid: true, fill: { txHash: "f", ledger: 1, closedAt: 1 } }; + }); + await expect(pipeline.cancelByFillProof(intentId, solver, "f")).rejects.toBeInstanceOf(ConflictException); + }); + }); + + describe("scenario: admin cancel", () => { + it("cancels during the window, compensates, and expires the intent", async () => { + await detect(); + const cancelled = await pipeline.cancelByAdmin(intentId, "ops@vortex", "event delayed, fill confirmed manually"); + expect(cancelled).toMatchObject({ + state: "cancelled", + cancelledBy: "ops@vortex", + cancelReason: "admin: event delayed, fill confirmed manually", + }); + expect(await fillsFailed()).toBe(0); + expect((await intents.get(intentId))?.state).toBe("expired"); + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + }); + + it("compensates at most once", async () => { + await detect(); + await pipeline.cancelByAdmin(intentId, "ops", "first cancel"); + await expect(pipeline.cancelByAdmin(intentId, "ops", "second cancel")).rejects.toBeInstanceOf(ConflictException); + expect(await fillsFailed()).toBe(0); + }); + + it("refuses once submitted, while a worker holds the lease, or for an unknown intent", async () => { + await expect(pipeline.cancelByAdmin("unknown", "ops", "why not")).rejects.toBeInstanceOf(NotFoundException); + + await detect(); + await slashes.claim(intentId, new Date(), new Date(Date.now() + 60_000)); + await expect(pipeline.cancelByAdmin(intentId, "ops", "mid submit")).rejects.toThrow(/submission in progress/); + + await slashes.transition(intentId, ["challenge_window"], { state: "submitted" }); + await expect(pipeline.cancelByAdmin(intentId, "ops", "too late")).rejects.toThrow(/state=submitted/); + }); + }); + + describe("scenario: RPC failure during submit", () => { + it("retries with backoff and does not submit blind when re-verification fails", async () => { + await detect(); + verifier.findLandedFill.mockRejectedValueOnce(new Error("RPC timeout")); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "challenge_window", attempts: 1, lastError: expect.stringContaining("RPC timeout") }); + expect(row!.nextAttemptAt).toEqual(new Date(at(WINDOW).getTime() + 5_000)); + + await pipeline.processDue(at(WINDOW + 1)); + expect(registry.slashSolver).not.toHaveBeenCalled(); // still backing off + await pipeline.processDue(at(WINDOW + 5)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + expect(await state()).toBe("submitted"); + }); + + it("gives up after max attempts: cancels, compensates, and alerts", async () => { + await detect(); + registry.slashSolver.mockResolvedValue(ok({ submitted: false, failed: true, detail: "simulation failed" })); + const errorLog = jest.spyOn((pipeline as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + + let t = WINDOW; + for (let i = 0; i < MAX_ATTEMPTS; i++) { + await pipeline.processDue(at(t)); + t += 3600; + } + + expect(registry.slashSolver).toHaveBeenCalledTimes(MAX_ATTEMPTS); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "cancelled", + cancelReason: "submit_failed", + attempts: MAX_ATTEMPTS, + }); + expect(await fillsFailed()).toBe(0); + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("ALERT")); + }); + + it("an unexpected throw is retried, not fatal to the batch", async () => { + await detect(); + registry.slashSolver.mockRejectedValueOnce(new Error("kaboom")); + await pipeline.processDue(at(WINDOW)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "challenge_window", + lastError: expect.stringContaining("kaboom"), + }); + }); + }); + + describe("scenario: kill-switch pause (issue #477)", () => { + it("defers the slash without consuming attempts, then submits after resume", async () => { + await detect(); + registry.slashSolver.mockRejectedValue( + new KillSwitchActiveException({ + reasonCode: "INCIDENT", + reason: "paused", + scope: "operation", + chain: "stellar", + token: null, + operation: "slash", + } as never), + ); + + let t = WINDOW; + for (let i = 0; i < MAX_ATTEMPTS + 2; i++) { + await pipeline.processDue(at(t)); + t += 61; + } + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "challenge_window", attempts: 0, lastError: expect.stringContaining("kill-switch") }); + expect(await fillsFailed()).toBe(1); // not compensated: the slash is only paused + + registry.slashSolver.mockReset(); + registry.slashSolver.mockResolvedValue(ok()); + await pipeline.processDue(at(t + 61)); + expect(await state()).toBe("submitted"); + }); + }); + + describe("confirmation", () => { + beforeEach(async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + }); + + it("confirms on success", async () => { + confirmation.check.mockResolvedValueOnce({ status: "success", ledger: 1 }); + await pipeline.processDue(at(WINDOW + 10)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "confirmed", confirmedAt: at(WINDOW + 10) }); + expect(metrics.recordSlashTransition).toHaveBeenCalledWith("confirmed"); + }); + + it("waits while not found within the grace period", async () => { + await pipeline.processDue(at(WINDOW + 10)); + expect(await state()).toBe("submitted"); + }); + + it("re-opens for re-verification and resubmission when the tx failed or vanished", async () => { + confirmation.check.mockResolvedValueOnce({ status: "failed", ledger: 1 }); + await pipeline.processDue(at(WINDOW + 10)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "challenge_window", attempts: 1 }); + + await pipeline.processDue(at(WINDOW + 100)); + expect(registry.slashSolver).toHaveBeenCalledTimes(2); + expect(verifier.findLandedFill).toHaveBeenCalledTimes(2); + + await pipeline.processDue(at(WINDOW + 100 + 121)); // not found past grace + expect(await slashes.findByIntent(intentId)).toMatchObject({ + state: "challenge_window", + lastError: expect.stringContaining("not found"), + }); + }); + + it("backs off when the confirmation lookup itself fails", async () => { + confirmation.check.mockRejectedValueOnce(new Error("RPC down")); + await pipeline.processDue(at(WINDOW + 10)); + const row = await slashes.findByIntent(intentId); + expect(row).toMatchObject({ state: "submitted", lastError: expect.stringContaining("RPC down") }); + expect(row!.nextAttemptAt.getTime()).toBeGreaterThan(at(WINDOW + 10).getTime()); + }); + }); + + describe("edge cases", () => { + it("clock skew: judges timeliness against fillDeadline + tolerance, on chain time", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + expect(verifier.findLandedFill.mock.calls[0][2]).toBe(fillDeadline + SKEW); + }); + + it("solver deregistered mid-window is still slashed (deregistration is not an escape hatch)", async () => { + await detect(); + await solvers.deregister(solver); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + expect(intents.getAuditLog(intentId).at(-1)?.metadata).toMatchObject({ solverActive: false }); + }); + + it("solver with no record is cancelled and compensated rather than submitted", async () => { + await detect(); + jest.spyOn(solvers, "get").mockResolvedValueOnce(undefined); + const rollback = jest.spyOn(solvers, "rollbackPenalty"); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "cancelled", cancelReason: "solver_not_found" }); + expect(rollback).toHaveBeenCalledWith(intentId, solver); + }); + + it("compensation survives a restart that lost the in-memory penalty record", async () => { + await detect(); + (solvers as unknown as { pendingPenalties: Map }).pendingPenalties.clear(); + await pipeline.cancelByAdmin(intentId, "ops", "after restart"); + expect(await fillsFailed()).toBe(0); + }); + + it("leaves the intent alone if it is no longer `slashed`", async () => { + await detect(); + await intents.update(intentId, { state: "filled" }); + await pipeline.cancelByAdmin(intentId, "ops", "already filled"); + expect((await intents.get(intentId))?.state).toBe("filled"); + }); + + it("skips rows another worker has leased, and overlapping passes", async () => { + await detect(); + await slashes.claim(intentId, at(0), at(WINDOW + 60)); + await pipeline.processDue(at(WINDOW)); + expect(registry.slashSolver).not.toHaveBeenCalled(); + + const claim = jest.spyOn(slashes, "claim").mockResolvedValueOnce(false); + await pipeline.processDue(at(WINDOW + 61)); + expect(claim).toHaveBeenCalled(); + expect(registry.slashSolver).not.toHaveBeenCalled(); + + let release!: () => void; + jest.spyOn(slashes, "findDue").mockImplementationOnce( + () => new Promise((resolve) => (release = () => resolve([]))), + ); + const first = pipeline.processDue(at(WINDOW + 62)); + await pipeline.processDue(at(WINDOW + 62)); // overlapping → no-op + release(); + await first; + }); + + it("exposes lookup and listing", async () => { + await detect(); + expect((await pipeline.getByIntent(intentId))?.intentId).toBe(intentId); + expect(await pipeline.list("challenge_window", 10)).toHaveLength(1); + expect(await pipeline.list("confirmed", 10)).toHaveLength(0); + }); + + it("runs on an interval and logs failures", () => { + jest.useFakeTimers(); + try { + const process = jest.spyOn(pipeline, "processDue").mockRejectedValue(new Error("tick failed")); + const errorLog = jest.spyOn((pipeline as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + pipeline.onModuleInit(); + jest.advanceTimersByTime(15_000); + expect(process).toHaveBeenCalledTimes(1); + return Promise.resolve().then(() => expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("tick failed"))); + } finally { + jest.useRealTimers(); + } + }); + }); + + describe("lost races (another worker or an admin won the conditional transition)", () => { + const loseNextTransition = () => jest.spyOn(slashes, "transition").mockResolvedValueOnce(null); + + it("detect returns the stored row if the window was already opened elsewhere", async () => { + loseNextTransition(); + const slash = await detect(); + expect(slash.state).toBe("detected"); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("challenge_window"); + }); + + it("does not record a submission it lost", async () => { + await detect(); + loseNextTransition(); + await pipeline.processDue(at(WINDOW)); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("submitted", expect.anything()); + }); + + it("does not record a confirmation or reopen it lost", async () => { + await detect(); + await pipeline.processDue(at(WINDOW)); + + confirmation.check.mockResolvedValueOnce({ status: "success", ledger: 1 }); + loseNextTransition(); + await pipeline.processDue(at(WINDOW + 10)); + expect(metrics.recordSlashTransition).not.toHaveBeenCalledWith("confirmed"); + + confirmation.check.mockResolvedValueOnce({ status: "failed", ledger: 1 }); + loseNextTransition(); + await pipeline.processDue(at(WINDOW + 20)); + expect(registry.slashSolver).toHaveBeenCalledTimes(1); + }); + + it("releases a broadcast row with no tx hash, and treats a missing submittedAt as past grace", async () => { + await detect(); + await slashes.transition(intentId, ["challenge_window"], { state: "submitted", simulated: false }); + await pipeline.processDue(at(WINDOW)); // no txHash → released, nothing checked + expect(confirmation.check).not.toHaveBeenCalled(); + + await slashes.transition(intentId, ["submitted"], { txHash: "h" }); + await pipeline.processDue(at(WINDOW + 1)); + expect(await slashes.findByIntent(intentId)).toMatchObject({ state: "challenge_window" }); + }); + + it("stringifies non-Error failures and defaults processDue's clock", async () => { + await slashes.createIfAbsent({ + intentId, solverAddress: solver, reason: "r", fillDeadline, detectedAt: new Date(0), challengeEndsAt: new Date(0), + }); + await slashes.transition(intentId, ["detected"], { state: "challenge_window" }); + verifier.findLandedFill.mockRejectedValueOnce("plain string"); + await pipeline.processDue(); + expect((await slashes.findByIntent(intentId))?.lastError).toContain("plain string"); + }); + }); +}); diff --git a/src/intents/slashing-pipeline.service.ts b/src/intents/slashing-pipeline.service.ts new file mode 100644 index 00000000..7bb8ba84 --- /dev/null +++ b/src/intents/slashing-pipeline.service.ts @@ -0,0 +1,459 @@ +import { + ConflictException, + ForbiddenException, + Inject, + Injectable, + Logger, + NotFoundException, + OnModuleDestroy, + OnModuleInit, + UnprocessableEntityException, +} from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { SolversService } from "../solvers/solvers.service"; +import { + IPendingSlashesRepository, + PENDING_SLASHES_REPOSITORY, + PendingSlash, + PendingSlashState, +} from "../solvers/pending-slashes.repository"; +import { FillVerifierService } from "../soroban/fill-verifier.service"; +import { SolverRegistryService } from "../soroban/solver-registry.service"; +import { TxConfirmationService } from "../soroban/tx-confirmation.service"; +import { IntentsGateway } from "./intents.gateway"; +import { IntentsService } from "./intents.service"; + +const PROCESS_INTERVAL_MS = 15_000; +const PROCESS_BATCH_SIZE = 25; +/** Lease while one worker verifies/submits a slash; also blocks admin cancel mid-submit. */ +const LEASE_SECONDS = 120; +const BASE_BACKOFF_MS = 5_000; +const MAX_BACKOFF_MS = 10 * 60_000; +/** Re-check interval while a kill-switch pause blocks slashing. */ +const PAUSED_RECHECK_MS = 60_000; +/** How long a submitted slash may stay unseen by RPC before it is treated as dropped. */ +const SUBMIT_NOT_FOUND_GRACE_SECONDS = 120; + +/** Why a slash was cancelled — also the `reason` label on the metric. */ +export type SlashCancelReason = + | "fill_landed" + | "admin" + | "solver_not_found" + | "submit_failed"; + +/** Cancellable before broadcast only; after that the chain decides. */ +const CANCELLABLE: PendingSlashState[] = ["detected", "challenge_window"]; + +export interface DetectMissedFillInput { + intentId: string; + solverAddress: string; + reason: string; + /** The accepted intent's fill deadline, unix seconds. */ + fillDeadline: number; + /** Server time at detection, unix seconds. */ + detectedAt: number; +} + +/** + * Saga connecting sweeper detection to the on-chain slash (issue #397): + * + * detected → challenge_window → submitted → confirmed | cancelled + * + * - **Durable + exactly-once**: one `pending_slashes` row per intent (unique + * constraint). Re-detection is a no-op; every step is a conditional state + * transition, so a crash anywhere resumes from the stored state. + * - **Challenge window** (SLASH_CHALLENGE_WINDOW_SECONDS, default 10 min): the + * slash is not broadcast before it ends. During it a solver fill-proof or an + * admin cancels the slash. + * - **Re-verification**: when the window ends the chain is checked again for a + * fill that landed by `fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS` + * (ledger close time). If one landed, the slash is cancelled. If the check + * itself fails, the slash is retried later — never submitted blind. + * - **Compensation**: the sweeper's optimistic `recordFailedFill` is reverted + * via `SolversService.rollbackPenalty` on every cancellation, and the intent + * leaves `slashed` (→ `filled` for a proven fill, → `expired` otherwise). + * Compensation runs only on the winning `→ cancelled` transition, so it + * happens at most once. + */ +@Injectable() +export class SlashingPipelineService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(SlashingPipelineService.name); + private readonly settings: AppConfig["slashing"]; + private interval?: NodeJS.Timeout; + private running = false; + + constructor( + @Inject(PENDING_SLASHES_REPOSITORY) private readonly slashes: IPendingSlashesRepository, + private readonly intentsService: IntentsService, + private readonly intentsGateway: IntentsGateway, + private readonly solversService: SolversService, + private readonly solverRegistry: SolverRegistryService, + private readonly fillVerifier: FillVerifierService, + private readonly confirmation: TxConfirmationService, + private readonly metrics: MetricsService, + configService: ConfigService, + ) { + this.settings = configService.get("slashing", { infer: true }); + } + + onModuleInit() { + this.interval = setInterval(() => { + this.processDue().catch((err) => + this.logger.error(`[slashing] processing failed: ${errorMessage(err)}`), + ); + }, PROCESS_INTERVAL_MS); + this.interval.unref?.(); + } + + onModuleDestroy() { + if (this.interval) clearInterval(this.interval); + } + + /** + * Records a missed fill and opens its challenge window. Idempotent per + * intent: a second detection returns the existing slash unchanged. + */ + async detect(input: DetectMissedFillInput): Promise { + const detectedAt = new Date(input.detectedAt * 1000); + const { slash, created } = await this.slashes.createIfAbsent({ + intentId: input.intentId, + solverAddress: input.solverAddress, + reason: input.reason, + fillDeadline: input.fillDeadline, + detectedAt, + challengeEndsAt: new Date(detectedAt.getTime() + this.settings.challengeWindowSeconds * 1000), + }); + if (!created) { + this.logger.warn(`[slashing] intent=${input.intentId} already has a slash (state=${slash.state}); ignoring`); + return slash; + } + this.metrics.recordSlashTransition("detected"); + return (await this.openChallengeWindow(slash)) ?? slash; + } + + /** Runs one pass over every slash that needs action now. */ + async processDue(now: Date = new Date()): Promise { + if (this.running) return; + this.running = true; + try { + const due = await this.slashes.findDue(now, PROCESS_BATCH_SIZE); + for (const slash of due) { + const leaseUntil = new Date(now.getTime() + LEASE_SECONDS * 1000); + if (!(await this.slashes.claim(slash.intentId, now, leaseUntil))) continue; + try { + await this.step(slash, now); + } catch (err) { + await this.retryOrGiveUp(slash, `unexpected error: ${errorMessage(err)}`, now); + } + } + } finally { + this.running = false; + } + } + + /** + * Cancels a slash that has not been broadcast yet (admin action). + * @throws NotFoundException / ConflictException when there is nothing to cancel. + */ + async cancelByAdmin(intentId: string, actor: string, note: string): Promise { + const slash = await this.slashes.findByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + const cancelled = await this.cancel(slash, "admin", actor, { note }, new Date()); + if (!cancelled) { + throw new ConflictException( + `Slash for intent ${intentId} cannot be cancelled (state=${slash.state}` + + `${slash.lockedUntil ? ", submission in progress — retry shortly" : ""})`, + ); + } + return cancelled; + } + + /** + * Cancels a slash on a solver-supplied proof that its fill landed in time. + * @throws when the slash is not cancellable, the solver is not the slashed + * one, or the proof does not verify on-chain. + */ + async cancelByFillProof(intentId: string, solver: string, txHash: string): Promise { + const slash = await this.slashes.findByIntent(intentId); + if (!slash) throw new NotFoundException(`No slash recorded for intent ${intentId}`); + if (slash.solverAddress !== solver) { + throw new ForbiddenException("Only the slashed solver may submit a fill proof"); + } + if (!CANCELLABLE.includes(slash.state)) { + throw new ConflictException(`Slash for intent ${intentId} is ${slash.state}; the challenge window is over`); + } + + const proof = await this.fillVerifier.verifyFillProof(txHash, intentId, this.latestAcceptable(slash)); + if (!proof.valid) throw new UnprocessableEntityException(`Fill proof rejected: ${proof.reason}`); + + const cancelled = await this.cancel(slash, "fill_landed", solver, { fillTxHash: txHash }, new Date()); + if (!cancelled) { + throw new ConflictException(`Slash for intent ${intentId} changed state while verifying; retry`); + } + return cancelled; + } + + getByIntent(intentId: string): Promise { + return this.slashes.findByIntent(intentId); + } + + list(state: PendingSlashState | undefined, limit: number): Promise { + return this.slashes.list({ state, limit }); + } + + // ── saga steps ──────────────────────────────────────────────────────────── + + private async step(slash: PendingSlash, now: Date): Promise { + switch (slash.state) { + case "detected": + await this.openChallengeWindow(slash); + return; + case "challenge_window": + await this.verifyAndSubmit(slash, now); + return; + case "submitted": + await this.confirm(slash, now); + return; + // confirmed / cancelled are terminal and never returned by findDue. + } + } + + private async openChallengeWindow(slash: PendingSlash): Promise { + const opened = await this.slashes.transition(slash.intentId, ["detected"], { state: "challenge_window" }); + if (!opened) return null; + this.metrics.recordSlashTransition("challenge_window"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash pending: challenge window open", { + solver: slash.solverAddress, + challengeEndsAt: opened.challengeEndsAt.toISOString(), + }); + this.logger.log( + `[slashing] intent=${slash.intentId} solver=${slash.solverAddress} challenge window open until ` + + opened.challengeEndsAt.toISOString(), + ); + return opened; + } + + private async verifyAndSubmit(slash: PendingSlash, now: Date): Promise { + // 1. Re-verify: a fill that landed in time (late event, missed API call) + // must never be slashed. Failure to check is not evidence of absence. + let fill; + try { + fill = await this.fillVerifier.findLandedFill( + slash.intentId, + slash.fillDeadline, + this.latestAcceptable(slash), + Math.floor(now.getTime() / 1000), + ); + } catch (err) { + await this.retryOrGiveUp(slash, `fill re-verification failed: ${errorMessage(err)}`, now); + return; + } + if (fill) { + await this.cancel(slash, "fill_landed", "system", { fillTxHash: fill.txHash }, undefined); + return; + } + + // 2. Solver deregistered mid-window: deregistration must not be an escape + // hatch, so the slash proceeds. A solver with no record at all cannot + // be penalised by the registry — cancel and compensate. + const solver = await this.solversService.get(slash.solverAddress); + if (!solver) { + await this.cancel(slash, "solver_not_found", "system", {}, undefined); + return; + } + if (!solver.isActive) { + this.logger.warn( + `[slashing] solver=${slash.solverAddress} deregistered during the challenge window; slashing intent=${slash.intentId} anyway`, + ); + } + + // 3. Submit. The registry contract keys slashes by intent id, so a + // resubmission after a crash cannot double-slash on-chain. + let result; + try { + result = await this.solverRegistry.slashSolver({ + solverAddress: slash.solverAddress, + intentId: slash.intentId, + reason: slash.reason, + }); + } catch (err) { + if (!(err instanceof KillSwitchActiveException)) throw err; + // Issue #477 — a pause on `slash`/`onchain` defers the slash without + // consuming an attempt, so a long pause can never make the saga give up. + await this.slashes.transition(slash.intentId, ["challenge_window"], { + nextAttemptAt: new Date(now.getTime() + PAUSED_RECHECK_MS), + lastError: `paused by kill-switch: ${err.message}`, + }); + return; + } + if (result.failed) { + await this.retryOrGiveUp(slash, result.detail, now); + return; + } + + const submitted = await this.slashes.transition(slash.intentId, ["challenge_window"], { + state: "submitted", + txHash: result.txHash, + simulated: !result.submitted, + submittedAt: now, + nextAttemptAt: now, + lastError: undefined, + }); + if (!submitted) return; + this.metrics.recordSlashTransition("submitted", result.submitted ? "broadcast" : "simulated"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash submitted", { + solver: slash.solverAddress, + txHash: result.txHash, + simulated: !result.submitted, + solverActive: solver.isActive, + detail: result.detail, + }); + this.logger.log( + `[slashing] intent=${slash.intentId} solver=${slash.solverAddress} submitted ` + + `(${result.submitted ? `tx ${result.txHash}` : "simulated only"}): ${result.detail}`, + ); + } + + private async confirm(slash: PendingSlash, now: Date): Promise { + if (!slash.txHash) { + await this.slashes.transition(slash.intentId, ["submitted"], {}); + return; + } + let status; + try { + status = (await this.confirmation.check(slash.txHash)).status; + } catch (err) { + await this.slashes.transition(slash.intentId, ["submitted"], { + nextAttemptAt: this.backoff(slash.attempts, now), + lastError: `confirmation lookup failed: ${errorMessage(err)}`, + }); + return; + } + + if (status === "success") { + const confirmed = await this.slashes.transition(slash.intentId, ["submitted"], { + state: "confirmed", + confirmedAt: now, + }); + if (confirmed) { + this.metrics.recordSlashTransition("confirmed"); + this.intentsService.appendAuditEntry(slash.intentId, "slashed", "system", "slash confirmed on-chain", { + txHash: slash.txHash, + }); + } + return; + } + + const age = slash.submittedAt ? now.getTime() - slash.submittedAt.getTime() : Infinity; + if (status === "failed" || age > SUBMIT_NOT_FOUND_GRACE_SECONDS * 1000) { + // Back into the window state so the next pass re-verifies and resubmits. + const reopened = await this.slashes.transition(slash.intentId, ["submitted"], { + state: "challenge_window", + txHash: undefined, + }); + if (reopened) { + await this.retryOrGiveUp(reopened, `slash tx ${slash.txHash} ${status === "failed" ? "failed on-chain" : "not found"}`, now); + } + return; + } + + await this.slashes.transition(slash.intentId, ["submitted"], { + nextAttemptAt: new Date(now.getTime() + BASE_BACKOFF_MS), + }); + } + + private async retryOrGiveUp(slash: PendingSlash, error: string, now: Date): Promise { + const attempts = slash.attempts + 1; + if (attempts >= this.settings.maxSubmitAttempts) { + this.logger.error( + `[slashing] ALERT giving up on slash for intent=${slash.intentId} solver=${slash.solverAddress} ` + + `after ${attempts} attempts: ${error}. Cancelling and compensating — see docs/runbooks/slash-cancellation.md`, + ); + await this.cancel({ ...slash, attempts }, "submit_failed", "system", { lastError: error }, undefined); + return; + } + await this.slashes.transition(slash.intentId, [slash.state], { + attempts, + nextAttemptAt: this.backoff(attempts, now), + lastError: error, + }); + this.logger.warn( + `[slashing] intent=${slash.intentId} attempt ${attempts}/${this.settings.maxSubmitAttempts} failed: ${error}`, + ); + } + + /** + * Compensating transaction. Runs only if this call wins the `→ cancelled` + * transition, so the rollback happens at most once per slash. + * + * @param now pass for externally-triggered cancels so an in-flight submission + * (active lease) is never cancelled underneath the worker. + */ + private async cancel( + slash: PendingSlash, + reason: SlashCancelReason, + actor: string, + extra: { fillTxHash?: string; note?: string; lastError?: string }, + now: Date | undefined, + ): Promise { + const cancelled = await this.slashes.transition( + slash.intentId, + reason === "submit_failed" ? ["challenge_window", "submitted"] : CANCELLABLE, + { + state: "cancelled", + cancelledAt: new Date(), + cancelReason: extra.note ? `${reason}: ${extra.note}` : reason, + cancelledBy: actor, + fillTxHash: extra.fillTxHash, + lastError: extra.lastError, + attempts: slash.attempts, + }, + now, + ); + if (!cancelled) return null; + + await this.solversService.rollbackPenalty(slash.intentId, slash.solverAddress); + + const intent = await this.intentsService.get(slash.intentId); + if (intent?.state === "slashed") { + if (reason === "fill_landed") { + await this.intentsService.update(slash.intentId, { state: "filled", txHash: extra.fillTxHash }); + } else { + await this.intentsService.update(slash.intentId, { state: "expired" }); + } + } + const toState = reason === "fill_landed" ? "filled" : "expired"; + this.intentsService.appendAuditEntry(slash.intentId, toState, actor, `slash cancelled: ${reason}`, { + solver: slash.solverAddress, + fillTxHash: extra.fillTxHash, + note: extra.note, + }); + await this.intentsGateway.broadcast({ + type: "intent_slash_cancelled", + intentId: slash.intentId, + solver: slash.solverAddress, + reason, + }); + this.metrics.recordSlashTransition("cancelled", reason); + this.logger.warn( + `[slashing] slash for intent=${slash.intentId} solver=${slash.solverAddress} cancelled by ${actor}: ${reason}`, + ); + return cancelled; + } + + private latestAcceptable(slash: PendingSlash): number { + return slash.fillDeadline + this.settings.clockSkewToleranceSeconds; + } + + private backoff(attempts: number, now: Date): Date { + const delay = Math.min(BASE_BACKOFF_MS * 2 ** Math.max(0, attempts - 1), MAX_BACKOFF_MS); + return new Date(now.getTime() + delay); + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/metrics/metrics.service.ts b/src/metrics/metrics.service.ts index 51d5047f..f1a184d0 100644 --- a/src/metrics/metrics.service.ts +++ b/src/metrics/metrics.service.ts @@ -101,6 +101,13 @@ export class MetricsService implements OnModuleInit { // ── Solver-registry event ingestion (issue #399) ────────────────────────── public readonly solverRegistryEventsTotal: client.Counter; + /** Transactional outbox relay (issue #396). */ + public readonly outboxRelayOutcomes: client.Counter; + public readonly outboxDeadTotal: client.Counter; + public readonly outboxBacklog: client.Gauge; + + /** Slashing saga (issue #397). */ + public readonly slashTransitions: client.Counter; constructor(private readonly configService: ConfigService) { this.register = new client.Registry(); @@ -391,6 +398,34 @@ export class MetricsService implements OnModuleInit { provider: () => Promise>, ): void { this.queueDepthProvider = provider; + // ── Outbox relay (issue #396) ─────────────────────────────────────────── + this.outboxRelayOutcomes = new client.Counter({ + name: `${prefix}outbox_relay_outcomes_total`, + help: "Outbox rows processed by the relay, by outcome (submitted|simulated|confirmed|retry|dead)", + labelNames: ["outcome"], + registers: [this.register], + }); + + this.outboxDeadTotal = new client.Counter({ + name: `${prefix}outbox_dead_total`, + help: "Outbox rows moved to dead after exhausting OUTBOX_MAX_ATTEMPTS (page on any increase)", + registers: [this.register], + }); + + this.outboxBacklog = new client.Gauge({ + name: `${prefix}outbox_rows`, + help: "Current number of outbox rows by status", + labelNames: ["status"], + registers: [this.register], + }); + + // ── Slashing saga (issue #397) ────────────────────────────────────────── + this.slashTransitions = new client.Counter({ + name: `${prefix}slash_pipeline_transitions_total`, + help: "Pending-slash state transitions, by target state and reason", + labelNames: ["to_state", "reason"], + registers: [this.register], + }); } onModuleInit() { @@ -537,4 +572,20 @@ export class MetricsService implements OnModuleInit { this.leaderElectionIsLeader.set({ worker: workerName }, 0); this.leaderElectionChangesTotal.inc({ worker: workerName, transition: "lost" }); } + + /** One outbox row outcome; `dead` also feeds the alerting counter. */ + recordOutboxOutcome(outcome: "submitted" | "simulated" | "confirmed" | "retry" | "dead"): void { + this.outboxRelayOutcomes.inc({ outcome }); + if (outcome === "dead") this.outboxDeadTotal.inc(); + } + + setOutboxBacklog(counts: Record): void { + for (const [status, count] of Object.entries(counts)) { + this.outboxBacklog.set({ status }, count); + } + } + + recordSlashTransition(toState: string, reason = "none"): void { + this.slashTransitions.inc({ to_state: toState, reason }); + } } diff --git a/src/solvers/pending-slashes.repository.spec.ts b/src/solvers/pending-slashes.repository.spec.ts new file mode 100644 index 00000000..995dbe71 --- /dev/null +++ b/src/solvers/pending-slashes.repository.spec.ts @@ -0,0 +1,127 @@ +import { InMemoryPendingSlashesRepository, NewPendingSlash } from "./pending-slashes.repository"; +import { PrismaPendingSlashesRepository } from "./prisma-pending-slashes.repository"; +import { PrismaService } from "../prisma/prisma.service"; + +const t = (s: number) => new Date(1_900_000_000_000 + s * 1000); +const input = (intentId = "i1"): NewPendingSlash => ({ + intentId, + solverAddress: "GSOLVER", + reason: "missed", + fillDeadline: 1_899_999_990, + detectedAt: t(0), + challengeEndsAt: t(600), +}); + +describe("InMemoryPendingSlashesRepository (#397)", () => { + let repo: InMemoryPendingSlashesRepository; + beforeEach(() => (repo = new InMemoryPendingSlashesRepository())); + + it("enforces one slash per intent", async () => { + const first = await repo.createIfAbsent(input()); + const second = await repo.createIfAbsent({ ...input(), reason: "dup" }); + expect(first.created).toBe(true); + expect(second).toMatchObject({ created: false, slash: { id: first.slash.id, reason: "missed" } }); + expect(await repo.findByIntent("nope")).toBeUndefined(); + }); + + it("finds due rows by state, window, backoff and lease", async () => { + await repo.createIfAbsent(input("detected")); + await repo.createIfAbsent(input("window")); + await repo.transition("window", ["detected"], { state: "challenge_window" }); + await repo.createIfAbsent(input("sim")); + await repo.transition("sim", ["detected"], { state: "submitted", simulated: true }); + await repo.createIfAbsent(input("live")); + await repo.transition("live", ["detected"], { state: "submitted", simulated: false, nextAttemptAt: t(0) }); + + expect((await repo.findDue(t(599), 10)).map((r) => r.intentId).sort()).toEqual(["detected", "live"]); + expect((await repo.findDue(t(600), 10)).map((r) => r.intentId).sort()).toEqual(["detected", "live", "window"]); + expect(await repo.findDue(t(600), 1)).toHaveLength(1); + + expect(await repo.claim("window", t(600), t(700))).toBe(true); + expect(await repo.claim("window", t(650), t(750))).toBe(false); + expect(await repo.claim("missing", t(650), t(750))).toBe(false); + expect((await repo.findDue(t(650), 10)).map((r) => r.intentId)).not.toContain("window"); + expect((await repo.findDue(t(701), 10)).map((r) => r.intentId)).toContain("window"); + }); + + it("transition is guarded by state and, when `now` is given, by the lease", async () => { + await repo.createIfAbsent(input()); + expect(await repo.transition("i1", ["submitted"], { state: "confirmed" })).toBeNull(); + expect(await repo.transition("missing", ["detected"], {})).toBeNull(); + + await repo.claim("i1", t(0), t(100)); + expect(await repo.transition("i1", ["detected"], { state: "cancelled" }, t(50))).toBeNull(); + const moved = await repo.transition("i1", ["detected"], { state: "challenge_window" }); + expect(moved).toMatchObject({ state: "challenge_window", lockedUntil: undefined }); + }); + + it("lists newest first with an optional state filter", async () => { + await repo.createIfAbsent(input("a")); + await repo.createIfAbsent({ ...input("b"), detectedAt: t(10) }); + expect((await repo.list({ limit: 10 })).map((r) => r.intentId)).toEqual(["b", "a"]); + expect(await repo.list({ state: "confirmed", limit: 10 })).toEqual([]); + }); +}); + +describe("PrismaPendingSlashesRepository (#397)", () => { + const row = { ...input(), id: "s1", state: "detected", attempts: 0, nextAttemptAt: t(0), lockedUntil: null, txHash: null, + simulated: false, submittedAt: null, confirmedAt: null, cancelledAt: null, cancelReason: null, cancelledBy: null, + fillTxHash: null, lastError: null, createdAt: t(0), updatedAt: t(0) }; + let pendingSlash: Record; + let repo: PrismaPendingSlashesRepository; + + beforeEach(() => { + pendingSlash = { + create: jest.fn().mockResolvedValue(row), + findUnique: jest.fn().mockResolvedValue(row), + findUniqueOrThrow: jest.fn().mockResolvedValue(row), + findMany: jest.fn().mockResolvedValue([row]), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }; + repo = new PrismaPendingSlashesRepository({ pendingSlash } as unknown as PrismaService); + }); + + it("maps a unique violation (P2002) to created=false and rethrows anything else", async () => { + expect(await repo.createIfAbsent(input())).toMatchObject({ created: true, slash: { id: "s1", lockedUntil: undefined } }); + pendingSlash.create.mockRejectedValueOnce(Object.assign(new Error("dup"), { code: "P2002" })); + expect(await repo.createIfAbsent(input())).toMatchObject({ created: false }); + pendingSlash.create.mockRejectedValueOnce(Object.assign(new Error("down"), { code: "P1001" })); + await expect(repo.createIfAbsent(input())).rejects.toThrow("down"); + }); + + it("reads by intent and lists with filters", async () => { + expect(await repo.findByIntent("i1")).toMatchObject({ intentId: "i1" }); + pendingSlash.findUnique.mockResolvedValueOnce(null); + expect(await repo.findByIntent("x")).toBeUndefined(); + await repo.list({ state: "submitted", limit: 5 }); + expect(pendingSlash.findMany).toHaveBeenLastCalledWith({ where: { state: "submitted" }, orderBy: { detectedAt: "desc" }, take: 5 }); + await repo.list({ limit: 5 }); + expect(pendingSlash.findMany).toHaveBeenLastCalledWith(expect.objectContaining({ where: undefined })); + }); + + it("queries due, unleased rows", async () => { + await repo.findDue(t(1), 3); + const args = pendingSlash.findMany.mock.calls[0][0]; + expect(JSON.stringify(args.where)).toContain("challenge_window"); + expect(args.take).toBe(3); + }); + + it("claims and transitions with conditional updates", async () => { + expect(await repo.claim("i1", t(0), t(100))).toBe(true); + expect(pendingSlash.updateMany).toHaveBeenLastCalledWith({ + where: { intentId: "i1", OR: [{ lockedUntil: null }, { lockedUntil: { lt: t(0) } }] }, + data: { lockedUntil: t(100) }, + }); + + await repo.transition("i1", ["detected"], { state: "challenge_window", txHash: undefined }, t(5)); + expect(pendingSlash.updateMany).toHaveBeenLastCalledWith({ + where: { intentId: "i1", state: { in: ["detected"] }, OR: [{ lockedUntil: null }, { lockedUntil: { lt: t(5) } }] }, + data: { state: "challenge_window", lockedUntil: null }, + }); + + pendingSlash.updateMany.mockResolvedValueOnce({ count: 0 }); + expect(await repo.transition("i1", ["detected"], {})).toBeNull(); + pendingSlash.findUnique.mockResolvedValueOnce(null); + expect(await repo.transition("i1", ["detected"], {})).toBeNull(); + }); +}); diff --git a/src/solvers/pending-slashes.repository.ts b/src/solvers/pending-slashes.repository.ts new file mode 100644 index 00000000..f7584214 --- /dev/null +++ b/src/solvers/pending-slashes.repository.ts @@ -0,0 +1,167 @@ +import { Injectable } from "@nestjs/common"; +import { v4 as uuidv4 } from "uuid"; + +/** Injection token for {@link IPendingSlashesRepository} (issue #397). */ +export const PENDING_SLASHES_REPOSITORY = Symbol("PENDING_SLASHES_REPOSITORY"); + +/** + * Slash saga states: + * + * detected ──▶ challenge_window ──(window over, re-verified)──▶ submitted ──▶ confirmed + * │ │ │ + * └───────────────┴──(fill proof / admin / give-up)──▶ cancelled ◀┘ (tx failed past retries) + */ +export const PENDING_SLASH_STATES = [ + "detected", + "challenge_window", + "submitted", + "confirmed", + "cancelled", +] as const; +export type PendingSlashState = (typeof PENDING_SLASH_STATES)[number]; + +export interface PendingSlash { + id: string; + intentId: string; + solverAddress: string; + reason: string; + state: PendingSlashState; + /** The fill deadline (unix seconds) the solver missed. */ + fillDeadline: number; + detectedAt: Date; + challengeEndsAt: Date; + attempts: number; + nextAttemptAt: Date; + lockedUntil?: Date; + txHash?: string; + /** Registry client simulated but did not broadcast (dry-run / gated submit). */ + simulated: boolean; + submittedAt?: Date; + confirmedAt?: Date; + cancelledAt?: Date; + cancelReason?: string; + cancelledBy?: string; + fillTxHash?: string; + lastError?: string; + createdAt: Date; + updatedAt: Date; +} + +export interface NewPendingSlash { + intentId: string; + solverAddress: string; + reason: string; + fillDeadline: number; + detectedAt: Date; + challengeEndsAt: Date; +} + +export type PendingSlashPatch = Partial< + Omit +>; + +export interface IPendingSlashesRepository { + /** + * Inserts a `detected` row unless one already exists for the intent — the + * unique constraint on intent_id is what makes slashing exactly-once. + */ + createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }>; + + findByIntent(intentId: string): Promise; + + list(filter: { state?: PendingSlashState; limit: number }): Promise; + + /** + * Rows the pipeline should act on now and that are not leased: + * `detected`; `challenge_window` past both challengeEndsAt and nextAttemptAt; + * non-simulated `submitted` past nextAttemptAt. + */ + findDue(now: Date, limit: number): Promise; + + /** Takes the processing lease if it is free or expired. */ + claim(intentId: string, now: Date, leaseUntil: Date): Promise; + + /** + * Conditional update: applies `patch` only if the row is in one of `from`, + * and — when `now` is given — only if no *other* holder's lease is active. + * Always clears the lease. Returns the updated row, or null if the guard failed. + */ + transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise; +} + +/** In-memory adapter (dev/test). Each method is atomic — none of them await. */ +@Injectable() +export class InMemoryPendingSlashesRepository implements IPendingSlashesRepository { + private readonly rows = new Map(); + + async createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }> { + const existing = this.rows.get(input.intentId); + if (existing) return { slash: { ...existing }, created: false }; + const now = new Date(); + const row: PendingSlash = { + ...input, + id: uuidv4(), + state: "detected", + attempts: 0, + nextAttemptAt: now, + simulated: false, + createdAt: now, + updatedAt: now, + }; + this.rows.set(row.intentId, row); + return { slash: { ...row }, created: true }; + } + + async findByIntent(intentId: string): Promise { + const row = this.rows.get(intentId); + return row ? { ...row } : undefined; + } + + async list(filter: { state?: PendingSlashState; limit: number }): Promise { + return [...this.rows.values()] + .filter((r) => !filter.state || r.state === filter.state) + .sort((a, b) => b.detectedAt.getTime() - a.detectedAt.getTime()) + .slice(0, filter.limit) + .map((r) => ({ ...r })); + } + + async findDue(now: Date, limit: number): Promise { + return [...this.rows.values()] + .filter((r) => !r.lockedUntil || r.lockedUntil < now) + .filter( + (r) => + r.state === "detected" || + (r.state === "challenge_window" && r.challengeEndsAt <= now && r.nextAttemptAt <= now) || + (r.state === "submitted" && !r.simulated && r.nextAttemptAt <= now), + ) + .sort((a, b) => a.detectedAt.getTime() - b.detectedAt.getTime()) + .slice(0, limit) + .map((r) => ({ ...r })); + } + + async claim(intentId: string, now: Date, leaseUntil: Date): Promise { + const row = this.rows.get(intentId); + if (!row || (row.lockedUntil && row.lockedUntil >= now)) return false; + row.lockedUntil = leaseUntil; + row.updatedAt = now; + return true; + } + + async transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise { + const row = this.rows.get(intentId); + if (!row || !from.includes(row.state)) return null; + if (now && row.lockedUntil && row.lockedUntil >= now) return null; + Object.assign(row, patch, { lockedUntil: undefined, updatedAt: new Date() }); + return { ...row }; + } +} diff --git a/src/solvers/prisma-pending-slashes.repository.ts b/src/solvers/prisma-pending-slashes.repository.ts new file mode 100644 index 00000000..a618c8f7 --- /dev/null +++ b/src/solvers/prisma-pending-slashes.repository.ts @@ -0,0 +1,127 @@ +import { Prisma, PendingSlash as PendingSlashRow } from "@prisma/client"; +import { PrismaService } from "../prisma/prisma.service"; +import { + IPendingSlashesRepository, + NewPendingSlash, + PendingSlash, + PendingSlashPatch, + PendingSlashState, +} from "./pending-slashes.repository"; + +/** + * Prisma adapter for the slashing saga (issue #397). Every state change is a + * single conditional `updateMany`, so concurrent pipeline workers and admin + * requests are arbitrated by the database. + */ +export class PrismaPendingSlashesRepository implements IPendingSlashesRepository { + constructor(private readonly prisma: PrismaService) {} + + async createIfAbsent(input: NewPendingSlash): Promise<{ slash: PendingSlash; created: boolean }> { + try { + const row = await this.prisma.pendingSlash.create({ data: { ...input, state: "detected" } }); + return { slash: fromRow(row), created: true }; + } catch (err) { + // P2002 = unique constraint violation on intent_id: already detected. + if ((err as Prisma.PrismaClientKnownRequestError).code !== "P2002") throw err; + const existing = await this.prisma.pendingSlash.findUniqueOrThrow({ + where: { intentId: input.intentId }, + }); + return { slash: fromRow(existing), created: false }; + } + } + + async findByIntent(intentId: string): Promise { + const row = await this.prisma.pendingSlash.findUnique({ where: { intentId } }); + return row ? fromRow(row) : undefined; + } + + async list(filter: { state?: PendingSlashState; limit: number }): Promise { + const rows = await this.prisma.pendingSlash.findMany({ + where: filter.state ? { state: filter.state } : undefined, + orderBy: { detectedAt: "desc" }, + take: filter.limit, + }); + return rows.map(fromRow); + } + + async findDue(now: Date, limit: number): Promise { + const rows = await this.prisma.pendingSlash.findMany({ + where: { + AND: [ + { OR: [{ lockedUntil: null }, { lockedUntil: { lt: now } }] }, + { + OR: [ + { state: "detected" }, + { state: "challenge_window", challengeEndsAt: { lte: now }, nextAttemptAt: { lte: now } }, + { state: "submitted", simulated: false, nextAttemptAt: { lte: now } }, + ], + }, + ], + }, + orderBy: { detectedAt: "asc" }, + take: limit, + }); + return rows.map(fromRow); + } + + async claim(intentId: string, now: Date, leaseUntil: Date): Promise { + const result = await this.prisma.pendingSlash.updateMany({ + where: { intentId, OR: [{ lockedUntil: null }, { lockedUntil: { lt: now } }] }, + data: { lockedUntil: leaseUntil }, + }); + return result.count > 0; + } + + async transition( + intentId: string, + from: PendingSlashState[], + patch: PendingSlashPatch, + now?: Date, + ): Promise { + const where: Prisma.PendingSlashWhereInput = { intentId, state: { in: from } }; + if (now) where.OR = [{ lockedUntil: null }, { lockedUntil: { lt: now } }]; + const result = await this.prisma.pendingSlash.updateMany({ + where, + data: { ...toData(patch), lockedUntil: null }, + }); + if (result.count === 0) return null; + const row = await this.prisma.pendingSlash.findUnique({ where: { intentId } }); + return row ? fromRow(row) : null; + } +} + +function toData(patch: PendingSlashPatch): Prisma.PendingSlashUpdateManyMutationInput { + const data: Prisma.PendingSlashUpdateManyMutationInput = {}; + for (const [key, value] of Object.entries(patch)) { + // undefined means "leave as is"; clearing a column is not needed by the saga. + if (value !== undefined) (data as Record)[key] = value; + } + return data; +} + +function fromRow(row: PendingSlashRow): PendingSlash { + return { + id: row.id, + intentId: row.intentId, + solverAddress: row.solverAddress, + reason: row.reason, + state: row.state as PendingSlashState, + fillDeadline: row.fillDeadline, + detectedAt: row.detectedAt, + challengeEndsAt: row.challengeEndsAt, + attempts: row.attempts, + nextAttemptAt: row.nextAttemptAt, + lockedUntil: row.lockedUntil ?? undefined, + txHash: row.txHash ?? undefined, + simulated: row.simulated, + submittedAt: row.submittedAt ?? undefined, + confirmedAt: row.confirmedAt ?? undefined, + cancelledAt: row.cancelledAt ?? undefined, + cancelReason: row.cancelReason ?? undefined, + cancelledBy: row.cancelledBy ?? undefined, + fillTxHash: row.fillTxHash ?? undefined, + lastError: row.lastError ?? undefined, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} diff --git a/src/solvers/solvers.service.spec.ts b/src/solvers/solvers.service.spec.ts index 9841c5ea..99383a34 100644 --- a/src/solvers/solvers.service.spec.ts +++ b/src/solvers/solvers.service.spec.ts @@ -98,4 +98,29 @@ describe("SolversService", () => { expect(solver?.isActive).toBe(false); expect((await service.get(ALPHA_ADDR))?.isActive).toBe(false); }); + + describe("rollbackPenalty durable fallback (#397)", () => { + it("reverts fillsFailed from the solver address when the in-memory penalty is gone", async () => { + await service.recordFailedFill(ALPHA_ADDR, "intent-x"); + const bumped = (await service.get(ALPHA_ADDR))!.fillsFailed; + (service as unknown as { pendingPenalties: Map }).pendingPenalties.clear(); + + const result = await service.rollbackPenalty("intent-x", ALPHA_ADDR); + expect(result?.fillsFailed).toBe(bumped - 1); + }); + + it("returns null for an unknown solver and keeps the legacy no-record behaviour without an address", async () => { + expect(await service.rollbackPenalty("intent-y", "GUNKNOWN")).toBeNull(); + expect(await service.rollbackPenalty("intent-y")).toBeNull(); + }); + + it("uses the in-memory record when present", async () => { + await service.recordFailedFill(ALPHA_ADDR, "intent-z"); + const bumped = (await service.get(ALPHA_ADDR))!.fillsFailed; + await service.rollbackPenalty("intent-z", ALPHA_ADDR); + expect((await service.get(ALPHA_ADDR))!.fillsFailed).toBe(bumped - 1); + // Second rollback is a no-op: the record is now "failed", not missing. + expect(await service.rollbackPenalty("intent-z", ALPHA_ADDR)).toBeNull(); + }); + }); }); diff --git a/src/solvers/solvers.service.ts b/src/solvers/solvers.service.ts index fc0a4db9..a940b1cc 100644 --- a/src/solvers/solvers.service.ts +++ b/src/solvers/solvers.service.ts @@ -356,9 +356,22 @@ export class SolversService { * investigate the discrepancy. * * @param intentId The intent whose slash submission failed. + * @param solverAddress Optional fallback for callers that track the + * penalty durably (the slashing saga, issue #397): when the in-memory + * pending entry is gone — e.g. lost in a restart — the fillsFailed + * increment is still reverted for this solver. Callers passing it must + * guarantee they compensate at most once per intent. */ - async rollbackPenalty(intentId: string): Promise { + async rollbackPenalty(intentId: string, solverAddress?: string): Promise { const penalty = this.pendingPenalties.get(intentId); + if (!penalty && solverAddress) { + const solver = await this.repo.findByAddress(solverAddress); + if (!solver) return null; + this.logger.warn( + `[penalty] rolled back without in-memory record: solver=${solverAddress} intent=${intentId}`, + ); + return this.repo.save({ ...solver, fillsFailed: Math.max(0, solver.fillsFailed - 1) }); + } if (!penalty || penalty.state !== "pending") { this.logger.warn( `rollbackPenalty called for intentId=${intentId} but no pending penalty found (state=${penalty?.state ?? "none"})`, diff --git a/src/soroban/fill-verifier.service.spec.ts b/src/soroban/fill-verifier.service.spec.ts new file mode 100644 index 00000000..a5d8fd67 --- /dev/null +++ b/src/soroban/fill-verifier.service.spec.ts @@ -0,0 +1,136 @@ +import { ConfigService } from "@nestjs/config"; +import { SorobanRpc, StrKey, nativeToScVal, xdr } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { FillVerifierService } from "./fill-verifier.service"; +import { SorobanService } from "./soroban.service"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; +const OTHER_CONTRACT = StrKey.encodeContract(Buffer.alloc(32, 7)); +const DEADLINE = 1_900_000_000; +const BOUND = DEADLINE + 30; + +const topic = (name: string, intentId: string) => [ + nativeToScVal(name, { type: "symbol" }), + nativeToScVal(intentId, { type: "string" }), +]; +const event = (name: string, intentId: string, closedAt: number, txHash = "fill") => ({ + topic: topic(name, intentId), + ledgerClosedAt: new Date(closedAt * 1000).toISOString(), + txHash, + ledger: 42, + pagingToken: `pt-${closedAt}`, +}); + +function metaWith(events: Array<{ contract?: string; topics: xdr.ScVal[] }>) { + const contractEvents = events.map((e) => ({ + contractId: () => (e.contract ? StrKey.decodeContract(e.contract) : null), + body: () => ({ v0: () => ({ topics: () => e.topics }) }), + })); + return { v3: () => ({ sorobanMeta: () => ({ events: () => contractEvents }) }) } as unknown as xdr.TransactionMeta; +} + +describe("FillVerifierService (#397)", () => { + let soroban: { getLatestLedger: jest.Mock; getEvents: jest.Mock; getTransaction: jest.Mock }; + const build = (contractId = CONTRACT_ID) => + new FillVerifierService( + soroban as unknown as SorobanService, + { get: () => contractId } as unknown as ConfigService, + ); + + beforeEach(() => { + soroban = { + getLatestLedger: jest.fn().mockResolvedValue({ sequence: 100_000 }), + getEvents: jest.fn().mockResolvedValue({ events: [], latestLedger: 100_000 }), + getTransaction: jest.fn(), + }; + }); + + describe("findLandedFill", () => { + it("returns null without a settlement contract (nothing to verify against)", async () => { + expect(await build("").findLandedFill("i1", DEADLINE, BOUND)).toBeNull(); + expect(soroban.getEvents).not.toHaveBeenCalled(); + }); + + it("looks back far enough to cover the fill window and finds an in-time fill", async () => { + soroban.getEvents.mockResolvedValueOnce({ + events: [event("intent_filled", "other", BOUND), event("intent_accepted", "i1", BOUND), event("intent_filled", "i1", BOUND)], + }); + const now = DEADLINE + 600; + expect(await build().findLandedFill("i1", DEADLINE, BOUND, now)).toEqual({ txHash: "fill", ledger: 42, closedAt: BOUND }); + expect(soroban.getEvents.mock.calls[0][0]).toMatchObject({ + startLedger: 100_000 - Math.ceil((600 + 1800) / 5), + filters: [{ type: "contract", contractIds: [CONTRACT_ID] }], + }); + }); + + it("ignores fills that closed after deadline + tolerance (chain time)", async () => { + soroban.getEvents.mockResolvedValueOnce({ events: [event("intent_filled", "i1", BOUND + 1)] }); + expect(await build().findLandedFill("i1", DEADLINE, BOUND)).toBeNull(); + }); + + it("paginates with the last paging token", async () => { + const page = Array.from({ length: 200 }, (_, i) => event("noise", "x", DEADLINE + i)); + soroban.getEvents + .mockResolvedValueOnce({ events: page }) + .mockResolvedValueOnce({ events: [event("intent_filled", "i1", DEADLINE)] }); + expect(await build().findLandedFill("i1", DEADLINE, BOUND)).not.toBeNull(); + expect(soroban.getEvents.mock.calls[1][0]).toMatchObject({ cursor: `pt-${DEADLINE + 199}` }); + expect(soroban.getEvents.mock.calls[1][0].startLedger).toBeUndefined(); + }); + + it("stops after the page cap and tolerates undecodable topics", async () => { + const junk = { ...event("x", "y", DEADLINE), topic: [{} as xdr.ScVal] }; + soroban.getEvents.mockResolvedValue({ events: Array.from({ length: 200 }, () => junk) }); + expect(await build().findLandedFill("i1", DEADLINE, BOUND)).toBeNull(); + expect(soroban.getEvents).toHaveBeenCalledTimes(10); + }); + + it("propagates RPC errors so the saga retries instead of slashing", async () => { + soroban.getLatestLedger.mockRejectedValueOnce(new Error("RPC down")); + await expect(build().findLandedFill("i1", DEADLINE, BOUND)).rejects.toThrow("RPC down"); + }); + }); + + describe("verifyFillProof", () => { + const success = (meta: xdr.TransactionMeta, createdAt = BOUND) => ({ + status: SorobanRpc.Api.GetTransactionStatus.SUCCESS, ledger: 5, createdAt, resultMetaXdr: meta, + }); + + it("accepts a successful, timely tx that emitted intent_filled from the settlement contract", async () => { + soroban.getTransaction.mockResolvedValueOnce(success(metaWith([{ contract: CONTRACT_ID, topics: topic("intent_filled", "i1") }]))); + expect(await build().verifyFillProof("h", "i1", BOUND)).toEqual({ + valid: true, fill: { txHash: "h", ledger: 5, closedAt: BOUND }, + }); + }); + + it.each([ + ["not successful", { status: SorobanRpc.Api.GetTransactionStatus.NOT_FOUND }, /status is NOT_FOUND/], + ["too late", success(metaWith([{ contract: CONTRACT_ID, topics: topic("intent_filled", "i1") }]), BOUND + 1), /after the acceptable bound/], + ["wrong contract", success(metaWith([{ contract: OTHER_CONTRACT, topics: topic("intent_filled", "i1") }])), /did not emit/], + ["no contract id", success(metaWith([{ topics: topic("intent_filled", "i1") }])), /did not emit/], + ["other intent", success(metaWith([{ contract: CONTRACT_ID, topics: topic("intent_filled", "i2") }])), /did not emit/], + ["unreadable meta", success({ v3: () => { throw new Error("v2 meta"); } } as unknown as xdr.TransactionMeta), /did not emit/], + ])("rejects a proof that is %s", async (_label, tx, reason) => { + soroban.getTransaction.mockResolvedValueOnce(tx); + const result = await build().verifyFillProof("h", "i1", BOUND); + expect(result.valid).toBe(false); + expect((result as { reason: string }).reason).toMatch(reason); + }); + + it("skips the contract check when no settlement contract is configured, and survives malformed bodies", async () => { + const meta = metaWith([{ topics: topic("intent_filled", "i1") }]); + soroban.getTransaction.mockResolvedValueOnce(success(meta)); + expect((await build("").verifyFillProof("h", "i1", BOUND)).valid).toBe(true); + + const broken = { + v3: () => ({ sorobanMeta: () => ({ events: () => [{ contractId: () => null, body: () => { throw new Error("x"); } }] }) }), + } as unknown as xdr.TransactionMeta; + soroban.getTransaction.mockResolvedValueOnce(success(broken)); + expect((await build("").verifyFillProof("h", "i1", BOUND)).valid).toBe(false); + + const noSorobanMeta = { v3: () => ({ sorobanMeta: () => null }) } as unknown as xdr.TransactionMeta; + soroban.getTransaction.mockResolvedValueOnce(success(noSorobanMeta)); + expect((await build("").verifyFillProof("h", "i1", BOUND)).valid).toBe(false); + }); + }); +}); diff --git a/src/soroban/fill-verifier.service.ts b/src/soroban/fill-verifier.service.ts new file mode 100644 index 00000000..84751130 --- /dev/null +++ b/src/soroban/fill-verifier.service.ts @@ -0,0 +1,151 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { SorobanRpc, StrKey, scValToNative, xdr } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { SorobanService } from "./soroban.service"; + +/** Approximate Stellar ledger close interval, used to size the event lookback. */ +const LEDGER_SECONDS = 5; +/** Longest per-chain fill window (CHAIN_FILL_WINDOW_DEFAULTS.ethereum) — a fill can't predate acceptance by more. */ +const MAX_FILL_WINDOW_SECONDS = 1800; +const EVENTS_PAGE_LIMIT = 200; +const MAX_EVENT_PAGES = 10; + +export interface LandedFill { + txHash: string; + ledger: number; + /** Ledger close time, unix seconds — chain time, not server time. */ + closedAt: number; +} + +/** + * Re-verification for the slashing saga (issue #397): answers "did a fill for + * this intent land on-chain in time?" using chain data only. + * + * Timeliness is judged on the ledger close time (chain clock) against + * `latestAcceptable = fillDeadline + SLASH_CLOCK_SKEW_TOLERANCE_SECONDS`, so + * skew between the sweeper host and the network never slashes a solver whose + * fill closed a few seconds "late" by server time. + * + * Methods throw on RPC failure: the caller must treat "couldn't check" as + * "don't slash yet", never as "no fill". + */ +@Injectable() +export class FillVerifierService { + private readonly logger = new Logger(FillVerifierService.name); + private readonly settlementContractId: string; + + constructor( + private readonly sorobanService: SorobanService, + configService: ConfigService, + ) { + this.settlementContractId = configService.get("stellar.settlementContractId", { infer: true }); + } + + /** + * Scans the settlement contract's recent events for an `intent_filled` + * event for `intentId` that closed by `latestAcceptable`. + * + * Returns null when none is found — including when SETTLEMENT_CONTRACT_ID is + * unset, since there is then no on-chain fill path to verify against. + */ + async findLandedFill( + intentId: string, + fillDeadline: number, + latestAcceptable: number, + now: number = Math.floor(Date.now() / 1000), + ): Promise { + if (!this.settlementContractId) return null; + + const latest = await this.sorobanService.getLatestLedger(); + const lookbackSeconds = Math.max(0, now - fillDeadline) + MAX_FILL_WINDOW_SECONDS; + const startLedger = Math.max(1, latest.sequence - Math.ceil(lookbackSeconds / LEDGER_SECONDS)); + + let cursor: string | undefined; + for (let page = 0; page < MAX_EVENT_PAGES; page++) { + const response = await this.sorobanService.getEvents({ + ...(cursor ? { cursor } : { startLedger }), + filters: [{ type: "contract", contractIds: [this.settlementContractId] }], + limit: EVENTS_PAGE_LIMIT, + }); + + for (const event of response.events) { + if (!isIntentFilled(event.topic, intentId)) continue; + const closedAt = Math.floor(Date.parse(event.ledgerClosedAt) / 1000); + if (closedAt <= latestAcceptable) { + return { txHash: event.txHash, ledger: event.ledger, closedAt }; + } + this.logger.log( + `[fill-verifier] intent=${intentId} fill ${event.txHash} closed at ${closedAt}, ` + + `after the acceptable bound ${latestAcceptable} — does not cancel the slash`, + ); + } + + if (response.events.length < EVENTS_PAGE_LIMIT) break; + cursor = response.events[response.events.length - 1].pagingToken; + } + return null; + } + + /** + * Verifies a solver-supplied fill proof: `txHash` must be a successful + * transaction, closed by `latestAcceptable`, that emitted `intent_filled` + * for `intentId` (from the settlement contract, when configured). + */ + async verifyFillProof( + txHash: string, + intentId: string, + latestAcceptable: number, + ): Promise<{ valid: true; fill: LandedFill } | { valid: false; reason: string }> { + const tx = await this.sorobanService.getTransaction(txHash); + if (tx.status !== SorobanRpc.Api.GetTransactionStatus.SUCCESS) { + return { valid: false, reason: `transaction status is ${tx.status}` }; + } + if (tx.createdAt > latestAcceptable) { + return { + valid: false, + reason: `fill closed at ${tx.createdAt}, after the acceptable bound ${latestAcceptable}`, + }; + } + if (!this.emitsIntentFilled(tx.resultMetaXdr, intentId)) { + return { valid: false, reason: "transaction did not emit intent_filled for this intent" }; + } + return { valid: true, fill: { txHash, ledger: tx.ledger, closedAt: tx.createdAt } }; + } + + private emitsIntentFilled(meta: xdr.TransactionMeta, intentId: string): boolean { + let events: xdr.ContractEvent[] = []; + try { + events = meta.v3().sorobanMeta()?.events() ?? []; + } catch { + return false; + } + return events.some((event) => { + if (this.settlementContractId) { + const contractId = event.contractId(); + if (!contractId || !this.matchesSettlementContract(contractId)) return false; + } + try { + return isIntentFilled(event.body().v0().topics(), intentId); + } catch { + return false; + } + }); + } + + private matchesSettlementContract(contractId: Buffer): boolean { + return StrKey.encodeContract(contractId) === this.settlementContractId; + } +} + +/** Topic layout shared with EventIngestionService: [event name, intentId, ...]. */ +function isIntentFilled(topic: xdr.ScVal[], intentId: string): boolean { + const decoded = topic.slice(0, 2).map((scVal) => { + try { + return scValToNative(scVal); + } catch { + return undefined; + } + }); + return decoded[0] === "intent_filled" && decoded[1] === intentId; +} diff --git a/src/soroban/outbox-admin.controller.spec.ts b/src/soroban/outbox-admin.controller.spec.ts new file mode 100644 index 00000000..26f32f85 --- /dev/null +++ b/src/soroban/outbox-admin.controller.spec.ts @@ -0,0 +1,34 @@ +import { NotFoundException } from "@nestjs/common"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { InMemoryOutboxRepository } from "./outbox.repository"; +import { OutboxAdminController } from "./outbox-admin.controller"; + +describe("OutboxAdminController (#396)", () => { + const principal = { id: "ops-1", role: "admin" as const }; + + it("audits and requeues dead rows, 404s everything else", async () => { + const outbox = new InMemoryOutboxRepository(); + const audit = { record: jest.fn().mockResolvedValue(undefined) }; + const controller = new OutboxAdminController(outbox, audit as unknown as AdminAuditService); + const row = await outbox.enqueue({ intentId: "i1", operation: "create_intent", payload: {} }); + const [claimed] = await outbox.claimDue(new Date(Date.now() + 1000), 1, new Date(Date.now() + 60_000)); + await outbox.markDead(claimed, "poison"); + + await expect(controller.requeue(row.id, principal)).resolves.toEqual({ id: row.id, status: "pending" }); + expect(audit.record).toHaveBeenCalledWith({ actor: "ops-1", action: "outbox.requeue", target: `outbox:${row.id}` }); + + await expect(controller.requeue(row.id, principal)).rejects.toBeInstanceOf(NotFoundException); + audit.record.mockClear(); + await expect(controller.requeue("not-a-number", principal)).rejects.toBeInstanceOf(NotFoundException); + expect(audit.record).not.toHaveBeenCalled(); + }); + + it("does not requeue when the audit write fails", async () => { + const outbox = new InMemoryOutboxRepository(); + const requeue = jest.spyOn(outbox, "requeueDead"); + const audit = { record: jest.fn().mockRejectedValue(new Error("audit down")) }; + const controller = new OutboxAdminController(outbox, audit as unknown as AdminAuditService); + await expect(controller.requeue("1", principal)).rejects.toThrow("audit down"); + expect(requeue).not.toHaveBeenCalled(); + }); +}); diff --git a/src/soroban/outbox-admin.controller.ts b/src/soroban/outbox-admin.controller.ts new file mode 100644 index 00000000..a3da1a62 --- /dev/null +++ b/src/soroban/outbox-admin.controller.ts @@ -0,0 +1,35 @@ +import { Controller, Inject, NotFoundException, Param, Post, UseGuards } from "@nestjs/common"; +import { ApiHeader, ApiNotFoundResponse, ApiTags, ApiUnauthorizedResponse } from "@nestjs/swagger"; +import { AdminGuard, CurrentAdmin, RequireAdminRole } from "../admin/admin.guard"; +import { AdminPrincipal } from "../admin/admin-auth"; +import { AdminAuditService } from "../admin/admin-audit.service"; +import { IOutboxRepository, OUTBOX_REPOSITORY } from "./outbox.repository"; + +/** + * Operator actions on the transactional outbox (issue #396). Registered in + * IntentsModule, which owns the OUTBOX_REPOSITORY binding. + */ +@ApiTags("admin") +@ApiHeader({ name: "x-admin-key", required: true }) +@Controller("api/v1/admin/outbox") +@UseGuards(AdminGuard) +@RequireAdminRole("admin") +export class OutboxAdminController { + constructor( + @Inject(OUTBOX_REPOSITORY) private readonly outbox: IOutboxRepository, + private readonly audit: AdminAuditService, + ) {} + + /** Moves a `dead` row back to `pending` with attempts reset, unblocking its intent. */ + @Post(":id/requeue") + @ApiUnauthorizedResponse({ description: "Missing or invalid admin key" }) + @ApiNotFoundResponse({ description: "No dead outbox row with this id" }) + async requeue(@Param("id") id: string, @CurrentAdmin() admin: AdminPrincipal) { + if (!/^\d+$/.test(id)) throw new NotFoundException(`No dead outbox row with id ${id}`); + await this.audit.record({ actor: admin.id, action: "outbox.requeue", target: `outbox:${id}` }); + if (!(await this.outbox.requeueDead(id))) { + throw new NotFoundException(`No dead outbox row with id ${id}`); + } + return { id, status: "pending" }; + } +} diff --git a/src/soroban/outbox-operations.spec.ts b/src/soroban/outbox-operations.spec.ts new file mode 100644 index 00000000..53b2d847 --- /dev/null +++ b/src/soroban/outbox-operations.spec.ts @@ -0,0 +1,65 @@ +import { Keypair, scValToNative } from "@stellar/stellar-sdk"; +import { Intent } from "../intents/intents.types"; +import { + acceptIntentEntry, + buildOutboxInvocation, + cancelIntentEntry, + createIntentEntry, + fillIntentEntry, +} from "./outbox-operations"; +import { OutboxOperation } from "./outbox.repository"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; +const USER = Keypair.random().publicKey(); +const SOLVER = Keypair.random().publicKey(); + +const intent: Intent = { + intentId: "11111111-2222-3333-4444-555555555555", + user: USER, + srcChain: "ethereum", + srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" }, + srcAmount: "1000000", + dstToken: { contract: CONTRACT_ID, symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + state: "filled", + createdAt: 1, + deadline: 1_900_000_000, + solver: SOLVER, + fillAmount: "995000", + txHash: "ab".repeat(32), +}; + +const decode = (entry: { operation: OutboxOperation; payload: Record }) => + buildOutboxInvocation(entry, CONTRACT_ID).args.map((a) => scValToNative(a)); + +describe("outbox operations (#396)", () => { + it("encodes create_intent with bigint amounts and the deadline", () => { + const entry = createIntentEntry(intent); + expect(entry).toMatchObject({ intentId: intent.intentId, operation: "create_intent" }); + // Payload survives a JSON round-trip (JSONB column). + const roundTripped = { ...entry, payload: JSON.parse(JSON.stringify(entry.payload)) }; + const params = buildOutboxInvocation(roundTripped, CONTRACT_ID); + expect(params).toMatchObject({ contractId: CONTRACT_ID, method: "create_intent" }); + expect(decode(roundTripped)).toEqual([ + intent.intentId, USER, "ethereum", "0xabc", 1_000_000n, CONTRACT_ID, 990_000n, 1_900_000_000n, + ]); + }); + + it("encodes accept_intent, fill_intent and cancel_intent", () => { + expect(decode(acceptIntentEntry(intent))).toEqual([intent.intentId, SOLVER, 1_900_000_000n]); + expect(decode(fillIntentEntry(intent))).toEqual([intent.intentId, SOLVER, 995_000n, "ab".repeat(32)]); + expect(decode(fillIntentEntry({ ...intent, txHash: undefined }))[3]).toBe(""); + expect(decode(cancelIntentEntry(intent))).toEqual([intent.intentId, USER]); + }); + + it("rejects malformed payloads so they fail at enqueue time", () => { + expect(() => buildOutboxInvocation(acceptIntentEntry({ ...intent, solver: undefined }), CONTRACT_ID)).toThrow( + /missing a required address/, + ); + expect(() => buildOutboxInvocation(createIntentEntry({ ...intent, user: "nope" }), CONTRACT_ID)).toThrow(); + expect(() => buildOutboxInvocation(createIntentEntry({ ...intent, srcAmount: "1.5" }), CONTRACT_ID)).toThrow(); + expect(() => + buildOutboxInvocation({ operation: "bogus" as OutboxOperation, payload: {} }, CONTRACT_ID), + ).toThrow(/unknown outbox operation/); + }); +}); diff --git a/src/soroban/outbox-operations.ts b/src/soroban/outbox-operations.ts new file mode 100644 index 00000000..937f860d --- /dev/null +++ b/src/soroban/outbox-operations.ts @@ -0,0 +1,118 @@ +import { Address, nativeToScVal, xdr } from "@stellar/stellar-sdk"; +import type { Intent } from "../intents/intents.types"; +import type { NewOutboxEntry, OutboxOperation } from "./outbox.repository"; +import type { InvokeContractParams } from "./stellar-tx.service"; + +/** + * Payload builders and ScVal encoders for settlement-contract operations that + * flow through the outbox (issue #396). + * + * Payloads are plain JSON (bigint amounts as strings) so they survive the + * `payload JSONB` column; ScVal encoding happens at relay time. The contract + * method names follow docs/architecture/onchain-settlement.md and stay + * provisional until the settlement ADR (issue #19) fixes the interface. + */ + +export function createIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "create_intent", + payload: { + intentId: intent.intentId, + user: intent.user, + srcChain: intent.srcChain, + srcTokenAddress: intent.srcToken.address, + srcAmount: intent.srcAmount, + dstTokenContract: intent.dstToken.contract, + minDstAmount: intent.minDstAmount, + deadline: intent.deadline, + }, + }; +} + +export function acceptIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "accept_intent", + payload: { intentId: intent.intentId, solver: intent.solver, fillDeadline: intent.deadline }, + }; +} + +export function fillIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "fill_intent", + payload: { + intentId: intent.intentId, + solver: intent.solver, + fillAmount: intent.fillAmount, + fillTxHash: intent.txHash ?? "", + }, + }; +} + +export function cancelIntentEntry(intent: Intent): NewOutboxEntry { + return { + intentId: intent.intentId, + operation: "cancel_intent", + payload: { intentId: intent.intentId, user: intent.user }, + }; +} + +/** + * Encodes an outbox payload as a contract invocation. + * + * @throws on a malformed payload (bad address, non-integer amount). Callers + * run this at enqueue time too, so bad input fails the HTTP request + * instead of becoming a poison row. + */ +export function buildOutboxInvocation( + entry: { operation: OutboxOperation; payload: Record }, + settlementContractId: string, +): InvokeContractParams { + const p = entry.payload; + let args: xdr.ScVal[]; + switch (entry.operation) { + case "create_intent": + args = [ + str(p.intentId), + address(p.user), + nativeToScVal(String(p.srcChain), { type: "symbol" }), + str(p.srcTokenAddress), + i128(p.srcAmount), + address(p.dstTokenContract), + i128(p.minDstAmount), + nativeToScVal(Number(p.deadline), { type: "u64" }), + ]; + break; + case "accept_intent": + args = [str(p.intentId), address(p.solver), nativeToScVal(Number(p.fillDeadline), { type: "u64" })]; + break; + case "fill_intent": + args = [str(p.intentId), address(p.solver), i128(p.fillAmount), str(p.fillTxHash)]; + break; + case "cancel_intent": + args = [str(p.intentId), address(p.user)]; + break; + default: { + const unknown: never = entry.operation; + throw new Error(`unknown outbox operation: ${String(unknown)}`); + } + } + return { contractId: settlementContractId, method: entry.operation, args }; +} + +function str(value: unknown): xdr.ScVal { + return nativeToScVal(String(value ?? ""), { type: "string" }); +} + +function address(value: unknown): xdr.ScVal { + if (typeof value !== "string" || value.length === 0) { + throw new Error("outbox payload is missing a required address"); + } + return new Address(value).toScVal(); +} + +function i128(value: unknown): xdr.ScVal { + return nativeToScVal(BigInt(String(value)), { type: "i128" }); +} diff --git a/src/soroban/outbox-relay.service.spec.ts b/src/soroban/outbox-relay.service.spec.ts new file mode 100644 index 00000000..9f3f874d --- /dev/null +++ b/src/soroban/outbox-relay.service.spec.ts @@ -0,0 +1,475 @@ +import { ConfigService } from "@nestjs/config"; +import { Keypair } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { InMemoryIntentsRepository } from "../intents/intents.repository"; +import { InMemoryIntentsUnitOfWork } from "../intents/intents.unit-of-work"; +import { IntentsService } from "../intents/intents.service"; +import { PrismaService } from "../prisma/prisma.service"; +import { ProtocolParamsService } from "../governance/params.service"; +import { InMemoryOutboxRepository } from "./outbox.repository"; +import { createIntentEntry } from "./outbox-operations"; +import { OutboxRelayService } from "./outbox-relay.service"; +import { InvokeContractOptions, InvokeContractParams, StellarTxService } from "./stellar-tx.service"; +import { TxConfirmation, TxConfirmationService } from "./tx-confirmation.service"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; +const LEASE_SECONDS = 120; +const MAX_ATTEMPTS = 3; + +function intentFor(intentId: string) { + return { + intentId, + user: Keypair.random().publicKey(), + srcChain: "ethereum" as const, + srcToken: { address: "0xabc", symbol: "USDC", name: "USD Coin", decimals: 6, chain: "ethereum" as const }, + srcAmount: "1000000", + dstToken: { contract: CONTRACT_ID, symbol: "USDC", decimals: 7 }, + minDstAmount: "990000", + state: "open" as const, + createdAt: 1, + deadline: 2_000_000_000, + }; +} + +function config(overrides: Partial = {}, settlementContractId = CONTRACT_ID) { + const outbox: AppConfig["outbox"] = { + relayEnabled: true, + relayIntervalMs: 1000, + batchSize: 10, + maxAttempts: MAX_ATTEMPTS, + leaseSeconds: LEASE_SECONDS, + ...overrides, + }; + return { + get: (key: string) => + key === "outbox" ? outbox : key === "stellar.settlementContractId" ? settlementContractId : undefined, + } as unknown as ConfigService; +} + +describe("OutboxRelayService (#396)", () => { + let outbox: InMemoryOutboxRepository; + let invokeContract: jest.Mock, [InvokeContractParams, InvokeContractOptions?]>; + let check: jest.Mock, [string]>; + let metrics: { recordOutboxOutcome: jest.Mock; setOutboxBacklog: jest.Mock }; + let relay: OutboxRelayService; + let now: Date; + + /** Default fake network: sign → beforeSubmit(hash) → accepted as PENDING. */ + function submitsAs(hash: string) { + invokeContract.mockImplementationOnce(async (_params, options) => { + await options?.beforeSubmit?.(hash); + return { hash, status: "PENDING", dryRun: false }; + }); + } + + function build(cfg = config()) { + relay = new OutboxRelayService( + outbox, + { invokeContract } as unknown as StellarTxService, + { check } as unknown as TxConfirmationService, + metrics as unknown as MetricsService, + cfg, + ); + } + + const later = (seconds: number) => new Date(now.getTime() + seconds * 1000); + + beforeEach(() => { + outbox = new InMemoryOutboxRepository(); + invokeContract = jest.fn(); + check = jest.fn().mockResolvedValue({ status: "not_found" }); + metrics = { recordOutboxOutcome: jest.fn(), setOutboxBacklog: jest.fn() }; + now = new Date(Date.now() + 1000); + build(); + }); + + afterEach(() => relay.onModuleDestroy()); + + it("submits a pending row, records the envelope before broadcast, then confirms it", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + let hashAtSubmit: string | undefined; + invokeContract.mockImplementationOnce(async (params, options) => { + expect(params).toMatchObject({ contractId: CONTRACT_ID, method: "create_intent" }); + await options?.beforeSubmit?.("h1"); + hashAtSubmit = (await outbox.findByIntent("i1"))[0].envelopeHash; + return { hash: "h1", status: "PENDING", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ claimed: 1, submitted: 1 }); + expect(hashAtSubmit).toBe("h1"); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "submitted", txHash: "h1" }); + + check.mockResolvedValueOnce({ status: "success", ledger: 10 }); + expect(await relay.tick(later(5))).toMatchObject({ confirmed: 1 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("confirmed"); + expect(metrics.recordOutboxOutcome).toHaveBeenCalledWith("confirmed"); + expect(metrics.setOutboxBacklog).toHaveBeenCalled(); + }); + + it("confirms immediately when the live path already waited for confirmation (SUCCESS)", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-sync"); + return { hash: "h-sync", status: "SUCCESS", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ submitted: 1, confirmed: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "confirmed", txHash: "h-sync" }); + expect(check).not.toHaveBeenCalled(); + + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockResolvedValueOnce({ hash: "h2", status: "SUCCESS", dryRun: false }); + expect(await relay.tick(later(1))).toMatchObject({ confirmed: 0 }); + }); + + it("marks rows simulated under ONCHAIN_DRY_RUN and lets the next row for the intent proceed", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await outbox.enqueue({ intentId: "i1", operation: "cancel_intent", payload: { intentId: "i1", user: Keypair.random().publicKey() } }); + invokeContract.mockResolvedValue({ hash: "dry-run-no-hash", status: "DRY_RUN", dryRun: true }); + + expect(await relay.tick(now)).toMatchObject({ simulated: 1 }); + expect(await relay.tick(later(1))).toMatchObject({ simulated: 1 }); + expect((await outbox.findByIntent("i1")).map((r) => r.status)).toEqual(["simulated", "simulated"]); + }); + + describe("crash injection", () => { + it("crash between DB commit and submit: the committed row is picked up by the next relay", async () => { + // The unit of work commits intent + outbox row, then the process dies + // before any relay tick — nothing was submitted. + const repo = new InMemoryIntentsRepository(); + const intents = new IntentsService( + repo, + { get: (k: string) => (k === "onchainIntentsEnabled" ? true : k === "stellar.settlementContractId" ? CONTRACT_ID : undefined) } as unknown as ConfigService, + {} as StellarTxService, + { intentAuditLog: { create: jest.fn().mockResolvedValue({}) } } as unknown as PrismaService, + undefined, + undefined, + { snapshotForChain: jest.fn().mockReturnValue({ version: 0, deadlineSeconds: 1800, fillWindowSeconds: 600 }) } as unknown as ProtocolParamsService, + undefined, + new InMemoryIntentsUnitOfWork(repo, outbox), + ); + const { intentId: _ignored, state: _state, createdAt: _createdAt, ...data } = intentFor("ignored"); + const created = await intents.create(data); + expect(invokeContract).not.toHaveBeenCalled(); + + // "Restart": a fresh relay over the same durable outbox. + build(); + submitsAs("h-after-restart"); + expect(await relay.tick(now)).toMatchObject({ submitted: 1 }); + expect((await outbox.findByIntent(created.intentId))[0].txHash).toBe("h-after-restart"); + }); + + it("crash after broadcast, before markSubmitted: detects the landed tx and does not resubmit", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-landed"); + throw new Error("process killed"); + }); + // Simulate the kill: the row is left `processing` with the envelope hash + // (the catch path's retry write is lost along with the process). + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "processing", envelopeHash: "h-landed" }); + + // Before the lease expires nobody touches it. + expect(await relay.tick(later(LEASE_SECONDS - 1))).toMatchObject({ claimed: 0 }); + + check.mockResolvedValueOnce({ status: "success", ledger: 7 }); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ claimed: 1, confirmed: 1 }); + expect(invokeContract).toHaveBeenCalledTimes(1); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "confirmed", txHash: "h-landed" }); + }); + + it("crash after signing, envelope never landed: rebuilds and resubmits once the lease expires", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-lost"); + throw new Error("process killed"); + }); + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + + check.mockResolvedValueOnce({ status: "not_found" }); + submitsAs("h-rebuilt"); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ submitted: 1 }); + expect(check).toHaveBeenCalledWith("h-lost"); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "submitted", txHash: "h-rebuilt", attempts: 2 }); + }); + + it("a reclaimed row whose earlier envelope failed on-chain is retried", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h-failed"); + throw new Error("process killed"); + }); + const retrySpy = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + await relay.tick(now); + retrySpy.mockRestore(); + + check.mockResolvedValueOnce({ status: "failed", ledger: 9 }); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ + status: "pending", + lastError: expect.stringContaining("h-failed"), + }); + }); + }); + + it("a kill-switch pause puts the row back without consuming an attempt (never dead-letters)", async () => { + build(config({ maxAttempts: 1 })); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + const paused = new KillSwitchActiveException({ + reasonCode: "INCIDENT", + reason: "paused", + scope: "operation", + chain: "stellar", + token: null, + operation: "onchain", + } as never); + invokeContract.mockRejectedValue(paused); + + for (let i = 0; i < 5; i++) { + const t = i === 0 ? now : (await outbox.findByIntent("i1"))[0].nextAttemptAt; + expect(await relay.tick(t)).toMatchObject({ claimed: 1, paused: 1, dead: 0, retried: 0 }); + } + const [row] = await outbox.findByIntent("i1"); + expect(row).toMatchObject({ status: "pending", attempts: 0, lastError: expect.stringContaining("kill-switch") }); + + // Resume: the next attempt submits normally. + invokeContract.mockReset(); + submitsAs("h-after-resume"); + expect(await relay.tick(row.nextAttemptAt)).toMatchObject({ submitted: 1 }); + + // A lost fence on release is not counted. + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockRejectedValueOnce(paused); + jest.spyOn(outbox, "release").mockResolvedValueOnce(false); + expect(await relay.tick(later(3600))).toMatchObject({ paused: 0 }); + }); + + it("does not submit when the lease was lost before broadcast", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + jest.spyOn(outbox, "recordEnvelope").mockResolvedValueOnce(false); + const sent = jest.fn(); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h"); + sent(); + return { hash: "h", status: "PENDING", dryRun: false }; + }); + + await relay.tick(now); + expect(sent).not.toHaveBeenCalled(); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "pending", lastError: expect.stringContaining("lost lease") }); + }); + + it("keeps per-intent order across ticks while other intents proceed in parallel", async () => { + await outbox.enqueue(createIntentEntry(intentFor("a"))); + await outbox.enqueue({ intentId: "a", operation: "cancel_intent", payload: { intentId: "a", user: Keypair.random().publicKey() } }); + await outbox.enqueue(createIntentEntry(intentFor("b"))); + const order: string[] = []; + invokeContract.mockImplementation(async (params, options) => { + order.push(params.method); + const hash = `h${order.length}`; + await options?.beforeSubmit?.(hash); + return { hash, status: "PENDING", dryRun: false }; + }); + + expect(await relay.tick(now)).toMatchObject({ claimed: 2, submitted: 2 }); + expect(order).toEqual(["create_intent", "create_intent"]); + + // a's cancel must wait for a's create to confirm. + expect(await relay.tick(later(1))).toMatchObject({ claimed: 0 }); + + check.mockImplementation(async (hash) => ({ status: hash === "h1" ? "success" : "not_found" })); + expect(await relay.tick(later(2))).toMatchObject({ confirmed: 1, claimed: 1, submitted: 1 }); + expect(order).toEqual(["create_intent", "create_intent", "cancel_intent"]); + }); + + it("retries with exponential backoff and moves a poison row to dead with an alert", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await outbox.enqueue({ intentId: "i1", operation: "cancel_intent", payload: { intentId: "i1", user: Keypair.random().publicKey() } }); + invokeContract.mockRejectedValue(new Error("RPC 503")); + const errorLog = jest.spyOn((relay as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + + expect(await relay.tick(now)).toMatchObject({ retried: 1 }); + let [row] = await outbox.findByIntent("i1"); + expect(row.nextAttemptAt.getTime() - now.getTime()).toBe(1000); + + expect(await relay.tick(new Date(row.nextAttemptAt.getTime()))).toMatchObject({ retried: 1 }); + const t2 = row.nextAttemptAt.getTime(); + [row] = await outbox.findByIntent("i1"); + expect(row.nextAttemptAt.getTime() - t2).toBe(2000); + + expect(await relay.tick(new Date(row.nextAttemptAt.getTime()))).toMatchObject({ dead: 1 }); + [row] = await outbox.findByIntent("i1"); + expect(row).toMatchObject({ status: "dead", attempts: MAX_ATTEMPTS, lastError: "RPC 503" }); + expect(metrics.recordOutboxOutcome).toHaveBeenCalledWith("dead"); + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("ALERT")); + + // The dead row blocks the intent's later operations. + expect(await relay.tick(later(3600))).toMatchObject({ claimed: 0 }); + }); + + it("caps the backoff", async () => { + build(config({ maxAttempts: 100 })); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + invokeContract.mockRejectedValue(new Error("nope")); + let t = now; + for (let i = 0; i < 12; i++) { + await relay.tick(t); + t = (await outbox.findByIntent("i1"))[0].nextAttemptAt; + } + const [row] = await outbox.findByIntent("i1"); + await relay.tick(row.nextAttemptAt); + const [after] = await outbox.findByIntent("i1"); + expect(after.nextAttemptAt.getTime() - row.nextAttemptAt.getTime()).toBe(5 * 60_000); + }); + + it("retries when SETTLEMENT_CONTRACT_ID is missing at relay time", async () => { + build(config({}, "")); + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + await relay.tick(now); + expect(invokeContract).not.toHaveBeenCalled(); + expect((await outbox.findByIntent("i1"))[0].lastError).toContain("SETTLEMENT_CONTRACT_ID"); + }); + + describe("confirmation of submitted rows", () => { + beforeEach(async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + await relay.tick(now); + }); + + it("rebuilds when the transaction failed on-chain", async () => { + check.mockResolvedValueOnce({ status: "failed", ledger: 3 }); + expect(await relay.tick(later(5))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0]).toMatchObject({ status: "pending", txHash: undefined }); + }); + + it("waits while not found within the lease, rebuilds after it", async () => { + expect(await relay.tick(later(LEASE_SECONDS - 10))).toMatchObject({ retried: 0, confirmed: 0 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("submitted"); + + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ retried: 1 }); + expect((await outbox.findByIntent("i1"))[0].lastError).toContain("not found after lease"); + }); + + it("treats a lookup failure as no evidence and looks again next tick", async () => { + check.mockRejectedValueOnce(new Error("RPC down")); + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ retried: 0, confirmed: 0 }); + expect((await outbox.findByIntent("i1"))[0].status).toBe("submitted"); + }); + }); + + it("skips overlapping ticks", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + let release!: () => void; + invokeContract.mockImplementationOnce( + (_p, options) => + new Promise((resolve) => { + release = () => { + void options?.beforeSubmit?.("h").then(() => resolve({ hash: "h", status: "PENDING", dryRun: false })); + }; + }), + ); + const first = relay.tick(now); + await new Promise((r) => setImmediate(r)); + expect(await relay.tick(now)).toMatchObject({ claimed: 0 }); + release(); + expect(await first).toMatchObject({ submitted: 1 }); + }); + + it("survives a backlog gauge failure", async () => { + jest.spyOn(outbox, "countByStatus").mockRejectedValueOnce(new Error("db")); + await expect(relay.tick(now)).resolves.toMatchObject({ claimed: 0 }); + }); + + it("starts an interval on init only when enabled", () => { + jest.useFakeTimers(); + try { + const tick = jest.spyOn(relay, "tick").mockResolvedValue({} as never); + relay.onModuleInit(); + jest.advanceTimersByTime(1000); + expect(tick).toHaveBeenCalledTimes(1); + relay.onModuleDestroy(); + + build(config({ relayEnabled: false })); + const disabledTick = jest.spyOn(relay, "tick"); + relay.onModuleInit(); + jest.advanceTimersByTime(5000); + expect(disabledTick).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + } + }); + + it("logs, but does not crash, when an interval tick throws", () => { + jest.useFakeTimers(); + try { + jest.spyOn(relay, "tick").mockRejectedValue(new Error("boom")); + const errorLog = jest.spyOn((relay as unknown as { logger: { error: () => void } }).logger, "error").mockImplementation(); + relay.onModuleInit(); + jest.advanceTimersByTime(1000); + return Promise.resolve().then(() => { + expect(errorLog).toHaveBeenCalledWith(expect.stringContaining("boom")); + }); + } finally { + jest.useRealTimers(); + } + }); + + describe("lost fences (another worker reclaimed the row)", () => { + it("does not count outcomes whose fenced write lost", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + jest.spyOn(outbox, "markSubmitted").mockResolvedValueOnce(false); + expect(await relay.tick(now)).toMatchObject({ claimed: 1, submitted: 0 }); + + invokeContract.mockResolvedValueOnce({ hash: "x", status: "DRY_RUN", dryRun: true }); + jest.spyOn(outbox, "markSimulated").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 1))).toMatchObject({ simulated: 0 }); + }); + + it("does not count a lost confirmation, dead-letter, or retry", async () => { + await outbox.enqueue(createIntentEntry(intentFor("i1"))); + submitsAs("h1"); + await relay.tick(now); + check.mockResolvedValue({ status: "success", ledger: 1 }); + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + expect(await relay.tick(later(1))).toMatchObject({ confirmed: 0 }); + + // Reclaimed row whose envelope landed, but the confirm write loses. + await outbox.enqueue(createIntentEntry(intentFor("i2"))); + invokeContract.mockImplementationOnce(async (_p, options) => { + await options?.beforeSubmit?.("h2"); + throw new Error("killed"); + }); + const retry = jest.spyOn(outbox, "scheduleRetry").mockResolvedValueOnce(false); + expect(await relay.tick(later(2))).toMatchObject({ retried: 0 }); + retry.mockRestore(); + jest.spyOn(outbox, "findSubmitted").mockResolvedValueOnce([]); + jest.spyOn(outbox, "markConfirmed").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 5))).toMatchObject({ claimed: 1, confirmed: 0 }); + + build(config({ maxAttempts: 1 })); + await outbox.enqueue(createIntentEntry(intentFor("i3"))); + invokeContract.mockRejectedValueOnce("not an Error"); + jest.spyOn(outbox, "markDead").mockResolvedValueOnce(false); + expect(await relay.tick(later(LEASE_SECONDS + 10))).toMatchObject({ dead: 0 }); + }); + + it("skips submitted rows without a tx hash and defaults the clock", async () => { + jest.spyOn(outbox, "findSubmitted").mockResolvedValueOnce([ + { id: "1", intentId: "i", operation: "create_intent", payload: {}, status: "submitted", attempts: 1, + nextAttemptAt: now, createdAt: now, updatedAt: now }, + ]); + await expect(relay.tick()).resolves.toMatchObject({ confirmed: 0, retried: 0 }); + expect(check).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/soroban/outbox-relay.service.ts b/src/soroban/outbox-relay.service.ts new file mode 100644 index 00000000..9482d4ce --- /dev/null +++ b/src/soroban/outbox-relay.service.ts @@ -0,0 +1,265 @@ +import { Inject, Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchActiveException } from "../killswitch/killswitch.guard"; +import { MetricsService } from "../metrics/metrics.service"; +import { IOutboxRepository, OUTBOX_REPOSITORY, OutboxEntry } from "./outbox.repository"; +import { buildOutboxInvocation } from "./outbox-operations"; +import { StellarTxService } from "./stellar-tx.service"; +import { TxConfirmationService } from "./tx-confirmation.service"; + +/** Upper bound for the exponential retry backoff. */ +const MAX_BACKOFF_MS = 5 * 60_000; +const BASE_BACKOFF_MS = 1_000; + +/** Re-check interval for rows blocked by a kill-switch pause. */ +const PAUSED_RECHECK_MS = 30_000; + +/** Counts from one relay tick — returned for tests and manual triggers. */ +export interface RelayTickResult { + claimed: number; + /** Rows put back untouched because a kill-switch pause blocked the write. */ + paused: number; + submitted: number; + simulated: number; + confirmed: number; + retried: number; + dead: number; +} + +/** + * Relay worker for the transactional outbox (issue #396). + * + * Each tick: + * 1. **Confirm** — polls `submitted` rows through TxConfirmationService and + * marks them `confirmed`, or schedules a rebuild when the transaction + * failed on-chain or expired unseen. + * 2. **Relay** — claims due head-of-intent rows (SKIP LOCKED in Postgres, so + * several instances can run this safely) and submits them via + * StellarTxService. + * + * StellarTxService.invokeContract blocks until the transaction confirms (or + * throws on FAILED / confirmation TIMEOUT). A TIMEOUT retry is safe: the wait + * (120 s) outlasts the envelope's 30 s time bound, so the timed-out envelope + * can no longer land. + * + * Crash idempotency: the signed envelope's hash is persisted *before* + * broadcast (`beforeSubmit`). If the process dies between broadcast and + * `markSubmitted`, the lease expires, the row is reclaimed with its + * `envelopeHash` set, and the relay looks that hash up first — SUCCESS means + * the earlier submission landed and the row is confirmed without + * resubmitting. NOT_FOUND is only trusted because the lease + * (OUTBOX_LEASE_SECONDS) is longer than the envelope's time bound, so by + * reclaim time an unseen envelope can no longer be included. + * + * Poison rows: after OUTBOX_MAX_ATTEMPTS claims a row moves to `dead`, + * increments `vortex_outbox_dead_total` (alerted on) and blocks later rows + * for the same intent until an operator requeues it — see + * docs/runbooks/on-call.md. + */ +@Injectable() +export class OutboxRelayService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(OutboxRelayService.name); + private readonly settings: AppConfig["outbox"]; + private readonly settlementContractId: string; + private interval?: NodeJS.Timeout; + private running = false; + + constructor( + @Inject(OUTBOX_REPOSITORY) private readonly outbox: IOutboxRepository, + private readonly stellarTxService: StellarTxService, + private readonly confirmation: TxConfirmationService, + private readonly metrics: MetricsService, + configService: ConfigService, + ) { + this.settings = configService.get("outbox", { infer: true }); + this.settlementContractId = configService.get("stellar.settlementContractId", { infer: true }); + } + + onModuleInit() { + if (!this.settings.relayEnabled) { + this.logger.warn("[outbox] relay disabled (OUTBOX_RELAY_ENABLED=false) — rows will accumulate"); + return; + } + this.interval = setInterval(() => { + this.tick().catch((err) => + this.logger.error(`[outbox] relay tick failed: ${errorMessage(err)}`), + ); + }, this.settings.relayIntervalMs); + this.interval.unref?.(); + } + + onModuleDestroy() { + if (this.interval) clearInterval(this.interval); + } + + /** Runs one confirm + relay cycle. Overlapping calls are skipped. */ + async tick(now: Date = new Date()): Promise { + const result: RelayTickResult = { + claimed: 0, + paused: 0, + submitted: 0, + simulated: 0, + confirmed: 0, + retried: 0, + dead: 0, + }; + if (this.running) return result; + this.running = true; + try { + await this.confirmSubmitted(now, result); + await this.relayDue(now, result); + await this.refreshBacklogGauge(); + return result; + } finally { + this.running = false; + } + } + + private async confirmSubmitted(now: Date, result: RelayTickResult): Promise { + const submitted = await this.outbox.findSubmitted(this.settings.batchSize); + for (const entry of submitted) { + if (!entry.txHash) continue; + let status; + try { + status = (await this.confirmation.check(entry.txHash)).status; + } catch (err) { + // RPC outage is not evidence either way — look again next tick. + this.logger.warn(`[outbox] confirmation lookup failed for row ${entry.id}: ${errorMessage(err)}`); + continue; + } + + if (status === "success") { + if (await this.outbox.markConfirmed(entry, entry.txHash)) { + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + } else if (status === "failed") { + await this.fail(entry, `transaction ${entry.txHash} failed on-chain`, now, result); + } else if (now.getTime() - entry.updatedAt.getTime() > this.settings.leaseSeconds * 1000) { + await this.fail(entry, `transaction ${entry.txHash} not found after lease; rebuilding`, now, result); + } + } + } + + private async relayDue(now: Date, result: RelayTickResult): Promise { + const leaseUntil = new Date(now.getTime() + this.settings.leaseSeconds * 1000); + const claimed = await this.outbox.claimDue(now, this.settings.batchSize, leaseUntil); + result.claimed += claimed.length; + // Sequential on purpose: every submission draws the next sequence number + // from the single signing account (SignerService serializes anyway). + for (const entry of claimed) { + await this.process(entry, now, result); + } + } + + private async process(entry: OutboxEntry, now: Date, result: RelayTickResult): Promise { + try { + if (entry.envelopeHash) { + const previous = await this.confirmation.check(entry.envelopeHash); + if (previous.status === "success") { + this.logger.warn( + `[outbox] row ${entry.id} (${entry.operation} intent=${entry.intentId}) was already ` + + `submitted before a crash (tx ${entry.envelopeHash}); confirming without resubmitting`, + ); + if (await this.outbox.markConfirmed(entry, entry.envelopeHash)) { + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + return; + } + if (previous.status === "failed") { + throw new Error(`previous envelope ${entry.envelopeHash} failed on-chain`); + } + // not_found: the lease outlived the envelope's time bound — rebuild. + } + + if (!this.settlementContractId) { + throw new Error("SETTLEMENT_CONTRACT_ID is not configured"); + } + const invocation = buildOutboxInvocation(entry, this.settlementContractId); + const sent = await this.stellarTxService.invokeContract(invocation, { + beforeSubmit: async (hash) => { + if (!(await this.outbox.recordEnvelope(entry, hash))) { + throw new Error(`lost lease on row ${entry.id} before submit`); + } + }, + }); + + if (sent.dryRun) { + if (await this.outbox.markSimulated(entry)) { + result.simulated++; + this.metrics.recordOutboxOutcome("simulated"); + } + return; + } + + // StellarTxService's live path waits for confirmation and reports + // SUCCESS; anything else (e.g. PENDING) is confirmed by a later tick. + if (sent.status === "SUCCESS") { + if (await this.outbox.markConfirmed(entry, sent.hash)) { + result.submitted++; + result.confirmed++; + this.metrics.recordOutboxOutcome("confirmed"); + } + return; + } + if (await this.outbox.markSubmitted(entry, sent.hash)) { + result.submitted++; + this.metrics.recordOutboxOutcome("submitted"); + } + } catch (err) { + if (err instanceof KillSwitchActiveException) { + // Issue #477 — a pause is not a failure: put the row back without + // consuming an attempt so a long pause cannot dead-letter it. + if (await this.outbox.release(entry, `paused by kill-switch: ${err.message}`, new Date(now.getTime() + PAUSED_RECHECK_MS))) { + result.paused++; + } + return; + } + await this.fail(entry, errorMessage(err), now, result); + } + } + + private async fail( + entry: OutboxEntry, + error: string, + now: Date, + result: RelayTickResult, + ): Promise { + if (entry.attempts >= this.settings.maxAttempts) { + if (await this.outbox.markDead(entry, error)) { + result.dead++; + this.metrics.recordOutboxOutcome("dead"); + this.logger.error( + `[outbox] ALERT row ${entry.id} (${entry.operation} intent=${entry.intentId}) moved to dead ` + + `after ${entry.attempts} attempts: ${error}. Later operations for this intent are blocked ` + + `until it is requeued — see docs/runbooks/on-call.md`, + ); + } + return; + } + + const delay = Math.min(BASE_BACKOFF_MS * 2 ** Math.max(0, entry.attempts - 1), MAX_BACKOFF_MS); + if (await this.outbox.scheduleRetry(entry, error, new Date(now.getTime() + delay))) { + result.retried++; + this.metrics.recordOutboxOutcome("retry"); + this.logger.warn( + `[outbox] row ${entry.id} (${entry.operation} intent=${entry.intentId}) attempt ` + + `${entry.attempts}/${this.settings.maxAttempts} failed: ${error}; retrying in ${delay}ms`, + ); + } + } + + private async refreshBacklogGauge(): Promise { + try { + this.metrics.setOutboxBacklog(await this.outbox.countByStatus()); + } catch (err) { + this.logger.warn(`[outbox] backlog gauge refresh failed: ${errorMessage(err)}`); + } + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/soroban/outbox.repository.spec.ts b/src/soroban/outbox.repository.spec.ts new file mode 100644 index 00000000..429031b6 --- /dev/null +++ b/src/soroban/outbox.repository.spec.ts @@ -0,0 +1,145 @@ +import { InMemoryOutboxRepository, NewOutboxEntry } from "./outbox.repository"; + +const at = (ms: number) => new Date(1_700_000_000_000 + ms); +const entry = (intentId: string, operation: NewOutboxEntry["operation"] = "create_intent"): NewOutboxEntry => ({ + intentId, + operation, + payload: { intentId }, +}); + +describe("InMemoryOutboxRepository (#396)", () => { + let repo: InMemoryOutboxRepository; + + beforeEach(() => { + repo = new InMemoryOutboxRepository(); + }); + + it("assigns monotonic ids and starts rows as pending with zero attempts", async () => { + const a = await repo.enqueue(entry("i1")); + const b = await repo.enqueue(entry("i1", "accept_intent")); + expect(BigInt(b.id)).toBeGreaterThan(BigInt(a.id)); + expect(a).toMatchObject({ status: "pending", attempts: 0 }); + }); + + it("claims only the head row per intent, but runs different intents in parallel", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + await repo.enqueue(entry("i2")); + + const claimed = await repo.claimDue(new Date(Date.now() + 1000), 10, at(60_000)); + + expect(claimed.map((r) => [r.intentId, r.operation])).toEqual([ + ["i1", "create_intent"], + ["i2", "create_intent"], + ]); + expect(claimed.every((r) => r.status === "processing" && r.attempts === 1)).toBe(true); + }); + + it("releases the next row for an intent only once the previous one is confirmed or simulated", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + const now = new Date(Date.now() + 1000); + + const [first] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markSubmitted(first, "tx1"); + expect(await repo.claimDue(now, 10, new Date(now.getTime() + 60_000))).toEqual([]); + + await repo.markConfirmed({ ...first }, "tx1"); + const [second] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(second.operation).toBe("accept_intent"); + + await repo.markSimulated(second); + await repo.enqueue(entry("i1", "fill_intent")); + const [third] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(third.operation).toBe("fill_intent"); + }); + + it("respects the batch limit and nextAttemptAt", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i2")); + const now = new Date(Date.now() + 1000); + const [claimed] = await repo.claimDue(now, 1, new Date(now.getTime() + 60_000)); + expect(claimed.intentId).toBe("i1"); + + await repo.scheduleRetry(claimed, "boom", new Date(now.getTime() + 10_000)); + const next = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + expect(next.map((r) => r.intentId)).toEqual(["i2"]); + const later = await repo.claimDue(new Date(now.getTime() + 10_001), 10, new Date(now.getTime() + 70_000)); + expect(later.map((r) => r.intentId)).toEqual(["i1"]); + expect(later[0].attempts).toBe(2); + }); + + it("reclaims a processing row only after its lease expires (crashed worker)", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const leaseUntil = new Date(now.getTime() + 60_000); + await repo.claimDue(now, 10, leaseUntil); + + expect(await repo.claimDue(new Date(leaseUntil.getTime() - 1), 10, at(0))).toEqual([]); + const [reclaimed] = await repo.claimDue(new Date(leaseUntil.getTime() + 1), 10, at(0)); + expect(reclaimed.attempts).toBe(2); + }); + + it("fences writes on attempts so a stale worker cannot overwrite a reclaimed row", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const [stale] = await repo.claimDue(now, 10, new Date(now.getTime() + 1)); + const [fresh] = await repo.claimDue(new Date(now.getTime() + 2), 10, new Date(now.getTime() + 60_000)); + + expect(await repo.recordEnvelope(stale, "old")).toBe(false); + expect(await repo.markSubmitted(stale, "old")).toBe(false); + expect(await repo.recordEnvelope(fresh, "new")).toBe(true); + expect((await repo.findByIntent("i1"))[0].envelopeHash).toBe("new"); + }); + + it("clears envelope and tx hash on retry so the relay rebuilds", async () => { + await repo.enqueue(entry("i1")); + const now = new Date(Date.now() + 1000); + const [row] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.recordEnvelope(row, "h"); + await repo.markSubmitted(row, "h"); + await repo.scheduleRetry(row, "failed on-chain", now); + + const [stored] = await repo.findByIntent("i1"); + expect(stored).toMatchObject({ status: "pending", lastError: "failed on-chain" }); + expect(stored.envelopeHash).toBeUndefined(); + expect(stored.txHash).toBeUndefined(); + }); + + it("dead rows block later rows for the same intent until requeued", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i1", "accept_intent")); + const now = new Date(Date.now() + 1000); + const [row] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markDead(row, "poison"); + + expect(await repo.claimDue(now, 10, new Date(now.getTime() + 60_000))).toEqual([]); + expect((await repo.countByStatus()).dead).toBe(1); + + expect(await repo.requeueDead("999")).toBe(false); + expect(await repo.requeueDead(row.id)).toBe(true); + expect(await repo.requeueDead(row.id)).toBe(false); + const [again] = await repo.claimDue(new Date(Date.now() + 1000), 10, at(0)); + expect(again).toMatchObject({ id: row.id, attempts: 1, operation: "create_intent" }); + }); + + it("lists submitted rows oldest first and counts by status", async () => { + await repo.enqueue(entry("i1")); + await repo.enqueue(entry("i2")); + const now = new Date(Date.now() + 1000); + const [a, b] = await repo.claimDue(now, 10, new Date(now.getTime() + 60_000)); + await repo.markSubmitted(b, "tb"); + await repo.markSubmitted(a, "ta"); + + expect((await repo.findSubmitted(10)).map((r) => r.txHash)).toEqual(["ta", "tb"]); + expect((await repo.findSubmitted(1)).map((r) => r.txHash)).toEqual(["ta"]); + expect(await repo.countByStatus()).toEqual({ + pending: 0, + processing: 0, + submitted: 2, + confirmed: 0, + simulated: 0, + dead: 0, + }); + }); +}); diff --git a/src/soroban/outbox.repository.ts b/src/soroban/outbox.repository.ts new file mode 100644 index 00000000..ef13c845 --- /dev/null +++ b/src/soroban/outbox.repository.ts @@ -0,0 +1,269 @@ +import { Injectable } from "@nestjs/common"; + +/** + * NestJS injection token for the on-chain outbox repository (issue #396). + * Bound in IntentsModule to the in-memory or Prisma adapter, following + * INTENTS_PERSISTENCE. + */ +export const OUTBOX_REPOSITORY = Symbol("OUTBOX_REPOSITORY"); + +/** Settlement-contract operations that flow through the outbox. */ +export const OUTBOX_OPERATIONS = [ + "create_intent", + "accept_intent", + "fill_intent", + "cancel_intent", +] as const; +export type OutboxOperation = (typeof OUTBOX_OPERATIONS)[number]; + +/** + * Row lifecycle: + * + * pending ──claim──▶ processing ──submit──▶ submitted ──confirm──▶ confirmed + * ▲ │ │ │ + * └──── retry ────────┘ └─dry-run─▶ simulated│ + * ▲ │ + * └──────────── tx failed / expired ─────────┘ + * any retry past OUTBOX_MAX_ATTEMPTS ──▶ dead (alerted; blocks later rows + * for the same intent) + */ +export type OutboxStatus = + | "pending" + | "processing" + | "submitted" + | "confirmed" + | "simulated" + | "dead"; + +/** Statuses after which the next row for the same intent may proceed. */ +export const OUTBOX_DONE_STATUSES: readonly OutboxStatus[] = ["confirmed", "simulated"]; + +export interface OutboxEntry { + /** Monotonic sequence (bigint serialized as string); defines per-intent order. */ + id: string; + intentId: string; + operation: OutboxOperation; + payload: Record; + status: OutboxStatus; + /** + * Number of claims so far. Also the fencing token: every state change after + * a claim is conditional on `attempts` still matching, so a worker whose + * lease expired cannot overwrite the row after another worker reclaimed it. + */ + attempts: number; + nextAttemptAt: Date; + lockedUntil?: Date; + envelopeHash?: string; + txHash?: string; + lastError?: string; + createdAt: Date; + updatedAt: Date; +} + +export interface NewOutboxEntry { + intentId: string; + operation: OutboxOperation; + payload: Record; +} + +/** The only outbox capability intent-mutating code needs inside a transaction. */ +export interface IOutboxWriter { + enqueue(entry: NewOutboxEntry): Promise; +} + +export interface IOutboxRepository extends IOutboxWriter { + /** + * Atomically claims up to `limit` due rows and moves them to `processing` + * (attempts + 1, lockedUntil = `leaseUntil`). A row is due when it is + * `pending` with nextAttemptAt <= now, or `processing` with an expired lease + * (its worker crashed). Only the head row per intent is eligible — every + * earlier row for that intent must be in {@link OUTBOX_DONE_STATUSES} — + * which gives per-intent ordering while different intents run in parallel. + * The Prisma adapter uses `FOR UPDATE SKIP LOCKED`. + */ + claimDue(now: Date, limit: number, leaseUntil: Date): Promise; + + /** Rows waiting on confirmation, oldest first. */ + findSubmitted(limit: number): Promise; + + findByIntent(intentId: string): Promise; + + countByStatus(): Promise>; + + /** Persists the signed envelope hash before broadcast. Fenced on `attempts`. */ + recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise; + + markSubmitted(entry: OutboxEntry, txHash: string): Promise; + + markConfirmed(entry: OutboxEntry, txHash: string): Promise; + + markSimulated(entry: OutboxEntry): Promise; + + /** Back to `pending` at `nextAttemptAt`; clears the envelope so it is rebuilt. */ + scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise; + + markDead(entry: OutboxEntry, error: string): Promise; + + /** + * Back to `pending` at `nextAttemptAt` *without* consuming the claim's + * attempt (attempts - 1). Used when a kill-switch pause blocked the write, + * so an emergency pause can never dead-letter rows. + */ + release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise; + + /** Operator action: `dead` → `pending` with attempts reset. */ + requeueDead(id: string): Promise; +} + +export function emptyStatusCounts(): Record { + return { pending: 0, processing: 0, submitted: 0, confirmed: 0, simulated: 0, dead: 0 }; +} + +/** + * In-memory adapter — development and tests. Single-process only; the Node + * event loop makes each method atomic because none of them await. + */ +@Injectable() +export class InMemoryOutboxRepository implements IOutboxRepository { + private readonly rows = new Map(); + private sequence = 0n; + + async enqueue(entry: NewOutboxEntry): Promise { + const now = new Date(); + const row: OutboxEntry = { + ...entry, + id: (++this.sequence).toString(), + status: "pending", + attempts: 0, + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }; + this.rows.set(row.id, row); + return { ...row }; + } + + async claimDue(now: Date, limit: number, leaseUntil: Date): Promise { + const claimed: OutboxEntry[] = []; + const headSeen = new Set(); + for (const row of this.ordered()) { + if (claimed.length >= limit) break; + if (OUTBOX_DONE_STATUSES.includes(row.status)) continue; + // First unfinished row for this intent is its head; anything after it waits. + if (headSeen.has(row.intentId)) continue; + headSeen.add(row.intentId); + + const due = + (row.status === "pending" && row.nextAttemptAt <= now) || + (row.status === "processing" && row.lockedUntil !== undefined && row.lockedUntil < now); + if (!due) continue; + + Object.assign(row, { + status: "processing", + attempts: row.attempts + 1, + lockedUntil: leaseUntil, + updatedAt: now, + }); + claimed.push({ ...row }); + } + return claimed; + } + + async findSubmitted(limit: number): Promise { + return this.ordered() + .filter((r) => r.status === "submitted") + .slice(0, limit) + .map((r) => ({ ...r })); + } + + async findByIntent(intentId: string): Promise { + return this.ordered() + .filter((r) => r.intentId === intentId) + .map((r) => ({ ...r })); + } + + async countByStatus(): Promise> { + const counts = emptyStatusCounts(); + for (const row of this.rows.values()) counts[row.status]++; + return counts; + } + + async recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise { + return this.fenced(entry, ["processing"], { envelopeHash }); + } + + async markSubmitted(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing"], { status: "submitted", txHash, lockedUntil: undefined }); + } + + async markConfirmed(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "confirmed", + txHash, + lockedUntil: undefined, + }); + } + + async markSimulated(entry: OutboxEntry): Promise { + return this.fenced(entry, ["processing"], { status: "simulated", lockedUntil: undefined }); + } + + async scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "pending", + nextAttemptAt, + lastError: error, + lockedUntil: undefined, + envelopeHash: undefined, + txHash: undefined, + }); + } + + async markDead(entry: OutboxEntry, error: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "dead", + lastError: error, + lockedUntil: undefined, + }); + } + + async release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing"], { + status: "pending", + attempts: Math.max(0, entry.attempts - 1), + nextAttemptAt, + lastError: note, + lockedUntil: undefined, + envelopeHash: undefined, + }); + } + + async requeueDead(id: string): Promise { + const row = this.rows.get(id); + if (!row || row.status !== "dead") return false; + Object.assign(row, { + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + envelopeHash: undefined, + txHash: undefined, + updatedAt: new Date(), + }); + return true; + } + + private ordered(): OutboxEntry[] { + return [...this.rows.values()].sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1)); + } + + private fenced( + entry: OutboxEntry, + from: OutboxStatus[], + patch: Partial, + ): boolean { + const row = this.rows.get(entry.id); + if (!row || row.attempts !== entry.attempts || !from.includes(row.status)) return false; + Object.assign(row, patch, { updatedAt: new Date() }); + return true; + } +} diff --git a/src/soroban/prisma-outbox.repository.spec.ts b/src/soroban/prisma-outbox.repository.spec.ts new file mode 100644 index 00000000..09135966 --- /dev/null +++ b/src/soroban/prisma-outbox.repository.spec.ts @@ -0,0 +1,129 @@ +import { Prisma } from "@prisma/client"; +import { OutboxEntry } from "./outbox.repository"; +import { OutboxPrismaClient, PrismaOutboxRepository } from "./prisma-outbox.repository"; + +const created = new Date("2026-09-27T00:00:00Z"); +const modelRow = (overrides: Record = {}) => ({ + id: 7n, + intentId: "i1", + operation: "create_intent", + payload: { intentId: "i1" }, + status: "pending", + attempts: 0, + nextAttemptAt: created, + lockedUntil: null, + envelopeHash: null, + txHash: null, + lastError: null, + createdAt: created, + updatedAt: created, + ...overrides, +}); + +const entry = { id: "7", attempts: 2 } as OutboxEntry; + +function fakeClient() { + return { + onchainOutbox: { + create: jest.fn().mockResolvedValue(modelRow()), + findMany: jest.fn().mockResolvedValue([modelRow({ status: "submitted", txHash: "h" })]), + groupBy: jest.fn().mockResolvedValue([ + { status: "pending", _count: { _all: 3 } }, + { status: "dead", _count: { _all: 1 } }, + ]), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, + $queryRaw: jest.fn(), + }; +} + +describe("PrismaOutboxRepository (#396)", () => { + let client: ReturnType; + let repo: PrismaOutboxRepository; + + beforeEach(() => { + client = fakeClient(); + repo = new PrismaOutboxRepository(client as unknown as OutboxPrismaClient); + }); + + it("enqueues through the (transaction) client and maps bigint ids to strings", async () => { + const row = await repo.enqueue({ intentId: "i1", operation: "create_intent", payload: { intentId: "i1" } }); + expect(client.onchainOutbox.create).toHaveBeenCalledWith({ + data: { intentId: "i1", operation: "create_intent", payload: { intentId: "i1" } }, + }); + expect(row).toMatchObject({ id: "7", status: "pending", lockedUntil: undefined, envelopeHash: undefined }); + }); + + it("claims with a single SKIP LOCKED statement that enforces per-intent ordering", async () => { + client.$queryRaw.mockResolvedValue([ + { + id: 9n, intent_id: "b", operation: "create_intent", payload: {}, status: "processing", attempts: 1, + next_attempt_at: created, locked_until: created, envelope_hash: null, tx_hash: null, last_error: null, + created_at: created, updated_at: created, + }, + { + id: 3n, intent_id: "a", operation: "accept_intent", payload: null, status: "processing", attempts: 1, + next_attempt_at: created, locked_until: created, envelope_hash: "e", tx_hash: null, last_error: "x", + created_at: created, updated_at: created, + }, + ]); + const now = new Date("2026-09-27T01:00:00Z"); + const lease = new Date("2026-09-27T01:02:00Z"); + + const rows = await repo.claimDue(now, 5, lease); + + const sql = client.$queryRaw.mock.calls[0][0] as Prisma.Sql; + expect(sql.sql).toMatch(/FOR UPDATE SKIP LOCKED/); + expect(sql.sql).toMatch(/NOT EXISTS/); + expect(sql.sql).toMatch(/p\.id < c\.id/); + expect(sql.sql).toMatch(/attempts = o\.attempts \+ 1/); + expect(sql.values).toEqual(expect.arrayContaining([now, 5, lease, "confirmed", "simulated"])); + expect(rows.map((r) => r.id)).toEqual(["3", "9"]); + expect(rows[0]).toMatchObject({ intentId: "a", envelopeHash: "e", lastError: "x", payload: {} }); + }); + + it("reads submitted rows, rows by intent, and status counts", async () => { + expect(await repo.findSubmitted(4)).toHaveLength(1); + expect(client.onchainOutbox.findMany).toHaveBeenLastCalledWith({ + where: { status: "submitted" }, + orderBy: { id: "asc" }, + take: 4, + }); + await repo.findByIntent("i1"); + expect(client.onchainOutbox.findMany).toHaveBeenLastCalledWith({ where: { intentId: "i1" }, orderBy: { id: "asc" } }); + expect(await repo.countByStatus()).toEqual({ + pending: 3, processing: 0, submitted: 0, confirmed: 0, simulated: 0, dead: 1, + }); + }); + + it.each([ + ["recordEnvelope", () => repo.recordEnvelope(entry, "h"), ["processing"], { envelopeHash: "h" }], + ["markSubmitted", () => repo.markSubmitted(entry, "h"), ["processing"], { status: "submitted", txHash: "h", lockedUntil: null }], + ["markConfirmed", () => repo.markConfirmed(entry, "h"), ["processing", "submitted"], { status: "confirmed", txHash: "h", lockedUntil: null }], + ["markSimulated", () => repo.markSimulated(entry), ["processing"], { status: "simulated", lockedUntil: null }], + ["markDead", () => repo.markDead(entry, "e"), ["processing", "submitted"], { status: "dead", lastError: "e", lockedUntil: null }], + ])("%s is fenced on id + attempts + status", async (_name, call, from, data) => { + expect(await (call as () => Promise)()).toBe(true); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith({ + where: { id: 7n, attempts: 2, status: { in: from } }, + data, + }); + }); + + it("scheduleRetry clears the envelope so the relay rebuilds, and reports a lost fence", async () => { + client.onchainOutbox.updateMany.mockResolvedValueOnce({ count: 0 }); + const next = new Date(); + expect(await repo.scheduleRetry(entry, "boom", next)).toBe(false); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith({ + where: { id: 7n, attempts: 2, status: { in: ["processing", "submitted"] } }, + data: { status: "pending", nextAttemptAt: next, lastError: "boom", lockedUntil: null, envelopeHash: null, txHash: null }, + }); + }); + + it("requeues only dead rows", async () => { + expect(await repo.requeueDead("7")).toBe(true); + expect(client.onchainOutbox.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ where: { id: 7n, status: "dead" } }), + ); + }); +}); diff --git a/src/soroban/prisma-outbox.repository.ts b/src/soroban/prisma-outbox.repository.ts new file mode 100644 index 00000000..1172b27b --- /dev/null +++ b/src/soroban/prisma-outbox.repository.ts @@ -0,0 +1,229 @@ +import { Prisma, OnchainOutbox, OutboxStatus as PrismaOutboxStatus } from "@prisma/client"; +import { + IOutboxRepository, + NewOutboxEntry, + OUTBOX_DONE_STATUSES, + OutboxEntry, + OutboxOperation, + OutboxStatus, + emptyStatusCounts, +} from "./outbox.repository"; + +/** PrismaService or the client handed to a `$transaction` callback. */ +export type OutboxPrismaClient = Prisma.TransactionClient; + +/** Raw row shape returned by the claim query (snake_case columns). */ +interface RawOutboxRow { + id: bigint; + intent_id: string; + operation: string; + payload: Prisma.JsonValue; + status: PrismaOutboxStatus; + attempts: number; + next_attempt_at: Date; + locked_until: Date | null; + envelope_hash: string | null; + tx_hash: string | null; + last_error: string | null; + created_at: Date; + updated_at: Date; +} + +/** + * Prisma-backed outbox (issue #396). + * + * Constructed either with PrismaService (relay worker) or with a transaction + * client (inside IntentsUnitOfWork) so `enqueue` commits atomically with the + * intent mutation it mirrors. + */ +export class PrismaOutboxRepository implements IOutboxRepository { + constructor(private readonly prisma: OutboxPrismaClient) {} + + async enqueue(entry: NewOutboxEntry): Promise { + const row = await this.prisma.onchainOutbox.create({ + data: { + intentId: entry.intentId, + operation: entry.operation, + payload: entry.payload as Prisma.InputJsonObject, + }, + }); + return fromModel(row); + } + + /** + * Single statement: pick the per-intent head rows that are due, lock them + * with SKIP LOCKED so concurrent relays partition the work, and flip them to + * `processing`. The NOT EXISTS clause treats every non-done earlier row + * (including `dead`) as blocking, which is what guarantees per-intent order. + */ + async claimDue(now: Date, limit: number, leaseUntil: Date): Promise { + const done = Prisma.join(OUTBOX_DONE_STATUSES.map((s) => Prisma.sql`${s}::"OutboxStatus"`)); + const rows = await this.prisma.$queryRaw(Prisma.sql` + WITH candidates AS ( + SELECT c.id + FROM onchain_outbox c + WHERE ( + (c.status = 'pending' AND c.next_attempt_at <= ${now}) + OR (c.status = 'processing' AND c.locked_until < ${now}) + ) + AND NOT EXISTS ( + SELECT 1 FROM onchain_outbox p + WHERE p.intent_id = c.intent_id + AND p.id < c.id + AND p.status NOT IN (${done}) + ) + ORDER BY c.id + LIMIT ${limit} + FOR UPDATE SKIP LOCKED + ) + UPDATE onchain_outbox o + SET status = 'processing', + attempts = o.attempts + 1, + locked_until = ${leaseUntil}, + updated_at = NOW() + FROM candidates + WHERE o.id = candidates.id + RETURNING o.* + `); + return rows.map(fromRaw).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1)); + } + + async findSubmitted(limit: number): Promise { + const rows = await this.prisma.onchainOutbox.findMany({ + where: { status: "submitted" }, + orderBy: { id: "asc" }, + take: limit, + }); + return rows.map(fromModel); + } + + async findByIntent(intentId: string): Promise { + const rows = await this.prisma.onchainOutbox.findMany({ + where: { intentId }, + orderBy: { id: "asc" }, + }); + return rows.map(fromModel); + } + + async countByStatus(): Promise> { + const groups = await this.prisma.onchainOutbox.groupBy({ + by: ["status"], + _count: { _all: true }, + }); + const counts = emptyStatusCounts(); + for (const g of groups) counts[g.status as OutboxStatus] = g._count._all; + return counts; + } + + recordEnvelope(entry: OutboxEntry, envelopeHash: string): Promise { + return this.fenced(entry, ["processing"], { envelopeHash }); + } + + markSubmitted(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing"], { status: "submitted", txHash, lockedUntil: null }); + } + + markConfirmed(entry: OutboxEntry, txHash: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "confirmed", + txHash, + lockedUntil: null, + }); + } + + markSimulated(entry: OutboxEntry): Promise { + return this.fenced(entry, ["processing"], { status: "simulated", lockedUntil: null }); + } + + scheduleRetry(entry: OutboxEntry, error: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "pending", + nextAttemptAt, + lastError: error, + lockedUntil: null, + envelopeHash: null, + txHash: null, + }); + } + + markDead(entry: OutboxEntry, error: string): Promise { + return this.fenced(entry, ["processing", "submitted"], { + status: "dead", + lastError: error, + lockedUntil: null, + }); + } + + release(entry: OutboxEntry, note: string, nextAttemptAt: Date): Promise { + return this.fenced(entry, ["processing"], { + status: "pending", + attempts: Math.max(0, entry.attempts - 1), + nextAttemptAt, + lastError: note, + lockedUntil: null, + envelopeHash: null, + }); + } + + async requeueDead(id: string): Promise { + const result = await this.prisma.onchainOutbox.updateMany({ + where: { id: BigInt(id), status: "dead" }, + data: { + status: "pending", + attempts: 0, + nextAttemptAt: new Date(), + envelopeHash: null, + txHash: null, + }, + }); + return result.count > 0; + } + + private async fenced( + entry: OutboxEntry, + from: OutboxStatus[], + data: Prisma.OnchainOutboxUpdateManyMutationInput, + ): Promise { + const result = await this.prisma.onchainOutbox.updateMany({ + where: { id: BigInt(entry.id), attempts: entry.attempts, status: { in: from } }, + data, + }); + return result.count > 0; + } +} + +function fromModel(row: OnchainOutbox): OutboxEntry { + return { + id: row.id.toString(), + intentId: row.intentId, + operation: row.operation as OutboxOperation, + payload: (row.payload ?? {}) as Record, + status: row.status as OutboxStatus, + attempts: row.attempts, + nextAttemptAt: row.nextAttemptAt, + lockedUntil: row.lockedUntil ?? undefined, + envelopeHash: row.envelopeHash ?? undefined, + txHash: row.txHash ?? undefined, + lastError: row.lastError ?? undefined, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function fromRaw(row: RawOutboxRow): OutboxEntry { + return fromModel({ + id: row.id, + intentId: row.intent_id, + operation: row.operation, + payload: row.payload, + status: row.status, + attempts: row.attempts, + nextAttemptAt: row.next_attempt_at, + lockedUntil: row.locked_until, + envelopeHash: row.envelope_hash, + txHash: row.tx_hash, + lastError: row.last_error, + createdAt: row.created_at, + updatedAt: row.updated_at, + }); +} diff --git a/src/soroban/solver-registry.service.spec.ts b/src/soroban/solver-registry.service.spec.ts index 043fafdb..cfea2726 100644 --- a/src/soroban/solver-registry.service.spec.ts +++ b/src/soroban/solver-registry.service.spec.ts @@ -78,6 +78,8 @@ function makeConfigService( eventLoopMaxLagMs: 1000, rpcHealthUrls: ["https://soroban-testnet.stellar.org"], }, + outbox: { relayEnabled: false, relayIntervalMs: 2000, batchSize: 10, maxAttempts: 8, leaseSeconds: 120 }, + slashing: { challengeWindowSeconds: 600, clockSkewToleranceSeconds: 30, maxSubmitAttempts: 5 }, }; return { get: (key: string) => { diff --git a/src/soroban/solver-registry.service.ts b/src/soroban/solver-registry.service.ts index 50d513fb..59a35a49 100644 --- a/src/soroban/solver-registry.service.ts +++ b/src/soroban/solver-registry.service.ts @@ -45,6 +45,12 @@ export interface SlashResult { * depending on whether the contract is configured. */ dryRun: boolean; + /** + * true when the call errored (RPC failure, simulation error) and should be + * retried by the caller. false for successful, dry-run, and unconfigured + * (no-op) outcomes (issue #397). + */ + failed: boolean; } /** @@ -117,6 +123,7 @@ export class SolverRegistryService { submitted: false, simulated: false, dryRun: true, + failed: false, detail: "ONCHAIN_DRY_RUN=true — simulated log only, no transaction submitted", }; } @@ -128,7 +135,7 @@ export class SolverRegistryService { this.logger.log( `[solver-registry] would slash solver=${params.solverAddress} intent=${params.intentId} reason="${params.reason}" (${detail})`, ); - return { submitted: false, simulated: false, dryRun: false, detail }; + return { submitted: false, simulated: false, dryRun: false, failed: false, detail }; } try { @@ -158,7 +165,7 @@ export class SolverRegistryService { this.logger.error( `[solver-registry] slash simulation errored for solver=${params.solverAddress} intent=${params.intentId}: ${detail}`, ); - return { submitted: false, simulated: true, dryRun: false, detail }; + return { submitted: false, simulated: true, dryRun: false, failed: true, detail }; } // TODO: Once issue #23 confirms the real contract interface, replace @@ -172,7 +179,7 @@ export class SolverRegistryService { this.logger.log( `[solver-registry] simulated slash tx for solver=${params.solverAddress} intent=${params.intentId} (${detail})`, ); - return { submitted: false, simulated: true, dryRun: false, detail }; + return { submitted: false, simulated: true, dryRun: false, failed: false, detail }; } catch (err) { // Issue #300 — the SDK may include serialized transaction/XDR details in // thrown errors; do not log the signing key or any raw secret here. @@ -180,7 +187,7 @@ export class SolverRegistryService { this.logger.error( `[solver-registry] slash call errored for solver=${params.solverAddress} intent=${params.intentId}: ${detail}`, ); - return { submitted: false, simulated: false, dryRun: false, detail }; + return { submitted: false, simulated: false, dryRun: false, failed: true, detail }; } } diff --git a/src/soroban/soroban.module.ts b/src/soroban/soroban.module.ts index f8b97d06..086465e3 100644 --- a/src/soroban/soroban.module.ts +++ b/src/soroban/soroban.module.ts @@ -10,6 +10,7 @@ import { SolverRegistryService } from "./solver-registry.service"; import { SignerService } from "./signer.service"; import { StellarTxService } from "./stellar-tx.service"; import { TxConfirmationService } from "./tx-confirmation.service"; +import { FillVerifierService } from "./fill-verifier.service"; import { SolverRegistryEventsService } from "./events/solver-registry-events.service"; import { SIGNER_TOKEN, signerFactory } from "./signers/signer.factory"; import { SolversModule } from "../solvers/solvers.module"; @@ -67,6 +68,8 @@ import { IntentsModule } from "../intents/intents.module"; // Issue #401 — shadow-mode divergence monitor. Exported so IntentsService // can report off-chain transitions to it without importing Soroban internals. ShadowService, + // Issue #397 — slashing saga's on-chain fill re-verification. + FillVerifierService, ], exports: [ SorobanService, @@ -77,6 +80,7 @@ import { IntentsModule } from "../intents/intents.module"; EventIngestionService, SolverRegistryEventsService, ShadowService, + FillVerifierService, ], }) export class SorobanModule {} diff --git a/src/soroban/stellar-tx.before-submit.spec.ts b/src/soroban/stellar-tx.before-submit.spec.ts new file mode 100644 index 00000000..d9f0b3b9 --- /dev/null +++ b/src/soroban/stellar-tx.before-submit.spec.ts @@ -0,0 +1,96 @@ +import { ConfigService } from "@nestjs/config"; +import { Keypair, Networks, Transaction } from "@stellar/stellar-sdk"; +import { AppConfig } from "../config/configuration"; +import { KillSwitchService } from "../killswitch/killswitch.service"; +import { SignerService } from "./signer.service"; +import { SorobanService } from "./soroban.service"; +import { INVOKE_TX_TIMEOUT_SECONDS, StellarTxService } from "./stellar-tx.service"; +import { TxConfirmationService } from "./tx-confirmation.service"; + +const CONTRACT_ID = "CBIELTK6YBZJU5UP2WWQEUCYKLPU6AUNZ2BQ4WWFEIE3USCIHMXQDAMA"; + +/** + * Issue #396 — the outbox relay persists the signed envelope hash through + * `invokeContract`'s `beforeSubmit` hook, which must run after signing and + * strictly before broadcast. + */ +describe("StellarTxService.invokeContract beforeSubmit (#396)", () => { + const signerKeypair = Keypair.random(); + let events: string[]; + let submitTransaction: jest.Mock; + let service: StellarTxService; + + beforeEach(() => { + events = []; + submitTransaction = jest.fn(async (tx: Transaction) => { + events.push("submit"); + return { status: "PENDING", hash: tx.hash().toString("hex") }; + }); + const soroban = { + getFeeStats: jest.fn().mockRejectedValue(new Error("no stats")), + simulateTransaction: jest.fn().mockResolvedValue({ minResourceFee: "0", latestLedger: 1 }), + prepareTransaction: jest.fn(async (tx: Transaction) => tx), + submitTransaction, + }; + const signer = { + withNextSequence: (fn: (seq: string) => Promise) => fn("100"), + getPublicKey: () => signerKeypair.publicKey(), + getNetworkPassphrase: () => Networks.TESTNET, + sign: async (tx: Transaction) => { + tx.sign(signerKeypair); + return tx; + }, + }; + const confirmation = { + waitForConfirmation: jest.fn(async (hash: string) => ({ hash, status: "SUCCESS", durationMs: 1 })), + }; + const config = { + get: (key: string) => + ({ onchainDryRun: false, "stellar.network": "testnet", "stellar.feePercentile": "p50" })[key], + }; + service = new StellarTxService( + soroban as unknown as SorobanService, + signer as unknown as SignerService, + confirmation as unknown as TxConfirmationService, + config as unknown as ConfigService, + undefined, + { evaluateTarget: () => ({ paused: false }) } as unknown as KillSwitchService, + ); + }); + + it("hands the signed envelope hash to beforeSubmit before broadcasting", async () => { + let hookHash = ""; + const result = await service.invokeContract( + { contractId: CONTRACT_ID, method: "create_intent", args: [] }, + { + beforeSubmit: async (hash) => { + events.push("beforeSubmit"); + hookHash = hash; + }, + }, + ); + + expect(events).toEqual(["beforeSubmit", "submit"]); + const submitted = submitTransaction.mock.calls[0][0] as Transaction; + expect(submitted.signatures).toHaveLength(1); + expect(hookHash).toBe(submitted.hash().toString("hex")); + expect(result).toMatchObject({ hash: hookHash, status: "SUCCESS", dryRun: false }); + // The envelope's time bound is what makes NOT_FOUND-after-lease conclusive. + const maxTime = Number(submitted.timeBounds?.maxTime); + expect(maxTime - Math.floor(Date.now() / 1000)).toBeLessThanOrEqual(INVOKE_TX_TIMEOUT_SECONDS); + }); + + it("never broadcasts when beforeSubmit throws", async () => { + await expect( + service.invokeContract( + { contractId: CONTRACT_ID, method: "create_intent", args: [] }, + { + beforeSubmit: async () => { + throw new Error("lost lease"); + }, + }, + ), + ).rejects.toThrow("lost lease"); + expect(submitTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/src/soroban/stellar-tx.service.ts b/src/soroban/stellar-tx.service.ts index 7ff3d4e6..ddbbbba4 100644 --- a/src/soroban/stellar-tx.service.ts +++ b/src/soroban/stellar-tx.service.ts @@ -90,6 +90,24 @@ export interface InvokeContractParams { args: xdr.ScVal[]; } +/** + * Time bound (seconds) on every transaction built by {@link StellarTxService.invokeContract}. + * After this the network rejects the envelope, which is what lets the outbox + * relay treat a NOT_FOUND envelope hash as "never landed, safe to rebuild" + * once its processing lease (OUTBOX_LEASE_SECONDS) has expired (issue #396). + */ +export const INVOKE_TX_TIMEOUT_SECONDS = 30; + +export interface InvokeContractOptions { + /** + * Called with the signed envelope's hash after signing and *before* + * broadcast (issue #396). If it throws, nothing is submitted. The outbox + * relay uses this to durably record the hash so a crash mid-submit can be + * detected on retry instead of double-submitting. + */ + beforeSubmit?: (envelopeHash: string) => Promise; +} + export interface InvokeContractResult { hash: string; status: string; @@ -266,7 +284,10 @@ export class StellarTxService { * 3. Sign and submit the (now-prepared) original transaction. * 4. Confirm and return the result. */ - async invokeContract(params: InvokeContractParams): Promise { + async invokeContract( + params: InvokeContractParams, + options: InvokeContractOptions = {}, + ): Promise { // Issue #477 — the last gate before anything touches the chain. Checking // here rather than only in controllers also covers background callers (the // sweeper, event ingestion) that never pass through an HTTP guard. @@ -303,7 +324,7 @@ export class StellarTxService { .addOperation( new Contract(params.contractId).call(params.method, ...params.args), ) - .setTimeout(30) + .setTimeout(INVOKE_TX_TIMEOUT_SECONDS) .build(); let simulation = await this.sorobanService.simulateTransaction(rawTx); @@ -334,6 +355,8 @@ export class StellarTxService { const prepared = await this.sorobanService.prepareTransaction(rawTx); const signed = await this.signerService.sign(prepared as Transaction); + await options.beforeSubmit?.(signed.hash().toString("hex")); + const submittedAt = Date.now(); const sendResponse = await this.sorobanService.submitTransaction(signed); diff --git a/src/soroban/tx-confirmation.service.spec.ts b/src/soroban/tx-confirmation.service.spec.ts new file mode 100644 index 00000000..ad0bf96d --- /dev/null +++ b/src/soroban/tx-confirmation.service.spec.ts @@ -0,0 +1,23 @@ +import { SorobanRpc } from "@stellar/stellar-sdk"; +import { SorobanService } from "./soroban.service"; +import { TxConfirmationService } from "./tx-confirmation.service"; + +describe("TxConfirmationService", () => { + const getTransaction = jest.fn(); + const service = new TxConfirmationService({ getTransaction } as unknown as SorobanService); + + it.each([ + [SorobanRpc.Api.GetTransactionStatus.SUCCESS, { status: "success", ledger: 5, ledgerCloseTime: 100 }], + [SorobanRpc.Api.GetTransactionStatus.FAILED, { status: "failed", ledger: 5, ledgerCloseTime: 100 }], + [SorobanRpc.Api.GetTransactionStatus.NOT_FOUND, { status: "not_found" }], + ])("maps %s", async (status, expected) => { + getTransaction.mockResolvedValueOnce({ status, ledger: 5, createdAt: 100 }); + await expect(service.check("h")).resolves.toEqual(expected); + expect(getTransaction).toHaveBeenLastCalledWith("h"); + }); + + it("propagates transport errors instead of reporting not_found", async () => { + getTransaction.mockRejectedValueOnce(new Error("ECONNRESET")); + await expect(service.check("h")).rejects.toThrow("ECONNRESET"); + }); +}); diff --git a/src/soroban/tx-confirmation.service.ts b/src/soroban/tx-confirmation.service.ts index e8df7ada..c49a2447 100644 --- a/src/soroban/tx-confirmation.service.ts +++ b/src/soroban/tx-confirmation.service.ts @@ -19,6 +19,25 @@ import { MetricsService } from "../metrics/metrics.service"; const DEFAULT_POLL_INTERVAL_MS = 3_000; const DEFAULT_TIMEOUT_MS = 120_000; // 2 minutes +/** + * Normalized outcome of a single, non-blocking lookup ({@link TxConfirmationService.check}). + * + * success — applied successfully in a closed ledger. + * failed — included in a ledger but the invocation failed. + * not_found — unknown to the RPC node: still pending, dropped, or expired + * past its time bound. Callers decide which using their own + * notion of elapsed time. + */ +export type TxConfirmationStatus = "success" | "failed" | "not_found"; + +export interface TxConfirmation { + status: TxConfirmationStatus; + /** Ledger the transaction was included in (success/failed only). */ + ledger?: number; + /** Ledger close time, unix seconds (success/failed only). */ + ledgerCloseTime?: number; +} + export interface ConfirmationResult { hash: string; status: "SUCCESS" | "FAILED" | "TIMEOUT"; @@ -39,6 +58,29 @@ export class TxConfirmationService { @Optional() private readonly metricsService?: MetricsService, ) {} + /** + * Single, non-blocking lookup of `hash` (issues #396 / #397). + * + * For durable callers — the outbox relay and the slashing saga — that + * persist the hashes they wait on and re-check on their own schedule, so a + * restart never loses an in-flight transaction and a worker tick never + * blocks for the full {@link waitForConfirmation} timeout. RPC transport + * errors propagate so callers can retry rather than mistake an outage for + * NOT_FOUND. + */ + async check(hash: string): Promise { + const response = await this.sorobanService.getTransaction(hash); + switch (response.status) { + case SorobanRpc.Api.GetTransactionStatus.SUCCESS: + return { status: "success", ledger: response.ledger, ledgerCloseTime: response.createdAt }; + case SorobanRpc.Api.GetTransactionStatus.FAILED: + this.logger.warn(`[tx-confirmation] tx ${hash} failed in ledger ${response.ledger}`); + return { status: "failed", ledger: response.ledger, ledgerCloseTime: response.createdAt }; + default: + return { status: "not_found" }; + } + } + /** * Poll until the transaction identified by `hash` reaches a terminal state. *