Skip to content

Commit 7e688ba

Browse files
committed
fix(billing): let operators resolve billing dead letters; skip Stripe syncs only on positive evidence
A seat sync whose Stripe subscription could not be found retried into a dead letter, and the hourly billing reconcile cron logged it at ERROR forever: nothing could close a dead letter, and retention never prunes one. - POST /api/v1/admin/outbox/[id]/resolve closes a dead-lettered billing Stripe event (status dead_letter -> completed). last_error keeps who resolved it, why, and the last failure; the action is audited. Other domains count completed rows as delivered, so only billing Stripe events can be resolved. - Outbox handlers can return skipOutboxHandler(reason) to complete with the reason kept in last_error. Billing uses it only on positive evidence: the cancel_at_period_end sync when Stripe reports the subscription canceled or incomplete_expired, and the contact sync when Stripe reports the customer deleted. A resource_missing subscription or customer still retries into a dead letter: live Stripe keeps canceled subscriptions retrievable, so a 404 points at a wrong key, account, or mode and must fail loudly. - The admin dashboard reports a cancellation that completed with a reason as resolved, never applied. - The reconcile cron logs dead letters from the last two runs at ERROR and older unresolved ones at WARN; the dead-letter scan returns newest first. - The in-memory Stripe throws SDK-shaped resource_missing errors with Stripe's path params, keeps canceled subscriptions retrievable, rejects updates to ended ones, and returns deleted customers as stubs.
1 parent 1f570d3 commit 7e688ba

12 files changed

Lines changed: 594 additions & 39 deletions

File tree

‎apps/sim/app/api/cron/reconcile-billing-seats/route.ts‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { createLogger } from '@sim/logger'
2+
import { toStringOrNull } from '@sim/utils/coerce'
23
import { toError } from '@sim/utils/errors'
4+
import { toRecord } from '@sim/utils/object'
35
import { type NextRequest, NextResponse } from 'next/server'
46
import { verifyCronAuth } from '@/lib/auth/internal'
57
import { reconcileTeamSeatDrift } from '@/lib/billing/organizations/seat-drift'
@@ -17,6 +19,22 @@ const BILLING_SYNC_EVENT_TYPES = [
1719
OUTBOX_EVENT_TYPES.STRIPE_SYNC_CANCEL_AT_PERIOD_END,
1820
]
1921

22+
/**
23+
* Dead letters newer than two hourly runs log at ERROR, so one missed run cannot hide one; older
24+
* ones repeat at WARN until requeued or resolved through the admin outbox API.
25+
*/
26+
const NEW_DEAD_LETTER_WINDOW_MS = 2 * 60 * 60 * 1000
27+
28+
function describeDeadLetter(event: Awaited<ReturnType<typeof findDeadLetteredEvents>>[number]) {
29+
return {
30+
id: event.id,
31+
eventType: event.eventType,
32+
subscriptionId: toStringOrNull(toRecord(event.payload).subscriptionId),
33+
deadLetteredAt: event.processedAt?.toISOString() ?? null,
34+
lastError: event.lastError,
35+
}
36+
}
37+
2038
/**
2139
* Periodic billing-seat reconciliation. Self-heals Team organizations whose
2240
* stored seat count drifted from their member count, and reports any
@@ -38,18 +56,28 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
3856
const drift = await reconcileTeamSeatDrift()
3957

4058
const deadLettered = await findDeadLetteredEvents(BILLING_SYNC_EVENT_TYPES)
41-
if (deadLettered.length > 0) {
59+
const newSince = Date.now() - NEW_DEAD_LETTER_WINDOW_MS
60+
const isNew = (event: (typeof deadLettered)[number]) =>
61+
event.processedAt !== null && event.processedAt.getTime() >= newSince
62+
const newlyDeadLettered = deadLettered.filter(isNew)
63+
const previouslyReported = deadLettered.filter((event) => !isNew(event))
64+
if (newlyDeadLettered.length > 0) {
4265
logger.error(
4366
'Dead-lettered billing sync events require manual remediation — a billing state change (seat charge or cancellation) never reached Stripe',
4467
{
4568
requestId,
46-
count: deadLettered.length,
47-
events: deadLettered.map((event) => ({
48-
id: event.id,
49-
eventType: event.eventType,
50-
subscriptionId: (event.payload as { subscriptionId?: string } | null)?.subscriptionId,
51-
lastError: event.lastError,
52-
})),
69+
count: newlyDeadLettered.length,
70+
events: newlyDeadLettered.map(describeDeadLetter),
71+
}
72+
)
73+
}
74+
if (previouslyReported.length > 0) {
75+
logger.warn(
76+
'Previously reported billing sync dead letters are still unresolved — requeue or resolve them through the admin outbox API',
77+
{
78+
requestId,
79+
count: previouslyReported.length,
80+
events: previouslyReported.map(describeDeadLetter),
5381
}
5482
)
5583
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
2+
import { createLogger } from '@sim/logger'
3+
import { toError } from '@sim/utils/errors'
4+
import { NextResponse } from 'next/server'
5+
import { adminV1ResolveOutboxEventContract } from '@/lib/api/contracts/v1/admin'
6+
import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
7+
import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events'
8+
import { resolveDeadLetteredOutboxEvent } from '@/lib/core/outbox/service'
9+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
10+
import { withAdminAuthParams } from '@/app/api/v1/admin/middleware'
11+
12+
const logger = createLogger('AdminOutboxResolveAPI')
13+
14+
export const dynamic = 'force-dynamic'
15+
16+
/**
17+
* Billing Stripe events only: their readers tell a resolved row from delivered work (the admin
18+
* dashboard reports a completed cancellation with a `lastError` as `resolved`). Other domains
19+
* count every `completed` row as delivered and retry through their own admin actions.
20+
*/
21+
const RESOLVABLE_EVENT_TYPES = Object.values(OUTBOX_EVENT_TYPES)
22+
23+
const invalidOutboxEventResponse = (message: string) =>
24+
NextResponse.json({ success: false, error: message }, { status: 400 })
25+
26+
/**
27+
* POST /api/v1/admin/outbox/[id]/resolve
28+
*
29+
* Close a dead-lettered billing Stripe event an operator has remediated by hand or decided needs
30+
* nothing (see `resolveDeadLetteredOutboxEvent`). Use `/requeue` to retry one instead.
31+
*
32+
* Body: `{ "reason": string, "resolvedBy": string }`
33+
*/
34+
export const POST = withRouteHandler(
35+
withAdminAuthParams<{ id: string }>(async (request, context) => {
36+
const parsed = await parseRequest(adminV1ResolveOutboxEventContract, request, context, {
37+
validationErrorResponse: (error) =>
38+
invalidOutboxEventResponse(getValidationErrorMessage(error, 'Invalid resolve request')),
39+
invalidJsonResponse: () => invalidOutboxEventResponse('Request body must be valid JSON'),
40+
})
41+
if (!parsed.success) return parsed.response
42+
43+
const { id } = parsed.data.params
44+
45+
try {
46+
const resolved = await resolveDeadLetteredOutboxEvent(
47+
id,
48+
RESOLVABLE_EVENT_TYPES,
49+
parsed.data.body
50+
)
51+
if (!resolved) {
52+
return NextResponse.json(
53+
{
54+
success: false,
55+
error:
56+
'Event not found, not in dead_letter status, or not a billing Stripe event. Only dead-lettered billing Stripe events can be resolved.',
57+
},
58+
{ status: 404 }
59+
)
60+
}
61+
62+
recordAudit({
63+
workspaceId: null,
64+
actorId: null,
65+
actorName: `Admin API (${parsed.data.body.resolvedBy})`,
66+
action: AuditAction.BILLING_SYNC_RESOLVED,
67+
resourceType: AuditResourceType.BILLING,
68+
resourceId: resolved.id,
69+
description: `Admin API resolved a dead-lettered ${resolved.eventType} event`,
70+
metadata: { eventType: resolved.eventType, ...parsed.data.body },
71+
request,
72+
})
73+
74+
return NextResponse.json({ success: true, resolved })
75+
} catch (error) {
76+
logger.error('Failed to resolve outbox event', { eventId: id, error: toError(error).message })
77+
return NextResponse.json(
78+
{ success: false, error: 'Failed to resolve outbox event' },
79+
{ status: 500 }
80+
)
81+
}
82+
})
83+
)

‎apps/sim/lib/admin/subscription-lifecycle.ts‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,24 @@ async function listRecentSubscriptionPayments(stripe: Stripe, stripeSubscription
177177
}
178178
}
179179

180+
/**
181+
* Where a dashboard cancellation event stands. One that completed with a `lastError` never
182+
* applied as requested: its handler found the Stripe subscription already ended, or an operator
183+
* resolved its dead letter. It reads as `resolved` with that reason, never as `applied`.
184+
*/
185+
function cancellationSyncOutcome(status: string, lastError: string | null) {
186+
if (status === 'completed') {
187+
return lastError
188+
? { status: 'resolved' as const, error: lastError }
189+
: { status: 'applied' as const, error: null }
190+
}
191+
if (status === 'dead_letter') return { status: 'failed' as const, error: lastError }
192+
return {
193+
status: status === 'processing' ? ('processing' as const) : ('pending' as const),
194+
error: null,
195+
}
196+
}
197+
180198
async function getDashboardCancellationSync(organizationId: string, subscriptionId: string) {
181199
const [row] = await db
182200
.select({
@@ -205,15 +223,7 @@ async function getDashboardCancellationSync(organizationId: string, subscription
205223
row.eventType === OUTBOX_EVENT_TYPES.STRIPE_CANCEL_SUBSCRIPTION_IMMEDIATELY
206224
? ('immediate' as const)
207225
: ('period_end' as const),
208-
status:
209-
row.status === 'completed'
210-
? ('applied' as const)
211-
: row.status === 'dead_letter'
212-
? ('failed' as const)
213-
: row.status === 'processing'
214-
? ('processing' as const)
215-
: ('pending' as const),
216-
error: row.status === 'dead_letter' ? row.error : null,
226+
...cancellationSyncOutcome(row.status, row.error),
217227
}
218228
}
219229

@@ -286,6 +296,7 @@ export async function requestDashboardSubscriptionCancellation({
286296
id: outboxEvent.id,
287297
eventType: outboxEvent.eventType,
288298
status: outboxEvent.status,
299+
lastError: outboxEvent.lastError,
289300
subscriptionId: sql<string>`${outboxEvent.payload} ->> 'subscriptionId'`,
290301
reason: sql<string | null>`${outboxEvent.payload} ->> 'reason'`,
291302
})
@@ -351,12 +362,8 @@ export async function requestDashboardSubscriptionCancellation({
351362
operationId,
352363
outboxEventId: existingOperation.id,
353364
subscriptionId: existingOperation.subscriptionId,
354-
status:
355-
existingOperation.status === 'completed'
356-
? ('applied' as const)
357-
: existingOperation.status === 'processing'
358-
? ('processing' as const)
359-
: ('pending' as const),
365+
status: cancellationSyncOutcome(existingOperation.status, existingOperation.lastError)
366+
.status,
360367
}
361368
}
362369

‎apps/sim/lib/api/contracts/v1/admin/dashboard.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ const adminDashboardBillingActionsSchema = z.object({
663663
.object({
664664
operationId: z.string().uuid(),
665665
timing: z.enum(['period_end', 'immediate']),
666-
status: z.enum(['pending', 'processing', 'applied', 'failed']),
666+
status: z.enum(['pending', 'processing', 'applied', 'resolved', 'failed']),
667667
error: z.string().nullable(),
668668
})
669669
.nullable(),
@@ -684,7 +684,7 @@ const adminDashboardBillingActionsSchema = z.object({
684684
const adminDashboardCancellationResultSchema = z.object({
685685
success: z.literal(true),
686686
operationId: z.string(),
687-
status: z.enum(['pending', 'processing', 'applied', 'failed']),
687+
status: z.enum(['pending', 'processing', 'applied', 'resolved', 'failed']),
688688
})
689689
const adminDashboardRefundResultSchema = z.object({
690690
success: z.literal(true),

‎apps/sim/lib/api/contracts/v1/admin/outbox.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,28 @@ const adminV1OutboxRequeueResultSchema = z.object({
6262
}),
6363
})
6464

65+
const adminV1ResolveOutboxEventBodySchema = z.object({
66+
reason: z
67+
.string({ error: 'reason is required' })
68+
.trim()
69+
.min(1, 'reason is required')
70+
.max(500, 'reason must be at most 500 characters'),
71+
resolvedBy: z
72+
.string({ error: 'resolvedBy is required' })
73+
.trim()
74+
.min(1, 'resolvedBy is required')
75+
.max(100, 'resolvedBy must be at most 100 characters'),
76+
})
77+
78+
const adminV1OutboxResolveResultSchema = z.object({
79+
success: z.literal(true),
80+
resolved: z.object({
81+
id: z.string(),
82+
eventType: z.string(),
83+
lastError: z.string().nullable(),
84+
}),
85+
})
86+
6587
export const adminV1ListOutboxContract = defineRouteContract({
6688
method: 'GET',
6789
path: '/api/v1/admin/outbox',
@@ -82,6 +104,17 @@ export const adminV1RequeueOutboxEventContract = defineRouteContract({
82104
},
83105
})
84106

107+
export const adminV1ResolveOutboxEventContract = defineRouteContract({
108+
method: 'POST',
109+
path: '/api/v1/admin/outbox/[id]/resolve',
110+
params: adminV1IdParamsSchema,
111+
body: adminV1ResolveOutboxEventBodySchema,
112+
response: {
113+
mode: 'json',
114+
schema: adminV1OutboxResolveResultSchema,
115+
},
116+
})
117+
85118
export type AdminV1ListOutboxQueryInput = ContractQueryInput<typeof adminV1ListOutboxContract>
86119
export type AdminV1ListOutboxQuery = ContractQuery<typeof adminV1ListOutboxContract>
87120
export type AdminV1RequeueOutboxEventParams = ContractParams<

‎apps/sim/lib/billing/webhooks/outbox-handlers.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {
1717
readRecordedSyncValue,
1818
type SubscriptionSeatsSyncPayload,
1919
} from '@/lib/billing/webhooks/subscription-sync'
20-
import type { OutboxHandler } from '@/lib/core/outbox/service'
20+
import { type OutboxHandler, skipOutboxHandler } from '@/lib/core/outbox/service'
2121

2222
const logger = createLogger('BillingOutboxHandlers')
2323

@@ -35,6 +35,14 @@ function syncWriteIdempotencyKey(eventId: string): string {
3535
return `outbox:${eventId}:${generateShortId()}`
3636
}
3737

38+
/**
39+
* Stripe subscription states that accept no further cancellation changes, so a sync that sees one
40+
* ends instead of retrying. A subscription Stripe cannot find is not one: live Stripe keeps
41+
* canceled subscriptions retrievable, so `resource_missing` means a wrong key, account, or mode
42+
* and dead-letters for an operator.
43+
*/
44+
const ENDED_STRIPE_SUBSCRIPTION_STATUSES = new Set(['canceled', 'incomplete_expired'])
45+
3846
interface StripeCancelSubscriptionImmediatelyPayload {
3947
stripeSubscriptionId: string
4048
subscriptionId: string
@@ -149,6 +157,11 @@ const stripeSyncCancelAtPeriodEnd: OutboxHandler<CancelAtPeriodEndSyncPayload> =
149157
}
150158

151159
const stripeSubscription = await stripe.subscriptions.retrieve(payload.stripeSubscriptionId)
160+
if (ENDED_STRIPE_SUBSCRIPTION_STATUSES.has(stripeSubscription.status)) {
161+
return skipOutboxHandler(
162+
`Stripe subscription ${payload.stripeSubscriptionId} has already ended (${stripeSubscription.status})`
163+
)
164+
}
152165
const needsUpdate = stripeSubscription.cancel_at_period_end !== desiredValue
153166
if (needsUpdate) {
154167
await stripe.subscriptions.update(
@@ -495,7 +508,7 @@ const stripeSyncCustomerContact: OutboxHandler<StripeSyncCustomerContactPayload>
495508

496509
const customer = await stripe.customers.retrieve(contact.stripeCustomerId)
497510
if (customer.deleted) {
498-
throw new Error(`Stripe customer ${contact.stripeCustomerId} is deleted`)
511+
return skipOutboxHandler(`Stripe customer ${contact.stripeCustomerId} is deleted`)
499512
}
500513
const needsUpdate =
501514
customer.email !== contact.email || Boolean(contact.name && customer.name !== contact.name)

0 commit comments

Comments
 (0)