Skip to content

Commit 22c5789

Browse files
committed
fix(trigger): do not alert on runs whose workspace was deleted mid-run
When an account deletion cascades a workspace (or its workflow) away while a run is in flight, #9144 makes LoggingSession record the deleted scope and core throw ExecutionScopeDeletedError when the scope is already gone at start. The background tasks still rethrew both cases: a start-time ExecutionScopeDeletedError is not "recorded by core", and a block that then fails (e.g. a file-upload FK error) cannot be finalized because the log row is gone, so hasCompleted() is false. The Trigger run FAILED and paged Slack for a workspace that no longer exists. workflow-execution and webhook-execution now treat a deleted scope as a quiet outcome: when the error is ExecutionScopeDeletedError, or the session reports a deleted scope once core's post-execution settles (or once the task's own failure log discovers it), they log at info and complete with the #9139 failure result. The deleted-scope check runs before the timeout rethrow: with the workspace gone there is no owner to alert and no log to fix, and the admission slot is already released. Live-workspace timeouts, unrecorded failures, and infra faults still throw. The /api/jobs projection reads the string `error` as `failed`, matching the FAILED status these runs reported before. schedule-execution already swallows execution errors into schedule bookkeeping (no Trigger failure), and resume-execution has no task-level logging session, so neither changes here.
1 parent e5feb08 commit 22c5789

5 files changed

Lines changed: 253 additions & 12 deletions

File tree

‎apps/sim/background/async-preprocessing-correlation.test.ts‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ import {
1919
createSessionPrincipal,
2020
createWorkspaceApiKeyPrincipal,
2121
} from '@sim/testing/factories/principal.factory'
22-
import { beforeEach, describe, expect, it, vi } from 'vitest'
22+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
2323
import { ADMISSION_ERROR_CODE } from '@/lib/core/admission/transient-failure'
24+
import { ExecutionScopeDeletedError } from '@/lib/core/errors/execution-scope-deleted'
2425

2526
const {
2627
mockExecuteWorkflowCore,
@@ -440,6 +441,103 @@ describe('async preprocessing correlation threading', () => {
440441
})
441442
})
442443

444+
describe('a run whose workspace was deleted', () => {
445+
afterEach(() => {
446+
loggingSessionMockFns.mockGetDeletedScope.mockReset()
447+
loggingSessionMockFns.mockSafeCompleteWithError.mockReset()
448+
})
449+
450+
function runInDeletedWorkspace(executionId: string) {
451+
mockPreprocessExecution.mockResolvedValueOnce({
452+
success: true,
453+
actorUserId: 'actor-1',
454+
workflowRecord: {
455+
id: 'workflow-1',
456+
userId: 'owner-1',
457+
workspaceId: 'workspace-1',
458+
variables: {},
459+
},
460+
billingAttribution,
461+
executionTimeout: {},
462+
})
463+
return executeWorkflowJob({
464+
principal,
465+
workflowId: 'workflow-1',
466+
userId: 'actor-1',
467+
workspaceId: 'workspace-1',
468+
billingAttribution,
469+
triggerType: 'api',
470+
executionId,
471+
requestId: 'request-deleted',
472+
})
473+
}
474+
475+
it('completes without alerting when core finds the workspace gone at start', async () => {
476+
mockExecuteWorkflowCore.mockRejectedValueOnce(
477+
new ExecutionScopeDeletedError('execution-deleted-start', 'workspace')
478+
)
479+
480+
await expect(runInDeletedWorkspace('execution-deleted-start')).resolves.toMatchObject({
481+
success: false,
482+
executionId: 'execution-deleted-start',
483+
error: 'The workspace of execution execution-deleted-start was deleted',
484+
})
485+
expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled()
486+
expect(workflowExecutionLogger.info).toHaveBeenCalledWith(
487+
'[request-deleted] Workflow execution ended: its workspace was deleted',
488+
{ workflowId: 'workflow-1', executionId: 'execution-deleted-start' }
489+
)
490+
})
491+
492+
it('completes without alerting when a block fails after the workspace is deleted mid-run', async () => {
493+
const blockError = new Error('insert or update on table "workspace_files" violates FK')
494+
mockExecuteWorkflowCore.mockRejectedValueOnce(blockError)
495+
let deletedScope: 'workspace' | null = null
496+
loggingSessionMockFns.mockGetDeletedScope.mockImplementation(() => deletedScope)
497+
loggingSessionMockFns.mockWaitForPostExecution.mockImplementationOnce(async () => {
498+
await Promise.resolve()
499+
deletedScope = 'workspace'
500+
})
501+
502+
await expect(runInDeletedWorkspace('execution-deleted-mid')).resolves.toMatchObject({
503+
success: false,
504+
executionId: 'execution-deleted-mid',
505+
error: blockError.message,
506+
})
507+
expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled()
508+
})
509+
510+
it('completes without alerting when its own failure log finds the workspace gone', async () => {
511+
const pipelineError = new Error('Workflow state not found')
512+
mockExecuteWorkflowCore.mockRejectedValueOnce(pipelineError)
513+
loggingSessionMockFns.mockSafeCompleteWithError.mockImplementationOnce(async () => {
514+
loggingSessionMockFns.mockGetDeletedScope.mockReturnValue('workspace')
515+
})
516+
517+
await expect(runInDeletedWorkspace('execution-deleted-log')).resolves.toMatchObject({
518+
success: false,
519+
error: pipelineError.message,
520+
})
521+
expect(loggingSessionMockFns.mockSafeCompleteWithError).toHaveBeenCalledTimes(1)
522+
})
523+
524+
it('completes without alerting when a run in a deleted workspace times out', async () => {
525+
mockIsWorkflowTimedOut.mockReturnValue(true)
526+
mockExecuteWorkflowCore.mockResolvedValueOnce({
527+
success: false,
528+
status: 'cancelled',
529+
output: undefined,
530+
metadata: { duration: 120_000, userId: 'actor-1' },
531+
})
532+
loggingSessionMockFns.mockGetDeletedScope.mockReturnValue('workspace')
533+
534+
await expect(runInDeletedWorkspace('execution-deleted-timeout')).resolves.toMatchObject({
535+
success: false,
536+
error: 'Execution timed out after 2 minutes',
537+
})
538+
})
539+
})
540+
443541
it('fails the backing job after a cooperative workflow timeout is finalized', async () => {
444542
mockPreprocessExecution.mockResolvedValueOnce({
445543
success: true,

‎apps/sim/background/webhook-execution.test.ts‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,16 @@ import {
2424
workflowsPersistenceUtilsMockFns,
2525
} from '@sim/testing/mocks/workflows-persistence-utils.mock'
2626
import type { Mock } from 'vitest'
27-
import { afterAll, beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest'
27+
import {
28+
afterAll,
29+
afterEach,
30+
beforeEach,
31+
describe,
32+
expect,
33+
it,
34+
type MockInstance,
35+
vi,
36+
} from 'vitest'
2837
import { getBlock } from '@/blocks'
2938

3039
const {
@@ -119,6 +128,7 @@ vi.mock('@/tools/safe-assign', () => ({ safeAssign: vi.fn() }))
119128
vi.mock('@/triggers', () => triggersMock)
120129

121130
import * as usageReservation from '@/lib/billing/calculations/usage-reservation'
131+
import { ExecutionScopeDeletedError } from '@/lib/core/errors/execution-scope-deleted'
122132
import { isRetryableSetupError } from '@/lib/core/errors/retryable-infrastructure'
123133
import { executeWebhookJob, type WebhookExecutionPayload } from '@/background/webhook-execution'
124134

@@ -239,6 +249,7 @@ describe('executeWebhookJob fault vs error handling', () => {
239249
safeComplete: loggingSessionMockFns.mockSafeComplete,
240250
safeCompleteWithError: loggingSessionMockFns.mockSafeCompleteWithError,
241251
waitForPostExecution: loggingSessionMockFns.mockWaitForPostExecution,
252+
getDeletedScope: loggingSessionMockFns.mockGetDeletedScope,
242253
markAsFailed: loggingSessionMockFns.mockMarkAsFailed,
243254
setExecutionDeadlineAt: loggingSessionMockFns.mockSetExecutionDeadlineAt,
244255
setResolvedSecretTraceRegistry: mockSetResolvedSecretTraceRegistry,
@@ -501,6 +512,55 @@ describe('executeWebhookJob fault vs error handling', () => {
501512
expect(rawError.message).toContain(secret)
502513
})
503514

515+
describe('a run whose workspace was deleted', () => {
516+
afterEach(() => {
517+
loggingSessionMockFns.mockGetDeletedScope.mockReset()
518+
})
519+
520+
it('completes without alerting when core finds the workspace gone at start', async () => {
521+
mockExecuteWorkflowCore.mockRejectedValue(
522+
new ExecutionScopeDeletedError('execution-1', 'workspace')
523+
)
524+
mockWasExecutionFinalizedByCore.mockReturnValue(false)
525+
526+
await expect(executeWebhookJob(payload)).resolves.toMatchObject({
527+
success: false,
528+
executionId: 'execution-1',
529+
provider: 'gmail',
530+
})
531+
expect(loggingSessionMockFns.mockSafeStart).toHaveBeenCalledTimes(0)
532+
expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled()
533+
})
534+
535+
it('completes without alerting when a block fails after the workspace is deleted mid-run', async () => {
536+
mockExecuteWorkflowCore.mockRejectedValue(new Error('violates foreign key constraint'))
537+
mockWasExecutionFinalizedByCore.mockReturnValue(false)
538+
let deletedScope: 'workspace' | null = null
539+
loggingSessionMockFns.mockGetDeletedScope.mockImplementation(() => deletedScope)
540+
loggingSessionMockFns.mockWaitForPostExecution.mockImplementationOnce(async () => {
541+
await Promise.resolve()
542+
deletedScope = 'workspace'
543+
})
544+
545+
await expect(executeWebhookJob(payload)).resolves.toMatchObject({
546+
success: false,
547+
executionId: 'execution-1',
548+
})
549+
expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled()
550+
})
551+
552+
it('completes without alerting when its own failure log finds the workspace gone', async () => {
553+
mockExecuteWorkflowCore.mockRejectedValue(new Error('Workflow state not found'))
554+
mockWasExecutionFinalizedByCore.mockReturnValue(false)
555+
loggingSessionMockFns.mockSafeStart.mockImplementationOnce(async () => {
556+
loggingSessionMockFns.mockGetDeletedScope.mockReturnValue('workspace')
557+
return false
558+
})
559+
560+
await expect(executeWebhookJob(payload)).resolves.toMatchObject({ success: false })
561+
})
562+
})
563+
504564
it('executes against the deployment version admitted by webhook ingress', async () => {
505565
mockExecuteWorkflowCore.mockResolvedValue({
506566
success: true,

‎apps/sim/background/webhook-execution.ts‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import {
2525
import { getJobQueue } from '@/lib/core/async-jobs'
2626
import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
2727
import { env, envNumber } from '@/lib/core/config/env'
28+
import type { ExecutionScope } from '@/lib/core/errors/execution-scope-deleted'
2829
import { logFailureOnce } from '@/lib/core/errors/failure-log'
2930
import {
3031
describeRetryableInfrastructureError,
@@ -76,7 +77,10 @@ import {
7677
type PreloadedExecutionEnvironment,
7778
} from '@/lib/workflows/executor/execution-core'
7879
import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
79-
import { isRecordedWorkflowFailure } from '@/lib/workflows/executor/recorded-failure'
80+
import {
81+
isRecordedWorkflowFailure,
82+
resolveDeletedExecutionScope,
83+
} from '@/lib/workflows/executor/recorded-failure'
8084
import {
8185
loadDeployedWorkflowState,
8286
loadWorkflowDeploymentVersionState,
@@ -1277,19 +1281,33 @@ async function executeWebhookJobInternal(
12771281
executionId,
12781282
})
12791283

1284+
const failedRunResult = () => ({
1285+
success: false,
1286+
workflowId: payload.workflowId,
1287+
executionId,
1288+
output: hasExecutionResult(error) ? error.executionResult.output : {},
1289+
executedAt: new Date().toISOString(),
1290+
provider: payload.provider,
1291+
})
1292+
/** A run whose workspace or workflow was deleted has no owner left to alert and no log to write. */
1293+
const logDeletedScope = (scope: ExecutionScope) =>
1294+
logger.info(`[${requestId}] Webhook execution ended: its ${scope} was deleted`, {
1295+
workflowId: payload.workflowId,
1296+
executionId,
1297+
})
1298+
1299+
const deletedScope = await resolveDeletedExecutionScope(error, loggingSession)
1300+
if (deletedScope) {
1301+
logDeletedScope(deletedScope)
1302+
return failedRunResult()
1303+
}
1304+
12801305
// A failure inside workflow execution (block error, provider 4xx, missing required field, etc.)
12811306
// is finalized by core and already recorded in the execution logs. That is a user/workflow error,
12821307
// not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors
12831308
// that were not finalized came from the webhook pipeline itself, so we re-throw to fault below.
12841309
if (await isRecordedWorkflowFailure(error, executionId, loggingSession)) {
1285-
return {
1286-
success: false,
1287-
workflowId: payload.workflowId,
1288-
executionId,
1289-
output: hasExecutionResult(error) ? error.executionResult.output : {},
1290-
executedAt: new Date().toISOString(),
1291-
provider: payload.provider,
1292-
}
1310+
return failedRunResult()
12931311
}
12941312

12951313
try {
@@ -1332,6 +1350,12 @@ async function executeWebhookJobInternal(
13321350
)
13331351
}
13341352

1353+
const scopeDeletedOnCompletion = loggingSession.getDeletedScope()
1354+
if (scopeDeletedOnCompletion) {
1355+
logDeletedScope(scopeDeletedOnCompletion)
1356+
return failedRunResult()
1357+
}
1358+
13351359
if (retryableSetupCause) {
13361360
throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause })
13371361
}

‎apps/sim/background/workflow-execution.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
} from '@/lib/billing/core/attribution'
1919
import { buildRecordedJobFailureResult } from '@/lib/core/async-jobs/recorded-failure'
2020
import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
21+
import type { ExecutionScope } from '@/lib/core/errors/execution-scope-deleted'
2122
import { logFailureOnce } from '@/lib/core/errors/failure-log'
2223
import {
2324
capExecutionTimeoutMs,
@@ -35,7 +36,10 @@ import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
3536
import { cleanupExecutionBase64Cache } from '@/lib/uploads/utils/user-file-base64.server'
3637
import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core'
3738
import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
38-
import { isRecordedWorkflowFailure } from '@/lib/workflows/executor/recorded-failure'
39+
import {
40+
isRecordedWorkflowFailure,
41+
resolveDeletedExecutionScope,
42+
} from '@/lib/workflows/executor/recorded-failure'
3943
import { WORKFLOW_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits'
4044
import { ExecutionSnapshot } from '@/executor/execution/snapshot'
4145
import type { ExecutionMetadata } from '@/executor/execution/types'
@@ -126,6 +130,24 @@ function completeWithRecordedFailure(workflowId: string, executionId: string, er
126130
return buildRecordedJobFailureResult({ workflowId, executionId, error: failure.message })
127131
}
128132

133+
/**
134+
* Completes, without alerting, a job whose workspace or workflow was deleted: no owner is left to
135+
* act on an alert and no log is left to record the failure in.
136+
*/
137+
function completeWithDeletedScope(
138+
requestId: string,
139+
workflowId: string,
140+
executionId: string,
141+
scope: ExecutionScope,
142+
error: unknown
143+
) {
144+
logger.info(`[${requestId}] Workflow execution ended: its ${scope} was deleted`, {
145+
workflowId,
146+
executionId,
147+
})
148+
return completeWithRecordedFailure(workflowId, executionId, error)
149+
}
150+
129151
function requireLegacyWorkflowJobString(value: unknown, field: string): string {
130152
if (typeof value !== 'string' || !value.trim()) {
131153
throw new Error(`Legacy workflow job ${field} must be a non-empty string`)
@@ -336,6 +358,12 @@ export async function executeWorkflowJob(
336358
executionId,
337359
})
338360

361+
// Checked before the timeout: a deleted scope leaves nobody to alert about a slow run.
362+
const deletedScope = await resolveDeletedExecutionScope(error, loggingSession)
363+
if (deletedScope) {
364+
return completeWithDeletedScope(requestId, workflowId, executionId, deletedScope, error)
365+
}
366+
339367
if (error instanceof ExecutionTimeoutError) throw error
340368

341369
// A failure core already recorded is the workflow's outcome, not a job fault; a run past
@@ -359,6 +387,17 @@ export async function executeWorkflowJob(
359387
executionState: executionResult?.executionState,
360388
})
361389

390+
const scopeDeletedOnCompletion = loggingSession.getDeletedScope()
391+
if (scopeDeletedOnCompletion) {
392+
return completeWithDeletedScope(
393+
requestId,
394+
workflowId,
395+
executionId,
396+
scopeDeletedOnCompletion,
397+
error
398+
)
399+
}
400+
362401
// Complete only when the rejection is durably logged; an unsaved one must still alert.
363402
if (
364403
preprocessingRejection &&

‎apps/sim/lib/workflows/executor/recorded-failure.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import {
2+
type ExecutionScope,
3+
ExecutionScopeDeletedError,
4+
} from '@/lib/core/errors/execution-scope-deleted'
15
import type { LoggingSession } from '@/lib/logs/execution/logging-session'
26
import { wasExecutionFinalizedByCore } from '@/lib/workflows/executor/execution-core'
37

@@ -17,3 +21,19 @@ export async function isRecordedWorkflowFailure(
1721
await loggingSession.waitForPostExecution()
1822
return wasExecutionFinalizedByCore(error, executionId)
1923
}
24+
25+
/**
26+
* The run's workspace or workflow that was hard-deleted, or `null`. Core throws
27+
* {@link ExecutionScopeDeletedError} when the scope is gone at start; a scope
28+
* deleted mid-run surfaces only on the session once core's post-execution
29+
* finalization fails to write the log, so this settles that promise first.
30+
* Such a run has no owner left to alert and no log it could ever record.
31+
*/
32+
export async function resolveDeletedExecutionScope(
33+
error: unknown,
34+
loggingSession: Pick<LoggingSession, 'waitForPostExecution' | 'getDeletedScope'>
35+
): Promise<ExecutionScope | null> {
36+
if (error instanceof ExecutionScopeDeletedError) return error.scope
37+
await loggingSession.waitForPostExecution()
38+
return loggingSession.getDeletedScope()
39+
}

0 commit comments

Comments
 (0)