From 6723f38ce2e81bd793bb8d8018bf8a0633084584 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:29:05 +0900 Subject: [PATCH 1/4] fix(webv2): stop retrying an enqueue that keeps failing with a 500 Signed-off-by: kevin9327 <5299031+kevin9327@users.noreply.github.com> --- .../webv2/src/features/queue/runtime.test.ts | 62 +++++++++++++++++++ .../webv2/src/features/queue/runtime.ts | 28 +++++++++ 2 files changed, 90 insertions(+) diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts index 9f9e7a86260..7d493d88589 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts @@ -1570,6 +1570,68 @@ describe('queue runtime', () => { runtime.dispose(); }); + it('fails a run whose enqueue keeps returning a 500 instead of retrying forever', async () => { + const queueItem = createPendingQueueItem(); + const project = { id: 'project-1', queue: { items: [queueItem] } }; + const commands = createTestCommands(); + const enqueueGenerate = vi.fn().mockRejectedValue(new ApiError('graph validation crashed', 500)); + const runtime = createQueueRuntime({ + ...runtimeServices, + backend: createTestBackend({ enqueueGenerate, listItems: vi.fn().mockResolvedValue([]) }), + history: { + commands, + getSnapshot: () => ({ connectionStatus: 'connected', isHydrated: true, projects: [project] }), + subscribe: vi.fn(() => vi.fn()), + }, + journal: createTestJournal(), + }); + + runtime.start(); + + await vi.waitFor( + () => + expect(commands.setStatus).toHaveBeenCalledWith( + expect.objectContaining({ projectId: project.id, queueItemId: queueItem.id, status: 'failed' }) + ), + { timeout: 4_000 } + ); + expect(enqueueGenerate).toHaveBeenCalledTimes(3); + await new Promise((resolve) => { + setTimeout(resolve, 600); + }); + expect(enqueueGenerate).toHaveBeenCalledTimes(3); + await runtime.dispose(); + }); + + it('keeps retrying an enqueue that returns 503 while the server is unavailable', async () => { + const queueItem = createPendingQueueItem(); + const project = { id: 'project-1', queue: { items: [queueItem] } }; + const commands = createTestCommands(); + const enqueueGenerate = vi + .fn() + .mockRejectedValueOnce(new ApiError('unavailable', 503)) + .mockRejectedValueOnce(new ApiError('unavailable', 503)) + .mockRejectedValueOnce(new ApiError('unavailable', 503)) + .mockResolvedValue({ batchId: 'backend-batch', enqueued: 1, itemIds: [88], requested: 1 }); + const runtime = createQueueRuntime({ + ...runtimeServices, + backend: createTestBackend({ enqueueGenerate, listItems: vi.fn().mockResolvedValue([]) }), + history: { + commands, + getSnapshot: () => ({ connectionStatus: 'connected', isHydrated: true, projects: [project] }), + subscribe: vi.fn(() => vi.fn()), + }, + journal: createTestJournal(), + }); + + runtime.start(); + + await vi.waitFor(() => expect(commands.markBackendSubmitted).toHaveBeenCalled(), { timeout: 4_000 }); + expect(enqueueGenerate).toHaveBeenCalledTimes(4); + expect(commands.setStatus).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + await runtime.dispose(); + }); + it('keeps reconciling after three consecutive lost enqueue responses', async () => { const queueItem = createPendingQueueItem(); const project = { id: 'project-1', queue: { items: [queueItem] } }; diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.ts b/invokeai/frontend/webv2/src/features/queue/runtime.ts index faf82093555..be25e1e71b5 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.ts @@ -370,6 +370,16 @@ const getJournalSignature = (item: QueueItem): string => item.cancelledBackendItemIds, ]); +/** + * A 5xx answer to an enqueue is usually caused by the request itself (e.g. a graph that + * crashes server-side validation), so retrying the same submission forever never succeeds. + * Gateway and availability errors (502/503/504) stay transient and keep retrying. + */ +const MAX_SERVER_ERROR_SUBMISSIONS = 3; +const TRANSIENT_SERVER_STATUSES = [502, 503, 504]; +const isDeterministicServerError = (error: unknown): boolean => + error instanceof ApiError && error.status >= 500 && !TRANSIENT_SERVER_STATUSES.includes(error.status); + export const createQueueRuntime = ({ backend, destinations, @@ -426,6 +436,7 @@ export const createQueueRuntime = ({ const submissionOperations = new Map>(); const cancelRequestedRunKeys = new Set(); let reconcileRetryDelayMs = 100; + const serverErrorCounts = new Map(); let retryTimer: ReturnType | undefined; let hasInitialized = false; let isInitializing = false; @@ -1330,6 +1341,7 @@ export const createQueueRuntime = ({ await request .then(async ({ batchId, enqueued, itemIds, requested }) => { + serverErrorCounts.delete(runKey); if (!isAttemptCurrent(attempt)) { return; } @@ -1399,6 +1411,22 @@ export const createQueueRuntime = ({ }); return; } + if (isDeterministicServerError(error)) { + const failures = (serverErrorCounts.get(runKey) ?? 0) + 1; + if (failures >= MAX_SERVER_ERROR_SUBMISSIONS) { + serverErrorCounts.delete(runKey); + commands.setStatus({ + error: `The server failed this run ${failures} times in a row: ${toErrorMessage(error)}`, + projectId: project.id, + queueItemId: currentQueueItem.id, + status: 'failed', + }); + return; + } + serverErrorCounts.set(runKey, failures); + } else { + serverErrorCounts.delete(runKey); + } startedRunKeys.delete(runKey); reconcileRetryRunKeys.add(runKey); scheduleRetry(); From ecba8616a877afd2faed2473449ed987058cf092 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:13:41 +0900 Subject: [PATCH 2/4] fix(webv2): reconcile the last 500 before failing the run The server can commit a batch and still answer 500, so after the last allowed 500 the run goes through reconciliation once more: it is adopted if the backend queue has it, and only fails when the backend has nothing. The failure now carries the server's error as is, like a 4xx, instead of a hard-coded English prefix. --- .../webv2/src/features/queue/runtime.test.ts | 40 +++++++++++++++++++ .../webv2/src/features/queue/runtime.ts | 28 +++++++++---- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts index 9952b30a297..5186df1e74e 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts @@ -1656,6 +1656,46 @@ describe('queue runtime', () => { await runtime.dispose(); }); + it('adopts a run the server accepted before its last 500 instead of failing it', async () => { + const queueItem = createPendingQueueItem(); + const project = { id: 'project-1', queue: { items: [queueItem] } }; + const commands = createTestCommands(); + const enqueueGenerate = vi.fn().mockRejectedValue(new ApiError('response failed after commit', 500)); + const acceptedItem = { + batchId: 'backend-batch', + id: 88, + origin: buildQueueItemOrigin(queueItem.id, project.id), + status: 'pending', + }; + // Nothing is on the backend queue until the third enqueue, which the server commits before answering 500. + const listItems = vi + .fn() + .mockImplementation(() => Promise.resolve(enqueueGenerate.mock.calls.length >= 3 ? [acceptedItem] : [])); + const runtime = createQueueRuntime({ + ...runtimeServices, + backend: createTestBackend({ enqueueGenerate, listItems }), + history: { + commands, + getSnapshot: () => ({ connectionStatus: 'connected', isHydrated: true, projects: [project] }), + subscribe: vi.fn(() => vi.fn()), + }, + journal: createTestJournal(), + }); + + runtime.start(); + + await vi.waitFor( + () => + expect(commands.markBackendSubmitted).toHaveBeenCalledWith( + expect.objectContaining({ backendItemIds: [88], projectId: project.id, queueItemId: queueItem.id }) + ), + { timeout: 4_000 } + ); + expect(enqueueGenerate).toHaveBeenCalledTimes(3); + expect(commands.setStatus).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + await runtime.dispose(); + }); + it('keeps retrying an enqueue that returns 503 while the server is unavailable', async () => { const queueItem = createPendingQueueItem(); const project = { id: 'project-1', queue: { items: [queueItem] } }; diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.ts b/invokeai/frontend/webv2/src/features/queue/runtime.ts index ce676b0e0e8..137034fdbf3 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.ts @@ -414,6 +414,8 @@ const getJournalSignature = (item: QueueItem): string => * A 5xx answer to an enqueue is usually caused by the request itself (e.g. a graph that * crashes server-side validation), so retrying the same submission forever never succeeds. * Gateway and availability errors (502/503/504) stay transient and keep retrying. + * After the last allowed 500 the run is reconciled once more, and only fails if the + * backend queue really has nothing for it. */ const MAX_SERVER_ERROR_SUBMISSIONS = 3; const TRANSIENT_SERVER_STATUSES = [502, 503, 504]; @@ -482,6 +484,7 @@ export const createQueueRuntime = ({ const cancelRequestedRunKeys = new Set(); let reconcileRetryDelayMs = 100; const serverErrorCounts = new Map(); + const exhaustedServerErrors = new Map(); let retryTimer: ReturnType | undefined; let renewTimer: ReturnType | undefined; let hasInitialized = false; @@ -1874,16 +1877,12 @@ export const createQueueRuntime = ({ if (isDeterministicServerError(error)) { const failures = (serverErrorCounts.get(runKey) ?? 0) + 1; if (failures >= MAX_SERVER_ERROR_SUBMISSIONS) { + // The server may have accepted the batch before answering 500, so reconcile before failing the run. serverErrorCounts.delete(runKey); - commands.setStatus({ - error: `The server failed this run ${failures} times in a row: ${toErrorMessage(error)}`, - projectId: project.id, - queueItemId: currentQueueItem.id, - status: 'failed', - }); - return; + exhaustedServerErrors.set(runKey, toErrorMessage(error)); + } else { + serverErrorCounts.set(runKey, failures); } - serverErrorCounts.set(runKey, failures); } else { serverErrorCounts.delete(runKey); } @@ -2173,6 +2172,8 @@ export const createQueueRuntime = ({ const cancelRequested = cancelRequestedRunKeys.has(runKey); reconcileRetryRunKeys.delete(runKey); const outcome = outcomes.get(queueItem.id); + const exhaustedServerError = exhaustedServerErrors.get(runKey); + exhaustedServerErrors.delete(runKey); if ((outcome?.kind === 'adopted' || outcome?.kind === 'resumed') && outcome.missingBackendItemIds?.length) { commands.recordError({ @@ -2198,6 +2199,15 @@ export const createQueueRuntime = ({ switch (outcome?.kind) { case 'enqueue': + if (exhaustedServerError !== undefined) { + commands.setStatus({ + error: exhaustedServerError, + projectId: project.id, + queueItemId: currentQueueItem.id, + status: 'failed', + }); + break; + } startedRunKeys.delete(runKey); break; case 'adopted': @@ -2503,6 +2513,8 @@ export const createQueueRuntime = ({ fallbackAckOperations.clear(); reconcileRetryRunKeys.clear(); pendingReconcileRunKeys.clear(); + serverErrorCounts.clear(); + exhaustedServerErrors.clear(); trackedRuns.clear(); activeAttempts.clear(); ownershipEpochs.clear(); From 1e97eff53678c18a6c4f511b9d7329bec14886d9 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:10:11 +0900 Subject: [PATCH 3/4] fix(webv2): keep retrying a 500 for at least 10 seconds before failing the run Three 500s can arrive within about 300 ms, so a short-lived internal error could fail a run that would have recovered. The run now gives up only after at least three 500s spread over at least 10 seconds. --- .../webv2/src/features/queue/runtime.test.ts | 78 +++++++++++++------ .../webv2/src/features/queue/runtime.ts | 14 +++- 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts index 5186df1e74e..a8b18380f40 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts @@ -1624,6 +1624,7 @@ describe('queue runtime', () => { }); it('fails a run whose enqueue keeps returning a 500 instead of retrying forever', async () => { + vi.useFakeTimers(); const queueItem = createPendingQueueItem(); const project = { id: 'project-1', queue: { items: [queueItem] } }; const commands = createTestCommands(); @@ -1641,36 +1642,71 @@ describe('queue runtime', () => { runtime.start(); - await vi.waitFor( - () => - expect(commands.setStatus).toHaveBeenCalledWith( - expect.objectContaining({ projectId: project.id, queueItemId: queueItem.id, status: 'failed' }) - ), - { timeout: 4_000 } + await vi.advanceTimersByTimeAsync(20_000); + expect(commands.setStatus).toHaveBeenCalledWith( + expect.objectContaining({ projectId: project.id, queueItemId: queueItem.id, status: 'failed' }) ); - expect(enqueueGenerate).toHaveBeenCalledTimes(3); - await new Promise((resolve) => { - setTimeout(resolve, 600); + const attempts = enqueueGenerate.mock.calls.length; + expect(attempts).toBeGreaterThanOrEqual(3); + await vi.advanceTimersByTimeAsync(30_000); + expect(enqueueGenerate).toHaveBeenCalledTimes(attempts); + await runtime.dispose(); + vi.useRealTimers(); + }); + + it('recovers from 500s that stop within a second instead of failing the run', async () => { + vi.useFakeTimers(); + const queueItem = createPendingQueueItem(); + const project = { id: 'project-1', queue: { items: [queueItem] } }; + const commands = createTestCommands(); + const startedAt = Date.now(); + const enqueueGenerate = vi.fn(() => + Date.now() - startedAt < 1_000 + ? Promise.reject(new ApiError('transient internal error', 500)) + : Promise.resolve({ batchId: 'backend-batch', enqueued: 1, itemIds: [88], requested: 1 }) + ); + const runtime = createQueueRuntime({ + ...runtimeServices, + backend: createTestBackend({ enqueueGenerate, listItems: vi.fn().mockResolvedValue([]) }), + history: { + commands, + getSnapshot: () => ({ connectionStatus: 'connected', isHydrated: true, projects: [project] }), + subscribe: vi.fn(() => vi.fn()), + }, + journal: createTestJournal(), }); - expect(enqueueGenerate).toHaveBeenCalledTimes(3); + + runtime.start(); + + await vi.advanceTimersByTimeAsync(10_000); + expect(enqueueGenerate.mock.calls.length).toBeGreaterThan(3); + expect(commands.markBackendSubmitted).toHaveBeenCalledWith( + expect.objectContaining({ backendItemIds: [88], projectId: project.id, queueItemId: queueItem.id }) + ); + expect(commands.setStatus).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); await runtime.dispose(); + vi.useRealTimers(); }); it('adopts a run the server accepted before its last 500 instead of failing it', async () => { + vi.useFakeTimers(); const queueItem = createPendingQueueItem(); const project = { id: 'project-1', queue: { items: [queueItem] } }; const commands = createTestCommands(); - const enqueueGenerate = vi.fn().mockRejectedValue(new ApiError('response failed after commit', 500)); + const startedAt = Date.now(); + let committed = false; + // Once the 500s have lasted long enough to give up, the server commits the next enqueue but still answers 500. + const enqueueGenerate = vi.fn(() => { + committed = Date.now() - startedAt >= 10_000; + return Promise.reject(new ApiError('response failed after commit', 500)); + }); const acceptedItem = { batchId: 'backend-batch', id: 88, origin: buildQueueItemOrigin(queueItem.id, project.id), status: 'pending', }; - // Nothing is on the backend queue until the third enqueue, which the server commits before answering 500. - const listItems = vi - .fn() - .mockImplementation(() => Promise.resolve(enqueueGenerate.mock.calls.length >= 3 ? [acceptedItem] : [])); + const listItems = vi.fn().mockImplementation(() => Promise.resolve(committed ? [acceptedItem] : [])); const runtime = createQueueRuntime({ ...runtimeServices, backend: createTestBackend({ enqueueGenerate, listItems }), @@ -1684,16 +1720,14 @@ describe('queue runtime', () => { runtime.start(); - await vi.waitFor( - () => - expect(commands.markBackendSubmitted).toHaveBeenCalledWith( - expect.objectContaining({ backendItemIds: [88], projectId: project.id, queueItemId: queueItem.id }) - ), - { timeout: 4_000 } + await vi.advanceTimersByTimeAsync(20_000); + expect(committed).toBe(true); + expect(commands.markBackendSubmitted).toHaveBeenCalledWith( + expect.objectContaining({ backendItemIds: [88], projectId: project.id, queueItemId: queueItem.id }) ); - expect(enqueueGenerate).toHaveBeenCalledTimes(3); expect(commands.setStatus).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); await runtime.dispose(); + vi.useRealTimers(); }); it('keeps retrying an enqueue that returns 503 while the server is unavailable', async () => { diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.ts b/invokeai/frontend/webv2/src/features/queue/runtime.ts index 137034fdbf3..00b698230e2 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.ts @@ -414,10 +414,13 @@ const getJournalSignature = (item: QueueItem): string => * A 5xx answer to an enqueue is usually caused by the request itself (e.g. a graph that * crashes server-side validation), so retrying the same submission forever never succeeds. * Gateway and availability errors (502/503/504) stay transient and keep retrying. + * A run only gives up after at least MAX_SERVER_ERROR_SUBMISSIONS 500s spread over at least + * MIN_SERVER_ERROR_WINDOW_MS, so a short-lived internal error can still recover. * After the last allowed 500 the run is reconciled once more, and only fails if the * backend queue really has nothing for it. */ const MAX_SERVER_ERROR_SUBMISSIONS = 3; +const MIN_SERVER_ERROR_WINDOW_MS = 10_000; const TRANSIENT_SERVER_STATUSES = [502, 503, 504]; const isDeterministicServerError = (error: unknown): boolean => error instanceof ApiError && error.status >= 500 && !TRANSIENT_SERVER_STATUSES.includes(error.status); @@ -483,7 +486,7 @@ export const createQueueRuntime = ({ const submissionOperations = new Map>(); const cancelRequestedRunKeys = new Set(); let reconcileRetryDelayMs = 100; - const serverErrorCounts = new Map(); + const serverErrorCounts = new Map(); const exhaustedServerErrors = new Map(); let retryTimer: ReturnType | undefined; let renewTimer: ReturnType | undefined; @@ -1875,13 +1878,16 @@ export const createQueueRuntime = ({ return; } if (isDeterministicServerError(error)) { - const failures = (serverErrorCounts.get(runKey) ?? 0) + 1; - if (failures >= MAX_SERVER_ERROR_SUBMISSIONS) { + const now = Date.now(); + const previous = serverErrorCounts.get(runKey); + const failures = (previous?.count ?? 0) + 1; + const firstAt = previous?.firstAt ?? now; + if (failures >= MAX_SERVER_ERROR_SUBMISSIONS && now - firstAt >= MIN_SERVER_ERROR_WINDOW_MS) { // The server may have accepted the batch before answering 500, so reconcile before failing the run. serverErrorCounts.delete(runKey); exhaustedServerErrors.set(runKey, toErrorMessage(error)); } else { - serverErrorCounts.set(runKey, failures); + serverErrorCounts.set(runKey, { count: failures, firstAt }); } } else { serverErrorCounts.delete(runKey); From 1ffb652ce74de87d32be531d0896397d1794838d Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:47:56 +0900 Subject: [PATCH 4/4] fix(webv2): clear 500 retry state when a run settles or its project closes --- .../webv2/src/features/queue/runtime.test.ts | 47 +++++++++++++++++++ .../webv2/src/features/queue/runtime.ts | 4 ++ 2 files changed, 51 insertions(+) diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts index baca92b029a..6dc5b9ed388 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.test.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.test.ts @@ -1695,6 +1695,53 @@ describe('queue runtime', () => { vi.useRealTimers(); }); + it('forgets earlier 500s once the run settles, so a later run of the item starts fresh', async () => { + vi.useFakeTimers(); + const queueItem = createPendingQueueItem(); + const project = { id: 'project-1', queue: { items: [queueItem] } }; + const listeners = new Set<() => void>(); + const notify = () => listeners.forEach((listener) => listener()); + const commands = createTestCommands({ + setStatus: vi.fn(({ status }) => { + queueItem.status = status; + notify(); + }), + }); + const enqueueGenerate = vi.fn().mockRejectedValue(new ApiError('graph validation crashed', 500)); + const runtime = createQueueRuntime({ + ...runtimeServices, + backend: createTestBackend({ enqueueGenerate, listItems: vi.fn().mockResolvedValue([]) }), + history: { + commands, + getSnapshot: () => ({ connectionStatus: 'connected', isHydrated: true, projects: [project] }), + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }, + journal: createTestJournal(), + }); + + runtime.start(); + await vi.advanceTimersByTimeAsync(1_000); + expect(enqueueGenerate.mock.calls.length).toBeGreaterThanOrEqual(2); + + // The user cancels the run before the 500s are given up on, which settles it. + queueItem.status = 'cancelled'; + notify(); + await vi.advanceTimersByTimeAsync(15_000); + + // Running the item again must not inherit the earlier run's 500 count and window. + enqueueGenerate.mockClear(); + queueItem.status = 'pending'; + notify(); + await vi.advanceTimersByTimeAsync(1_000); + expect(enqueueGenerate).toHaveBeenCalled(); + expect(commands.setStatus).not.toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + await runtime.dispose(); + vi.useRealTimers(); + }); + it('recovers from 500s that stop within a second instead of failing the run', async () => { vi.useFakeTimers(); const queueItem = createPendingQueueItem(); diff --git a/invokeai/frontend/webv2/src/features/queue/runtime.ts b/invokeai/frontend/webv2/src/features/queue/runtime.ts index 65439bdd37b..3aa86522a10 100644 --- a/invokeai/frontend/webv2/src/features/queue/runtime.ts +++ b/invokeai/frontend/webv2/src/features/queue/runtime.ts @@ -812,6 +812,8 @@ export const createQueueRuntime = ({ reconcileRetryRunKeys.delete(key); cancelRequestedRunKeys.delete(key); reportedCancelFailures.delete(key); + serverErrorCounts.delete(key); + exhaustedServerErrors.delete(key); invalidateAttempt(projectId, queueItemId); pendingJournalRecords.delete(key); const ownershipEpoch = ownershipEpochs.get(key) ?? 0; @@ -1966,6 +1968,8 @@ export const createQueueRuntime = ({ pendingReconcileRunKeys.delete(key); reconcileRetryRunKeys.delete(key); reportedCancelFailures.delete(key); + serverErrorCounts.delete(key); + exhaustedServerErrors.delete(key); pendingJournalRecords.delete(key); releaseRunOwnershipAfterJournal(tracked.projectId, tracked.queueItemId); }