@@ -3,7 +3,8 @@ import { knowledgeConnectorMemberSyncLog, knowledgeConnectorSyncLog } from '@sim
33import { createLogger } from '@sim/logger'
44import { describeError } from '@sim/utils/errors'
55import { randomInt } from '@sim/utils/random'
6- import { and , desc , eq , ne } from 'drizzle-orm'
6+ import { and , desc , eq , ne , sql } from 'drizzle-orm'
7+ import type { DbTransaction } from '@/lib/db/types'
78import {
89 CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES ,
910 CONNECTOR_FAILURE_BACKOFF_STEP_MINUTES ,
@@ -33,6 +34,14 @@ const LADDER_RUNGS = Math.ceil(
3334 CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES / CONNECTOR_FAILURE_BACKOFF_STEP_MINUTES
3435)
3536
37+ /**
38+ * Limits on the run-history read. It runs on the failure path, usually while the database is the
39+ * thing failing, so it must give up fast rather than queue behind the slow window; a read that
40+ * times out falls back to counting this run alone.
41+ */
42+ export const RUN_HISTORY_STATEMENT_TIMEOUT_MS = 2_000
43+ export const RUN_HISTORY_LOCK_TIMEOUT_MS = 500
44+
3645/** Which run log a sync writes: the content sync's, or the members-mode run's. */
3746export type SyncRunLogKind = 'content' | 'member'
3847
@@ -43,14 +52,15 @@ interface LoggedRun {
4352
4453/** Earlier runs of this connector, newest first, and whether each one moved the sync forward. */
4554async function readEarlierRuns (
55+ tx : DbTransaction ,
4656 kind : SyncRunLogKind ,
4757 connectorId : string ,
4858 runId : string ,
4959 limit : number
5060) : Promise < LoggedRun [ ] > {
5161 if ( kind === 'content' ) {
5262 const log = knowledgeConnectorSyncLog
53- const rows = await db
63+ const rows = await tx
5464 . select ( {
5565 status : log . status ,
5666 docsAdded : log . docsAdded ,
@@ -67,20 +77,21 @@ async function readEarlierRuns(
6777 } ) )
6878 }
6979 const log = knowledgeConnectorMemberSyncLog
70- const rows = await db
80+ const rows = await tx
7181 . select ( {
7282 status : log . status ,
7383 membersCompleted : log . membersCompleted ,
7484 docsAdded : log . docsAdded ,
7585 docsUpdated : log . docsUpdated ,
86+ docsPurged : log . docsPurged ,
7687 } )
7788 . from ( log )
7889 . where ( and ( eq ( log . connectorId , connectorId ) , ne ( log . id , runId ) ) )
7990 . orderBy ( desc ( log . startedAt ) )
8091 . limit ( limit )
8192 return rows . map ( ( row ) => ( {
8293 status : row . status ,
83- progressed : row . membersCompleted + row . docsAdded + row . docsUpdated > 0 ,
94+ progressed : row . membersCompleted + row . docsAdded + row . docsUpdated + row . docsPurged > 0 ,
8495 } ) )
8596}
8697
@@ -93,17 +104,24 @@ async function readEarlierRuns(
93104 * cannot spend the breaker that disables connectors for persistent source failures. The run log
94105 * already records every attempt and what it wrote, so it measures the streak instead: a statement
95106 * that fails every run without progress still backs off rung by rung, while a run that added,
96- * updated, or deleted documents (or, in members mode, completed a member) before failing ends it.
97- * The read uses the log's `(connector_id, started_at DESC)` index. If it fails too, the streak
98- * counts only this run and the caller's own floor applies.
107+ * updated, or deleted documents (in members mode, purged by the document lifecycle and logged as
108+ * `docs_purged`), or completed a member, before failing ends it.
109+ * The read uses the log's `(connector_id, started_at DESC)` index and is bounded by
110+ * {@link RUN_HISTORY_STATEMENT_TIMEOUT_MS}. If it fails or times out, the streak counts only this
111+ * run and the caller's own floor applies.
99112 */
100113export async function countZeroProgressFailedRuns (
101114 kind : SyncRunLogKind ,
102115 connectorId : string ,
103116 runId : string
104117) : Promise < number > {
105118 try {
106- const earlier = await readEarlierRuns ( kind , connectorId , runId , LADDER_RUNGS - 1 )
119+ const earlier = await db . transaction ( async ( tx ) => {
120+ await tx . execute (
121+ sql `SELECT set_config('statement_timeout', ${ String ( RUN_HISTORY_STATEMENT_TIMEOUT_MS ) } , true), set_config('lock_timeout', ${ String ( RUN_HISTORY_LOCK_TIMEOUT_MS ) } , true)`
122+ )
123+ return readEarlierRuns ( tx , kind , connectorId , runId , LADDER_RUNGS - 1 )
124+ } )
107125 const streakEnd = earlier . findIndex ( ( run ) => run . status !== 'failed' || run . progressed )
108126 return 1 + ( streakEnd === - 1 ? earlier . length : streakEnd )
109127 } catch ( error ) {
0 commit comments