@@ -103,6 +103,33 @@ const LEASE_LOOKUP_RETRY_MS = 5_000
103103/** How long one lease lookup may take before it counts as failed. */
104104const LEASE_LOOKUP_TIMEOUT_MS = 5_000
105105
106+ /** The resume wait's watch over one pending tool call. */
107+ interface PendingToolWatchdog {
108+ promise : Promise < AsyncCompletionSignal >
109+ settlement : Promise < { toolCallId : string ; promise : Promise < AsyncCompletionSignal > } >
110+ deadlineAt : number
111+ waitBudgetMs : number
112+ /** When this wait began. */
113+ startedAt : number
114+ /** Past this, the call is given up however its lease stands: any client tool's cap. */
115+ ceilingAt : number
116+ /** Why the deadline was moved past the plain budget, if it was. */
117+ extendedBy ?: 'lease' | 'lease_lookup'
118+ /** Until when a failing lease lookup is retried before the call is given up. */
119+ leaseLookupRetryUntil ?: number
120+ }
121+
122+ /** Why a pending call's wait ran out, in the words its force-fail is logged with. */
123+ function pendingToolExpiryMessage ( watchdog : PendingToolWatchdog , now : number ) : string {
124+ if ( ! watchdog . extendedBy )
125+ return 'Pending tool execution exceeded its resume wait budget; force-failing'
126+ if ( now >= watchdog . ceilingAt )
127+ return 'Pending tool execution reached the client tool result cap; force-failing'
128+ if ( watchdog . leaseLookupRetryUntil !== undefined )
129+ return 'Pending tool execution lease could not be read for a whole lease; force-failing'
130+ return 'Pending tool execution has no live lease past its budget; force-failing'
131+ }
132+
106133/** A pending call's chat-view lease, or why it could not be read within `timeoutMs`. */
107134async function readLeaseWithin (
108135 toolCallId : string ,
@@ -1390,26 +1417,7 @@ async function runCheckpointLoop(
13901417 } )
13911418 let maximumWaitBudgetMs = 0
13921419 let timedOutCount = 0
1393- const pendingWatchdogs = new Map <
1394- string ,
1395- {
1396- promise : Promise < AsyncCompletionSignal >
1397- settlement : Promise < {
1398- toolCallId : string
1399- promise : Promise < AsyncCompletionSignal >
1400- } >
1401- deadlineAt : number
1402- waitBudgetMs : number
1403- /** When this wait began. */
1404- startedAt : number
1405- /** Past this, the call is given up however its lease stands: any client tool's cap. */
1406- ceilingAt : number
1407- /** Why the deadline was moved past the plain budget, if it was. */
1408- extendedBy ?: 'lease' | 'lease_lookup'
1409- /** Until when a failing lease lookup is retried before the call is given up. */
1410- leaseLookupRetryUntil ?: number
1411- }
1412- > ( )
1420+ const pendingWatchdogs = new Map < string , PendingToolWatchdog > ( )
14131421
14141422 /**
14151423 * A long-running approval must not lend its deadline to an unrelated
@@ -1450,59 +1458,62 @@ async function runCheckpointLoop(
14501458 context . pendingToolPromises . get ( toolCallId ) === watchdog . promise
14511459 )
14521460 // A desktop import the chat view is running renews its lease while it works: its budget
1453- // runs to the end of that lease, and only a lapsed lease fails it.
1461+ // runs to the end of that lease, and only a lapsed lease fails it. At its ceiling a call is
1462+ // given up however its lease stands, so its lease is not read.
14541463 // A lookup that fails says nothing about the lease, so it is retried, for at most one lease.
14551464 // Each lookup is bounded, so a stalled read cannot hold the wait past its deadlines or Stop.
1465+ const leaseChecked = overdueTools . filter ( ( [ , watchdog ] ) => now < watchdog . ceilingAt )
14561466 const leases = await Promise . all (
1457- overdueTools . map ( ( [ toolCallId ] ) =>
1467+ leaseChecked . map ( ( [ toolCallId ] ) =>
14581468 readLeaseWithin ( toolCallId , LEASE_LOOKUP_TIMEOUT_MS , options . abortSignal )
14591469 )
14601470 )
14611471 if ( isAborted ( options , context ) ) break
1462- const expiredTools = overdueTools . filter ( ( [ toolCallId , watchdog ] , index ) => {
1463- // A call replaced while its lease was read belongs to its new watchdog.
1464- if ( context . pendingToolPromises . get ( toolCallId ) !== watchdog . promise ) return false
1465- const lease = leases [ index ]
1466- const checkedAt = Date . now ( )
1467- if ( checkedAt >= watchdog . ceilingAt ) return true
1468- const extendTo = ( deadlineAt : number , reason : 'lease' | 'lease_lookup' ) => {
1469- watchdog . deadlineAt = Math . min ( deadlineAt , watchdog . ceilingAt )
1470- watchdog . extendedBy = reason
1471- maximumWaitBudgetMs = Math . max (
1472- maximumWaitBudgetMs ,
1473- watchdog . deadlineAt - watchdog . startedAt
1474- )
1475- return false
1476- }
1477- if ( 'error' in lease ) {
1478- watchdog . leaseLookupRetryUntil ??= checkedAt + SIM_TOOL_EXECUTION_LEASE_SECONDS * 1000
1479- if ( checkedAt >= watchdog . leaseLookupRetryUntil ) return true
1480- logger . warn ( 'Could not read a pending tool call lease; checking again' , {
1481- toolCallId,
1482- error : getErrorMessage ( lease . error ) ,
1483- } )
1484- return extendTo ( checkedAt + LEASE_LOOKUP_RETRY_MS , 'lease_lookup' )
1485- }
1486- watchdog . leaseLookupRetryUntil = undefined
1487- if ( typeof lease . remainingMs !== 'number' || lease . remainingMs <= 0 ) return true
1488- return extendTo ( checkedAt + lease . remainingMs + LEASE_RECHECK_SLACK_MS , 'lease' )
1489- } )
1472+ const expiredTools = [
1473+ ...overdueTools . filter (
1474+ ( [ toolCallId , watchdog ] ) =>
1475+ now >= watchdog . ceilingAt &&
1476+ context . pendingToolPromises . get ( toolCallId ) === watchdog . promise
1477+ ) ,
1478+ ...leaseChecked . filter ( ( [ toolCallId , watchdog ] , index ) => {
1479+ // A call replaced while its lease was read belongs to its new watchdog.
1480+ if ( context . pendingToolPromises . get ( toolCallId ) !== watchdog . promise ) return false
1481+ const lease = leases [ index ]
1482+ const checkedAt = Date . now ( )
1483+ const extendTo = ( deadlineAt : number , reason : 'lease' | 'lease_lookup' ) => {
1484+ watchdog . deadlineAt = Math . min ( deadlineAt , watchdog . ceilingAt )
1485+ watchdog . extendedBy = reason
1486+ maximumWaitBudgetMs = Math . max (
1487+ maximumWaitBudgetMs ,
1488+ watchdog . deadlineAt - watchdog . startedAt
1489+ )
1490+ return false
1491+ }
1492+ if ( 'error' in lease ) {
1493+ watchdog . leaseLookupRetryUntil ??= checkedAt + SIM_TOOL_EXECUTION_LEASE_SECONDS * 1000
1494+ if ( checkedAt >= watchdog . leaseLookupRetryUntil ) return true
1495+ logger . warn ( 'Could not read a pending tool call lease; checking again' , {
1496+ toolCallId,
1497+ error : getErrorMessage ( lease . error ) ,
1498+ } )
1499+ return extendTo ( checkedAt + LEASE_LOOKUP_RETRY_MS , 'lease_lookup' )
1500+ }
1501+ watchdog . leaseLookupRetryUntil = undefined
1502+ if ( typeof lease . remainingMs !== 'number' || lease . remainingMs <= 0 ) return true
1503+ return extendTo ( checkedAt + lease . remainingMs + LEASE_RECHECK_SLACK_MS , 'lease' )
1504+ } ) ,
1505+ ]
14901506 if ( expiredTools . length > 0 ) {
14911507 await Promise . all (
14921508 expiredTools . map ( async ( [ toolCallId , watchdog ] ) => {
14931509 const waitedMs = Date . now ( ) - watchdog . startedAt
1494- logger . error (
1495- watchdog . extendedBy
1496- ? 'Pending tool execution outlived its renewed lease or its cap; force-failing'
1497- : 'Pending tool execution exceeded its resume wait budget; force-failing' ,
1498- {
1499- checkpointId : continuation . checkpointId ,
1500- toolCallId,
1501- waitBudgetMs : watchdog . waitBudgetMs ,
1502- waitedMs,
1503- ...( watchdog . extendedBy ? { extendedBy : watchdog . extendedBy } : { } ) ,
1504- }
1505- )
1510+ logger . error ( pendingToolExpiryMessage ( watchdog , now ) , {
1511+ checkpointId : continuation . checkpointId ,
1512+ toolCallId,
1513+ waitBudgetMs : watchdog . waitBudgetMs ,
1514+ waitedMs,
1515+ ...( watchdog . extendedBy ? { extendedBy : watchdog . extendedBy } : { } ) ,
1516+ } )
15061517 await failPendingToolCall ( toolCallId , context , execContext )
15071518 if ( context . pendingToolPromises . get ( toolCallId ) === watchdog . promise ) {
15081519 context . pendingToolPromises . delete ( toolCallId )
0 commit comments