@@ -104,6 +104,7 @@ export class ConcurrencySystem {
104104 } ,
105105 resetTotalConcurrencyLimit : ( environment : AuthenticatedEnvironment , queue : QueueInput ) => {
106106 return findQueueFromInput ( this . db , environment , queue )
107+ . andThen ( ( queue ) => syncQueueTotalConcurrencyResetToEngine ( environment , queue ) )
107108 . andThen ( ( queue ) => resetQueueTotalConcurrencyLimit ( this . db , queue ) )
108109 . andThen ( ( queue ) => syncQueueTotalConcurrencyToEngine ( environment , queue ) )
109110 . andThen ( ( queue ) => getQueueStats ( environment , queue ) ) ;
@@ -134,7 +135,9 @@ export class ConcurrencySystem {
134135 concurrencyKey : string
135136 ) => {
136137 return findQueueFromInput ( this . db , environment , queue )
137- . andThen ( ( queue ) => resetQueueConcurrencyKeyLimit ( this . db , environment , queue , concurrencyKey ) )
138+ . andThen ( ( queue ) =>
139+ resetQueueConcurrencyKeyLimit ( this . db , environment , queue , concurrencyKey )
140+ )
138141 . andThen ( ( queue ) => getQueueStats ( environment , queue ) ) ;
139142 } ,
140143 /**
@@ -420,6 +423,35 @@ function overrideQueueTotalConcurrencyLimit(
420423 ) ;
421424}
422425
426+ /**
427+ * Enforce first, then persist: syncs the engine to the declared base BEFORE clearing
428+ * the override marker, so an engine failure leaves the marker set and a retry
429+ * converges instead of being rejected while the overridden limit stays enforced.
430+ */
431+ function syncQueueTotalConcurrencyResetToEngine (
432+ environment : AuthenticatedEnvironment ,
433+ queue : TaskQueue
434+ ) {
435+ if ( queue . totalConcurrencyLimitOverriddenAt === null ) {
436+ return errAsync ( { type : "queue_not_overridden" as const } ) ;
437+ }
438+
439+ if ( typeof queue . totalConcurrencyLimitBase === "number" ) {
440+ return fromPromise (
441+ updateQueueTotalConcurrencyLimits ( environment , queue . name , queue . totalConcurrencyLimitBase ) ,
442+ ( error ) => ( {
443+ type : "sync_queue_concurrency_to_engine_failed" as const ,
444+ cause : error ,
445+ } )
446+ ) . andThen ( ( ) => okAsync ( queue ) ) ;
447+ }
448+
449+ return fromPromise ( removeQueueTotalConcurrencyLimits ( environment , queue . name ) , ( error ) => ( {
450+ type : "sync_queue_concurrency_to_engine_failed" as const ,
451+ cause : error ,
452+ } ) ) . andThen ( ( ) => okAsync ( queue ) ) ;
453+ }
454+
423455function resetQueueTotalConcurrencyLimit ( db : PrismaClientOrTransaction , queue : TaskQueue ) {
424456 if ( queue . totalConcurrencyLimitOverriddenAt === null ) {
425457 return errAsync ( { type : "queue_not_overridden" as const } ) ;
@@ -466,6 +498,13 @@ function syncQueueTotalConcurrencyToEngine(
466498 } ) ) . andThen ( ( ) => okAsync ( queue ) ) ;
467499}
468500
501+ /**
502+ * Persist first, then enforce: a database failure leaves nothing enforced and the
503+ * request errors cleanly, while an engine failure after persistence leaves a durable
504+ * record and a retry converges (the upsert is idempotent). When the engine rejects a
505+ * NEW key for exceeding the per-queue override cap, the just-created row is removed
506+ * again so the record never claims an override the engine refused.
507+ */
469508function overrideQueueConcurrencyKeyLimit (
470509 db : PrismaClientOrTransaction ,
471510 environment : AuthenticatedEnvironment ,
@@ -491,20 +530,13 @@ function overrideQueueConcurrencyKeyLimit(
491530 }
492531
493532 return fromPromise (
494- engine . runQueue . updateQueueConcurrencyKeyLimit (
495- environment ,
496- queue . name ,
497- concurrencyKey ,
498- concurrencyLimit
499- ) ,
500- ( error ) => {
501- if ( error instanceof RunQueueConcurrencyKeyLimitExceededError ) {
502- return { type : "too_many_key_overrides" as const , message : error . message } ;
503- }
504- return { type : "sync_queue_concurrency_to_engine_failed" as const , cause : error } ;
505- }
533+ db . taskQueueConcurrencyKeyOverride . findFirst ( {
534+ where : { taskQueueId : queue . id , concurrencyKey } ,
535+ select : { id : true } ,
536+ } ) ,
537+ ( error ) => ( { type : "other" as const , cause : error } )
506538 )
507- . andThen ( ( ) =>
539+ . andThen ( ( existing ) =>
508540 fromPromise (
509541 db . taskQueueConcurrencyKeyOverride . upsert ( {
510542 where : { taskQueueId_concurrencyKey : { taskQueueId : queue . id , concurrencyKey } } ,
@@ -524,27 +556,56 @@ function overrideQueueConcurrencyKeyLimit(
524556 type : "queue_update_failed" as const ,
525557 cause : error ,
526558 } )
527- )
559+ ) . map ( ( ) => existing )
560+ )
561+ . andThen ( ( existing ) =>
562+ fromPromise (
563+ engine . runQueue . updateQueueConcurrencyKeyLimit (
564+ environment ,
565+ queue . name ,
566+ concurrencyKey ,
567+ concurrencyLimit
568+ ) ,
569+ ( error ) => {
570+ if ( error instanceof RunQueueConcurrencyKeyLimitExceededError ) {
571+ return { type : "too_many_key_overrides" as const , message : error . message } ;
572+ }
573+ return { type : "sync_queue_concurrency_to_engine_failed" as const , cause : error } ;
574+ }
575+ ) . orElse ( ( error ) => {
576+ if ( ! existing && error . type === "too_many_key_overrides" ) {
577+ return fromPromise (
578+ db . taskQueueConcurrencyKeyOverride . deleteMany ( {
579+ where : { taskQueueId : queue . id , concurrencyKey } ,
580+ } ) ,
581+ ( ) => error
582+ ) . andThen ( ( ) => errAsync ( error ) ) ;
583+ }
584+ return errAsync ( error ) ;
585+ } )
528586 )
529587 . andThen ( ( ) => okAsync ( queue ) ) ;
530588}
531589
590+ /**
591+ * Enforce first, then persist: removing the engine limit is idempotent, so an engine
592+ * failure leaves the override row in place and a retry converges instead of being
593+ * rejected as not overridden while the old limit is still enforced.
594+ */
532595function resetQueueConcurrencyKeyLimit (
533596 db : PrismaClientOrTransaction ,
534597 environment : AuthenticatedEnvironment ,
535598 queue : TaskQueue ,
536599 concurrencyKey : string
537600) {
538601 return fromPromise (
539- db . taskQueueConcurrencyKeyOverride . deleteMany ( {
602+ db . taskQueueConcurrencyKeyOverride . findFirst ( {
540603 where : { taskQueueId : queue . id , concurrencyKey } ,
604+ select : { id : true } ,
541605 } ) ,
542- ( error ) => ( {
543- type : "queue_update_failed" as const ,
544- cause : error ,
545- } )
546- ) . andThen ( ( deleted ) => {
547- if ( deleted . count === 0 ) {
606+ ( error ) => ( { type : "other" as const , cause : error } )
607+ ) . andThen ( ( existing ) => {
608+ if ( ! existing ) {
548609 return errAsync ( { type : "queue_not_overridden" as const } ) ;
549610 }
550611
@@ -554,7 +615,19 @@ function resetQueueConcurrencyKeyLimit(
554615 type : "sync_queue_concurrency_to_engine_failed" as const ,
555616 cause : error ,
556617 } )
557- ) . andThen ( ( ) => okAsync ( queue ) ) ;
618+ )
619+ . andThen ( ( ) =>
620+ fromPromise (
621+ db . taskQueueConcurrencyKeyOverride . deleteMany ( {
622+ where : { taskQueueId : queue . id , concurrencyKey } ,
623+ } ) ,
624+ ( error ) => ( {
625+ type : "queue_update_failed" as const ,
626+ cause : error ,
627+ } )
628+ )
629+ )
630+ . andThen ( ( ) => okAsync ( queue ) ) ;
558631 } ) ;
559632}
560633
0 commit comments