Remove benchmark cutoffs and renew active stage leases

Author: icecrasher321

apps/sim/app/api/organizations/[id]/benchmarks/[benchmarkId]/run/route.ts

@@ -9,8 +9,6 @@ import { requireBenchmarkOperator } from '@/lib/benchmarks/application/access'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
 import { runBenchmarkStage } from '@/lib/benchmarks/application/run-stage'
 
-export const maxDuration = 660
-
 export const POST = defineInternalJsonRoute({
   contract: runBenchmarkStageContract,
   auth: internalSessionAuth,

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-json.tsx

@@ -12,7 +12,7 @@ import { getErrorMessage } from '@sim/utils/errors'
 import { isRecordLike } from '@sim/utils/object'
 import type { BenchmarkCase } from '@/lib/api/contracts/benchmarks'
 import { validateBenchmarkRedaction } from '@/lib/benchmarks/artifacts'
-import { BENCHMARK_SPEC_MAX_LENGTH, benchmarkArtifactsSchema } from '@/lib/benchmarks/types'
+import { benchmarkArtifactsSchema } from '@/lib/benchmarks/types'
 
 interface BenchmarkJsonProps {
   artifacts: BenchmarkCase['artifacts']
@@ -21,8 +21,6 @@ interface BenchmarkJsonProps {
   onClose: () => void
 }
 
-const MAPPING_MAX_LENGTH = BENCHMARK_SPEC_MAX_LENGTH * 6 + 5_000
-
 export function BenchmarkJson({ artifacts, disabled, onApply, onClose }: BenchmarkJsonProps) {
   const [redactedSpec, setRedactedSpec] = useState(
     artifacts.redactedSpec || artifacts.referenceSpec
@@ -53,7 +51,7 @@ export function BenchmarkJson({ artifacts, disabled, onApply, onClose }: Benchma
       )
       if (!parsed.success) {
         throw new Error(
-          'Use up to 50 blank IDs (letters, numbers, underscores or hyphens, at most 64 characters), each mapped to a nonempty string of at most 10,000 characters.'
+          'Use blank IDs with letters, numbers, underscores or hyphens (at most 64 characters), each mapped to a nonempty answer string.'
         )
       }
       const patch = { redactedSpec, blanks: parsed.data }
@@ -82,7 +80,6 @@ export function BenchmarkJson({ artifacts, disabled, onApply, onClose }: Benchma
             setRedactedSpec(value)
             setError(null)
           }}
-          maxLength={BENCHMARK_SPEC_MAX_LENGTH}
           rows={6}
           resizable
           disabled={disabled}
@@ -99,7 +96,6 @@ export function BenchmarkJson({ artifacts, disabled, onApply, onClose }: Benchma
           placeholder={
             '{\n  "queue": "Customer Escalations",\n  "handoff": "Engineering explicitly accepts the case"\n}'
           }
-          maxLength={MAPPING_MAX_LENGTH}
           rows={10}
           mono
           resizable

apps/sim/app/o/[organizationId]/benchmark/components/benchmark-reference.tsx

@@ -5,7 +5,6 @@ import { Chip, ChipInput, ChipTextarea, toast } from '@sim/emcn'
 import { Code, Plus, Trash, Upload } from '@sim/emcn/icons'
 import { getErrorMessage } from '@sim/utils/errors'
 import type { BenchmarkCase, RunBenchmarkStageBody } from '@/lib/api/contracts/benchmarks'
-import { BENCHMARK_MAX_BLANKS, BENCHMARK_SPEC_MAX_LENGTH } from '@/lib/benchmarks/types'
 import { BenchmarkJson } from '@/app/o/[organizationId]/benchmark/components/benchmark-json'
 import { BenchmarkStep } from '@/app/o/[organizationId]/benchmark/components/benchmark-step'
 
@@ -40,13 +39,7 @@ export function BenchmarkReference({
   const importReference = async (file: File | undefined) => {
     if (!file) return
     try {
-      if (file.size > BENCHMARK_SPEC_MAX_LENGTH * 4) {
-        throw new Error('The reference is too large. Use a text file under 4 MB.')
-      }
       const text = await file.text()
-      if (text.length > BENCHMARK_SPEC_MAX_LENGTH) {
-        throw new Error('The reference must be at most 1,000,000 characters.')
-      }
       onChange({ referenceSpec: text })
     } catch (error) {
       toast.error(getErrorMessage(error, 'Could not import the reference'))
@@ -79,7 +72,6 @@ export function BenchmarkReference({
             value={taskBrief}
             onChange={(event) => onChange({ taskBrief: event.target.value })}
             placeholder='What should the new Mothership plan?'
-            maxLength={20_000}
             rows={4}
             resizable
           />
@@ -113,7 +105,6 @@ export function BenchmarkReference({
             value={referenceSpec}
             onChange={(event) => onChange({ referenceSpec: event.target.value })}
             placeholder='Generate a spec, paste one, or import Markdown, text, or JSON.'
-            maxLength={BENCHMARK_SPEC_MAX_LENGTH}
             rows={12}
             resizable
           />
@@ -158,7 +149,6 @@ export function BenchmarkReference({
             aria-label='Redacted reference spec'
             value={redactedSpec}
             onChange={(event) => onChange({ redactedSpec: event.target.value })}
-            maxLength={BENCHMARK_SPEC_MAX_LENGTH}
             rows={10}
             resizable
           />
@@ -188,7 +178,6 @@ export function BenchmarkReference({
                 <ChipTextarea
                   aria-label={`Expected answer for blank ${index + 1}`}
                   value={blank.answer}
-                  maxLength={10_000}
                   rows={2}
                   resizable
                   onChange={(event) =>
@@ -215,7 +204,6 @@ export function BenchmarkReference({
           <div>
             <Chip
               leftIcon={Plus}
-              disabled={blanks.length >= BENCHMARK_MAX_BLANKS}
               onClick={() => {
                 let suffix = blanks.length + 1
                 while (blanks.some((blank) => blank.id === `detail_${suffix}`)) suffix += 1

apps/sim/app/o/[organizationId]/benchmark/components/create-benchmark.tsx

@@ -82,7 +82,6 @@ export function CreateBenchmark({ organizationId, runAsUserId, onCreated }: Crea
           onChange={(event) => setTaskBrief(event.target.value)}
           placeholder='Paste the original request, or generate a brief from the workspace in step 1.'
           rows={4}
-          maxLength={20_000}
           resizable
           disabled={createBenchmark.isPending}
         />

apps/sim/lib/benchmarks/README.md

@@ -23,9 +23,9 @@ Grading opens the saved report for human review. On any detail, choose **Mark as
 
 Reference specs can be prose or JSON text; **Import spec** accepts Markdown, text, and JSON files without reformatting their contents. **Generate blanks with AI** creates the redacted reference and expected answers together. **Edit JSON** also lets you paste or edit a mapping such as `{"queue":"Customer Escalations","handoff":"Engineering explicitly accepts the case"}` alongside the matching redacted reference. Keys are blank IDs and values are exact original passages as strings. Applying validates that the mapping restores the reference exactly and updates the draft; **Save changes** persists it.
 
-Every stage is limited to ten minutes, with a twelve-minute persistence lease. Concurrent starts or stale completions cannot overwrite newer results. Interrupted steps are retryable; a hard server failure becomes retryable when its lease expires. This first version runs a stage within its HTTP request, so the deployment's request timeout must accommodate the run. Page reload may interrupt a pending request; saved completed stages remain available.
+Benchmark stages have no benchmark-specific duration cutoff. The active process renews its persistence lease every 30 seconds; a two-minute lease detects an abandoned process rather than limiting a healthy run. Concurrent starts or stale completions cannot overwrite newer results. Interrupted steps are retryable; a hard server failure becomes retryable when its lease expires. This version runs a stage within its HTTP request, so the deployment must support long-lived requests. Page reload may interrupt a pending request; saved completed stages remain available.
 
-Reference generation uses the existing Mothership agent loop and workspace CLI to inspect active workflows and supporting resources incrementally. There is no benchmark-wide workflow-count or export-size gate. Reads remain authorized as the selected user; the benchmark transport refuses source mutations, workflow execution, services and scratch writes. Existing CLI pagination and searchable stored outputs keep individual model inputs bounded. Specs can contain up to 1,000,000 characters; structured output uses the worker’s 32,768-token per-response limit. The planner uses the dev worker's model configuration; keep worker/model configuration constant when comparing agent changes.
+Reference generation uses the existing Mothership agent loop and workspace CLI to inspect active workflows and supporting resources incrementally. There is no benchmark-wide workflow-count or export-size gate. Reads remain authorized as the selected user; the benchmark transport refuses source mutations, workflow execution, services and scratch writes. Existing CLI pagination and searchable stored outputs keep individual model inputs bounded. Benchmark-specific caps on spec length, blank count, answer length and model output are removed. The normal model/provider and transport constraints still apply; page sizes bound individual tool reads without limiting the complete workspace or document. The planner uses the dev worker's model configuration; keep worker/model configuration constant when comparing agent changes.
 
 This version uses **live enterprise context**, not a frozen historical snapshot. The completed Sim workspace is excluded from planning, but historical solution documents in connected enterprise sources can still reveal answers. Treat these cases as retrospective evaluations and review source availability. The score measures recovery of the selected requirements, not execution correctness or every claim in the plan.
 

apps/sim/lib/benchmarks/application/run-stage.ts

@@ -6,6 +6,10 @@ import { defineAuthorizedBenchmarkUseCase } from '@/lib/benchmarks/application/a
 import { requireBenchmarkCaseAccess } from '@/lib/benchmarks/application/cases'
 import { benchmarkOperations } from '@/lib/benchmarks/application/operations'
 import { prepareBenchmarkReference } from '@/lib/benchmarks/application/prepare-reference'
+import {
+  BENCHMARK_LEASE_MS,
+  withBenchmarkStageLease,
+} from '@/lib/benchmarks/application/stage-lease'
 import { applyBenchmarkPatch, validateBenchmarkRedaction } from '@/lib/benchmarks/artifacts'
 import { getBenchmarkMothershipUrl } from '@/lib/benchmarks/config'
 import { gradeReconstruction, validateReconstruction } from '@/lib/benchmarks/evaluation'
@@ -21,7 +25,6 @@ import {
   failBenchmarkStage,
 } from '@/lib/benchmarks/repository'
 import {
-  BENCHMARK_MAX_BLANKS,
   type BenchmarkArtifacts,
   type BenchmarkCase,
   type BenchmarkStage,
@@ -35,24 +38,20 @@ import { executeBenchmarkJson, executeBenchmarkPlan } from '@/lib/benchmarks/wor
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 
 const logger = createLogger('BenchmarkStage')
-const STAGE_TIMEOUT_MS = 10 * 60 * 1000
-const STAGE_LEASE_MS = STAGE_TIMEOUT_MS + 2 * 60 * 1000
 
 const distillationSchema = z
   .object({ taskBrief: benchmarkBriefSchema.min(1), referenceSpec: benchmarkSpecSchema.min(1) })
   .strict()
 const redactionSchema = z
   .object({
     redactedSpec: benchmarkSpecSchema.min(1),
-    blanks: z.array(benchmarkBlankSchema).min(1).max(BENCHMARK_MAX_BLANKS),
+    blanks: z.array(benchmarkBlankSchema).min(1),
   })
   .strict()
 const reconstructionSchema = z
-  .object({ answers: z.array(benchmarkReconstructionSchema).min(1).max(BENCHMARK_MAX_BLANKS) })
-  .strict()
-const gradingSchema = z
-  .object({ judgments: z.array(benchmarkGradeSchema).min(1).max(BENCHMARK_MAX_BLANKS) })
+  .object({ answers: z.array(benchmarkReconstructionSchema).min(1) })
   .strict()
+const gradingSchema = z.object({ judgments: z.array(benchmarkGradeSchema).min(1) }).strict()
 
 interface RunBenchmarkStageInput {
   organizationId: string
@@ -186,19 +185,19 @@ export const runBenchmarkStage = defineAuthorizedBenchmarkUseCase({
       expectedVersion: input.version,
       stage: input.stage,
       attemptId,
-      leaseExpiresAt: new Date(Date.now() + STAGE_LEASE_MS),
+      leaseExpiresAt: new Date(Date.now() + BENCHMARK_LEASE_MS),
     })
     const attempt = { ...scope, version: claimed.version, stage: input.stage, attemptId }
     logger.info('Benchmark step started', {
       ...attempt,
       operatorUserId: current.userId,
       runAsUserId: current.runAsUserId ?? current.userId,
     })
-    const timeout = AbortSignal.timeout(STAGE_TIMEOUT_MS)
-    const signal = request?.signal ? AbortSignal.any([request.signal, timeout]) : timeout
     try {
-      const output = await performStage(principal, claimed, input.stage, signal)
-      signal.throwIfAborted()
+      const output = await withBenchmarkStageLease(attempt, request?.signal, (signal) =>
+        performStage(principal, claimed, input.stage, signal)
+      )
+      request?.signal?.throwIfAborted()
       await requireBenchmarkCaseAccess(principal, input)
       return {
         benchmark: await completeBenchmarkStage({
@@ -208,8 +207,8 @@ export const runBenchmarkStage = defineAuthorizedBenchmarkUseCase({
         }),
       }
     } catch (error) {
-      const message = signal.aborted
-        ? 'This step was interrupted or exceeded ten minutes. Retry it.'
+      const message = request?.signal?.aborted
+        ? 'This step was interrupted. Retry it.'
         : error instanceof OrchestrationError
           ? error.message
           : 'This benchmark step failed. Retry it or inspect the server logs.'
@@ -229,8 +228,8 @@ export const runBenchmarkStage = defineAuthorizedBenchmarkUseCase({
           errorType: persistenceError instanceof Error ? persistenceError.name : 'UnknownError',
         })
       }
-      if (!signal.aborted && error instanceof OrchestrationError) throw error
-      throw new OrchestrationError(signal.aborted ? 'validation' : 'internal', message)
+      if (!request?.signal?.aborted && error instanceof OrchestrationError) throw error
+      throw new OrchestrationError(request?.signal?.aborted ? 'validation' : 'internal', message)
     }
   },
 })

apps/sim/lib/benchmarks/application/stage-lease.test.ts

@@ -0,0 +1,58 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { renew } = vi.hoisted(() => ({ renew: vi.fn() }))
+vi.mock('@/lib/benchmarks/repository', () => ({ renewBenchmarkStage: renew }))
+
+import { withBenchmarkStageLease } from '@/lib/benchmarks/application/stage-lease'
+
+/** Covers healthy runs beyond the old cutoff, loss of ownership, and caller cancellation. */
+describe('benchmark stage lifetime', () => {
+  const attempt = {
+    organizationId: 'org',
+    userId: 'operator',
+    benchmarkId: 'benchmark',
+    version: 2,
+    stage: 'distill' as const,
+    attemptId: 'attempt',
+  }
+
+  beforeEach(() => {
+    vi.useFakeTimers()
+    renew.mockResolvedValue(true)
+  })
+  afterEach(() => vi.useRealTimers())
+
+  it('keeps a healthy inspection alive beyond ten minutes and stops its heartbeat on completion', async () => {
+    const result = withBenchmarkStageLease(attempt, undefined, async (signal) => {
+      await vi.advanceTimersByTimeAsync(60 * 60_000)
+      signal.throwIfAborted()
+      return 'complete reference'
+    })
+    await expect(result).resolves.toBe('complete reference')
+    expect(vi.getTimerCount()).toBe(0)
+  })
+
+  it.each([false, new Error('database unavailable')])(
+    'aborts the inspection when its ownership cannot be renewed (%s)',
+    async (outcome) => {
+      if (outcome instanceof Error) renew.mockRejectedValue(outcome)
+      else renew.mockResolvedValue(outcome)
+      const result = withBenchmarkStageLease(attempt, undefined, async (signal) => {
+        await vi.advanceTimersByTimeAsync(60_000)
+        signal.throwIfAborted()
+      }).catch((error: unknown) => error)
+      await expect(result).resolves.toMatchObject({ code: 'conflict' })
+      expect(vi.getTimerCount()).toBe(0)
+    }
+  )
+
+  it('propagates caller cancellation and clears its heartbeat when work throws', async () => {
+    const controller = new AbortController()
+    const result = withBenchmarkStageLease(attempt, controller.signal, async (signal) => {
+      controller.abort(new Error('cancelled by caller'))
+      signal.throwIfAborted()
+    })
+    await expect(result).rejects.toThrow('cancelled by caller')
+    expect(vi.getTimerCount()).toBe(0)
+  })
+})

apps/sim/lib/benchmarks/application/stage-lease.ts

@@ -0,0 +1,48 @@
+import { type BenchmarkAttempt, renewBenchmarkStage } from '@/lib/benchmarks/repository'
+import { OrchestrationError } from '@/lib/core/orchestration/types'
+
+/** This is crash detection, not a deadline: the current owner continually extends its lease. */
+export const BENCHMARK_LEASE_MS = 2 * 60_000
+const HEARTBEAT_MS = 30_000
+
+export async function withBenchmarkStageLease<T>(
+  attempt: BenchmarkAttempt,
+  callerSignal: AbortSignal | undefined,
+  execute: (signal: AbortSignal) => Promise<T>
+): Promise<T> {
+  const ownership = new AbortController()
+  const signal = callerSignal ? AbortSignal.any([callerSignal, ownership.signal]) : ownership.signal
+  let pending: Promise<void> | undefined
+  const heartbeat = setInterval(() => {
+    if (pending || signal.aborted) return
+    pending = renewBenchmarkStage({
+      ...attempt,
+      leaseExpiresAt: new Date(Date.now() + BENCHMARK_LEASE_MS),
+    })
+      .then((renewed) => {
+        if (!renewed) throw new Error('Benchmark lease lost')
+      })
+      .catch(() => {
+        ownership.abort(
+          new OrchestrationError(
+            'conflict',
+            'This step lost its run ownership. Refresh and retry it.'
+          )
+        )
+      })
+      .finally(() => {
+        pending = undefined
+      })
+  }, HEARTBEAT_MS)
+  heartbeat.unref?.()
+  try {
+    signal.throwIfAborted()
+    const result = await execute(signal)
+    await pending
+    signal.throwIfAborted()
+    return result
+  } finally {
+    clearInterval(heartbeat)
+    await pending
+  }
+}

apps/sim/lib/benchmarks/repository.integration.ts

@@ -42,6 +42,7 @@ import {
   getBenchmarkRecord,
   getBenchmarkRunRecord,
   listBenchmarkRunRecords,
+  renewBenchmarkStage,
   updateBenchmarkRecord,
 } from '@/lib/benchmarks/repository'
 import { type BenchmarkArtifacts, emptyBenchmarkArtifacts } from '@/lib/benchmarks/types'
@@ -359,6 +360,15 @@ describe('private benchmark persistence and attempt fencing', () => {
       attemptId: 'next',
       leaseExpiresAt: new Date(Date.now() + 60_000),
     })
+    expect(
+      await renewBenchmarkStage({
+        ...scope,
+        version: old.version,
+        stage: 'plan',
+        attemptId: 'old',
+        leaseExpiresAt: new Date(Date.now() + 120_000),
+      })
+    ).toBe(false)
     await expect(
       completeBenchmarkStage({
         ...scope,
@@ -388,6 +398,29 @@ describe('private benchmark persistence and attempt fencing', () => {
     expect(completed.runningStage).toBeNull()
   })
 
+  it('extends only the current live lease without invalidating the attempt version', async () => {
+    const claimed = await claimBenchmarkStage({
+      ...scope,
+      expectedVersion: 1,
+      stage: 'distill',
+      attemptId: 'long-inspection',
+      leaseExpiresAt: new Date(Date.now() + 60_000),
+    })
+    const attempt = {
+      ...scope,
+      version: claimed.version,
+      stage: 'distill' as const,
+      attemptId: 'long-inspection',
+    }
+    const leaseExpiresAt = new Date(Date.now() + 120_000)
+    expect(await renewBenchmarkStage({ ...attempt, leaseExpiresAt })).toBe(true)
+    const current = await getBenchmarkRecord(scope)
+    expect(current.version).toBe(claimed.version)
+    expect(current.leaseExpiresAt).toBe(leaseExpiresAt.toISOString())
+    await completeBenchmarkStage({ ...attempt, artifacts })
+    expect(await renewBenchmarkStage({ ...attempt, leaseExpiresAt })).toBe(false)
+  })
+
   it('keeps artifacts private even from another member who can read the same workspace', async () => {
     await expect(
       getBenchmark.execute({