Generate benchmark redactions from exact passages

Author: Vikhyath Mondreti

apps/sim/lib/benchmarks/README.md

@@ -11,7 +11,7 @@ Open **Settings → Admin → Open benchmarks** (or `/benchmark`). Select an org
 Each case fixes the execution user when it is created. Changing the selector shows that user's saved cases; it never changes an existing case's identity. Legacy cases retain their original owner's execution identity. **New benchmark** always opens a creation form, and saved cases appear in a list with an **Open** action.
 
 1. Select a workspace. Generate the reference by letting Mothership inspect its workflows and supporting resources, or paste/import a reference produced by another client. Review the original task brief and reference.
-2. Generate the blanks and review their expected answers. Only select facts discoverable from the permitted enterprise sources. Replacing `[[BLANK:id]]` with each exact answer must restore the reference byte for byte. Remove answer hints left elsewhere in the reference.
+2. Generate the blanks and review their expected answers. The LLM selects exact passages; the server masks all matching occurrences without rewriting the reference. Only select facts discoverable from the permitted enterprise sources. Replacing `[[BLANK:id]]` with each exact answer must restore the reference byte for byte. Remove answer hints left elsewhere in the reference.
 3. Run the planner. It receives the brief in a fresh organization Plan conversation, with enterprise search/document reads and isolated memory. It receives neither the completed workspace inventory nor the reference, masks, or expected answers. Its final response is the generated spec.
 4. Reconstruct and grade separately. Reconstruction is a fresh execution with the redacted questions in context and one immutable document, `generated-spec.md`, available through a bounded read/search tool. That document contains only the planner’s generated spec; enterprise tools, other documents and history are unavailable. Grading compares the recovered answers to the hidden targets and checks support in the generated spec. An exact supporting passage is required for credit. The score is correct details divided by the fixed number of blanks.
 

apps/sim/lib/benchmarks/application/run-stage.ts

@@ -10,7 +10,11 @@ import {
   BENCHMARK_LEASE_MS,
   withBenchmarkStageLease,
 } from '@/lib/benchmarks/application/stage-lease'
-import { applyBenchmarkPatch, validateBenchmarkRedaction } from '@/lib/benchmarks/artifacts'
+import {
+  applyBenchmarkPatch,
+  redactBenchmarkSpec,
+  validateBenchmarkRedaction,
+} from '@/lib/benchmarks/artifacts'
 import { getBenchmarkMothershipUrl } from '@/lib/benchmarks/config'
 import { gradeReconstruction, validateReconstruction } from '@/lib/benchmarks/evaluation'
 import {
@@ -44,7 +48,6 @@ const distillationSchema = z
   .strict()
 const redactionSchema = z
   .object({
-    redactedSpec: benchmarkSpecSchema.min(1),
     blanks: z.array(benchmarkBlankSchema).min(1),
   })
   .strict()
@@ -110,7 +113,12 @@ async function performStage(
         schema: redactionSchema,
         messages: redactionMessages(artifacts.referenceSpec),
       })
-      return { artifacts: applyBenchmarkPatch(artifacts, result) }
+      return {
+        artifacts: applyBenchmarkPatch(artifacts, {
+          ...result,
+          redactedSpec: redactBenchmarkSpec(artifacts.referenceSpec, result.blanks),
+        }),
+      }
     }
     case 'plan': {
       const result = await executeBenchmarkPlan({ principal, benchmark, signal })

apps/sim/lib/benchmarks/artifacts.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from 'vitest'
-import { applyBenchmarkPatch, validateBenchmarkRedaction } from '@/lib/benchmarks/artifacts'
+import {
+  applyBenchmarkPatch,
+  redactBenchmarkSpec,
+  validateBenchmarkRedaction,
+} from '@/lib/benchmarks/artifacts'
 import type { BenchmarkArtifacts } from '@/lib/benchmarks/types'
 
 const artifacts: BenchmarkArtifacts = {
@@ -14,6 +18,32 @@ const artifacts: BenchmarkArtifacts = {
 }
 
 describe('benchmark reference integrity', () => {
+  it('builds masks from exact selected passages without rewriting the reference', () => {
+    const referenceSpec = 'Queue: Ops [L2].\nNotify Ops [L2] and team+$ at $5.\n'
+    expect(
+      redactBenchmarkSpec(referenceSpec, [
+        { id: 'queue', answer: 'Ops [L2]' },
+        { id: 'team', answer: 'team+$' },
+      ])
+    ).toBe('Queue: [[BLANK:queue]].\nNotify [[BLANK:queue]] and [[BLANK:team]] at $5.\n')
+  })
+
+  it('rejects missing, duplicate, or overlapping passages that cannot produce every blank', () => {
+    for (const blanks of [
+      [{ id: 'missing', answer: 'Elsewhere' }],
+      [
+        { id: 'first', answer: 'Support' },
+        { id: 'second', answer: 'Support' },
+      ],
+      [
+        { id: 'short', answer: 'Support' },
+        { id: 'long', answer: 'Support team' },
+      ],
+    ]) {
+      expect(() => redactBenchmarkSpec('Support team owns follow-up.', blanks)).toThrow()
+    }
+  })
+
   it('accepts repeated masks for one requirement without changing surviving reference text', () => {
     expect(() => validateBenchmarkRedaction(artifacts)).not.toThrow()
   })

apps/sim/lib/benchmarks/artifacts.ts

@@ -1,10 +1,29 @@
+import { escapeRegExp } from '@sim/utils/string'
 import {
   type BenchmarkArtifacts,
   type BenchmarkEditablePatch,
   benchmarkArtifactsSchema,
 } from '@/lib/benchmarks/types'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 
+/** Replace selected passages in one pass so markers and surviving text cannot be rewritten. */
+export function redactBenchmarkSpec(
+  referenceSpec: string,
+  blanks: BenchmarkArtifacts['blanks']
+): string {
+  if (!blanks.length || blanks.some((blank) => !blank.answer)) {
+    throw new OrchestrationError('validation', 'Select at least one nonempty passage to redact')
+  }
+  const ids = new Map(blanks.map((blank) => [blank.answer, blank.id]))
+  const passages = [...ids.keys()].sort((left, right) => right.length - left.length)
+  const redactedSpec = referenceSpec.replace(
+    new RegExp(passages.map(escapeRegExp).join('|'), 'g'),
+    (answer) => `[[BLANK:${ids.get(answer)}]]`
+  )
+  validateBenchmarkRedaction({ referenceSpec, redactedSpec, blanks })
+  return redactedSpec
+}
+
 /** Every mask must restore the exact reference, including repeated occurrences. */
 export function validateBenchmarkRedaction(
   artifacts: Pick<BenchmarkArtifacts, 'referenceSpec' | 'redactedSpec' | 'blanks'>

apps/sim/lib/benchmarks/prompts.ts

@@ -14,13 +14,13 @@ function messages(instruction: string, data: unknown): ExecuteMessage[] {
 export function distillationMessages(taskBrief: string): ExecuteMessage[] {
   return messages(
     'Explore the selected workspace using the read-only workspace CLI and describe its implemented behavior as a detailed, self-contained reference specification. Begin by listing all active workflows, follow list pagination, inspect each workflow and its referenced workspace resources, and follow large-output continuations. Cover exact triggers and input shapes, conditions, ownership, field mappings, prompts and code behavior, actions, destinations, cross-workflow relationships, outputs and failure/recovery behavior. Preserve concrete names, values and business rules; do not compress them into a high-level overview. Distinguish implemented behavior from unresolved configuration or inferred intent. Do not invent missing values. Omit editor layout and credential values. Also draft a short taskBrief expressing the business goal a user would originally request, without disclosing the enterprise-specific answers. If a taskBrief was supplied, preserve it. The reference is for human review before evaluation. Use tools to inspect; return the final JSON only when inspection is complete.',
-    { taskBrief }
+    taskBrief.trim() ? { taskBrief } : {}
   )
 }
 
 export function redactionMessages(referenceSpec: string): ExecuteMessage[] {
   return messages(
-    'Select 5 to 15 meaningful enterprise-specific facts from this reference specification that an agent should discover from enterprise context. Prefer ownership, conditions, mappings, existing mechanisms and business destinations. Exclude arbitrary implementation choices or details unlikely to exist outside this completed workflow. Produce redactedSpec by replacing exact passages with [[BLANK:id]] markers and blanks containing each id and its exact original answer. Preserve every other character of the reference, including whitespace and punctuation. Use the same marker for repeated identical answers and redact all occurrences that reveal an answer. Use short meaningful IDs. Do not paraphrase or add text. Return at least one blank; the human will review whether each is discoverable.',
+    'Select 5 to 15 meaningful enterprise-specific facts from this reference specification that an agent should discover from enterprise context. Prefer ownership, conditions, mappings, existing mechanisms and business destinations. Exclude arbitrary implementation choices or details unlikely to exist outside this completed workflow. Return blanks containing a short meaningful id and an answer copied as an exact contiguous passage of the reference. Select distinct, non-overlapping answers; the server replaces every exact occurrence with the same marker and preserves all other text. Avoid facts whose answer is revealed by a different surviving passage. Do not paraphrase or return a rewritten reference. Return at least one blank; the human will review whether each is discoverable.',
     { referenceSpec }
   )
 }