fix(logs): mark withheld trace span content so logs query stops reporting it as missing (#9314)

- Secret projection sets `contentWithheld: true` on a span when it omits any of its content (input/output/thinking/errorMessage, tool-call or provider-segment content) and on every span of a structural-only fallback; marker only, no content restored
- v2 `LogTraceSpan` contract documents the optional marker (additive); regenerated openapi-v2-logs.json
- `logs query` reports a withheld note and a `contentWithheld` count instead of "path not found on span"; under --where, withheld runs are kept (with --field still resolved) instead of silently filtered out

Author: waleedlatif1

apps/docs/openapi-v2-logs.json

@@ -1448,6 +1448,11 @@
             "type": "number",
             "description": "Offset from the root span in milliseconds."
           },
+          "contentWithheld": {
+            "type": "boolean",
+            "const": true,
+            "description": "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
+          },
           "toolCalls": {
             "type": "array",
             "items": {

apps/sim/lib/api/contracts/logs.ts

@@ -218,6 +218,7 @@ export type LogTraceSpan = {
   tokens?: number | { total?: number; input?: number; output?: number }
   cost?: { total?: number; input?: number; output?: number; toolCost?: number }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<z.output<typeof toolCallSchema>>
   children?: LogTraceSpan[]
 }
@@ -273,6 +274,12 @@ export const traceSpanSchema: z.ZodType<LogTraceSpan> = z
           .number()
           .describe('Offset from the root span in milliseconds.')
           .optional(),
+        contentWithheld: z
+          .literal(true)
+          .describe(
+            "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
+          )
+          .optional(),
         toolCalls: z.array(toolCallSchema).describe('Tool calls recorded by the span.').optional(),
         children: z.array(traceSpanSchema).describe('Nested child trace spans.').optional(),
       })

apps/sim/lib/logs/execution/trace-secret-projection.test.ts

@@ -502,6 +502,85 @@ describe('projectTraceSpansForSecrets', () => {
     expect(span.children?.[0]).not.toHaveProperty('output')
   })
 
+  it('marks every span of a structural-only fallback as content-withheld', async () => {
+    const source = [
+      createSpan({
+        output: { secret: 'raw' },
+        children: [createSpan({ id: 'child-1', output: { secret: 'raw' } })],
+      }),
+    ]
+
+    const incomplete = await projectTraceSpansForSecrets(source, {
+      registry: createRegistry([], false),
+      store: STORE,
+    })
+    const failedMatcher = await projectTraceSpansForSecrets(source, {
+      registry: createRegistry([
+        { plaintext: 's'.repeat(64 * 1024 + 1), replacement: '{{TOO_LARGE}}' },
+      ]),
+      store: STORE,
+    })
+    const failedInvariant = await enforceTraceSpanSecretInvariant(
+      [createSpan({ output: { apiKey: '[REDACTED]' } })],
+      {
+        registry: createRegistry([{ plaintext: 'REDACTED', replacement: '{{X}}' }]),
+        store: STORE,
+      }
+    )
+
+    for (const [span] of [incomplete, failedMatcher]) {
+      expect(span.contentWithheld).toBe(true)
+      expect(span.children?.[0].contentWithheld).toBe(true)
+    }
+    expect(failedInvariant[0].contentWithheld).toBe(true)
+  })
+
+  it('marks a span whose content field was omitted and leaves fully projected spans unmarked', async () => {
+    const [omitted, intact] = await projectTraceSpansForSecrets(
+      [
+        createSpan({ input: { safe: 'keep' }, output: { secretval: 1, '{{TOKEN}}': 2 } }),
+        createSpan({ id: 'span-2', output: { value: 'secretval' } }),
+      ],
+      {
+        registry: createRegistry([{ plaintext: 'secretval', replacement: '{{TOKEN}}' }]),
+        store: STORE,
+      }
+    )
+
+    expect(omitted.input).toEqual({ safe: 'keep' })
+    expect(omitted.contentWithheld).toBe(true)
+    expect(intact.output).toEqual({ value: '{{TOKEN}}' })
+    expect(intact).not.toHaveProperty('contentWithheld')
+  })
+
+  it('marks a span whose tool-call content was omitted', async () => {
+    const [span] = await projectTraceSpansForSecrets(
+      [
+        createSpan({
+          output: { safe: 'keep' },
+          toolCalls: [
+            {
+              name: 'legacy',
+              duration: 1,
+              startTime: '2026-07-01T00:00:00.000Z',
+              endTime: '2026-07-01T00:00:00.001Z',
+              status: 'success',
+              output: { secretval: 1, '{{TOKEN}}': 2 },
+            },
+          ],
+        }),
+      ],
+      {
+        registry: createRegistry([{ plaintext: 'secretval', replacement: '{{TOKEN}}' }]),
+        store: STORE,
+      }
+    )
+
+    expect(span.output).toEqual({ safe: 'keep' })
+    expect(span.toolCalls?.[0]).not.toHaveProperty('output')
+    expect(span.contentWithheld).toBe(true)
+  })
+
   it('fails closed when a later log transform reintroduces an active literal', async () => {
     const source = [createSpan({ output: { apiKey: '[REDACTED]' } })]
 

apps/sim/lib/logs/execution/trace-secret-projection.ts

@@ -82,6 +82,8 @@ interface ProjectionContext {
   sourceLargeValueBytes: number
   storedLargeValueBytes: number
   storedLargeValueCount: number
+  /** Content fields omitted so far; a span compares it across its own fields to set `contentWithheld`. */
+  omittedContentFields: number
 }
 
 interface OversizedHydrationGate {
@@ -162,6 +164,7 @@ function createProjectionContext(
     manifestIds: new WeakMap<object, string>(),
     nextManifestId: 0,
     largeValueCount: 0,
+    omittedContentFields: 0,
     sourceLargeValueBytes: 0,
     storedLargeValueBytes: 0,
     storedLargeValueCount: 0,
@@ -808,6 +811,7 @@ async function sanitizeContentField(
     logger.warn('Omitting trace content that could not be sanitized', {
       failure: describeProjectionFailure(error),
     })
+    context.omittedContentFields += 1
     return OMIT
   }
 }
@@ -927,6 +931,7 @@ async function sanitizeTraceSpan(
   }
 
   let projected = omit(span, ['displayResolvedSecretTraceProvenance']) as TraceSpan
+  const omittedBefore = context.omittedContentFields
 
   for (const key of ['input', 'output', 'thinking', 'errorMessage'] as const) {
     const value = span[key]
@@ -961,6 +966,8 @@ async function sanitizeTraceSpan(
     projected.providerTiming = { ...span.providerTiming, segments }
   }
 
+  if (context.omittedContentFields > omittedBefore) projected.contentWithheld = true
+
   assertContentIsSafe(projected, depth)
 
   if (span.children !== undefined) {
@@ -1074,6 +1081,7 @@ function structuralOnlySpan(
 
   return {
     ...structural,
+    contentWithheld: true,
     ...(modelToolCalls
       ? {
           modelToolCalls: projectBoundedArray(modelToolCalls, (call) =>

apps/sim/lib/logs/types.ts

@@ -308,6 +308,14 @@ export interface TraceSpan {
    * one read, not the run.
    */
   childTraceAccess?: 'granted' | 'disabled' | 'missing' | 'truncated'
+  /**
+   * Set by secret projection when it omitted content it could not prove safe from
+   * this span (`input`, `output`, `thinking`, `errorMessage`, or its tool-call and
+   * provider-segment content), or reduced the span to structure only. Status, timing,
+   * and usage remain. Without it a withheld field is
+   * indistinguishable from one the block never produced.
+   */
+  contentWithheld?: true
   model?: string
   cost?: {
     input?: number

apps/sim/lib/mothership/agent-cli/engines.test.ts

@@ -101,6 +101,68 @@ describe('logs query', () => {
   })
 })
 
+const WITHHELD_RUNS = {
+  [RUNS_PATH]: {
+    data: [
+      { runId: 'run-1', status: 'completed' },
+      { runId: 'run-2', status: 'completed' },
+    ],
+  },
+  '/api/v2/logs/run-1': {
+    data: {
+      traceSpans: [{ name: 'Count rows', status: 'success', output: { result: { count: 6 } } }],
+    },
+  },
+  // The log projection withheld this span's content; its structure survived.
+  '/api/v2/logs/run-2': {
+    data: {
+      traceSpans: [{ name: 'Count rows', status: 'success', duration: 12, contentWithheld: true }],
+    },
+  },
+}
+
+async function logsQueryWithheld(flags: Record<string, string>) {
+  const result = await runEngine('logs query', ['wf-1'], runtimeWith(WITHHELD_RUNS), {
+    block: 'Count rows',
+    ...flags,
+  })
+  expect(result.exitCode).toBe(0)
+  return JSON.parse(result.stdout)
+}
+
+describe('logs query on a span whose content was withheld', () => {
+  it('reports withheld content instead of a missing path, and counts it', async () => {
+    const report = await logsQueryWithheld({ field: 'result.count' })
+    expect(report.rows[0]).toMatchObject({ runId: 'run-1', value: 6 })
+    expect(report.rows[1]).toMatchObject({ runId: 'run-2', hits: 1, value: null })
+    expect(report.rows[1].note).not.toBe('path not found on span')
+    expect(report.rows[1].note).toMatch(/withheld/)
+    expect(report.contentWithheld).toBe(1)
+  })
+
+  it('still answers structural paths on a withheld span', async () => {
+    const report = await logsQueryWithheld({ field: 'duration' })
+    expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 })
+    expect(report.rows[1].note).toBeUndefined()
+    expect(report.contentWithheld).toBeUndefined()
+  })
+
+  it('keeps a withheld run under --where instead of counting it as filtered out', async () => {
+    const report = await logsQueryWithheld({ where: 'result.count=6' })
+    expect(report.filteredOut).toBe(0)
+    expect(report.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1', 'run-2'])
+    expect(report.rows[1].note).toMatch(/withheld/)
+    expect(report.contentWithheld).toBe(1)
+  })
+
+  it('still resolves a readable --field when --where hits withheld content', async () => {
+    const report = await logsQueryWithheld({ field: 'duration', where: 'result.count=6' })
+    expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 })
+    expect(report.rows[1].note).toMatch(/--where/)
+    expect(report.contentWithheld).toBe(1)
+  })
+})
+
 const DEPS_STATE = {
   blocks: {
     'trigger-1': { type: 'starter', name: 'Start' },

apps/sim/lib/mothership/agent-cli/engines/query.ts

@@ -23,6 +23,13 @@ const MAX_LIMIT = 50
 const TRACE_FETCH_CONCURRENCY = 5
 const VALUE_MAX_CHARS = 600
 const PATH_NOT_FOUND_NOTE = 'path not found on span'
+/**
+ * The log projection withheld this span's content rather than return what it could not
+ * prove safe to display, so an unresolved content path says nothing about the run.
+ */
+const CONTENT_WITHHELD_NOTE = 'span content withheld by the log projection; value unknown'
+const WHERE_WITHHELD_NOTE =
+  'span content withheld by the log projection; --where could not be checked, run kept'
 
 /**
  * Keys that live on the span itself. A path whose head is anything else is read
@@ -56,6 +63,7 @@ interface QueryTraceSpan {
   duration?: number
   input?: Record<string, unknown>
   output?: Record<string, unknown>
+  contentWithheld?: true
   children?: QueryTraceSpan[]
 }
 
@@ -158,6 +166,7 @@ export const logsQueryCommand: AgentCliEngine = {
 
     let filteredOut = 0
     let missingTrace = 0
+    let contentWithheld = 0
     let fieldResolvedUnder: 'output' | undefined
     let whereResolvedUnder: 'output' | undefined
     const rows = await mapWithConcurrency(
@@ -193,23 +202,34 @@ export const logsQueryCommand: AgentCliEngine = {
           }
           return { ...base, hits: 0, value: null }
         }
+        const matchedRow = {
+          ...base,
+          hits: matches.length,
+          blockStatus: last.status ?? 'success',
+        }
+        let whereWithheld = false
         if (wherePath) {
           const matched = resolveSpanPathWithFallback(last, wherePath)
           if (matched.resolvedUnder) whereResolvedUnder = matched.resolvedUnder
-          if (String(matched.value) !== whereValue) {
+          // Filtering on a withheld value would silently drop a run that may well match.
+          whereWithheld = matched.value === undefined && last.contentWithheld === true
+          if (!whereWithheld && String(matched.value) !== whereValue) {
             filteredOut++
             return undefined
           }
         }
         const resolved = resolveSpanPathWithFallback(last, field)
         if (resolved.resolvedUnder) fieldResolvedUnder = resolved.resolvedUnder
-        return {
-          ...base,
-          hits: matches.length,
-          blockStatus: last.status ?? 'success',
-          value: clipValue(resolved.value),
-          ...(resolved.value === undefined ? { note: PATH_NOT_FOUND_NOTE } : {}),
-        }
+        const fieldWithheld = resolved.value === undefined && last.contentWithheld === true
+        if (whereWithheld || fieldWithheld) contentWithheld++
+        const note = fieldWithheld
+          ? CONTENT_WITHHELD_NOTE
+          : resolved.value === undefined
+            ? PATH_NOT_FOUND_NOTE
+            : whereWithheld
+              ? WHERE_WITHHELD_NOTE
+              : undefined
+        return { ...matchedRow, value: clipValue(resolved.value), ...(note ? { note } : {}) }
       }
     )
 
@@ -230,6 +250,7 @@ export const logsQueryCommand: AgentCliEngine = {
               }
             : {}),
           ...(missingTrace ? { missingTrace } : {}),
+          ...(contentWithheld ? { contentWithheld } : {}),
           rows: kept,
         },
         null,

packages/sim-cli/src/generated/v2-api.ts

@@ -6244,6 +6244,7 @@ type GetLogResponseRef1 = {
     toolCost?: number
   }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<{
     id?: string
     name?: string
@@ -9037,6 +9038,7 @@ type ListLogsResponseRef2 = {
     toolCost?: number
   }
   relativeStartMs?: number
+  contentWithheld?: true
   toolCalls?: Array<{
     id?: string
     name?: string