fix(content): preserve canonical provenance across reads (#9036)

Author: icecrasher321

apps/sim/lib/knowledge/__integration__/content-delivery.integration.ts

@@ -16,6 +16,7 @@ import {
 } from '@sim/db/schema'
 import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
 import { sha256Hex } from '@sim/security/hash'
+import { createExecutorPrincipal } from '@sim/testing/factories/principal.factory'
 import { generateId } from '@sim/utils/id'
 import { isRecordLike } from '@sim/utils/object'
 import { eq, inArray } from 'drizzle-orm'
@@ -33,6 +34,12 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({
 import { observeContentDelivery } from '@/lib/core/application'
 import { encryptSecret } from '@/lib/core/security/encryption'
 import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance'
+import {
+  PRIVATE_TOOL_METADATA_REQUEST_HEADER,
+  RESOLVED_SECRET_PROVENANCE_FIELD,
+  RESOLVED_SECRET_PROVENANCE_METADATA_V1,
+} from '@/lib/execution/private-tool-metadata'
+import { listDocumentsOperation } from '@/lib/internal/knowledge/operations'
 import {
   createKnowledgeAclFixtureIds,
   seedKnowledgeAclFixture,
@@ -51,6 +58,7 @@ import {
   readDetailedKnowledgeTagUsage,
   readKnowledgeTagUsage,
 } from '@/lib/knowledge/application/tags'
+import * as documentService from '@/lib/knowledge/documents/service'
 import {
   bindKnowledgeDocumentFieldSecretProvenance,
   classifyKnowledgeDocumentMetadata,
@@ -63,6 +71,7 @@ import { embeddingVectorValues } from '@/lib/knowledge/vector-columns'
 import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation'
 import { createRegistryContentGate } from '@/lib/mothership/content-delivery/registry-gate'
 import { uploadFile } from '@/lib/uploads/core/storage-service'
+import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 const SECRET = 'fixture-knowledge-token-41875'
@@ -95,6 +104,7 @@ async function seedDocument(
     content?: string
     provenance?: DurableSecretProvenance
     original?: boolean
+    contentHash?: string | null
   } = {}
 ) {
   const content = options.content ?? 'page public content'
@@ -105,6 +115,7 @@ async function seedDocument(
       id,
       knowledgeBaseId: ids.knowledgeBaseId,
       filename: options.name ?? 'page.txt',
+      contentHash: options.contentHash ?? null,
       fileUrl:
         options.original === false
           ? ''
@@ -229,6 +240,81 @@ afterAll(async () => {
 })
 
 describe('knowledge content delivery', () => {
+  it.each([null, sha256Hex('stable connector source')])(
+    'keeps workflow list provenance complete for an unchanged document with content hash %s',
+    async (contentHash) => {
+      const doc = await seedDocument({
+        name: `${SECRET}.txt`,
+        contentHash,
+        provenance: bindKnowledgeDocumentFieldSecretProvenance(exact, 'filename', `${SECRET}.txt`),
+      })
+      const context = {
+        principal: createExecutorPrincipal({
+          subjectUserId: ids.aliceId,
+          workspaceId: ids.workspaceId,
+          audience: KNOWLEDGE_DELEGATION_AUDIENCE,
+          delegationContext: { kind: 'workflow_execution' as const, workflowId: generateId() },
+        }),
+        headers: new Headers({
+          [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1,
+        }),
+      }
+      const result = await listDocumentsOperation(
+        ids.knowledgeBaseId,
+        { limit: 50, offset: 0 },
+        context
+      )
+      const provenance = result.bodyFields?.[RESOLVED_SECRET_PROVENANCE_FIELD]
+      expect(provenance).toMatchObject({ complete: true })
+      expect(result.body).toMatchObject({
+        data: { documents: [{ id: doc.id, filename: `${SECRET}.txt` }] },
+      })
+      expect(JSON.stringify(result.body)).not.toContain('contentHash')
+      const registry = new ResolvedSecretTraceRegistry([], {
+        userId: ids.aliceId,
+        workspaceId: ids.workspaceId,
+      })
+      expect(await registry.importProvenance(provenance, { trusted: true })).toBe(true)
+      const projected = projectResolvedSecretModelContent(result.body, registry)
+      expect(projected.safe).toBe(true)
+      expect(JSON.stringify(projected)).toContain('{{FIXTURE_TOKEN}}.txt')
+      expect(JSON.stringify(projected)).not.toContain(SECRET)
+
+      const read = documentService.getDocuments
+      const replace = vi
+        .spyOn(documentService, 'getDocuments')
+        .mockImplementation(async (...args) => {
+          const snapshot = await read(...args)
+          await db.transaction(async (tx) => {
+            const [replacement] = await tx
+              .update(document)
+              .set({ contentHash: sha256Hex('replacement connector source') })
+              .where(eq(document.id, doc.id))
+              .returning()
+            await replaceKnowledgeDocumentSecretProvenanceInTx(
+              tx,
+              doc.id,
+              createKnowledgeDocumentSourceValue(replacement),
+              bindKnowledgeDocumentFieldSecretProvenance(exact, 'filename', `${SECRET}.txt`)
+            )
+          })
+          return snapshot
+        })
+      try {
+        const changed = await listDocumentsOperation(
+          ids.knowledgeBaseId,
+          { limit: 50, offset: 0 },
+          context
+        )
+        expect(changed.bodyFields?.[RESOLVED_SECRET_PROVENANCE_FIELD]).toMatchObject({
+          complete: false,
+        })
+      } finally {
+        replace.mockRestore()
+      }
+    }
+  )
+
   it('classifies a mixed metadata page without materializing multiple inline originals together', async () => {
     const content = 'inline source content '.repeat(8192)
     const legacy = await seedDocument({ content })

apps/sim/lib/knowledge/documents/service.ts

@@ -217,7 +217,7 @@ const logger = createLogger('DocumentService')
  * and a machine-readable name would only invite a client to read resource
  * existence into it.
  */
-export class KnowledgeBaseFileOwnershipError extends OrchestrationError {
+class KnowledgeBaseFileOwnershipError extends OrchestrationError {
   constructor(public readonly storageKey: string) {
     super('forbidden', 'Document file is not owned by this knowledge base')
     this.name = 'KnowledgeBaseFileOwnershipError'
@@ -2861,6 +2861,7 @@ export async function getDocuments(
     knowledgeBaseId: string
     filename: string
     fileUrl: string
+    contentHash: string | null
     fileSize: number
     mimeType: string
     chunkCount: number
@@ -2968,6 +2969,7 @@ export async function getDocuments(
         knowledgeBaseId: document.knowledgeBaseId,
         filename: document.filename,
         fileUrl: document.fileUrl,
+        contentHash: document.contentHash,
         fileSize: document.fileSize,
         mimeType: document.mimeType,
         chunkCount: document.chunkCount,
@@ -3084,6 +3086,7 @@ export async function getDocuments(
       knowledgeBaseId: doc.knowledgeBaseId,
       filename: doc.filename,
       fileUrl: doc.fileUrl,
+      contentHash: doc.contentHash,
       fileSize: doc.fileSize,
       mimeType: doc.mimeType,
       chunkCount: doc.chunkCount,

apps/sim/lib/mothership/__integration__/content-delivery.integration.ts

@@ -152,6 +152,7 @@ import { storeCompiledDoc } from '@/lib/uploads/documents/compiled-store'
 import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization'
 import { downloadWorkspaceFileItems } from '@/lib/workspace-files/application/download-workspace-file-items'
 import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content'
+import { readWorkspaceFileSecretProvenance } from '@/lib/workspace-files/application/read-workspace-file-secret-provenance'
 import {
   getWorkspaceFileShares,
   updateWorkspaceFileShare,
@@ -1130,21 +1131,42 @@ describe('content reaching Chat is admitted per item', () => {
         userId: fixture.ids.aliceId,
         workspaceId: ws(),
       })
-      const result = await observeContentDelivery(
-        createRegistryContentGate({
-          registry,
-          actorUserId: fixture.ids.aliceId,
-          workspaceId: ws(),
-        }),
-        () =>
-          readWorkspaceFileContent.execute({
-            principal,
-            input: { fileId: created.fileId, includeSecretProvenance: true },
-          })
+      const gate = createRegistryContentGate({
+        registry,
+        actorUserId: fixture.ids.aliceId,
+        workspaceId: ws(),
+      })
+      const result = await observeContentDelivery(gate, () =>
+        readWorkspaceFileContent.execute({
+          principal,
+          input: { fileId: created.fileId, includeSecretProvenance: true },
+        })
       )
       expect(result.content.toString()).toBe(content)
       expect(result.file.vfsNamespace).toBe(context === 'test' ? 'tests' : 'changelog')
       expect(result.secretProvenance).toEqual({ status: 'exact', entries: [] })
+      const evidence = await observeContentDelivery(gate, () =>
+        readWorkspaceFileSecretProvenance.execute({
+          principal,
+          input: { fileId: created.fileId },
+        })
+      )
+      expect.soft(evidence.provenance).toEqual({ status: 'exact', entries: [] })
+      const staleEvidence = await observeContentDelivery(gate, () =>
+        readWorkspaceFileSecretProvenance.execute({
+          principal,
+          input: { fileId: created.fileId, expectedContentUpdatedAt: new Date(0) },
+        })
+      )
+      expect(staleEvidence.provenance).toEqual({ status: 'unknown' })
+      if (context === 'changelog') {
+        const version = await cli(
+          'GET',
+          `/api/v2/files/${created.fileId}/versions/1/text?workspaceId=${ws()}`
+        )
+        expect.soft(version.status).toBe(200)
+        expect.soft(modelView(version)).toContain(content)
+      }
       featureFlagsMockFns.mockIsFeatureEnabled.mockReset()
     }
   )

apps/sim/lib/mothership/request/tools/client.ts

@@ -347,10 +347,7 @@ export async function waitForWorkflowToolCompletion({
 
   try {
     completion = await waitForToolCompletion(toolCallId, timeoutMs, abortSignal)
-    if (!completion) {
-      toolRegistry?.markIncomplete('client-tool-completion-missing')
-      return null
-    }
+    if (!completion) return null
 
     const executionId = getWorkflowToolCompletionExecutionId(completion.data)
     const launchError = getWorkflowToolLaunchError(completion.data)

apps/sim/lib/workspace-files/application/file-versions.ts

@@ -30,7 +30,10 @@ import {
 import { hasObjectNotFoundCause } from '@/lib/uploads/core/errors'
 import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
 import { formatFileSize } from '@/lib/uploads/utils/file-utils'
-import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content'
+import {
+  requireAdmittedWorkspaceFile,
+  workspaceFileProvenanceIdentity,
+} from '@/lib/workspace-files/application/admit-file-content'
 import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
 import {
   type DownloadWorkspaceFileStreamResult,
@@ -128,12 +131,10 @@ async function readVersionSecretProvenance(
 ): Promise<WorkspaceFileSecretProvenance | undefined> {
   if (!mustClassifyContent(principal)) return undefined
   if (version.isCurrent) {
-    return getBoundWorkspaceFileSecretProvenance(file.workspaceId, {
-      fileId: file.id,
-      key: file.key,
-      context: file.storageContext ?? 'workspace',
-      contentUpdatedAt: file.contentUpdatedAt ?? undefined,
-    })
+    return getBoundWorkspaceFileSecretProvenance(
+      file.workspaceId,
+      workspaceFileProvenanceIdentity(file)
+    )
   }
   const snapshot = await getWorkspaceFileVersionProvenance(file.id, version.version, version.key)
   return snapshot ? workspaceFileSecretProvenanceFromSnapshot(snapshot) : { status: 'unknown' }

apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts

@@ -6,6 +6,7 @@ import {
   getBoundWorkspaceFileSecretProvenance,
   type WorkspaceFileSecretProvenance,
 } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
+import { workspaceFileProvenanceIdentity } from '@/lib/workspace-files/application/admit-file-content'
 import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
 import { fileOperations } from '@/lib/workspace-files/application/operations'
 import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context'
@@ -39,9 +40,7 @@ export const readWorkspaceFileSecretProvenance = defineAuthorizedWorkspaceFileUs
     await admitContent(principal, [])
     return {
       provenance: await getBoundWorkspaceFileSecretProvenance(context.workspaceId, {
-        fileId: file.id,
-        key: file.key,
-        context: 'workspace',
+        ...workspaceFileProvenanceIdentity(file),
         contentUpdatedAt: input.expectedContentUpdatedAt,
       }),
       ownerUserId: file.uploadedBy,