diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fce5f4d359c..c57afab1d3b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -243,6 +243,14 @@ jobs: working-directory: packages/db run: bun run db:migrate + - name: Build native isolate addon for Project file rendering + if: matrix.group == 'project-files' + working-directory: apps/sim + run: | + addon_dir="$(node -p 'require("node:path").dirname(require.resolve("isolated-vm/package.json", { paths: ["./apps/sim"] }))')" + cd "$addon_dir" + bun run install + # No step timeout: the job's bound covers a hang without cutting a slow but healthy suite # short of writing its report. - name: Run end-to-end suites diff --git a/apps/sim/app/api/desktop/tool/file/route.ts b/apps/sim/app/api/desktop/tool/file/route.ts index 9829a64e681..9a2e923dc03 100644 --- a/apps/sim/app/api/desktop/tool/file/route.ts +++ b/apps/sim/app/api/desktop/tool/file/route.ts @@ -21,8 +21,8 @@ import { import { PayloadSizeLimitError, readStreamToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { workspaceFileVfsPath } from '@/lib/uploads/contexts/workspace' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { internalFileErrorPolicies } from '@/lib/workspace-files/api' -import { encodeFilenameForHeader } from '@/app/api/files/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index eb27d10003f..9837d594961 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -13,6 +13,7 @@ import { captureServerEvent } from '@/lib/posthog/server' import type { StorageContext } from '@/lib/uploads/config' import { getServeStoragePrefix } from '@/lib/uploads/config' import { downloadFile } from '@/lib/uploads/core/storage-service' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { extractEmbeddedFileRefs } from '@/lib/uploads/server/embedded-image-refs' import { createMarkdownExport, @@ -27,7 +28,6 @@ import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' import { storedFileId } from '@/lib/uploads/utils/embedded-image-ref' import { formatFileSize } from '@/lib/uploads/utils/file' import { verifyFileAccess } from '@/app/api/files/authorization' -import { encodeFilenameForHeader } from '@/app/api/files/utils' const logger = createLogger('FilesExportAPI') diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index 447460a1b8e..d0289ae7e21 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -13,7 +13,12 @@ import { createSessionPrincipal, } from '@sim/testing/factories/principal.factory' import { createRouteContext } from '@sim/testing/helpers/http' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { fileUtilsMock, fileUtilsMockFns } from '@sim/testing/mocks/file-utils.mock' +import { + fileUtilsServerMock, + fileUtilsServerMockFns, +} from '@sim/testing/mocks/file-utils-server.mock' import { filesAuthorizationMock, filesAuthorizationMockFns, @@ -36,6 +41,8 @@ const { mockReadFile, mockAuthenticateWorkspaceFile, mockReadWorkspaceFileContentByKey, + mockReadWorkspaceFileRecordByKey, + mockAuthorizeWorkspaceFileRecordByKey, mockResolveServableDocBytes, mockGetContentType, mockFindLocalFile, @@ -57,6 +64,8 @@ const { mockReadFile: vi.fn(), mockAuthenticateWorkspaceFile: vi.fn(), mockReadWorkspaceFileContentByKey: vi.fn(), + mockReadWorkspaceFileRecordByKey: vi.fn(), + mockAuthorizeWorkspaceFileRecordByKey: vi.fn(async () => undefined), mockResolveServableDocBytes: vi.fn(), mockGetContentType: vi.fn(), mockFindLocalFile: vi.fn(), @@ -86,6 +95,9 @@ vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/uploads/utils/file', () => fileUtilsMock) +vi.mock('@/lib/uploads/utils/file.server', () => fileUtilsServerMock) +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) + vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@/lib/uploads/setup.server', () => ({})) @@ -106,6 +118,10 @@ vi.mock('@/lib/workspace-files/api', () => ({ vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ readWorkspaceFileContentByKey: { execute: mockReadWorkspaceFileContentByKey }, + readWorkspaceFileRecordByKey: { + execute: mockReadWorkspaceFileRecordByKey, + authorize: mockAuthorizeWorkspaceFileRecordByKey, + }, })) vi.mock('@/lib/uploads/documents/compile', () => ({ @@ -161,6 +177,17 @@ describe('File Serve API Route', () => { }, content: Buffer.from('generated source'), }) + mockReadWorkspaceFileRecordByKey.mockResolvedValue({ + file: { + id: 'file-1', + workspaceId: 'test-workspace-id', + name: 'report.pdf', + key: 'workspace/test-workspace-id/report.pdf', + type: 'text/x-pdflibjs', + size: 16, + contentUpdatedAt: new Date('2026-09-06'), + }, + }) mockResolveServableDocBytes.mockImplementation( async ({ rawBuffer, fileName }: { rawBuffer: Buffer; fileName: string }) => ({ buffer: rawBuffer, @@ -173,24 +200,23 @@ describe('File Serve API Route', () => { mockReadFile(filePath) ) mockCreateFileResponse.mockImplementation( - (file: { buffer: Buffer; contentType: string; filename: string }) => { + (file: { buffer: Buffer; contentType: string; filename: string; cacheControl?: string }) => { return new Response(file.buffer, { status: 200, headers: { 'Content-Type': file.contentType, + ...(file.cacheControl ? { 'Cache-Control': file.cacheControl } : {}), 'Content-Disposition': `inline; filename="${file.filename}"`, }, }) } ) - // Delegates so the existing assertions on the response payload — including its - // Cache-Control — read the same call list whichever helper the route reached for. mockCreateConditionalFileResponse.mockImplementation((file: unknown) => mockCreateFileResponse(file) ) - mockFileErrorResponse.mockImplementation((error: Error) => { + mockFileErrorResponse.mockImplementation((error: Error, status = 500) => { return new Response(JSON.stringify({ error: error.name, message: error.message }), { - status: error.name === 'FileNotFoundError' ? 404 : 500, + status: error.name === 'FileNotFoundError' ? 404 : status, headers: { 'Content-Type': 'application/json' }, }) }) @@ -284,9 +310,9 @@ describe('File Serve API Route', () => { // The real createFileErrorResponse owns the status mapping; mirror it here so the // route's own error path is what decides, not the mock's default 500. mockFileErrorResponse.mockImplementation( - (error: Error) => + (error: Error, status = 500) => new Response(JSON.stringify({ error: error.name }), { - status: error.name === 'PayloadSizeLimitError' ? 413 : 500, + status: error.name === 'PayloadSizeLimitError' ? 413 : status, }) ) @@ -312,32 +338,48 @@ describe('File Serve API Route', () => { }, }) - async function serveVersionedDoc(dependsOnReferencedFiles: boolean) { + async function serveVersionedDoc( + dependsOnReferencedFiles: boolean, + method: 'GET' | 'HEAD' = 'GET' + ) { mockResolveStoredFileContext.mockResolvedValue('workspace') mockParseWorkspaceFileKey.mockReturnValue('test-workspace-id') mockAuthenticateWorkspaceFile.mockResolvedValue(principal) - mockResolveServableDocBytes.mockResolvedValue({ + fileUtilsServerMockFns.mockDownloadServableFileFromStorage.mockResolvedValue({ buffer: Buffer.from('compiled'), contentType: 'application/pdf', ...(dependsOnReferencedFiles ? { dependsOnReferencedFiles: true } : {}), }) const req = createMockRequest({ + method, url: 'http://localhost:3000/api/files/serve/workspace/test-workspace-id/report.pdf?v=1756684800000', }) - await GET(req, createRouteContext({ path: ['workspace', 'test-workspace-id', 'report.pdf'] })) - return mockCreateFileResponse.mock.calls.at(-1)?.[0] + const response = await GET( + req, + createRouteContext({ path: ['workspace', 'test-workspace-id', 'report.pdf'] }) + ) + expect(response.status).toBe(200) + return response } + it('authorizes HEAD without compiling a representation or inventing its length', async () => { + const response = await serveVersionedDoc(true, 'HEAD') + expect(await response.text()).toBe('') + expect(response.headers.get('Content-Length')).toBeNull() + expect(response.headers.get('Cache-Control')).toBe('private, no-store') + expect(mockReadWorkspaceFileRecordByKey).toHaveBeenCalled() + expect(fileUtilsServerMockFns.mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + }) + it('caches a versioned document immutably when its bytes derive from the stored source alone', async () => { - expect(await serveVersionedDoc(false)).toEqual( - expect.objectContaining({ cacheControl: 'private, max-age=31536000, immutable' }) - ) + const response = await serveVersionedDoc(false) + expect(response.headers.get('Cache-Control')).toBe('private, max-age=31536000, immutable') }) - it('attaches a validator to a revalidated response so the next check can be answered 304', async () => { - await serveVersionedDoc(true) - expect(mockCreateConditionalFileResponse).toHaveBeenCalled() + it('requires revalidation for a versioned document with referenced inputs', async () => { + const response = await serveVersionedDoc(true) + expect(response.headers.get('Cache-Control')).toBe('private, no-cache, must-revalidate') }) }) }) diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index fb06b9262a1..9e0d5c52732 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -1,4 +1,4 @@ -import { type Principal, requirePrincipalSubjectUserId } from '@sim/auth/principal' +import type { Principal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' @@ -10,7 +10,7 @@ import { internalSessionAuth, } from '@/lib/api/server/routes' import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads' @@ -20,12 +20,18 @@ import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/file-man import { downloadFile } from '@/lib/uploads/core/storage-service' import { resolveServableDocBytes } from '@/lib/uploads/documents/compile' import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' +import { FILE_CACHE_CONTROL, workspaceFileCacheControl } from '@/lib/uploads/server/delivery' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' import { resolveStoredFileContext } from '@/lib/uploads/server/metadata' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { inferContextFromKey } from '@/lib/uploads/utils/file' import { internalWorkspaceFileServeAuth } from '@/lib/workspace-files/api' -import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { fetchAuthorizedServableWorkspaceFileBuffer } from '@/lib/workspace-files/application/fetch-servable-workspace-file-buffer' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' +import { + readWorkspaceFileContentByKey, + readWorkspaceFileRecordByKey, +} from '@/lib/workspace-files/application/read-workspace-file-content-by-key' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' @@ -212,11 +218,11 @@ function getWorkspaceIdForCompile(key: string): string | undefined { return parseWorkspaceFileKey(key) ?? undefined } -const IMMUTABLE_CACHE_CONTROL = 'private, max-age=31536000, immutable' -const WORKSPACE_REVALIDATE_CACHE_CONTROL = 'private, no-cache, must-revalidate' +const IMMUTABLE_CACHE_CONTROL = FILE_CACHE_CONTROL.immutable +const WORKSPACE_REVALIDATE_CACHE_CONTROL = FILE_CACHE_CONTROL.revalidate /** For the genuinely-public, pre-auth asset routes (avatars, OG images, workspace logos) — these are * intentionally shared-cacheable. Passed EXPLICITLY so the default response cache stays `private`. */ -const PUBLIC_ASSET_CACHE_CONTROL = 'public, max-age=31536000' +const PUBLIC_ASSET_CACHE_CONTROL = FILE_CACHE_CONTROL.publicAsset /** * Cache-Control for a served file. @@ -238,6 +244,7 @@ function resolveServeCacheControl( context: string | undefined, dependsOnReferencedFiles: boolean ): string | undefined { + if (context === 'workspace') return workspaceFileCacheControl(versioned, dependsOnReferencedFiles) if (versioned && !dependsOnReferencedFiles) return IMMUTABLE_CACHE_CONTROL return context === 'workspace' || dependsOnReferencedFiles ? WORKSPACE_REVALIDATE_CACHE_CONTROL @@ -269,6 +276,8 @@ export const GET = withRouteHandler( throw new FileNotFoundError('No file path provided') } + if (path[0] === 'project') throw new FileNotFoundError('File not found') + logger.info('File serve request:', { path }) const fullPath = path.join('/') @@ -331,7 +340,16 @@ export const GET = withRouteHandler( path, error: legacyAuthResult.error || 'Missing userId', }) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + return NextResponse.json( + { error: 'Unauthorized' }, + { + status: 401, + headers: { + 'Cache-Control': FILE_CACHE_CONTROL.noStore, + 'X-Content-Type-Options': 'nosniff', + }, + } + ) } const query = fileServeQuerySchema.parse({ @@ -363,7 +381,8 @@ export const GET = withRouteHandler( options, request.signal, storageContext, - knowledgeAccess + knowledgeAccess, + request.method === 'HEAD' ) } @@ -373,12 +392,22 @@ export const GET = withRouteHandler( options, request.signal, storageContext, - knowledgeAccess + knowledgeAccess, + request.method === 'HEAD' ) } catch (error) { if (error instanceof InternalUnauthenticatedError) { logger.warn('Unauthorized file access attempt', { error: error.message }) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + return NextResponse.json( + { error: 'Unauthorized' }, + { + status: 401, + headers: { + 'Cache-Control': FILE_CACHE_CONTROL.noStore, + 'X-Content-Type-Options': 'nosniff', + }, + } + ) } // An in-progress/incomplete doc source fails to compile — this is expected @@ -401,6 +430,15 @@ export const GET = withRouteHandler( return createFileErrorResponse(notFound) } + if (orchestrationError) { + if (orchestrationError.code === 'internal') + logServeFailure('Error serving file:', orchestrationError) + return createFileErrorResponse( + orchestrationError, + statusForOrchestrationError(orchestrationError.code) + ) + } + logServeFailure('Error serving file:', error) if (error instanceof FileNotFoundError) { @@ -423,23 +461,42 @@ async function handleWorkspaceFile( const workspaceId = getWorkspaceIdForCompile(key) if (!workspaceId) throw new FileNotFoundError(`File not found: ${key}`) - const { file, content } = await readWorkspaceFileContentByKey.execute({ - principal, - input: { key, assertedWorkspaceId: workspaceId }, - request, - }) - const ownerKey = `user:${requirePrincipalSubjectUserId(principal)}` - const resolved = await resolveServableBytes({ - buffer: content, - filename: file.name, - storageKey: key, - workspaceId, - options, - ownerKey, - filePrincipal: principal, - fileType: file.type, - signal: request.signal, - }) + const input = { key, assertedWorkspaceId: workspaceId } + const { file } = await readWorkspaceFileRecordByKey.execute({ principal, input, request }) + if (request.method === 'HEAD') { + return new NextResponse(null, { + headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, 'X-Content-Type-Options': 'nosniff' }, + }) + } + let resolved: ServableBytes + if (options.raw) { + const { content } = await readWorkspaceFileContentByKey.execute({ principal, input, request }) + resolved = { + buffer: content, + contentType: getContentType(file.name), + dependsOnReferencedFiles: false, + } + } else { + const rendered = await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: request.signal, + }) + const preview = options.preview ? await resolveServableImageBytes(rendered.buffer, key) : null + resolved = { + buffer: preview?.buffer ?? rendered.buffer, + contentType: preview?.contentType ?? rendered.contentType, + dependsOnReferencedFiles: rendered.dependsOnReferencedFiles, + } + assertKnownSizeWithinLimit( + resolved.buffer.length, + MAX_BUFFERED_TRANSFER_BYTES, + 'served file response' + ) + await finishFileDelivery({ + authorize: () => readWorkspaceFileRecordByKey.authorize({ principal, input, request }), + receipt: rendered.receipt, + }) + } logger.info('Workspace file served', { fileId: file.id, @@ -467,7 +524,8 @@ async function handleLocalFile( options: ServeOptions, signal: AbortSignal | undefined, context: StorageContext, - knowledgeAccess: KnowledgeFileAccess | undefined + knowledgeAccess: KnowledgeFileAccess | undefined, + head: boolean ): Promise { const ownerKey = `user:${userId}` try { @@ -484,6 +542,13 @@ async function handleLocalFile( throw new FileNotFoundError(`File not found: ${filename}`) } + if (head) + return new NextResponse(null, { + headers: { + 'Cache-Control': FILE_CACHE_CONTROL.noStore, + 'X-Content-Type-Options': 'nosniff', + }, + }) const filePath = await findLocalFile(filename) if (!filePath) { @@ -539,7 +604,8 @@ async function handleCloudProxy( options: ServeOptions, signal: AbortSignal | undefined, context: StorageContext, - knowledgeAccess: KnowledgeFileAccess | undefined + knowledgeAccess: KnowledgeFileAccess | undefined, + head: boolean ): Promise { const ownerKey = `user:${userId}` try { @@ -558,6 +624,13 @@ async function handleCloudProxy( throw new FileNotFoundError(`File not found: ${cloudKey}`) } + if (head) + return new NextResponse(null, { + headers: { + 'Cache-Control': FILE_CACHE_CONTROL.noStore, + 'X-Content-Type-Options': 'nosniff', + }, + }) let rawBuffer: Buffer if (context === 'copilot') { diff --git a/apps/sim/app/api/files/utils.test.ts b/apps/sim/app/api/files/utils.test.ts index 02536725612..f324c33f148 100644 --- a/apps/sim/app/api/files/utils.test.ts +++ b/apps/sim/app/api/files/utils.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { createConditionalFileResponse, createFileResponse, - encodeFilenameForHeader, extractFilename, findLocalFile, } from '@/app/api/files/utils' diff --git a/apps/sim/app/api/files/utils.ts b/apps/sim/app/api/files/utils.ts index b668142b9e9..f8b018c6c7f 100644 --- a/apps/sim/app/api/files/utils.ts +++ b/apps/sim/app/api/files/utils.ts @@ -1,11 +1,15 @@ -import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' import { isPayloadSizeLimitError, readNodeStreamToBufferWithLimit, } from '@/lib/core/utils/stream-limits' -import { ensureFileNameExtension, sanitizeFileKey } from '@/lib/uploads/utils/file' +import { + bufferedRepresentationEtag, + FILE_CACHE_CONTROL, + fileDeliveryHeaders, +} from '@/lib/uploads/server/delivery' +import { sanitizeFileKey } from '@/lib/uploads/utils/file' const logger = createLogger('FilesUtils') @@ -151,116 +155,20 @@ export async function findLocalFile(filename: string): Promise { } } -const SAFE_INLINE_TYPES = new Set([ - 'image/png', - 'image/jpeg', - 'image/jpg', - 'image/gif', - 'image/svg+xml', - 'image/webp', - 'image/avif', - 'image/bmp', - 'image/x-icon', - 'application/pdf', - 'text/plain', - 'text/csv', - 'application/json', -]) - -const FORCE_ATTACHMENT_EXTENSIONS = new Set(['html', 'htm', 'js', 'css', 'xml']) - -export function getSecureFileHeaders(filename: string, originalContentType: string) { - const extension = filename.split('.').pop()?.toLowerCase() || '' - - if (FORCE_ATTACHMENT_EXTENSIONS.has(extension)) { - return { - contentType: 'application/octet-stream', - disposition: 'attachment', - } - } - - let safeContentType = originalContentType - - if (originalContentType === 'text/html') { - safeContentType = 'text/plain' - } - - const disposition = SAFE_INLINE_TYPES.has(safeContentType) ? 'inline' : 'attachment' - - return { - contentType: safeContentType, - disposition, - } -} - -/** - * Percent-encode a filename as an RFC 8187 `ext-value`. - * - * `encodeURIComponent` alone is not enough: it leaves `'`, `(`, `)` and `*` raw, and - * none of those are `attr-char`. The apostrophe is the specific hazard — it is the - * delimiter in `UTF-8''name`, so a filename like `it's.pdf` would emit a third `'` - * and desync the parser. - */ -function encodeExtValue(filename: string): string { - return encodeURIComponent(filename).replace( - /['()*]/g, - (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}` - ) -} - -/** - * Build the `filename` parameters for a Content-Disposition header. - * - * The name is attacker-controlled (it is the user's `originalName`), so it can never - * be interpolated raw: a `"` closes the quoted-string early and everything after it - * is parsed as further parameters. An injected `filename*` is the payload that - * matters, because RFC 6266 tells clients to prefer `filename*` over `filename` — - * so the attacker's value wins and the download lands under a name the product UI - * never showed. Both parameters are therefore always emitted from sanitized input: - * the quoted form keeps only printable ASCII minus `"` and `\`, and the `filename*` - * form is fully percent-encoded. - * - * `;` is neutralized too, even though a quoted string may legally contain one: the - * quoted parameter exists as the fallback for clients that do not implement - * `filename*`, and those are the same clients liable to split parameters on a bare - * `;` without honouring the quoting. The exact name still survives in `filename*`. - */ -export function encodeFilenameForHeader(storageKey: string): string { - const filename = storageKey.split('/').pop() || storageKey - const asciiSafe = filename.replace(/[^\x20-\x7E]/g, '_').replace(/["\\;]/g, '_') - // Unchanged input proves the name is printable ASCII with no `"` or `\`, so the - // quoted form alone is both safe and sufficient — `filename*` buys nothing here. - if (asciiSafe === filename) { - return `filename="${filename}"` - } - return `filename="${asciiSafe}"; filename*=UTF-8''${encodeExtValue(filename)}` -} - /** * Derives the served filename from the CALLER's content type (`getSecureFileHeaders` * downgrades `text/html`) before the header decision, so a derived `.html` name gets the * same forced-attachment treatment a stored `.html` file gets. */ export function createFileResponse(file: FileResponse): NextResponse { - const servedFilename = ensureFileNameExtension(file.filename, file.contentType) - - const { contentType, disposition } = getSecureFileHeaders(servedFilename, file.contentType) - - const headers: Record = { - 'Content-Type': contentType, - 'Content-Disposition': `${disposition}; ${encodeFilenameForHeader(servedFilename)}`, - // Default to PRIVATE: this response is served only after access verification, so it must never be - // stored by a shared cache/CDN and re-served cross-user. Genuinely public assets (avatars, OG images, - // workspace logos) pass an explicit `cacheControl` (see PUBLIC_ASSET_CACHE_CONTROL in the serve route). - 'Cache-Control': file.cacheControl || 'private, no-cache', - 'X-Content-Type-Options': 'nosniff', - } - - if (contentType === 'image/svg+xml') { - headers['Content-Security-Policy'] = "default-src 'none'; style-src 'unsafe-inline'; sandbox;" - } - - return new NextResponse(file.buffer as BodyInit, { status: 200, headers }) + return new NextResponse(file.buffer as BodyInit, { + status: 200, + headers: fileDeliveryHeaders({ + ...file, + cacheControl: file.cacheControl || FILE_CACHE_CONTROL.private, + contentLength: file.buffer.length, + }), + }) } /** @@ -294,18 +202,15 @@ export function createConditionalFileResponse( file: FileResponse, ifNoneMatch: string | null ): NextResponse { - const etag = `"${createHash('sha256').update(file.buffer).digest('base64url')}"` + const etag = bufferedRepresentationEtag(file.buffer) if (ifNoneMatchHolds(ifNoneMatch, etag)) { - // A 304 repeats the headers that govern caching, so the stored response is refreshed with the - // lifetime this request would have granted it rather than keeping the one it was stored with. - return new NextResponse(null, { - status: 304, - headers: { - ETag: etag, - 'Cache-Control': file.cacheControl || 'private, no-cache', - }, + const headers = fileDeliveryHeaders({ + ...file, + cacheControl: file.cacheControl || FILE_CACHE_CONTROL.private, }) + headers.set('ETag', etag) + return new NextResponse(null, { status: 304, headers }) } const response = createFileResponse(file) @@ -331,7 +236,10 @@ export function createFileErrorResponse(error: Error, status = 500): NextRespons error: error.name, message: error.message, }, - { status: statusCode } + { + status: statusCode, + headers: { 'Cache-Control': FILE_CACHE_CONTROL.noStore, 'X-Content-Type-Options': 'nosniff' }, + } ) } diff --git a/apps/sim/app/api/knowledge/[id]/export/route.ts b/apps/sim/app/api/knowledge/[id]/export/route.ts index b7c9c25e560..2c4ad52080e 100644 --- a/apps/sim/app/api/knowledge/[id]/export/route.ts +++ b/apps/sim/app/api/knowledge/[id]/export/route.ts @@ -12,7 +12,7 @@ import { buildKnowledgeBundleArchive, knowledgeBundleFileName, } from '@/lib/knowledge/transfer/export-archive' -import { encodeFilenameForHeader } from '@/app/api/files/utils' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' /** GET /api/knowledge/[id]/export — stream a knowledge base as a bundle archive. */ export const GET = defineInternalBinaryRoute({ diff --git a/apps/sim/app/api/projects/[id]/files/[fileId]/artifact/route.ts b/apps/sim/app/api/projects/[id]/files/[fileId]/artifact/route.ts new file mode 100644 index 00000000000..c18a1989e84 --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/[fileId]/artifact/route.ts @@ -0,0 +1,30 @@ +import { readProjectFileArtifactContract } from '@/lib/api/contracts/project-files' +import { + defineInternalBinaryRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { presentProjectFileContent } from '@/lib/projects/files/api' +import { readProjectFileArtifact } from '@/lib/projects/files/application' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +export const GET = defineInternalBinaryRoute({ + contract: readProjectFileArtifactContract, + auth: internalSessionAuth, + operation: readProjectFileArtifact.operation, + rateLimit: internalRateLimits.none({ + reason: 'Authenticated bounded file rendering follows the existing private preview policy', + }), + errorPolicy: internalOrchestrationErrorPolicy, + headSafe: false, + mapInput: ({ params, query }) => ({ + preview: query.preview === '1', + projectId: params.id, + fileId: params.fileId, + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }), + useCase: readProjectFileArtifact, + present: ({ file, buffer, contentType }) => + presentProjectFileContent({ file: { ...file, type: contentType }, content: buffer }), +}) diff --git a/apps/sim/app/api/projects/[id]/files/[fileId]/csv-preview/route.ts b/apps/sim/app/api/projects/[id]/files/[fileId]/csv-preview/route.ts new file mode 100644 index 00000000000..f4e1500490f --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/[fileId]/csv-preview/route.ts @@ -0,0 +1,25 @@ +import { getProjectCsvPreviewContract } from '@/lib/api/contracts/project-files' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { readProjectFileCsvPreview } from '@/lib/projects/files/application' + +export const GET = defineInternalJsonRoute({ + contract: getProjectCsvPreviewContract, + auth: internalSessionAuth, + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, + operation: readProjectFileCsvPreview.operation, + rateLimit: internalRateLimits.user({ bucketName: 'project-files.csv-preview' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ params, query }, { request }) => ({ + projectId: params.id, + fileId: params.fileId, + key: query.key, + signal: request.signal, + }), + useCase: readProjectFileCsvPreview, + present: ({ success, headers, rows, truncated }) => ({ success, headers, rows, truncated }), +}) diff --git a/apps/sim/app/api/projects/[id]/files/inline/route.ts b/apps/sim/app/api/projects/[id]/files/inline/route.ts new file mode 100644 index 00000000000..8caab953b32 --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/inline/route.ts @@ -0,0 +1,25 @@ +import { getInlineProjectFileContract } from '@/lib/api/contracts/project-files' +import { + defineInternalBinaryRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { presentProjectFileContent } from '@/lib/projects/files/api' +import { readProjectInlineFile } from '@/lib/projects/files/application' + +export const GET = defineInternalBinaryRoute({ + contract: getInlineProjectFileContract, + auth: internalSessionAuth, + headSafe: false, + operation: readProjectInlineFile.operation, + rateLimit: internalRateLimits.none({ reason: 'Authenticated embedded file delivery' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ params, query }) => ({ + projectId: params.id, + key: query.key, + referenceFileId: query.fileId, + }), + useCase: readProjectInlineFile, + present: presentProjectFileContent, +}) diff --git a/apps/sim/app/api/v1/admin/folders/[id]/export/route.ts b/apps/sim/app/api/v1/admin/folders/[id]/export/route.ts index eeab2d40d80..915bbc33cd2 100644 --- a/apps/sim/app/api/v1/admin/folders/[id]/export/route.ts +++ b/apps/sim/app/api/v1/admin/folders/[id]/export/route.ts @@ -25,10 +25,10 @@ import { adminV1ExportFolderContract } from '@/lib/api/contracts/v1/admin' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { isWorkspaceFolder } from '@/lib/folders/scope' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { exportFolderToZip, sanitizePathSegment } from '@/lib/workflows/operations/import-export' import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' import { parseWorkflowVariables } from '@/lib/workflows/variables/parse' -import { encodeFilenameForHeader } from '@/app/api/files/utils' import { withAdminAuthParams } from '@/app/api/v1/admin/middleware' import { adminInternalErrorResponse, diff --git a/apps/sim/app/api/v1/admin/workspaces/[id]/export/route.ts b/apps/sim/app/api/v1/admin/workspaces/[id]/export/route.ts index 3f1617e585c..3e40349d344 100644 --- a/apps/sim/app/api/v1/admin/workspaces/[id]/export/route.ts +++ b/apps/sim/app/api/v1/admin/workspaces/[id]/export/route.ts @@ -20,10 +20,10 @@ import { NextResponse } from 'next/server' import { adminV1ExportWorkspaceContract } from '@/lib/api/contracts/v1/admin' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { exportWorkspaceToZip, sanitizePathSegment } from '@/lib/workflows/operations/import-export' import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' import { parseWorkflowVariables } from '@/lib/workflows/variables/parse' -import { encodeFilenameForHeader } from '@/app/api/files/utils' import { withAdminAuthParams } from '@/app/api/v1/admin/middleware' import { adminInternalErrorResponse, diff --git a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts index 4d8b8a2cc8f..5c1a842bcae 100644 --- a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts +++ b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts @@ -8,7 +8,7 @@ import { buildKnowledgeBundleArchive, knowledgeBundleFileName, } from '@/lib/knowledge/transfer/export-archive' -import { encodeFilenameForHeader } from '@/app/api/files/utils' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' export const dynamic = 'force-dynamic' export const revalidate = 0 diff --git a/apps/sim/app/api/v2/projects/[projectId]/files/search/route.ts b/apps/sim/app/api/v2/projects/[projectId]/files/search/route.ts new file mode 100644 index 00000000000..acb8e777195 --- /dev/null +++ b/apps/sim/app/api/v2/projects/[projectId]/files/search/route.ts @@ -0,0 +1,28 @@ +import { parseFolderPathList } from '@/lib/api/contracts/v2/files' +import { v2SearchProjectFileContentContract } from '@/lib/api/contracts/v2/project-file-search' +import { + defineV2JsonRoute, + v2ApiKeyAuth, + v2OrchestrationErrorPolicy, + v2RateLimits, +} from '@/lib/api/server/routes' +import { searchProjectFileContent } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' + +export const GET = defineV2JsonRoute({ + contract: v2SearchProjectFileContentContract, + auth: v2ApiKeyAuth, + operation: projectFileOperations.searchContent, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + mapInput: ({ params, query }) => ({ + ...params, + ...query, + folderPaths: + query.folderPaths === undefined ? undefined : parseFolderPathList(query.folderPaths), + }), + useCase: searchProjectFileContent, + present: ({ results, count, truncated, complete, indexStatus }) => ({ + data: { results, count, truncated, complete, indexStatus }, + }), +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/[fileId]/export/route.ts b/apps/sim/app/api/workspaces/[id]/files/[fileId]/export/route.ts index c6d573e4d5c..1aad1b2521b 100644 --- a/apps/sim/app/api/workspaces/[id]/files/[fileId]/export/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/[fileId]/export/route.ts @@ -5,9 +5,9 @@ import { internalSessionAuth, } from '@/lib/api/server/routes' import { captureServerEvent } from '@/lib/posthog/server' +import { encodeFilenameForHeader } from '@/lib/uploads/server/delivery' import { internalFileErrorPolicies } from '@/lib/workspace-files/api' import { exportWorkspaceFileSnapshot } from '@/lib/workspace-files/application/export-workspace-file-snapshot' -import { encodeFilenameForHeader } from '@/app/api/files/utils' export const POST = defineInternalBinaryRoute({ contract: exportWorkspaceFileSnapshotContract, diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts index 85f09365f27..4cfc126a79c 100644 --- a/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.test.ts @@ -9,6 +9,7 @@ vi.mock('@/lib/workspace-files/application/read-workspace-inline-file', () => ({ readWorkspaceInlineFile: { operation: { id: 'files.read_content', minimumRole: 'read', workspaceApiKey: 'allow' }, execute: mockReadInline, + authorize: vi.fn(async () => undefined), }, })) diff --git a/apps/sim/app/api/workspaces/[id]/files/inline/route.ts b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts index 0a3f20bf4ec..bcb9ca37b24 100644 --- a/apps/sim/app/api/workspaces/[id]/files/inline/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/inline/route.ts @@ -4,40 +4,16 @@ import { internalRateLimits, internalSessionAuth, } from '@/lib/api/server/routes' +import { presentFileDelivery, workspaceFileCacheControl } from '@/lib/uploads/server/delivery' import { internalFileErrorPolicies } from '@/lib/workspace-files/api' import { readWorkspaceInlineFile } from '@/lib/workspace-files/application/read-workspace-inline-file' -import { encodeFilenameForHeader, getSecureFileHeaders } from '@/app/api/files/utils' export const dynamic = 'force-dynamic' -/** - * How long the browser may reuse an embedded image, decided by whether the URL names the exact object - * that was streamed (see {@link ReadWorkspaceInlineFileResult.contentAddressed}). - * - * A content write never rewrites a storage object, so a URL that names one addresses bytes that can - * never change and the browser needs no round trip — which is the difference between an embedded image - * reappearing instantly and being downloaded again. Every document render asks for the same image at - * least twice (ProseMirror's own DOM, then the React node view) and every editor mounts twice (the - * read-only placeholder, then the live editor), so revalidating each time meant re-fetching the whole - * image on every open and reload — measured at ~1 MB per open on a real document, with the image area - * blank until it landed. `private` keeps it out of shared caches: the bytes are authorized per user. - * - * Anything else — a request that names the FILE, whose bytes move under it, or one whose object was - * rotated away mid-request — keeps revalidating. - */ -const IMMUTABLE_CACHE_CONTROL = 'private, max-age=31536000, immutable' -const REVALIDATE_CACHE_CONTROL = 'private, no-cache, must-revalidate' - -/** - * GET /api/workspaces/[id]/files/inline?key=|fileId= - * - * Serves an authenticated workspace-scoped image. Authentication and the - * `files.read_content` authorization check happen before resolving or reading - * the referenced object, preserving cross-workspace concealment. - */ export const GET = defineInternalBinaryRoute({ contract: getInlineWorkspaceFileContract, auth: internalSessionAuth, + headSafe: false, operation: readWorkspaceInlineFile.operation, rateLimit: internalRateLimits.none({ reason: 'Internal workspace inline image delivery' }), errorPolicy: internalFileErrorPolicies.inline, @@ -47,25 +23,12 @@ export const GET = defineInternalBinaryRoute({ fileId: query.fileId, }), useCase: readWorkspaceInlineFile, - present: ({ file, stream, contentAddressed }) => { - const secure = getSecureFileHeaders(file.name, file.type) - const headers = new Headers({ - 'Content-Type': secure.contentType, - 'Content-Disposition': `${secure.disposition}; ${encodeFilenameForHeader(file.name)}`, - 'Cache-Control': contentAddressed ? IMMUTABLE_CACHE_CONTROL : REVALIDATE_CACHE_CONTROL, - 'X-Content-Type-Options': 'nosniff', - }) - if (secure.contentType === 'image/svg+xml') { - headers.set( - 'Content-Security-Policy', - "default-src 'none'; style-src 'unsafe-inline'; sandbox;" - ) - } - return { + present: ({ file, stream, contentAddressed }) => + presentFileDelivery({ body: stream, - contentType: secure.contentType, + filename: file.name, + contentType: file.type, contentLength: file.size, - headers, - } - }, + cacheControl: workspaceFileCacheControl(contentAddressed), + }), }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts index d6508627285..1bc87099399 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.test.ts @@ -23,4 +23,10 @@ describe('content-source resolveImageSrc', () => { '/api/files/public/tok_1/inline?fileId=wf_abc' ) }) + it('preserves the canonical Project owner on public sources', () => { + const owner = { entityType: 'project', entityId: 'project-1' } as const + const source = createPublicFileContentSource('tok_1', '/api/files/public/tok_1/content', owner) + expect(source.owner).toEqual(owner) + expect(source.buildUrl('ignored')).toBe('/api/files/public/tok_1/content') + }) }) diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts index d5408ec0d88..04f804a32ad 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -77,6 +77,7 @@ export type ResolvedSecretIncompletenessReason = | 'table-run-state-provenance-unavailable' | 'mounted-file-provenance-unavailable' | 'workspace-file-provenance-unknown' + | 'project-file-provenance-unavailable' | 'file-source-unidentified' | 'table-snapshot-unsafe-for-mount' | 'restored-provenance-untrusted' diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index 4c342369166..4c44376812d 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -3,9 +3,10 @@ import { createContext, useContext } from 'react' import { type EmbeddedFileRef, - extractEmbeddedFileRef, + resolveEmbeddedFileRef, storedFileId, } from '@/lib/uploads/utils/embedded-image-ref' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' export interface FileContentUrlOptions { /** Request the uncompiled source instead of the rendered/compiled bytes. */ @@ -56,6 +57,7 @@ export interface ImageDimensionsSource { * this source so the same components work in both contexts. */ export interface FileContentSource { + owner?: EditableFileOwner buildUrl: (key: string, opts?: FileContentUrlOptions) => string /** * Map an embedded image `src` to a display URL scoped to the current context: the in-app source @@ -85,14 +87,21 @@ function buildServeUrl( /** Build a source whose embeds resolve through `inlineBase` (the workspace- or token-scoped inline route). */ function inlineImageSource( buildUrl: FileContentSource['buildUrl'], - inlineBase: string + inlineBase: string, + owner?: EditableFileOwner ): FileContentSource { return { + owner, buildUrl, resolveImageSrc: (src) => { if (!src) return src - const ref = extractEmbeddedFileRef(src) - return ref ? `${inlineBase}?${inlineRefQuery(ref)}` : src + const resolved = resolveEmbeddedFileRef( + src, + owner, + typeof window === 'undefined' ? undefined : window.location.origin + ) + if (resolved.kind === 'rejected') return undefined + return resolved.kind === 'file' ? `${inlineBase}?${inlineRefQuery(resolved.reference)}` : src }, } } @@ -110,7 +119,8 @@ export function createWorkspaceFileContentSource( return { ...inlineImageSource( (key, opts) => buildServeUrl(key, opts, storageContext), - `/api/workspaces/${workspaceId}/files/inline` + `/api/workspaces/${workspaceId}/files/inline`, + { entityType: 'workspace', entityId: workspaceId } ), ...imageDimensions, } @@ -119,16 +129,18 @@ export function createWorkspaceFileContentSource( /** * Public share source. Direct file bytes come from the token content URL; embedded images route through * `/api/files/public/{token}/inline`, which serves them only when referenced by the shared document and - * in its workspace. + * in its canonical owner. */ export function createPublicFileContentSource( token: string, - contentUrl: string + contentUrl: string, + owner?: EditableFileOwner ): FileContentSource { return inlineImageSource( (_key, opts) => opts?.preview ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` : contentUrl, - `/api/files/public/${token}/inline` + `/api/files/public/${token}/inline`, + owner ) } diff --git a/apps/sim/lib/api/contracts/v2/openapi/project-files.ts b/apps/sim/lib/api/contracts/v2/openapi/project-files.ts index 4af74a6231e..7f06eb3de45 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/project-files.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/project-files.ts @@ -11,6 +11,7 @@ import { v2RenameProjectFileContract, v2RestoreProjectFileContract, } from '@/lib/api/contracts/v2/project-file-lifecycle' +import { v2SearchProjectFileContentContract } from '@/lib/api/contracts/v2/project-file-search' import { v2CreateProjectFileContract, v2GetProjectFileMetadataContract, @@ -22,6 +23,41 @@ import { defineOpenApiRoute } from '@/lib/api/openapi/types' import { projectFileOperations } from '@/lib/projects/files/application/operations' export const projectFileOpenApiRoutes = [ + defineOpenApiRoute( + v2SearchProjectFileContentContract, + { + applicationOperation: projectFileOperations.searchContent, + operationId: 'searchProjectFileContent', + summary: 'Search Project File Content', + description: `Search indexed text in active Project files, returning matching lines with file IDs and line numbers. Folder filters narrow both results and reported coverage. Missing matches are inconclusive when complete is false, or skippedFiles or partialFiles is nonzero. truncated means additional matches exist beyond maxResults. ${WORKSPACE_API_KEY_DENIED}`, + tags: ['Files'], + errors: [...RESOURCE_CONFLICT_ERRORS, 'Locked'], + success: { + description: 'Matching lines and the index coverage they were drawn from.', + headers: RATE_LIMIT_HEADERS, + }, + }, + { + params: documentedSchema( + v2SearchProjectFileContentContract.params, + 'ProjectFilesParams', + 'Project file collection', + 'The Project that owns the requested files.' + ), + query: documentedSchema( + v2SearchProjectFileContentContract.query, + 'SearchProjectFileContentQuery', + 'Project file search query', + 'The search text, mode, result limit, and optional folder scope.' + ), + response: documentedSchema( + v2SearchProjectFileContentContract.response.schema, + 'V2ProjectFileSearchResultsResponse', + 'File search results response', + 'Matching lines from indexed file content.' + ), + } + ), defineOpenApiRoute( v2RenameProjectFileContract, { diff --git a/apps/sim/lib/api/contracts/v2/project-file-search.ts b/apps/sim/lib/api/contracts/v2/project-file-search.ts new file mode 100644 index 00000000000..02ee7fdfb37 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/project-file-search.ts @@ -0,0 +1,25 @@ +import type { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { + v2FileSearchResultsSchema, + v2SearchFileContentQuerySchema, +} from '@/lib/api/contracts/v2/files' +import { v2ProjectFilesParamsSchema } from '@/lib/api/contracts/v2/project-files' +import { v2DataResponse } from '@/lib/api/contracts/v2/shared' + +export const v2SearchProjectFileContentQuerySchema = v2SearchFileContentQuerySchema + .omit({ workspaceId: true }) + .extend({ + folderPaths: v2SearchFileContentQuerySchema.shape.folderPaths.describe( + 'Comma-separated folder paths within the Project. Omit to search the entire Project; index coverage applies to the selected folders.' + ), + }) +export type V2SearchProjectFileContentQuery = z.output + +export const v2SearchProjectFileContentContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/projects/[projectId]/files/search', + params: v2ProjectFilesParamsSchema, + query: v2SearchProjectFileContentQuerySchema, + response: { mode: 'json', schema: v2DataResponse(v2FileSearchResultsSchema) }, +}) diff --git a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts index ef65bd3c03e..eb42013361e 100644 --- a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts +++ b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts @@ -520,6 +520,10 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ pattern: '/api/v2/projects/{projectId}/files/move', load: () => import('@/app/api/v2/projects/[projectId]/files/move/route'), }, + { + pattern: '/api/v2/projects/{projectId}/files/search', + load: () => import('@/app/api/v2/projects/[projectId]/files/search/route'), + }, { pattern: '/api/v2/projects/{projectId}/files/uploads', load: () => import('@/app/api/v2/projects/[projectId]/files/uploads/route'), diff --git a/apps/sim/lib/execution/sandbox/brokers/workspace-file.ts b/apps/sim/lib/execution/sandbox/brokers/workspace-file.ts index 83cac379378..482c852955f 100644 --- a/apps/sim/lib/execution/sandbox/brokers/workspace-file.ts +++ b/apps/sim/lib/execution/sandbox/brokers/workspace-file.ts @@ -32,6 +32,19 @@ export const workspaceFileBroker: SandboxBroker MAX_SANDBOX_IMAGE_DATA_URI_CHARS || + JSON.stringify({ dataUri: prefix }).length + encodedLength > + MAX_ISOLATED_VM_BROKER_RESULT_JSON_CHARS + ) { + throw new Error('Prepared file exceeds the document broker byte limit') + } + return { dataUri: `${prefix}${file.content.toString('base64')}` } + } if (!ctx.workspaceId) { throw new Error('workspaceFile broker requires a workspaceId') } diff --git a/apps/sim/lib/execution/sandbox/run-task.ts b/apps/sim/lib/execution/sandbox/run-task.ts index 78bdac1bf3d..055c89c3443 100644 --- a/apps/sim/lib/execution/sandbox/run-task.ts +++ b/apps/sim/lib/execution/sandbox/run-task.ts @@ -13,6 +13,8 @@ import { getSandboxTask, type SandboxTaskId } from '@/sandboxes/registry' const logger = createLogger('SandboxRunTask') export interface RunSandboxTaskOptions { + /** Private document compiles expose only their already-authorized, fixed input manifest. */ + resolvePreparedFile?: SandboxBrokerContext['resolvePreparedFile'] /** * Owner key used by the isolated-vm pool for fairness + distributed leases. * Typically `user:` or `workspace:`. @@ -61,6 +63,7 @@ export async function runSandboxTask( const brokerContext: SandboxBrokerContext = { workspaceId: input.workspaceId, requestId, + resolvePreparedFile: options.resolvePreparedFile, onWorkspaceFileAccess: options.onWorkspaceFileAccess, filePrincipal: options.filePrincipal, } diff --git a/apps/sim/lib/execution/sandbox/types.ts b/apps/sim/lib/execution/sandbox/types.ts index eaa3066da29..776e0a659dd 100644 --- a/apps/sim/lib/execution/sandbox/types.ts +++ b/apps/sim/lib/execution/sandbox/types.ts @@ -24,8 +24,9 @@ export interface SandboxBroker { } export interface SandboxBrokerContext { - workspaceId: string + workspaceId?: string requestId: string + resolvePreparedFile?: (fileId: string) => { content: Buffer; contentType: string } /** Host-side identity used to authorize and admit each actual broker read. Never sent to the isolate. */ filePrincipal?: Principal onWorkspaceFileAccess?: (identity: { @@ -37,7 +38,7 @@ export interface SandboxBrokerContext { } export interface SandboxTaskInput { - workspaceId: string + workspaceId?: string code: string /** Module filename for `code` when the task sets `userModules`; appears in stack traces. */ codeFilename?: string diff --git a/apps/sim/lib/file-parsers/csv-preview-slice.test.ts b/apps/sim/lib/file-parsers/csv-preview-slice.test.ts index 000fcacfa49..526b7d585e7 100644 --- a/apps/sim/lib/file-parsers/csv-preview-slice.test.ts +++ b/apps/sim/lib/file-parsers/csv-preview-slice.test.ts @@ -81,3 +81,37 @@ describe('getCsvPreviewSlice', () => { expect(destroySpy).toHaveBeenCalled() }) }) + +it('rejects an oversized streamed record before consuming the entire file', async () => { + let chunksRead = 0 + const source = Readable.from( + (async function* () { + yield Buffer.from('header\n"') + for (let i = 0; i < 1000; i++) { + chunksRead++ + yield Buffer.alloc(16 * 1024, 'x') + } + })() + ) + mockDownloadFileStream.mockResolvedValue(source) + await expect(getCsvPreviewSlice(args)).rejects.toMatchObject({ code: 'CSV_MAX_RECORD_SIZE' }) + expect(chunksRead).toBeLessThan(30) + expect(source.destroyed).toBe(true) +}) + +it('bounds bytes consumed even when blank records produce no preview rows', async () => { + let chunksRead = 0 + const source = Readable.from( + (async function* () { + yield Buffer.from('header\n') + for (let i = 0; i < 1000; i++) { + chunksRead++ + yield Buffer.alloc(64 * 1024, '\n') + } + })() + ) + mockDownloadFileStream.mockResolvedValue(source) + await expect(getCsvPreviewSlice(args)).rejects.toThrow('CSV preview') + expect(chunksRead).toBeLessThan(140) + expect(source.destroyed).toBe(true) +}) diff --git a/apps/sim/lib/file-parsers/csv-preview-slice.ts b/apps/sim/lib/file-parsers/csv-preview-slice.ts index 2f4c9369ed9..b0521d3d0f9 100644 --- a/apps/sim/lib/file-parsers/csv-preview-slice.ts +++ b/apps/sim/lib/file-parsers/csv-preview-slice.ts @@ -2,11 +2,14 @@ import { Readable } from 'node:stream' import { truncate } from '@sim/utils/string' import { parse as parseCsvStream } from 'csv-parse' import { CSV_PREVIEW_MAX_ROWS } from '@/lib/api/contracts/workspace-file-table' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' import type { StorageContext } from '@/lib/uploads/config' import { downloadFileStream } from '@/lib/uploads/core/storage-service' /** Cap a single cell so one pathological field can't bloat the preview payload. */ const MAX_CELL_LENGTH = 2_000 +const MAX_RECORD_SIZE = 256 * 1024 +const MAX_PREVIEW_BYTES = 8 * 1024 * 1024 /** Read at most this many bytes while sniffing the first line for the delimiter. */ const DELIMITER_SNIFF_MAX_BYTES = 256 * 1024 @@ -44,7 +47,7 @@ function cell(value: unknown): string { /** * Streams the first {@link CSV_PREVIEW_MAX_ROWS} rows of a CSV/TSV from storage without * ever buffering the whole file. The source stream is destroyed as soon as enough rows are - * read (one past the cap, to detect truncation), so a multi-GB file costs O(rows) of memory. + * read (one past the cap, to detect truncation), with record and total-byte limits bounding parser memory and work. */ export async function getCsvPreviewSlice({ key, @@ -60,6 +63,17 @@ export async function getCsvPreviewSlice({ signal?.addEventListener('abort', onAbort, { once: true }) const reader = source[Symbol.asyncIterator]() + let bytesRead = 0 + let piped: Readable | undefined + let parser: ReturnType | undefined + async function nextChunk() { + const next = await reader.next() + if (!next.done) { + bytesRead += Buffer.byteLength(next.value) + assertKnownSizeWithinLimit(bytesRead, MAX_PREVIEW_BYTES, 'CSV preview') + } + return next + } try { // Pull chunks until the first newline so the delimiter can be sniffed before parsing. @@ -71,12 +85,14 @@ export async function getCsvPreviewSlice({ let firstLine = '' let sniffedBytes = 0 while (true) { - const { value, done } = await reader.next() + const { value, done } = await nextChunk() if (done) break const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value) sniffed.push(chunk) sniffedBytes += chunk.length - const text = chunk.toString('utf-8') + const text = chunk + .subarray(0, Math.max(0, DELIMITER_SNIFF_MAX_BYTES - sniffedBytes + chunk.length)) + .toString('utf-8') const nl = text.indexOf('\n') if (nl !== -1) { firstLine += text.slice(0, nl) @@ -91,13 +107,13 @@ export async function getCsvPreviewSlice({ } const delimiter = detectDelimiter(firstLine) - const parser = parseCsvStream({ + parser = parseCsvStream({ columns: false, skip_empty_lines: true, trim: true, relax_column_count: true, relax_quotes: true, - skip_records_with_error: true, + max_record_size: MAX_RECORD_SIZE, cast: false, bom: true, delimiter, @@ -107,13 +123,13 @@ export async function getCsvPreviewSlice({ async function* rejoin() { for (const chunk of sniffed) yield chunk while (true) { - const { value, done } = await reader.next() + const { value, done } = await nextChunk() if (done) return yield value } } - const piped = Readable.from(rejoin()) - piped.on('error', (err) => parser.destroy(err)) + piped = Readable.from(rejoin()) + piped.on('error', (err) => parser?.destroy(err)) piped.pipe(parser) let headers: string[] = [] @@ -141,6 +157,8 @@ export async function getCsvPreviewSlice({ if (signal?.aborted) signal.throwIfAborted() throw error } finally { + piped?.destroy() + parser?.destroy() signal?.removeEventListener('abort', onAbort) source.destroy() } diff --git a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts index ed4ceb43c83..70c74fca5a3 100644 --- a/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts +++ b/apps/sim/lib/mothership/__integration__/content-delivery.integration.ts @@ -132,6 +132,7 @@ import { } from '@/lib/mothership/__integration__/content-delivery-fixture' import { observePrivateFile } from '@/lib/mothership/agent-cli/engines/observe-private-file' import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport' +import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation' import { createRegistryContentGate } from '@/lib/mothership/content-delivery/registry-gate' import { proxySandboxResourceRequest } from '@/lib/mothership/tools/sandbox-resource-transport' @@ -251,6 +252,12 @@ describe('content reaching Chat is admitted per item', () => { chatId: generateId(), }, registry, + files: createWorkbenchFileProvenance({ + userId: fixture.ids.aliceId, + workspaceId: fixture.ids.workspaceId, + sessionKey: generateId(), + resolvedSecretTraceRegistry: registry, + }), resources: [], observeReads: true, }) diff --git a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts index 6fee8cbd8ed..a0638b0f092 100644 --- a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts +++ b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts @@ -59,10 +59,12 @@ vi.mock('@/lib/mothership/agent-cli/scoped-transport', () => ({ import { createFileReadTransport } from '@/lib/mothership/agent-cli/file-read-transport' import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport' +import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation' import { createRegistryContentGate } from '@/lib/mothership/content-delivery' import { inspectToolResultForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result' import { executeSimCli } from '@/lib/mothership/tools/handlers/sim-cli' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' import { fileOperations } from '@/lib/workspace-files/application/operations' import { readWorkspaceFileArtifact } from '@/lib/workspace-files/application/read-workspace-file-artifact' @@ -118,6 +120,31 @@ function registry() { return new ResolvedSecretTraceRegistry([], { userId: 'reader', workspaceId: 'workspace' }) } +function classify( + status: 'exact' | 'unknown', + fileRevision = revision, + entries: Extract['entries'] = [], + includePublicationRows = true +) { + resetDbChainMock() + const sidecar = { + id: file.id, + key: file.key, + context: file.storageContext, + fileContentUpdatedAt: fileRevision, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: fileRevision, + status, + entries, + } + queueTableRows(workspaceFiles, [sidecar]) + if (!includePublicationRows) return + queueTableRows(workspaceFiles, [ + { ...file, context: file.storageContext, secretProvenanceVersion: 1 }, + ]) + queueTableRows(workspaceFiles, [sidecar]) +} + type SidecarStatus = 'exact' | 'unknown' function sidecarRow(status: SidecarStatus, fileRevision = revision, entries: unknown[] = []) { @@ -131,7 +158,7 @@ function sidecarRow(status: SidecarStatus, fileRevision = revision, entries: unk } /** Queues one sidecar per status, in the order the reads consume them. */ -function classify(...statuses: [SidecarStatus, ...SidecarStatus[]]) { +function classifySequence(...statuses: [SidecarStatus, ...SidecarStatus[]]) { resetDbChainMock() for (const status of statuses) queueTableRows(workspaceFiles, [sidecarRow(status)]) } @@ -178,11 +205,19 @@ function runCli(invocation: CliInvocation, trace = registry()) { const invocation = { userId: 'reader', workspaceId: 'workspace', chatId: 'chat' } /** Chat's CLI transport as every invocation composes it: the registry gate around the routes. */ -function readTransport(trace = registry()): typeof fetch { +function readTransport(trace = registry(), tracked = true): typeof fetch { const transport = createAgentCliTransport({ endpoint: 'https://sim.test', invocation, registry: trace, + files: tracked + ? createWorkbenchFileProvenance({ + userId: 'reader', + workspaceId: 'workspace', + sessionKey: 'chat', + resolvedSecretTraceRegistry: trace, + }) + : undefined, resources: [], observeReads: true, }) @@ -220,15 +255,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { mocks.stream.mockImplementation(async () => Readable.from(Buffer.from(content))) mocks.buffer.mockResolvedValue(Buffer.from(content)) mocks.decrypt.mockResolvedValue({ decrypted: content }) - queueTableRows(workspaceFiles, [ - { - fileContentUpdatedAt: revision, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: revision, - status: 'unknown', - entries: [], - }, - ]) + classify('unknown') }) it('uses current Copilot file authority with personal keys disabled and rejects a foreign ID-only owner', async () => { @@ -411,18 +438,19 @@ describe('file provenance at the actual CLI and model-result boundary', () => { ).not.toContain(content) }) + it('refuses opaque protected downloads without workbench provenance tracking', async () => { + classify('exact', revision, [ + { name: 'FILE_SECRET', encryptedValue: 'fixture-ciphertext', sourceUserId: 'reader' }, + ]) + const response = await readTransport(registry(), false)(fileRequest()) + expect(response.status).toBe(403) + expect(await response.text()).not.toContain(content) + expect(mocks.stream).not.toHaveBeenCalled() + }) + it('imports encrypted file provenance into a fresh registry before model projection', async () => { - resetDbChainMock() - queueTableRows(workspaceFiles, [ - { - fileContentUpdatedAt: revision, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: revision, - status: 'exact', - entries: [ - { name: 'FILE_SECRET', encryptedValue: 'fixture-ciphertext', sourceUserId: 'reader' }, - ], - }, + classify('exact', revision, [ + { name: 'FILE_SECRET', encryptedValue: 'fixture-ciphertext', sourceUserId: 'reader' }, ]) const trace = registry() const response = await readTransport(trace)(fileRequest()) @@ -481,7 +509,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { nextCursor: null, }) }) - classify('unknown') + classify('unknown', revision, [], false) const grep = async () => { const trace = registry() const result = await runCli(grepFiles(content), trace) @@ -523,7 +551,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { nextCursor: null, }) ) - classify('exact', 'unknown') + classifySequence('exact', 'unknown') const refusals = getMockLogger('DurableSecretProvenancePersistence').error refusals.mockClear() const trace = registry() diff --git a/apps/sim/lib/mothership/agent-cli/file-read-transport.ts b/apps/sim/lib/mothership/agent-cli/file-read-transport.ts index fcc56801d63..3a62be5e96a 100644 --- a/apps/sim/lib/mothership/agent-cli/file-read-transport.ts +++ b/apps/sim/lib/mothership/agent-cli/file-read-transport.ts @@ -38,6 +38,7 @@ export function createFileReadTransport(context: { transport: typeof fetch invocation: CopilotChatDelegationContext & { workspaceId: string } registry?: ResolvedSecretTraceRegistry + tracksWorkbenchDownloads?: boolean }): typeof fetch { const base = new URL(context.endpoint) const basePath = base.pathname.replace(/\/$/, '') @@ -105,6 +106,7 @@ export function createFileReadTransport(context: { workspaceId: parsed.data.query.workspaceId, actorUserId: context.invocation.userId, unknown: 'taint', + destination: context.tracksWorkbenchDownloads ? 'workbench' : 'model', }), () => downloadWorkspaceFileStream.execute({ diff --git a/apps/sim/lib/mothership/agent-cli/file-round-trip-provenance.test.ts b/apps/sim/lib/mothership/agent-cli/file-round-trip-provenance.test.ts index 885a539382c..98a664531b3 100644 --- a/apps/sim/lib/mothership/agent-cli/file-round-trip-provenance.test.ts +++ b/apps/sim/lib/mothership/agent-cli/file-round-trip-provenance.test.ts @@ -190,15 +190,29 @@ function queueRead(file: typeof SOURCE, provenance: WorkspaceFileSecretProvenanc }, ]) queueTableRows(workspaceFiles, [file]) + const sidecar = { + id: file.id, + key: file.key, + context: file.context, + fileContentUpdatedAt: REVISION, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: REVISION, + ...provenance, + entries: provenance.status === 'exact' ? provenance.entries : [], + } + queueTableRows(workspaceFiles, [sidecar]) queueTableRows(workspaceFiles, [ { - fileContentUpdatedAt: REVISION, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: REVISION, - ...provenance, - entries: provenance.status === 'exact' ? provenance.entries : [], + workspaceId: WORKSPACE, + fileId: file.id, + ownership: file, + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'owner', }, ]) + queueTableRows(workspaceFiles, [file]) + queueTableRows(workspaceFiles, [sidecar]) } function execute(argv: string[], trace: ResolvedSecretTraceRegistry, sink?: string) { diff --git a/apps/sim/lib/mothership/agent-cli/transport.ts b/apps/sim/lib/mothership/agent-cli/transport.ts index f400833f499..c56690b4d61 100644 --- a/apps/sim/lib/mothership/agent-cli/transport.ts +++ b/apps/sim/lib/mothership/agent-cli/transport.ts @@ -40,6 +40,7 @@ export function createAgentCliTransport(context: { transport: createTracedCliTransport(endpoint, createScopedCliTransport(endpoint, invocation)), invocation, registry, + tracksWorkbenchDownloads: Boolean(files), }) const reads: typeof fetch = async (input, init) => { const { response, admitted } = await dispatchUnderContentGate(gate, () => routes(input, init)) diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts index 82671cb137c..dfb4d04b14d 100644 --- a/apps/sim/lib/mothership/chat/post.ts +++ b/apps/sim/lib/mothership/chat/post.ts @@ -634,7 +634,10 @@ async function resolveAgentContexts(params: { { userId, workspaceId, organizationId, chatId }, resource.workspaceId ) - : { workspaceId: workspaceId! } + : workspaceId + ? { workspaceId } + : null + if (!target) return null const ctx = await withWorkspaceInvocationScope( { workspaceId: target.workspaceId, organizationId }, () => diff --git a/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts b/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts index 61fcec2f5da..2eea660c1d7 100644 --- a/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts +++ b/apps/sim/lib/mothership/content-delivery/registry-gate.test.ts @@ -135,6 +135,23 @@ describe('createRegistryContentGate', () => { expect(await gate.admit([item('config', exact(SECRET))])).toEqual([false]) }) + it('tracks opaque workbench downloads without relaxing model or excerpt admission', async () => { + const registry = newRegistry() + const gate = createRegistryContentGate({ + registry, + actorUserId: 'user-1', + destination: 'workbench', + }) + expect(await gate.admit([item('download', exact(SECRET), 'opaque')])).toEqual([true]) + expect(modelView(SECRET, registry)).not.toContain(SECRET) + expect(await gate.admit([item('snippet', exact(SECRET), 'excerpt')])).toEqual([false]) + expect( + await createRegistryContentGate({ actorUserId: 'user-1', destination: 'workbench' }).admit([ + item('download', exact(SECRET), 'opaque'), + ]) + ).toEqual([false]) + }) + it('refuses an unknown item without latching the registry', async () => { const registry = newRegistry() const gate = createRegistryContentGate({ registry, actorUserId: 'user-1' }) diff --git a/apps/sim/lib/mothership/content-delivery/registry-gate.ts b/apps/sim/lib/mothership/content-delivery/registry-gate.ts index 6dc499c4324..ba6bd14fa90 100644 --- a/apps/sim/lib/mothership/content-delivery/registry-gate.ts +++ b/apps/sim/lib/mothership/content-delivery/registry-gate.ts @@ -23,6 +23,8 @@ export interface RegistryContentGateScope { * anything persisted from it stays unknown. Only a direct single-file download takes it (D7). */ unknown?: 'refuse' | 'taint' + /** Private workbench downloads persist imported provenance; opaque bytes never go directly to the model. */ + destination?: 'model' | 'workbench' } /** @@ -56,7 +58,11 @@ export function createRegistryContentGate(scope: RegistryContentGateScope): Cont return true } if (provenance.entries.length === 0) return true - if (source.view === 'opaque' || source.view === 'excerpt') return refuse(source, 'opaque') + if ( + source.view === 'excerpt' || + (source.view === 'opaque' && scope.destination !== 'workbench') + ) + return refuse(source, 'opaque') const registry = scope.registry if (!registry) return refuse(source, 'registry') /** A derived view that shows nothing needs no entry; `undefined` would mean the whole item. */ diff --git a/apps/sim/lib/projects/files/__integration__/content-rendered.integration.ts b/apps/sim/lib/projects/files/__integration__/content-rendered.integration.ts new file mode 100644 index 00000000000..e98a1e4b4b5 --- /dev/null +++ b/apps/sim/lib/projects/files/__integration__/content-rendered.integration.ts @@ -0,0 +1,863 @@ +import { mkdtempSync } from 'node:fs' +import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import type { ResourceDelegatedPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { + folder, + idempotencyKey, + member, + organization, + outboxEvent, + permissions, + subscription, + user, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { emailMailerMock } from '@sim/testing/mocks/email-mailer.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' +import { eq, inArray, sql } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) +vi.mock('@/lib/messaging/email/mailer', () => emailMailerMock) + +import { observeContentDelivery } from '@/lib/core/application/content-delivery' +import * as sandboxTask from '@/lib/execution/sandbox/run-task' +import { readProjectFileArtifact } from '@/lib/projects/files/application/artifacts' +import { + createProjectFile, + readProjectFileContent, + updateProjectFileContent, +} from '@/lib/projects/files/application/content' +import { + readProjectFileCsvPreview, + readProjectInlineFile, +} from '@/lib/projects/files/application/previews' +import { searchProjectFileContent } from '@/lib/projects/files/application/search' +import { revertProjectFileVersion } from '@/lib/projects/files/application/versions' +import * as storageCleanup from '@/lib/uploads/contexts/workspace/file-storage-cleanup-outbox' +import * as storage from '@/lib/uploads/core/storage-service' +import { compileFileDocument } from '@/lib/uploads/documents/compile' +import * as heic from '@/lib/uploads/server/heic' +import { observeWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import * as searchDelivery from '@/lib/workspace-files/search/delivery' +import * as searchRepository from '@/lib/workspace-files/search/repository' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +const localStorageRoot = mkdtempSync(join(tmpdir(), 'sim-project-content-')) +setUploadDirServer(localStorageRoot) +const fixtures: { + ownerId: string + editorId: string + organizationId: string + workspaceId: string + projectId: string +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + vi.restoreAllMocks() + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const ownerId = generateId() + const editorId = generateId() + const organizationId = generateId() + const workspaceId = generateId() + await db.insert(user).values( + [ownerId, editorId].map((id) => ({ + id, + email: `${id}@content.invalid`, + name: 'File content fixture', + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db + .insert(organization) + .values({ id: organizationId, name: 'Content', slug: organizationId, createdAt: new Date() }) + await db.insert(member).values( + [ownerId, editorId].map((userId) => ({ + id: generateId(), + organizationId, + userId, + role: userId === ownerId ? 'owner' : 'member', + createdAt: new Date(), + })) + ) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId, + billedAccountUserId: ownerId, + organizationId, + workspaceMode: 'organization', + name: 'Content environment', + }) + const [binding] = await db.select().from(workspace).where(eq(workspace.id, workspaceId)) + if (!binding) throw new Error('Project fixture missing') + await db.insert(permissions).values({ + id: generateId(), + userId: editorId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const ids = { ownerId, editorId, organizationId, workspaceId, projectId: binding.projectId } + fixtures.push(ids) + return { ...ids, principal: createSessionPrincipal({ userId: editorId }) } +} + +function createInput(projectId: string, content = 'Project architecture') { + return { + projectId, + name: 'architecture.md', + contentType: 'text/markdown', + content, + encoding: 'utf-8' as const, + exactName: true, + } +} + +describe('Project file content against PostgreSQL and the local object store', () => { + for (const code of ['55P03', '25P04']) { + check(`Project search classifies delivery contention ${code} as retryable`, async () => { + const f = await fixture() + const delivery = vi + .spyOn(searchDelivery, 'loadFileSearchDelivery') + .mockRejectedValueOnce( + new Error('Failed query', { cause: Object.assign(new Error('lock timeout'), { code }) }) + ) + try { + await expect( + searchProjectFileContent.execute({ + principal: f.principal, + input: { projectId: f.projectId, query: 'needle', mode: 'exact', maxResults: 10 }, + }) + ).rejects.toMatchObject({ code: 'locked' }) + } finally { + delivery.mockRestore() + } + }) + } +}) + +describe('Project delivery admission', () => { + for (const kind of ['inline', 'csv', 'artifact'] as const) { + for (const refusal of ['before storage', 'after storage'] as const) { + check(`${kind} gates content ${refusal}`, async () => { + const f = await fixture() + const created = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'name,value\nrow,secret'), + name: 'rows.csv', + contentType: 'text/csv', + }, + }) + const issuedAt = new Date() + const principal: ResourceDelegatedPrincipal = { + kind: 'resource_delegated', + serviceId: 'copilot', + subjectUserId: f.editorId, + delegationId: generateId(), + audience: 'sim:project-files', + issuedAt, + expiresAt: new Date(issuedAt.getTime() + 60_000), + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + scope: { kind: 'entity', entityType: 'project', entityId: f.projectId }, + } + const download = vi.spyOn(storage, 'downloadFile') + const stream = vi.spyOn(storage, 'downloadFileStream') + let calls = 0 + const input = { projectId: f.projectId, fileId: created.file.id, key: created.file.key } + await expect( + observeContentDelivery( + { + async admit(sources) { + calls++ + expect( + sources.every((source) => source.view === (kind === 'csv' ? 'excerpt' : 'opaque')) + ).toBe(true) + return sources.map(() => refusal === 'after storage' && calls === 1) + }, + }, + () => + kind === 'inline' + ? readProjectInlineFile.execute({ + principal, + input: { projectId: f.projectId, referenceFileId: created.file.id }, + }) + : kind === 'csv' + ? readProjectFileCsvPreview.execute({ principal, input }) + : readProjectFileArtifact.execute({ + principal, + input: { ...input, maxBytes: 1024 }, + }) + ) + ).rejects.toMatchObject({ code: 'forbidden' }) + if (refusal === 'before storage') { + expect(download).not.toHaveBeenCalled() + expect(stream).not.toHaveBeenCalled() + } else expect(calls).toBeGreaterThan(1) + }) + } + } + for (const kind of ['inline', 'csv', 'artifact'] as const) { + check(`${kind} awaits an asynchronous transport observer refusal`, async () => { + const f = await fixture() + const created = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'name,value\nrow,secret'), + name: 'rows.csv', + contentType: 'text/csv', + }, + }) + const error = new Error('Transport observer refused delivery') + const input = { projectId: f.projectId, fileId: created.file.id, key: created.file.key } + await expect( + observeWorkspaceFileDelivery( + async () => { + throw error + }, + () => + kind === 'inline' + ? readProjectInlineFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, referenceFileId: created.file.id }, + }) + : kind === 'csv' + ? readProjectFileCsvPreview.execute({ principal: f.principal, input }) + : readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, maxBytes: 1024 }, + }) + ) + ).rejects.toBe(error) + }) + } + + check('Project search withholds stale previews and removes moved-out results', async () => { + const f = await fixture() + const created = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId), + }) + const identity = { + fileId: created.file.id, + key: created.file.key, + contentUpdatedAt: new Date(), + } + const prepared: searchRepository.FileSearchResult = { + results: [ + { fileId: created.file.id, text: 'stale secret', lineNumber: 1 }, + { fileId: 'moved', text: 'out of scope', lineNumber: 1 }, + ], + count: 2, + truncated: false, + complete: true, + indexStatus: { + readyFiles: 0, + pendingFiles: 0, + failedFiles: 0, + skippedFiles: 0, + partialFiles: 0, + }, + sources: [ + { + owner: { entityType: 'project', entityId: f.projectId }, + identity, + buildId: 'old-build', + dependencies: [], + }, + ], + } + vi.spyOn(searchRepository, 'searchFileIndex').mockResolvedValue(prepared) + vi.spyOn(searchDelivery, 'loadFileSearchDelivery').mockResolvedValue({ + provenance: new Map(), + outOfScope: new Set(['moved']), + inputs: [], + }) + const result = await searchProjectFileContent.execute({ + principal: f.principal, + input: { projectId: f.projectId, query: 'secret', mode: 'exact', maxResults: 10 }, + }) + expect(result.count).toBe(1) + expect(result.results[0].text).toContain('Preview withheld') + expect(result.sources).toEqual([]) + expect(JSON.stringify(result)).not.toContain('stale secret') + }) +}) + +describe('Project outer transaction cleanup', () => { + for (const { operation, code } of [ + { operation: 'create', code: undefined }, + { operation: 'update', code: undefined }, + { operation: 'revert', code: undefined }, + { operation: 'preview', code: undefined }, + { operation: 'update', code: '40003' }, + { operation: 'update', code: 'CONNECTION_CLOSED' }, + ] as const) { + check( + `retains committed Project ${operation} bytes after acknowledgement loss (${code ?? 'uncoded'})`, + async () => { + const f = await fixture() + const sourceBytes = Buffer.alloc(16) + sourceBytes.writeUInt32BE(16, 0) + sourceBytes.write('ftypheic', 4, 'ascii') + const source = await createProjectFile.execute({ + principal: f.principal, + input: + operation === 'preview' + ? { + ...createInput(f.projectId), + name: 'preview.heic', + contentType: 'image/heic', + content: sourceBytes.toString('base64'), + encoding: 'base64', + } + : createInput(f.projectId, 'original'), + }) + if (operation === 'revert') + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: 'second', + encoding: 'utf-8', + }, + }) + vi.spyOn(heic, 'transcodeHeicToJpeg').mockResolvedValue(Buffer.alloc(128, 255)) + const upload = storage.uploadFile + const written: string[] = [] + vi.spyOn(storage, 'uploadFile').mockImplementation(async (args) => { + const result = await upload(args) + written.push(result.key) + return result + }) + const transaction = db.transaction.bind(db) + const primary = Object.assign( + new Error('Commit acknowledgement lost'), + code ? { code } : {} + ) + let lost = false + vi.spyOn(db, 'transaction').mockImplementation(async (callback, config) => { + const result = await transaction(callback, config) + if ( + !lost && + written.length > 0 && + isRecordLike(result) && + isRecordLike(result.result) && + 'file' in result.result + ) { + lost = true + throw primary + } + return result + }) + const input = { projectId: f.projectId, fileId: source.file.id } + const action = + operation === 'create' + ? createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'committed'), name: 'new.md' }, + }) + : operation === 'update' + ? updateProjectFileContent.execute({ + principal: f.principal, + input: { ...input, content: 'committed', encoding: 'utf-8' }, + }) + : operation === 'revert' + ? revertProjectFileVersion.execute({ + principal: f.principal, + input: { ...input, version: 1, expectedCurrentVersion: 2 }, + }) + : readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, preview: true, maxBytes: 1024 }, + }) + await expect(action).rejects.toBe(primary) + expect(lost).toBe(true) + expect(written).toHaveLength(1) + for (const key of written) { + expect((await readFile(join(localStorageRoot, key))).length).toBeGreaterThan(0) + expect( + await db + .select() + .from(outboxEvent) + .where(sql`${outboxEvent.payload}::jsonb ->> 'key' = ${key}`) + ).toEqual([]) + } + } + ) + } + + check('preserves size rejection and a concurrently published image derivative', async () => { + const f = await fixture() + const bytes = Buffer.alloc(16) + bytes.writeUInt32BE(16, 0) + bytes.write('ftypheic', 4, 'ascii') + const source = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId), + name: 'preview.heic', + contentType: 'image/heic', + content: bytes.toString('base64'), + encoding: 'base64', + }, + }) + vi.spyOn(storageCleanup, 'enqueueWorkspaceFileStorageCleanups').mockRejectedValue( + new Error('Cleanup database unavailable') + ) + const input = { projectId: f.projectId, fileId: source.file.id, preview: true } + let winner: Buffer | undefined + vi.spyOn(heic, 'transcodeHeicToJpeg') + .mockResolvedValue(Buffer.alloc(128, 255)) + .mockImplementationOnce(async () => { + winner = ( + await readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, maxBytes: 1024 }, + }) + ).buffer + return Buffer.alloc(128, 255) + }) + await expect + .soft( + readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, preview: true, maxBytes: 64 }, + }) + ) + .rejects.toMatchObject({ name: 'PayloadSizeLimitError' }) + expect( + await readdir(join(localStorageRoot, 'project', f.projectId, 'image-derivative')) + ).toHaveLength(1) + const subsequent = await readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, maxBytes: 1024 }, + }) + expect(winner).toBeDefined() + expect(subsequent.buffer).toEqual(winner) + }) +}) + +describe('Project rendered artifacts against PostgreSQL and private storage', () => { + for (const failure of ['size rejection', 'revision change'] as const) { + check( + `a preview with ${failure} retains the shared derivative and preserves the original error`, + async () => { + const f = await fixture() + const sourceBytes = Buffer.alloc(16) + sourceBytes.writeUInt32BE(16, 0) + sourceBytes.write('ftypheic', 4, 'ascii') + const source = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId), + name: 'preview.heic', + contentType: 'image/heic', + content: sourceBytes.toString('base64'), + encoding: 'base64', + }, + }) + vi.spyOn(heic, 'transcodeHeicToJpeg').mockImplementation(async () => { + if (failure === 'revision change') { + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: 'replacement source', + encoding: 'utf-8', + }, + }) + } + return Buffer.alloc(128, 255) + }) + const rendering = readProjectFileArtifact.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + preview: true, + maxBytes: failure === 'size rejection' ? 64 : 1024, + }, + }) + await expect + .soft(rendering) + .rejects.toMatchObject( + failure === 'size rejection' ? { name: 'PayloadSizeLimitError' } : { code: 'conflict' } + ) + expect + .soft(await readdir(join(localStorageRoot, 'project', f.projectId, 'image-derivative'))) + .toHaveLength(1) + const current = await readProjectFileContent.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + expect(current.content).toEqual( + failure === 'size rejection' ? sourceBytes : Buffer.from('replacement source') + ) + } + ) + } + + for (const failure of ['revoked read', 'uncertain pointer upload'] as const) { + check( + `a render with ${failure} preserves shared published objects without scheduling deletion`, + async () => { + const { readProjectFileArtifact } = await import( + '@/lib/projects/files/application/artifacts' + ) + const f = await fixture() + const asset = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'image'), + name: 'logo.png', + contentType: 'image/png', + }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, `getFileBase64('${asset.file.id}')`), + name: 'design.pptx', + contentType: 'text/x-pptxgenjs', + }, + }) + vi.spyOn(sandboxTask, 'runSandboxTask').mockResolvedValue( + Buffer.from('PK\u0003\u0004document') + ) + const upload = storage.uploadFile + const writtenKeys: string[] = [] + vi.spyOn(storage, 'uploadFile').mockImplementation(async (options) => { + const result = await upload(options) + if (result.key.startsWith(`project/${f.projectId}/compiled/`)) { + writtenKeys.push(result.key) + if (result.key.endsWith('.published.json')) { + if (failure === 'uncertain pointer upload') throw new Error('Lost upload response') + await db.delete(permissions).where(eq(permissions.userId, f.editorId)) + } + } + return result + }) + vi.spyOn(storage, 'deleteFile').mockRejectedValue(new Error('Cleanup unavailable')) + const rendering = readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, maxBytes: 1_000_000 }, + }) + if (failure === 'revoked read') { + await expect(rendering).rejects.toMatchObject({ code: 'not_found' }) + } else { + await expect(rendering).rejects.toThrow('Lost upload response') + } + expect(writtenKeys).toHaveLength(2) + const events = await db + .select() + .from(outboxEvent) + .where( + sql`${outboxEvent.payload}::jsonb ->> 'key' LIKE ${`project/${f.projectId}/compiled/%`}` + ) + expect(events).toEqual([]) + for (const key of writtenKeys) + expect((await readFile(join(localStorageRoot, key))).length).toBeGreaterThan(0) + expect(await readFile(join(localStorageRoot, source.file.key))).toEqual( + Buffer.from(`getFileBase64('${asset.file.id}')`) + ) + } + ) + } + + check( + 'artifact pages bind embedded asset provenance and separately gate opaque model delivery', + async () => { + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const f = await fixture() + const secret = { + status: 'exact' as const, + entries: [ + { + encryptedValue: 'fixture-ciphertext', + sourceUserId: f.editorId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + const asset = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'private image bytes'), + name: 'diagram.png', + contentType: 'image/png', + secretProvenance: secret, + }, + }) + const source = `---\ntitle: Design\n---\n\n![Diagram](sim:file/${asset.file.id})` + const page = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, source), + name: 'design.html', + contentType: 'text/x-sim-page', + }, + }) + const input = { projectId: f.projectId, fileId: page.file.id, maxBytes: 1_000_000 } + const rendered = await readProjectFileArtifact.execute({ principal: f.principal, input }) + expect(rendered.contentType).toBe('text/html') + expect(rendered.buffer.toString()).toContain( + `data:image/png;base64,${Buffer.from('private image bytes').toString('base64')}` + ) + expect(rendered.secretProvenance).toEqual(secret) + await expect( + readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, forModel: true }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: asset.file.id, + content: 'public image bytes', + encoding: 'utf-8', + expectedUpdatedAt: asset.file.contentUpdatedAt, + }, + }) + const clean = await readProjectFileArtifact.execute({ + principal: f.principal, + input: { ...input, forModel: true }, + }) + expect(clean.secretProvenance).toEqual({ status: 'exact', entries: [] }) + expect(clean.buffer.toString()).toContain( + Buffer.from('public image bytes').toString('base64') + ) + } + ) + + check( + 'artifact source cannot import a foreign Project asset through a valid source-file capability', + async () => { + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const f = await fixture() + const foreign = await fixture() + const asset = await createProjectFile.execute({ + principal: foreign.principal, + input: { + ...createInput(foreign.projectId, 'foreign bytes'), + name: 'private.png', + contentType: 'image/png', + }, + }) + const page = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput( + f.projectId, + `---\ntitle: Design\n---\n\n![Diagram](sim:file/${asset.file.id})` + ), + name: 'design.html', + contentType: 'text/x-sim-page', + }, + }) + await expect( + readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: page.file.id, maxBytes: 1_000_000 }, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + } + ) + + check('a source renamed during rendering is refused', async () => { + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const { renameProjectFile } = await import('@/lib/projects/files/application/lifecycle') + const f = await fixture() + const source = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'const title = "test"'), + name: 'before.pptx', + contentType: 'text/x-pptxgenjs', + }, + }) + vi.spyOn(sandboxTask, 'runSandboxTask').mockImplementation(async () => { + await renameProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, name: 'after.docx' }, + }) + return Buffer.from('PK\u0003\u0004document') + }) + await expect( + readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, maxBytes: 1_000_000 }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + }) + + check( + 'an asset changed during render cannot be delivered with stale provenance or source identity', + async () => { + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const f = await fixture() + const asset = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'before'), + name: 'diagram.png', + contentType: 'image/png', + }, + }) + const page = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput( + f.projectId, + `---\ntitle: Design\n---\n\n![Diagram](sim:file/${asset.file.id})` + ), + name: 'design.html', + contentType: 'text/x-sim-page', + }, + }) + const download = storage.downloadFile + let changed = false + vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const bytes = await download(options) + if (!changed && options.key === asset.file.key) { + changed = true + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: asset.file.id, + content: 'after', + encoding: 'utf-8', + expectedUpdatedAt: asset.file.contentUpdatedAt, + secretProvenance: { status: 'unknown' }, + }, + }) + } + return bytes + }) + await expect( + readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: page.file.id, maxBytes: 1_000_000 }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + } + ) + + check( + 'generated Office artifacts use the canonical owner cache without exposing generator source', + async () => { + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const a = await fixture() + const b = await fixture() + const source = 'const title = "architecture"' + const mime = 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + const prepared = [] + for (const [f, marker] of [ + [a, 'project A'], + [b, 'project B'], + ] as const) { + const created = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, source), + name: 'architecture.pptx', + contentType: 'text/x-pptxgenjs', + }, + }) + const artifact = Buffer.from(`PK\u0003\u0004${marker}`) + vi.spyOn(sandboxTask, 'runSandboxTask').mockResolvedValue(artifact) + await compileFileDocument({ + owner: { entityType: 'project', entityId: f.projectId }, + source, + sourceMime: 'text/x-pptxgenjs', + fileName: 'architecture.pptx', + inputs: [], + maxBytes: 1_000_000, + }) + prepared.push({ f, created, artifact }) + } + for (const { f, created, artifact } of prepared) { + const rendered = await readProjectFileArtifact.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: created.file.id, + maxBytes: 1_000_000, + forModel: true, + }, + }) + expect(rendered.buffer).toEqual(artifact) + expect(rendered.contentType).toBe(mime) + expect(rendered.secretProvenance).toEqual({ status: 'exact', entries: [] }) + } + } + ) +}) + +afterAll(async () => { + vi.unstubAllEnvs() + vi.restoreAllMocks() + for (const f of fixtures) { + await db.delete(idempotencyKey).where(sql`${idempotencyKey.key} LIKE ${`%${f.projectId}%`}`) + await db + .delete(outboxEvent) + .where( + sql`${outboxEvent.payload}::jsonb ->> 'key' LIKE ${`project/${f.projectId}/%`} OR ${outboxEvent.payload}::jsonb -> 'owner' ->> 'entityId' = ${f.projectId}` + ) + await db.delete(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + await db.delete(folder).where(eq(folder.projectId, f.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + await db.delete(subscription).where(eq(subscription.referenceId, f.organizationId)) + await db.delete(organization).where(eq(organization.id, f.organizationId)) + await db.delete(user).where(inArray(user.id, [f.ownerId, f.editorId])) + } + await rm(localStorageRoot, { recursive: true, force: true }) + const report = + process.env.PROJECT_FILE_RENDERED_CONTENT_REPORT_PATH ?? + 'test-results/project-file-rendered-content.json' + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + await db.$client.end() +}) diff --git a/apps/sim/lib/projects/files/api/content-presenter.ts b/apps/sim/lib/projects/files/api/content-presenter.ts index 31ce36ac8d3..6edf243a029 100644 --- a/apps/sim/lib/projects/files/api/content-presenter.ts +++ b/apps/sim/lib/projects/files/api/content-presenter.ts @@ -2,17 +2,17 @@ import type { OwnedFileRecord } from '@/lib/uploads/contexts/workspace' import { FILE_CACHE_CONTROL, presentFileDelivery } from '@/lib/uploads/server/delivery' /** Presents authorized bytes without caching the mutable file URL. */ -export function presentProjectFileContent({ - file, - stream, - contentLength, -}: { - file: OwnedFileRecord - stream: ReadableStream - contentLength: number -}) { +export function presentProjectFileContent( + result: { file: OwnedFileRecord } & ( + | { stream: ReadableStream; contentLength: number } + | { content: Buffer } + ) +) { + const { file } = result + const body = 'content' in result ? result.content : result.stream + const contentLength = 'content' in result ? result.content.byteLength : result.contentLength return presentFileDelivery({ - body: stream, + body, filename: file.name, contentType: file.type, contentLength, diff --git a/apps/sim/lib/projects/files/application/artifacts.ts b/apps/sim/lib/projects/files/application/artifacts.ts new file mode 100644 index 00000000000..fd3e4853aee --- /dev/null +++ b/apps/sim/lib/projects/files/application/artifacts.ts @@ -0,0 +1,364 @@ +import type { WorkspaceFileRow } from '@sim/db/schema' +import { workspaceFiles } from '@sim/db/schema' +import { and, asc, inArray, isNull } from 'drizzle-orm' +import { requireAdmittedContent } from '@/lib/core/application/content-delivery' +import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import type { DbTransaction } from '@/lib/db/types' +import type { + ProjectFileAuthorizationContext, + ProjectFileTarget, +} from '@/lib/projects/files/application/authorization' +import { defineAuthorizedProjectFileUseCase } from '@/lib/projects/files/application/authorized-use-case' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { + buildWorkspaceFileFolderPathMap, + listFileFolders, + mapFileRecord, + type OwnedFileRecord, +} from '@/lib/uploads/contexts/workspace' +import { + getBoundWorkspaceFileSecretProvenanceByMetadata, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, + mergeWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { + collectReferencedFileIds, + compileFileDocument, + getDocumentSourceLanguage, + getE2BDocFormat, + isCompiledDocumentBuffer, + resolveDocumentRender, +} from '@/lib/uploads/documents' +import { fileDocumentInputIdentity } from '@/lib/uploads/documents/input-identity' +import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query' +import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' +import { renderSimPageDocument } from '@/lib/workspace-files/page-document' +import { + collectSimPageFileReferences, + inlineSimPageImages, +} from '@/lib/workspace-files/page-document.server' +import { createFileReadReceipt, type FileReadReceipt } from '@/lib/workspace-files/read-receipt' +import { markFileSearchArtifactReadyInTx } from '@/lib/workspace-files/search/artifact-ready' + +type ProjectOwner = { entityType: 'project'; entityId: string } +interface ArtifactInput extends ProjectFileTarget { + fileId: string + maxBytes: number + forModel?: boolean + preview?: boolean +} +interface SourceSnapshot { + file: WorkspaceFileRow + content: Buffer +} +interface ArtifactManifest { + source: WorkspaceFileRow + dependencies: WorkspaceFileRow[] + pageHtml?: string + generatedDocument: boolean + secretProvenance: WorkspaceFileSecretProvenance +} +interface PreparedArtifact { + artifactKey?: string + manifest: ArtifactManifest + buffer: Buffer + contentType: string + dependsOnReferencedFiles: boolean +} +interface ArtifactResult { + file: OwnedFileRecord + buffer: Buffer + contentType: string + dependsOnReferencedFiles: boolean + secretProvenance: WorkspaceFileSecretProvenance + receipt: FileReadReceipt +} + +function requireSameRevision(current: WorkspaceFileRow | undefined, expected: WorkspaceFileRow) { + if ( + !current || + current.key !== expected.key || + current.originalName !== expected.originalName || + current.contentType !== expected.contentType || + current.contentUpdatedAt.getTime() !== expected.contentUpdatedAt.getTime() + ) { + throw new OrchestrationError('conflict', 'A document source or input changed during rendering') + } +} + +async function loadDependencies(tx: DbTransaction, owner: ProjectOwner, ids: string[]) { + if (ids.length === 0) return [] + const rows = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(owner), + inArray(workspaceFiles.id, ids), + isNull(workspaceFiles.deletedAt) + ) + ) + .orderBy(asc(workspaceFiles.id)) + .for('share') + if (rows.length !== ids.length) + throw new OrchestrationError('not_found', 'Document input not found') + return rows +} + +const snapshotSource = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput, + { file: WorkspaceFileRow; secretProvenance: WorkspaceFileSecretProvenance | undefined } +>({ + operation: projectFileOperations.readArtifact, + async execute({ + context, + tx, + }: { + context: ProjectFileAuthorizationContext + tx: DbTransaction + input: ArtifactInput + }) { + const file = context.file + if (!file) throw new OrchestrationError('not_found', 'File not found') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) + return { file, secretProvenance: evidence.get(file.id) } + }, + afterSuccess: ({ principal, result }) => + requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance ?? { status: 'unknown' }, + view: 'opaque', + }), +}) + +const snapshotArtifact = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput & { source: SourceSnapshot }, + ArtifactManifest +>({ + operation: projectFileOperations.readArtifact, + async execute({ input, context, tx }) { + requireSameRevision(context.file, input.source.file) + const source = input.source.content.toString('utf-8') + const page = + (input.source.file.contentType === SIM_PAGE_CONTENT_TYPE || + input.source.file.originalName.toLowerCase().endsWith('.html')) && + isSimPageSource(source) + const pageHtml = page + ? renderSimPageDocument(source, { projectId: context.projectId }) + : undefined + const format = await getE2BDocFormat(input.source.file.originalName) + const generatedDocument = + !page && + format !== null && + !isCompiledDocumentBuffer(input.source.file.originalName, input.source.content) + const references = pageHtml ? collectSimPageFileReferences(pageHtml) : [] + if ( + references.some( + (reference) => reference.projectId && reference.projectId !== context.projectId + ) + ) { + throw new OrchestrationError('not_found', 'Document input not found') + } + const ids = [ + ...new Set( + pageHtml + ? references.map((reference) => reference.fileId) + : generatedDocument && format + ? collectReferencedFileIds( + source, + isDocSandboxEnabled + ? getDocumentSourceLanguage(source, format, input.source.file.contentType) + : 'javascript' + ) + : [] + ), + ] + if (ids.length > (pageHtml ? 256 : 500)) + throw new OrchestrationError( + 'payload_too_large', + 'Document exceeds the referenced input limit' + ) + const dependencies = await loadDependencies(tx, context.owner, ids) + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ + input.source.file, + ...dependencies, + ]) + return { + source: input.source.file, + dependencies, + pageHtml, + generatedDocument, + secretProvenance: mergeWorkspaceFileSecretProvenance(...evidence.values()), + } + }, + afterSuccess: ({ principal, result }) => + requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.source.id, + provenance: result.secretProvenance, + view: 'opaque', + }), +}) + +/** Renders only source-owned dependencies, then rechecks the source, every input, and current authority. */ +export const readProjectFileArtifact = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readArtifact, + ArtifactInput, + ArtifactResult, + PreparedArtifact +>({ + operation: projectFileOperations.readArtifact, + async prepare({ principal, input, context }) { + if ( + !Number.isSafeInteger(input.maxBytes) || + input.maxBytes <= 0 || + input.maxBytes > MAX_BUFFERED_TRANSFER_BYTES + ) { + throw new OrchestrationError('validation', 'Invalid document byte limit') + } + const { file } = await snapshotSource.execute({ principal, input }) + const artifact = await resolveDocumentRender( + file.originalName, + { maxBytes: input.maxBytes }, + async () => { + const content = await downloadFile({ + key: file.key, + context: 'project', + maxBytes: input.maxBytes, + }) + const manifest = await snapshotArtifact.execute({ + principal, + input: { ...input, source: { file, content } }, + }) + let remaining = manifest.pageHtml ? 32 * 1024 * 1024 : 50 * 1024 * 1024 + const inputs = [] + for (const dependency of manifest.dependencies) { + const bytes = await downloadFile({ + key: dependency.key, + context: 'project', + maxBytes: Math.min(remaining, manifest.pageHtml ? 8 * 1024 * 1024 : 25 * 1024 * 1024), + }) + remaining -= bytes.length + inputs.push({ + fileId: dependency.id, + contentType: dependency.contentType, + content: bytes, + }) + } + if (manifest.pageHtml) { + const byId = new Map(inputs.map((input) => [input.fileId, input])) + const page = await inlineSimPageImages( + manifest.pageHtml, + async ({ fileId }) => { + const asset = byId.get(fileId) + if (!asset) throw new OrchestrationError('not_found', 'Document input not found') + return { bytes: asset.content, contentType: asset.contentType, identity: fileId } + }, + { strict: true } + ) + const buffer = Buffer.from(page.html, 'utf-8') + assertKnownSizeWithinLimit(buffer.length, input.maxBytes, 'rendered page') + return { + manifest, + buffer, + contentType: 'text/html', + dependsOnReferencedFiles: inputs.length > 0, + } + } + if (manifest.generatedDocument) { + const inputIdentity = fileDocumentInputIdentity(context.owner, manifest.dependencies) + const compiled = await compileFileDocument({ + owner: context.owner, + sourceMime: file.contentType, + source: content.toString('utf-8'), + fileName: file.originalName, + inputs, + inputIdentity, + maxBytes: input.maxBytes, + }) + return { manifest, ...compiled } + } + return { + manifest, + buffer: content, + contentType: file.contentType, + dependsOnReferencedFiles: false, + } + } + ) + const derivative = input.preview + ? await resolveServableImageBytes(artifact.buffer, artifact.manifest.source.key, { + owner: context.owner, + }) + : null + if (derivative) + assertKnownSizeWithinLimit(derivative.buffer.length, input.maxBytes, 'image preview') + return { ...artifact, ...derivative } + }, + async execute({ input, context, tx, prepared }) { + if (!prepared) throw new Error('Prepared document is unavailable') + requireSameRevision(context.file, prepared.manifest.source) + const dependencies = await loadDependencies( + tx, + context.owner, + prepared.manifest.dependencies.map((file) => file.id) + ) + const currentById = new Map(dependencies.map((file) => [file.id, file])) + for (const expected of prepared.manifest.dependencies) + requireSameRevision(currentById.get(expected.id), expected) + const source = context.file + if (!source) throw new OrchestrationError('not_found', 'File not found') + if (prepared.artifactKey) + await markFileSearchArtifactReadyInTx(tx, { + owner: context.owner, + file: { fileId: source.id, key: source.key, contentUpdatedAt: source.contentUpdatedAt }, + dependencies: dependencies.map((file) => ({ + fileId: file.id, + key: file.key, + sourceContentUpdatedAt: file.contentUpdatedAt, + })), + artifactKey: prepared.artifactKey, + }) + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [ + source, + ...dependencies, + ]) + const secretProvenance = mergeWorkspaceFileSecretProvenance(...evidence.values()) + if ( + input.forModel && + (secretProvenance.status !== 'exact' || secretProvenance.entries.length !== 0) + ) { + throw new OrchestrationError('forbidden', MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE) + } + const folders = source.folderId + ? await listFileFolders(context.owner, { scope: 'all' }, tx) + : [] + return { + file: mapFileRecord(source, context.owner, buildWorkspaceFileFolderPathMap(folders)), + buffer: prepared.buffer, + contentType: prepared.contentType, + dependsOnReferencedFiles: prepared.dependsOnReferencedFiles, + secretProvenance, + receipt: createFileReadReceipt(context.owner, [source, ...dependencies]), + } + }, + async afterSuccess({ principal, result }) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance, + view: 'opaque', + }) + await reportWorkspaceFileDelivery(result.secretProvenance, result.file.id) + }, +}) diff --git a/apps/sim/lib/projects/files/application/index.ts b/apps/sim/lib/projects/files/application/index.ts index 65435ae4f2f..6909e95ef84 100644 --- a/apps/sim/lib/projects/files/application/index.ts +++ b/apps/sim/lib/projects/files/application/index.ts @@ -1,3 +1,4 @@ +export { readProjectFileArtifact } from './artifacts' export { getProjectFileListAccess } from './collection-access' export { createProjectFile, @@ -13,12 +14,14 @@ export { restoreProjectFileFolder, } from './lifecycle' export { projectFileOperations } from './operations' +export { readProjectFileCsvPreview, readProjectInlineFile } from './previews' export { getProjectFileMetadata, listProjectFileItems, listProjectFiles, resolveProjectFileReference, } from './read' +export { searchProjectFileContent } from './search' export { abortProjectFileUploadSession, completeProjectFileUploadSession, diff --git a/apps/sim/lib/projects/files/application/previews.ts b/apps/sim/lib/projects/files/application/previews.ts new file mode 100644 index 00000000000..9b51da02bc8 --- /dev/null +++ b/apps/sim/lib/projects/files/application/previews.ts @@ -0,0 +1,189 @@ +import type { Principal } from '@sim/auth/principal' +import { type WorkspaceFileRow, workspaceFiles } from '@sim/db/schema' +import { and, eq, isNull } from 'drizzle-orm' +import { requireAdmittedContent } from '@/lib/core/application/content-delivery' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import { getCsvPreviewSlice } from '@/lib/file-parsers/csv-preview-slice' +import type { + ProjectFileAuthorizationContext, + ProjectFileTarget, +} from '@/lib/projects/files/application/authorization' +import { defineAuthorizedProjectFileUseCase } from '@/lib/projects/files/application/authorized-use-case' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { + buildWorkspaceFileFolderPathMap, + listFileFolders, + mapFileRecord, +} from '@/lib/uploads/contexts/workspace' +import { + getBoundWorkspaceFileSecretProvenanceByMetadata, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query' + +interface InlineInput { + projectId: string + key?: string + referenceFileId?: string + view?: 'opaque' | 'excerpt' +} + +const resolveInlineRecord = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.readInline, + InlineInput, + { file: WorkspaceFileRow; secretProvenance: WorkspaceFileSecretProvenance | undefined } +>({ + operation: projectFileOperations.readInline, + async execute({ + input, + context, + tx, + }: { + input: InlineInput + context: ProjectFileAuthorizationContext + tx: DbTransaction + }) { + if (Boolean(input.key) === Boolean(input.referenceFileId)) + throw new OrchestrationError('validation', 'Provide exactly one file reference') + const [file] = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(context.owner), + isNull(workspaceFiles.deletedAt), + input.referenceFileId + ? eq(workspaceFiles.id, input.referenceFileId) + : eq(workspaceFiles.key, input.key ?? '') + ) + ) + .for('share') + .limit(1) + if (!file) throw new OrchestrationError('not_found', 'File not found') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) + return { file, secretProvenance: evidence.get(file.id) } + }, + afterSuccess: ({ principal, input, result }) => + requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance ?? { status: 'unknown' }, + view: input.view ?? 'opaque', + }), +}) + +/** Resolves a private object under its Project before reading, then fences its current head again. */ +const readInlineFile = defineAuthorizedProjectFileUseCase({ + operation: projectFileOperations.readInline, + async prepare({ principal, input }: { principal: Principal; input: InlineInput }) { + const { file } = await resolveInlineRecord.execute({ principal, input }) + const content = await downloadFile({ + key: file.key, + context: 'project', + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + return { file, content } + }, + async execute({ context, tx, prepared }) { + if (!prepared) throw new Error('Inline file bytes are unavailable') + const [file] = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(context.owner), + eq(workspaceFiles.id, prepared.file.id), + isNull(workspaceFiles.deletedAt) + ) + ) + .for('share') + .limit(1) + if ( + !file || + file.key !== prepared.file.key || + file.contentUpdatedAt.getTime() !== prepared.file.contentUpdatedAt.getTime() + ) + throw new OrchestrationError('conflict', 'File changed while preparing its preview') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) + const folders = file.folderId ? await listFileFolders(context.owner, { scope: 'all' }, tx) : [] + return { + file: mapFileRecord(file, context.owner, buildWorkspaceFileFolderPathMap(folders)), + content: prepared.content, + secretProvenance: evidence.get(file.id), + } + }, + async afterSuccess({ principal, result }) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance ?? { status: 'unknown' }, + view: 'opaque', + }) + await reportWorkspaceFileDelivery(result.secretProvenance, result.file.id) + }, +}) + +/** HEAD resolves the canonical image without fetching its bytes. */ +export const readProjectInlineFile = { + ...readInlineFile, + async authorize(args: Parameters[0]) { + await resolveInlineRecord.execute(args) + }, +} + +interface CsvPreviewInput extends ProjectFileTarget { + fileId: string + key: string + signal?: AbortSignal +} + +/** Reads only the bounded CSV slice and rejects head changes before exposing its rows. */ +export const readProjectFileCsvPreview = defineAuthorizedProjectFileUseCase({ + operation: projectFileOperations.readContent, + async prepare({ + input, + context, + principal, + }: { + principal: Principal + input: CsvPreviewInput + context: ProjectFileAuthorizationContext + }) { + const file = context.file + if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') + const { file: admitted } = await resolveInlineRecord.execute({ + principal, + input: { projectId: input.projectId, referenceFileId: file.id, view: 'excerpt' }, + }) + if ( + admitted.key !== file.key || + admitted.contentUpdatedAt.getTime() !== file.contentUpdatedAt.getTime() + ) + throw new OrchestrationError('conflict', 'File changed while preparing its preview') + return { + file, + slice: await getCsvPreviewSlice({ key: file.key, context: 'project', signal: input.signal }), + } + }, + async execute({ input, context, tx, prepared }) { + const file = context.file + if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') + if (!prepared || file.contentUpdatedAt.getTime() !== prepared.file.contentUpdatedAt.getTime()) + throw new OrchestrationError('conflict', 'File changed while preparing its preview') + const evidence = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file]) + return { success: true as const, ...prepared.slice, secretProvenance: evidence.get(file.id) } + }, + async afterSuccess({ principal, result, input }) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: input.fileId, + provenance: result.secretProvenance ?? { status: 'unknown' }, + view: 'excerpt', + }) + await reportWorkspaceFileDelivery(result.secretProvenance, input.fileId) + }, +}) diff --git a/apps/sim/lib/projects/files/application/search.ts b/apps/sim/lib/projects/files/application/search.ts new file mode 100644 index 00000000000..bc793df2d8d --- /dev/null +++ b/apps/sim/lib/projects/files/application/search.ts @@ -0,0 +1,123 @@ +import { getPostgresErrorCode } from '@sim/utils/errors' +import { admitEach } from '@/lib/core/application/content-delivery' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { defineAuthorizedProjectFileUseCase } from '@/lib/projects/files/application/authorized-use-case' +import { listProjectFileFolders } from '@/lib/projects/files/application/folders' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { listFileFolders } from '@/lib/uploads/contexts/workspace' +import { + mergeWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' +import type { SearchWorkspaceFileContentInput } from '@/lib/workspace-files/application/search-workspace-file-content' +import { + loadFileSearchDelivery, + resolveFileSearchFolderScope, +} from '@/lib/workspace-files/search/delivery' +import { WorkspaceFileSearchUnavailableError } from '@/lib/workspace-files/search/errors' +import { + compileFileSearchPattern, + FileSearchPatternError, +} from '@/lib/workspace-files/search/pattern' +import { type FileSearchResult, searchFileIndex } from '@/lib/workspace-files/search/repository' + +export interface SearchProjectFileContentInput + extends Omit { + projectId: string +} + +export const searchProjectFileContent = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.searchContent, + SearchProjectFileContentInput, + FileSearchResult & { secretProvenance: WorkspaceFileSecretProvenance }, + FileSearchResult +>({ + operation: projectFileOperations.searchContent, + isolationLevel: 'repeatable read', + async prepare({ input, principal, context, request }) { + const signal = input.signal ?? request?.signal + signal?.throwIfAborted() + const folders = + input.folderPaths === undefined + ? [] + : ( + await listProjectFileFolders.execute({ + principal, + input: { projectId: context.projectId }, + }) + ).folders + try { + return await searchFileIndex({ + owner: context.owner, + pattern: compileFileSearchPattern(input.query, input.mode), + maxResults: input.maxResults, + folderScope: resolveFileSearchFolderScope(folders, input), + signal, + }) + } catch (error) { + if (error instanceof FileSearchPatternError) + throw new OrchestrationError('validation', error.message) + if (error instanceof WorkspaceFileSearchUnavailableError) + throw new OrchestrationError('locked', error.message) + throw error + } + }, + async execute({ tx, input, context, prepared, request, principal }) { + if (!prepared) throw new Error('Search snapshot is missing') + try { + const folders = + input.folderPaths === undefined ? [] : await listFileFolders(context.owner, {}, tx) + const delivery = await loadFileSearchDelivery(tx, { + owner: context.owner, + classify: true, + prepared, + scope: resolveFileSearchFolderScope(folders, input), + signal: input.signal ?? request?.signal, + }) + let secretProvenance: WorkspaceFileSecretProvenance = { status: 'exact', entries: [] } + const current = prepared.sources.filter( + (source) => + !delivery.outOfScope.has(source.identity.fileId) && + delivery.provenance.has(source.identity.fileId) + ) + const admitted = await admitEach(principal, current, async (sources) => + sources.map((source) => ({ + kind: 'workspace-file' as const, + id: source.identity.fileId, + provenance: delivery.provenance.get(source.identity.fileId) ?? { + status: 'unknown' as const, + }, + view: 'excerpt' as const, + })) + ) + const admittedIds = new Set(admitted.map((source) => source.identity.fileId)) + const results = prepared.results + .filter((hit) => !delivery.outOfScope.has(hit.fileId)) + .map((hit) => + admittedIds.has(hit.fileId) + ? hit + : { + ...hit, + text: '[Preview withheld: read or grep this file to inspect protected text.]', + } + ) + for (const source of admitted) { + secretProvenance = mergeWorkspaceFileSecretProvenance( + secretProvenance, + delivery.provenance.get(source.identity.fileId) ?? { status: 'unknown' } + ) + } + return { ...prepared, results, count: results.length, sources: admitted, secretProvenance } + } catch (error) { + if (['55P03', '25P04'].includes(getPostgresErrorCode(error) ?? '')) { + throw new OrchestrationError( + 'locked', + 'File search is briefly unavailable. Try again shortly.' + ) + } + throw error + } + }, + afterSuccess: ({ result }) => reportWorkspaceFileDelivery(result.secretProvenance), +}) diff --git a/apps/sim/lib/uploads/core/storage-client.ts b/apps/sim/lib/uploads/core/storage-client.ts index 485841ec286..d23713fea8f 100644 --- a/apps/sim/lib/uploads/core/storage-client.ts +++ b/apps/sim/lib/uploads/core/storage-client.ts @@ -11,32 +11,15 @@ export function getServePathPrefix(): string { } /** - * Get file metadata from storage provider + * Read raw object metadata from the configured provider; it is not canonical file ownership. * @param key File key/name * @param customConfig Optional custom storage configuration * @returns File metadata object with userId, workspaceId, originalName, uploadedAt, etc. */ -export async function getFileMetadata( +export async function getStorageObjectMetadata( key: string, customConfig?: StorageConfig ): Promise> { - const { getFileMetadataByKey } = await import('../server/metadata') - const metadataRecord = await getFileMetadataByKey(key) - - if (metadataRecord) { - return { - ...(metadataRecord.projectId == null && - metadataRecord.context !== 'project' && - metadataRecord.userId - ? { userId: metadataRecord.userId } - : {}), - workspaceId: metadataRecord.workspaceId || '', - originalName: metadataRecord.originalName, - uploadedAt: metadataRecord.uploadedAt.toISOString(), - purpose: metadataRecord.context, - } - } - if (USE_BLOB_STORAGE) { const { headBlobObject } = await import('@/lib/uploads/providers/blob/client') const { BLOB_CONFIG } = await import('@/lib/uploads/config') diff --git a/apps/sim/lib/uploads/documents/compile.test.ts b/apps/sim/lib/uploads/documents/compile.test.ts index 13a2e6c42c2..ad6c18edbf4 100644 --- a/apps/sim/lib/uploads/documents/compile.test.ts +++ b/apps/sim/lib/uploads/documents/compile.test.ts @@ -40,7 +40,11 @@ vi.mock('@/lib/uploads/documents/compiled-store', async (importOriginal) => ({ import { type ContentDeliveryGate, observeContentDelivery } from '@/lib/core/application' import { createCopilotChatFilePrincipal } from '@/lib/mothership/auth/file-delegation' -import { collectReferencedFileIds, compileDoc } from '@/lib/uploads/documents/compile' +import { + collectReferencedFileIds, + compileDoc, + compileFileDocument, +} from '@/lib/uploads/documents/compile' import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' const executeInSandboxMock = remoteSandboxMockFns.mockExecuteInSandbox @@ -391,3 +395,51 @@ describe('isolated-VM compile for Chat', () => { expect(runSandboxTaskMock).toHaveBeenCalledTimes(2) }) }) + +describe('compileFileDocument source engine', () => { + it('uses the JavaScript PDF task even with the document sandbox enabled', async () => { + setEnvFlags({ isDocSandboxEnabled: true }) + loadCompiledDocMock.mockResolvedValue(null) + runSandboxTaskMock.mockResolvedValue(Buffer.from('%PDF-1.7')) + runSandboxTaskMock.mockClear() + executeInSandboxMock.mockClear() + const source = 'const doc = await PDFDocument.create()' + const result = await compileFileDocument({ + owner: { entityType: 'project', entityId: ID }, + source, + sourceMime: 'text/x-pdflibjs', + fileName: 'report.pdf', + inputs: [], + maxBytes: 1024, + }) + expect(result.buffer.toString()).toBe('%PDF-1.7') + expect(runSandboxTaskMock).toHaveBeenCalledWith( + 'pdf-generate', + { code: source }, + expect.any(Object) + ) + expect(executeInSandboxMock).not.toHaveBeenCalled() + }) +}) + +it('separates cached PDF artifacts by effective source dialect', async () => { + setEnvFlags({ isDocSandboxEnabled: true }) + const identities: string[] = [] + loadCompiledDocMock.mockImplementation(async (_owner, _source, _ext, identity) => { + identities.push(identity) + return Buffer.from(identity.includes('javascript') ? '%PDF-JS' : '%PDF-Python') + }) + const input = { + owner: { entityType: 'project' as const, entityId: ID }, + source: 'shared source', + fileName: 'report.pdf', + inputs: [], + maxBytes: 1024, + } + const js = await compileFileDocument({ ...input, sourceMime: 'text/x-pdflibjs' }) + const python = await compileFileDocument({ ...input, sourceMime: 'text/x-python' }) + expect(js.buffer.toString()).toBe('%PDF-JS') + expect(python.buffer.toString()).toBe('%PDF-Python') + expect(js.artifactKey).not.toBe(python.artifactKey) + expect(identities[0]).not.toBe(identities[1]) +}) diff --git a/apps/sim/lib/uploads/documents/compile.ts b/apps/sim/lib/uploads/documents/compile.ts index 915e2ee123a..fc1da9f357f 100644 --- a/apps/sim/lib/uploads/documents/compile.ts +++ b/apps/sim/lib/uploads/documents/compile.ts @@ -4,6 +4,7 @@ import { sha256Hex } from '@sim/security/hash' import { compareStrings } from '@sim/utils/string' import { ContentRefusedError, mustClassifyContent } from '@/lib/core/application' import { isDocSandboxEnabled } from '@/lib/core/config/env-flags' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' import { CodeLanguage } from '@/lib/execution/languages' import { executeInSandbox, @@ -35,6 +36,7 @@ import { } from '@/lib/workspace-files/application/admit-file-content' import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content' import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' import type { SandboxTaskId } from '@/sandboxes/registry' const logger = createLogger('CopilotDocCompile') @@ -55,7 +57,7 @@ const PYTHON_XLSX_SOURCE_MIME = 'text/x-python-xlsx' type DocEngine = 'node' | 'python' -interface E2BDocFormat { +export interface E2BDocFormat { ext: 'pptx' | 'docx' | 'pdf' | 'xlsx' engine: DocEngine formatName: 'PPTX' | 'DOCX' | 'PDF' | 'XLSX' @@ -159,7 +161,7 @@ interface ReferencedImageResolution { artifactIdentity?: string } -interface CompiledDocResult { +export interface CompiledDocResult { artifactKey?: string buffer: Buffer contentType: string @@ -618,6 +620,101 @@ async function renderPreparedDocument(source: string, fmt: E2BDocFormat, inputs: : compileDocViaE2BPython(source, fmt, inputs) } +/** Shared identity for owner-bound rendering and readers of its compiled artifacts. */ +export function fileDocumentCacheIdentity( + source: string, + format: E2BDocFormat, + sourceMime: string, + inputIdentity?: string +): string { + return JSON.stringify({ + sourceLanguage: getDocumentSourceLanguage(source, format, sourceMime), + renderer: + isDocSandboxEnabled && sourceMime !== 'text/x-pdflibjs' ? 'document-sandbox' : 'native', + inputs: inputIdentity ?? null, + }) +} + +/** Compiles a document against a fixed, authorized input manifest and canonical owner cache. */ +export async function compileFileDocument(args: { + owner: EditableFileOwner + sourceMime: string + source: string + fileName: string + inputs: readonly { fileId: string; contentType: string; content: Buffer }[] + inputIdentity?: string + maxBytes: number + signal?: AbortSignal +}): Promise { + const fmt = await getE2BDocFormat(args.fileName) + if (!fmt) throw new DocCompileUserError(`Unsupported document format: ${args.fileName}`) + if ( + args.inputs.length > MAX_REFERENCED_INPUTS || + args.inputs.some((input) => input.content.length > MAX_STAGED_FILE_BYTES) || + args.inputs.reduce((sum, input) => sum + input.content.length, 0) > MAX_STAGED_TOTAL_BYTES + ) { + throw new DocCompileUserError('Document inputs exceed the rendering limit') + } + const useDocumentSandbox = isDocSandboxEnabled && args.sourceMime !== 'text/x-pdflibjs' + const cacheIdentity = fileDocumentCacheIdentity( + args.source, + fmt, + args.sourceMime, + args.inputIdentity + ) + const cached = await loadCompiledDoc(args.owner, args.source, fmt.ext, cacheIdentity, { + maxBytes: args.maxBytes, + signal: args.signal, + }) + if (cached) + return { + buffer: cached, + artifactKey: compiledArtifactKey(args.owner, args.source, fmt.ext, cacheIdentity), + contentType: fmt.contentType, + dependsOnReferencedFiles: args.inputs.length > 0, + } + let buffer: Buffer + if (useDocumentSandbox) { + buffer = await renderPreparedDocument( + args.source, + fmt, + args.inputs.map((input) => ({ + path: `/home/user/inputs/${input.fileId}`, + content: input.content.toString('base64'), + encoding: 'base64' as const, + })) + ) + } else { + const format = COMPILABLE_FORMATS[`.${fmt.ext}`] + if (!format || isSimdocDeckSource(args.source)) { + throw new DocCompileUserError('This document format requires the document sandbox') + } + const inputs = new Map(args.inputs.map((input) => [input.fileId, input])) + buffer = await runSandboxTask( + format.taskId, + { code: args.source }, + { + ownerKey: `${args.owner.entityType}:${args.owner.entityId}`, + signal: args.signal, + resolvePreparedFile(fileId) { + const input = inputs.get(fileId) + if (!input) + throw new Error('Document requested a file outside its authorized input manifest') + return input + }, + } + ) + } + assertKnownSizeWithinLimit(buffer.length, args.maxBytes, 'compiled document') + await storeCompiledDoc(args.owner, args.source, fmt.ext, fmt.contentType, buffer, cacheIdentity) + return { + buffer, + artifactKey: compiledArtifactKey(args.owner, args.source, fmt.ext, cacheIdentity), + contentType: fmt.contentType, + dependsOnReferencedFiles: args.inputs.length > 0, + } +} + // Template-clone scripts author against the simdoc Deck (pptx) / Doc (docx) // API. The prelude supplies input_path (staged workspace files) and // OUTPUT_PATH; the finalizer saves the expected variable (scripts never save @@ -936,7 +1033,7 @@ export function isCompiledDocumentBuffer(fileName: string, buffer: Buffer): bool * not exist yet; the raw bytes are source, so serving them under the file's binary * content type would be corrupt. The caller should signal "not ready" instead. */ -type ServableDoc = +export type ServableDoc = | { kind: 'passthrough' } | { kind: 'artifact'; buffer: Buffer; contentType: string } | { kind: 'unavailable' } diff --git a/apps/sim/lib/uploads/documents/index.ts b/apps/sim/lib/uploads/documents/index.ts index 5641079904e..732f104e8cf 100644 --- a/apps/sim/lib/uploads/documents/index.ts +++ b/apps/sim/lib/uploads/documents/index.ts @@ -1,5 +1,6 @@ export { collectReferencedFileIds, + compileFileDocument, getDocumentSourceLanguage, getE2BDocFormat, isCompiledDocumentBuffer, diff --git a/apps/sim/lib/uploads/index.ts b/apps/sim/lib/uploads/index.ts index 3c026bae8f0..8974b287c06 100644 --- a/apps/sim/lib/uploads/index.ts +++ b/apps/sim/lib/uploads/index.ts @@ -5,5 +5,6 @@ export { } from '@/lib/uploads/config' export * as ChatFiles from '@/lib/uploads/contexts/chat' export * as CopilotFiles from '@/lib/uploads/contexts/copilot' -export { getFileMetadata, getServePathPrefix } from '@/lib/uploads/core/storage-client' +export { getServePathPrefix } from '@/lib/uploads/core/storage-client' export * as StorageService from '@/lib/uploads/core/storage-service' +export { getFileMetadata } from '@/lib/uploads/server/legacy-metadata' diff --git a/apps/sim/lib/uploads/server/delivery.ts b/apps/sim/lib/uploads/server/delivery.ts index 4358a128e14..a3ed7d483cd 100644 --- a/apps/sim/lib/uploads/server/delivery.ts +++ b/apps/sim/lib/uploads/server/delivery.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { ensureFileNameExtension } from '@/lib/uploads/utils/file' import { normalizeMimeType } from '@/lib/uploads/utils/mime' @@ -125,7 +126,7 @@ export const FILE_CACHE_CONTROL = { } as const /** Constructs identical security and filename headers for buffered and streamed representations. */ -function fileDeliveryHeaders(input: { +export function fileDeliveryHeaders(input: { filename: string contentType: string cacheControl: string @@ -144,6 +145,11 @@ function fileDeliveryHeaders(input: { return headers } +/** Hash only already-buffered representations; streamed downloads never need materializing. */ +export function bufferedRepresentationEtag(buffer: Buffer): string { + return `"${createHash('sha256').update(buffer).digest('base64url')}"` +} + /** Binary presenters share header assembly without coupling storage mechanics to a route framework. */ export function presentFileDelivery(input: { body: Buffer | ReadableStream @@ -167,3 +173,13 @@ export function presentFileDelivery(input: { headers, } } + +/** Immutable workspace caching requires a fixed storage object and no live referenced inputs. */ +export function workspaceFileCacheControl( + contentAddressed: boolean, + dependsOnReferencedFiles = false +): string { + return contentAddressed && !dependsOnReferencedFiles + ? FILE_CACHE_CONTROL.immutable + : FILE_CACHE_CONTROL.revalidate +} diff --git a/apps/sim/lib/uploads/server/embedded-image-refs.ts b/apps/sim/lib/uploads/server/embedded-image-refs.ts index 0fbb78ff65c..a5156c10db1 100644 --- a/apps/sim/lib/uploads/server/embedded-image-refs.ts +++ b/apps/sim/lib/uploads/server/embedded-image-refs.ts @@ -11,6 +11,7 @@ import { extractEmbeddedFileRef, storedFileId, } from '@/lib/uploads/utils/embedded-image-ref' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' /** Hard cap on embedded images collected for a bulk export bundle. */ export const MAX_EMBEDDED_IMAGES = 50 @@ -43,33 +44,48 @@ function childrenOf(token: Token): Token[] { * Links are excluded for the same reason: a link is navigated to, not displayed, so it is neither an * exportable asset nor something a document's public share should cascade to. */ -export function extractEmbeddedFileRefs(content: string): { keys: string[]; ids: string[] } { +export function extractEmbeddedFileRefs( + content: string, + owner?: EditableFileOwner +): { keys: string[]; ids: string[] } { const keys = new Set() const ids = new Set() - visitEmbeddedFileRefs(content, (ref) => { - if ('key' in ref) keys.add(ref.key) - else ids.add(ref.fileId) - return keys.size + ids.size >= MAX_EMBEDDED_IMAGES - }) + visitEmbeddedFileRefs( + content, + (ref) => { + if ('key' in ref) keys.add(ref.key) + else ids.add(ref.fileId) + return keys.size + ids.size >= MAX_EMBEDDED_IMAGES + }, + owner + ) return { keys: [...keys], ids: [...ids] } } /** Matches one stored image reference without imposing a bulk export's asset-count limit. */ -export function hasEmbeddedFileRef(content: string, target: NonNullable): boolean { - return visitEmbeddedFileRefs(content, (ref) => - 'fileId' in target - ? 'fileId' in ref && storedFileId(ref.fileId) === target.fileId - : 'key' in ref && ref.key === target.key +export function hasEmbeddedFileRef( + content: string, + target: NonNullable, + owner?: EditableFileOwner +): boolean { + return visitEmbeddedFileRefs( + content, + (ref) => + 'fileId' in target + ? 'fileId' in ref && storedFileId(ref.fileId) === target.fileId + : 'key' in ref && ref.key === target.key, + owner ) } /** Stops at the first accepted reference; callers bound document bytes before parsing. */ function visitEmbeddedFileRefs( content: string, - visit: (ref: NonNullable) => boolean + visit: (ref: NonNullable) => boolean, + owner?: EditableFileOwner ): boolean { const record = (src: string) => { - const ref = extractEmbeddedFileRef(src) + const ref = extractEmbeddedFileRef(src, owner) return ref !== null && visit(ref) } diff --git a/apps/sim/lib/uploads/server/image-derivative.ts b/apps/sim/lib/uploads/server/image-derivative.ts index b17559fbefc..82367c5f97d 100644 --- a/apps/sim/lib/uploads/server/image-derivative.ts +++ b/apps/sim/lib/uploads/server/image-derivative.ts @@ -4,17 +4,24 @@ import { getErrorMessage } from '@sim/utils/errors' import { downloadFile, uploadFile } from '@/lib/uploads/core/storage-service' import { isHevcHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' const logger = createLogger('ImageDerivative') +interface ImageDerivativeOptions { + owner: EditableFileOwner +} + /** * Keyed by the source's storage key rather than a hash of its bytes. Workspace keys * are regenerated on every content replacement, so the key is already a content * version — using it avoids streaming the whole original just to hash it. */ -function derivativeKey(storageKey: string): string { +function derivativeKey(storageKey: string, owner?: EditableFileOwner): string { const hash = createHash('sha256').update(storageKey, 'utf-8').digest('hex') - return `image-derivative/${hash}.jpg` + return owner?.entityType === 'project' + ? `project/${owner.entityId}/image-derivative/${hash}.jpg` + : `image-derivative/${hash}.jpg` } /** @@ -27,11 +34,14 @@ function derivativeKey(storageKey: string): string { * in that band into a miss, re-transcoding the original on each preview, which costs * more than serving the cached copy would have. */ -async function loadDerivative(storageKey: string): Promise { +async function loadDerivative( + storageKey: string, + options?: ImageDerivativeOptions +): Promise { try { return await downloadFile({ - key: derivativeKey(storageKey), - context: 'copilot', + key: derivativeKey(storageKey, options?.owner), + context: options?.owner.entityType === 'project' ? 'project' : 'copilot', maxBytes: MAX_BUFFERED_TRANSFER_BYTES, }) } catch { @@ -47,14 +57,19 @@ async function loadDerivative(storageKey: string): Promise { * read transcodes again. Failing the request would turn a cache problem into a broken * image for bytes we have already rendered successfully. */ -async function storeDerivative(storageKey: string, jpeg: Buffer): Promise { +async function storeDerivative( + storageKey: string, + jpeg: Buffer, + options?: ImageDerivativeOptions +): Promise { + const key = derivativeKey(storageKey, options?.owner) try { await uploadFile({ file: jpeg, fileName: 'derivative.jpg', contentType: 'image/jpeg', - context: 'copilot', - customKey: derivativeKey(storageKey), + context: options?.owner.entityType === 'project' ? 'project' : 'copilot', + customKey: key, preserveKey: true, }) } catch (error) { @@ -81,16 +96,17 @@ async function storeDerivative(storageKey: string, jpeg: Buffer): Promise */ export async function resolveServableImageBytes( buffer: Buffer, - storageKey: string + storageKey: string, + options?: ImageDerivativeOptions ): Promise<{ buffer: Buffer; contentType: string } | null> { if (!isHevcHeifContainer(buffer)) return null - const cached = await loadDerivative(storageKey) + const cached = await loadDerivative(storageKey, options) if (cached) return { buffer: cached, contentType: 'image/jpeg' } const jpeg = await transcodeHeicToJpeg(buffer) if (!jpeg) return null - await storeDerivative(storageKey, jpeg) + await storeDerivative(storageKey, jpeg, options) return { buffer: jpeg, contentType: 'image/jpeg' } } diff --git a/apps/sim/lib/uploads/core/storage-client.test.ts b/apps/sim/lib/uploads/server/legacy-metadata.test.ts similarity index 80% rename from apps/sim/lib/uploads/core/storage-client.test.ts rename to apps/sim/lib/uploads/server/legacy-metadata.test.ts index 25ead64beb3..6e1387a9d9f 100644 --- a/apps/sim/lib/uploads/core/storage-client.test.ts +++ b/apps/sim/lib/uploads/server/legacy-metadata.test.ts @@ -17,7 +17,7 @@ vi.mock('@/lib/uploads/providers/s3/client', () => ({ vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) -import { getFileMetadata } from '@/lib/uploads/core/storage-client' +import { getFileMetadata } from '@/lib/uploads/server/legacy-metadata' setUploadsConfig({ USE_S3_STORAGE: true, @@ -49,6 +49,20 @@ describe('getFileMetadata', () => { await expect(getFileMetadata('workspace/ws/key.md')).rejects.toThrow('AccessDenied') }) + it.each(['general', 'project'])( + 'does not grant a legacy uploader read for %s files', + async (context) => { + mockGetFileMetadataByKey.mockResolvedValue({ + userId: 'uploader', + context, + workspaceId: null, + originalName: 'shared.md', + uploadedAt: new Date(), + }) + expect(await getFileMetadata(`${context}/shared.md`)).not.toHaveProperty('userId') + } + ) + it('prefers the database record when one exists', async () => { mockGetFileMetadataByKey.mockResolvedValue({ userId: 'user-1', diff --git a/apps/sim/lib/uploads/server/legacy-metadata.ts b/apps/sim/lib/uploads/server/legacy-metadata.ts new file mode 100644 index 00000000000..e9d6a62e917 --- /dev/null +++ b/apps/sim/lib/uploads/server/legacy-metadata.ts @@ -0,0 +1,32 @@ +import { getStorageObjectMetadata } from '@/lib/uploads/core/storage-client' +import type { StorageConfig } from '@/lib/uploads/shared/types' + +/** + * Compatibility projection for legacy key-based authorization. Shared Project files require + * their Principal-aware operation; their creator must never become a legacy user grant. + * Owner-aware operations consume canonical metadata through resolveFileOwner instead. + */ +export async function getFileMetadata( + key: string, + customConfig?: StorageConfig +): Promise> { + const { getFileMetadataByKey } = await import('@/lib/uploads/server/metadata') + const metadataRecord = await getFileMetadataByKey(key) + + if (metadataRecord) { + return { + ...(metadataRecord.projectId == null && + metadataRecord.context !== 'project' && + metadataRecord.context !== 'general' && + metadataRecord.userId + ? { userId: metadataRecord.userId } + : {}), + workspaceId: metadataRecord.workspaceId || '', + originalName: metadataRecord.originalName, + uploadedAt: metadataRecord.uploadedAt.toISOString(), + purpose: metadataRecord.context, + } + } + + return getStorageObjectMetadata(key, customConfig) +} diff --git a/apps/sim/lib/uploads/server/markdown-export.ts b/apps/sim/lib/uploads/server/markdown-export.ts index f60a205b8cc..680c9697609 100644 --- a/apps/sim/lib/uploads/server/markdown-export.ts +++ b/apps/sim/lib/uploads/server/markdown-export.ts @@ -10,10 +10,14 @@ import remarkParse from 'remark-parse' import remarkStringify from 'remark-stringify' import { unified } from 'unified' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { StorageContext } from '@/lib/uploads/config' import { downloadFile } from '@/lib/uploads/core/storage-service' +import { bufferZipWithinLimit } from '@/lib/uploads/server/zip' import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' import { formatFileSize } from '@/lib/uploads/utils/file' +import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' const logger = createLogger('MarkdownExport') @@ -29,7 +33,11 @@ const markdownProcessor = unified() const markdownLexer = new Marked() /** Source ranges keep unrelated links, definitions, code, and whitespace byte-for-byte intact. */ -function rewriteImageSources(source: string, filenames: ReadonlyMap): string { +function rewriteImageSources( + source: string, + filenames: ReadonlyMap, + owner?: EditableFileOwner +): string { const { frontmatter, body: content } = splitFrontmatter(source) const root = markdownProcessor.parse(content) const definitions = new Map() @@ -45,8 +53,8 @@ function rewriteImageSources(source: string, filenames: ReadonlyMap { - const ref = extractEmbeddedFileRef(src) - const filename = ref && 'fileId' in ref ? filenames.get(ref.fileId) : undefined + const ref = extractEmbeddedFileRef(src, owner) + const filename = ref ? filenames.get('fileId' in ref ? ref.fileId : ref.key) : undefined return filename === undefined ? null : `./assets/${encodeURIComponent(filename)}` } const replacements: Array<{ start: number; end: number; value: string }> = [] @@ -160,6 +168,8 @@ export interface MarkdownExportAsset { context: StorageContext originalName: string size: number + /** Prepared authorized bytes let a compound export fence every asset after external IO. */ + buffer?: Buffer } export interface MarkdownExportResult { @@ -180,10 +190,11 @@ export class MarkdownExportSizeError extends Error { } function safeFilename(name: string): string { - return path + const filename = path .basename(name) .replace(/["\\]/g, '_') .replace(/[\r\n\t]/g, '') + return !filename || filename === '.' || filename === '..' ? safeZipLeafName(name) : filename } function deduplicatedFilename(preferred: string, existing: Set, imageId: string): string { @@ -203,10 +214,12 @@ export async function createMarkdownExport({ content, fileName, assets, + owner, }: { content: Buffer fileName: string assets: readonly MarkdownExportAsset[] + owner?: EditableFileOwner }): Promise { const plainMarkdown: MarkdownExportResult = { buffer: content, @@ -226,11 +239,14 @@ export async function createMarkdownExport({ if (overflowBytes !== undefined) return null let buffer: Buffer try { - buffer = await downloadFile({ - key: asset.key, - context: asset.context, - maxBytes: MAX_EXPORT_ASSET_BYTES, - }) + buffer = + asset.buffer ?? + (await downloadFile({ + key: asset.key, + context: asset.context, + maxBytes: MAX_EXPORT_ASSET_BYTES, + })) + assertKnownSizeWithinLimit(buffer.length, MAX_EXPORT_ASSET_BYTES, 'Markdown export asset') } catch (error) { logger.warn('Failed to fetch asset for export', { imageId: asset.imageId, @@ -265,16 +281,27 @@ export async function createMarkdownExport({ const markdown = rewriteImageSources( content.toString('utf-8'), - new Map([...assetMap].map(([imageId, asset]) => [imageId, asset.filename])) + new Map([...assetMap].map(([imageId, asset]) => [imageId, asset.filename])), + owner ) + const exportedBytes = retainedBytes - content.length + Buffer.byteLength(markdown, 'utf-8') + if (exportedBytes > MAX_EXPORT_TOTAL_BYTES) throw new MarkdownExportSizeError(exportedBytes) const zip = new JSZip() zip.file(safeFilename(fileName), markdown) const assetsFolder = zip.folder('assets')! for (const { filename, buffer } of assetMap.values()) assetsFolder.file(filename, buffer) + let buffer: Buffer + try { + buffer = await bufferZipWithinLimit(zip, MAX_EXPORT_TOTAL_BYTES) + } catch (error) { + if (isPayloadSizeLimitError(error)) + throw new MarkdownExportSizeError(error.observedBytes ?? MAX_EXPORT_TOTAL_BYTES + 1) + throw error + } return { - buffer: await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }), + buffer, fileName: safeFilename(`${fileName.replace(/\.[^.]+$/, '')}.zip`), contentType: 'application/zip', format: 'zip', diff --git a/apps/sim/lib/uploads/server/zip.ts b/apps/sim/lib/uploads/server/zip.ts new file mode 100644 index 00000000000..5b58f1ec46a --- /dev/null +++ b/apps/sim/lib/uploads/server/zip.ts @@ -0,0 +1,10 @@ +import type JSZip from 'jszip' +import { readNodeStreamToBufferWithLimit } from '@/lib/core/utils/stream-limits' + +/** Bounds emitted ZIP bytes while compression runs, before allocating the final output buffer. */ +export function bufferZipWithinLimit(zip: JSZip, maxBytes: number): Promise { + return readNodeStreamToBufferWithLimit( + zip.generateNodeStream({ streamFiles: true, compression: 'DEFLATE' }), + { maxBytes, label: 'file archive' } + ) +} diff --git a/apps/sim/lib/uploads/utils/embedded-image-ref.ts b/apps/sim/lib/uploads/utils/embedded-image-ref.ts index f49a97b1269..4df92926927 100644 --- a/apps/sim/lib/uploads/utils/embedded-image-ref.ts +++ b/apps/sim/lib/uploads/utils/embedded-image-ref.ts @@ -1,3 +1,6 @@ +import { parseSimFileReference } from '@/lib/uploads/utils/file-reference' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' + /** * The grammar of a markup-embedded workspace image reference: how one `src` maps to the workspace * file it points at ({@link extractEmbeddedFileRef}), and how to find the `src` values in a raw HTML @@ -25,21 +28,37 @@ export function storedFileId(spelledId: string): string { } } +/** A rejected private reference must not fall back to its credential-bearing URL. */ +export type EmbeddedFileResolution = + | { kind: 'file'; reference: NonNullable } + | { kind: 'external' } + | { kind: 'rejected' } + /** - * Parse a single embed `src` into the workspace file it references, normalizing the spellings the - * editor and file agent produce: `/api/files/serve/` (incl. `s3/`/`blob/`/`gcs/` prefixes), `/api/files/view/`, - * and the in-app path `/workspace//files/`. Returns null for absolute, `data:`, or non-workspace - * URLs (e.g. public `profile-pictures/` assets), which render as-is. - * - * A key is percent-decoded — it is matched against stored keys. An id is returned exactly as it is - * spelled in the `src`, because the export bundler rewrites embeds by searching the document for - * that spelling; handing it a decoded id it cannot find would bundle an asset and leave the markdown - * pointing at the API URL, which renders as a broken image offline. + * Classifies embeds using one grammar for inline authorization and display. Absolute internal + * references require a caller-supplied trusted origin; without it they remain rejected for SSR. */ -export function extractEmbeddedFileRef(src: string): EmbeddedFileRef { +export function resolveEmbeddedFileRef( + src: string, + owner?: EditableFileOwner, + trustedOrigin?: string +): EmbeddedFileResolution { try { - const parsed = new URL(src, 'http://placeholder') - if (parsed.origin !== 'http://placeholder') return null + const value = src.trim() + if (value.startsWith('sim:file/')) { + if (!owner) return { kind: 'rejected' } + const reference = parseSimFileReference(value.slice('sim:file/'.length), owner) + return reference?.owner?.entityType === owner.entityType && + reference.owner.entityId === owner.entityId + ? { kind: 'file', reference: { fileId: reference.fileId } } + : { kind: 'rejected' } + } + const absolute = /^[a-z][a-z0-9+.-]*:/i.test(value) || value.startsWith('//') + const origin = trustedOrigin ? new URL(trustedOrigin).origin : undefined + const parsed = new URL(value, origin ?? 'http://placeholder') + if (absolute && origin && parsed.origin !== origin) return { kind: 'external' } + const resolved = (reference: NonNullable): EmbeddedFileResolution => + absolute && !origin ? { kind: 'rejected' } : { kind: 'file', reference } const segs = parsed.pathname.split('/') if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'serve') { let keySegs = segs.slice(4) @@ -47,22 +66,57 @@ export function extractEmbeddedFileRef(src: string): EmbeddedFileRef { keySegs = keySegs.slice(1) } const raw = keySegs.join('/') - if (!raw) return null + if (!raw) return { kind: 'rejected' } const key = decodeURIComponent(raw) - return key.startsWith('workspace/') ? { key } : null + if (!key.startsWith('workspace/') && !key.startsWith('project/')) return { kind: 'external' } + return key.startsWith(owner ? `${owner.entityType}/${owner.entityId}/` : 'workspace/') + ? resolved({ key }) + : { kind: 'rejected' } + } + if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'view') { + return segs[4] ? resolved({ fileId: segs[4] }) : { kind: 'rejected' } } - if (segs[1] === 'api' && segs[2] === 'files' && segs[3] === 'view' && segs[4]) { - return { fileId: segs[4] } + if (segs[1] === 'api' && segs[2] === 'projects' && segs[4] === 'files') { + return owner?.entityType === 'project' && + decodeURIComponent(segs[3]) === owner.entityId && + segs[5] && + segs[6] === 'content' && + segs.length === 7 + ? resolved({ fileId: segs[5] }) + : { kind: 'rejected' } } - if (segs[1] === 'workspace' && segs[3] === 'files' && segs[4]) { - return { fileId: segs[4] } + if (segs[1] === 'projects' && segs[3] === 'files') { + return owner?.entityType === 'project' && + decodeURIComponent(segs[2]) === owner.entityId && + segs[4] && + segs.length === 5 + ? resolved({ fileId: segs[4] }) + : { kind: 'rejected' } } - return null + if (segs[1] === 'workspace' && segs[3] === 'files') { + if ( + !segs[4] || + (owner && + (owner.entityType !== 'workspace' || decodeURIComponent(segs[2]) !== owner.entityId)) + ) + return { kind: 'rejected' } + return resolved({ fileId: segs[4] }) + } + return { kind: 'external' } } catch { - return null + return { kind: 'rejected' } } } +/** + * Returns stored-key or document-spelled ID references for existing scanners and exporters. + * Display code uses the full classification so a rejected internal reference cannot pass through. + */ +export function extractEmbeddedFileRef(src: string, owner?: EditableFileOwner): EmbeddedFileRef { + const result = resolveEmbeddedFileRef(src, owner) + return result.kind === 'file' ? result.reference : null +} + /** * Matches `` `src` attribute values: double-quoted, single-quoted, or (validly) unquoted per * the HTML spec — the browser's own clipboard serialization always quotes it, but other producers diff --git a/apps/sim/lib/uploads/utils/file-reference.ts b/apps/sim/lib/uploads/utils/file-reference.ts new file mode 100644 index 00000000000..784b8e605ed --- /dev/null +++ b/apps/sim/lib/uploads/utils/file-reference.ts @@ -0,0 +1,29 @@ +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' + +/** Parses the ID, optional single owner, and fragment of a sim:file resource reference. */ +export function parseSimFileReference(reference: string, defaultOwner?: EditableFileOwner) { + const match = /^([^?#]+)(?:\?([^#]*))?(#.*)?$/.exec(reference) + if (!match) return null + let fileId: string + try { + fileId = decodeURIComponent(match[1]) + } catch { + return null + } + if (!/^[A-Za-z0-9_-]+$/.test(fileId)) return null + let owner = defaultOwner + if (match[2] !== undefined) { + const entries = [...new URLSearchParams(match[2])] + const entry = entries[0] + if (entries.length !== 1 || !entry) return null + const [entityType, entityId] = entry + if ( + (entityType !== 'workspace' && entityType !== 'project') || + !/^[A-Za-z0-9_-]+$/.test(entityId) + ) { + return null + } + owner = { entityType, entityId } + } + return { fileId, owner, fragment: match[3] ?? '' } +} diff --git a/apps/sim/lib/uploads/utils/file.server.test.ts b/apps/sim/lib/uploads/utils/file.server.test.ts index 6a7d21de758..8cadad75acc 100644 --- a/apps/sim/lib/uploads/utils/file.server.test.ts +++ b/apps/sim/lib/uploads/utils/file.server.test.ts @@ -222,6 +222,7 @@ describe('servable page provenance', () => { buffer: Buffer.from('rendered image'), contentType: 'text/html', contributingFiles: [contributor], + dependsOnReferencedFiles: true, }) }) }) diff --git a/apps/sim/lib/uploads/utils/file.server.ts b/apps/sim/lib/uploads/utils/file.server.ts index 96dd5c47782..8996149f1b4 100644 --- a/apps/sim/lib/uploads/utils/file.server.ts +++ b/apps/sim/lib/uploads/utils/file.server.ts @@ -410,6 +410,7 @@ export async function downloadFileFromStorage( * actually attach/upload, plus the content type that matches those bytes. */ export interface ServableFile { + dependsOnReferencedFiles?: boolean artifactKey?: string buffer: Buffer contentType: string @@ -473,6 +474,7 @@ export async function downloadServableFileFromStorage( buffer: rendered, contentType: 'text/html', contributingFiles: page.contributingFiles, + dependsOnReferencedFiles: page.contributingFiles.length > 0, } } } diff --git a/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts b/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts index cc10a685eaf..3bd9f212779 100644 --- a/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/copilot-file-versions.integration.ts @@ -40,6 +40,7 @@ import { import { runCli } from '@/lib/mothership/agent-cli/run-cli' import { createScopedCliTransport } from '@/lib/mothership/agent-cli/scoped-transport' import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport' +import { createWorkbenchFileProvenance } from '@/lib/mothership/agent-cli/workbench-file-provenance' import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation' import { updateWorkspaceFileContent, @@ -145,6 +146,12 @@ describe('chat-delegated file version history', () => { endpoint: ORIGIN, invocation, ...(registry ? { registry } : {}), + files: createWorkbenchFileProvenance({ + userId, + workspaceId: fixture.workspaceId, + sessionKey: invocation.chatId, + resolvedSecretTraceRegistry: registry, + }), resources: [], observeReads: true, }) diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file.test.ts b/apps/sim/lib/workspace-files/application/download-workspace-file.test.ts index b002a7dd1be..48bb73d7dcf 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file.test.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file.test.ts @@ -4,6 +4,7 @@ import { createSessionPrincipal, } from '@sim/testing/factories/principal.factory' import { auditMock, auditMockFns } from '@sim/testing/mocks/audit.mock' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' import { @@ -22,6 +23,7 @@ import { observeContentDelivery, } from '@/lib/core/application' import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' class FakeDocNotReadyError extends Error {} @@ -29,6 +31,8 @@ const hoisted = vi.hoisted(() => ({ fetchServable: vi.fn(), })) +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) + vi.mock('@sim/audit', () => auditMock) vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) @@ -142,7 +146,7 @@ describe('workspace file downloads', () => { downloadWorkspaceFileStream.execute({ principal: CHAT, input: { fileId: plainFile.id } }) ) expect(admit).toHaveBeenCalledWith([ - { kind: 'workspace-file', id: plainFile.id, provenance, view: 'complete' }, + { kind: 'workspace-file', id: plainFile.id, provenance, view: 'opaque' }, ]) expect(admit.mock.invocationCallOrder[0]).toBeLessThan( mocks.downloadStream.mock.invocationCallOrder[0] @@ -161,6 +165,22 @@ describe('workspace file downloads', () => { expect(mocks.downloadStream).not.toHaveBeenCalled() }) + it('refuses raw bytes when final receipt provenance is no longer admitted', async () => { + mocks.getFile.mockResolvedValue(plainFile) + mocks.provenance.mockResolvedValue({ status: 'exact', entries: [] }) + vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ + status: 'unknown', + }) + await expect( + observeContentDelivery( + { admit: async (sources) => sources.map((source) => source.provenance.status === 'exact') }, + () => + downloadWorkspaceFileStream.execute({ principal: CHAT, input: { fileId: plainFile.id } }) + ) + ).rejects.toBeInstanceOf(ContentRefusedError) + expect(mocks.downloadStream).toHaveBeenCalled() + }) + it('refuses a Chat download no gate observes', async () => { mocks.getFile.mockResolvedValue(plainFile) await expect( diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file.ts b/apps/sim/lib/workspace-files/application/download-workspace-file.ts index 7c57b67de71..e6b6952c19f 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file.ts @@ -15,8 +15,10 @@ import { } from '@/lib/workspace-files/application/admit-file-content' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/file-context' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' import { resolveRenderedWorkspaceArtifact } from '@/lib/workspace-files/application/resolve-rendered-workspace-artifact' +import { createFileReadReceipt, type FileReadReceipt } from '@/lib/workspace-files/read-receipt' export interface DownloadWorkspaceFileInput { fileId: string @@ -36,6 +38,7 @@ export interface DownloadWorkspaceFileStreamResult extends DownloadWorkspaceFile */ contentLength: number contentType: string + receipt?: FileReadReceipt } /** Audits the bytes actually handed out, which for a generated doc is not `file.size`. */ @@ -88,6 +91,7 @@ function resolveRenderedArtifact( } async function executeDownloadWorkspaceFileStream({ + input, context, principal, }: AuthorizedWorkspaceUseCaseContext< @@ -99,8 +103,24 @@ async function executeDownloadWorkspaceFileStream({ throwOnError: true, }) if (!file) throw new OrchestrationError('not_found', 'File not found') - await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'complete') - return streamWorkspaceFileRecord(file, principal) + await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque') + const result = await streamWorkspaceFileRecord(file, principal) + const provenance = await finishFileDelivery({ + authorize: () => downloadWorkspaceFileStream.authorize({ principal, input }), + receipt: + result.receipt ?? + createFileReadReceipt({ entityType: 'workspace', entityId: context.workspaceId }, [ + { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, + ]), + stream: result.stream, + }) + try { + await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque', provenance) + return result + } catch (error) { + await result.stream.cancel(error).catch(() => undefined) + throw error + } } /** @@ -121,11 +141,11 @@ export async function streamWorkspaceFileRecord( * double peak memory. */ if (needsRenderedArtifact(file.type, file.name)) { - const { buffer, contentType, contributingFiles } = await resolveRenderedArtifact( + const { buffer, contentType, contributingFiles, receipt } = await resolveRenderedArtifact( file, principal ) - await requireAdmittedContributors(principal, file.workspaceId, contributingFiles, 'complete') + await requireAdmittedContributors(principal, file.workspaceId, contributingFiles, 'opaque') return { file, stream: new ReadableStream({ @@ -138,6 +158,7 @@ export async function streamWorkspaceFileRecord( }), contentLength: buffer.length, contentType, + receipt, } } diff --git a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts index 30304f6f85e..cee5f9601bc 100644 --- a/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts +++ b/apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts @@ -1,9 +1,11 @@ -import type { Principal } from '@sim/auth/principal' +import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { downloadServableFileFromStorage, type ServableFile } from '@/lib/uploads/utils/file.server' +import { createFileReadReceipt, type FileReadReceipt } from '@/lib/workspace-files/read-receipt' import { requeueFileSearchForRenderedArtifact } from '@/lib/workspace-files/search/artifact-ready' const logger = createLogger('FetchServableWorkspaceFileBuffer') @@ -17,7 +19,7 @@ export async function fetchAuthorizedServableWorkspaceFileBuffer( fileRecord: WorkspaceFileRecord, filePrincipal: Principal, options: { maxBytes: number; signal?: AbortSignal; requestId?: string } -): Promise { +): Promise { const result = await downloadServableFileFromStorage( { id: fileRecord.id, @@ -33,6 +35,8 @@ export async function fetchAuthorizedServableWorkspaceFileBuffer( { ...options, filePrincipal, + ownerKey: + resolvePrincipalSubjectUserId(filePrincipal) ?? `workspace:${fileRecord.workspaceId}`, } ) const artifactKey = result.artifactKey @@ -68,5 +72,20 @@ export async function fetchAuthorizedServableWorkspaceFileBuffer( }) } } - return result + const receipt = createFileReadReceipt( + { entityType: 'workspace', entityId: fileRecord.workspaceId }, + [ + { ...fileRecord, contentUpdatedAt: fileRecord.contentUpdatedAt ?? fileRecord.updatedAt }, + ...(result.contributingFiles ?? []).map((file) => { + if (!['workspace', 'test', 'changelog'].includes(file.context) || !file.contentUpdatedAt) + throw new OrchestrationError('conflict', 'Document input has no canonical revision') + return { id: file.fileId, key: file.key, contentUpdatedAt: file.contentUpdatedAt } + }), + ] + ) + return { + ...result, + receipt, + dependsOnReferencedFiles: result.dependsOnReferencedFiles ?? receipt.files.length > 1, + } } diff --git a/apps/sim/lib/workspace-files/application/file-versions.test.ts b/apps/sim/lib/workspace-files/application/file-versions.test.ts index 8c2ff3e8e28..f9abd524c5b 100644 --- a/apps/sim/lib/workspace-files/application/file-versions.test.ts +++ b/apps/sim/lib/workspace-files/application/file-versions.test.ts @@ -1,5 +1,9 @@ -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { + createDelegatedPrincipal, + createSessionPrincipal, +} from '@sim/testing/factories/principal.factory' import { auditMock, auditMockFns } from '@sim/testing/mocks/audit.mock' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { realtimeNotifyMock, realtimeNotifyMockFns } from '@sim/testing/mocks/realtime-notify.mock' import { uploadsMock } from '@sim/testing/mocks/uploads.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' @@ -20,6 +24,8 @@ const hoisted = vi.hoisted(() => ({ streamRecord: vi.fn(), })) +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) + vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) vi.mock('@sim/audit', () => auditMock) @@ -45,6 +51,7 @@ import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { workspaceFileRevision } from '@/lib/workspace-files/application/file-revision' import { deleteWorkspaceFileVersion, + downloadWorkspaceFileVersion, revertWorkspaceFileVersion, } from '@/lib/workspace-files/application/file-versions' @@ -130,6 +137,16 @@ describe('file version use cases', () => { mocks.notify.mockResolvedValue(undefined) }) + it('rejects Copilot version downloads before storage because the operation accepts direct principals only', async () => { + await expect( + downloadWorkspaceFileVersion.execute({ + principal: createDelegatedPrincipal({ audience: 'sim:workspace-files' }), + input: { fileId: file.id, version: 2 }, + }) + ).rejects.toMatchObject({ name: 'PrincipalKindAuthorizationError' }) + expect(mocks.streamRecord).not.toHaveBeenCalled() + }) + describe('revertWorkspaceFileVersion', () => { it('writes the version as a new revert version carrying its provenance snapshot', async () => { mocks.getVersion.mockImplementation(getVersionAfterRevert) diff --git a/apps/sim/lib/workspace-files/application/file-versions.ts b/apps/sim/lib/workspace-files/application/file-versions.ts index 247ff84cafa..dde2023f9e7 100644 --- a/apps/sim/lib/workspace-files/application/file-versions.ts +++ b/apps/sim/lib/workspace-files/application/file-versions.ts @@ -41,6 +41,7 @@ import { import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/file-context' import { workspaceFileRevisionField } from '@/lib/workspace-files/application/file-revision' import { resolveWorkspaceFileVersionWrite } from '@/lib/workspace-files/application/file-version-write' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' import { prepareOwnedFileContentWrite } from '@/lib/workspace-files/application/owned-file-content' import { @@ -250,6 +251,24 @@ const downloadVersion = defineAuthorizedWorkspaceFileUseCase({ const result = await readFileVersionObject(version.version, () => streamWorkspaceFileRecord(recordAtVersion(file, version), principal) ) + await finishFileDelivery({ + async authorize() { + await downloadVersion.authorize({ principal, input }) + const currentFile = await loadActiveFile(context) + const current = await loadVersion(currentFile, input.version) + if ( + current.key !== version.key || + (version.isCurrent && + currentFile.contentUpdatedAt?.getTime() !== file.contentUpdatedAt?.getTime()) + ) + throw new OrchestrationError('conflict', 'The selected version changed during download') + }, + receipt: { + owner: { entityType: 'workspace', entityId: context.workspaceId }, + files: result.receipt?.files.filter((entry) => entry.id !== file.id) ?? [], + }, + stream: result.stream, + }) return { ...result, file, version } }, projectAudit: ({ result }) => ({ diff --git a/apps/sim/lib/workspace-files/application/finish-file-delivery.ts b/apps/sim/lib/workspace-files/application/finish-file-delivery.ts new file mode 100644 index 00000000000..4cd41c3e57b --- /dev/null +++ b/apps/sim/lib/workspace-files/application/finish-file-delivery.ts @@ -0,0 +1,20 @@ +import { db } from '@sim/db' +import { type FileReadReceipt, recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +/** Recheck current application authority and byte identities before publishing; dispose rejected streams. */ +export async function finishFileDelivery(input: { + authorize(): Promise + receipt: FileReadReceipt + stream?: ReadableStream +}) { + try { + await input.authorize() + return await db.transaction((tx) => recheckFileReadReceipt(tx, input.receipt), { + isolationLevel: 'repeatable read', + accessMode: 'read only', + }) + } catch (error) { + await input.stream?.cancel(error).catch(() => undefined) + throw error + } +} diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts index 3c27fe48383..d19acf2e6f2 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts @@ -1,17 +1,35 @@ import { + createDelegatedPrincipal, createSessionPrincipal, createWorkspaceApiKeyPrincipal, } from '@sim/testing/factories/principal.factory' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { uploadsMetadataMock, uploadsMetadataMockFns, } from '@sim/testing/mocks/uploads-metadata.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' +import { + workspaceFileSecretProvenanceMock, + workspaceFileSecretProvenanceMockFns, +} from '@sim/testing/mocks/workspace-file-secret-provenance.mock' import { workspaceUploadsMock, workspaceUploadsMockFns, } from '@sim/testing/mocks/workspace-uploads.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + ContentRefusedError, + observeContentDelivery, +} from '@/lib/core/application/content-delivery' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) +vi.mock( + '@/lib/uploads/contexts/workspace/file-secret-provenance', + () => workspaceFileSecretProvenanceMock +) vi.mock('@/lib/uploads/contexts/workspace', () => workspaceUploadsMock) @@ -68,6 +86,21 @@ describe('readWorkspaceFileContentByKey', () => { mocks.fetchContent.mockResolvedValue(Buffer.from('source')) }) + it('refuses bytes when final receipt provenance changes after storage opens', async () => { + const chat = createDelegatedPrincipal({ audience: WORKSPACE_FILES_DELEGATION_AUDIENCE }) + workspaceFileSecretProvenanceMockFns.mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue( + { status: 'exact', entries: [] } + ) + vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ status: 'unknown' }) + await expect( + observeContentDelivery( + { admit: async (sources) => sources.map((source) => source.provenance.status === 'exact') }, + () => readWorkspaceFileContentByKey.execute({ principal: chat, input: { key: file.key } }) + ) + ).rejects.toBeInstanceOf(ContentRefusedError) + expect(mocks.fetchContent).toHaveBeenCalled() + }) + it('authorizes the canonical file before returning its stored bytes', async () => { await expect( readWorkspaceFileContentByKey.execute({ diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts index f7cb38a5c5a..ae0313ced01 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts @@ -13,8 +13,10 @@ import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { assertOwnedFileAccess } from '@/lib/workspace-files/application/file-context' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' import { OWNED_FILE_CONTEXTS, ownedFileKind } from '@/lib/workspace-files/owned-files' +import { createFileReadReceipt } from '@/lib/workspace-files/read-receipt' export interface ReadWorkspaceFileByKeyInput { key: string @@ -53,10 +55,15 @@ async function executeReadWorkspaceFileContentByKey({ >): Promise { const file = await loadCurrentWorkspaceFileByKey(input, context) await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque') - return { - file, - content: await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }), - } + const content = await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }) + const provenance = await finishFileDelivery({ + authorize: () => readWorkspaceFileContentByKey.authorize({ principal, input }), + receipt: createFileReadReceipt({ entityType: 'workspace', entityId: context.workspaceId }, [ + { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, + ]), + }) + await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque', provenance) + return { file, content } } /** Workspace files and owned files share the `workspace/` key prefix; the row says which. */ diff --git a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.test.ts index 59ad41f7c45..a5bf732fca3 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.test.ts @@ -1,5 +1,9 @@ import { Readable } from 'node:stream' -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { + createDelegatedPrincipal, + createSessionPrincipal, +} from '@sim/testing/factories/principal.factory' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { uploadsMetadataMock, @@ -10,7 +14,23 @@ import { workspaceFileManagerMock, workspaceFileManagerMockFns, } from '@sim/testing/mocks/workspace-file-manager.mock' +import { + workspaceFileSecretProvenanceMock, + workspaceFileSecretProvenanceMockFns, +} from '@sim/testing/mocks/workspace-file-secret-provenance.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + ContentRefusedError, + observeContentDelivery, +} from '@/lib/core/application/content-delivery' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' +import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) +vi.mock( + '@/lib/uploads/contexts/workspace/file-secret-provenance', + () => workspaceFileSecretProvenanceMock +) vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) vi.mock('@/lib/uploads/contexts/workspace/file-manager', () => workspaceFileManagerMock) @@ -50,6 +70,28 @@ describe('readWorkspaceInlineFile', () => { mockDownloadFileStream.mockResolvedValue(Readable.from(Buffer.from('png'))) }) + it('refuses bytes when final receipt provenance changes after storage opens', async () => { + const chat = createDelegatedPrincipal({ + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + workspaceId: 'ws-1', + }) + workspaceFileSecretProvenanceMockFns.mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue( + { status: 'exact', entries: [] } + ) + vi.mocked(recheckFileReadReceipt).mockResolvedValueOnce({ status: 'unknown' }) + await expect( + observeContentDelivery( + { admit: async (sources) => sources.map((source) => source.provenance.status === 'exact') }, + () => + readWorkspaceInlineFile.execute({ + principal: chat, + input: { workspaceId: 'ws-1', fileId: 'f1' }, + }) + ) + ).rejects.toBeInstanceOf(ContentRefusedError) + expect(mockDownloadFileStream).toHaveBeenCalled() + }) + it('authorizes a file-id reference against the asserted workspace before reading bytes', async () => { const result = await readWorkspaceInlineFile.execute({ principal, diff --git a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts index 6ab39d205ab..d44e838c7aa 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-inline-file.ts @@ -11,7 +11,9 @@ import { downloadFileStream } from '@/lib/uploads/core/storage-service' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' import { requireAdmittedWorkspaceFile } from '@/lib/workspace-files/application/admit-file-content' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' +import { finishFileDelivery } from '@/lib/workspace-files/application/finish-file-delivery' import { fileOperations } from '@/lib/workspace-files/application/operations' +import { createFileReadReceipt } from '@/lib/workspace-files/read-receipt' export interface ReadWorkspaceInlineFileInput { workspaceId: string @@ -50,10 +52,25 @@ async function executeReadWorkspaceInlineFile({ if (!file) throw new OrchestrationError('not_found', 'Not found') await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque') - const stream = await downloadFileStream({ key: file.key, context: 'workspace' }) + const stream = nodeReadableToWebStream( + await downloadFileStream({ key: file.key, context: 'workspace' }) + ) + const provenance = await finishFileDelivery({ + authorize: () => readWorkspaceInlineFile.authorize({ principal, input }), + receipt: createFileReadReceipt({ entityType: 'workspace', entityId: context.workspaceId }, [ + { ...file, contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt }, + ]), + stream, + }) + try { + await requireAdmittedWorkspaceFile(principal, context.workspaceId, file, 'opaque', provenance) + } catch (error) { + await stream.cancel(error).catch(() => undefined) + throw error + } return { file, - stream: nodeReadableToWebStream(stream), + stream, contentAddressed: input.key !== undefined && input.key === file.key, } } diff --git a/apps/sim/lib/workspace-files/application/resolve-rendered-workspace-artifact.ts b/apps/sim/lib/workspace-files/application/resolve-rendered-workspace-artifact.ts index 89d9ded9ac0..5428ee2249a 100644 --- a/apps/sim/lib/workspace-files/application/resolve-rendered-workspace-artifact.ts +++ b/apps/sim/lib/workspace-files/application/resolve-rendered-workspace-artifact.ts @@ -1,7 +1,6 @@ import type { Principal } from '@sim/auth/principal' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { resolveDocumentRender } from '@/lib/uploads/documents' -import type { ServableFile } from '@/lib/uploads/utils/file.server' +import { resolveDocumentRender } from '@/lib/uploads/documents/render' import { fetchAuthorizedServableWorkspaceFileBuffer } from '@/lib/workspace-files/application/fetch-servable-workspace-file-buffer' /** @@ -29,7 +28,7 @@ export async function resolveRenderedWorkspaceArtifact( file: WorkspaceFileRecord, filePrincipal: Principal, options: { maxBytes: number; signal?: AbortSignal; tooLargeMessage?: (limit: string) => string } -): Promise { +) { return resolveDocumentRender(file.name, options, () => fetchAuthorizedServableWorkspaceFileBuffer(file, filePrincipal, { maxBytes: options.maxBytes, diff --git a/apps/sim/lib/workspace-files/delivery.integration.ts b/apps/sim/lib/workspace-files/delivery.integration.ts new file mode 100644 index 00000000000..c659027629f --- /dev/null +++ b/apps/sim/lib/workspace-files/delivery.integration.ts @@ -0,0 +1,310 @@ +import { mkdtempSync } from 'node:fs' +import { mkdir, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { db } from '@sim/db' +import { + changelogRelease, + permissions, + user, + userStats, + workflowTest, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { + createDelegatedPrincipal, + createSessionPrincipal, +} from '@sim/testing/factories/principal.factory' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) + +import { type ContentProvenance, observeContentDelivery } from '@/lib/core/application' +import { + createOwnedWorkspaceFile, + uploadWorkspaceFile, +} from '@/lib/uploads/contexts/workspace/file-manager' +import { markWorkspaceFileSecretProvenanceUnknown } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import * as storage from '@/lib/uploads/core/storage-service' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { downloadWorkspaceFileStream } from '@/lib/workspace-files/application/download-workspace-file' +import { readWorkspaceInlineFile } from '@/lib/workspace-files/application/read-workspace-inline-file' +import { createFileReadReceipt, recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' + +const uploadRoot = mkdtempSync(join(tmpdir(), 'sim-delivery-test-')) +setUploadDirServer(uploadRoot) +const fixtures: { ownerId: string; editorId: string; workspaceId: string }[] = [] +const checks: { name: string; status: string; durationMs: number; error?: string }[] = [] +beforeEach(() => vi.restoreAllMocks()) +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} +async function fixture() { + const ownerId = generateId() + const editorId = generateId() + const workspaceId = generateId() + await db.insert(user).values( + [ownerId, editorId].map((id) => ({ + id, + name: 'Delivery fixture', + email: `${id}@delivery.invalid`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db.insert(userStats).values({ id: generateId(), userId: ownerId }) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId, + billedAccountUserId: ownerId, + name: 'Delivery', + workspaceMode: 'personal', + }) + await db.insert(permissions).values({ + id: generateId(), + userId: editorId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + fixtures.push({ ownerId, editorId, workspaceId }) + return { ownerId, editorId, workspaceId, principal: createSessionPrincipal({ userId: editorId }) } +} +describe('Workspace delivery against current PostgreSQL authority and local storage', () => { + for (const mutation of ['membership', 'revision', 'deletion'] as const) { + check( + `workspace inline delivery cancels its opened stream after ${mutation} changes`, + async () => { + const f = await fixture() + const file = await uploadWorkspaceFile( + f.workspaceId, + f.editorId, + Buffer.alloc(1024 * 1024, 42), + 'image.png', + 'image/png', + { notifyWorkspaceChange: false } + ) + const open = storage.downloadFileStream + let source: Awaited> | undefined + vi.spyOn(storage, 'downloadFileStream').mockImplementationOnce(async (options) => { + source = await open(options) + if (mutation === 'membership') { + await db + .delete(permissions) + .where( + and(eq(permissions.userId, f.editorId), eq(permissions.entityId, f.workspaceId)) + ) + } else { + await db + .update(workspaceFiles) + .set( + mutation === 'deletion' + ? { deletedAt: new Date() } + : { contentUpdatedAt: new Date(Date.now() + 1000) } + ) + .where(eq(workspaceFiles.id, file.id)) + } + return source + }) + await expect( + readWorkspaceInlineFile.execute({ + principal: f.principal, + input: { workspaceId: f.workspaceId, fileId: file.id }, + }) + ).rejects.toBeInstanceOf(Error) + expect(source?.destroyed).toBe(true) + } + ) + } +}) + +describe('Rendered input receipts preserve owned-file eligibility', () => { + for (const context of ['test', 'changelog'] as const) { + for (const mutation of ['owner-deletion', 'foreign-workspace', 'revision', 'key'] as const) { + check(`${context} input receipt accepts its live owner and rejects ${mutation}`, async () => { + const f = await fixture() + const foreign = mutation === 'foreign-workspace' ? await fixture() : undefined + const body = await createOwnedWorkspaceFile({ + context, + workspaceId: f.workspaceId, + userId: f.ownerId, + content: 'owned delivery input', + contentType: 'text/plain', + fileName: (id) => `${id}.txt`, + insertOwner: async (tx, bodyFileId) => { + if (context === 'test') + await tx.insert(workflowTest).values({ + id: generateId(), + workspaceId: f.workspaceId, + name: generateId(), + title: 'Delivery input', + bodyFileId, + sourceHash: 'fixture', + }) + else + await tx.insert(changelogRelease).values({ + id: generateId(), + workspaceId: f.workspaceId, + title: 'Delivery input', + bodyFileId, + versionMajor: 1, + versionMinor: 0, + versionPatch: 0, + bumpReason: 'fixture', + }) + }, + }) + const [file] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, body.fileId)) + const receipt = createFileReadReceipt( + { entityType: 'workspace', entityId: f.workspaceId }, + [file] + ) + await expect( + db.transaction((tx) => recheckFileReadReceipt(tx, receipt)) + ).resolves.toBeDefined() + if (mutation === 'owner-deletion') { + if (context === 'test') + await db + .update(workflowTest) + .set({ deletedAt: new Date() }) + .where(eq(workflowTest.bodyFileId, body.fileId)) + else await db.delete(changelogRelease).where(eq(changelogRelease.bodyFileId, body.fileId)) + } else if (mutation === 'foreign-workspace') { + if (!foreign) throw new Error('Foreign workspace fixture is missing') + const foreignReceipt = createFileReadReceipt( + { entityType: 'workspace', entityId: foreign.workspaceId }, + [file] + ) + await expect( + db.transaction((tx) => recheckFileReadReceipt(tx, foreignReceipt)) + ).rejects.toThrow('A file changed') + return + } else { + await db + .update(workspaceFiles) + .set( + mutation === 'revision' + ? { contentUpdatedAt: new Date(file.contentUpdatedAt.getTime() + 1000) } + : { key: `${file.key}.changed` } + ) + .where(eq(workspaceFiles.id, file.id)) + } + await expect(db.transaction((tx) => recheckFileReadReceipt(tx, receipt))).rejects.toThrow( + 'A file changed' + ) + }) + } + } +}) + +check( + 'downloads above the buffering ceiling stream incrementally and release storage on cancellation', + async () => { + const f = await fixture() + const file = await uploadWorkspaceFile( + f.workspaceId, + f.editorId, + Buffer.from('streamed'), + 'large.bin', + 'application/octet-stream', + { notifyWorkspaceChange: false } + ) + const size = MAX_BUFFERED_TRANSFER_BYTES + 1024 + await truncate(join(uploadRoot, file.key), size) + await db.update(workspaceFiles).set({ sizeBytes: size }).where(eq(workspaceFiles.id, file.id)) + const open = storage.downloadFileStream + let source: Awaited> | undefined + vi.spyOn(storage, 'downloadFileStream').mockImplementationOnce(async (options) => { + source = await open(options) + return source + }) + const result = await downloadWorkspaceFileStream.execute({ + principal: f.principal, + input: { fileId: file.id }, + }) + expect(result.contentLength).toBe(size) + const reader = result.stream.getReader() + const first = await reader.read() + expect(first.done).toBe(false) + expect(first.value?.length).toBeLessThan(size) + await reader.cancel() + expect(source?.destroyed).toBe(true) + } +) + +check('same-revision provenance downgrade reaches the content admission gate', async () => { + const f = await fixture() + const file = await uploadWorkspaceFile( + f.workspaceId, + f.editorId, + Buffer.from('classification race'), + 'race.txt', + 'text/plain', + { notifyWorkspaceChange: false, secretProvenance: { status: 'exact', entries: [] } } + ) + const open = storage.downloadFileStream + vi.spyOn(storage, 'downloadFileStream').mockImplementationOnce(async (options) => { + const source = await open(options) + await markWorkspaceFileSecretProvenanceUnknown(f.workspaceId, [file.id]) + return source + }) + const observed: ContentProvenance[] = [] + const result = await observeContentDelivery( + { + async admit(sources) { + observed.push(...sources.map((source) => source.provenance)) + return sources.map(() => true) + }, + }, + () => + downloadWorkspaceFileStream.execute({ + principal: createDelegatedPrincipal({ + subjectUserId: f.editorId, + workspaceId: f.workspaceId, + audience: 'sim:workspace-files', + }), + input: { fileId: file.id }, + }) + ) + await result.stream.cancel() + const [current] = await db.select().from(workspaceFiles).where(eq(workspaceFiles.id, file.id)) + expect(current.contentUpdatedAt).toEqual(file.contentUpdatedAt) + expect(observed.at(-1)).toEqual({ status: 'unknown' }) +}) + +afterAll(async () => { + for (const fixture of fixtures) { + await deleteWorkspaceFixture(db, eq(workspace.id, fixture.workspaceId)) + await db.delete(user).where(inArray(user.id, [fixture.ownerId, fixture.editorId])) + } + await rm(uploadRoot, { recursive: true, force: true }) + const report = process.env.FILE_DELIVERY_INTEGRATION_REPORT_PATH + if (report) { + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + } +}) diff --git a/apps/sim/lib/workspace-files/page-compile.test.ts b/apps/sim/lib/workspace-files/page-compile.test.ts index b9eae33f957..3cc407fb9b8 100644 --- a/apps/sim/lib/workspace-files/page-compile.test.ts +++ b/apps/sim/lib/workspace-files/page-compile.test.ts @@ -62,6 +62,56 @@ describe('compileSimPage', () => { expect(html).toContain('src="/api/files/view/img9"') }) + it('resolves a qualified Project file without treating its owner query as part of the ID', () => { + const html = compileSimPage( + '---\ntitle: T\n---\n![diagram](sim:file/img%2D9?project=project%2Done)\n[open](sim:file/img%2D9?project=project%2Done#details)', + { workspaceId: 'ambient-workspace', baseUrl: 'https://sim.example/' } + ) + expect(html).toContain('src="https://sim.example/api/projects/project-one/files/img-9/content"') + expect(html).toContain( + 'href="https://sim.example/projects/project-one/files/img-9#details" data-sim-link=""' + ) + }) + + it('retains an explicit workspace owner when a Project page contains a workspace file link', () => { + const html = compileSimPage( + '---\ntitle: T\n---\n[open](sim:file/img9?workspace=source-workspace)', + { projectId: 'ambient-project' } + ) + expect(html).toContain('href="/workspace/source-workspace/files/img9" data-sim-link=""') + expect(html).not.toContain('/api/projects/ambient-project/files/img9') + }) + + it('rejects cross-owner images in a Project page before compiling byte URLs', () => { + for (const owner of ['workspace=source-workspace', 'project=other-project']) { + expect(() => + compileSimPage(`---\ntitle: T\n---\n![diagram](sim:file/img9?${owner})`, { + projectId: 'ambient-project', + }) + ).toThrow('Project pages can only embed files owned by the same Project') + } + }) + + it('leaves ambiguous and malformed file ownership inert instead of inventing a byte URL', () => { + const links = [ + 'sim:file/img9?project=one&workspace=two', + 'sim:file/img9?project=one&project=two', + 'sim:file/img9?organization=one', + 'sim:file/img9?project=', + 'sim:file/img9?project=one%2Ftwo', + 'sim:file/img%252F9?project=one', + 'sim:file/%E0%A4%A?project=one', + ] + const html = compileSimPage( + `---\ntitle: T\n---\n${links.map((link) => `[open](${link}) ![image](${link})`).join('\n')}`, + { projectId: 'ambient-project' } + ) + expect(html).not.toContain('/api/projects/') + expect(html).not.toContain('/api/files/view/') + expect(html.match(/href="sim:file\//g)).toHaveLength(links.length) + expect(html.match(/src="sim:file\//g)).toHaveLength(links.length) + }) + it('escapes html in yaml-derived values', () => { const html = compileSimPage( '---\ntitle: T\n---\n```sim:kv\n- { key: "'], + ['code.js', 'application/javascript', 'globalThis.deliveryExecuted=true'], + ['mismatch.txt', 'text/html', ''], + ]) { + const file = await create('project', name, type, content) + const path = `${prefix}/${required(file.id)}/content` + const response = await request(path) + assert.equal(response.status, 200) + assert.equal(response.headers.get('cache-control'), 'private, no-store') + assert.equal(response.headers.get('x-content-type-options'), 'nosniff') + if (name.endsWith('.svg')) + assert.match(response.headers.get('content-security-policy') ?? '', /sandbox/) + if (name.endsWith('.html') || name.endsWith('.js')) + assert.match(response.headers.get('content-disposition') ?? '', /^attachment;/) + if (!name.endsWith('.svg')) + assert.match(response.headers.get('content-security-policy') ?? '', /default-src/) + const head = await request(path, { method: 'HEAD' }) + assert.equal(head.status, 200) + assert.equal(await head.text(), '') + assert.equal(head.headers.get('cache-control'), 'private, no-store') + const denied = await request(path, { method: 'HEAD' }, true) + assert.equal(denied.status, 401) + assert.equal(denied.headers.get('content-disposition'), null) + assert.match(denied.headers.get('content-security-policy') ?? '', /default-src/) + const bypass = await request(`/api/files/serve/${required(file.key)}`) + assert.equal(bypass.status, 404) + } + const generated = await create( + 'project', + 'head.pdf', + 'text/x-pdflibjs', + 'throw new Error("HEAD must never execute this source")' + ) + const head = await request(`${prefix}/${required(generated.id)}/artifact`, { method: 'HEAD' }) + assert.equal(head.status, 200) + } +) +await check( + 'v2 Project SVG delivery and current, retained, missing and generated HEAD admission', + async () => { + const source = '' + const file = await create('project', 'v2-sandbox.svg', 'image/svg+xml', source) + const detail = `/api/v2/projects/${projectId}/files/${required(file.id)}` + const headers = { 'X-API-Key': personalKey } + const response = await request(`${detail}/content`, { headers }, true) + assert.equal(response.status, 200) + assert.equal(await response.text(), source) + assert.match(response.headers.get('content-security-policy') ?? '', /sandbox/) + assert.equal(response.headers.get('cache-control'), 'private, no-store') + const metadataResponse = await request(`${detail}/metadata`, { headers }, true) + assert.equal(metadataResponse.status, 200) + const metadata = toRecord(toRecord(await metadataResponse.json()).data) + const updated = await request( + `${detail}/content`, + { + method: 'PUT', + headers, + body: JSON.stringify({ + content: `${source}\n`, + encoding: 'utf-8', + expectedRevision: required(metadata.revision), + }), + }, + true + ) + assert.equal(updated.status, 200) + for (const suffix of ['/content', '/versions/1/content']) { + const head = await request(`${detail}${suffix}`, { method: 'HEAD', headers }, true) + assert.equal(head.status, 200) + assert.equal(await head.text(), '') + assert.equal(head.headers.get('cache-control'), 'private, no-store') + const denied = await request(`${detail}${suffix}`, { method: 'HEAD' }, true) + assert.equal(denied.status, 401) + assert.equal(denied.headers.get('content-disposition'), null) + } + const missing = await request( + `${detail}/versions/999999/content`, + { method: 'HEAD', headers }, + true + ) + assert.equal(missing.status, 404) + const generated = await create( + 'project', + 'v2-head.pdf', + 'text/x-pdflibjs', + 'throw new Error("HEAD must not compile")' + ) + for (const path of [`/api/v2/projects/${projectId}/files/${required(generated.id)}/content`]) { + const head = await request(path, { method: 'HEAD', headers }, true) + assert.equal(head.status, 200) + assert.equal(await head.text(), '') + assert.equal(head.headers.get('cache-control'), 'private, no-store') + } + } +) +await check('Workspace immutable URLs, authorized 304 and browser cache reuse', async () => { + const file = await create('workspace', 'cache.txt', 'text/plain', 'workspace cache bytes') + const path = `/api/files/serve/${required(file.key)}` + const first = await request(path) + assert.equal(first.status, 200) + const etag = required(first.headers.get('etag')) + const conditional = await request(path, { headers: { 'If-None-Match': etag } }) + assert.equal(conditional.status, 304) + assert.equal(conditional.headers.get('x-content-type-options'), 'nosniff') + const denied = await request(path, { headers: { 'If-None-Match': etag } }, true) + assert.equal(denied.status, 401) + const immutable = await request(`${path}?v=1`) + assert.equal(immutable.headers.get('cache-control'), 'private, max-age=31536000, immutable') + const browser = await chromium.launch({ channel: 'chrome', headless: true }) + try { + const context = await browser.newContext() + await context.addCookies( + cookie.split('; ').map((entry) => ({ + name: entry.slice(0, entry.indexOf('=')), + value: entry.slice(entry.indexOf('=') + 1), + url: base.origin, + })) + ) + const page = await context.newPage() + await page.goto(new URL('/api/health', base).href) + const cdp = await context.newCDPSession(page) + await cdp.send('Network.enable') + const cached: string[] = [] + cdp.on('Network.requestServedFromCache', (event) => cached.push(event.requestId)) + for (let attempt = 0; attempt < 2; attempt++) { + const text = await page.evaluate( + async (url) => (await fetch(url)).text(), + `${path}?v=browser` + ) + assert.equal(text, 'workspace cache bytes') + } + const timing = await page.evaluate(() => + performance.getEntriesByType('resource').map((entry) => { + const resource = entry as PerformanceResourceTiming + return { + name: resource.name, + transferSize: resource.transferSize, + encodedBodySize: resource.encodedBodySize, + } + }) + ) + await writeFile(`${reportPath}.cache.json`, JSON.stringify({ cached, timing }, null, 2)) + assert.ok( + cached.length > 0 || + timing + .filter((entry) => entry.name.endsWith('?v=browser')) + .some((entry) => entry.transferSize === 0 && entry.encodedBodySize > 0), + 'Second browser fetch must use its private cache' + ) + const svg = await create( + 'workspace', + 'sandbox.svg', + 'image/svg+xml', + '' + ) + await page.goto(new URL(`/api/files/serve/${required(svg.key)}`, base).href) + assert.equal(await page.evaluate('globalThis.deliveryExecuted'), undefined) + await page.screenshot({ path: `${reportPath}.svg.png` }) + } finally { + await browser.close() + } +}) +await check('Nonfile and file JSON responses retain the application CSP', async () => { + for (const path of ['/api/health', prefix]) { + const response = await request(path) + assert.equal(response.status, 200) + assert.match(response.headers.get('content-security-policy') ?? '', /default-src/) + assert.doesNotMatch(response.headers.get('content-security-policy') ?? '', /sandbox/) + } +}) +if (checks.some((check) => !check.passed)) process.exitCode = 1 diff --git a/apps/sim/scripts/test-project-file-rendered-e2e.ts b/apps/sim/scripts/test-project-file-rendered-e2e.ts new file mode 100644 index 00000000000..fd3a4d5ca1b --- /dev/null +++ b/apps/sim/scripts/test-project-file-rendered-e2e.ts @@ -0,0 +1,410 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { basename, dirname, join } from 'node:path' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { toArray, toRecord } from '@sim/utils/object' +import JSZip from 'jszip' +import { PDFDocument, PDFName } from 'pdf-lib' +import sharp from 'sharp' + +type Auth = 'session' | 'personal' | 'workspace' | 'none' +type Format = 'docx' | 'pptx' | 'pdf' +interface CreatedFile { + id: string + projectId: string + name: string + revision: string + source?: string +} +interface Check { + name: string + passed: boolean + durationMs: number + error?: string +} + +const reportPath = required( + process.env.PROJECT_FILE_RENDERED_REPORT_PATH, + 'PROJECT_FILE_RENDERED_REPORT_PATH is required' +) +const fixtureDir = required(process.env.PROJECT_FILE_RENDERED_FIXTURE_DIR) +const runId = generateId() +const artifactDir = join(dirname(reportPath), `${basename(reportPath, '.json')}-${runId}`) +const checks: Check[] = [] +const requests: { method: string; path: string; auth: Auth; status: number; durationMs: number }[] = + [] +const artifacts: { format: Format; path: string; size: number; sha256: string }[] = [] +const files: CreatedFile[] = [] +const archived = new Set() +const documents = new Map() +const rendered = new Map() +const marker = `ProjectRendered${runId.replaceAll('-', '')}` +const mimeTypes = { + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + pdf: 'application/pdf', +} as const +const sourceMimeTypes = { + docx: 'text/x-docxjs', + pptx: 'text/x-pptxgenjs', + pdf: 'text/x-pdflibjs', +} as const +let base: URL +let cookie = '' +let keys: Record = {} +let projectId = '' +let foreignProjectId = '' +let asset: CreatedFile | undefined +let foreignAsset: CreatedFile | undefined +let firstImage = Buffer.alloc(0) +let secondImage = Buffer.alloc(0) + +function required(value: unknown, message = 'Required fixture field missing'): string { + assert.ok(typeof value === 'string' && value.length > 0, message) + return value +} + +async function fixture(name: string) { + return toRecord(JSON.parse(await readFile(join(fixtureDir, name), 'utf8'))) +} + +async function saveReport() { + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile( + reportPath, + JSON.stringify( + { + runId, + boundary: 'Running HTTP application, real isolated-vm document tasks and Project assets', + limitation: 'Remote Python/XLSX rendering is not exercised without configured credentials', + checks, + requests, + artifacts, + }, + null, + 2 + ) + ) +} + +async function check(name: string, run: () => Promise) { + const start = performance.now() + try { + await run() + checks.push({ name, passed: true, durationMs: performance.now() - start }) + return true + } catch (error) { + checks.push({ + name, + passed: false, + durationMs: performance.now() - start, + error: getErrorMessage(error), + }) + return false + } finally { + await saveReport() + } +} + +async function request( + method: string, + path: string, + auth: Auth = 'personal', + body?: Record +) { + const headers = new Headers({ Origin: base.origin, 'Content-Type': 'application/json' }) + if (auth === 'session') headers.set('Cookie', cookie) + if (auth === 'personal' || auth === 'workspace') headers.set('X-API-Key', required(keys[auth])) + const start = performance.now() + // boundary-raw-fetch: this standalone proof validates real document bytes through the disposable local HTTP app. + const response = await fetch(new URL(path, base), { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(120_000), + }) + const buffer = Buffer.from(await response.arrayBuffer()) + requests.push({ + method, + path, + auth, + status: response.status, + durationMs: performance.now() - start, + }) + return { status: response.status, headers: response.headers, buffer } +} + +function json(response: Awaited>, status = 200) { + assert.equal(response.status, status, `Expected HTTP ${status}, received ${response.status}`) + assert.match(response.headers.get('content-type') ?? '', /application\/json/) + return toRecord(JSON.parse(response.buffer.toString('utf8'))) +} + +function filePath(file: CreatedFile) { + return `/api/v2/projects/${file.projectId}/files/${file.id}` +} + +function artifactPath(file: CreatedFile) { + return `/api/projects/${file.projectId}/files/${file.id}/artifact` +} + +async function create( + ownerId: string, + name: string, + content: string | Buffer, + contentType: string +) { + const data = toRecord( + json( + await request('POST', `/api/v2/projects/${ownerId}/files`, 'personal', { + name, + content: typeof content === 'string' ? content : content.toString('base64'), + encoding: typeof content === 'string' ? 'utf-8' : 'base64', + contentType, + }), + 201 + ).data + ) + const file = { + id: required(data.id), + projectId: ownerId, + name, + revision: required(data.revision), + ...(typeof content === 'string' ? { source: content } : {}), + } + files.push(file) + return file +} + +function program(format: Format, imageId: string) { + if (format === 'docx') + return `addSection({ children: [new docx.Paragraph(${JSON.stringify(marker)}), new docx.Paragraph({ children: [await addImage(${JSON.stringify(imageId)}, {width: 32, height: 32})] })] });` + if (format === 'pptx') + return `const slide = pptx.addSlide(); slide.addText(${JSON.stringify(marker)}, {x: 0.5, y: 0.5, w: 8, h: 0.5}); await addImage(slide, ${JSON.stringify(imageId)}, {x: 1, y: 1, w: 1, h: 1});` + return `const page = pdf.addPage([320, 240]); const font = await pdf.embedFont(StandardFonts.Helvetica); page.drawText(${JSON.stringify(marker)}, {x: 10, y: 210, size: 8, font}); await drawImage(page, ${JSON.stringify(imageId)}, {x: 20, y: 30, width: 64, height: 64});` +} + +async function assertDocument(format: Format, buffer: Buffer, image: Buffer) { + if (format === 'pdf') { + assert.equal(buffer.subarray(0, 5).toString(), '%PDF-') + const pdf = await PDFDocument.load(buffer) + assert.equal(pdf.getPageCount(), 1) + const page = pdf.getPage(0) + assert.deepEqual(page.getSize(), { width: 320, height: 240 }) + assert.ok( + page.node.Resources()?.has(PDFName.of('XObject')), + 'PDF must contain the prepared image' + ) + return + } + const zip = await JSZip.loadAsync(buffer) + const xml = zip.file(format === 'docx' ? 'word/document.xml' : 'ppt/slides/slide1.xml') + assert.ok(xml, 'Rendered Office archive must contain its document XML') + assert.ok( + (await xml.async('string')).includes(marker), + 'Rendered document must contain the source marker' + ) + const media = zip.file(format === 'docx' ? /^word\/media\// : /^ppt\/media\//) + const bytes = await Promise.all(media.map((entry) => entry.async('nodebuffer'))) + assert.ok( + bytes.some((value) => value.equals(image)), + 'Rendered document must embed the current Project image bytes' + ) +} + +async function readArtifact(file: CreatedFile, format: Format) { + const response = await request('GET', artifactPath(file), 'session') + assert.equal(response.status, 200, `Real ${format} compilation returned HTTP ${response.status}`) + assert.equal(response.headers.get('content-type')?.split(';')[0], mimeTypes[format]) + return response.buffer +} + +try { + const ready = await check( + 'Local fixtures create source documents and images through real APIs', + async () => { + base = new URL(process.env.PROJECT_FILE_RENDERED_BASE_URL ?? 'http://127.0.0.1:3300') + assert.ok( + ['127.0.0.1', 'localhost', '[::1]'].includes(base.hostname), + 'Requires a disposable loopback runtime' + ) + keys = await fixture('v2-fixture-keys.json') + cookie = toArray((await fixture('owner-account.json')).cookies) + .map((value) => required(value).split(';')[0]) + .join('; ') + assert.ok(cookie) + projectId = required(toRecord((await fixture('http-project-fixture.json')).project).id) + foreignProjectId = + process.env.PROJECT_FILE_RENDERED_FOREIGN_PROJECT_ID ?? + required( + toRecord((await fixture('public-browser-wrong-owner-fixtures.json')).wrongOwner).projectId + ) + assert.notEqual(projectId, foreignProjectId) + firstImage = await sharp({ + create: { width: 16, height: 16, channels: 3, background: '#0055ff' }, + }) + .png() + .toBuffer() + secondImage = await sharp({ + create: { width: 16, height: 16, channels: 3, background: '#ff2200' }, + }) + .png() + .toBuffer() + asset = await create(projectId, `rendered-${runId}.png`, firstImage, 'image/png') + foreignAsset = await create( + foreignProjectId, + `rendered-foreign-${runId}.png`, + secondImage, + 'image/png' + ) + for (const format of ['docx', 'pptx', 'pdf'] as const) { + documents.set( + format, + await create( + projectId, + `rendered-${runId}.${format}`, + program(format, asset.id), + sourceMimeTypes[format] + ) + ) + } + } + ) + if (ready) { + await check('Artifact HEAD authorizes without executing a document program', async () => { + const file = await create( + projectId, + `head-only-${runId}.pdf`, + 'throw new Error("Artifact HEAD must not execute this program");', + sourceMimeTypes.pdf + ) + const head = await request('HEAD', artifactPath(file), 'session') + assert.equal(head.status, 200) + assert.equal(head.buffer.length, 0) + assert.equal((await request('HEAD', artifactPath(file), 'none')).status, 401) + const rendered = await request('GET', artifactPath(file), 'session') + assert.ok(rendered.status >= 400, 'The program must fail if actually rendered') + }) + for (const format of ['docx', 'pptx', 'pdf'] as const) { + await check( + `Real ${format.toUpperCase()} rendering returns a parseable document with a Project asset`, + async () => { + const file = documents.get(format) + assert.ok(file) + const buffer = await readArtifact(file, format) + await assertDocument(format, buffer, firstImage) + rendered.set(format, buffer) + const path = join(artifactDir, `rendered.${format}`) + await mkdir(artifactDir, { recursive: true }) + await writeFile(path, buffer) + artifacts.push({ + format, + path, + size: buffer.length, + sha256: createHash('sha256').update(buffer).digest('hex'), + }) + } + ) + } + await check( + 'Same-ID asset content revision invalidates the compiled document cache', + async () => { + assert.ok(asset) + const file = documents.get('docx') + const original = rendered.get('docx') + assert.ok(file && original, 'A successful initial render is required') + assert.deepEqual(await readArtifact(file, 'docx'), original) + const updated = toRecord( + json( + await request('PUT', `${filePath(asset)}/content`, 'personal', { + content: secondImage.toString('base64'), + encoding: 'base64', + expectedRevision: asset.revision, + }) + ).data + ) + assert.equal(updated.id, asset.id) + assert.notEqual(required(updated.revision), asset.revision) + const buffer = await readArtifact(file, 'docx') + await assertDocument('docx', buffer, secondImage) + assert.equal(buffer.equals(original), false) + await writeFile(join(artifactDir, 'rendered-updated.docx'), buffer) + } + ) + await check('A caller with both Projects cannot compile a foreign-owner asset', async () => { + assert.ok(foreignAsset) + const ownRead = await request('GET', `${filePath(foreignAsset)}/content`) + assert.equal(ownRead.status, 200) + assert.deepEqual(ownRead.buffer, secondImage) + const file = await create( + projectId, + `rendered-foreign-${runId}.docx`, + program('docx', foreignAsset.id), + sourceMimeTypes.docx + ) + assert.equal((await request('GET', artifactPath(file), 'session')).status, 404) + }) + await check( + 'Cached private document bytes still reject anonymous and workspace-key callers', + async () => { + const file = documents.get('docx') + assert.ok(file) + await readArtifact(file, 'docx') + assert.equal((await request('GET', artifactPath(file), 'none')).status, 401) + assert.equal((await request('GET', artifactPath(file), 'workspace')).status, 401) + } + ) + await check( + 'An archived dependency cannot be delivered from a warm artifact cache', + async () => { + assert.ok(asset) + const file = documents.get('docx') + assert.ok(file) + await readArtifact(file, 'docx') + json( + await request('POST', `/api/v2/projects/${projectId}/files/archive`, 'personal', { + fileIds: [asset.id], + }) + ) + archived.add(asset.id) + assert.equal((await request('GET', artifactPath(file), 'session')).status, 404) + } + ) + await check( + 'Rendering and denied re-reads preserve every editable source and revision', + async () => { + for (const file of files.filter((file) => file.source !== undefined)) { + const response = await request('GET', `${filePath(file)}/content`) + assert.equal(response.status, 200) + assert.equal(response.buffer.toString('utf8'), file.source) + const metadata = toRecord(json(await request('GET', `${filePath(file)}/metadata`)).data) + assert.equal(metadata.revision, file.revision) + } + } + ) + } +} catch (error) { + checks.push({ + name: 'Harness completion', + passed: false, + durationMs: 0, + error: getErrorMessage(error), + }) +} finally { + await check('Archive only fixtures created by this run', async () => { + const owners = new Set(files.map((file) => file.projectId)) + for (const owner of owners) { + const fileIds = files + .filter((file) => file.projectId === owner && !archived.has(file.id)) + .map((file) => file.id) + if (fileIds.length) + json( + await request('POST', `/api/v2/projects/${owner}/files/archive`, 'personal', { fileIds }) + ) + } + }) + await saveReport() + process.exitCode = checks.some((entry) => !entry.passed) ? 1 : 0 +} diff --git a/apps/sim/scripts/test-project-files-e2e.ts b/apps/sim/scripts/test-project-files-e2e.ts index 984fa495cf1..8804315717d 100644 --- a/apps/sim/scripts/test-project-files-e2e.ts +++ b/apps/sim/scripts/test-project-files-e2e.ts @@ -299,8 +299,10 @@ process.once('SIGINT', interrupt) process.once('SIGTERM', interrupt) const suites = [ + ['delivery', 'FILE_DELIVERY', 'test-file-delivery-e2e.ts'], ['browser', 'PROJECT_FILE_BROWSER', 'test-project-file-browser-e2e.ts'], ['history', 'PROJECT_FILE_HISTORY', 'test-project-file-history-e2e.ts'], + ['rendered', 'PROJECT_FILE_RENDERED', 'test-project-file-rendered-e2e.ts'], ['realtime', 'FILE_LIST_REALTIME', 'test-file-list-realtime-e2e.ts'], ] as const diff --git a/packages/sim-cli/src/contract/commands.ts b/packages/sim-cli/src/contract/commands.ts index 1d6e415cbb3..b51a87bdb16 100644 --- a/packages/sim-cli/src/contract/commands.ts +++ b/packages/sim-cli/src/contract/commands.ts @@ -803,6 +803,19 @@ export const CLI_CONTRACT: CliContract = { createProjectFileFolder: { command: 'projects files folders create' }, updateProjectFileFolder: { command: 'projects files folders update' }, restoreProjectFileFolder: { command: 'projects files folders restore' }, + searchProjectFileContent: { + command: 'projects files search', + flags: { + folderPaths: FOLDER_PATHS_FLAG, + includeSubfolders: { boolean: true, negatable: true }, + }, + itemsPath: 'results', + columns: [ + { header: 'file', path: 'fileId' }, + { header: 'line', path: 'lineNumber' }, + { header: 'text' }, + ], + }, moveProjectFileItems: { command: 'projects files move', flags: { diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index e59ef1d56a5..1132e8338d7 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -13514,6 +13514,53 @@ export type SearchKnowledgeResponse = { data: SearchKnowledgeResponseRef1 } +/** `GET /api/v2/projects/[projectId]/files/search` */ +export type SearchProjectFileContentParams = { + projectId: string +} + +export type SearchProjectFileContentQuery = { + query: string + mode?: 'exact' | 'regex' + maxResults?: number + folderPaths?: string + includeSubfolders?: + | 'true' + | '1' + | 'yes' + | 'on' + | 'y' + | 'enabled' + | 'false' + | '0' + | 'no' + | 'off' + | 'n' + | 'disabled' +} + +type SearchProjectFileContentResponseRef0 = { + results: Array<{ + fileId: string + lineNumber: number + text: string + }> + count: number + truncated: boolean + complete: boolean + indexStatus: { + readyFiles: number + pendingFiles: number + failedFiles: number + skippedFiles: number + partialFiles: number + } +} + +export type SearchProjectFileContentResponse = { + data: SearchProjectFileContentResponseRef0 +} + /** `POST /api/v2/tables/[tableId]/rows/search` */ export type SearchTableRowsParams = { tableId: string @@ -23185,6 +23232,53 @@ export const V2_OPERATIONS = { }, }, }, + searchProjectFileContent: { + method: 'GET', + path: '/api/v2/projects/[projectId]/files/search', + pathParams: ['projectId'] as const, + pathParamDocs: { projectId: 'Project identifier.' }, + responseMode: 'json', + summary: 'Search Project File Content', + workspaceKeyUnsupported: true, + query: { + query: { + kind: 'string', + required: true, + describe: 'Regular expression, or exact text when `mode` is `exact`.', + }, + mode: { + kind: 'enum', + values: ['exact', 'regex'] as const, + default: 'regex', + describe: 'How `query` is read.', + }, + maxResults: { kind: 'integer', default: 50, describe: 'Maximum matching lines to return.' }, + folderPaths: { + kind: 'string', + describe: + 'Comma-separated folder paths within the Project. Omit to search the entire Project; index coverage applies to the selected folders.', + }, + includeSubfolders: { + kind: 'enum', + values: [ + 'true', + '1', + 'yes', + 'on', + 'y', + 'enabled', + 'false', + '0', + 'no', + 'off', + 'n', + 'disabled', + ] as const, + describe: + 'Whether the scope descends into nested folders. Absent means yes. The listed spellings are the whole accepted vocabulary and are case-sensitive; any other value is rejected.', + }, + }, + }, searchTableRows: { method: 'POST', path: '/api/v2/tables/[tableId]/rows/search', diff --git a/packages/testing/src/mocks/file-read-receipt.mock.ts b/packages/testing/src/mocks/file-read-receipt.mock.ts new file mode 100644 index 00000000000..1fdacb3a286 --- /dev/null +++ b/packages/testing/src/mocks/file-read-receipt.mock.ts @@ -0,0 +1,14 @@ +import { vi } from 'vitest' + +const fileReadReceiptMockFns = { + mockCreateFileReadReceipt: vi.fn(() => ({ + owner: { entityType: 'workspace' as const, entityId: 'workspace-1' }, + files: [], + })), + mockRecheckFileReadReceipt: vi.fn(async () => ({ status: 'exact' as const, entries: [] })), +} + +export const fileReadReceiptMock = { + createFileReadReceipt: fileReadReceiptMockFns.mockCreateFileReadReceipt, + recheckFileReadReceipt: fileReadReceiptMockFns.mockRecheckFileReadReceipt, +} diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 738a21d39ea..1f3e57a79e0 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1582,7 +1582,6 @@ "apps/sim/lib/mothership/chat/payload.test.ts": 1, "apps/sim/lib/mothership/chat/persisted-message.ts": 4, "apps/sim/lib/mothership/chat/post.test.ts": 2, - "apps/sim/lib/mothership/chat/post.ts": 1, "apps/sim/lib/mothership/chat/process-contents.test.ts": 17, "apps/sim/lib/mothership/chat/stream-liveness.ts": 1, "apps/sim/lib/mothership/chat/title-context.test.ts": 4, diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index ca3bcffbbc6..ecc1d0dc95d 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -283,7 +283,6 @@ "apps/sim/lib/uploads/contexts/workspace/track-chat-upload.test.ts": 1, "apps/sim/lib/uploads/contexts/workspace/file-manager-errors.test.ts": 1, "apps/sim/lib/uploads/contexts/workspace/file-manager.test.ts": 1, - "apps/sim/lib/uploads/core/storage-client.ts": 1, "apps/sim/lib/webhooks/providers/table.ts": 1, "apps/sim/lib/webhooks/providers/whatsapp.test.ts": 1, "apps/sim/lib/workflows/comparison/normalize.test.ts": 1, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index ac3c533a2a3..e1f9d134e00 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -3254,7 +3254,6 @@ "apps/sim/lib/api/contracts/v2/files.ts#v2FileMetadataSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2FileReferenceParamsSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2FileScopeSchema", - "apps/sim/lib/api/contracts/v2/files.ts#v2FileSearchResultsSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2FileShareSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2FileUploadParamsSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2FileUploadWorkspaceQuerySchema", @@ -5060,7 +5059,6 @@ "apps/sim/lib/uploads/contexts/workspace/file-manager.ts#matchesWorkspaceFilePattern", "apps/sim/lib/uploads/contexts/workspace/file-secret-provenance.ts#WorkspaceFileSecretProvenanceEnvelope", "apps/sim/lib/uploads/index.ts#getStorageConfig", - "apps/sim/lib/uploads/server/heic.ts#MAX_TRANSCODE_INPUT_BYTES", "apps/sim/lib/uploads/shared/types.ts#WORKSPACE_SCOPED_CONTEXTS", "apps/sim/lib/uploads/upload-session/application.ts#UploadSessionCreateResult", "apps/sim/lib/uploads/upload-session/application.ts#UploadSessionTransfer",