diff --git a/apps/docs/content/docs/cli/files.mdx b/apps/docs/content/docs/cli/files.mdx index 315ec4600c8..597b8fbb2d8 100644 --- a/apps/docs/content/docs/cli/files.mdx +++ b/apps/docs/content/docs/cli/files.mdx @@ -802,6 +802,27 @@ sim files mkdir API request and response schemas: [createFileFolder](/api-reference/files/createFilesFolder). +## Other commands + +### Copy file items + +OAuth login or personal API key required. + +```bash +sim files copy [options] +``` + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--source ` | Yes | Source owner and selected fileIds or folderIds (JSON, or @path / @- to read a file or stdin). | +| `--destination ` | Yes | Destination owner and optional folderId (JSON, or @path / @- to read a file or stdin). | + + + ## Related docs - [Files](/files) diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 27318d9e39e..de34d2524d6 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -988,6 +988,25 @@ sim files batch-delete [options] API request and response schemas: [bulkDeleteFiles](/api-reference/files/bulkDeleteFiles). +### sim files copy + +Copy file items (OAuth login or personal API key required) + +```bash +sim files copy [options] +``` + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--source ` | Yes | Source owner and selected fileIds or folderIds (JSON, or @path / @- to read a file or stdin). | +| `--destination ` | Yes | Destination owner and optional folderId (JSON, or @path / @- to read a file or stdin). | + + + ### sim files create Create file diff --git a/apps/sim/app/api/files/copy/route.ts b/apps/sim/app/api/files/copy/route.ts new file mode 100644 index 00000000000..d4755885e0b --- /dev/null +++ b/apps/sim/app/api/files/copy/route.ts @@ -0,0 +1,21 @@ +import { copyFileItemsContract } from '@/lib/api/contracts/file-copy' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { presentCopiedFileItems } from '@/lib/workspace-files/api/copy-presenter' +import { copyFileItems } from '@/lib/workspace-files/application/copy-file-items' +import { fileCopyOperation } from '@/lib/workspace-files/application/copy-operation' + +export const POST = defineInternalJsonRoute({ + contract: copyFileItemsContract, + auth: internalSessionAuth, + operation: fileCopyOperation, + rateLimit: internalRateLimits.user({ bucketName: 'files.copy' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: copyFileItems, + present: presentCopiedFileItems, +}) diff --git a/apps/sim/app/api/projects/[id]/files/[fileId]/export/route.ts b/apps/sim/app/api/projects/[id]/files/[fileId]/export/route.ts new file mode 100644 index 00000000000..e6f575dfc15 --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/[fileId]/export/route.ts @@ -0,0 +1,24 @@ +import { + exportProjectFileSnapshotContract, + MAX_PROJECT_FILE_SNAPSHOT_BODY_BYTES, +} from '@/lib/api/contracts/project-file-downloads' +import { + defineInternalBinaryRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { presentProjectFileDownload } from '@/lib/projects/files/api/download-presenter' +import { exportProjectFileSnapshot } from '@/lib/projects/files/application' + +export const POST = defineInternalBinaryRoute({ + contract: exportProjectFileSnapshotContract, + parseOptions: { maxBodyBytes: MAX_PROJECT_FILE_SNAPSHOT_BODY_BYTES }, + auth: internalSessionAuth, + operation: exportProjectFileSnapshot.operation, + rateLimit: internalRateLimits.none({ reason: 'Authenticated bounded Markdown snapshot export' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ params, body }) => ({ projectId: params.id, fileId: params.fileId, ...body }), + useCase: exportProjectFileSnapshot, + present: presentProjectFileDownload, +}) diff --git a/apps/sim/app/api/projects/[id]/files/[fileId]/extract/route.ts b/apps/sim/app/api/projects/[id]/files/[fileId]/extract/route.ts new file mode 100644 index 00000000000..41b5d123f4a --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/[fileId]/extract/route.ts @@ -0,0 +1,21 @@ +import { extractProjectFileContract } from '@/lib/api/contracts/project-file-extraction' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { extractProjectFile } from '@/lib/projects/files/application' +import { internalFileErrorPolicies } from '@/lib/workspace-files/api' + +export const maxDuration = 300 + +export const POST = defineInternalJsonRoute({ + contract: extractProjectFileContract, + auth: internalSessionAuth, + operation: extractProjectFile.operation, + rateLimit: internalRateLimits.user({ bucketName: 'project-files.write' }), + errorPolicy: internalFileErrorPolicies.extractArchive, + mapInput: ({ params }) => ({ projectId: params.id, fileId: params.fileId }), + useCase: extractProjectFile, + present: (result) => ({ success: true, ...result }), +}) diff --git a/apps/sim/app/api/projects/[id]/files/download/route.ts b/apps/sim/app/api/projects/[id]/files/download/route.ts new file mode 100644 index 00000000000..961374cebae --- /dev/null +++ b/apps/sim/app/api/projects/[id]/files/download/route.ts @@ -0,0 +1,21 @@ +import { downloadProjectFileItemsContract } from '@/lib/api/contracts/project-file-downloads' +import { + defineInternalBinaryRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { presentProjectFileDownload } from '@/lib/projects/files/api/download-presenter' +import { downloadProjectFileItems } from '@/lib/projects/files/application' + +export const GET = defineInternalBinaryRoute({ + contract: downloadProjectFileItemsContract, + auth: internalSessionAuth, + operation: downloadProjectFileItems.operation, + rateLimit: internalRateLimits.none({ reason: 'Authenticated bounded Project archive download' }), + errorPolicy: internalOrchestrationErrorPolicy, + headSafe: false, + mapInput: ({ params, query }) => ({ projectId: params.id, ...query }), + useCase: downloadProjectFileItems, + present: presentProjectFileDownload, +}) diff --git a/apps/sim/app/api/v2/files/bulk-download/route.ts b/apps/sim/app/api/v2/files/bulk-download/route.ts index 5587caa1a16..00d255c5d31 100644 --- a/apps/sim/app/api/v2/files/bulk-download/route.ts +++ b/apps/sim/app/api/v2/files/bulk-download/route.ts @@ -1,33 +1,13 @@ -import { Readable } from 'node:stream' -import { createLogger } from '@sim/logger' -import { ZipArchive } from 'archiver' import { v2BulkDownloadFilesContract } from '@/lib/api/contracts/v2/files' import { defineV2BinaryRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' -import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { downloadFileStream } from '@/lib/uploads/core/storage-service' -import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' -import { v2FileErrorPolicies } from '@/lib/workspace-files/api' +import { presentWorkspaceFileArchive, v2FileErrorPolicies } from '@/lib/workspace-files/api' import { downloadWorkspaceFileItems } from '@/lib/workspace-files/application/download-workspace-file-items' import { fileOperations } from '@/lib/workspace-files/application/operations' -const logger = createLogger('V2FilesBulkDownloadAPI') - export const dynamic = 'force-dynamic' export const revalidate = 0 /** Opens each object only as the archiver reaches it, so peak memory stays flat. */ -function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { - return Readable.from( - (async function* () { - yield* await downloadFileStream({ - key: file.key, - context: file.storageContext ?? 'workspace', - }) - })(), - { objectMode: false } - ) -} /** * GET /api/v2/files/bulk-download — stream a selection of files as one zip. @@ -54,36 +34,5 @@ export const GET = defineV2BinaryRoute({ folderPaths: query.folderPaths, }), useCase: downloadWorkspaceFileItems, - present: ({ filesToZip, folderPaths, renderedDocuments, excludedFileIds }) => { - const report = excludedFileIds.length > 0 ? { partial: true, excludedFileIds } : undefined - const entryPaths = buildZipEntryPaths([ - ...filesToZip.map((file) => ({ - name: file.name, - folderPath: file.folderId ? folderPaths.get(file.folderId) : null, - contentType: file.type, - })), - ...(report ? [{ name: 'sim-download-report.json', contentType: 'application/json' }] : []), - ]) - const archive = new ZipArchive({ store: true }) - archive.on('warning', (error: Error) => { - logger.warn('Archive warning while streaming workspace files', { error }) - }) - filesToZip.forEach((file, index) => { - archive.append(renderedDocuments.get(file.id) ?? lazyWorkspaceFileStream(file), { - name: entryPaths[index], - }) - }) - if (report) { - archive.append(JSON.stringify(report, null, 2), { name: entryPaths[filesToZip.length] }) - } - archive.finalize().catch((error) => { - logger.error('Failed to finalize workspace file archive', { error }) - }) - - return { - body: nodeReadableToWebStream(archive), - contentType: 'application/zip', - contentDisposition: 'attachment; filename="workspace-files.zip"', - } - }, + present: presentWorkspaceFileArchive, }) diff --git a/apps/sim/app/api/v2/files/copy/route.ts b/apps/sim/app/api/v2/files/copy/route.ts new file mode 100644 index 00000000000..3de70475a44 --- /dev/null +++ b/apps/sim/app/api/v2/files/copy/route.ts @@ -0,0 +1,21 @@ +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' +import { + defineV2JsonRoute, + v2ApiKeyAuth, + v2OrchestrationErrorPolicy, + v2RateLimits, +} from '@/lib/api/server/routes' +import { presentCopiedFileItems } from '@/lib/workspace-files/api/copy-presenter' +import { copyFileItems } from '@/lib/workspace-files/application/copy-file-items' +import { fileCopyOperations } from '@/lib/workspace-files/application/operations' + +export const POST = defineV2JsonRoute({ + contract: v2CopyFileItemsContract, + auth: v2ApiKeyAuth, + operation: fileCopyOperations.copy, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: copyFileItems, + present: (result) => ({ data: presentCopiedFileItems(result) }), +}) diff --git a/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/export/route.ts b/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/export/route.ts new file mode 100644 index 00000000000..282d07f3947 --- /dev/null +++ b/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/export/route.ts @@ -0,0 +1,23 @@ +import { MAX_PROJECT_FILE_SNAPSHOT_BODY_BYTES } from '@/lib/api/contracts/project-file-downloads' +import { v2ExportProjectFileSnapshotContract } from '@/lib/api/contracts/v2/project-file-downloads' +import { + defineV2BinaryRoute, + v2ApiKeyAuth, + v2OrchestrationErrorPolicy, + v2RateLimits, +} from '@/lib/api/server/routes' +import { presentProjectFileDownload } from '@/lib/projects/files/api/download-presenter' +import { exportProjectFileSnapshot } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' + +export const POST = defineV2BinaryRoute({ + contract: v2ExportProjectFileSnapshotContract, + parseOptions: { maxBodyBytes: MAX_PROJECT_FILE_SNAPSHOT_BODY_BYTES }, + auth: v2ApiKeyAuth, + operation: projectFileOperations.exportSnapshot, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + mapInput: ({ params, body }) => ({ ...params, content: body.content }), + useCase: exportProjectFileSnapshot, + present: presentProjectFileDownload, +}) diff --git a/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/unzip/route.ts b/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/unzip/route.ts new file mode 100644 index 00000000000..899587397a0 --- /dev/null +++ b/apps/sim/app/api/v2/projects/[projectId]/files/[fileId]/unzip/route.ts @@ -0,0 +1,26 @@ +import { v2UnzipProjectFileContract } from '@/lib/api/contracts/v2/project-file-extraction' +import { defineV2JsonRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' +import { buildFolderPath } from '@/lib/folders/paths' +import { extractProjectFile } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { v2FileErrorPolicies } from '@/lib/workspace-files/api' +import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' + +export const maxDuration = 300 + +export const POST = defineV2JsonRoute({ + contract: v2UnzipProjectFileContract, + auth: v2ApiKeyAuth, + operation: projectFileOperations.extractArchive, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2FileErrorPolicies.concealExtractionAuthorization, + mapInput: ({ params }) => ({ projectId: params.projectId, fileId: params.fileId }), + useCase: extractProjectFile, + present: (result) => ({ + data: { + folderPath: buildFolderPath(parseWorkspaceFileFolderDisplayPath(result.folderDisplayPath)), + extractedFileCount: result.extractedCount, + skippedFileCount: result.skippedCount, + }, + }), +}) diff --git a/apps/sim/app/api/v2/projects/[projectId]/files/bulk-download/route.ts b/apps/sim/app/api/v2/projects/[projectId]/files/bulk-download/route.ts new file mode 100644 index 00000000000..f50f95dfa3c --- /dev/null +++ b/apps/sim/app/api/v2/projects/[projectId]/files/bulk-download/route.ts @@ -0,0 +1,22 @@ +import { v2DownloadProjectFileItemsContract } from '@/lib/api/contracts/v2/project-file-downloads' +import { + defineV2BinaryRoute, + v2ApiKeyAuth, + v2OrchestrationErrorPolicy, + v2RateLimits, +} from '@/lib/api/server/routes' +import { presentProjectFileDownload } from '@/lib/projects/files/api/download-presenter' +import { downloadProjectFileItems } from '@/lib/projects/files/application' +import { projectFileOperations } from '@/lib/projects/files/application/operations' + +export const GET = defineV2BinaryRoute({ + contract: v2DownloadProjectFileItemsContract, + auth: v2ApiKeyAuth, + operation: projectFileOperations.downloadItems, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrchestrationErrorPolicy, + headSafe: false, + mapInput: ({ params, query }) => ({ projectId: params.projectId, ...query }), + useCase: downloadProjectFileItems, + present: presentProjectFileDownload, +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route-archive.test.ts b/apps/sim/app/api/workspaces/[id]/files/download/route-archive.test.ts index 77c794ffc6e..a7acf73517d 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route-archive.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route-archive.test.ts @@ -15,6 +15,7 @@ import { createMockRequest } from '@sim/testing' import { createRouteContext } from '@sim/testing/helpers/http' import { auditMock } from '@sim/testing/mocks/audit.mock' import { authMockFns } from '@sim/testing/mocks/auth.mock' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' @@ -29,6 +30,7 @@ const run = promisify(execFile) vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) vi.mock('@/lib/uploads/contexts/workspace', () => workspaceUploadsMock) vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) vi.mock('@sim/audit', () => auditMock) vi.mock('@/lib/posthog/server', () => posthogServerMock) diff --git a/apps/sim/app/api/workspaces/[id]/files/download/route.ts b/apps/sim/app/api/workspaces/[id]/files/download/route.ts index f1ccd1ce2bd..74892fca96e 100644 --- a/apps/sim/app/api/workspaces/[id]/files/download/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/download/route.ts @@ -1,36 +1,20 @@ -import { Readable } from 'node:stream' -import { createLogger } from '@sim/logger' -import { ZipArchive } from 'archiver' import { downloadWorkspaceFileItemsContract } from '@/lib/api/contracts/workspace-file-folders' import { defineInternalBinaryRoute, internalRateLimits, internalSessionAuth, } from '@/lib/api/server/routes' -import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { downloadFileStream } from '@/lib/uploads/core/storage-service' -import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' -import { internalFileAnalytics, internalFileErrorPolicies } from '@/lib/workspace-files/api' +import { + internalFileAnalytics, + internalFileErrorPolicies, + presentWorkspaceFileArchive, +} from '@/lib/workspace-files/api' import { downloadWorkspaceFileItems } from '@/lib/workspace-files/application/download-workspace-file-items' -const logger = createLogger('WorkspaceFilesDownloadAPI') - -function lazyWorkspaceFileStream(file: WorkspaceFileRecord): Readable { - return Readable.from( - (async function* () { - yield* await downloadFileStream({ - key: file.key, - context: file.storageContext ?? 'workspace', - }) - })(), - { objectMode: false } - ) -} - export const GET = defineInternalBinaryRoute({ contract: downloadWorkspaceFileItemsContract, auth: internalSessionAuth, + headSafe: false, operation: downloadWorkspaceFileItems.operation, rateLimit: internalRateLimits.none({ reason: 'Internal workspace zip download' }), errorPolicy: internalFileErrorPolicies.downloadArchive, @@ -41,32 +25,5 @@ export const GET = defineInternalBinaryRoute({ }), useCase: downloadWorkspaceFileItems, onSuccess: internalFileAnalytics.bulkDownloaded, - present: ({ filesToZip, folderPaths, renderedDocuments }) => { - const entryPaths = buildZipEntryPaths( - filesToZip.map((file) => ({ - name: file.name, - folderPath: file.folderId ? folderPaths.get(file.folderId) : null, - contentType: file.type, - })) - ) - const archive = new ZipArchive({ store: true }) - archive.on('warning', (error: Error) => { - logger.warn('Archive warning while streaming workspace files', { error }) - }) - filesToZip.forEach((file, index) => { - archive.append(renderedDocuments.get(file.id) ?? lazyWorkspaceFileStream(file), { - name: entryPaths[index], - }) - }) - archive.finalize().catch((error) => { - logger.error('Failed to finalize workspace file archive', { error }) - }) - - return { - body: nodeReadableToWebStream(archive), - contentType: 'application/zip', - contentDisposition: 'attachment; filename="workspace-files.zip"', - headers: { 'Cache-Control': 'no-store' }, - } - }, + present: presentWorkspaceFileArchive, }) diff --git a/apps/sim/lib/api/contracts/file-copy-input.ts b/apps/sim/lib/api/contracts/file-copy-input.ts new file mode 100644 index 00000000000..b032fafac43 --- /dev/null +++ b/apps/sim/lib/api/contracts/file-copy-input.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { MAX_WORKSPACE_FILE_BULK_REQUEST_IDS } from '@/lib/workspace-files/limits' + +const copyIdSchema = z.string().trim().min(1).max(200) +const copyOwnerSchema = z.strictObject({ + entityType: z.enum(['workspace', 'project']).describe('Resource scope that owns these files.'), + entityId: z.string().min(1).max(200).describe('Identifier of the owning workspace or Project.'), +}) +const copyIdsSchema = z + .array(copyIdSchema) + .max(MAX_WORKSPACE_FILE_BULK_REQUEST_IDS, 'Too many selected file items') + .refine((ids) => new Set(ids).size === ids.length, 'Selected identifiers must be unique') + .default([]) + +export const fileCopySourceSchema = z + .strictObject({ + owner: copyOwnerSchema.describe('Owner from which to read the selected files and folders.'), + fileIds: copyIdsSchema.describe('Identifiers of individual files to copy.'), + folderIds: copyIdsSchema.describe( + 'Identifiers of folders to copy with their active descendants and files.' + ), + }) + .refine((source) => source.fileIds.length + source.folderIds.length > 0, { + message: 'Select at least one file or folder to copy', + }) +export type FileCopySource = z.output + +export const fileCopyDestinationSchema = z.strictObject({ + owner: copyOwnerSchema.describe('Owner in which to create the copied files and folders.'), + folderId: copyIdSchema + .nullable() + .default(null) + .describe('Existing destination folder identifier. Omit or use null for the owner root.'), +}) +export type FileCopyDestination = z.output + +export const fileCopyInputSchema = z.strictObject({ + source: fileCopySourceSchema.describe('Selection to read under the source owner.'), + destination: fileCopyDestinationSchema.describe('Destination requiring file write access.'), +}) +export type FileCopyInput = z.output diff --git a/apps/sim/lib/api/contracts/file-copy.ts b/apps/sim/lib/api/contracts/file-copy.ts new file mode 100644 index 00000000000..3912a4fcf56 --- /dev/null +++ b/apps/sim/lib/api/contracts/file-copy.ts @@ -0,0 +1,85 @@ +import { z } from 'zod' +import { fileCopyDestinationSchema, fileCopyInputSchema } from '@/lib/api/contracts/file-copy-input' +import { noInputSchema, nonEmptyIdSchema } from '@/lib/api/contracts/primitives' +import { projectFileFolderRecordSchema } from '@/lib/api/contracts/project-file-folders' +import { projectFileRecordSchema } from '@/lib/api/contracts/project-files' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const copyFileItemsBodySchema = fileCopyInputSchema +export type CopyFileItemsBody = z.input + +export const copiedFileSchema = projectFileRecordSchema + .omit({ key: true, path: true, url: true, share: true }) + .extend({ + id: projectFileRecordSchema.shape.id.describe('Identifier of the new file.'), + name: projectFileRecordSchema.shape.name.describe( + 'Name of the copied file, including its extension.' + ), + size: projectFileRecordSchema.shape.size.describe('Current content size in bytes.'), + type: projectFileRecordSchema.shape.type.describe('MIME type of the copied file.'), + width: projectFileRecordSchema.shape.width.describe( + 'Known image width in pixels, when available.' + ), + height: projectFileRecordSchema.shape.height.describe( + 'Known image height in pixels, when available.' + ), + uploadedBy: projectFileRecordSchema.shape.uploadedBy.describe( + 'User who performed the copy, when available.' + ), + folderId: projectFileRecordSchema.shape.folderId.describe( + 'Containing folder identifier, or null at the owner root.' + ), + folderPath: projectFileRecordSchema.shape.folderPath.describe( + 'Containing folder path, or null at the owner root.' + ), + deletedAt: z.iso.datetime().nullable().describe('Archive time, or null for an active file.'), + uploadedAt: z.iso.datetime().describe('Time the new file was created.'), + updatedAt: z.iso.datetime().describe('Time the file metadata last changed.'), + contentUpdatedAt: z.iso + .datetime() + .nullable() + .describe('Time the current file content was written.'), + owner: fileCopyDestinationSchema.shape.owner.describe( + 'Canonical destination owner of the copied file.' + ), + revision: nonEmptyIdSchema.describe('Current content revision for subsequent edits.'), + }) +export type CopiedFile = z.output + +export const copiedFileFolderSchema = projectFileFolderRecordSchema.extend({ + id: projectFileFolderRecordSchema.shape.id.describe('Identifier of the new folder.'), + userId: projectFileFolderRecordSchema.shape.userId.describe( + 'User who performed the copy, when available.' + ), + name: projectFileFolderRecordSchema.shape.name.describe('Name of the copied folder.'), + parentId: projectFileFolderRecordSchema.shape.parentId.describe( + 'Parent folder identifier, or null at the owner root.' + ), + path: projectFileFolderRecordSchema.shape.path.describe( + 'Path of the copied folder within its owner.' + ), + sortOrder: projectFileFolderRecordSchema.shape.sortOrder.describe( + 'Display ordering value within the parent folder.' + ), + deletedAt: z.iso.datetime().nullable().describe('Archive time, or null for an active folder.'), + createdAt: z.iso.datetime().describe('Time the new folder was created.'), + updatedAt: z.iso.datetime().describe('Time the folder metadata last changed.'), + owner: fileCopyDestinationSchema.shape.owner.describe( + 'Canonical destination owner of the copied folder.' + ), +}) +export type CopiedFileFolder = z.output + +export const copyFileItemsResponseSchema = z.object({ + files: z.array(copiedFileSchema).describe('Created files with new identities.'), + folders: z.array(copiedFileFolderSchema).describe('Created folders in their new hierarchy.'), +}) +export type CopyFileItemsResponse = z.output + +export const copyFileItemsContract = defineRouteContract({ + method: 'POST', + path: '/api/files/copy', + query: noInputSchema, + body: copyFileItemsBodySchema, + response: { mode: 'json', schema: copyFileItemsResponseSchema, status: 201 }, +}) diff --git a/apps/sim/lib/api/contracts/project-file-downloads.ts b/apps/sim/lib/api/contracts/project-file-downloads.ts new file mode 100644 index 00000000000..4914f5ff464 --- /dev/null +++ b/apps/sim/lib/api/contracts/project-file-downloads.ts @@ -0,0 +1,34 @@ +import type { z } from 'zod' +import { + projectFileParamsSchema, + projectFilesParamsSchema, +} from '@/lib/api/contracts/project-files' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { downloadWorkspaceFileItemsQuerySchema } from '@/lib/api/contracts/workspace-file-folders' +import { exportWorkspaceFileSnapshotBodySchema } from '@/lib/api/contracts/workspace-files' + +export const downloadProjectFileItemsQuerySchema = downloadWorkspaceFileItemsQuerySchema.strict() +export type DownloadProjectFileItemsQuery = z.output +export const downloadProjectFileItemsContract = defineRouteContract({ + method: 'GET', + path: '/api/projects/[id]/files/download', + params: projectFilesParamsSchema, + query: downloadProjectFileItemsQuerySchema, + response: { mode: 'binary' }, +}) + +/** + * The proxy accepts at most 10 MiB of encoded JSON, independently of the decoded Markdown cap. + * Heavily escaped snapshots can hit this wire limit before their decoded-content limit. + */ +export const MAX_PROJECT_FILE_SNAPSHOT_BODY_BYTES = 10 * 1024 * 1024 + +export const exportProjectFileSnapshotBodySchema = exportWorkspaceFileSnapshotBodySchema.strict() +export type ExportProjectFileSnapshotBody = z.input +export const exportProjectFileSnapshotContract = defineRouteContract({ + method: 'POST', + path: '/api/projects/[id]/files/[fileId]/export', + params: projectFileParamsSchema, + body: exportProjectFileSnapshotBodySchema, + response: { mode: 'binary' }, +}) diff --git a/apps/sim/lib/api/contracts/project-file-extraction.ts b/apps/sim/lib/api/contracts/project-file-extraction.ts new file mode 100644 index 00000000000..456f40ff8db --- /dev/null +++ b/apps/sim/lib/api/contracts/project-file-extraction.ts @@ -0,0 +1,18 @@ +import { z } from 'zod' +import { nonEmptyIdSchema } from '@/lib/api/contracts/primitives' +import { projectFileParamsSchema } from '@/lib/api/contracts/project-files' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { extractWorkspaceFileResponseSchema } from '@/lib/api/contracts/workspace-files' + +export const extractProjectFileResponseSchema = extractWorkspaceFileResponseSchema.extend({ + folderId: nonEmptyIdSchema, + folderDisplayPath: z.string(), +}) +export type ExtractProjectFileResponse = z.output + +export const extractProjectFileContract = defineRouteContract({ + method: 'POST', + path: '/api/projects/[id]/files/[fileId]/extract', + params: projectFileParamsSchema, + response: { mode: 'json', schema: extractProjectFileResponseSchema }, +}) diff --git a/apps/sim/lib/api/contracts/v2/file-copy.ts b/apps/sim/lib/api/contracts/v2/file-copy.ts new file mode 100644 index 00000000000..65adf70f339 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/file-copy.ts @@ -0,0 +1,12 @@ +import { copyFileItemsBodySchema, copyFileItemsResponseSchema } from '@/lib/api/contracts/file-copy' +import { noInputSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { v2DataResponse } from '@/lib/api/contracts/v2/shared' + +export const v2CopyFileItemsContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/files/copy', + query: noInputSchema, + body: copyFileItemsBodySchema, + response: { mode: 'json', schema: v2DataResponse(copyFileItemsResponseSchema), status: 201 }, +}) diff --git a/apps/sim/lib/api/contracts/v2/files.ts b/apps/sim/lib/api/contracts/v2/files.ts index 323fcaf36f7..113bad59775 100644 --- a/apps/sim/lib/api/contracts/v2/files.ts +++ b/apps/sim/lib/api/contracts/v2/files.ts @@ -1070,7 +1070,7 @@ export const v2MoveFileItemsContract = defineRouteContract({ * Comma-separated only: v2 rejects a query parameter sent more than once, so a * repeated-parameter form would never reach this schema. */ -function v2QuerySelectionListSchema(field: string) { +export function v2QuerySelectionListSchema(field: string) { return z .string() .optional() diff --git a/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts b/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts index fe19ef283db..792a3b3ef8f 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/files-audit.ts @@ -1,4 +1,5 @@ import { v2GetAuditLogContract, v2ListAuditLogsContract } from '@/lib/api/contracts/v2/audit-logs' +import { v2CopyFileItemsContract } from '@/lib/api/contracts/v2/file-copy' import { v2DeleteFileVersionContract, v2DownloadFileVersionContract, @@ -63,6 +64,7 @@ import { type OpenApiSuccessMetadata, } from '@/lib/api/openapi/types' import { auditLogOperations } from '@/lib/audit-logs/application/operations' +import { fileCopyOperation } from '@/lib/workspace-files/application/copy-operation' import { fileOperations } from '@/lib/workspace-files/application/operations' import { MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' @@ -153,6 +155,35 @@ function auditOperation( } } +export const unpublishedFileCopyOpenApiRoutes = [ + defineOpenApiRoute( + v2CopyFileItemsContract, + filesOperation({ + applicationOperation: fileCopyOperation, + operationId: 'copyFileItems', + summary: 'Copy File Items', + description: `Copy selected files and folder trees between workspace or Project owners. Source read and destination write access are checked independently. Copies receive new identities and retain source secret provenance. Registration is atomic, with destination names resolved by the existing copy rules. ${WORKSPACE_API_KEY_DENIED}`, + errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge', 'UnsupportedMediaType'], + success: { description: 'New file and folder identities with their destination owner.' }, + }), + { + query: v2CopyFileItemsContract.query, + body: documentedSchema( + v2CopyFileItemsContract.body, + 'CopyFileItemsRequest', + 'Copy file items request', + 'Exact source owner and selected identifiers, plus the destination owner and folder.' + ), + response: documentedSchema( + v2CopyFileItemsContract.response.schema, + 'V2CopyFileItemsResponse', + 'Copy file items response', + 'Created files and folders, without private storage keys.' + ), + } + ), +] + const declaredRoutes = [ defineOpenApiRoute( v2ListFilesContract, diff --git a/apps/sim/lib/api/contracts/v2/openapi/project-file-downloads.ts b/apps/sim/lib/api/contracts/v2/openapi/project-file-downloads.ts new file mode 100644 index 00000000000..187d91c73b9 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/openapi/project-file-downloads.ts @@ -0,0 +1,78 @@ +import { + documentedSchema, + HEAD_MIRRORS_GET, + HEAD_OMITS_PAYLOAD_HEADERS, + RESOURCE_CONFLICT_ERRORS, + WORKSPACE_API_KEY_DENIED, +} from '@/lib/api/contracts/v2/openapi/shared' +import { + v2DownloadProjectFileItemsContract, + v2ExportProjectFileSnapshotContract, +} from '@/lib/api/contracts/v2/project-file-downloads' +import { defineOpenApiRoute } from '@/lib/api/openapi/types' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' + +export const projectFileDownloadOpenApiRoutes = [ + defineOpenApiRoute( + v2DownloadProjectFileItemsContract, + { + applicationOperation: projectFileOperations.downloadItems, + operationId: 'downloadProjectFileItems', + summary: 'Download Project File Items', + description: `Download selected files and recursive folder contents as one ZIP archive. Duplicate selections are included once. Select at most ${MAX_ZIP_DOWNLOAD_FILES} files in total; archive bytes are bounded. Unknown or archived selections are rejected. Downloads record an audit event. ${WORKSPACE_API_KEY_DENIED} ${HEAD_MIRRORS_GET} ${HEAD_OMITS_PAYLOAD_HEADERS}`, + tags: ['Files'], + errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge'], + success: { + description: 'Selected files in their Project folder paths.', + headers: ['Content-Type', 'Content-Disposition', 'Content-Length'], + contentTypes: ['application/zip'], + }, + }, + { + params: documentedSchema( + v2DownloadProjectFileItemsContract.params, + 'ProjectFilesParams', + 'Project file collection', + 'The Project that owns the requested files.' + ), + query: documentedSchema( + v2DownloadProjectFileItemsContract.query, + 'DownloadProjectFileItemsQuery', + 'Project archive selection', + 'File and folder identifiers within the Project.' + ), + } + ), + defineOpenApiRoute( + v2ExportProjectFileSnapshotContract, + { + applicationOperation: projectFileOperations.exportSnapshot, + operationId: 'exportProjectFileSnapshot', + summary: 'Export Project File Snapshot', + description: `Export the supplied visible Markdown snapshot without changing the stored file or its history. Readable embedded Project assets are bundled in a ZIP; a snapshot without bundled assets is returned as Markdown. Missing or unreadable assets remain as references. Total bytes are bounded. ${WORKSPACE_API_KEY_DENIED}`, + tags: ['Files'], + errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge'], + success: { + description: 'Markdown snapshot or its archive with embedded assets.', + headers: ['Content-Type', 'Content-Disposition', 'Content-Length'], + contentTypes: ['text/markdown', 'application/zip'], + }, + }, + { + params: documentedSchema( + v2ExportProjectFileSnapshotContract.params, + 'ProjectFileParams', + 'Project file identity', + 'The owning Project and the file identifier.' + ), + query: v2ExportProjectFileSnapshotContract.query, + body: documentedSchema( + v2ExportProjectFileSnapshotContract.body, + 'ExportProjectFileSnapshotRequest', + 'Project Markdown snapshot', + 'The visible document content to export.' + ), + } + ), +] as const diff --git a/apps/sim/lib/api/contracts/v2/openapi/project-files.ts b/apps/sim/lib/api/contracts/v2/openapi/project-files.ts index 7f06eb3de45..3588768689b 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/project-files.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/project-files.ts @@ -5,6 +5,7 @@ import { RESOURCE_ERRORS, WORKSPACE_API_KEY_DENIED, } from '@/lib/api/contracts/v2/openapi/shared' +import { v2UnzipProjectFileContract } from '@/lib/api/contracts/v2/project-file-extraction' import { v2ArchiveProjectFileItemsContract, v2MoveProjectFileItemsContract, @@ -23,6 +24,36 @@ import { defineOpenApiRoute } from '@/lib/api/openapi/types' import { projectFileOperations } from '@/lib/projects/files/application/operations' export const projectFileOpenApiRoutes = [ + defineOpenApiRoute( + v2UnzipProjectFileContract, + { + applicationOperation: projectFileOperations.extractArchive, + operationId: 'unzipProjectFile', + summary: 'Unzip Project File', + description: `Extract a ZIP archive into a new sibling folder in the same Project. Use List Project Files to inspect its contents. Concurrent extraction of the same archive returns \`409\`; size or processing-time limits return \`413\`. ${WORKSPACE_API_KEY_DENIED}`, + tags: ['Files'], + errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge'], + success: { + description: 'Counts and destination folder for the unpacked archive.', + headers: RATE_LIMIT_HEADERS, + }, + }, + { + params: documentedSchema( + v2UnzipProjectFileContract.params, + 'ProjectFileParams', + 'Project file identity', + 'The owning Project and the file identifier.' + ), + query: v2UnzipProjectFileContract.query, + response: documentedSchema( + v2UnzipProjectFileContract.response.schema, + 'V2ProjectFileUnzipResponse', + 'Unzip Project file response', + 'Counts and destination folder for the unpacked archive.' + ), + } + ), defineOpenApiRoute( v2SearchProjectFileContentContract, { diff --git a/apps/sim/lib/api/contracts/v2/openapi/unpublished-project-files.ts b/apps/sim/lib/api/contracts/v2/openapi/unpublished-project-files.ts index b4f2c8acd91..a454f1ebc08 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/unpublished-project-files.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/unpublished-project-files.ts @@ -1,3 +1,5 @@ +import { unpublishedFileCopyOpenApiRoutes } from '@/lib/api/contracts/v2/openapi/files-audit' +import { projectFileDownloadOpenApiRoutes } from '@/lib/api/contracts/v2/openapi/project-file-downloads' import { projectFileFolderOpenApiRoutes } from '@/lib/api/contracts/v2/openapi/project-file-folders' import { projectFileUploadOpenApiRoutes } from '@/lib/api/contracts/v2/openapi/project-file-uploads' import { projectFileVersionOpenApiRoutes } from '@/lib/api/contracts/v2/openapi/project-file-versions' @@ -8,6 +10,8 @@ export const unpublishedProjectFileDocumentation = { reason: 'Project files are unreleased. Publish these routes and the Projects CLI documentation together at launch.', routes: [ + ...unpublishedFileCopyOpenApiRoutes, + ...projectFileDownloadOpenApiRoutes, ...projectFileOpenApiRoutes, ...projectFileVersionOpenApiRoutes, ...projectFileFolderOpenApiRoutes, diff --git a/apps/sim/lib/api/contracts/v2/project-file-downloads.ts b/apps/sim/lib/api/contracts/v2/project-file-downloads.ts new file mode 100644 index 00000000000..57033617102 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/project-file-downloads.ts @@ -0,0 +1,46 @@ +import { z } from 'zod' +import { noInputSchema } from '@/lib/api/contracts/primitives' +import { exportProjectFileSnapshotBodySchema } from '@/lib/api/contracts/project-file-downloads' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { v2QuerySelectionListSchema } from '@/lib/api/contracts/v2/files' +import { + v2ProjectFileParamsSchema, + v2ProjectFilesParamsSchema, +} from '@/lib/api/contracts/v2/project-files' +import { MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' + +export const v2DownloadProjectFileItemsQuerySchema = z + .object({ + fileIds: v2QuerySelectionListSchema('fileIds').describe( + `File identifiers to include, comma-separated. At most ${MAX_ZIP_DOWNLOAD_FILES} entries.` + ), + folderIds: v2QuerySelectionListSchema('folderIds').describe( + `Folder identifiers to include recursively, comma-separated. The resolved selection allows at most ${MAX_ZIP_DOWNLOAD_FILES} files.` + ), + }) + .strict() +export type V2DownloadProjectFileItemsQuery = z.output + +export const v2DownloadProjectFileItemsContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/projects/[projectId]/files/bulk-download', + params: v2ProjectFilesParamsSchema, + query: v2DownloadProjectFileItemsQuerySchema, + response: { mode: 'binary' }, +}) + +export const v2ExportProjectFileSnapshotBodySchema = exportProjectFileSnapshotBodySchema.extend({ + content: exportProjectFileSnapshotBodySchema.shape.content.describe( + 'Visible Markdown snapshot to export. This does not replace the stored file or create a version.' + ), +}) +export type V2ExportProjectFileSnapshotBody = z.input + +export const v2ExportProjectFileSnapshotContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/projects/[projectId]/files/[fileId]/export', + params: v2ProjectFileParamsSchema, + query: noInputSchema, + body: v2ExportProjectFileSnapshotBodySchema, + response: { mode: 'binary' }, +}) diff --git a/apps/sim/lib/api/contracts/v2/project-file-extraction.ts b/apps/sim/lib/api/contracts/v2/project-file-extraction.ts new file mode 100644 index 00000000000..afae040004e --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/project-file-extraction.ts @@ -0,0 +1,21 @@ +import type { z } from 'zod' +import { noInputSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { v2UnzipFileDataSchema } from '@/lib/api/contracts/v2/files' +import { v2ProjectFileParamsSchema } from '@/lib/api/contracts/v2/project-files' +import { v2DataResponse } from '@/lib/api/contracts/v2/shared' + +export const v2UnzipProjectFileDataSchema = v2UnzipFileDataSchema.meta({ + id: 'V2ProjectFileUnzipResult', + title: 'Project unzip result', + description: 'Outcome of unzipping a Project archive into a sibling folder.', +}) +export type V2ProjectFileUnzipResult = z.output + +export const v2UnzipProjectFileContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/projects/[projectId]/files/[fileId]/unzip', + params: v2ProjectFileParamsSchema, + query: noInputSchema, + response: { mode: 'json', schema: v2DataResponse(v2UnzipProjectFileDataSchema) }, +}) diff --git a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts index eb42013361e..07fcf763782 100644 --- a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts +++ b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts @@ -138,6 +138,10 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ pattern: '/api/v2/files/bulk-download', load: () => import('@/app/api/v2/files/bulk-download/route'), }, + { + pattern: '/api/v2/files/copy', + load: () => import('@/app/api/v2/files/copy/route'), + }, { pattern: '/api/v2/files/folders', load: () => import('@/app/api/v2/files/folders/route'), @@ -474,6 +478,10 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ pattern: '/api/v2/projects/{projectId}/files/{fileId}/content', load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/content/route'), }, + { + pattern: '/api/v2/projects/{projectId}/files/{fileId}/export', + load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/export/route'), + }, { pattern: '/api/v2/projects/{projectId}/files/{fileId}/metadata', load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/metadata/route'), @@ -482,6 +490,10 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ pattern: '/api/v2/projects/{projectId}/files/{fileId}/restore', load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/restore/route'), }, + { + pattern: '/api/v2/projects/{projectId}/files/{fileId}/unzip', + load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/unzip/route'), + }, { pattern: '/api/v2/projects/{projectId}/files/{fileId}/versions', load: () => import('@/app/api/v2/projects/[projectId]/files/[fileId]/versions/route'), @@ -504,6 +516,10 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ pattern: '/api/v2/projects/{projectId}/files/archive', load: () => import('@/app/api/v2/projects/[projectId]/files/archive/route'), }, + { + pattern: '/api/v2/projects/{projectId}/files/bulk-download', + load: () => import('@/app/api/v2/projects/[projectId]/files/bulk-download/route'), + }, { pattern: '/api/v2/projects/{projectId}/files/folders', load: () => import('@/app/api/v2/projects/[projectId]/files/folders/route'), diff --git a/apps/sim/lib/core/application/resource-delegation.ts b/apps/sim/lib/core/application/resource-delegation.ts index 693aecb52c8..4956229fda6 100644 --- a/apps/sim/lib/core/application/resource-delegation.ts +++ b/apps/sim/lib/core/application/resource-delegation.ts @@ -38,7 +38,7 @@ function isSelectionIds(value: unknown): value is string[] { } /** Validates the paired scope shared by delegation admission and the copy application boundary. */ -function isResourceFileCopyScope(scope: unknown): scope is ResourceFileCopyScope { +export function isResourceFileCopyScope(scope: unknown): scope is ResourceFileCopyScope { if (!isRecordLike(scope) || scope.kind !== 'file_copy') return false const { source, destination } = scope return ( diff --git a/apps/sim/lib/projects/files/__integration__/authorization-bulk.integration.ts b/apps/sim/lib/projects/files/__integration__/authorization-bulk.integration.ts new file mode 100644 index 00000000000..a9c4e7c77e1 --- /dev/null +++ b/apps/sim/lib/projects/files/__integration__/authorization-bulk.integration.ts @@ -0,0 +1,618 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import type { + OAuthAccessTokenPrincipal, + ResourceDelegatedPrincipal, + ResourceFileCopyScope, +} from '@sim/auth/principal' +import { db } from '@sim/db' +import { + copilotChats, + folder, + member, + organization, + permissionGroup, + permissionGroupWorkspace, + permissions, + project, + user, + workspace, + workspaceFiles, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { + createExecutorPrincipal, + createPersonalApiKeyPrincipal, + createSessionPrincipal, + createWorkspaceApiKeyPrincipal, +} from '@sim/testing/factories/principal.factory' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { acquirePermissionGroupOrgLock } from '@/lib/permission-groups/locks' +import { createProjectFileFolder, listProjectFileFolders } from '@/lib/projects/files/application' +import { createFileCopyAuthorizer } from '@/lib/workspace-files/application/copy-authorization' +import { copyFileItems } from '@/lib/workspace-files/application/copy-file-items' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +const fixtures: { + users: string[] + organizationId: string + projectId: string + workspaces: string[] +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const ownerId = generateId() + const readerId = generateId() + const organizationId = generateId() + const projectId = generateId() + const workspaces = [generateId(), generateId()] + fixtures.push({ users: [ownerId, readerId], organizationId, projectId, workspaces }) + await db.insert(user).values( + [ownerId, readerId].map((id) => ({ + id, + name: 'Project file fixture', + email: `${id}@files.invalid`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db.insert(organization).values({ + id: organizationId, + name: 'Project files', + slug: organizationId, + createdAt: new Date(), + }) + await db.insert(member).values([ + { id: generateId(), organizationId, userId: ownerId, role: 'owner', createdAt: new Date() }, + { id: generateId(), organizationId, userId: readerId, role: 'member', createdAt: new Date() }, + ]) + await db.transaction(async (tx) => { + await tx + .insert(project) + .values({ id: projectId, organizationId, ownerId, name: 'Project files' }) + await tx.insert(workspace).values( + workspaces.map((id, index) => ({ + id, + organizationId, + ownerId, + billedAccountUserId: ownerId, + workspaceMode: 'organization' as const, + name: 'Environment', + projectId, + forkedFromWorkspaceId: index ? workspaces[0] : null, + })) + ) + }) + await db.insert(permissions).values({ + id: generateId(), + entityType: 'workspace', + entityId: workspaces[0], + userId: readerId, + permissionType: 'read', + }) + return { + ownerId, + readerId, + organizationId, + projectId, + workspaces, + reader: createSessionPrincipal({ userId: readerId }), + } +} + +async function grant( + userId: string, + workspaceId: string, + permissionType: 'read' | 'write' | 'admin' +) { + await db + .delete(permissions) + .where( + and( + eq(permissions.userId, userId), + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, workspaceId) + ) + ) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType, + }) +} + +function delegated( + userId: string, + projectId: string, + workspaceId: string +): Extract { + return { + kind: 'resource_delegated', + serviceId: 'copilot', + subjectUserId: userId, + audience: 'sim:project-files', + delegationId: generateId(), + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 30_000), + scope: { kind: 'entity', entityType: 'project', entityId: projectId }, + invocation: { kind: 'workspace', workspaceId }, + } +} + +afterAll(async () => { + const reportPath = + process.env.PROJECT_FILE_AUTHORIZATION_REPORT_PATH ?? + resolve('test-results/project-file-authorization.json') + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) + for (const f of fixtures) { + const memberships = await db + .select({ projectId: workspace.projectId }) + .from(workspace) + .where(inArray(workspace.id, f.workspaces)) + const projectIds = [f.projectId, ...memberships.map((row) => row.projectId)] + await db.delete(workspaceFiles).where(inArray(workspaceFiles.projectId, projectIds)) + await db.delete(folder).where(inArray(folder.projectId, projectIds)) + await deleteWorkspaceFixture(db, inArray(workspace.id, f.workspaces)) + await db.delete(project).where(inArray(project.id, projectIds)) + await db.delete(organization).where(eq(organization.id, f.organizationId)) + await db.delete(user).where(inArray(user.id, f.users)) + } +}) + +describe('compound copy owner authority', () => { + function selection(f: Awaited>): Omit { + return { + source: { + owner: { entityType: 'project', entityId: f.projectId }, + fileIds: [generateId()], + folderIds: [], + }, + destination: { + owner: { entityType: 'workspace', entityId: f.workspaces[1] }, + folderId: null, + }, + } + } + + function copyPrincipal( + f: Awaited>, + input: Omit + ) { + return { + ...delegated(f.readerId, f.projectId, f.workspaces[0]), + audience: 'sim:files:copy', + scope: { kind: 'file_copy' as const, ...input }, + } + } + + check( + 'copy checks source read and destination write independently and rechecks after preparation', + async () => { + const f = await fixture() + await grant(f.readerId, f.workspaces[1], 'write') + const input = selection(f) + const authorize = await createFileCopyAuthorizer(f.reader, input) + expect(await db.transaction(authorize)).toMatchObject({ + source: { owner: input.source.owner, canWrite: false, ownerUserId: f.ownerId }, + destination: { owner: input.destination.owner, billedAccountUserId: f.ownerId }, + }) + const reverse = { + source: { ...input.source, owner: input.destination.owner }, + destination: { ...input.destination, owner: input.source.owner }, + } + const reverseAuthorize = await createFileCopyAuthorizer(f.reader, reverse) + await expect(db.transaction(reverseAuthorize)).rejects.toMatchObject({ code: 'forbidden' }) + await grant(f.readerId, f.workspaces[1], 'read') + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'forbidden' }) + } + ) + + check( + 'copy refuses widened paired grants, entity grants, workspace keys and executors', + async () => { + const f = await fixture() + await grant(f.readerId, f.workspaces[1], 'write') + const input = selection(f) + const principal = copyPrincipal(f, input) + expect(await db.transaction(await createFileCopyAuthorizer(principal, input))).toMatchObject({ + source: { owner: input.source.owner }, + destination: { owner: input.destination.owner }, + }) + for (const changed of [ + { ...input, source: { ...input.source, fileIds: [generateId()] } }, + { ...input, destination: { ...input.destination, folderId: generateId() } }, + { ...input, source: { ...input.source, owner: input.destination.owner } }, + ]) { + await expect(createFileCopyAuthorizer(principal, changed)).rejects.toMatchObject({ + code: 'forbidden', + }) + } + const realtime: ResourceDelegatedPrincipal = { + kind: 'resource_delegated', + serviceId: 'realtime', + subjectUserId: f.readerId, + audience: 'sim:files:copy', + delegationId: generateId(), + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 30_000), + scope: { + kind: 'entity', + entityType: 'project', + entityId: f.projectId, + fileId: input.source.fileIds[0], + }, + invocation: { kind: 'realtime', connectionId: 'copy-fixture' }, + } + for (const rejected of [ + realtime, + delegated(f.readerId, f.projectId, f.workspaces[0]), + createWorkspaceApiKeyPrincipal({ workspaceId: f.workspaces[1] }), + createExecutorPrincipal({ subjectUserId: f.readerId, workspaceId: f.workspaces[1] }), + ]) { + await expect(createFileCopyAuthorizer(rejected, input)).rejects.toMatchObject({ + code: 'forbidden', + }) + } + } + ) + + check('copy cannot escape the origin organization despite access to both owners', async () => { + const f = await fixture() + const other = await fixture() + for (const workspaceId of other.workspaces) await grant(f.ownerId, workspaceId, 'admin') + const input = { + ...selection(f), + destination: { + owner: { entityType: 'project' as const, entityId: other.projectId }, + folderId: null, + }, + } + const principal = { ...copyPrincipal(f, input), subjectUserId: f.ownerId } + const authorize = await createFileCopyAuthorizer(principal, input) + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'not_found' }) + expect( + await db.transaction( + await createFileCopyAuthorizer(createSessionPrincipal({ userId: f.ownerId }), input) + ) + ).toMatchObject({ destination: { owner: input.destination.owner } }) + }) + + check('copy rechecks organization chat actor and mode at commit', async () => { + const f = await fixture() + await grant(f.readerId, f.workspaces[1], 'write') + const input = selection(f) + const chatId = generateId() + await db.insert(copilotChats).values({ + id: chatId, + userId: f.readerId, + organizationId: f.organizationId, + workspaceId: null, + type: 'mothership', + config: { conversationMode: 'agent' }, + title: 'Copy files', + }) + const principal = { ...copyPrincipal(f, input), invocation: { kind: 'chat' as const, chatId } } + const authorize = await createFileCopyAuthorizer(principal, input) + expect(await db.transaction(authorize)).toMatchObject({ source: { owner: input.source.owner } }) + await db + .update(copilotChats) + .set({ config: { conversationMode: 'ask' } }) + .where(eq(copilotChats.id, chatId)) + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'not_found' }) + await db + .update(copilotChats) + .set({ config: { conversationMode: 'agent' }, userId: f.ownerId }) + .where(eq(copilotChats.id, chatId)) + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'not_found' }) + }) + + check('copy applies personal-key availability and OAuth write scope to both owners', async () => { + const f = await fixture() + await grant(f.readerId, f.workspaces[1], 'write') + await db + .update(workspace) + .set({ allowPersonalApiKeys: true }) + .where(inArray(workspace.id, f.workspaces)) + const input = selection(f) + const key = createPersonalApiKeyPrincipal({ userId: f.readerId }) + expect(await db.transaction(await createFileCopyAuthorizer(key, input))).toMatchObject({ + destination: { owner: input.destination.owner }, + }) + await db + .update(workspace) + .set({ allowPersonalApiKeys: false }) + .where(eq(workspace.id, f.workspaces[0])) + await expect(db.transaction(await createFileCopyAuthorizer(key, input))).rejects.toMatchObject({ + code: 'forbidden', + }) + const oauth: OAuthAccessTokenPrincipal = { + kind: 'oauth_access_token', + userId: f.readerId, + tokenId: generateId(), + clientId: 'test-client', + scopes: ['api:read'], + expiresAt: new Date(Date.now() + 60_000), + } + await expect(createFileCopyAuthorizer(oauth, input)).rejects.toMatchObject({ + code: 'forbidden', + }) + }) + + check('copy blocks a suspended actor after preparation for workspace-only owners', async () => { + const f = await fixture() + await grant(f.readerId, f.workspaces[1], 'write') + const original = selection(f) + const input = { + ...original, + source: { + ...original.source, + owner: { entityType: 'workspace' as const, entityId: f.workspaces[0] }, + }, + } + const authorize = await createFileCopyAuthorizer(f.reader, input) + expect(await db.transaction(authorize)).toMatchObject({ source: { owner: input.source.owner } }) + await db.update(user).set({ banned: true }).where(eq(user.id, f.readerId)) + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'forbidden' }) + }) + + check( + 'copy locks opposite Project directions consistently in overlapping transactions', + async () => { + const f = await fixture() + const other = await fixture() + for (const workspaceId of other.workspaces) await grant(f.ownerId, workspaceId, 'admin') + const principal = createSessionPrincipal({ userId: f.ownerId }) + const input = { + ...selection(f), + destination: { + owner: { entityType: 'project' as const, entityId: other.projectId }, + folderId: null, + }, + } + const reverse = { + source: { ...input.source, owner: input.destination.owner }, + destination: { ...input.destination, owner: input.source.owner }, + } + const forward = await createFileCopyAuthorizer(principal, input) + const backward = await createFileCopyAuthorizer(principal, reverse) + const results = await Promise.all([db.transaction(forward), db.transaction(backward)]) + expect(results.map((result) => result.destination.owner.entityId)).toEqual([ + other.projectId, + f.projectId, + ]) + } + ) + + check( + 'copy does not upgrade shared membership locks across two Projects in one organization', + async () => { + const f = await fixture() + const extraWorkspaceId = generateId() + f.workspaces.push(extraWorkspaceId) + await insertWorkspaceFixture(db, { + id: extraWorkspaceId, + name: 'Separate Project', + organizationId: f.organizationId, + ownerId: f.ownerId, + billedAccountUserId: f.ownerId, + workspaceMode: 'organization', + }) + const [extra] = await db.select().from(workspace).where(eq(workspace.id, extraWorkspaceId)) + const first = selection(f) + const second = { + source: { + ...first.source, + owner: { entityType: 'project' as const, entityId: extra.projectId }, + }, + destination: { + owner: { entityType: 'workspace' as const, entityId: extraWorkspaceId }, + folderId: null, + }, + } + const principal = createSessionPrincipal({ userId: f.ownerId }) + const authorizeFirst = await createFileCopyAuthorizer(principal, first) + const authorizeSecond = await createFileCopyAuthorizer(principal, second) + const ready = createDeferred() + const release = createDeferred() + const organizationEdit = db.transaction(async (tx) => { + await acquirePermissionGroupOrgLock(tx, f.organizationId) + const [connection] = await tx.execute<{ pid: number }>(sql`SELECT pg_backend_pid() AS pid`) + ready.resolve(connection.pid) + await release.promise + }) + const pid = await Promise.race([ + ready.promise, + organizationEdit.then(() => { + throw new Error('Organization edit exited early') + }), + ]) + const outcomes = Promise.allSettled([ + db.transaction(authorizeFirst), + db.transaction(authorizeSecond), + ]) + try { + let blocked = 0 + for (let attempt = 0; attempt < 100; attempt++) { + const [state] = await db.execute<{ count: number }>(sql` + SELECT count(*)::int AS count FROM pg_stat_activity + WHERE ${pid} = ANY(pg_blocking_pids(pid)) AND wait_event = 'advisory' + `) + blocked = state.count + if (blocked === 2) break + await sleep(10) + } + expect(blocked).toBe(2) + } finally { + release.resolve() + await organizationEdit + await outcomes + } + const results = await outcomes + expect(results.map((result) => result.status)).toEqual(['fulfilled', 'fulfilled']) + } + ) +}) + +describe('Mothership origin navigation and independent Project authority', () => { + async function target(f: Awaited>) { + const workspaceId = generateId() + f.workspaces.push(workspaceId) + await insertWorkspaceFixture(db, { + id: workspaceId, + name: 'Independent target', + organizationId: f.organizationId, + ownerId: f.ownerId, + billedAccountUserId: f.ownerId, + workspaceMode: 'organization', + }) + const [binding] = await db.select().from(workspace).where(eq(workspace.id, workspaceId)) + if (!binding) throw new Error('Target Project missing') + await grant(f.readerId, workspaceId, 'admin') + return { workspaceId, projectId: binding.projectId } + } + + check( + 'origin A reads and writes B and authorizes A to B copy with independent live grants', + async () => { + const f = await fixture() + const b = await target(f) + const principal = delegated(f.readerId, b.projectId, f.workspaces[0]) + const result = await createProjectFileFolder.execute({ + principal, + input: { projectId: b.projectId, name: 'Target folder' }, + }) + expect( + ( + await listProjectFileFolders.execute({ principal, input: { projectId: b.projectId } }) + ).folders.map((row) => row.id) + ).toContain(result.folder.id) + const { folder: source } = await createProjectFileFolder.execute({ + principal: createSessionPrincipal({ userId: f.ownerId }), + input: { projectId: f.projectId, name: 'Copy me' }, + }) + const input = { + source: { + owner: { entityType: 'project' as const, entityId: f.projectId }, + fileIds: [], + folderIds: [source.id], + }, + destination: { + owner: { entityType: 'project' as const, entityId: b.projectId }, + folderId: null, + }, + } + const copy = { + ...principal, + audience: 'sim:files:copy', + scope: { kind: 'file_copy' as const, ...input }, + } + const copied = await copyFileItems.execute({ principal: copy, input }) + expect(copied.folders).toHaveLength(1) + expect( + ( + await listProjectFileFolders.execute({ principal, input: { projectId: b.projectId } }) + ).folders.map((row) => row.id) + ).toContain(copied.folders[0].id) + const authorize = await createFileCopyAuthorizer(copy, input) + expect(await db.transaction(authorize)).toMatchObject({ + source: { canWrite: false }, + destination: { canWrite: true }, + }) + await grant(f.readerId, b.workspaceId, 'read') + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'forbidden' }) + await db + .delete(permissions) + .where(and(eq(permissions.userId, f.readerId), eq(permissions.entityId, b.workspaceId))) + await expect( + listProjectFileFolders.execute({ principal, input: { projectId: b.projectId } }) + ).rejects.toMatchObject({ code: 'not_found' }) + } + ) + + check( + 'origin access and copilot capability are rechecked after preparation independently of B', + async () => { + for (const revoke of ['permission', 'copilot', 'files'] as const) { + const f = await fixture() + const b = await target(f) + const input = { + source: { + owner: { entityType: 'project' as const, entityId: b.projectId }, + fileIds: [generateId()], + folderIds: [], + }, + destination: { + owner: { entityType: 'project' as const, entityId: b.projectId }, + folderId: null, + }, + } + const principal = { + ...delegated(f.readerId, b.projectId, f.workspaces[0]), + audience: 'sim:files:copy', + scope: { kind: 'file_copy' as const, ...input }, + } + const authorize = await createFileCopyAuthorizer(principal, input) + await db.transaction(authorize) + if (revoke === 'permission') { + await db + .delete(permissions) + .where( + and(eq(permissions.userId, f.readerId), eq(permissions.entityId, f.workspaces[0])) + ) + } else { + const groupId = generateId() + await db.insert(permissionGroup).values({ + id: groupId, + organizationId: f.organizationId, + createdBy: f.ownerId, + name: 'Revoked capability', + membershipMode: 'inherit', + config: revoke === 'copilot' ? { hideCopilot: true } : { hideFilesTab: true }, + }) + await db.insert(permissionGroupWorkspace).values({ + id: generateId(), + permissionGroupId: groupId, + workspaceId: revoke === 'copilot' ? f.workspaces[0] : b.workspaceId, + organizationId: f.organizationId, + }) + } + await expect(db.transaction(authorize)).rejects.toMatchObject({ code: 'forbidden' }) + } + } + ) +}) diff --git a/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts b/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts new file mode 100644 index 00000000000..1ece5a545e8 --- /dev/null +++ b/apps/sim/lib/projects/files/__integration__/content-bulk.integration.ts @@ -0,0 +1,1472 @@ +import { mkdtempSync } from 'node:fs' +import { mkdir, readdir, readFile, rm, truncate, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import type { Principal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { + folder, + idempotencyKey, + member, + organization, + outboxEvent, + permissionGroup, + permissionGroupWorkspace, + permissions, + subscription, + user, + workspace, + workspaceFileSecretProvenance, + workspaceFiles, + workspaceFileVersion, +} from '@sim/db/schema' +import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { emailMailerMock } from '@sim/testing/mocks/email-mailer.mock' +import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock' +import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, sql } from 'drizzle-orm' +import JSZip from 'jszip' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) +vi.mock('@/lib/messaging/email/mailer', () => emailMailerMock) + +import { observeContentDelivery } from '@/lib/core/application/content-delivery' +import { + createProjectFile, + readProjectFileContent, + updateProjectFileContent, +} from '@/lib/projects/files/application/content' +import { createProjectFileFolder } from '@/lib/projects/files/application/folders' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import * as storage from '@/lib/uploads/core/storage-service' +import { observeWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' + +vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) + +const localStorageRoot = mkdtempSync(join(tmpdir(), 'sim-project-content-')) +setUploadDirServer(localStorageRoot) +const fixtures: { + ownerId: string + editorId: string + organizationId: string + workspaceId: string + projectId: string +}[] = [] +const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] = + [] + +beforeEach(() => { + vi.restoreAllMocks() + featureFlagsMockFns.mockIsFeatureEnabled.mockImplementation(async (flag) => flag === 'projects') + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '') +}) + +function check(name: string, run: () => Promise) { + it(name, async () => { + const started = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - started, + error: getErrorMessage(error), + }) + throw error + } + }) +} + +async function fixture() { + const ownerId = generateId() + const editorId = generateId() + const organizationId = generateId() + const workspaceId = generateId() + await db.insert(user).values( + [ownerId, editorId].map((id) => ({ + id, + email: `${id}@content.invalid`, + name: 'File content fixture', + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + })) + ) + await db + .insert(organization) + .values({ id: organizationId, name: 'Content', slug: organizationId, createdAt: new Date() }) + await db.insert(member).values( + [ownerId, editorId].map((userId) => ({ + id: generateId(), + organizationId, + userId, + role: userId === ownerId ? 'owner' : 'member', + createdAt: new Date(), + })) + ) + await insertWorkspaceFixture(db, { + id: workspaceId, + ownerId, + billedAccountUserId: ownerId, + organizationId, + workspaceMode: 'organization', + name: 'Content environment', + }) + const [binding] = await db.select().from(workspace).where(eq(workspace.id, workspaceId)) + if (!binding) throw new Error('Project fixture missing') + await db.insert(permissions).values({ + id: generateId(), + userId: editorId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + const ids = { ownerId, editorId, organizationId, workspaceId, projectId: binding.projectId } + fixtures.push(ids) + return { ...ids, principal: createSessionPrincipal({ userId: editorId }) } +} + +function createInput(projectId: string, content = 'Project architecture') { + return { + projectId, + name: 'architecture.md', + contentType: 'text/markdown', + content, + encoding: 'utf-8' as const, + exactName: true, + } +} + +async function keys(projectId: string) { + return readdir(join(localStorageRoot, 'project', projectId)).catch(() => []) +} + +async function ledger(organizationId: string) { + const [row] = await db + .select({ bytes: organization.storageUsedBytes }) + .from(organization) + .where(eq(organization.id, organizationId)) + return row?.bytes +} + +async function rows(projectId: string) { + return db.select().from(workspaceFiles).where(eq(workspaceFiles.projectId, projectId)) +} + +describe('Project file content against PostgreSQL and the local object store', () => { + check( + 'folder tree copy hydrates destination paths once while preserving nested paths', + async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const f = await fixture() + const root = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Root' }, + }) + await db.insert(folder).values( + Array.from({ length: 20 }, (_, index) => ({ + id: generateId(), + projectId: f.projectId, + parentId: root.folder.id, + resourceType: 'file' as const, + name: `Child-${index}`, + userId: f.ownerId, + })) + ) + const originalDebug = db.$client.options.debug + const queries: string[] = [] + db.$client.options.debug = (_connection, query) => { + queries.push(query) + } + try { + const result = await copyFileItems.execute({ + principal: f.principal, + input: { + source: { + owner: { entityType: 'project', entityId: f.projectId }, + fileIds: [], + folderIds: [root.folder.id], + }, + destination: { + owner: { entityType: 'workspace', entityId: f.workspaceId }, + folderId: null, + }, + }, + }) + expect(result.folders).toHaveLength(21) + const copiedRoot = result.folders.find((item) => item.parentId === null) + if (!copiedRoot) throw new Error('Copied root missing') + expect(result.folders.filter((item) => item.parentId === copiedRoot.id)).toHaveLength(20) + for (const item of result.folders.filter((item) => item.id !== copiedRoot.id)) + expect(item.path).toBe(`${copiedRoot.path}/${item.name}`) + expect( + queries.filter((query) => + query.startsWith('select "id", "name", "parent_id" from "folder"') + ) + ).toHaveLength(0) + } finally { + db.$client.options.debug = originalDebug + } + } + ) + + check( + 'copy selection ignores unrelated folders when the owner exceeds the selection limit', + async () => { + const { snapshotFileCopyInTx } = await import('@/lib/workspace-files/copy') + const f = await fixture() + const selected = generateId() + await db.insert(folder).values( + Array.from({ length: 5_001 }, (_, index) => ({ + id: index === 0 ? selected : generateId(), + projectId: f.projectId, + resourceType: 'file' as const, + name: `folder-${index}`, + userId: f.ownerId, + })) + ) + const result = await db.transaction((tx) => + snapshotFileCopyInTx(tx, { + owner: { entityType: 'project', entityId: f.projectId }, + fileIds: [], + folderIds: [selected], + }) + ) + expect(result.folders.map((item) => item.id)).toEqual([selected]) + expect(result.files).toHaveLength(0) + } + ) + + check('copy retains committed bytes when COMMIT acknowledgement is lost', async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const f = await fixture() + const source = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId, 'retained after lost acknowledgement'), + }) + const original = db.transaction.bind(db) + vi.spyOn(db, 'transaction').mockImplementation(async (callback, config) => { + const result = await original(callback, config) + if (result && typeof result === 'object' && 'usage' in result && 'result' in result) + throw new Error('Lost COMMIT acknowledgement') + return result + }) + await expect( + copyFileItems.execute({ + principal: f.principal, + input: { + source: { + owner: { entityType: 'project', entityId: f.projectId }, + fileIds: [source.file.id], + folderIds: [], + }, + destination: { owner: { entityType: 'project', entityId: f.projectId }, folderId: null }, + }, + }) + ).rejects.toThrow('Lost COMMIT acknowledgement') + vi.restoreAllMocks() + const copied = (await rows(f.projectId)).find((file) => file.id !== source.file.id) + if (!copied) throw new Error('Copy did not commit') + expect(await readFile(join(localStorageRoot, copied.key), 'utf8')).toBe( + 'retained after lost acknowledgement' + ) + const cleanup = await db + .select() + .from(outboxEvent) + .where(sql`${outboxEvent.payload}::text LIKE ${`%${copied.key}%`}`) + expect(cleanup).toHaveLength(0) + }) + + check( + 'compound copy remaps selected page assets before staging and bills the actual rendered source bytes', + async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const { createWorkspaceFileFromBuffer } = await import( + '@/lib/workspace-files/application/create-workspace-file' + ) + const { readProjectFileArtifact } = await import('@/lib/projects/files/application/artifacts') + const f = await fixture() + const image = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a9uoAAAAASUVORK5CYII=', + 'base64' + ) + const asset = await createWorkspaceFileFromBuffer.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'diagram.png', + contentType: 'image/png', + content: image, + exactName: true, + }, + }) + const original = Buffer.from( + `---\ntitle: Architecture\n---\n\n![Diagram](sim:file/${asset.file.id})\n\n![Stored](/api/files/serve/${encodeURIComponent(asset.file.key)})` + ) + const source = await createWorkspaceFileFromBuffer.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'Architecture', + contentType: 'text/x-sim-page', + content: original, + exactName: true, + }, + }) + const before = await ledger(f.organizationId) + const copied = await copyFileItems.execute({ + principal: f.principal, + input: { + source: { + owner: { entityType: 'workspace', entityId: f.workspaceId }, + fileIds: [asset.file.id, source.file.id], + folderIds: [], + }, + destination: { owner: { entityType: 'project', entityId: f.projectId }, folderId: null }, + }, + }) + const copiedSource = copied.files.find((file) => file.name === 'Architecture') + const copiedAsset = copied.files.find((file) => file.name === 'diagram.png') + if (!copiedSource || !copiedAsset) throw new Error('Copy omitted selected source or asset') + const rendered = await readProjectFileArtifact.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: copiedSource.id, maxBytes: 1_000_000 }, + }) + expect(rendered.buffer.toString()).toContain( + `data:image/png;base64,${image.toString('base64')}` + ) + const copiedBytes = await readFile(join(localStorageRoot, copiedSource.key)) + expect(copiedBytes.toString()).toContain(copiedAsset.id) + expect(copiedBytes.toString()).not.toContain(asset.file.id) + expect(await readFile(join(localStorageRoot, source.file.key))).toEqual(original) + expect(await readFile(join(localStorageRoot, copiedAsset.key))).toEqual(image) + expect(await ledger(f.organizationId)).toBe( + (before ?? 0) + copied.files.reduce((sum, file) => sum + file.size, 0) + ) + expect(copiedSource.size).toBe(copiedBytes.length) + } + ) + + check( + 'compound recursive copy preserves bytes and provenance under new owner and creator attribution', + async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const { createWorkspaceFileFromBuffer } = await import( + '@/lib/workspace-files/application/create-workspace-file' + ) + const { createWorkspaceFileFolderOperation } = await import( + '@/lib/workspace-files/application/folders' + ) + const f = await fixture() + const sourceOwner = { entityType: 'workspace' as const, entityId: f.workspaceId } + const destinationOwner = { entityType: 'project' as const, entityId: f.projectId } + const directory = await createWorkspaceFileFolderOperation.execute({ + principal: f.principal, + input: { workspaceId: f.workspaceId, name: 'Source' }, + }) + const child = await createWorkspaceFileFolderOperation.execute({ + principal: f.principal, + input: { workspaceId: f.workspaceId, name: 'Nested', parentId: directory.folder.id }, + }) + const secret = { + status: 'exact' as const, + entries: [ + { + name: 'TOKEN', + encryptedValue: 'encrypted-fixture', + sourceUserId: f.ownerId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + const source = await createWorkspaceFileFromBuffer.execute({ + principal: createSessionPrincipal({ userId: f.ownerId }), + input: { + workspaceId: f.workspaceId, + name: 'source.bin', + contentType: 'application/octet-stream', + content: Buffer.from([1, 255, 0]), + folderId: child.folder.id, + exactName: true, + secretProvenance: secret, + }, + }) + const copied = await copyFileItems.execute({ + principal: f.principal, + input: { + source: { owner: sourceOwner, fileIds: [], folderIds: [directory.folder.id] }, + destination: { owner: destinationOwner, folderId: null }, + }, + }) + expect(copied.files).toHaveLength(1) + expect(copied.folders).toHaveLength(2) + expect(copied.files[0]).toMatchObject({ + owner: destinationOwner, + uploadedBy: f.editorId, + folderPath: 'Source/Nested', + }) + expect(copied.files[0].id).not.toBe(source.file.id) + const read = await readProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: copied.files[0].id, + includeSecretProvenance: true, + }, + }) + expect(read.content).toEqual(Buffer.from([1, 255, 0])) + expect(read.secretProvenance).toEqual(secret) + expect(await ledger(f.organizationId)).toBe(6) + const sameOwner = await copyFileItems.execute({ + principal: f.principal, + input: { + source: { owner: destinationOwner, fileIds: [copied.files[0].id], folderIds: [] }, + destination: { owner: destinationOwner, folderId: null }, + }, + }) + expect(sameOwner.files[0].folderId).toBeNull() + expect(sameOwner.files[0].id).not.toBe(copied.files[0].id) + expect(await ledger(f.organizationId)).toBe(9) + } + ) + + check( + 'compound copy rejects source changes after staging without committing folders, files, or extra billing', + async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const { createWorkspaceFile, createWorkspaceFileFromBuffer } = await import( + '@/lib/workspace-files/application/create-workspace-file' + ) + const { updateWorkspaceFileContent } = await import( + '@/lib/workspace-files/application/update-workspace-file-content' + ) + const { createWorkspaceFileFolderOperation } = await import( + '@/lib/workspace-files/application/folders' + ) + const f = await fixture() + const directory = await createWorkspaceFileFolderOperation.execute({ + principal: f.principal, + input: { workspaceId: f.workspaceId, name: 'Source' }, + }) + const source = await createWorkspaceFile.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'notes.txt', + contentType: 'text/plain', + content: 'before', + encoding: 'utf-8', + folderId: directory.folder.id, + exactName: true, + }, + }) + await createWorkspaceFileFromBuffer.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'more.bin', + contentType: 'application/octet-stream', + content: Buffer.from([1]), + folderId: directory.folder.id, + exactName: true, + }, + }) + const upload = storage.uploadFile + vi.spyOn(storage, 'uploadFile').mockImplementationOnce(async (options) => { + const staged = await upload(options) + await updateWorkspaceFileContent.execute({ + principal: f.principal, + input: { + fileId: source.file.id, + assertedWorkspaceId: f.workspaceId, + content: 'after source change', + encoding: 'utf-8', + syncLiveDoc: false, + }, + }) + return staged + }) + await expect( + copyFileItems.execute({ + principal: f.principal, + input: { + source: { + owner: { entityType: 'workspace', entityId: f.workspaceId }, + fileIds: [], + folderIds: [directory.folder.id], + }, + destination: { + owner: { entityType: 'project', entityId: f.projectId }, + folderId: null, + }, + }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(await rows(f.projectId)).toEqual([]) + expect(await db.select().from(folder).where(eq(folder.projectId, f.projectId))).toEqual([]) + expect(await keys(f.projectId)).toEqual([]) + expect(await ledger(f.organizationId)).toBe(Buffer.byteLength('after source change') + 1) + } + ) + + check( + 'compound copy quota failure atomically rolls back the whole destination tree and releases staged bytes', + async () => { + const { copyFileItems } = await import('@/lib/workspace-files/application/copy-file-items') + const { createWorkspaceFile } = await import( + '@/lib/workspace-files/application/create-workspace-file' + ) + const { createWorkspaceFileFolderOperation } = await import( + '@/lib/workspace-files/application/folders' + ) + const f = await fixture() + const directory = await createWorkspaceFileFolderOperation.execute({ + principal: f.principal, + input: { workspaceId: f.workspaceId, name: 'Source' }, + }) + const source = await createWorkspaceFile.execute({ + principal: f.principal, + input: { + workspaceId: f.workspaceId, + name: 'notes.txt', + contentType: 'text/plain', + content: 'before', + encoding: 'utf-8', + folderId: directory.folder.id, + exactName: true, + }, + }) + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '1') + await db + .update(organization) + .set({ storageUsedBytes: 1024 ** 3 }) + .where(eq(organization.id, f.organizationId)) + await expect( + copyFileItems.execute({ + principal: f.principal, + input: { + source: { + owner: { entityType: 'workspace', entityId: f.workspaceId }, + fileIds: [], + folderIds: [directory.folder.id], + }, + destination: { + owner: { entityType: 'project', entityId: f.projectId }, + folderId: null, + }, + }, + }) + ).rejects.toMatchObject({ name: 'StorageLimitExceededError' }) + expect(await rows(f.projectId)).toEqual([]) + expect(await db.select().from(folder).where(eq(folder.projectId, f.projectId))).toEqual([]) + expect(await keys(f.projectId)).toEqual([]) + expect(await readFile(join(localStorageRoot, source.file.key), 'utf8')).toBe('before') + expect(await ledger(f.organizationId)).toBe(1024 ** 3) + } + ) +}) + +describe('Project archive download and Markdown snapshot export', () => { + check('snapshot export refuses an opaque embedded asset before fetching it', async () => { + const { exportProjectFileSnapshot } = await import('@/lib/projects/files/application/downloads') + const f = await fixture() + const source = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId, '# source'), + }) + const image = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'image bytes'), + name: 'image.png', + contentType: 'image/png', + }, + }) + const principal: Principal = { + kind: 'resource_delegated', + serviceId: 'copilot', + subjectUserId: f.editorId, + audience: 'sim:project-files', + delegationId: generateId(), + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 30_000), + scope: { kind: 'entity', entityType: 'project', entityId: f.projectId }, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + } + const read = vi.spyOn(storage, 'downloadFile') + await expect( + observeContentDelivery( + { + async admit(sources) { + expect(sources.every((item) => item.view === 'opaque')).toBe(true) + return sources.map((item) => item.id !== image.file.id) + }, + }, + () => + exportProjectFileSnapshot.execute({ + principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: `![image](sim:file/${image.file.id}?project=${f.projectId})`, + secretProvenance: { status: 'exact', entries: [] }, + }, + }) + ) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(read).not.toHaveBeenCalled() + }) + + for (const phase of ['before-read', 'before-delivery'] as const) { + check(`Project archive opaque admission refuses ${phase}`, async () => { + const { downloadProjectFileItems } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId, 'private bytes'), + }) + const principal: Principal = { + kind: 'resource_delegated', + serviceId: 'copilot', + subjectUserId: f.editorId, + audience: 'sim:project-files', + delegationId: generateId(), + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 30_000), + scope: { kind: 'entity', entityType: 'project', entityId: f.projectId }, + invocation: { kind: 'workspace', workspaceId: f.workspaceId }, + } + const read = vi.spyOn(storage, 'downloadFile') + let admissions = 0 + await expect( + observeContentDelivery( + { + async admit(sources) { + admissions++ + expect(sources.every((source) => source.view === 'opaque')).toBe(true) + return sources.map(() => phase === 'before-delivery' && admissions === 1) + }, + }, + () => + downloadProjectFileItems.execute({ + principal, + input: { projectId: f.projectId, fileIds: [file.id], folderIds: [] }, + }) + ) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(read).toHaveBeenCalledTimes(phase === 'before-read' ? 0 : 1) + }) + } + + check( + 'Project recursive download retains nested paths, deduplicates selections, and excludes archived or foreign items', + async () => { + const { downloadProjectFileItems } = await import( + '@/lib/projects/files/application/downloads' + ) + const { archiveProjectFileItems } = await import('@/lib/projects/files/application/lifecycle') + const f = await fixture() + const other = await fixture() + const root = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Architecture' }, + }) + const nested = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, parentId: root.folder.id, name: 'Diagrams' }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'nested bytes'), folderId: nested.folder.id }, + }) + const archived = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'retired bytes'), folderId: root.folder.id }, + }) + await archiveProjectFileItems.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileIds: [archived.file.id] }, + }) + const foreign = await createProjectFile.execute({ + principal: other.principal, + input: createInput(other.projectId, 'foreign bytes'), + }) + const foreignFolder = await createProjectFileFolder.execute({ + principal: other.principal, + input: { projectId: other.projectId, name: 'Architecture' }, + }) + const input = { + projectId: f.projectId, + fileIds: [source.file.id, source.file.id], + folderIds: [root.folder.id, nested.folder.id], + } + const result = await downloadProjectFileItems.execute({ principal: f.principal, input }) + const zip = await JSZip.loadAsync(result.buffer) + const entries = Object.values(zip.files).filter((entry) => !entry.dir) + expect(entries.map((entry) => entry.name)).toEqual(['Architecture/Diagrams/architecture.md']) + expect(await entries[0].async('string')).toBe('nested bytes') + expect(result.fileCount).toBe(1) + expect(result.secretProvenance).toEqual({ status: 'exact', entries: [] }) + for (const selection of [ + { ...input, fileIds: [foreign.file.id] }, + { ...input, folderIds: [foreignFolder.folder.id] }, + ]) { + await expect( + downloadProjectFileItems.execute({ principal: f.principal, input: selection }) + ).rejects.toMatchObject({ code: 'not_found' }) + } + } + ) + + check( + 'Project recursive download enforces expanded file counts and actual aggregate bytes despite understated metadata', + async () => { + const { downloadProjectFileItems } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const fileIds: string[] = [] + for (let index = 0; index < 3; index++) { + const result = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'x'), + name: `large-${index}.bin`, + contentType: 'application/octet-stream', + }, + }) + await truncate(join(localStorageRoot, result.file.key), 90 * 1024 * 1024) + fileIds.push(result.file.id) + } + await expect( + downloadProjectFileItems.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileIds, folderIds: [] }, + }) + ).rejects.toMatchObject({ code: 'payload_too_large' }) + expect(await ledger(f.organizationId)).toBe(3) + const [template] = await rows(f.projectId) + if (!template) throw new Error('Download fixture metadata is missing') + const selected = Array.from({ length: 101 }, (_, index) => ({ + ...template, + id: generateId(), + key: `project/${f.projectId}/count-${index}`, + originalName: `count-${index}.bin`, + })) + await db.insert(workspaceFiles).values(selected) + await expect( + downloadProjectFileItems.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileIds: selected.map((file) => file.id), + folderIds: [], + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + } + ) + + check( + 'Project recursive download rechecks source, descendant membership, and current access after object IO', + async () => { + const { downloadProjectFileItems } = await import( + '@/lib/projects/files/application/downloads' + ) + for (const change of ['source', 'selection', 'access'] as const) { + const f = await fixture() + const directory = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Selected' }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'before IO'), folderId: directory.folder.id }, + }) + const download = storage.downloadFile + let changed = false + const interception = vi + .spyOn(storage, 'downloadFile') + .mockImplementation(async (options) => { + const buffer = await download(options) + if (options.key === source.file.key && !changed) { + changed = true + if (change === 'source') { + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: 'after IO', + encoding: 'utf-8', + }, + }) + } else if (change === 'selection') { + await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'new descendant'), + name: 'later.md', + folderId: directory.folder.id, + }, + }) + } else { + await db.delete(permissions).where(eq(permissions.userId, f.editorId)) + } + } + return buffer + }) + try { + await expect( + downloadProjectFileItems.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileIds: [], folderIds: [directory.folder.id] }, + }) + ).rejects.toMatchObject({ code: change === 'access' ? 'not_found' : 'conflict' }) + } finally { + interception.mockRestore() + } + } + } + ) + + check( + 'Project recursive download aggregates bulk-download policy across accessible environments while retaining single-file reads', + async () => { + const { downloadProjectFileItems } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const directory = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Selected' }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId), folderId: directory.folder.id }, + }) + const secondId = generateId() + await insertWorkspaceFixture(db, { + id: secondId, + ownerId: f.ownerId, + billedAccountUserId: f.ownerId, + organizationId: f.organizationId, + workspaceMode: 'organization', + name: 'Restricted downloads', + forkedFromWorkspaceId: f.workspaceId, + }) + try { + await db.insert(subscription).values({ + id: generateId(), + plan: 'enterprise', + referenceId: f.organizationId, + status: 'active', + metadata: {}, + }) + const groupId = generateId() + await db.insert(permissionGroup).values({ + id: groupId, + organizationId: f.organizationId, + createdBy: f.ownerId, + name: 'Restricted downloads', + membershipMode: 'inherit', + config: { disableBulkFileDownload: true }, + }) + await db.insert(permissionGroupWorkspace).values({ + id: generateId(), + permissionGroupId: groupId, + workspaceId: secondId, + organizationId: f.organizationId, + }) + const input = { projectId: f.projectId, fileIds: [], folderIds: [directory.folder.id] } + await downloadProjectFileItems.execute({ principal: f.principal, input }) + await db.insert(permissions).values({ + id: generateId(), + userId: f.editorId, + entityType: 'workspace', + entityId: secondId, + permissionType: 'read', + }) + await expect( + downloadProjectFileItems.execute({ principal: f.principal, input }) + ).rejects.toMatchObject({ detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED' }) + const single = await downloadProjectFileItems.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileIds: [source.file.id], folderIds: [] }, + }) + expect(single.fileCount).toBe(1) + } finally { + await deleteWorkspaceFixture(db, eq(workspace.id, secondId)) + } + } + ) + + check( + 'Project Markdown snapshot export packages only same-owner images, preserves source fidelity, and delivers joined provenance without persisting edits', + async () => { + const { exportProjectFileSnapshot } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const other = await fixture() + const secret = { + status: 'exact' as const, + entries: [ + { + encryptedValue: 'asset-fixture', + sourceUserId: f.ownerId, + sourceWorkspaceId: f.workspaceId, + }, + ], + } + const image = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'same-owner image bytes'), + name: 'diagram.png', + contentType: 'image/png', + secretProvenance: secret, + }, + }) + const foreign = await createProjectFile.execute({ + principal: other.principal, + input: { + ...createInput(other.projectId, 'foreign image bytes'), + name: 'secret.png', + contentType: 'image/png', + }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId, 'durable draft'), + }) + const headBefore = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, source.file.id)) + const historyBefore = await db + .select() + .from(workspaceFileVersion) + .where(eq(workspaceFileVersion.fileId, source.file.id)) + const visible = `---\ntitle: Architecture\n---\n\n# Unsaved snapshot\n\n![diagram](sim:file/${image.file.id}?project=${f.projectId})\n\n![foreign](/api/projects/${other.projectId}/files/${foreign.file.id}/content)\n\n\`sim:file/${image.file.id}\`\n` + let delivered: unknown + const exported = await observeWorkspaceFileDelivery( + async (source) => { + delivered = source + }, + () => + exportProjectFileSnapshot.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, content: visible }, + }) + ) + const zip = await JSZip.loadAsync(exported.buffer) + expect( + Object.values(zip.files) + .filter((entry) => !entry.dir) + .map((entry) => entry.name) + ).toEqual(['architecture.md', 'assets/diagram.png']) + expect(await zip.file('assets/diagram.png')?.async('string')).toBe('same-owner image bytes') + expect(await zip.file('architecture.md')?.async('string')).toBe( + visible.replace(`sim:file/${image.file.id}?project=${f.projectId}`, './assets/diagram.png') + ) + expect(exported.secretProvenance).toEqual(secret) + expect(delivered).toEqual(secret) + expect(await readFile(join(localStorageRoot, source.file.key), 'utf8')).toBe('durable draft') + const history = await db + .select() + .from(workspaceFileVersion) + .where(eq(workspaceFileVersion.fileId, source.file.id)) + expect(history).toEqual(historyBefore) + const headAfter = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, source.file.id)) + expect(headAfter).toEqual(headBefore) + + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: 'unknown source', + encoding: 'utf-8', + secretProvenance: { status: 'unknown' }, + }, + }) + const unknown = await exportProjectFileSnapshot.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id, content: visible }, + }) + expect(unknown.secretProvenance.status).toBe('unknown') + } + ) + + check( + 'Project Markdown snapshot export never collapses admitted control-bearing names into ZIP traversal segments', + async () => { + const { exportProjectFileSnapshot } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const image = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'bundled asset'), + name: '.\r.', + contentType: 'image/png', + }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'durable draft'), name: '.\n.' }, + }) + const exported = await exportProjectFileSnapshot.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: `![asset](sim:file/${image.file.id}?project=${f.projectId})`, + }, + }) + const zip = await JSZip.loadAsync(exported.buffer) + const entries = Object.values(zip.files).filter((entry) => !entry.dir) + expect(entries).toHaveLength(2) + for (const entry of entries) { + const rawPath = entry.unsafeOriginalName ?? entry.name + expect(rawPath).not.toMatch(/[\x00-\x1f\x7f]/) + expect( + rawPath.split('/').every((segment) => segment && segment !== '.' && segment !== '..') + ).toBe(true) + } + const asset = entries.find((entry) => entry.name.startsWith('assets/')) + expect(await asset?.async('string')).toBe('bundled asset') + } + ) + + check( + 'Project Markdown snapshot export refuses an asset revision changed after its actual bytes were read', + async () => { + const { exportProjectFileSnapshot } = await import( + '@/lib/projects/files/application/downloads' + ) + const f = await fixture() + const image = await createProjectFile.execute({ + principal: f.principal, + input: { + ...createInput(f.projectId, 'image before'), + name: 'diagram.png', + contentType: 'image/png', + }, + }) + const source = await createProjectFile.execute({ + principal: f.principal, + input: createInput(f.projectId), + }) + const download = storage.downloadFile + let changed = false + vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const buffer = await download(options) + if (options.key === image.file.key && !changed) { + changed = true + await updateProjectFileContent.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: image.file.id, + content: 'image after', + encoding: 'utf-8', + }, + }) + } + return buffer + }) + await expect( + exportProjectFileSnapshot.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: source.file.id, + content: `![diagram](sim:file/${image.file.id}?project=${f.projectId})`, + }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + } + ) +}) + +describe('Project ZIP extraction against PostgreSQL and local storage', () => { + async function archive( + f: Awaited>, + entries: Record, + provenance?: WorkspaceFileSecretProvenance + ) { + const zip = new JSZip() + for (const [path, content] of Object.entries(entries)) zip.file(path, content) + const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) + return createProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + name: 'bundle.zip', + contentType: 'application/zip', + content: buffer.toString('base64'), + encoding: 'base64', + secretProvenance: provenance, + }, + }) + } + + check('extraction retains committed bytes when COMMIT acknowledgement is lost', async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const source = await archive(f, { 'retained.txt': 'retained extracted bytes' }) + const original = db.transaction.bind(db) + vi.spyOn(db, 'transaction').mockImplementation(async (callback, config) => { + const result = await original(callback, config) + if (result && typeof result === 'object' && 'usage' in result && 'result' in result) + throw new Error('Lost COMMIT acknowledgement') + return result + }) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toThrow('Lost COMMIT acknowledgement') + vi.restoreAllMocks() + const child = (await rows(f.projectId)).find((file) => file.id !== source.file.id) + if (!child) throw new Error('Extraction did not commit') + expect(await readFile(join(localStorageRoot, child.key), 'utf8')).toBe( + 'retained extracted bytes' + ) + const cleanup = await db + .select() + .from(outboxEvent) + .where(sql`${outboxEvent.payload}::text LIKE ${`%${child.key}%`}`) + expect(cleanup).toHaveLength(0) + }) + + check('extraction rejects an overlong directory before staging a member', async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const source = await archive(f, { [`${'x'.repeat(256)}/entry.txt`]: 'content' }) + const upload = vi.spyOn(storage, 'uploadFile') + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(upload).not.toHaveBeenCalled() + expect(await folders(f.projectId)).toHaveLength(0) + expect(await rows(f.projectId)).toHaveLength(1) + }) + + async function folders(projectId: string) { + return db.select().from(folder).where(eq(folder.projectId, projectId)) + } + + check( + 'Project ZIP extraction atomically creates nested bytes beside the archive and bills the canonical payer', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const parent = await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'Resources' }, + }) + const source = await archive(f, { + 'docs/readme.txt': 'architecture', + 'images/icon.txt': 'diagram', + '__MACOSX/._readme.txt': 'noise', + }) + await db + .update(workspaceFiles) + .set({ folderId: parent.folder.id }) + .where(eq(workspaceFiles.id, source.file.id)) + await createProjectFileFolder.execute({ + principal: f.principal, + input: { projectId: f.projectId, name: 'bundle', parentId: parent.folder.id }, + }) + const before = await ledger(f.organizationId) + const result = await extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + expect(result).toMatchObject({ extractedCount: 2, skippedCount: 1 }) + expect(result.folderName).not.toBe('bundle') + const tree = await folders(f.projectId) + const root = tree.find((row) => row.id === result.folderId) + expect(root).toMatchObject({ + parentId: parent.folder.id, + userId: f.editorId, + projectId: f.projectId, + workspaceId: null, + }) + const children = (await rows(f.projectId)).filter((row) => row.id !== source.file.id) + expect(children).toHaveLength(2) + for (const row of children) { + expect(row.userId).toBe(f.editorId) + expect(row.workspaceId).toBeNull() + const directory = tree.find((candidate) => candidate.id === row.folderId) + expect(directory?.parentId).toBe(result.folderId) + expect(await readFile(join(localStorageRoot, row.key), 'utf8')).toBe( + row.originalName === 'readme.txt' ? 'architecture' : 'diagram' + ) + } + expect(await ledger(f.organizationId)).toBe( + (before ?? 0) + Buffer.byteLength('architecturediagram') + ) + expect(await storage.downloadFile({ key: source.file.key, context: 'project' })).toHaveLength( + source.file.size + ) + } + ) + + check( + 'Project ZIP extraction conceals a foreign archive and refuses a read-only destination without publishing children', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const other = await fixture() + const source = await archive(f, { 'one.txt': 'one' }) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: (await archive(other, { 'secret.txt': 'secret' })).file.id, + }, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + await db + .update(permissions) + .set({ permissionType: 'read' }) + .where(and(eq(permissions.userId, f.editorId), eq(permissions.entityId, f.workspaceId))) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect((await rows(f.projectId)).map((row) => row.id)).toEqual([source.file.id]) + expect(await folders(f.projectId)).toEqual([]) + expect(await keys(f.projectId)).toHaveLength(1) + expect(await ledger(f.organizationId)).toBe(source.file.size) + } + ) + + for (const change of ['permission', 'source'] as const) { + check( + `Project ZIP extraction rejects ${change} changes after staging and leaves no partial tree`, + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const source = await archive(f, { 'first.txt': 'first', 'nested/second.txt': 'second' }) + const upload = storage.uploadFile + let changed = false + vi.spyOn(storage, 'uploadFile').mockImplementation(async (options) => { + const result = await upload(options) + if (!changed) { + changed = true + if (change === 'permission') + await db + .update(permissions) + .set({ permissionType: 'read' }) + .where( + and(eq(permissions.userId, f.editorId), eq(permissions.entityId, f.workspaceId)) + ) + else + await db + .update(workspaceFiles) + .set({ originalName: 'renamed.zip', updatedAt: new Date() }) + .where(eq(workspaceFiles.id, source.file.id)) + } + return result + }) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toMatchObject({ code: change === 'permission' ? 'forbidden' : 'conflict' }) + expect((await rows(f.projectId)).map((row) => row.id)).toEqual([source.file.id]) + expect(await folders(f.projectId)).toEqual([]) + expect(await keys(f.projectId)).toHaveLength(1) + expect(await ledger(f.organizationId)).toBe(source.file.size) + } + ) + } + + check( + 'Project ZIP extraction blocks concurrent work and fences a replaced lease before metadata commit', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const source = await archive(f, { 'one.txt': 'one' }) + const uploaded = createDeferred() + const resume = createDeferred() + const upload = storage.uploadFile + vi.spyOn(storage, 'uploadFile').mockImplementationOnce(async (options) => { + const result = await upload(options) + uploaded.resolve() + await resume.promise + return result + }) + const args = { + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + } + const first = extractProjectFile.execute(args) + const firstResult = first.catch((error: unknown) => error) + await uploaded.promise + try { + await expect(extractProjectFile.execute(args)).rejects.toMatchObject({ code: 'conflict' }) + const replaced = await db + .update(idempotencyKey) + .set({ + result: { + status: 'in-progress', + claimToken: generateId(), + inProgressExpiresAt: Date.now() + 60_000, + }, + }) + .where(sql`${idempotencyKey.key} LIKE ${`%${f.projectId}%${source.file.id}%`}`) + .returning({ key: idempotencyKey.key }) + expect(replaced).toHaveLength(1) + } finally { + resume.resolve() + } + expect(await firstResult).toMatchObject({ code: 'conflict' }) + expect((await rows(f.projectId)).map((row) => row.id)).toEqual([source.file.id]) + expect(await folders(f.projectId)).toEqual([]) + expect(await keys(f.projectId)).toHaveLength(1) + expect(await ledger(f.organizationId)).toBe(source.file.size) + } + ) + + check( + 'Project ZIP extraction quota failure rolls back every child and preserves the source archive', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const f = await fixture() + const source = await archive(f, { 'first.txt': 'first', 'nested/second.txt': 'second' }) + vi.stubEnv('FREE_STORAGE_LIMIT_GB', '1') + await db + .update(organization) + .set({ storageUsedBytes: 1024 ** 3 }) + .where(eq(organization.id, f.organizationId)) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toMatchObject({ name: 'StorageLimitExceededError' }) + expect((await rows(f.projectId)).map((row) => row.id)).toEqual([source.file.id]) + expect(await folders(f.projectId)).toEqual([]) + expect(await keys(f.projectId)).toHaveLength(1) + expect(await ledger(f.organizationId)).toBe(1024 ** 3) + } + ) + + check( + 'Project ZIP extraction rejects over-deep member paths before any destination mutation', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const { MAX_FOLDER_PATH_SEGMENTS } = await import('@/lib/folders/paths') + const f = await fixture() + const source = await archive(f, { + [`${Array.from({ length: MAX_FOLDER_PATH_SEGMENTS + 1 }, () => 'deep').join('/')}/file.txt`]: + 'too deep', + }) + await expect( + extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + ).rejects.toMatchObject({ reason: 'invalid' }) + expect((await rows(f.projectId)).map((row) => row.id)).toEqual([source.file.id]) + expect(await folders(f.projectId)).toEqual([]) + expect(await keys(f.projectId)).toHaveLength(1) + expect(await ledger(f.organizationId)).toBe(source.file.size) + } + ) + + check( + 'Project ZIP extraction preserves conservative provenance through decompression', + async () => { + const { extractProjectFile } = await import('@/lib/projects/files/application/extract') + const classifications = [ + { status: 'exact' as const, entries: [] }, + { status: 'unknown' as const }, + { status: 'unrecorded' as const }, + { + status: 'exact' as const, + entries: [ + { + encryptedValue: 'archive-fixture', + sourceUserId: 'origin', + }, + ], + }, + ] + for (const classification of classifications) { + const f = await fixture() + const source = await archive(f, { 'data.txt': 'extracted' }, classification) + await extractProjectFile.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: source.file.id }, + }) + const [child] = (await rows(f.projectId)).filter((row) => row.id !== source.file.id) + if (!child) throw new Error('Extraction did not publish a child') + const [provenance] = await db + .select() + .from(workspaceFileSecretProvenance) + .where(eq(workspaceFileSecretProvenance.fileId, child.id)) + const expected = + classification.status === 'exact' + ? classification.entries.length + ? 'unknown' + : 'exact' + : classification.status + expect(provenance.status).toBe(expected) + expect(provenance.entries).toEqual([]) + expect(await readFile(join(localStorageRoot, child.key), 'utf8')).toBe('extracted') + } + } + ) +}) + +afterAll(async () => { + vi.restoreAllMocks() + for (const f of fixtures) { + await db.delete(idempotencyKey).where(sql`${idempotencyKey.key} LIKE ${`%${f.projectId}%`}`) + await db + .delete(outboxEvent) + .where( + sql`${outboxEvent.payload}::jsonb ->> 'key' LIKE ${`project/${f.projectId}/%`} OR ${outboxEvent.payload}::jsonb -> 'owner' ->> 'entityId' = ${f.projectId}` + ) + await db.delete(workspaceFiles).where(eq(workspaceFiles.projectId, f.projectId)) + await db.delete(folder).where(eq(folder.projectId, f.projectId)) + await deleteWorkspaceFixture(db, eq(workspace.id, f.workspaceId)) + await db.delete(subscription).where(eq(subscription.referenceId, f.organizationId)) + await db.delete(organization).where(eq(organization.id, f.organizationId)) + await db.delete(user).where(inArray(user.id, [f.ownerId, f.editorId])) + } + await rm(localStorageRoot, { recursive: true, force: true }) + const report = + process.env.PROJECT_FILE_CONTENT_REPORT_PATH ?? 'test-results/project-file-content.json' + await mkdir(dirname(report), { recursive: true }) + await writeFile(report, JSON.stringify({ checks }, null, 2)) + await db.$client.end() +}) diff --git a/apps/sim/lib/projects/files/api/download-presenter.ts b/apps/sim/lib/projects/files/api/download-presenter.ts new file mode 100644 index 00000000000..b4cddc60067 --- /dev/null +++ b/apps/sim/lib/projects/files/api/download-presenter.ts @@ -0,0 +1,17 @@ +import { FILE_CACHE_CONTROL, presentFileDelivery } from '@/lib/uploads/server/delivery' + +/** Serves the authorized archive through a buffer view, without cloning its bounded payload. */ +export function presentProjectFileDownload(result: { + buffer: Buffer + fileName: string + contentType: string +}) { + return presentFileDelivery({ + body: result.buffer, + filename: result.fileName, + contentType: result.contentType, + contentLength: result.buffer.length, + attachment: true, + cacheControl: FILE_CACHE_CONTROL.noStore, + }) +} diff --git a/apps/sim/lib/projects/files/application/authorization.ts b/apps/sim/lib/projects/files/application/authorization.ts index b7fc97ce446..b180ec2fd35 100644 --- a/apps/sim/lib/projects/files/application/authorization.ts +++ b/apps/sim/lib/projects/files/application/authorization.ts @@ -94,7 +94,7 @@ function requirePrincipal( } /** Rechecks canonical conversation ownership and authoring capability while authorization locks are held. */ -async function requireCurrentCopilotProjectInvocation( +export async function requireCurrentCopilotProjectInvocation( tx: DbTransaction, principal: Extract, access: Awaited>, @@ -185,7 +185,7 @@ async function requireCurrentCopilotProjectInvocation( } /** Shared owner policy keeps compound copies subject to the same live Project edit rule. */ -async function requireProjectFileOwnerRole( +export async function requireProjectFileOwnerRole( tx: DbTransaction, principal: ProjectFilePrincipal, access: Awaited>, @@ -215,7 +215,7 @@ async function requireProjectFileOwnerRole( } /** Credential and Files capability checks apply to every currently visible environment. */ -async function requireProjectFileOwnerCapabilities( +export async function requireProjectFileOwnerCapabilities( tx: DbTransaction, principal: ProjectFilePrincipal, access: Awaited>, diff --git a/apps/sim/lib/projects/files/application/downloads.ts b/apps/sim/lib/projects/files/application/downloads.ts new file mode 100644 index 00000000000..2cdeee6a553 --- /dev/null +++ b/apps/sim/lib/projects/files/application/downloads.ts @@ -0,0 +1,503 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { type Principal, requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { type WorkspaceFileRow, workspaceFiles } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { PASTE_LIMITS, utf8ByteLength } from '@sim/utils/paste' +import { compareStrings } from '@sim/utils/string' +import { and, asc, inArray, isNull, or } from 'drizzle-orm' +import JSZip from 'jszip' +import { requireAdmittedContent } from '@/lib/core/application/content-delivery' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import type { DbTransaction } from '@/lib/db/types' +import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' +import { readProjectFileArtifact } from '@/lib/projects/files/application/artifacts' +import type { + ProjectFileAuthorizationContext, + ProjectFileTarget, +} from '@/lib/projects/files/application/authorization' +import { defineAuthorizedProjectFileUseCase } from '@/lib/projects/files/application/authorized-use-case' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { + buildWorkspaceFileFolderPathMap, + loadActiveFileFolderPathIndex, + mapFileRecord, +} from '@/lib/uploads/contexts/workspace' +import { + getBoundWorkspaceFileSecretProvenanceByMetadata, + mergeWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { extractEmbeddedFileRefs } from '@/lib/uploads/server/embedded-image-refs' +import { + createMarkdownExport, + MAX_EXPORT_ASSET_BYTES, + MAX_EXPORT_TOTAL_BYTES, + type MarkdownExportAsset, + type MarkdownExportResult, + MarkdownExportSizeError, +} from '@/lib/uploads/server/markdown-export' +import { bufferZipWithinLimit } from '@/lib/uploads/server/zip' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' +import { storedFileId } from '@/lib/uploads/utils/embedded-image-ref' +import { + isMarkdownFile, + MAX_RENDERED_DOCUMENT_BYTES, + needsRenderedArtifact, +} from '@/lib/uploads/utils/file' +import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' +import { + observeWorkspaceFileDelivery, + reportWorkspaceFileDelivery, +} from '@/lib/workspace-files/application/file-delivery-observer' +import { + expandFileDownloadFolders, + normalizeFileDownloadSelection, +} from '@/lib/workspace-files/download-selection' +import { MAX_ZIP_DOWNLOAD_BYTES, MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' +import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query' +import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' +import { + createFileReadReceipt, + type FileReadReceipt, + recheckFileReadReceipt, +} from '@/lib/workspace-files/read-receipt' + +const logger = createLogger('ProjectFileDownloads') + +interface DownloadProjectFileItemsInput extends ProjectFileTarget { + fileIds: string[] + folderIds: string[] +} + +interface DownloadResult { + buffer: Buffer + fileName: string + contentType: 'application/zip' + fileCount: number + secretProvenance: WorkspaceFileSecretProvenance +} + +export interface ExportProjectFileSnapshotInput extends ProjectFileTarget { + fileId: string + content: string + /** Host evidence for an agent's visible snapshot; never accepted by HTTP contracts. */ + secretProvenance?: WorkspaceFileSecretProvenance +} + +async function requireDownloadCapability( + tx: DbTransaction, + principal: Principal, + context: ProjectFileAuthorizationContext, + input: DownloadProjectFileItemsInput +) { + const selection = normalizeFileDownloadSelection(input) + if (selection.fileIds.length === 1 && !selection.folderIds.length) return selection + for (const workspaceId of context.visibleWorkspaceIds) { + // permission-group-enforced: files.bulk_download — all accessible active environments govern an archive. + await assertWorkspaceCapability( + requirePrincipalSubjectUserId(principal), + workspaceId, + 'files.bulk_download', + context.organizationId, + tx + ) + } + return selection +} + +async function snapshotSelection( + tx: DbTransaction, + context: ProjectFileAuthorizationContext, + input: DownloadProjectFileItemsInput +) { + const selection = normalizeFileDownloadSelection(input) + const index = await loadActiveFileFolderPathIndex(context.owner, tx, { + maxRows: MAX_FOLDERS_PER_WORKSPACE, + }) + const folders = [...index.rowById.values()] + const paths = buildWorkspaceFileFolderPathMap(folders) + if (selection.folderIds.some((id) => !index.rowById.has(id))) + throw new OrchestrationError('not_found', 'Folder not found') + const folderIds = expandFileDownloadFolders(selection, folders, paths) + const files = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(context.owner), + isNull(workspaceFiles.deletedAt), + or( + selection.fileIds.length ? inArray(workspaceFiles.id, selection.fileIds) : undefined, + folderIds.size ? inArray(workspaceFiles.folderId, [...folderIds]) : undefined + ) + ) + ) + .orderBy(asc(workspaceFiles.id)) + .limit(MAX_ZIP_DOWNLOAD_FILES + 1) + .for('share') + if (files.length > MAX_ZIP_DOWNLOAD_FILES) + throw new OrchestrationError( + 'validation', + `Too many files selected for download. Select ${MAX_ZIP_DOWNLOAD_FILES} or fewer files.` + ) + const ids = new Set(files.map((file) => file.id)) + if (selection.fileIds.some((id) => !ids.has(id))) + throw new OrchestrationError('not_found', 'File not found') + if (!files.length) throw new OrchestrationError('validation', 'No files selected for download') + const bytes = files.reduce((total, file) => total + getWorkspaceFileSize(file), 0) + assertKnownSizeWithinLimit(bytes, MAX_ZIP_DOWNLOAD_BYTES, 'selected files') + const identity = JSON.stringify({ + folders: folders + .filter((folder) => folderIds.has(folder.id)) + .sort((left, right) => compareStrings(left.id, right.id)) + .map((folder) => [folder.id, folder.name, folder.parentId]), + files: files.map((file) => [ + file.id, + file.key, + file.contentUpdatedAt.getTime(), + file.originalName, + file.folderId, + file.folderId ? paths.get(file.folderId) : null, + ]), + }) + const provenance = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, files) + return { files, paths, identity, provenance } +} + +const snapshotDownload = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.downloadItems, + DownloadProjectFileItemsInput, + Awaited> +>({ + operation: projectFileOperations.downloadItems, + async execute({ principal, input, context, tx }) { + await requireDownloadCapability(tx, principal, context, input) + return snapshotSelection(tx, context, input) + }, +}) + +interface PreparedDownload { + identity: string + receipts: FileReadReceipt[] + buffer: Buffer + fileCount: number +} + +/** A bounded archive is published only after the selected tree and all rendered inputs remain current. */ +const downloadProjectFileItemsUseCase = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.downloadItems, + DownloadProjectFileItemsInput, + DownloadResult, + PreparedDownload +>({ + operation: projectFileOperations.downloadItems, + async prepare({ principal, input, context }) { + try { + const selection = await snapshotDownload.execute({ principal, input }) + const zip = new JSZip() + const receipts: FileReadReceipt[] = [] + const paths = buildZipEntryPaths( + selection.files.map((file) => ({ + name: file.originalName, + contentType: file.contentType, + folderPath: file.folderId ? selection.paths.get(file.folderId) : null, + })) + ) + let remaining = MAX_ZIP_DOWNLOAD_BYTES + for (const [index, file] of selection.files.entries()) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: file.id, + provenance: selection.provenance.get(file.id) ?? { status: 'unknown' }, + view: 'opaque', + }) + let buffer: Buffer + if ( + needsRenderedArtifact(file.contentType, file.originalName) || + file.contentType === SIM_PAGE_CONTENT_TYPE || + file.originalName.toLowerCase().endsWith('.html') + ) { + const rendered = await observeWorkspaceFileDelivery( + async () => {}, + () => + readProjectFileArtifact.execute({ + principal, + input: { + projectId: context.projectId, + fileId: file.id, + maxBytes: Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES), + }, + }) + ) + buffer = rendered.buffer + receipts.push(rendered.receipt) + } else { + buffer = await downloadFile({ key: file.key, context: 'project', maxBytes: remaining }) + receipts.push(createFileReadReceipt(context.owner, [file])) + } + assertKnownSizeWithinLimit(buffer.length, remaining, 'selected files') + remaining -= buffer.length + zip.file(paths[index], buffer) + } + return { + identity: selection.identity, + receipts, + fileCount: selection.files.length, + buffer: await bufferZipWithinLimit(zip, MAX_ZIP_DOWNLOAD_BYTES), + } + } catch (error) { + throw asOrchestrationError(error) ?? error + } + }, + async execute({ principal, input, context, tx, prepared }) { + if (!prepared) throw new Error('Prepared archive is unavailable') + await requireDownloadCapability(tx, principal, context, input) + const selection = await snapshotSelection(tx, context, input) + if (selection.identity !== prepared.identity) + throw new OrchestrationError( + 'conflict', + 'File selection changed while preparing the download' + ) + const evidence: WorkspaceFileSecretProvenance[] = [] + for (const receipt of prepared.receipts) + evidence.push(await recheckFileReadReceipt(tx, receipt)) + return { + buffer: prepared.buffer, + fileName: 'project-files.zip', + contentType: 'application/zip', + fileCount: prepared.fileCount, + secretProvenance: mergeWorkspaceFileSecretProvenance(...evidence), + } + }, + afterSuccess: async ({ principal, input, result }) => { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: input.projectId, + provenance: result.secretProvenance, + view: 'opaque', + }) + await reportWorkspaceFileDelivery(result.secretProvenance) + }, + projectAudit: ({ context, result }) => ({ + action: AuditAction.FILE_DOWNLOADED, + resourceType: AuditResourceType.FILE, + description: `Downloaded ${result.fileCount} Project files as zip`, + metadata: { + projectId: context.projectId, + fileCount: result.fileCount, + totalBytes: result.buffer.length, + }, + }), +}) + +/** HEAD validates the same selected resources and capability without preparing archive bytes. */ +export const downloadProjectFileItems = { + ...downloadProjectFileItemsUseCase, + async authorize(args: Parameters[0]) { + await snapshotDownload.execute(args) + }, +} + +interface SnapshotAssets { + source: WorkspaceFileRow + provenance: Map + assets: { reference: string; file: WorkspaceFileRow }[] +} + +const snapshotMarkdownAssets = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.exportSnapshot, + ExportProjectFileSnapshotInput, + SnapshotAssets +>({ + operation: projectFileOperations.exportSnapshot, + async execute({ input, context, tx }) { + const source = context.file + if (!source) throw new OrchestrationError('not_found', 'File not found') + if ( + !isMarkdownFile({ name: source.originalName, type: source.contentType }) && + source.contentType !== 'text/x-markdown' + ) + throw new OrchestrationError('validation', 'Only Markdown files support snapshot export') + if ( + utf8ByteLength(input.content, PASTE_LIMITS.RICH_MARKDOWN_BYTES) > + PASTE_LIMITS.RICH_MARKDOWN_BYTES + ) + throw new OrchestrationError('validation', 'Markdown snapshot is too large') + const references = extractEmbeddedFileRefs(input.content, context.owner) + if (!references.ids.length && !references.keys.length) + return { + source, + assets: [], + provenance: await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [source]), + } + const rows = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(context.owner), + isNull(workspaceFiles.deletedAt), + or( + references.ids.length + ? inArray(workspaceFiles.id, references.ids.map(storedFileId)) + : undefined, + references.keys.length ? inArray(workspaceFiles.key, references.keys) : undefined + ) + ) + ) + .orderBy(asc(workspaceFiles.id)) + .for('share') + const byId = new Map(rows.map((file) => [file.id, file])) + const byKey = new Map(rows.map((file) => [file.key, file])) + const assets: SnapshotAssets['assets'] = [] + for (const reference of references.ids) { + const file = byId.get(storedFileId(reference)) + if (file) assets.push({ reference, file }) + } + for (const reference of references.keys) { + const file = byKey.get(reference) + if (file) assets.push({ reference, file }) + } + return { + source, + assets, + provenance: await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [source, ...rows]), + } + }, +}) + +interface PreparedSnapshot { + receipt: FileReadReceipt + export: MarkdownExportResult +} + +/** Exports the visible snapshot without advancing the durable file or collaborative document. */ +export const exportProjectFileSnapshot = defineAuthorizedProjectFileUseCase< + typeof projectFileOperations.exportSnapshot, + ExportProjectFileSnapshotInput, + MarkdownExportResult & { + file: ReturnType + secretProvenance: WorkspaceFileSecretProvenance + }, + PreparedSnapshot +>({ + operation: projectFileOperations.exportSnapshot, + async prepare({ principal, input, context }) { + const snapshot = await snapshotMarkdownAssets.execute({ principal, input }) + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: snapshot.source.id, + provenance: mergeWorkspaceFileSecretProvenance( + snapshot.provenance.get(snapshot.source.id) ?? { status: 'unknown' }, + input.secretProvenance ?? + (principal.kind === 'resource_delegated' + ? { status: 'unknown' } + : { status: 'exact', entries: [] }) + ), + view: 'opaque', + }) + const content = Buffer.from(input.content) + const assets: MarkdownExportAsset[] = [] + const consumed: WorkspaceFileRow[] = [snapshot.source] + let actualBytes = content.length + try { + assertKnownSizeWithinLimit( + content.length + + snapshot.assets.reduce((total, { file }) => total + getWorkspaceFileSize(file), 0), + MAX_EXPORT_TOTAL_BYTES, + 'Markdown export' + ) + for (const asset of snapshot.assets) { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: asset.file.id, + provenance: snapshot.provenance.get(asset.file.id) ?? { status: 'unknown' }, + view: 'opaque', + }) + let buffer: Buffer + try { + buffer = await downloadFile({ + key: asset.file.key, + context: 'project', + maxBytes: MAX_EXPORT_ASSET_BYTES, + }) + } catch (error) { + logger.warn('Skipped unavailable Markdown export asset', { fileId: asset.file.id, error }) + continue + } + actualBytes += buffer.length + assertKnownSizeWithinLimit(actualBytes, MAX_EXPORT_TOTAL_BYTES, 'Markdown export') + assets.push({ + imageId: asset.reference, + key: asset.file.key, + context: 'project', + originalName: asset.file.originalName, + size: buffer.length, + buffer, + }) + consumed.push(asset.file) + } + return { + receipt: createFileReadReceipt(context.owner, consumed), + export: await createMarkdownExport({ + content, + fileName: snapshot.source.originalName, + owner: context.owner, + assets, + }), + } + } catch (error) { + if (error instanceof MarkdownExportSizeError) + throw new OrchestrationError('validation', error.message) + throw asOrchestrationError(error) ?? error + } + }, + async execute({ principal, input, context, tx, prepared }) { + if (!prepared || !context.file) throw new Error('Prepared Markdown export is unavailable') + const evidence = await recheckFileReadReceipt(tx, prepared.receipt) + const secretProvenance = mergeWorkspaceFileSecretProvenance( + evidence, + input.secretProvenance ?? + (principal.kind === 'resource_delegated' + ? { status: 'unknown' } + : { status: 'exact', entries: [] }) + ) + const index = await loadActiveFileFolderPathIndex(context.owner, tx, { + maxRows: MAX_FOLDERS_PER_WORKSPACE, + }) + return { + ...prepared.export, + secretProvenance, + file: mapFileRecord( + context.file, + context.owner, + buildWorkspaceFileFolderPathMap([...index.rowById.values()]) + ), + } + }, + afterSuccess: async ({ principal, result }) => { + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: result.file.id, + provenance: result.secretProvenance, + view: 'opaque', + }) + await reportWorkspaceFileDelivery(result.secretProvenance, result.file.id) + }, + projectAudit: ({ context, result }) => ({ + action: AuditAction.FILE_DOWNLOADED, + resourceType: AuditResourceType.FILE, + resourceId: result.file.id, + resourceName: result.file.name, + description: `Exported Project file "${result.file.name}"`, + metadata: { + projectId: context.projectId, + bytes: result.buffer.length, + format: result.format, + assetCount: result.assetCount, + }, + }), +}) diff --git a/apps/sim/lib/projects/files/application/extract.ts b/apps/sim/lib/projects/files/application/extract.ts new file mode 100644 index 00000000000..19ac3d0da86 --- /dev/null +++ b/apps/sim/lib/projects/files/application/extract.ts @@ -0,0 +1,240 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { db } from '@sim/db' +import { workspaceFiles } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { describeError, isPostgresCommitRejection } from '@sim/utils/errors' +import { and, eq, isNull } from 'drizzle-orm' +import { maybeNotifyStorageLimitForBillingContext } from '@/lib/billing/storage/tracking' +import { + type AuthorizingUseCase, + recordProjectedUseCaseAuditEntries, +} from '@/lib/core/application/authorized-workspace-use-case' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import { + createProjectFileAuthorizer, + type ProjectFileAuthorizationContext, +} from '@/lib/projects/files/application/authorization' +import { projectFileOperations } from '@/lib/projects/files/application/operations' +import { prepareProjectFileAccounting } from '@/lib/projects/files/application/write-effects' +import { notifyFileListChanged } from '@/lib/realtime/notify' +import { + archiveFolderName, + MAX_ARCHIVE_BYTES, + prepareArchiveExtraction, +} from '@/lib/uploads/archive' +import { resolveFileFolderTarget } from '@/lib/uploads/contexts/workspace/file-folder-manager' +import { getBoundWorkspaceFileSecretProvenanceByMetadata } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' +import { isArchiveFileName } from '@/lib/uploads/utils/file' +import { + commitFileArchiveInTx, + discardFileArchive, + stageFileArchive, +} from '@/lib/workspace-files/archive-extraction' +import { + FILE_EXTRACTION_BUDGET_MS, + requireFileExtractionLeaseInTx, + withFileExtractionLease, +} from '@/lib/workspace-files/extraction-lease' +import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' +import { lockFileDirectories } from '@/lib/workspace-files/locks' +import { fileOwnerCondition } from '@/lib/workspace-files/ownership-query' + +const logger = createLogger('ProjectFileExtraction') + +interface ExtractProjectFileInput { + projectId: string + fileId: string +} + +async function snapshotArchive( + tx: DbTransaction, + context: ProjectFileAuthorizationContext, + fileId: string +) { + await lockFileDirectories(tx, [context.owner]) + const [file] = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(context.owner), + eq(workspaceFiles.id, fileId), + isNull(workspaceFiles.deletedAt) + ) + ) + .for('update') + .limit(1) + if (!file) throw new OrchestrationError('not_found', 'File not found') + const provenance = (await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, [file])).get( + file.id + ) + if (!provenance) throw new Error('Archive provenance is unavailable') + const parent = await resolveFileFolderTarget(context.owner, { folderId: file.folderId }, tx) + const parentSegments = parent ? parseWorkspaceFileFolderDisplayPath(parent.path) : [] + return { + file, + provenance, + parentSegments, + identity: JSON.stringify({ + key: file.key, + name: file.originalName, + size: getWorkspaceFileSize(file), + contentType: file.contentType, + contentUpdatedAt: file.contentUpdatedAt, + updatedAt: file.updatedAt, + folderId: file.folderId, + parentSegments, + provenance, + }), + } +} + +/** Stages bounded archive members, then atomically publishes their tree under fresh owner authority. */ +export const extractProjectFile: AuthorizingUseCase< + typeof projectFileOperations.extractArchive, + ExtractProjectFileInput, + Awaited> +> = { + operation: projectFileOperations.extractArchive, + delegationAudience: projectFileOperations.extractArchive.delegationAudience, + async authorize(args) { + const authorize = await createProjectFileAuthorizer( + args.principal, + projectFileOperations.extractArchive, + args.input + ) + await db.transaction(authorize) + }, + async execute(args) { + const input = structuredClone(args.input) + const authorize = await createProjectFileAuthorizer( + args.principal, + projectFileOperations.extractArchive, + input + ) + // actorless-unsupported: Project file authorization requires the original human subject. + const userId = requirePrincipalSubjectUserId(args.principal) + const prepared = await db.transaction(async (tx) => { + const context = await authorize(tx) + return { context, snapshot: await snapshotArchive(tx, context, input.fileId) } + }) + const { file } = prepared.snapshot + if (!isArchiveFileName(file.originalName)) + throw new OrchestrationError('validation', 'Only .zip files can be unzipped') + if (getWorkspaceFileSize(file) > MAX_ARCHIVE_BYTES) + throw new OrchestrationError('payload_too_large', 'Archive exceeds the unzip limit') + return withFileExtractionLease(prepared.context.owner, file.id, async (lease) => { + const deadline = AbortSignal.timeout(FILE_EXTRACTION_BUDGET_MS) + const signal = args.request?.signal + ? AbortSignal.any([deadline, args.request.signal]) + : deadline + try { + const staged = await runWithOutboundOrganization( + prepared.context.organizationId, + async () => { + const bytes = await downloadFile({ + key: file.key, + context: 'project', + maxBytes: MAX_ARCHIVE_BYTES, + signal, + }) + if (bytes.length !== getWorkspaceFileSize(file)) + throw new OrchestrationError('conflict', 'Archive bytes changed; retry') + const plan = await prepareArchiveExtraction(bytes, { + rootFolderSegments: [ + ...prepared.snapshot.parentSegments, + archiveFolderName(file.originalName), + ], + includeRootFolder: true, + skipNoiseEntries: true, + signal, + }) + if (!plan.entryCount) + throw new OrchestrationError( + 'validation', + 'No files could be unzipped from this archive' + ) + return stageFileArchive(plan, { owner: prepared.context.owner, userId, signal }) + } + ) + let callbackCompleted = false + let committed + try { + committed = await db.transaction(async (tx) => { + signal.throwIfAborted() + const context = await authorize(tx) + await requireFileExtractionLeaseInTx(tx, lease) + const accounting = await prepareProjectFileAccounting(tx, context) + const current = await snapshotArchive(tx, context, file.id) + if (current.identity !== prepared.snapshot.identity) + throw new OrchestrationError('conflict', 'Archive or destination changed; retry') + const result = await commitFileArchiveInTx(tx, { + owner: context.owner, + userId, + rootName: archiveFolderName(current.file.originalName), + parentId: current.file.folderId, + parentSegments: current.parentSegments, + staged, + secretProvenance: current.provenance, + signal, + }) + const usage = await accounting.mutation.applyDelta(staged.bytes) + signal.throwIfAborted() + await requireFileExtractionLeaseInTx(tx, lease) + callbackCompleted = true + return { context, result, billing: accounting.billing, usage } + }) + } catch (error) { + if (callbackCompleted && !isPostgresCommitRejection(error)) { + logger.error('Archive commit outcome is uncertain; retaining staged content', { + error: describeError(error), + fileId: file.id, + }) + throw error + } + try { + await discardFileArchive(staged) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + 'Archive publication and cleanup failed' + ) + } + throw error + } + await notifyFileListChanged(committed.context.owner) + recordProjectedUseCaseAuditEntries( + projectFileOperations.extractArchive, + null, + args.principal, + args.request, + [ + { + action: AuditAction.FILE_UPDATED, + resourceType: AuditResourceType.FILE, + resourceId: file.id, + resourceName: file.originalName, + description: `Unzipped Project file "${file.originalName}"`, + metadata: { projectId: committed.context.projectId, ...committed.result }, + }, + ], + committed.context.organizationId ?? undefined + ) + await maybeNotifyStorageLimitForBillingContext(committed.billing, committed.usage) + return committed.result + } catch (error) { + if (deadline.aborted && error === deadline.reason) + throw new OrchestrationError( + 'payload_too_large', + 'Unzipping took too long and was cancelled. Try a smaller archive.' + ) + throw error + } + }) + }, +} diff --git a/apps/sim/lib/projects/files/application/index.ts b/apps/sim/lib/projects/files/application/index.ts index 6909e95ef84..60afb052ef8 100644 --- a/apps/sim/lib/projects/files/application/index.ts +++ b/apps/sim/lib/projects/files/application/index.ts @@ -5,6 +5,8 @@ export { downloadProjectFileContent, updateProjectFileContent, } from './content' +export { downloadProjectFileItems, exportProjectFileSnapshot } from './downloads' +export { extractProjectFile } from './extract' export { createProjectFileFolder, listProjectFileFolders, updateProjectFileFolder } from './folders' export { archiveProjectFileItems, diff --git a/apps/sim/lib/uploads/archive.ts b/apps/sim/lib/uploads/archive.ts index e6924847366..cf182d56cb1 100644 --- a/apps/sim/lib/uploads/archive.ts +++ b/apps/sim/lib/uploads/archive.ts @@ -154,17 +154,27 @@ type InflateResult = const inflateEntryWithinCaps = ( entry: JSZip.JSZipObject, remainingTotalBudget: number, - retain: boolean + retain: boolean, + signal?: AbortSignal ): Promise => new Promise((resolve, reject) => { + signal?.throwIfAborted() const chunks: Buffer[] = [] let size = 0 let settled = false const stream = entry.nodeStream() as Readable + const abort = () => { + if (settled) return + settled = true + stream.destroy() + reject(signal?.reason) + } + signal?.addEventListener('abort', abort, { once: true }) const settle = (result: InflateResult) => { if (settled) return settled = true + signal?.removeEventListener('abort', abort) stream.destroy() resolve(result) } @@ -187,6 +197,7 @@ const inflateEntryWithinCaps = ( stream.on('error', () => { if (settled) return settled = true + signal?.removeEventListener('abort', abort) stream.destroy() // A stream error here means the entry's compressed data is corrupt or // truncated (it passed the central-directory parse) — surface it under the @@ -258,70 +269,43 @@ function throwInflateCapError(reason: 'entry' | 'total', entryName: string): nev ) } -/** - * Decompress an archive buffer into workspace files under `rootFolderSegments` - * (default: the workspace root). Reuses the same caps and zip-slip / zip-bomb / - * symlink guards everywhere. Throws {@link ArchiveError} for an invalid archive - * or a cap violation; returns `{ extracted: [] }` when every entry was skipped. - * - * All-or-nothing: pass 1 inflates every entry through a discarding counting sink - * to enforce the per-entry and total caps on REAL inflated bytes (declared sizes - * can lie), and nothing is uploaded until the whole archive has passed — so a - * mid-archive violation never leaves a partial tree in the workspace. Pass 2 - * re-inflates and uploads one entry at a time. Peak memory stays ~one entry in - * both passes; the cost is inflating twice (CPU only, bounded by the caps). - * - * `signal` aborts between entries in both passes. Callers that hold a lease or run - * under a request deadline must pass one: the write loop is otherwise unbounded, and a - * process killed mid-pass-2 strands a partial tree that no `catch` can roll back. - * Aborting instead unwinds through the same all-or-nothing rollback as any other failure. - * - * When `prepareRootFolder` is provided it takes precedence over - * `rootFolderSegments`: it runs once, only after the caps have been proven and - * only when at least one safe entry exists, and extraction lands under the - * segments it returns. It must create exactly one folder, and must pass its - * final segments to the supplied validator before inserting it so the complete - * destination path is rejected before any folder mutation. That one folder is - * counted by the `maxMaterializedItems` pre-check (files + implied folders + - * root folder), which rejects an over-limit archive before the callback - * materializes anything. That cap always applies — it defaults to - * {@link MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS} — because the file count alone - * does not bound folder creation. - * - * Filesystem-noise entries (`__MACOSX/`, `.DS_Store`, `Thumbs.db`) are extracted - * verbatim unless `skipNoiseEntries` is set — the HTTP decompress route preserves - * them; the agent-facing extract path drops them. Decompression is not byte-preserving, - * so every extracted file inherits the archive's full candidate set. Runtime consumers redact - * decoded content; direct opaque delivery still refuses known secret contributions. - */ -export async function decompressArchiveBufferToWorkspaceFiles( +/** Derives a safe, human-readable extraction root from the archive name. */ +export function archiveFolderName(fileName: string): string { + const stripped = fileName + .replace(/\.zip$/i, '') + .normalize('NFC') + .replace(/[\x00-\x1f\x7f]/g, '') + .replace(/[/\\]/g, '-') + .trim() + return stripped && stripped !== '.' && stripped !== '..' ? stripped : 'archive' +} + +export interface PreparedArchiveExtraction { + readonly entryCount: number + readonly skipped: number + readonly skippedUnsafePaths: readonly string[] + validateRootFolderSegments(segments: readonly string[]): void + entries(signal?: AbortSignal): AsyncGenerator<{ segments: readonly string[]; buffer: Buffer }> +} + +/** Validates one bounded archive before any destination mutation or storage upload. */ +export async function prepareArchiveExtraction( buffer: Buffer, - opts: { - workspaceId: string - principal: Principal - rootFolderSegments?: string[] - prepareRootFolder?: ( - validateRootFolderSegments: (rootFolderSegments: string[]) => void - ) => Promise + options: { + rootFolderSegments?: readonly string[] + includeRootFolder?: boolean signal?: AbortSignal maxMaterializedItems?: number skipNoiseEntries?: boolean - secretProvenance?: WorkspaceFileSecretProvenance - notifyWorkspaceChange?: boolean - } -): Promise { + } = {} +): Promise { const { - workspaceId, - principal, rootFolderSegments = [], - prepareRootFolder, + includeRootFolder = false, signal, maxMaterializedItems = MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS, skipNoiseEntries = false, - secretProvenance = { status: 'unknown' }, - notifyWorkspaceChange = true, - } = opts - + } = options assertCentralDirWithinCaps(buffer) let zip: JSZip @@ -362,7 +346,7 @@ export async function decompressArchiveBufferToWorkspaceFiles( ) } - const validateRootFolderSegments = (candidateRootFolderSegments: string[]): void => { + const validateRootFolderSegments = (candidateRootFolderSegments: readonly string[]): void => { for (const { entry, segments } of safeEntries) { try { buildFolderPath([...candidateRootFolderSegments, ...segments.slice(0, -1)]) @@ -388,13 +372,13 @@ export async function decompressArchiveBufferToWorkspaceFiles( } /** * Bounds the whole output tree, not just the file count: 1000 entries nested 64 deep imply - * far more folders than files, and nothing else caps folder creation. `prepareRootFolder` + * far more folders than files, and nothing else caps folder creation. `includeRootFolder` * contributes exactly one more folder when it runs. */ const materializedItems = safeEntries.length + impliedFolderPaths.size + - (safeEntries.length > 0 && prepareRootFolder ? 1 : 0) + (safeEntries.length > 0 && includeRootFolder ? 1 : 0) if (materializedItems > maxMaterializedItems) { throw new ArchiveError( 'too_many_entries', @@ -420,14 +404,112 @@ export async function decompressArchiveBufferToWorkspaceFiles( const result = await inflateEntryWithinCaps( entry, MAX_ARCHIVE_TOTAL_BYTES - validatedTotal, - false + false, + signal ) if (!result.ok) throwInflateCapError(result.reason, entry.name) validatedTotal += result.size } + return Object.freeze({ + entryCount: safeEntries.length, + skipped, + skippedUnsafePaths: Object.freeze(skippedUnsafePaths), + validateRootFolderSegments, + async *entries(entrySignal = signal) { + let totalBytes = 0 + for (const { entry, segments } of safeEntries) { + entrySignal?.throwIfAborted() + const result = await inflateEntryWithinCaps( + entry, + MAX_ARCHIVE_TOTAL_BYTES - totalBytes, + true, + entrySignal + ) + if (!result.ok) throwInflateCapError(result.reason, entry.name) + if (!result.buffer) throw new Error('Archive entry bytes are unavailable') + totalBytes += result.size + yield { segments: Object.freeze(segments), buffer: result.buffer } + } + }, + }) +} + +/** + * Decompress an archive buffer into workspace files under `rootFolderSegments` + * (default: the workspace root). Reuses the same caps and zip-slip / zip-bomb / + * symlink guards everywhere. Throws {@link ArchiveError} for an invalid archive + * or a cap violation; returns `{ extracted: [] }` when every entry was skipped. + * + * All-or-nothing: pass 1 inflates every entry through a discarding counting sink + * to enforce the per-entry and total caps on REAL inflated bytes (declared sizes + * can lie), and nothing is uploaded until the whole archive has passed — so a + * mid-archive violation never leaves a partial tree in the workspace. Pass 2 + * re-inflates and uploads one entry at a time. Peak memory stays ~one entry in + * both passes; the cost is inflating twice (CPU only, bounded by the caps). + * + * `signal` aborts between entries in both passes. Callers that hold a lease or run + * under a request deadline must pass one: the write loop is otherwise unbounded, and a + * process killed mid-pass-2 strands a partial tree that no `catch` can roll back. + * Aborting instead unwinds through the same all-or-nothing rollback as any other failure. + * + * When `prepareRootFolder` is provided it takes precedence over + * `rootFolderSegments`: it runs once, only after the caps have been proven and + * only when at least one safe entry exists, and extraction lands under the + * segments it returns. It must create exactly one folder, and must pass its + * final segments to the supplied validator before inserting it so the complete + * destination path is rejected before any folder mutation. That one folder is + * counted by the `maxMaterializedItems` pre-check (files + implied folders + + * root folder), which rejects an over-limit archive before the callback + * materializes anything. That cap always applies — it defaults to + * {@link MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS} — because the file count alone + * does not bound folder creation. + * + * Filesystem-noise entries (`__MACOSX/`, `.DS_Store`, `Thumbs.db`) are extracted + * verbatim unless `skipNoiseEntries` is set — the HTTP decompress route preserves + * them; the agent-facing extract path drops them. Decompression is not byte-preserving, + * so every extracted file inherits the archive's full candidate set. Runtime consumers redact + * decoded content; direct opaque delivery still refuses known secret contributions. + */ +export async function decompressArchiveBufferToWorkspaceFiles( + buffer: Buffer, + opts: { + workspaceId: string + principal: Principal + rootFolderSegments?: string[] + prepareRootFolder?: ( + validateRootFolderSegments: (rootFolderSegments: string[]) => void + ) => Promise + signal?: AbortSignal + maxMaterializedItems?: number + skipNoiseEntries?: boolean + secretProvenance?: WorkspaceFileSecretProvenance + notifyWorkspaceChange?: boolean + } +): Promise { + const { + workspaceId, + principal, + rootFolderSegments = [], + prepareRootFolder, + signal, + maxMaterializedItems = MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS, + skipNoiseEntries = false, + secretProvenance = { status: 'unknown' }, + notifyWorkspaceChange = true, + } = opts + + const prepared = await prepareArchiveExtraction(buffer, { + rootFolderSegments, + includeRootFolder: Boolean(prepareRootFolder), + signal, + maxMaterializedItems, + skipNoiseEntries, + }) + const { validateRootFolderSegments, skipped } = prepared + const skippedUnsafePaths = [...prepared.skippedUnsafePaths] const resolvedRootFolderSegments = - safeEntries.length > 0 && prepareRootFolder + prepared.entryCount > 0 && prepareRootFolder ? await prepareRootFolder(validateRootFolderSegments) : rootFolderSegments // Re-check what the callback actually returned; identical segments were proven above. @@ -447,14 +529,9 @@ export async function decompressArchiveBufferToWorkspaceFiles( const createdFolderIds: string[] = [] const createdFiles: WorkspaceFileRecord[] = [] const extracted: UserFile[] = [] - let totalBytes = 0 try { - for (const { entry, segments } of safeEntries) { + for await (const { segments, buffer: entryBuffer } of prepared.entries(signal)) { signal?.throwIfAborted() - const result = await inflateEntryWithinCaps(entry, MAX_ARCHIVE_TOTAL_BYTES - totalBytes, true) - if (!result.ok) throwInflateCapError(result.reason, entry.name) - totalBytes += result.size - const entryBuffer = result.buffer as Buffer const leafName = segments[segments.length - 1] const folderSegments = [...resolvedRootFolderSegments, ...segments.slice(0, -1)] diff --git a/apps/sim/lib/uploads/contexts/workspace/file-folder-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-folder-manager.ts index 3c7cb2b96f6..cb42467105f 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-folder-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-folder-manager.ts @@ -679,11 +679,21 @@ export async function createWorkspaceFileFolder( return workspaceFolderRecord(folder) } -/** Creates one folder under the caller's owner transaction and the shared hierarchy lock. */ +/** Creates one folder and hydrates its display path for single-item callers. */ export async function createFileFolder( params: CreateFileFolderParams, tx: DbTransaction ): Promise> { + const inserted = await createFileFolderRow(params, tx) + const paths = buildWorkspaceFileFolderPathMap(await selectFileFolderPathRows(params.owner, tx)) + return mapFileFolder(inserted, params.owner, paths) +} + +/** Uses canonical folder validation and locking without rebuilding paths during a bulk mutation. */ +export async function createFileFolderRow( + params: CreateFileFolderParams, + tx: DbTransaction +): Promise { const requestedName = normalizeWorkspaceFileItemName(params.name, 'Folder') await acquireFileFolderMutationLock(tx, params.owner) const parentId = await assertFileFolderTarget(params.owner, params.parentId, tx) @@ -745,8 +755,7 @@ export async function createFileFolder( if (getPostgresErrorCode(error) === '23505') throw new WorkspaceFileFolderConflictError(name) throw error } - const paths = buildWorkspaceFileFolderPathMap(await selectFileFolderPathRows(params.owner, tx)) - return mapFileFolder(inserted, params.owner, paths) + return inserted } /** diff --git a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts index 2b973a65d81..b14ce6e3d62 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts @@ -506,7 +506,7 @@ export interface PlannedFileIdentity { const plannedFileOwners = new WeakMap>() /** Reserves an in-process destination identity so a compound copy can rewrite selected references. */ -function planFileIdentity(owner: EditableFileOwner): PlannedFileIdentity { +export function planFileIdentity(owner: EditableFileOwner): PlannedFileIdentity { const identity = Object.freeze({ id: `wf_${generateShortId()}` }) plannedFileOwners.set(identity, Object.freeze({ ...owner })) return identity diff --git a/apps/sim/lib/uploads/server/markdown-export.ts b/apps/sim/lib/uploads/server/markdown-export.ts index 680c9697609..207253f79f8 100644 --- a/apps/sim/lib/uploads/server/markdown-export.ts +++ b/apps/sim/lib/uploads/server/markdown-export.ts @@ -25,7 +25,7 @@ const logger = createLogger('MarkdownExport') export const MAX_EXPORT_TOTAL_BYTES = 250 * 1024 * 1024 /** Larger documents remain available as exact Markdown without allocating a syntax tree. */ export const MAX_EXPORT_MARKDOWN_PARSE_BYTES = 10 * 1024 * 1024 -const MAX_EXPORT_ASSET_BYTES = 25 * 1024 * 1024 +export const MAX_EXPORT_ASSET_BYTES = 25 * 1024 * 1024 const markdownProcessor = unified() .use(remarkParse) .use(remarkGfm) diff --git a/apps/sim/lib/workspace-files/api/archive-presenter.test.ts b/apps/sim/lib/workspace-files/api/archive-presenter.test.ts new file mode 100644 index 00000000000..7c526896b41 --- /dev/null +++ b/apps/sim/lib/workspace-files/api/archive-presenter.test.ts @@ -0,0 +1,115 @@ +/** @vitest-environment node */ +import { Readable } from 'node:stream' +import { flushMacrotask } from '@sim/testing/helpers' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { storageServiceMock, storageServiceMockFns } from '@sim/testing/mocks/storage-service.mock' +import JSZip from 'jszip' +import { describe, expect, it, vi } from 'vitest' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' + +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) + +import { presentWorkspaceFileArchive } from '@/lib/workspace-files/api/archive-presenter' + +function plan() { + const filesToZip: WorkspaceFileRecord[] = Array.from({ length: 3 }, (_, index) => ({ + id: `file-${index}`, + workspaceId: 'workspace', + name: `${index}.txt`, + key: `workspace/${index}`, + path: `/files/${index}`, + size: 131072, + type: 'text/plain', + uploadedBy: 'creator', + uploadedAt: new Date(0), + updatedAt: new Date(0), + })) + return { + filesToZip, + excludedFileIds: [], + folderPaths: new Map(), + renderedDocuments: new Map(), + declaredBytes: 393216, + } +} + +describe('file archive resource lifetime', () => { + it('keeps only one storage stream open while preserving every entry', async () => { + let active = 0 + let peak = 0 + storageServiceMockFns.mockDownloadFileStream.mockImplementation(async () => { + active++ + peak = Math.max(peak, active) + const source = Readable.from( + (async function* () { + yield Buffer.alloc(65536, 'a') + await flushMacrotask() + yield Buffer.alloc(65536, 'b') + })() + ) + let released = false + const release = () => { + if (!released) { + released = true + active-- + } + } + source.once('end', release) + source.once('close', release) + return source + }) + const result = presentWorkspaceFileArchive(plan()) + const zip = await JSZip.loadAsync(await new Response(result.body).arrayBuffer()) + for (const name of ['0.txt', '1.txt', '2.txt']) { + expect(await zip.file(name)?.async('string')).toBe('a'.repeat(65536) + 'b'.repeat(65536)) + } + expect(peak).toBe(1) + expect(active).toBe(0) + }) + + it('rejects partial storage failure and closes the input', async () => { + const source = Readable.from( + (async function* () { + yield Buffer.from('partial') + throw new Error('Storage unavailable') + })() + ) + storageServiceMockFns.mockDownloadFileStream.mockResolvedValue(source) + const result = presentWorkspaceFileArchive(plan()) + await expect(new Response(result.body).arrayBuffer()).rejects.toThrow('Storage unavailable') + expect(source.destroyed).toBe(true) + }) + + it('closes a storage stream acquired after the client cancels', async () => { + const acquiring = createDeferred() + const acquired = createDeferred() + const source = new Readable({ read() {} }) + storageServiceMockFns.mockDownloadFileStream.mockImplementationOnce(() => { + acquiring.resolve() + return acquired.promise + }) + const reader = presentWorkspaceFileArchive(plan()).body.getReader() + try { + await acquiring.promise + await reader.cancel() + acquired.resolve(source) + await vi.waitFor(() => expect(source.destroyed).toBe(true)) + } finally { + acquired.resolve(source) + source.destroy() + } + }) + + it('closes the active source when the client cancels', async () => { + const source = new Readable({ + read() { + this.push(Buffer.alloc(65536)) + }, + }) + storageServiceMockFns.mockDownloadFileStream.mockResolvedValue(source) + const reader = presentWorkspaceFileArchive(plan()).body.getReader() + await reader.read() + await reader.cancel() + await vi.waitFor(() => expect(source.destroyed).toBe(true)) + }) +}) diff --git a/apps/sim/lib/workspace-files/api/archive-presenter.ts b/apps/sim/lib/workspace-files/api/archive-presenter.ts new file mode 100644 index 00000000000..ce628926fa4 --- /dev/null +++ b/apps/sim/lib/workspace-files/api/archive-presenter.ts @@ -0,0 +1,95 @@ +import { once } from 'node:events' +import { addAbortSignal, Readable } from 'node:stream' +import { finished } from 'node:stream/promises' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { ZipArchive } from 'archiver' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' +import { downloadFileStream } from '@/lib/uploads/core/storage-service' +import { FILE_CACHE_CONTROL, fileDeliveryHeaders } from '@/lib/uploads/server/delivery' +import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' +import type { DownloadWorkspaceFileItemsResult } from '@/lib/workspace-files/application/download-workspace-file-items' + +const logger = createLogger('FileArchiveDelivery') + +/** Stream an authorized archive plan, opening at most the current entry rather than buffering files. */ +export function presentWorkspaceFileArchive({ + filesToZip, + folderPaths, + renderedDocuments, + excludedFileIds = [], +}: DownloadWorkspaceFileItemsResult) { + const report = excludedFileIds.length > 0 ? { partial: true, excludedFileIds } : undefined + const entryPaths = buildZipEntryPaths([ + ...filesToZip.map((file) => ({ + name: file.name, + folderPath: file.folderId ? folderPaths.get(file.folderId) : null, + contentType: file.type, + })), + ...(report ? [{ name: 'sim-download-report.json', contentType: 'application/json' }] : []), + ]) + const archive = new ZipArchive({ store: true }) + archive.on('warning', (error: Error) => logger.warn('Archive warning', { error })) + const closed = new AbortController() + let activeInput: Readable | undefined + archive.once('close', () => { + closed.abort() + activeInput?.destroy() + }) + async function appendEntries() { + for (const [index, file] of filesToZip.entries()) { + closed.signal.throwIfAborted() + const rendered = renderedDocuments.get(file.id) + const input = + rendered ?? + Readable.from( + (async function* () { + const source = addAbortSignal( + closed.signal, + await downloadFileStream({ + key: file.key, + context: file.storageContext ?? 'workspace', + }) + ) + try { + yield* source + } finally { + source.destroy() + } + })(), + { objectMode: false } + ) + activeInput = input instanceof Readable ? input : undefined + const consumed = once(archive, 'entry', { signal: closed.signal }) + const sourceFinished = activeInput + ? finished(activeInput, { readable: true, writable: false, cleanup: true }) + : undefined + try { + archive.append(input, { name: entryPaths[index] }) + await Promise.all([consumed, sourceFinished]) + } finally { + activeInput?.destroy() + activeInput = undefined + } + } + if (report) { + archive.append(JSON.stringify(report, null, 2), { name: entryPaths[filesToZip.length] }) + } + await archive.finalize() + } + appendEntries().catch((error: unknown) => { + if (toError(error).name === 'AbortError') return + logger.error('Failed to build file archive', { error }) + archive.destroy(toError(error)) + }) + return { + body: nodeReadableToWebStream(archive), + contentType: 'application/zip', + headers: fileDeliveryHeaders({ + filename: 'workspace-files.zip', + contentType: 'application/zip', + attachment: true, + cacheControl: FILE_CACHE_CONTROL.noStore, + }), + } +} diff --git a/apps/sim/lib/workspace-files/api/copy-presenter.ts b/apps/sim/lib/workspace-files/api/copy-presenter.ts new file mode 100644 index 00000000000..c15e5da753b --- /dev/null +++ b/apps/sim/lib/workspace-files/api/copy-presenter.ts @@ -0,0 +1,27 @@ +import type { CopyFileItemsResponse } from '@/lib/api/contracts/file-copy' +import { workspaceFileRevision } from '@/lib/workspace-files/application/file-revision' +import type { CopiedFileItems } from '@/lib/workspace-files/copy' + +/** Both copy surfaces expose new canonical identities without object-storage addresses. */ +export function presentCopiedFileItems(result: CopiedFileItems): CopyFileItemsResponse { + return { + files: result.files.map(({ key, path, ...file }) => { + const revision = workspaceFileRevision(file) + if (revision === null) throw new Error('Copied file is missing its content revision') + return { + ...file, + uploadedAt: file.uploadedAt.toISOString(), + updatedAt: file.updatedAt.toISOString(), + contentUpdatedAt: file.contentUpdatedAt?.toISOString() ?? null, + deletedAt: file.deletedAt?.toISOString() ?? null, + revision, + } + }), + folders: result.folders.map((folder) => ({ + ...folder, + createdAt: folder.createdAt.toISOString(), + updatedAt: folder.updatedAt.toISOString(), + deletedAt: folder.deletedAt?.toISOString() ?? null, + })), + } +} diff --git a/apps/sim/lib/workspace-files/api/index.ts b/apps/sim/lib/workspace-files/api/index.ts index 8943f1051c3..144afebaf0e 100644 --- a/apps/sim/lib/workspace-files/api/index.ts +++ b/apps/sim/lib/workspace-files/api/index.ts @@ -6,4 +6,5 @@ export { internalWorkspaceFileServeAuth, v2FileErrorPolicies, } from '@/lib/workspace-files/api/route-policies' -export { toFileVersion } from '@/lib/workspace-files/api/version-presenters' +export { presentWorkspaceFileArchive } from './archive-presenter' +export { toFileVersion } from './version-presenters' diff --git a/apps/sim/lib/workspace-files/application/copy-authorization.ts b/apps/sim/lib/workspace-files/application/copy-authorization.ts new file mode 100644 index 00000000000..a7a185bb07b --- /dev/null +++ b/apps/sim/lib/workspace-files/application/copy-authorization.ts @@ -0,0 +1,253 @@ +import type { Principal, ResourceFileCopyScope } from '@sim/auth/principal' +import { workspace } from '@sim/db/schema' +import { compareStrings } from '@sim/utils/string' +import { and, asc, inArray, isNull } from 'drizzle-orm' +import { requireOAuthOperationScope } from '@/lib/core/application/oauth-authorization' +import { + isResourceFileCopyScope, + requireResourceDelegation, +} from '@/lib/core/application/resource-delegation' +import { + authorizeWorkspaceOperation, + PrincipalKindAuthorizationError, + requireCurrentHumanRole, +} from '@/lib/core/application/workspace-authorization' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' +import { acquirePermissionGroupOrgLock } from '@/lib/permission-groups/locks' +import { loadProjectAccess } from '@/lib/projects/application/authorization' +import { resolveCopilotProjectScope } from '@/lib/projects/application/discovery' +import { + type ProjectFileAuthorizationContext, + requireCurrentCopilotProjectInvocation, + requireProjectFileOwnerCapabilities, + requireProjectFileOwnerRole, +} from '@/lib/projects/files/application/authorization' +import { lockProject, lockProjectBackfillWrites } from '@/lib/projects/membership' +import { requireProjectApiEnabled } from '@/lib/projects/rollout.server' +import { + FILE_COPY_DELEGATION_TTL_MS, + fileCopyOperation, +} from '@/lib/workspace-files/application/copy-operation' +import { fileOperations } from '@/lib/workspace-files/application/operations' +import { + requireCurrentFileSubject, + requireFileSubject, +} from '@/lib/workspace-files/application/subject' +import { + type FileOwnerAdapters, + requireFileOwnerAdapter, +} from '@/lib/workspace-files/owner-adapters' + +export type CopyFileItemsInput = Omit + +type CopyPrincipal = + | Extract + | Extract + +type ProjectAccess = Awaited> + +type FileCopyOwnerContext = + | ProjectFileAuthorizationContext + | { + owner: { entityType: 'workspace'; entityId: string } + workspaceId: string + organizationId: string | null + billedAccountUserId: string + } + +export interface FileCopyAuthorizationContext { + source: FileCopyOwnerContext + destination: FileCopyOwnerContext +} + +interface CopyOwnerPolicyInput { + tx: DbTransaction + principal: CopyPrincipal + userId: string + owner: ResourceFileCopyScope['source']['owner'] + mode: 'read' | 'write' + projects: ReadonlyMap + workspaces: ReadonlyMap +} + +type CopyOwnerPolicy = (input: CopyOwnerPolicyInput) => Promise + +const COPY_OWNER_POLICIES: FileOwnerAdapters = { + async project({ tx, principal, owner, mode, projects }) { + const access = projects.get(owner.entityId) + if (!access) throw new OrchestrationError('not_found', 'Project not found') + const context = await requireProjectFileOwnerRole(tx, principal, access, mode) + await requireProjectFileOwnerCapabilities(tx, principal, access, fileCopyOperation.capability) + return context + }, + async workspace({ tx, principal, userId, owner, mode, workspaces }) { + const row = workspaces.get(owner.entityId) + if (!row) throw new OrchestrationError('not_found', 'Workspace not found') + const context = { + workspaceId: row.id, + workspaceOrganizationId: row.organizationId, + allowPersonalApiKeys: row.allowPersonalApiKeys, + } + // Project access already holds membership and permission SHARE locks; upgrading can deadlock other Projects. + if (principal.kind === 'resource_delegated') { + await requireCurrentHumanRole(userId, context, mode, { executor: tx }) + // permission-group-enforced: files.use — copy delegation retains the subject's workspace file policy. + await assertWorkspaceCapability( + userId, + row.id, + fileCopyOperation.capability, + row.organizationId, + tx + ) + } else { + await authorizeWorkspaceOperation( + principal, + mode === 'read' ? fileOperations.list : fileOperations.create, + context, + { executor: tx } + ) + } + return { + owner: { entityType: 'workspace', entityId: row.id }, + workspaceId: row.id, + organizationId: row.organizationId, + billedAccountUserId: row.billedAccountUserId, + } + }, +} + +function requireCopyPrincipal( + principal: Principal, + input: CopyFileItemsInput +): asserts principal is CopyPrincipal { + if (!fileCopyOperation.principalKinds.some((kind) => kind === principal.kind)) { + throw new PrincipalKindAuthorizationError(principal.kind, fileCopyOperation.id) + } + const scope = { kind: 'file_copy', ...input } + if (!isResourceFileCopyScope(scope)) { + throw new OrchestrationError( + 'validation', + 'Copy requires bounded source and destination owners' + ) + } + requireOAuthOperationScope(principal, fileCopyOperation) + if (principal.kind === 'resource_delegated') { + requireResourceDelegation(principal, { + audience: fileCopyOperation.delegationAudience, + services: fileCopyOperation.delegatedServices, + scope, + maxTtlMs: FILE_COPY_DELEGATION_TTL_MS, + }) + } +} + +/** Authorizes both owners under canonical locks; selected rows remain the copy manager's responsibility. */ +export async function createFileCopyAuthorizer( + principal: Principal, + input: CopyFileItemsInput +): Promise<(tx: DbTransaction) => Promise> { + requireCopyPrincipal(principal, input) + await requireProjectApiEnabled() + // actorless-unsupported: files.copy rejects executors and workspace keys; both owner policies require the acting human. + const userId = await requireFileSubject(principal) + const invocationScope = + principal.kind === 'resource_delegated' + ? await resolveCopilotProjectScope(principal) + : undefined + + return async (tx) => { + requireCopyPrincipal(principal, input) + const actingPrincipal = principal + const owners = [input.source.owner, input.destination.owner] + const workspaceIds = [ + ...new Set( + owners.filter((owner) => owner.entityType === 'workspace').map((owner) => owner.entityId) + ), + ].sort(compareStrings) + await lockProjectBackfillWrites(tx, workspaceIds) + const memberships = workspaceIds.length + ? await tx + .select({ workspaceId: workspace.id, projectId: workspace.projectId }) + .from(workspace) + .where(inArray(workspace.id, workspaceIds)) + : [] + const parents = new Map(memberships.map((row) => [row.workspaceId, row.projectId])) + if (workspaceIds.some((workspaceId) => !parents.has(workspaceId))) { + throw new OrchestrationError('not_found', 'Workspace not found in a Project') + } + const projectIds = [ + ...new Set([ + ...owners.filter((owner) => owner.entityType === 'project').map((owner) => owner.entityId), + ...parents.values(), + ]), + ].sort(compareStrings) + for (const projectId of projectIds) await lockProject(tx, projectId) + const currentMemberships = workspaceIds.length + ? await tx + .select({ workspaceId: workspace.id, projectId: workspace.projectId }) + .from(workspace) + .where(inArray(workspace.id, workspaceIds)) + : [] + if ( + currentMemberships.length !== memberships.length || + currentMemberships.some((row) => parents.get(row.workspaceId) !== row.projectId) + ) { + throw new OrchestrationError('conflict', 'Project membership changed; retry the copy') + } + + const projects = new Map() + for (const projectId of projectIds) { + projects.set(projectId, await loadProjectAccess(tx, userId, { projectId })) + } + const workspaces = workspaceIds.length + ? await tx + .select() + .from(workspace) + .where(and(inArray(workspace.id, workspaceIds), isNull(workspace.archivedAt))) + .orderBy(asc(workspace.id)) + .for('share') + : [] + const workspacesById = new Map(workspaces.map((row) => [row.id, row])) + const organizationIds = [ + ...new Set( + [ + ...[...projects.values()].map((access) => access.record.organizationId), + ...workspaces.map((row) => row.organizationId), + ].filter((id): id is string => id !== null) + ), + ].sort(compareStrings) + for (const organizationId of organizationIds) + await acquirePermissionGroupOrgLock(tx, organizationId) + await requireCurrentFileSubject(tx, principal) + if (principal.kind === 'resource_delegated') { + if (!invocationScope) throw new Error('Copilot copy invocation was not prepared') + for (const access of projects.values()) { + await requireCurrentCopilotProjectInvocation(tx, principal, access, invocationScope) + } + } + + function authorizeOwner( + owner: ResourceFileCopyScope['source']['owner'], + mode: 'read' | 'write' + ): Promise { + return requireFileOwnerAdapter( + COPY_OWNER_POLICIES, + owner + )({ + tx, + principal: actingPrincipal, + userId, + owner, + mode, + projects, + workspaces: workspacesById, + }) + } + + const source = await authorizeOwner(input.source.owner, 'read') + const destination = await authorizeOwner(input.destination.owner, 'write') + return { source, destination } + } +} diff --git a/apps/sim/lib/workspace-files/application/copy-file-items.ts b/apps/sim/lib/workspace-files/application/copy-file-items.ts new file mode 100644 index 00000000000..8e6883b9e44 --- /dev/null +++ b/apps/sim/lib/workspace-files/application/copy-file-items.ts @@ -0,0 +1,240 @@ +import { isUtf8 } from 'node:buffer' +import { AuditAction, AuditResourceType } from '@sim/audit' +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { db } from '@sim/db' +import { createLogger } from '@sim/logger' +import { describeError, isPostgresCommitRejection } from '@sim/utils/errors' +import { prepareFileAccountingInTx } from '@/lib/billing/storage/accounting' +import { maybeNotifyStorageLimitForBillingContext } from '@/lib/billing/storage/tracking' +import { + type AuthorizingUseCase, + recordProjectedUseCaseAuditEntries, + type WorkspaceUseCaseAuditEntry, +} from '@/lib/core/application/authorized-workspace-use-case' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { notifyFileListChanged } from '@/lib/realtime/notify' +import { assertFileFolderTarget } from '@/lib/uploads/contexts/workspace/file-folder-manager' +import { + discardStagedFileContent, + planFileIdentity, + type StagedFileContent, + stageFileContent, +} from '@/lib/uploads/contexts/workspace/file-manager' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { + getDocumentSourceLanguage, + getE2BDocFormat, + isCompiledDocumentBuffer, +} from '@/lib/uploads/documents' +import { getWorkspaceFileSize, MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { isMarkdownFile, isRenderableDocumentName } from '@/lib/uploads/utils/file' +import { + type CopyFileItemsInput, + createFileCopyAuthorizer, +} from '@/lib/workspace-files/application/copy-authorization' +import { fileCopyOperation } from '@/lib/workspace-files/application/copy-operation' +import { + type CopiedFileItems, + commitFileCopyInTx, + requireUnchangedFileCopy, + snapshotFileCopyInTx, +} from '@/lib/workspace-files/copy' +import { rewriteCopiedFileReferences } from '@/lib/workspace-files/copy-references' +import { lockFileDirectories } from '@/lib/workspace-files/locks' + +const logger = createLogger('FileCopy') + +/** Atomic source-read/destination-write copy; staging never holds authorization or accounting locks. */ +export const copyFileItems: AuthorizingUseCase< + typeof fileCopyOperation, + CopyFileItemsInput, + CopiedFileItems +> = { + operation: fileCopyOperation, + delegationAudience: fileCopyOperation.delegationAudience, + async authorize(args) { + const input = structuredClone(args.input) + const authorize = await createFileCopyAuthorizer(args.principal, input) + await db.transaction(async (tx) => { + const context = await authorize(tx) + await lockFileDirectories(tx, [context.source.owner, context.destination.owner]) + await assertFileFolderTarget(context.destination.owner, input.destination.folderId, tx) + await snapshotFileCopyInTx(tx, { ...input.source, owner: context.source.owner }) + }) + }, + async execute(args) { + const input = structuredClone(args.input) + const authorize = await createFileCopyAuthorizer(args.principal, input) + // actorless-unsupported: the copy authorizer has rejected actorless principals; creator attribution retains its human subject. + const userId = requirePrincipalSubjectUserId(args.principal) + const prepared = await db.transaction(async (tx) => { + const context = await authorize(tx) + await lockFileDirectories(tx, [context.source.owner, context.destination.owner]) + await assertFileFolderTarget(context.destination.owner, input.destination.folderId, tx) + return { + context, + snapshot: await snapshotFileCopyInTx(tx, { ...input.source, owner: context.source.owner }), + } + }) + const staged = new Map() + const identities = new Map( + prepared.snapshot.files.map((file) => [ + file.id, + planFileIdentity(prepared.context.destination.owner), + ]) + ) + const fileIds = new Map([...identities].map(([sourceId, identity]) => [sourceId, identity.id])) + const fileKeys = new Map( + prepared.snapshot.files.map((file) => { + const id = fileIds.get(file.id) + if (!id) throw new Error('Copied file identity plan is incomplete') + return [file.key, id] + }) + ) + let stagedBytes = 0 + let callbackCompleted = false + const committed = await (async () => { + try { + for (const source of prepared.snapshot.files) { + const content = await runWithOutboundOrganization( + prepared.context.source.organizationId, + () => + downloadFile({ + key: source.key, + context: prepared.context.source.owner.entityType, + maxBytes: getWorkspaceFileSize(source), + }) + ) + if (content.length !== getWorkspaceFileSize(source)) { + throw new OrchestrationError('conflict', 'Source file size changed during copy') + } + const textual = + source.contentType.startsWith('text/') || + ['application/json', 'application/xml', 'application/javascript'].includes( + source.contentType + ) || + isMarkdownFile({ name: source.originalName, type: source.contentType }) || + isRenderableDocumentName(source.originalName) + let copiedContent = content + if ( + textual && + isUtf8(content) && + !isCompiledDocumentBuffer(source.originalName, content) + ) { + const text = content.toString('utf8') + const format = await getE2BDocFormat(source.originalName) + copiedContent = Buffer.from( + rewriteCopiedFileReferences( + text, + { + sourceOwner: prepared.context.source.owner, + destinationOwner: prepared.context.destination.owner, + fileIds, + fileKeys, + }, + format ? getDocumentSourceLanguage(text, format, source.contentType) : null + ) + ) + } + stagedBytes += copiedContent.length + if (stagedBytes > MAX_BUFFERED_TRANSFER_BYTES) + throw new OrchestrationError( + 'payload_too_large', + 'Copied content exceeds the buffered transfer limit' + ) + staged.set( + source.id, + await runWithOutboundOrganization(prepared.context.destination.organizationId, () => + stageFileContent({ + owner: prepared.context.destination.owner, + userId, + name: source.originalName, + contentType: source.contentType, + content: copiedContent, + }) + ) + ) + } + return await db.transaction(async (tx) => { + const context = await authorize(tx) + const accounting = await prepareFileAccountingInTx(tx, context.destination.owner) + await lockFileDirectories(tx, [context.source.owner, context.destination.owner]) + const current = await snapshotFileCopyInTx(tx, { + ...input.source, + owner: context.source.owner, + }) + requireUnchangedFileCopy(prepared.snapshot, current) + const result = await commitFileCopyInTx(tx, { + snapshot: current, + destination: { owner: context.destination.owner, folderId: input.destination.folderId }, + userId, + staged, + identities, + }) + const usage = await accounting.mutation.applyDelta(stagedBytes) + callbackCompleted = true + return { + context, + result, + billing: accounting.billing, + usage, + } + }) + } catch (error) { + if (callbackCompleted && !isPostgresCommitRejection(error)) { + logger.error('Copy commit outcome is uncertain; retaining staged content', { + error: describeError(error), + owner: prepared.context.destination.owner, + }) + throw error + } + const cleanups = await Promise.allSettled( + [...staged.values()].map(discardStagedFileContent) + ) + const failed = cleanups.find((cleanup) => cleanup.status === 'rejected') + if (failed?.status === 'rejected') { + throw new AggregateError( + [error, failed.reason], + 'Copy failed and durable cleanup could not be recorded' + ) + } + throw error + } + })() + const owner = committed.context.destination.owner + const workspaceId = owner.entityType === 'workspace' ? owner.entityId : null + const metadata = { source: input.source, destination: input.destination } + const entries: WorkspaceUseCaseAuditEntry[] = [ + ...committed.result.files.map((file) => ({ + action: AuditAction.FILE_UPLOADED, + resourceType: AuditResourceType.FILE, + resourceId: file.id, + resourceName: file.name, + metadata, + })), + ...committed.result.folders.map((folder) => ({ + action: AuditAction.FOLDER_CREATED, + resourceType: AuditResourceType.FOLDER, + resourceId: folder.id, + resourceName: folder.name, + metadata, + })), + ] + if (committed.result.files.length || committed.result.folders.length) { + await notifyFileListChanged(committed.context.destination.owner) + } + recordProjectedUseCaseAuditEntries( + fileCopyOperation, + workspaceId, + args.principal, + args.request, + entries, + committed.context.destination.organizationId ?? undefined + ) + if (committed.usage !== undefined) { + await maybeNotifyStorageLimitForBillingContext(committed.billing, committed.usage) + } + return committed.result + }, +} diff --git a/apps/sim/lib/workspace-files/application/copy-operation.ts b/apps/sim/lib/workspace-files/application/copy-operation.ts new file mode 100644 index 00000000000..9e70495fe8f --- /dev/null +++ b/apps/sim/lib/workspace-files/application/copy-operation.ts @@ -0,0 +1,23 @@ +import { + assertOperationCapability, + assertOperationOAuthPolicy, +} from '@/lib/core/application/operation' + +export const FILE_COPY_DELEGATION_TTL_MS = 60_000 + +const policy = { + id: 'files.copy', + capability: 'files.use', + oauthScope: 'api:write', + principalKinds: ['session', 'personal_api_key', 'oauth_access_token', 'resource_delegated'], + delegatedServices: ['copilot'], + delegationAudience: 'sim:files:copy', +} as const + +assertOperationCapability(policy) +assertOperationOAuthPolicy(policy) +Object.freeze(policy.principalKinds) +Object.freeze(policy.delegatedServices) + +/** Compound copies bind independent source-read and destination-write authority. */ +export const fileCopyOperation = Object.freeze(policy) diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts b/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts index 6c4eaad3245..6864a907e6d 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file-items.test.ts @@ -3,6 +3,7 @@ import { createSessionPrincipal, } from '@sim/testing/factories/principal.factory' import { auditMock } from '@sim/testing/mocks/audit.mock' +import { fileReadReceiptMock } from '@sim/testing/mocks/file-read-receipt.mock' import { fileUtilsMock, fileUtilsMockFns } from '@sim/testing/mocks/file-utils.mock' import { permissionGroupsResolveMock, @@ -30,6 +31,8 @@ const { mockListWorkspaceFileFolders: mockListFolders, } = workspaceUploadsMockFns +vi.mock('@/lib/workspace-files/read-receipt', () => fileReadReceiptMock) + vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) vi.mock('@/lib/uploads/contexts/workspace', () => workspaceUploadsMock) @@ -59,6 +62,8 @@ const mockProvenance = import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { downloadWorkspaceFileItems } from '@/lib/workspace-files/application/download-workspace-file-items' +import { MAX_ZIP_DOWNLOAD_BYTES } from '@/lib/workspace-files/limits' +import { recheckFileReadReceipt } from '@/lib/workspace-files/read-receipt' const { mockResolveVerifiedUserAccessControlContext: mockAccessContext } = permissionGroupsResolveMockFns @@ -107,6 +112,7 @@ function file(id: string, name: string, folderId: string | null = null, size = 1 describe('downloadWorkspaceFileItems', () => { beforeEach(() => { events.length = 0 + vi.mocked(recheckFileReadReceipt).mockResolvedValue({ status: 'exact', entries: [] }) mockLoadContext.mockImplementation(async () => { events.push('resolve') return workspace @@ -144,7 +150,7 @@ describe('downloadWorkspaceFileItems', () => { ) expect(result.filesToZip.map((item) => item.id)).toEqual(['f1']) - expect(mockResolvePermission).toHaveBeenCalledOnce() + expect(mockResolvePermission).toHaveBeenCalledTimes(2) expect(mockProvenance).toHaveBeenCalledWith('ws-1', [ expect.objectContaining({ fileId: 'f1', key: 'key-f1' }), ]) @@ -172,38 +178,42 @@ describe('downloadWorkspaceFileItems', () => { expect(result.filesToZip.map((item) => item.id)).toEqual(['f1']) }) - it('counts only the zipped files once a rendered document is left out', async () => { - mockListFiles.mockResolvedValue([ - file('f1', 'clean.txt', null, 10), - { ...file('f2', 'report.docx', null, 25), type: 'text/x-generated-document' }, - ]) - mockIsGenerated.mockImplementation((type) => type === 'text/x-generated-document') - mockFetchServable.mockResolvedValue({ - buffer: Buffer.from('rendered'), - contributingFiles: [{ fileId: 'c1', key: 'key-c1', context: 'workspace' }], - }) - mockProvenance.mockImplementation( - async (_workspaceId: string, identities: readonly { fileId: string }[]) => - new Map( - identities.map((identity) => [ - identity.fileId, - identity.fileId === 'c1' ? { status: 'unknown' } : { status: 'exact', entries: [] }, - ]) - ) - ) - const result = await observeContentDelivery(secretFreeGate, () => - downloadWorkspaceFileItems.execute({ - principal: createDelegatedPrincipal({ - subjectUserId: 'u1', - workspaceId: 'ws-1', - audience: 'sim:workspace-files', - }), - input: { workspaceId: 'ws-1', fileIds: ['f1', 'f2'], folderIds: [] }, + it.each([10, MAX_ZIP_DOWNLOAD_BYTES])( + 'counts only admitted bytes with %i streamed bytes', + async (streamedBytes) => { + mockListFiles.mockResolvedValue([ + file('f1', 'clean.txt', null, streamedBytes), + { ...file('f2', 'report.docx', null, 25), type: 'text/x-generated-document' }, + ]) + mockIsGenerated.mockImplementation((type) => type === 'text/x-generated-document') + mockFetchServable.mockResolvedValue({ + buffer: Buffer.from('rendered'), + contributingFiles: [{ fileId: 'c1', key: 'key-c1', context: 'workspace' }], }) - ) - expect(result.filesToZip.map((item) => item.id)).toEqual(['f1']) - expect(result.declaredBytes).toBe(10) - }) + mockProvenance.mockImplementation( + async (_workspaceId: string, identities: readonly { fileId: string }[]) => + new Map( + identities.map((identity) => [ + identity.fileId, + identity.fileId === 'c1' ? { status: 'unknown' } : { status: 'exact', entries: [] }, + ]) + ) + ) + const result = await observeContentDelivery(secretFreeGate, () => + downloadWorkspaceFileItems.execute({ + principal: createDelegatedPrincipal({ + subjectUserId: 'u1', + workspaceId: 'ws-1', + audience: 'sim:workspace-files', + }), + input: { workspaceId: 'ws-1', fileIds: ['f1', 'f2'], folderIds: [] }, + }) + ) + expect(result.filesToZip.map((item) => item.id)).toEqual(['f1']) + expect(result.declaredBytes).toBe(streamedBytes) + expect(mockFetchServable.mock.calls[0][2].maxBytes).toBeGreaterThan(0) + } + ) it('refuses a Chat archive whose every file its gate refuses', async () => { mockProvenance.mockResolvedValue(new Map([['f1', { status: 'unknown' }]])) @@ -310,6 +320,27 @@ describe('downloadWorkspaceFileItems', () => { ).rejects.toMatchObject({ code: 'validation' }) }) + it('refuses refreshed unsafe provenance even after initial admission succeeds', async () => { + vi.mocked(recheckFileReadReceipt).mockResolvedValue({ status: 'unknown' }) + await expect( + observeContentDelivery(secretFreeGate, () => + downloadWorkspaceFileItems.execute({ + principal: delegatedPrincipal, + input: { workspaceId: 'ws-1', fileIds: ['f1'], folderIds: [] }, + }) + ) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(recheckFileReadReceipt).toHaveBeenCalled() + }) + + it('rejects an unknown folder in a mixed selection', async () => { + await expect( + downloadWorkspaceFileItems.execute({ + principal, + input: { workspaceId: 'ws-1', fileIds: ['f1'], folderIds: ['missing'] }, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + }) describe('permission-group capability', () => { beforeEach(() => { mockAccessContext.mockResolvedValue({ @@ -328,6 +359,15 @@ describe('downloadWorkspaceFileItems', () => { }) }) + it('refuses a HEAD authorization probe when bulk download is withheld', async () => { + await expect( + downloadWorkspaceFileItems.authorize({ + principal, + input: { workspaceId: 'ws-1', fileIds: [], folderIds: ['folder-1'] }, + }) + ).rejects.toMatchObject({ capability: 'files.bulk_download' }) + expect(events).not.toContain('execute') + }) it('refuses a folder archive when the group withholds files.bulk_download', async () => { await expect( downloadWorkspaceFileItems.execute({ diff --git a/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts b/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts index 24326849656..02d0da9d359 100644 --- a/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts +++ b/apps/sim/lib/workspace-files/application/download-workspace-file-items.ts @@ -1,12 +1,14 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import { db } from '@sim/db' +import { compareStrings } from '@sim/utils/string' import { type AuthorizedWorkspaceUseCaseContext, ContentRefusedError, capabilityGovernedPrincipalUserId, } from '@/lib/core/application' +import { requireAdmittedContent } from '@/lib/core/application/content-delivery' import { OrchestrationError } from '@/lib/core/orchestration/types' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { parseFolderPath } from '@/lib/folders/paths' import { assertWorkspaceCapability } from '@/lib/permission-groups/capability-assertions' import { buildWorkspaceFileFolderPathMap, @@ -15,6 +17,7 @@ import { loadWorkspaceFileOperationContext, type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace' +import { mergeWorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' import { formatFileSize, @@ -28,12 +31,16 @@ import { import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fetchAuthorizedServableWorkspaceFileBuffer } from '@/lib/workspace-files/application/fetch-servable-workspace-file-buffer' import { fileOperations } from '@/lib/workspace-files/application/operations' -import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' -import { MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' - -const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 -const MAX_REQUESTED_FILE_IDS = 1_000 -const MAX_REQUESTED_FOLDER_IDS = 1_000 +import { + expandFileDownloadFolders, + normalizeFileDownloadSelection, +} from '@/lib/workspace-files/download-selection' +import { MAX_ZIP_DOWNLOAD_BYTES, MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits' +import { + createFileReadReceipt, + type FileReadReceipt, + recheckFileReadReceipt, +} from '@/lib/workspace-files/read-receipt' export interface DownloadWorkspaceFileItemsInput { workspaceId: string @@ -55,51 +62,6 @@ export interface DownloadWorkspaceFileItemsResult { excludedFileIds: string[] } -function collectDescendantFolderIds( - selectedFolderIds: string[], - folders: Array<{ id: string; parentId: string | null }> -): Set { - const folderIds = new Set(selectedFolderIds) - let changed = true - while (changed) { - changed = false - for (const folder of folders) { - if (folder.parentId && folderIds.has(folder.parentId) && !folderIds.has(folder.id)) { - folderIds.add(folder.id) - changed = true - } - } - } - return folderIds -} - -/** - * Maps canonical folder paths onto the ids the selection walk uses. - * - * Resolved against the folder set the download already loads rather than by a - * separate path query, and a path that matches nothing is rejected rather than - * silently dropped — a caller that misspells a folder should not receive a zip - * of whatever else it happened to select. - */ -function resolveFolderIdsFromPaths( - paths: string[], - folders: Array<{ id: string }>, - displayPathById: Map -): string[] { - if (paths.length === 0) return [] - const idByPath = new Map() - for (const folder of folders) { - const displayPath = displayPathById.get(folder.id) - if (!displayPath) continue - idByPath.set(parseWorkspaceFileFolderDisplayPath(displayPath).join('\u0000'), folder.id) - } - return paths.map((path) => { - const id = idByPath.get(parseFolderPath(path).join('\u0000')) - if (!id) validationError(`Folder not found: ${path}`) - return id - }) -} - function validationError(message: string): never { throw new OrchestrationError('validation', message) } @@ -112,30 +74,14 @@ function validateDeclaredBytes(bytes: number): void { } } -async function executeDownloadWorkspaceFileItems({ - input, +async function requireDownloadCapability({ context, principal, }: AuthorizedWorkspaceUseCaseContext< typeof fileOperations.download, DownloadWorkspaceFileItemsInput, Awaited> ->): Promise { - const fileIds = [...new Set(input.fileIds)] - const folderIds = [...new Set(input.folderIds)] - const requestedFolderPaths = [...new Set(input.folderPaths ?? [])] - if (fileIds.length > MAX_REQUESTED_FILE_IDS) { - validationError(`Too many file IDs selected. Select ${MAX_REQUESTED_FILE_IDS} or fewer files.`) - } - if (folderIds.length + requestedFolderPaths.length > MAX_REQUESTED_FOLDER_IDS) { - validationError( - `Too many folders selected. Select ${MAX_REQUESTED_FOLDER_IDS} or fewer folders.` - ) - } - if (fileIds.length === 0 && folderIds.length === 0 && requestedFolderPaths.length === 0) { - validationError('No files selected for download') - } - +>) { /** * permission-group-enforced: files.bulk_download — one operation serves both * a single file and a whole folder tree, and only the archive is what the key @@ -160,17 +106,26 @@ async function executeDownloadWorkspaceFileItems({ ) } } +} + +async function executeDownloadWorkspaceFileItems({ + input, + context, + principal, +}: AuthorizedWorkspaceUseCaseContext< + typeof fileOperations.download, + DownloadWorkspaceFileItemsInput, + Awaited> +>): Promise { + const selection = normalizeFileDownloadSelection(input) const [files, folders] = await Promise.all([ listWorkspaceFiles(context.workspaceId, { hydrateFolderPaths: false, throwOnError: true }), listWorkspaceFileFolders(context.workspaceId), ]) const folderPaths = buildWorkspaceFileFolderPathMap(folders) - const selectedFolderIds = collectDescendantFolderIds( - [...folderIds, ...resolveFolderIdsFromPaths(requestedFolderPaths, folders, folderPaths)], - folders - ) - const requestedFileIds = new Set(fileIds) + const selectedFolderIds = expandFileDownloadFolders(selection, folders, folderPaths) + const requestedFileIds = new Set(selection.fileIds) const selectedFiles = files.filter( (file) => requestedFileIds.has(file.id) || @@ -197,25 +152,30 @@ async function executeDownloadWorkspaceFileItems({ let declaredBytes = reservedForStreamed validateDeclaredBytes(declaredBytes) const renderedDocuments = new Map() + const receipts: FileReadReceipt[] = [] const pendingNames: string[] = [] let renderedBytes = 0 for (const file of filesToZip) { if (!needsRenderedArtifact(file.type, file.name)) continue const remaining = Math.max(0, MAX_ZIP_DOWNLOAD_BYTES - reservedForStreamed - renderedBytes) - const allowance = Math.min(remaining, MAX_RENDERED_DOCUMENT_BYTES) try { - const { buffer, contributingFiles } = await fetchAuthorizedServableWorkspaceFileBuffer( - file, - principal, - { maxBytes: allowance } - ) + const { buffer, contributingFiles, receipt } = + await fetchAuthorizedServableWorkspaceFileBuffer(file, principal, { + maxBytes: MAX_RENDERED_DOCUMENT_BYTES, + }) if (!(await admitContributors(principal, context.workspaceId, contributingFiles, 'opaque'))) { filesToZip = filesToZip.filter((admitted) => admitted.id !== file.id) continue } + if (buffer.length > remaining) { + validationError( + `The selected files exceed the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit once documents are rendered. Select fewer files.` + ) + } declaredBytes += file.size validateDeclaredBytes(declaredBytes) + receipts.push(receipt) renderedBytes += buffer.length renderedDocuments.set(file.id, buffer) } catch (error) { @@ -225,9 +185,7 @@ async function executeDownloadWorkspaceFileItems({ } if (error instanceof PayloadSizeLimitError) { validationError( - allowance === MAX_RENDERED_DOCUMENT_BYTES - ? `"${file.name}" renders to more than ${formatFileSize(MAX_RENDERED_DOCUMENT_BYTES)} and is too large to include in a zip; download it on its own instead.` - : `The selected files exceed the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit once documents are rendered. Select fewer files.` + `"${file.name}" renders to more than ${formatFileSize(MAX_RENDERED_DOCUMENT_BYTES)} and is too large to include in a zip; download it on its own instead.` ) } if (!isDocNotReadyError(error)) throw error @@ -242,6 +200,67 @@ async function executeDownloadWorkspaceFileItems({ throw new ContentRefusedError('workspace-file') } + await downloadWorkspaceFileItems.authorize({ principal, input }) + if (context.fileId === undefined) { + const actingUserId = capabilityGovernedPrincipalUserId(principal) + if (actingUserId) { + // permission-group-enforced: files.bulk_download — publication must use the current archive capability. + await assertWorkspaceCapability( + actingUserId, + context.workspaceId, + 'files.bulk_download', + context.workspaceOrganizationId + ) + } + } + const [currentFiles, currentFolders] = await Promise.all([ + listWorkspaceFiles(context.workspaceId, { hydrateFolderPaths: false, throwOnError: true }), + listWorkspaceFileFolders(context.workspaceId), + ]) + const currentPaths = buildWorkspaceFileFolderPathMap(currentFolders) + const currentSelection = expandFileDownloadFolders(selection, currentFolders, currentPaths) + const currentSelectedFiles = currentFiles.filter( + (file) => + requestedFileIds.has(file.id) || + (file.folderId != null && currentSelection.has(file.folderId)) + ) + const identity = (selected: WorkspaceFileRecord[], paths: Map) => + JSON.stringify( + [...selected] + .sort((a, b) => compareStrings(a.id, b.id)) + .map((file) => [ + file.id, + file.key, + file.name, + file.folderId, + file.folderId ? paths.get(file.folderId) : null, + ]) + ) + if (identity(currentSelectedFiles, currentPaths) !== identity(selectedFiles, folderPaths)) + throw new OrchestrationError('conflict', 'File selection changed while preparing the download') + receipts.push( + createFileReadReceipt( + { entityType: 'workspace', entityId: context.workspaceId }, + filesToZip.map((file) => ({ + ...file, + contentUpdatedAt: file.contentUpdatedAt ?? file.updatedAt, + })) + ) + ) + const provenance = await db.transaction( + async (tx) => { + const evidence = [] + for (const receipt of receipts) evidence.push(await recheckFileReadReceipt(tx, receipt)) + return mergeWorkspaceFileSecretProvenance(...evidence) + }, + { isolationLevel: 'repeatable read', accessMode: 'read only' } + ) + await requireAdmittedContent(principal, { + kind: 'workspace-file', + id: context.workspaceId, + provenance, + view: 'opaque', + }) const included = new Set(filesToZip.map((file) => file.id)) return { excludedFileIds: selectedFiles.filter((file) => !included.has(file.id)).map((file) => file.id), @@ -275,6 +294,7 @@ async function resolveDownloadContext({ input }: { input: DownloadWorkspaceFileI export const downloadWorkspaceFileItems = defineAuthorizedWorkspaceFileUseCase({ operation: fileOperations.download, resolveContext: resolveDownloadContext, + authorizeResource: requireDownloadCapability, execute: executeDownloadWorkspaceFileItems, projectAudit({ result }) { return { diff --git a/apps/sim/lib/workspace-files/application/extract-workspace-file.test.ts b/apps/sim/lib/workspace-files/application/extract-workspace-file.test.ts index 2b3ca630d32..959c18bd3b0 100644 --- a/apps/sim/lib/workspace-files/application/extract-workspace-file.test.ts +++ b/apps/sim/lib/workspace-files/application/extract-workspace-file.test.ts @@ -37,6 +37,7 @@ vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) vi.mock('@/lib/core/idempotency/service', () => idempotencyServiceMock) vi.mock('@/lib/uploads/archive', () => ({ + archiveFolderName: vi.fn(() => 'bundle'), decompressArchiveBufferToWorkspaceFiles: hoisted.decompress, MAX_ARCHIVE_BYTES: 100 * 1024 * 1024, })) diff --git a/apps/sim/lib/workspace-files/application/extract-workspace-file.ts b/apps/sim/lib/workspace-files/application/extract-workspace-file.ts index 9dcf6d60e70..44f5176eb96 100644 --- a/apps/sim/lib/workspace-files/application/extract-workspace-file.ts +++ b/apps/sim/lib/workspace-files/application/extract-workspace-file.ts @@ -6,7 +6,11 @@ import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application' import { IdempotencyService } from '@/lib/core/idempotency/service' import { OrchestrationError } from '@/lib/core/orchestration/types' import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' -import { decompressArchiveBufferToWorkspaceFiles, MAX_ARCHIVE_BYTES } from '@/lib/uploads/archive' +import { + archiveFolderName, + decompressArchiveBufferToWorkspaceFiles, + MAX_ARCHIVE_BYTES, +} from '@/lib/uploads/archive' import { archiveWorkspaceFileFolderIfEmpty, createWorkspaceFileFolder, @@ -73,16 +77,6 @@ type ExtractWorkspaceFileUseCaseContext = AuthorizedWorkspaceUseCaseContext< ActiveWorkspaceFileContext > -function archiveFolderName(fileName: string): string { - const stripped = fileName - .replace(/\.zip$/i, '') - .normalize('NFC') - .replace(/[\x00-\x1f\x7f]/g, '') - .replace(/[/\\]/g, '-') - .trim() - return stripped && stripped !== '.' && stripped !== '..' ? stripped : 'archive' -} - async function withExtractionLease( workspaceId: string, fileId: string, diff --git a/apps/sim/lib/workspace-files/application/operations.ts b/apps/sim/lib/workspace-files/application/operations.ts index c96ff002871..d84ffe529ad 100644 --- a/apps/sim/lib/workspace-files/application/operations.ts +++ b/apps/sim/lib/workspace-files/application/operations.ts @@ -1,4 +1,8 @@ import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' +import { fileCopyOperation } from '@/lib/workspace-files/application/copy-operation' + +/** Compound copy admission remains separate from workspace-only file operations. */ +export const fileCopyOperations = Object.freeze({ copy: fileCopyOperation }) const ALL_COPILOT_PRINCIPAL_POLICY = { principalKinds: [ diff --git a/apps/sim/lib/workspace-files/archive-extraction.ts b/apps/sim/lib/workspace-files/archive-extraction.ts new file mode 100644 index 00000000000..54a389d6de2 --- /dev/null +++ b/apps/sim/lib/workspace-files/archive-extraction.ts @@ -0,0 +1,175 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import { MAX_ARCHIVE_TOTAL_BYTES, type PreparedArchiveExtraction } from '@/lib/uploads/archive' +import { createFileFolder } from '@/lib/uploads/contexts/workspace/file-folder-manager' +import { + commitFileCreateInTx, + discardStagedFileContent, + type StagedFileContent, + stageFileContent, +} from '@/lib/uploads/contexts/workspace/file-manager' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' +import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' +import { restoreSimPageSourceBuffer } from '@/lib/workspace-files/page-source-embed' + +interface StagedArchiveEntry { + readonly parentSegments: readonly string[] + readonly content: StagedFileContent +} + +interface StagedFileArchive { + readonly entries: readonly StagedArchiveEntry[] + readonly bytes: number + readonly plan: PreparedArchiveExtraction +} + +/** Releases staged objects through the existing durable compensation path. */ +export async function discardFileArchive(archive: Pick) { + const results = await Promise.allSettled( + archive.entries.map((entry) => discardStagedFileContent(entry.content)) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length) throw new AggregateError(failures, 'Archive cleanup could not be recorded') +} + +/** Inflates and stages one bounded entry at a time, without holding database locks. */ +export async function stageFileArchive( + plan: PreparedArchiveExtraction, + args: { owner: EditableFileOwner; userId: string; signal: AbortSignal } +): Promise { + const entries: StagedArchiveEntry[] = [] + let bytes = 0 + try { + for await (const entry of plan.entries(args.signal)) { + args.signal.throwIfAborted() + if (entry.segments.slice(0, -1).some((segment) => segment.length > 255)) + throw new OrchestrationError( + 'validation', + 'Archive folder names must be at most 255 characters' + ) + const name = entry.segments[entry.segments.length - 1] + const restored = restoreSimPageSourceBuffer(name, entry.buffer) + const content = restored?.buffer ?? entry.buffer + bytes += content.length + if (bytes > MAX_ARCHIVE_TOTAL_BYTES) { + throw new OrchestrationError( + 'payload_too_large', + 'Extracted content exceeds the archive limit' + ) + } + entries.push( + Object.freeze({ + parentSegments: Object.freeze(entry.segments.slice(0, -1)), + content: await stageFileContent({ + owner: args.owner, + userId: args.userId, + name: restored?.name ?? name, + contentType: restored + ? SIM_PAGE_CONTENT_TYPE + : getMimeTypeFromExtension(getFileExtension(name)), + content, + signal: args.signal, + }), + }) + ) + } + args.signal.throwIfAborted() + return Object.freeze({ entries: Object.freeze(entries), bytes, plan }) + } catch (error) { + try { + await discardFileArchive({ entries }) + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], 'Archive staging and cleanup failed') + } + throw error + } +} + +/** Publishes the complete tree through canonical shared writers in the caller's transaction. */ +export async function commitFileArchiveInTx( + tx: DbTransaction, + args: { + owner: EditableFileOwner + userId: string + rootName: string + parentId: string | null + parentSegments: readonly string[] + staged: StagedFileArchive + secretProvenance: WorkspaceFileSecretProvenance + signal: AbortSignal + } +) { + args.signal.throwIfAborted() + if (!args.staged.entries.length) { + throw new OrchestrationError('validation', 'No files could be unzipped from this archive') + } + if ( + args.staged.entries.some((entry) => + entry.parentSegments.some((segment) => segment.length > 255) + ) + ) + throw new OrchestrationError( + 'validation', + 'Archive folder names must be at most 255 characters' + ) + const root = await createFileFolder( + { + owner: args.owner, + userId: args.userId, + name: args.rootName, + parentId: args.parentId, + exactName: false, + validateResolvedName: (name) => + args.staged.plan.validateRootFolderSegments([...args.parentSegments, name]), + }, + tx + ) + const folders = new Map([['[]', root.id]]) + const provenance: WorkspaceFileSecretProvenance = + args.secretProvenance.status === 'unrecorded' || + (args.secretProvenance.status === 'exact' && args.secretProvenance.entries.length === 0) + ? args.secretProvenance + : { status: 'unknown' } + for (const entry of args.staged.entries) { + args.signal.throwIfAborted() + let parentId = root.id + for (let depth = 0; depth < entry.parentSegments.length; depth++) { + const key = JSON.stringify(entry.parentSegments.slice(0, depth + 1)) + let folderId = folders.get(key) + if (!folderId) { + const created = await createFileFolder( + { + owner: args.owner, + userId: args.userId, + name: entry.parentSegments[depth], + parentId, + }, + tx + ) + folderId = created.id + folders.set(key, folderId) + } + parentId = folderId + } + await commitFileCreateInTx(tx, { + owner: args.owner, + userId: args.userId, + staged: entry.content, + folderId: parentId, + exactName: false, + secretProvenance: provenance, + }) + } + args.signal.throwIfAborted() + return { + folderId: root.id, + folderName: root.name, + folderDisplayPath: root.path, + extractedCount: args.staged.entries.length, + skippedCount: args.staged.plan.skipped, + } +} diff --git a/apps/sim/lib/workspace-files/copy-references.test.ts b/apps/sim/lib/workspace-files/copy-references.test.ts new file mode 100644 index 00000000000..35dcfd0043c --- /dev/null +++ b/apps/sim/lib/workspace-files/copy-references.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from 'vitest' +import { rewriteCopiedFileReferences } from '@/lib/workspace-files/copy-references' + +const workspaceOwner = { entityType: 'workspace', entityId: 'source-ws' } as const +const projectOwner = { entityType: 'project', entityId: 'target-project' } as const +const sourceKey = 'workspace/source-ws/123-photo.png' +const maps = { + sourceOwner: workspaceOwner, + destinationOwner: projectOwner, + fileIds: new Map([['photo-1', 'copied-photo']]), + fileKeys: new Map([[sourceKey, 'copied-photo']]), +} + +const projectBytes = '/api/projects/target-project/files/copied-photo/content' + +describe('selected copy references', () => { + it('preserves sentence punctuation while rewriting bare references', () => { + expect( + rewriteCopiedFileReferences('See /api/files/view/photo-1. Then sim:file/photo-1!', maps) + ).toBe(`See ${projectBytes}. Then sim:file/copied-photo!`) + }) + + it('rewrites selected neutral identities without touching other IDs or resources', () => { + const source = [ + '![photo](/api/files/view/photo-1)', + '[selected](sim:file/photo-1)', + '[other](sim:file/photo-10)', + '[table](sim:table/photo-1)', + '![not selected](/api/files/view/other)', + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps)).toBe( + [ + `![photo](${projectBytes})`, + '[selected](sim:file/copied-photo)', + '[other](sim:file/photo-10)', + '[table](sim:table/photo-1)', + '![not selected](/api/files/view/other)', + ].join('\n') + ) + }) + + it('normalizes selected raw and encoded storage URLs to destination bytes', () => { + const source = [ + `/api/files/serve/${sourceKey}`, + `/api/files/serve/s3/${encodeURIComponent(sourceKey)}?context=workspace`, + `/api/files/serve/blob/${encodeURIComponent(sourceKey)}`, + `/api/files/serve/gcs/${encodeURIComponent(sourceKey)}`, + ] + .map((url) => `![photo](${url})`) + .join('\n') + expect(rewriteCopiedFileReferences(source, maps)).toBe( + Array.from({ length: 4 }, () => `![photo](${projectBytes})`).join('\n') + ) + }) + + it('requires the exact source owner prefix even if a foreign key is accidentally mapped', () => { + const foreign = 'workspace/source-ws-other/123-photo.png' + const source = `![photo](/api/files/serve/${encodeURIComponent(foreign)})` + expect( + rewriteCopiedFileReferences(source, { + ...maps, + fileKeys: new Map([[foreign, 'copied-photo']]), + }) + ).toBe(source) + }) + + it('moves both workspace-qualified identities to the destination Project and keeps fragments', () => { + const source = + '[app](/workspace/source-ws/files/photo-1#caption) [chip](sim:file/photo-1?workspace=source-ws#caption)' + expect(rewriteCopiedFileReferences(source, maps)).toBe( + '[app](/projects/target-project/files/copied-photo#caption) [chip](sim:file/copied-photo?project=target-project#caption)' + ) + }) + + it('moves Project-qualified identities and bytes to a workspace', () => { + const source = [ + '[app](/projects/target-project/files/photo-1)', + '[chip](sim:file/photo-1?project=target-project)', + '![photo](/api/projects/target-project/files/photo-1/content)', + '![key](/api/files/serve/project%2Ftarget-project%2F123-photo.png)', + ].join('\n') + expect( + rewriteCopiedFileReferences(source, { + ...maps, + sourceOwner: projectOwner, + destinationOwner: workspaceOwner, + fileKeys: new Map([['project/target-project/123-photo.png', 'copied-photo']]), + }) + ).toBe( + [ + '[app](/workspace/source-ws/files/copied-photo)', + '[chip](sim:file/copied-photo?workspace=source-ws)', + '![photo](/api/files/view/copied-photo)', + '![key](/api/files/view/copied-photo)', + ].join('\n') + ) + }) + + it('preserves foreign or ambiguous ownership and unsupported file paths', () => { + const source = [ + 'sim:file/photo-1?workspace=other', + 'sim:file/photo-1?project=target-project', + 'sim:file/photo-1?workspace=source-ws&project=target-project', + 'sim:file/photo-1?workspace=source-ws&workspace=source-ws', + '/workspace/other/files/photo-1', + '/projects/target-project/files/photo-1', + '/api/projects/target-project/files/photo-1/content', + '/api/files/view/photo-1/extra', + '/api/files/view/photo-1?project=other', + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps)).toBe(source) + }) + + it('does not turn absolute or protocol-relative URLs into local destination references', () => { + const source = [ + 'https://other.test/api/files/view/photo-1', + '//other.test/api/files/view/photo-1', + 'https://other.test/workspace/source-ws/files/photo-1', + 'https://other.test/?next=/api/files/view/photo-1', + 'https://other.test/a(/api/files/view/photo-1)', + 'prefixsim:file/photo-1', + 'x/api/files/view/photo-1', + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps)).toBe(source) + }) + + it('decodes once while preserving malformed and twice-encoded references', () => { + const source = [ + '/api/files/view/photo%2D1', + '/api/files/view/photo%252D1', + '/api/files/view/%E0%A4%A', + '/workspace/source%2Dws/files/photo%2D1', + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps)).toBe( + [ + projectBytes, + '/api/files/view/photo%252D1', + '/api/files/view/%E0%A4%A', + '/projects/target-project/files/copied-photo', + ].join('\n') + ) + }) + + it('applies each selected mapping once without cascading into another selected identity', () => { + expect( + rewriteCopiedFileReferences('sim:file/photo-1 sim:file/copied-photo', { + ...maps, + fileIds: new Map([ + ['photo-1', 'copied-photo'], + ['copied-photo', 'third-photo'], + ]), + }) + ).toBe('sim:file/copied-photo sim:file/third-photo') + }) + + it('rewrites only selected Office helper and input-path identities', () => { + const source = [ + "const image = await getFileBase64('photo-1')", + 'await addImage(slide, "photo-1", options)', + "await addImage('photo-1', options)", + "drawImage(page, 'photo-1', opts)", + "Doc.open(input_path('photo-1'))", + "fs.readFileSync('/home/user/inputs/photo-1')", + "getFileBase64('unselected')", + 'getFileBase64(variable)', + "slide.addText('photo-1')", + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps, 'javascript')).toBe( + [ + "const image = await getFileBase64('copied-photo')", + 'await addImage(slide, "copied-photo", options)', + "await addImage('copied-photo', options)", + "drawImage(page, 'copied-photo', opts)", + "Doc.open(input_path('copied-photo'))", + "fs.readFileSync('/home/user/inputs/copied-photo')", + "getFileBase64('unselected')", + 'getFileBase64(variable)', + "slide.addText('photo-1')", + ].join('\n') + ) + }) + + it.each(['javascript', 'python'] as const)( + 'copies unfinished %s document source without partial reference rewrites', + (language) => { + const source = [ + 'link = "/api/files/view/photo-1"', + 'image = getFileBase64("photo-1")', + 'unfinished = "unterminated', + ].join('\n') + expect(rewriteCopiedFileReferences(source, maps, language)).toBe(source) + } + ) +}) diff --git a/apps/sim/lib/workspace-files/copy-references.ts b/apps/sim/lib/workspace-files/copy-references.ts new file mode 100644 index 00000000000..08ac7719cd4 --- /dev/null +++ b/apps/sim/lib/workspace-files/copy-references.ts @@ -0,0 +1,187 @@ +import { + type DocumentSourceLanguage, + iterateDocumentFileReferences, +} from '@/lib/uploads/documents/references' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' + +interface CopiedFileReferenceMaps { + sourceOwner: EditableFileOwner + destinationOwner: EditableFileOwner + fileIds: ReadonlyMap + fileKeys: ReadonlyMap +} + +interface SourceReplacement { + start: number + end: number + value: string +} + +const FILE_REFERENCE_RE = + /(^|[\s("'<>`])((?:[A-Za-z][A-Za-z0-9+.-]*:\/\/|\/\/|sim:file\/|\/(?:api\/(?:files\/(?:view|serve)\/|projects\/)|workspace\/|projects\/))[^\s)"'<>`]+)/g + +function absoluteReferenceEnd(content: string, start: number): number { + let parentheses = 0 + let end = start + for (; end < content.length; end++) { + const character = content[end] + if (/[\s"'<>`]/.test(character)) break + if (character === '(') parentheses++ + if (character === ')') { + if (parentheses === 0) break + parentheses-- + } + } + return end +} + +function decodedIdentity(value: string): string | undefined { + try { + const decoded = decodeURIComponent(value) + return /^[A-Za-z0-9_-]+$/.test(decoded) ? decoded : undefined + } catch { + return undefined + } +} + +function destinationBytes(owner: EditableFileOwner, fileId: string): string { + return owner.entityType === 'project' + ? `/api/projects/${encodeURIComponent(owner.entityId)}/files/${encodeURIComponent(fileId)}/content` + : `/api/files/view/${encodeURIComponent(fileId)}` +} + +function destinationPage(owner: EditableFileOwner, fileId: string): string { + const namespace = owner.entityType === 'project' ? 'projects' : 'workspace' + return `/${namespace}/${encodeURIComponent(owner.entityId)}/files/${encodeURIComponent(fileId)}` +} + +function matchesOwner(owner: EditableFileOwner, entityType: string, entityId: string): boolean { + return owner.entityType === entityType && owner.entityId === decodedIdentity(entityId) +} + +function rewriteFileReference(reference: string, maps: CopiedFileReferenceMaps): string | null { + const hashIndex = reference.indexOf('#') + const fragment = hashIndex < 0 ? '' : reference.slice(hashIndex) + const withoutFragment = hashIndex < 0 ? reference : reference.slice(0, hashIndex) + const queryIndex = withoutFragment.indexOf('?') + const pathname = queryIndex < 0 ? withoutFragment : withoutFragment.slice(0, queryIndex) + const query = queryIndex < 0 ? null : new URLSearchParams(withoutFragment.slice(queryIndex + 1)) + const mappedId = (spelledId: string) => { + const fileId = decodedIdentity(spelledId) + return fileId === undefined ? undefined : maps.fileIds.get(fileId) + } + + const simLink = /^sim:file\/([^/]+)$/.exec(pathname) + if (simLink) { + const fileId = mappedId(simLink[1]) + if (!fileId) return null + if (query === null) return `sim:file/${encodeURIComponent(fileId)}${fragment}` + const entries = [...query] + const entry = entries[0] + if ( + entries.length !== 1 || + !entry || + entry[0] !== maps.sourceOwner.entityType || + entry[1] !== maps.sourceOwner.entityId + ) { + return null + } + return `sim:file/${encodeURIComponent(fileId)}?${maps.destinationOwner.entityType}=${encodeURIComponent(maps.destinationOwner.entityId)}${fragment}` + } + + const serve = /^\/api\/files\/serve\/(?:s3\/|blob\/|gcs\/)?(.+)$/.exec(pathname) + if (serve) { + if (query !== null) { + const entries = [...query] + const entry = entries[0] + if ( + entries.length !== 1 || + !entry || + entry[0] !== 'context' || + entry[1] !== maps.sourceOwner.entityType + ) { + return null + } + } + let key: string + try { + key = decodeURIComponent(serve[1]) + } catch { + return null + } + if (!key.startsWith(`${maps.sourceOwner.entityType}/${maps.sourceOwner.entityId}/`)) return null + const fileId = maps.fileKeys.get(key) + return fileId ? destinationBytes(maps.destinationOwner, fileId) + fragment : null + } + + if (query !== null) return null + const view = /^\/api\/files\/view\/([^/]+)$/.exec(pathname) + if (view) { + const fileId = mappedId(view[1]) + return fileId ? destinationBytes(maps.destinationOwner, fileId) + fragment : null + } + const page = /^\/(workspace|projects)\/([^/]+)\/files\/([^/]+)$/.exec(pathname) + if (page) { + const entityType = page[1] === 'projects' ? 'project' : 'workspace' + if (!matchesOwner(maps.sourceOwner, entityType, page[2])) return null + const fileId = mappedId(page[3]) + return fileId ? destinationPage(maps.destinationOwner, fileId) + fragment : null + } + const projectContent = /^\/api\/projects\/([^/]+)\/files\/([^/]+)\/content$/.exec(pathname) + if (projectContent) { + if (!matchesOwner(maps.sourceOwner, 'project', projectContent[1])) return null + const fileId = mappedId(projectContent[2]) + return fileId ? destinationBytes(maps.destinationOwner, fileId) + fragment : null + } + return null +} + +/** Repoints only the canonical selected identities; it neither discovers nor copies dependencies. */ +export function rewriteCopiedFileReferences( + content: string, + maps: CopiedFileReferenceMaps, + language: DocumentSourceLanguage | null = null +): string { + if (!content || (maps.fileIds.size === 0 && maps.fileKeys.size === 0)) return content + const replacements: SourceReplacement[] = [] + let protectedEnd = 0 + for (const match of content.matchAll(FILE_REFERENCE_RE)) { + const reference = match[2] + const start = match.index + match[1].length + if (start < protectedEnd) continue + if (/^(?:[A-Za-z][A-Za-z0-9+.-]*:\/\/|\/\/)/.test(reference)) { + protectedEnd = absoluteReferenceEnd(content, start) + continue + } + let value = rewriteFileReference(reference, maps) + if (value === null && !language && /[.,;:!?]+$/.test(reference)) { + const trimmed = reference.replace(/[.,;:!?]+$/, '') + const rewritten = rewriteFileReference(trimmed, maps) + if (rewritten !== null) value = rewritten + reference.slice(trimmed.length) + } + if (value === null) continue + replacements.push({ start, end: start + reference.length, value }) + } + if (language) { + try { + for (const { fileId, start, end } of iterateDocumentFileReferences(content, language)) { + const value = maps.fileIds.get(fileId) + if (value !== undefined) replacements.push({ start, end, value }) + } + } catch (error) { + if (error instanceof DocCompileUserError) return content + throw error + } + } + replacements.sort((left, right) => left.start - right.start) + const chunks: string[] = [] + let offset = 0 + for (const { start, end, value } of replacements) { + chunks.push(content.slice(offset, start), value) + offset = end + } + chunks.push(content.slice(offset)) + return chunks.join('') +} + +import { DocCompileUserError } from '@/lib/uploads/documents/compile-error' diff --git a/apps/sim/lib/workspace-files/copy.ts b/apps/sim/lib/workspace-files/copy.ts new file mode 100644 index 00000000000..f009d0321d8 --- /dev/null +++ b/apps/sim/lib/workspace-files/copy.ts @@ -0,0 +1,228 @@ +import { folder, type WorkspaceFileRow, workspaceFiles } from '@sim/db/schema' +import { and, asc, inArray, isNull, or } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import { + assertFileFolderTarget, + buildWorkspaceFileFolderPathMap, + createFileFolderRow, + listFileFolders, +} from '@/lib/uploads/contexts/workspace/file-folder-manager' +import { + commitFileCreateInTx, + mapFileRecord, + type OwnedFileRecord, + type PlannedFileIdentity, + type StagedFileContent, +} from '@/lib/uploads/contexts/workspace/file-manager' +import { + getBoundWorkspaceFileSecretProvenanceByMetadata, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/file-secret-provenance' +import { getWorkspaceFileSize, MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS } from '@/lib/workspace-files/limits' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' +import { fileFolderOwnerCondition, fileOwnerCondition } from '@/lib/workspace-files/ownership-query' + +interface CopySelection { + owner: EditableFileOwner + fileIds: readonly string[] + folderIds: readonly string[] +} + +interface CopyFolder { + id: string + name: string + parentId: string | null + updatedAt: Date +} + +interface FileCopySnapshot { + owner: EditableFileOwner + files: WorkspaceFileRow[] + folders: CopyFolder[] + provenance: Map + bytes: number +} + +export interface CopiedFileItems { + files: OwnedFileRecord[] + folders: Awaited> +} + +function requireBoundedSelection(size: number) { + if (size > MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS) { + throw new OrchestrationError('validation', 'Copy expands beyond the file selection limit') + } +} + +/** Captures only explicit source selections and their bounded active descendants. */ +export async function snapshotFileCopyInTx( + tx: DbTransaction, + source: CopySelection +): Promise { + if (!source.fileIds.length && !source.folderIds.length) { + throw new OrchestrationError('validation', 'Copy requires a source selection') + } + const loadFolders = (ids: readonly string[], descendants: boolean, limit: number) => + tx + .select({ + id: folder.id, + name: folder.name, + parentId: folder.parentId, + updatedAt: folder.updatedAt, + }) + .from(folder) + .where( + and( + fileFolderOwnerCondition(source.owner), + isNull(folder.deletedAt), + inArray(descendants ? folder.parentId : folder.id, [...ids]) + ) + ) + .orderBy(asc(folder.id)) + .limit(limit) + const candidates = source.folderIds.length + ? await loadFolders(source.folderIds, false, MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS + 1) + : [] + const selectedFolders = new Set(candidates.map((row) => row.id)) + if (source.folderIds.some((id) => !selectedFolders.has(id))) + throw new OrchestrationError('not_found', 'Source folder not found') + let frontier = [...selectedFolders] + while (frontier.length) { + requireBoundedSelection(selectedFolders.size) + const children = await loadFolders(frontier, true, MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS + 1) + frontier = [] + for (const child of children) { + if (selectedFolders.has(child.id)) continue + selectedFolders.add(child.id) + candidates.push(child) + frontier.push(child.id) + } + requireBoundedSelection(selectedFolders.size) + } + const files = await tx + .select() + .from(workspaceFiles) + .where( + and( + fileOwnerCondition(source.owner), + isNull(workspaceFiles.deletedAt), + or( + source.fileIds.length ? inArray(workspaceFiles.id, [...source.fileIds]) : undefined, + selectedFolders.size ? inArray(workspaceFiles.folderId, [...selectedFolders]) : undefined + ) + ) + ) + .orderBy(asc(workspaceFiles.id)) + .limit(MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS + 1) + .for('share') + requireBoundedSelection(files.length + selectedFolders.size) + const fileIds = new Set(files.map((row) => row.id)) + if (source.fileIds.some((id) => !fileIds.has(id))) { + throw new OrchestrationError('not_found', 'Source file not found') + } + const bytes = files.reduce((sum, row) => sum + getWorkspaceFileSize(row), 0) + if (!Number.isSafeInteger(bytes) || bytes > MAX_BUFFERED_TRANSFER_BYTES) { + throw new OrchestrationError('payload_too_large', 'Copy exceeds the buffered transfer limit') + } + const folders = candidates.filter((row) => selectedFolders.has(row.id)) + const provenance = await getBoundWorkspaceFileSecretProvenanceByMetadata(tx, files) + return { owner: source.owner, files, folders, provenance, bytes } +} + +function snapshotIdentity(snapshot: FileCopySnapshot) { + return JSON.stringify({ + owner: snapshot.owner, + folders: snapshot.folders, + files: snapshot.files.map((file) => ({ + id: file.id, + key: file.key, + name: file.originalName, + folderId: file.folderId, + size: getWorkspaceFileSize(file), + type: file.contentType, + updatedAt: file.updatedAt, + contentUpdatedAt: file.contentUpdatedAt, + provenance: snapshot.provenance.get(file.id), + })), + }) +} + +/** Refuses changed bytes, metadata, classification, or recursive membership after external staging. */ +export function requireUnchangedFileCopy(prepared: FileCopySnapshot, current: FileCopySnapshot) { + if (snapshotIdentity(prepared) !== snapshotIdentity(current)) { + throw new OrchestrationError('conflict', 'Source selection changed during copy; retry') + } +} + +/** Commits a fresh tree and current heads through the shared identity/provenance writers. */ +export async function commitFileCopyInTx( + tx: DbTransaction, + args: { + snapshot: FileCopySnapshot + destination: { owner: EditableFileOwner; folderId: string | null } + userId: string + staged: ReadonlyMap + identities: ReadonlyMap + } +): Promise { + const { snapshot, destination, userId } = args + const targetFolderId = await assertFileFolderTarget(destination.owner, destination.folderId, tx) + const pending = new Map(snapshot.folders.map((row) => [row.id, row])) + const mapped = new Map() + while (pending.size) { + let progressed = false + for (const [id, source] of pending) { + if (source.parentId && pending.has(source.parentId)) continue + const created = await createFileFolderRow( + { + owner: destination.owner, + userId, + name: source.name, + parentId: source.parentId + ? (mapped.get(source.parentId) ?? targetFolderId) + : targetFolderId, + exactName: false, + }, + tx + ) + mapped.set(id, created.id) + pending.delete(id) + progressed = true + } + if (!progressed) + throw new OrchestrationError('conflict', 'Source folder hierarchy contains a cycle') + } + const createdFiles: WorkspaceFileRow[] = [] + for (const source of snapshot.files) { + const staged = args.staged.get(source.id) + const secretProvenance = snapshot.provenance.get(source.id) + const identity = args.identities.get(source.id) + if (!staged || !secretProvenance || !identity) + throw new Error('Copied file staging is incomplete') + createdFiles.push( + await commitFileCreateInTx(tx, { + owner: destination.owner, + staged, + identity, + userId, + folderId: source.folderId + ? (mapped.get(source.folderId) ?? targetFolderId) + : targetFolderId, + secretProvenance, + }) + ) + } + const destinationFolders = await listFileFolders(destination.owner, undefined, tx) + const paths = buildWorkspaceFileFolderPathMap(destinationFolders) + const folderById = new Map(destinationFolders.map((folder) => [folder.id, folder])) + return { + files: createdFiles.map((file) => mapFileRecord(file, destination.owner, paths)), + folders: [...mapped.values()].map((id) => { + const folder = folderById.get(id) + if (!folder) throw new Error('Copied folder is missing from the destination') + return folder + }), + } +} diff --git a/apps/sim/lib/workspace-files/download-selection.ts b/apps/sim/lib/workspace-files/download-selection.ts new file mode 100644 index 00000000000..fc0e576ea52 --- /dev/null +++ b/apps/sim/lib/workspace-files/download-selection.ts @@ -0,0 +1,64 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { parseFolderPath } from '@/lib/folders/paths' +import { + collectDescendantFolderIdsFrom, + type FolderNode, + indexFolderChildren, +} from '@/lib/folders/subtree' +import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' +import { MAX_WORKSPACE_FILE_BULK_REQUEST_IDS } from '@/lib/workspace-files/limits' + +export interface FileDownloadSelection { + fileIds: readonly string[] + folderIds: readonly string[] + folderPaths?: readonly string[] +} + +/** Normalizes the bounded explicit selection before either owner's directory walk. */ +export function normalizeFileDownloadSelection(input: FileDownloadSelection) { + const fileIds = [...new Set(input.fileIds)] + const folderIds = [...new Set(input.folderIds)] + const folderPaths = [...new Set(input.folderPaths ?? [])] + if (fileIds.length > MAX_WORKSPACE_FILE_BULK_REQUEST_IDS) + throw new OrchestrationError( + 'validation', + `Too many file IDs selected. Select ${MAX_WORKSPACE_FILE_BULK_REQUEST_IDS} or fewer files.` + ) + if (folderIds.length + folderPaths.length > MAX_WORKSPACE_FILE_BULK_REQUEST_IDS) + throw new OrchestrationError( + 'validation', + `Too many folders selected. Select ${MAX_WORKSPACE_FILE_BULK_REQUEST_IDS} or fewer folders.` + ) + if (!fileIds.length && !folderIds.length && !folderPaths.length) + throw new OrchestrationError('validation', 'No files selected for download') + return { fileIds, folderIds, folderPaths } +} + +/** Expands canonical parent relationships, never textual path prefixes. */ +export function expandFileDownloadFolders( + input: FileDownloadSelection, + folders: readonly FolderNode[], + displayPaths: ReadonlyMap +): Set { + const known = new Set(folders.map((folder) => folder.id)) + if (input.folderIds.some((id) => !known.has(id))) + throw new OrchestrationError('not_found', 'Folder not found') + const selected = new Set(input.folderIds) + if (input.folderPaths?.length) { + const byPath = new Map() + for (const folder of folders) { + const path = displayPaths.get(folder.id) + if (path) byPath.set(parseWorkspaceFileFolderDisplayPath(path).join('\0'), folder.id) + } + for (const path of input.folderPaths) { + const id = byPath.get(parseFolderPath(path).join('\0')) + if (!id) throw new OrchestrationError('validation', `Folder not found: ${path}`) + selected.add(id) + } + } + const children = indexFolderChildren(folders) + for (const id of [...selected]) { + for (const descendant of collectDescendantFolderIdsFrom(children, id)) selected.add(descendant) + } + return selected +} diff --git a/apps/sim/lib/workspace-files/extraction-lease.ts b/apps/sim/lib/workspace-files/extraction-lease.ts new file mode 100644 index 00000000000..58271ee106f --- /dev/null +++ b/apps/sim/lib/workspace-files/extraction-lease.ts @@ -0,0 +1,79 @@ +import { idempotencyKey } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { and, eq, sql } from 'drizzle-orm' +import { IdempotencyService } from '@/lib/core/idempotency/service' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { DbTransaction } from '@/lib/db/types' +import type { EditableFileOwner } from '@/lib/workspace-files/ownership' + +const logger = createLogger('FileArchiveLease') +const leases = new IdempotencyService({ + namespace: 'file-archive', + ttlSeconds: 6 * 60, + forceStorage: 'database', +}) + +/** Cooperative work budget leaves time for compensation before the lease expires. */ +export const FILE_EXTRACTION_BUDGET_MS = 180_000 + +interface FileExtractionLease { + readonly key: string + readonly token: string +} + +/** Holds an exact, token-fenced database lease without memoizing any extraction result. */ +export async function withFileExtractionLease( + owner: EditableFileOwner, + fileId: string, + action: (lease: FileExtractionLease) => Promise +): Promise { + const claim = await leases.atomicallyClaim( + 'extract', + `${owner.entityType}:${owner.entityId}:${fileId}` + ) + if (!claim.claimed) + throw new OrchestrationError('conflict', 'This archive is already being unzipped') + if (!claim.claimToken) throw new Error('Archive extraction lease is missing its fencing token') + try { + return await action(Object.freeze({ key: claim.normalizedKey, token: claim.claimToken })) + } finally { + await leases + .release(claim.normalizedKey, claim.storageMethod, claim.claimToken) + .catch((error) => { + logger.warn('Failed to release archive extraction lease', { + owner, + fileId, + error: getErrorMessage(error), + }) + }) + } +} + +/** Serializes publication with claim replacement and refuses expired or superseded work. */ +export async function requireFileExtractionLeaseInTx( + tx: DbTransaction, + lease: FileExtractionLease +) { + const [claim] = await tx + .select({ result: idempotencyKey.result }) + .from(idempotencyKey) + .where( + and( + eq(idempotencyKey.key, lease.key), + sql`${idempotencyKey.result}->>'claimToken' = ${lease.token}` + ) + ) + .for('update') + .limit(1) + const result = claim?.result + if ( + !isRecordLike(result) || + result.status !== 'in-progress' || + typeof result.inProgressExpiresAt !== 'number' || + result.inProgressExpiresAt <= Date.now() + ) { + throw new OrchestrationError('conflict', 'Archive extraction lease expired; retry') + } +} diff --git a/apps/sim/lib/workspace-files/limits.ts b/apps/sim/lib/workspace-files/limits.ts index c1b80bad120..d5b89b45483 100644 --- a/apps/sim/lib/workspace-files/limits.ts +++ b/apps/sim/lib/workspace-files/limits.ts @@ -9,3 +9,4 @@ export const MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS = 5_000 * validate, resolve, and only then fail — and so the two ceilings cannot drift. */ export const MAX_ZIP_DOWNLOAD_FILES = 100 +export const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024 diff --git a/apps/sim/scripts/test-file-copy-e2e.ts b/apps/sim/scripts/test-file-copy-e2e.ts new file mode 100644 index 00000000000..a2de1292c4d --- /dev/null +++ b/apps/sim/scripts/test-file-copy-e2e.ts @@ -0,0 +1,242 @@ +import assert from 'node:assert/strict' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { toArray, toRecord } from '@sim/utils/object' + +const base = new URL(process.env.FILE_COPY_BASE_URL ?? 'http://127.0.0.1:3300') +if (!['127.0.0.1', 'localhost', '[::1]'].includes(base.hostname)) + throw new Error('File copy proof requires a disposable local runtime') +const fixtureDir = required(process.env.FILE_COPY_FIXTURE_DIR, 'FILE_COPY_FIXTURE_DIR is required') +const reportPath = required(process.env.FILE_COPY_REPORT_PATH, 'FILE_COPY_REPORT_PATH is required') +const workspaceId = required( + process.env.FILE_COPY_WORKSPACE_ID, + 'FILE_COPY_WORKSPACE_ID is required' +) +const checks: { name: string; passed: boolean; durationMs: number; error?: string }[] = [] +function required(value: unknown, message = 'Required fixture field missing'): string { + if (typeof value !== 'string' || !value) throw new Error(message) + return value +} +async function fixture(name: string) { + return toRecord(JSON.parse(await readFile(join(fixtureDir, name), 'utf8'))) +} +const keys = await fixture('v2-fixture-keys.json') +const cookie = toArray((await fixture('owner-account.json')).cookies) + .map((value) => required(value).split(';')[0]) + .join('; ') +const projectId = required(toRecord((await fixture('http-project-fixture.json')).project).id) +type Owner = { entityType: 'workspace' | 'project'; entityId: string } +const workspaceOwner: Owner = { entityType: 'workspace', entityId: workspaceId } +const projectOwner: Owner = { entityType: 'project', entityId: projectId } +const created: { owner: Owner; fileIds: string[]; folderIds: string[] }[] = [] +async function request( + method: string, + path: string, + body?: Record, + auth: 'personal' | 'workspace' | 'session' | 'none' = 'personal' +) { + const headers = new Headers({ Origin: base.origin, 'Content-Type': 'application/json' }) + if (auth === 'session') headers.set('Cookie', cookie) + if (auth === 'personal' || auth === 'workspace') headers.set('X-API-Key', required(keys[auth])) + // boundary-raw-fetch: this proof exercises actual authenticated HTTP against a disposable local runtime. + const response = await fetch(new URL(path, base), { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(60_000), + }) + return { status: response.status, text: await response.text() } +} +function json(response: Awaited>, status = 200) { + assert.equal(response.status, status, response.text.slice(0, 300)) + return toRecord(JSON.parse(response.text)) +} +async function check(name: string, run: () => Promise) { + const start = performance.now() + try { + await run() + checks.push({ name, passed: true, durationMs: performance.now() - start }) + } catch (error) { + checks.push({ + name, + passed: false, + durationMs: performance.now() - start, + error: getErrorMessage(error), + }) + } + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) +} +const projectPrefix = `/api/v2/projects/${projectId}/files` +function selection(owner: Owner, fileIds: string[], folderIds: string[] = []) { + return { owner, fileIds, folderIds } +} +function remember(owner: Owner, result: Record) { + const fileIds = toArray(result.files).map((file) => required(toRecord(file).id)) + const folderIds = toArray(result.folders).map((folder) => required(toRecord(folder).id)) + created.push({ owner, fileIds, folderIds }) + for (const file of toArray(result.files)) { + assert.deepEqual(toRecord(file).owner, owner) + assert.equal('key' in toRecord(file), false) + } + return { fileIds, folderIds } +} +try { + const name = `copy-http-${generateId()}.txt` + const content = 'Copy retains these source bytes.' + const projectFile = toRecord( + json(await request('POST', projectPrefix, { name, content }), 201).data + ) + const workspaceFile = toRecord( + json(await request('POST', '/api/v2/files', { workspaceId, name, content }), 201).data + ) + created.push({ owner: projectOwner, fileIds: [required(projectFile.id)], folderIds: [] }) + created.push({ owner: workspaceOwner, fileIds: [required(workspaceFile.id)], folderIds: [] }) + const body = { + source: selection(projectOwner, [required(projectFile.id)]), + destination: { owner: workspaceOwner, folderId: null }, + } + await check('copy requires a real API credential', async () => { + const result = json(await request('POST', '/api/v2/files/copy', body, 'none'), 401) + assert.equal(toRecord(result.error).code, 'UNAUTHORIZED') + }) + await check('workspace keys cannot acquire compound owner authority', async () => { + const result = json(await request('POST', '/api/v2/files/copy', body, 'workspace'), 403) + assert.equal(toRecord(result.error).code, 'FORBIDDEN') + }) + await check( + 'nested unknown fields and empty selections are rejected before copying', + async () => { + for (const invalid of [ + { ...body, source: { ...body.source, owner: { ...projectOwner, workspaceId } } }, + { ...body, source: selection(projectOwner, []) }, + { + ...body, + source: { ...body.source, owner: { entityType: 'organization', entityId: projectId } }, + }, + ]) { + const result = json(await request('POST', '/api/v2/files/copy', invalid), 400) + assert.equal(toRecord(result.error).code, 'BAD_REQUEST') + } + } + ) + await check('v2 copies Project bytes to a separately authorized workspace', async () => { + const result = toRecord(json(await request('POST', '/api/v2/files/copy', body), 201).data) + const copied = remember(workspaceOwner, result) + assert.equal(copied.fileIds.length, 1) + assert.notEqual(copied.fileIds[0], projectFile.id) + const read = await request( + 'GET', + `/api/v2/files/${copied.fileIds[0]}?workspaceId=${workspaceId}` + ) + assert.equal(read.status, 200) + assert.equal(read.text, content) + assert.equal((await request('GET', `${projectPrefix}/${projectFile.id}/content`)).text, content) + }) + await check( + 'internal session copy uses the same compound workspace to Project behavior', + async () => { + const result = json( + await request( + 'POST', + '/api/files/copy', + { + source: selection(workspaceOwner, [required(workspaceFile.id)]), + destination: { owner: projectOwner, folderId: null }, + }, + 'session' + ), + 201 + ) + const copied = remember(projectOwner, result) + assert.equal(copied.fileIds.length, 1) + const read = await request('GET', `${projectPrefix}/${copied.fileIds[0]}/content`) + assert.equal(read.status, 200) + assert.equal(read.text, content) + } + ) + await check('a file identity from another owner is concealed', async () => { + const result = json( + await request('POST', '/api/v2/files/copy', { + ...body, + source: selection(projectOwner, [required(workspaceFile.id)]), + }), + 404 + ) + assert.equal(toRecord(result.error).code, 'NOT_FOUND') + }) + await check( + 'recursive selected folder copy preserves nesting and excludes unselected siblings', + async () => { + const root = toRecord( + json( + await request( + 'POST', + `/api/projects/${projectId}/files/folders`, + { + name: `tree-${generateId()}`, + }, + 'session' + ), + 200 + ).folder + ) + created.push({ owner: projectOwner, fileIds: [], folderIds: [required(root.id)] }) + const nested = toRecord( + json( + await request( + 'POST', + `/api/projects/${projectId}/files/folders`, + { + name: 'nested', + parentId: required(root.id), + }, + 'session' + ), + 200 + ).folder + ) + json( + await request('POST', projectPrefix, { + name: 'inside.txt', + content: 'nested bytes', + folderPath: `/${required(root.name)}/nested`, + }), + 201 + ) + const result = toRecord( + json( + await request('POST', '/api/v2/files/copy', { + source: selection(projectOwner, [], [required(root.id)]), + destination: { owner: workspaceOwner, folderId: null }, + }), + 201 + ).data + ) + const copied = remember(workspaceOwner, result) + assert.equal(copied.fileIds.length, 1) + assert.equal(copied.folderIds.length, 2) + assert.equal(toRecord(toArray(result.files)[0]).name, 'inside.txt') + assert.equal( + toArray(result.folders).some((folder) => toRecord(folder).id === nested.id), + false + ) + assert.equal( + (await request('GET', `/api/v2/files/${copied.fileIds[0]}?workspaceId=${workspaceId}`)) + .text, + 'nested bytes' + ) + } + ) +} finally { + for (const item of created.reverse()) { + const path = + item.owner.entityType === 'project' + ? `/api/projects/${item.owner.entityId}/files/archive` + : `/api/workspaces/${item.owner.entityId}/files/bulk-archive` + await request('POST', path, { fileIds: item.fileIds, folderIds: item.folderIds }, 'session') + } +} +if (checks.some((result) => !result.passed)) process.exitCode = 1 diff --git a/apps/sim/scripts/test-file-delivery-e2e.ts b/apps/sim/scripts/test-file-delivery-e2e.ts index ce72e3541d5..09b7caf31ca 100644 --- a/apps/sim/scripts/test-file-delivery-e2e.ts +++ b/apps/sim/scripts/test-file-delivery-e2e.ts @@ -4,6 +4,7 @@ import { dirname, join } from 'node:path' import { chromium } from '@playwright/test' import { getErrorMessage } from '@sim/utils/errors' import { toArray, toRecord } from '@sim/utils/object' +import JSZip from 'jszip' const required = (value: unknown): string => { assert.ok(typeof value === 'string' && value.length > 0, 'Missing delivery fixture setting') @@ -115,6 +116,12 @@ await check( assert.match(response.headers.get('content-disposition') ?? '', /^attachment;/) if (!name.endsWith('.svg')) assert.match(response.headers.get('content-security-policy') ?? '', /default-src/) + const download = await request(`${prefix}/download?fileIds=${required(file.id)}`) + assert.equal(download.status, 200) + assert.match(download.headers.get('content-disposition') ?? '', /^attachment;/) + assert.match(download.headers.get('content-security-policy') ?? '', /default-src/) + const archive = await JSZip.loadAsync(await download.arrayBuffer()) + assert.equal(await archive.file(required(file.name))?.async('string'), content) const head = await request(path, { method: 'HEAD' }) assert.equal(head.status, 200) assert.equal(await head.text(), '') @@ -186,7 +193,10 @@ await check( 'text/x-pdflibjs', 'throw new Error("HEAD must not compile")' ) - for (const path of [`/api/v2/projects/${projectId}/files/${required(generated.id)}/content`]) { + for (const path of [ + `/api/v2/projects/${projectId}/files/${required(generated.id)}/content`, + `/api/v2/projects/${projectId}/files/bulk-download?fileIds=${required(generated.id)}`, + ]) { const head = await request(path, { method: 'HEAD', headers }, true) assert.equal(head.status, 200) assert.equal(await head.text(), '') diff --git a/apps/sim/scripts/test-file-list-realtime-e2e.ts b/apps/sim/scripts/test-file-list-realtime-e2e.ts index 9dff42c7726..847478b96a6 100644 --- a/apps/sim/scripts/test-file-list-realtime-e2e.ts +++ b/apps/sim/scripts/test-file-list-realtime-e2e.ts @@ -416,6 +416,38 @@ try { }) } ) + await check('copy emits only to the canonical destination owner', async () => { + const workspaceFile = toRecord( + json( + await request(`/api/workspaces/${fixture.workspaceId}/files`, 'POST', { + name: 'copy-source.txt', + content: 'copy bytes', + contentType: 'text/plain', + encoding: 'utf-8', + }), + 201 + ).file + ) + await sleep(200) + const before = workspaceEvents.length + await changed(async () => { + json( + await request('/api/files/copy', 'POST', { + source: { + owner: { entityType: 'workspace', entityId: fixture.workspaceId }, + fileIds: [required(workspaceFile.id)], + folderIds: [], + }, + destination: { + owner: { entityType: 'project', entityId: fixture.projectId }, + folderId: null, + }, + }), + 201 + ) + }) + assert.equal(workspaceEvents.length, before) + }) await check( 'upload allocation stays silent and only completed registration invalidates', async () => { diff --git a/apps/sim/scripts/test-project-file-rendered-e2e.ts b/apps/sim/scripts/test-project-file-rendered-e2e.ts index fd3a4d5ca1b..852d23860e3 100644 --- a/apps/sim/scripts/test-project-file-rendered-e2e.ts +++ b/apps/sim/scripts/test-project-file-rendered-e2e.ts @@ -150,6 +150,10 @@ function artifactPath(file: CreatedFile) { return `/api/projects/${file.projectId}/files/${file.id}/artifact` } +function archivePath(file: CreatedFile) { + return `/api/v2/projects/${file.projectId}/files/bulk-download?fileIds=${encodeURIComponent(file.id)}` +} + async function create( ownerId: string, name: string, @@ -308,6 +312,18 @@ try { } ) } + await check('Personal-key bulk download returns the same compiled Office bytes', async () => { + const file = documents.get('docx') + const previous = rendered.get('docx') + assert.ok(file && previous, 'A successful initial render is required') + const response = await request('GET', archivePath(file)) + assert.equal(response.status, 200) + assert.equal(response.headers.get('content-type')?.split(';')[0], 'application/zip') + const archive = await JSZip.loadAsync(response.buffer) + const entry = archive.file(file.name) + assert.ok(entry) + assert.deepEqual(await entry.async('nodebuffer'), previous) + }) await check( 'Same-ID asset content revision invalidates the compiled document cache', async () => { @@ -345,6 +361,7 @@ try { sourceMimeTypes.docx ) assert.equal((await request('GET', artifactPath(file), 'session')).status, 404) + assert.equal((await request('GET', archivePath(file))).status, 404) }) await check( 'Cached private document bytes still reject anonymous and workspace-key callers', @@ -354,6 +371,8 @@ try { await readArtifact(file, 'docx') assert.equal((await request('GET', artifactPath(file), 'none')).status, 401) assert.equal((await request('GET', artifactPath(file), 'workspace')).status, 401) + assert.equal((await request('GET', archivePath(file), 'none')).status, 401) + assert.equal((await request('GET', archivePath(file), 'workspace')).status, 403) } ) await check( @@ -370,6 +389,7 @@ try { ) archived.add(asset.id) assert.equal((await request('GET', artifactPath(file), 'session')).status, 404) + assert.equal((await request('GET', archivePath(file))).status, 404) } ) await check( diff --git a/apps/sim/scripts/test-project-files-e2e.ts b/apps/sim/scripts/test-project-files-e2e.ts index 8804315717d..423806ae1f9 100644 --- a/apps/sim/scripts/test-project-files-e2e.ts +++ b/apps/sim/scripts/test-project-files-e2e.ts @@ -303,6 +303,7 @@ const suites = [ ['browser', 'PROJECT_FILE_BROWSER', 'test-project-file-browser-e2e.ts'], ['history', 'PROJECT_FILE_HISTORY', 'test-project-file-history-e2e.ts'], ['rendered', 'PROJECT_FILE_RENDERED', 'test-project-file-rendered-e2e.ts'], + ['copy', 'FILE_COPY', 'test-file-copy-e2e.ts'], ['realtime', 'FILE_LIST_REALTIME', 'test-file-list-realtime-e2e.ts'], ] as const diff --git a/knip.jsonc b/knip.jsonc index a5142e18537..99e66d29f56 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -55,6 +55,14 @@ "lib/api/contracts/project-file-folders.ts": ["types"], "lib/api/contracts/project-file-lifecycle.ts": ["types"], "sandboxes/index.ts": ["files"], + "lib/api/contracts/file-copy.ts": ["exports", "types"], + "lib/api/contracts/file-copy-input.ts": ["exports", "types"], + // API contract conventions require named boundary schemas and wire aliases before client adoption. + "lib/api/contracts/project-file-downloads.ts": ["exports", "types"], + "lib/api/contracts/project-file-extraction.ts": ["exports", "types"], + "lib/api/contracts/v2/project-file-downloads.ts": ["exports", "types"], + "lib/api/contracts/v2/project-file-extraction.ts": ["exports", "types"], + "lib/api/contracts/v2/project-file-search.ts": ["exports", "types"], "components/mcp/index.ts": ["files"], "triggers/quickbooks/index.ts": ["files"], "lib/api/contracts/selectors/index.ts": ["files"], diff --git a/packages/sim-cli/src/commands/protocol/files-get.ts b/packages/sim-cli/src/commands/protocol/files-get.ts index 72685874fcb..bf8304c7815 100644 --- a/packages/sim-cli/src/commands/protocol/files-get.ts +++ b/packages/sim-cli/src/commands/protocol/files-get.ts @@ -16,6 +16,7 @@ import { } from '../../http/client' import { describeOperation } from '../../runtime/build' import { apiCommand, type Connection } from '../../runtime/called-operations' +import { buildRequest, readArgumentSource } from '../../runtime/request' import { printProtocolResult } from './result' function writeFailure(path: WriteStream['path'], error: unknown): SimApiError { @@ -291,6 +292,8 @@ interface DownloadOutputOptions { type DownloadOperation = | 'downloadFile' | 'downloadFileVersion' + | 'downloadProjectFileItems' + | 'exportProjectFileSnapshot' | 'readProjectFileContent' | 'readProjectFileVersionContent' @@ -347,6 +350,73 @@ async function downloadToOutput( }) } +interface ProjectFileDownloadOptions extends DownloadOutputOptions { + fileIds?: string[] + folderIds?: string[] +} + +/** Owner-specific selection uses the same bounded transfer and atomic output path as file reads. */ +export function attachProjectFileDownload(files: Command): void { + const [command, connect] = apiCommand(files, 'bulk-download', ['downloadProjectFileItems']) + command + .argument('', 'Project that owns the files') + .allowExcessArguments(false) + .description( + describeOperation( + V2_OPERATIONS.downloadProjectFileItems, + 'Download Project files and recursive folder contents as a zip archive' + ) + ) + .option('--file-ids ', 'File identifiers to include') + .option('--folder-ids ', 'Folder identifiers to include recursively') + .option('-o, --output-file ', 'Write the archive to a file instead of stdout') + .option('--force', 'Overwrite --output-file if it already exists') + .action(async (projectId: string, options: ProjectFileDownloadOptions) => { + const input = await buildRequest( + 'downloadProjectFileItems', + [projectId], + { ...options }, + null + ) + await downloadToOutput(connect, 'downloadProjectFileItems', { projectId }, options, 'owner', { + query: input.query, + }) + }) +} + +interface ProjectFileSnapshotOptions extends DownloadOutputOptions { + content: string +} + +/** Snapshot text comes from the caller's file reader, including the embedded workbench boundary. */ +export function attachProjectFileSnapshotExport(files: Command): void { + const [command, connect] = apiCommand(files, 'export', ['exportProjectFileSnapshot']) + command + .argument('', 'Project that owns the file') + .argument('', 'Markdown file whose visible snapshot to export') + .allowExcessArguments(false) + .description( + describeOperation( + V2_OPERATIONS.exportProjectFileSnapshot, + 'Export a visible Project Markdown snapshot with its embedded assets' + ) + ) + .requiredOption('--content ', 'Visible Markdown content or a file to read') + .option('-o, --output-file ', 'Write the export to a file instead of stdout') + .option('--force', 'Overwrite --output-file if it already exists') + .action(async (projectId: string, fileId: string, options: ProjectFileSnapshotOptions) => { + const source = await readArgumentSource(options.content, 'content') + await downloadToOutput( + connect, + 'exportProjectFileSnapshot', + { projectId, fileId }, + options, + 'owner', + { body: { content: source.text } } + ) + }) +} + export function attachFileGet(files: Command): void { const [command, connect] = apiCommand(files, 'get', ['downloadFile']) command diff --git a/packages/sim-cli/src/commands/protocol/index.ts b/packages/sim-cli/src/commands/protocol/index.ts index db2a27dff8a..edc1209833f 100644 --- a/packages/sim-cli/src/commands/protocol/index.ts +++ b/packages/sim-cli/src/commands/protocol/index.ts @@ -3,6 +3,8 @@ import { attachChat } from './chat' import { attachFileGet, attachFileVersionDownload, + attachProjectFileDownload, + attachProjectFileSnapshotExport, attachProjectFileSource, attachProjectFileVersionSource, } from './files-get' @@ -30,6 +32,8 @@ export function attachProtocolCommands(program: Command): void { const projectFiles = group(group(program, 'projects'), 'files') attachProjectFileUpload(projectFiles) attachProjectFileSource(projectFiles) + attachProjectFileDownload(projectFiles) + attachProjectFileSnapshotExport(projectFiles) attachProjectFileVersionSource(group(projectFiles, 'versions')) const files = group(program, 'files') attachFileUpload(files) diff --git a/packages/sim-cli/src/contract/commands.ts b/packages/sim-cli/src/contract/commands.ts index b51a87bdb16..2498dcd88ce 100644 --- a/packages/sim-cli/src/contract/commands.ts +++ b/packages/sim-cli/src/contract/commands.ts @@ -795,6 +795,13 @@ export const CLI_CONTRACT: CliContract = { }, }, }, + copyFileItems: { + command: 'files copy', + flags: { + source: { json: true, describe: 'Source owner and selected fileIds or folderIds' }, + destination: { json: true, describe: 'Destination owner and optional folderId' }, + }, + }, renameProjectFile: { command: 'projects files rename' }, listProjectFileFolders: { command: 'projects files folders list', @@ -1739,6 +1746,11 @@ export const CLI_CONTRACT: CliContract = { describe: 'Unzip an archive into a new folder beside it', confirm: 'This writes every file in the archive into the workspace.', }, + unzipProjectFile: { + command: 'projects files unzip', + describe: 'Unzip an archive into a new folder beside it in the Project', + confirm: 'This writes every file in the archive into the Project.', + }, /** * Configured even though `buildGeneratedCommands` skips it: it only builds * operations whose `responseMode` is `json` (runtime/build.ts), so this @@ -1754,6 +1766,18 @@ export const CLI_CONTRACT: CliContract = { folderPaths: FOLDER_PATHS_FLAG, }, }, + downloadProjectFileItems: { + command: 'projects files bulk-download', + describe: 'Download Project files and folders as a zip archive', + flags: { + fileIds: { list: true }, + folderIds: { list: true }, + }, + }, + exportProjectFileSnapshot: { + command: 'projects files export', + describe: 'Export a visible Project Markdown snapshot with its embedded assets', + }, editFileContent: { command: 'files edit', describe: 'Apply one exact or anchor-based edit to a text file', diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 1132e8338d7..f9bdda640cb 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -1820,6 +1820,67 @@ export type CompleteTableImportResponse = { data: CompleteTableImportResponseRef4 } +/** `POST /api/v2/files/copy` */ +export type CopyFileItemsQuery = Record + +export type CopyFileItemsBody = { + source: { + owner: { + entityType: 'workspace' | 'project' + entityId: string + } + fileIds?: Array + folderIds?: Array + } + destination: { + owner: { + entityType: 'workspace' | 'project' + entityId: string + } + folderId?: string | null + } +} + +export type CopyFileItemsResponse = { + data: { + files: Array<{ + id: string + name: string + size: number + type: string + width?: number | null + height?: number | null + uploadedBy: string + folderId: string | null + folderPath?: string | null + deletedAt: string | null + uploadedAt: string + updatedAt: string + contentUpdatedAt: string | null + revision: string + owner: { + entityType: 'workspace' | 'project' + entityId: string + } + }> + folders: Array<{ + id: string + userId: string + name: string + parentId: string | null + path: string + sortOrder: number + deletedAt: string | null + createdAt: string + updatedAt: string + owner: { + entityType: 'workspace' | 'project' + entityId: string + } + }> + } +} + /** `POST /api/v2/credentials/connections` */ export type CreateCredentialConnectionQuery = Record @@ -5139,6 +5200,19 @@ export type DownloadFileVersionQuery = { /** Non-JSON response (`binary`). */ export type DownloadFileVersionResponse = never +/** `GET /api/v2/projects/[projectId]/files/bulk-download` */ +export type DownloadProjectFileItemsParams = { + projectId: string +} + +export type DownloadProjectFileItemsQuery = { + fileIds?: string + folderIds?: string +} + +/** Non-JSON response (`binary`). */ +export type DownloadProjectFileItemsResponse = never + /** `GET /api/v2/workflows/[workflowId]/runs/[runId]/files/[fileId]` */ export type DownloadRunFileParams = { workflowId: string @@ -5369,6 +5443,21 @@ export type ExportKnowledgeBaseQuery = { /** Non-JSON response (`binary`). */ export type ExportKnowledgeBaseResponse = never +/** `POST /api/v2/projects/[projectId]/files/[fileId]/export` */ +export type ExportProjectFileSnapshotParams = { + projectId: string + fileId: string +} + +export type ExportProjectFileSnapshotQuery = Record + +export type ExportProjectFileSnapshotBody = { + content: string +} + +/** Non-JSON response (`binary`). */ +export type ExportProjectFileSnapshotResponse = never + /** `GET /api/v2/workflows/[workflowId]/export` */ export type ExportWorkflowParams = { workflowId: string @@ -13884,6 +13973,26 @@ export type UnzipFileResponse = { data: UnzipFileResponseRef0 } +/** `POST /api/v2/projects/[projectId]/files/[fileId]/unzip` */ +export type UnzipProjectFileParams = { + projectId: string + fileId: string +} + +export type UnzipProjectFileQuery = Record + +type UnzipProjectFileResponseRef0 = UnzipProjectFileResponseRef1 + +type UnzipProjectFileResponseRef1 = { + folderPath: string + extractedFileCount: number + skippedFileCount: number +} + +export type UnzipProjectFileResponse = { + data: UnzipProjectFileResponseRef0 +} + /** `PATCH /api/v2/credentials/[credentialId]` */ export type UpdateCredentialParams = { credentialId: string @@ -16195,6 +16304,26 @@ export const V2_OPERATIONS = { }, }, }, + copyFileItems: { + method: 'POST', + path: '/api/v2/files/copy', + pathParams: [] as const, + responseMode: 'json', + summary: 'Copy File Items', + workspaceKeyUnsupported: true, + body: { + source: { + kind: 'object', + required: true, + describe: 'Selection to read under the source owner.', + }, + destination: { + kind: 'object', + required: true, + describe: 'Destination requiring file write access.', + }, + }, + }, createCredentialConnection: { method: 'POST', path: '/api/v2/credentials/connections', @@ -17834,6 +17963,26 @@ export const V2_OPERATIONS = { workspaceId: { kind: 'string', required: true, describe: 'Workspace that owns the file.' }, }, }, + downloadProjectFileItems: { + method: 'GET', + path: '/api/v2/projects/[projectId]/files/bulk-download', + pathParams: ['projectId'] as const, + pathParamDocs: { projectId: 'Project identifier.' }, + responseMode: 'binary', + summary: 'Download Project File Items', + workspaceKeyUnsupported: true, + query: { + fileIds: { + kind: 'string', + describe: 'File identifiers to include, comma-separated. At most 100 entries.', + }, + folderIds: { + kind: 'string', + describe: + 'Folder identifiers to include recursively, comma-separated. The resolved selection allows at most 100 files.', + }, + }, + }, downloadRunFile: { method: 'GET', path: '/api/v2/workflows/[workflowId]/runs/[runId]/files/[fileId]', @@ -18016,6 +18165,26 @@ export const V2_OPERATIONS = { }, }, }, + exportProjectFileSnapshot: { + method: 'POST', + path: '/api/v2/projects/[projectId]/files/[fileId]/export', + pathParams: ['projectId', 'fileId'] as const, + pathParamDocs: { + projectId: 'Project identifier.', + fileId: 'File identifier within the Project.', + }, + responseMode: 'binary', + summary: 'Export Project File Snapshot', + workspaceKeyUnsupported: true, + body: { + content: { + kind: 'string', + required: true, + describe: + 'Visible Markdown snapshot to export. This does not replace the stored file or create a version.', + }, + }, + }, exportWorkflow: { method: 'GET', path: '/api/v2/workflows/[workflowId]/export', @@ -23439,6 +23608,18 @@ export const V2_OPERATIONS = { workspaceId: { kind: 'string', required: true, describe: 'Workspace that owns the archive.' }, }, }, + unzipProjectFile: { + method: 'POST', + path: '/api/v2/projects/[projectId]/files/[fileId]/unzip', + pathParams: ['projectId', 'fileId'] as const, + pathParamDocs: { + projectId: 'Project identifier.', + fileId: 'File identifier within the Project.', + }, + responseMode: 'json', + summary: 'Unzip Project File', + workspaceKeyUnsupported: true, + }, updateCredential: { method: 'PATCH', path: '/api/v2/credentials/[credentialId]', diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index 9a56cba9541..ef6beb2d219 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -419,15 +419,18 @@ describe('destructive operations are gated', () => { */ const NON_DESTRUCTIVE = new Set([ 'completeProjectFileUpload', + 'copyFileItems', 'createProjectFile', 'createProjectFileFolder', 'createProjectFileUpload', + 'exportProjectFileSnapshot', 'getProjectFileUploadPartUrls', 'moveProjectFileItems', 'renameProjectFile', 'restoreProjectFile', 'restoreProjectFileFolder', 'revertProjectFileVersion', + 'unzipProjectFile', 'updateProjectFileContent', 'updateProjectFileFolder', /** These amend access or request history without discarding a resource. */ diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index e1f9d134e00..5fc9df0e02c 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -3265,7 +3265,6 @@ "apps/sim/lib/api/contracts/v2/files.ts#v2RestoreFileFolderBodySchema", "apps/sim/lib/api/contracts/v2/files.ts#v2RestoreFileFolderDataSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2UnzipFileBodySchema", - "apps/sim/lib/api/contracts/v2/files.ts#v2UnzipFileDataSchema", "apps/sim/lib/api/contracts/v2/files.ts#v2WrittenFileSchema", "apps/sim/lib/api/contracts/v2/knowledge.ts#MAX_V2_BULK_KNOWLEDGE_DOCUMENTS", "apps/sim/lib/api/contracts/v2/knowledge.ts#MAX_V2_KNOWLEDGE_DOCUMENT_TAG_FILTERS", @@ -3927,7 +3926,6 @@ "apps/sim/lib/api/contracts/workflows.ts#workflowVariableReadSchema", "apps/sim/lib/api/contracts/workflows.ts#workflowVariableWriteSchema", "apps/sim/lib/api/contracts/workflows.ts#workflowVariablesBodySchema", - "apps/sim/lib/api/contracts/workspace-file-folders.ts#downloadWorkspaceFileItemsQuerySchema", "apps/sim/lib/api/contracts/workspace-file-folders.ts#workspaceFileFolderParamsSchema", "apps/sim/lib/api/contracts/workspace-file-folders.ts#workspaceFileFolderScopeSchema", "apps/sim/lib/api/contracts/workspace-file-folders.ts#workspaceFileFoldersParamsSchema", @@ -3936,7 +3934,6 @@ "apps/sim/lib/api/contracts/workspace-file-table.ts#workspaceCsvPreviewParamsSchema", "apps/sim/lib/api/contracts/workspace-files.ts#ExportWorkspaceFileSnapshotBody", "apps/sim/lib/api/contracts/workspace-files.ts#UpdateWorkspaceFileDimensionsBody", - "apps/sim/lib/api/contracts/workspace-files.ts#extractWorkspaceFileResponseSchema", "apps/sim/lib/api/contracts/workspace-files.ts#listWorkspaceFilesQuerySchema", "apps/sim/lib/api/contracts/workspace-files.ts#renameWorkspaceFileErrorSchema", "apps/sim/lib/api/contracts/workspace-files.ts#updateWorkspaceFileDimensionsBodySchema", @@ -5046,7 +5043,6 @@ "apps/sim/lib/tokenization/streaming.ts#processStreamingBlockLog", "apps/sim/lib/tokenization/types.ts#TokenizationInput", "apps/sim/lib/uploads/archive.ts#MAX_ARCHIVE_ENTRIES", - "apps/sim/lib/uploads/archive.ts#MAX_ARCHIVE_TOTAL_BYTES", "apps/sim/lib/uploads/client/upload-session.ts#UploadSessionTransportError", "apps/sim/lib/uploads/contexts/copilot/file-manager.ts#isSupportedFileType", "apps/sim/lib/uploads/contexts/copilot/index.ts#CopilotStoredFile",