From 2f8bc3dfcd5f0e5320df3b8381caf677d1c7c066 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 16:38:51 -0400 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=8E=9E=EF=B8=8F=20fix:=20Deliver=20Op?= =?UTF-8?q?ted-In=20Custom=20Endpoint=20Media=20Safely=20(#15937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: send video/audio to custom endpoints that opt in via supportedMimeTypes The video and audio encoders only emitted OpenAI-format parts (video_url, input_audio) for OpenRouter, and Google's media part for Google/Vertex. For every other provider the file was validated and then silently dropped, so a custom OpenAI-compatible endpoint (vLLM, LiteLLM, ...) never received the attachment even though the wire format is identical to OpenRouter's. Emit the OpenAI-format part for OpenAI-like providers when the endpoint's fileConfig supportedMimeTypes explicitly allows the file's type. The inherited default list does not count as opting in (isExplicitMimeConfig, keyed on referential identity like the client's picker), so endpoints whose gateway cannot handle media are unaffected. * feat: let a configured custom endpoint pick and attach video/audio The attach menu's Upload to Provider filter and the drag-drop viability check only opened video/audio for Google and OpenRouter, or for a custom endpoint with a fully permissive (.*) supportedMimeTypes. Honor any admin-configured allowlist on a custom endpoint instead, so a finite list that includes video/.* or audio/.* is enough, matching the server-side encoders. * fix: canonicalize input_audio format from MIME type The `input_audio.format` value was derived from the filename extension via `filename.split('.').pop()`, which produces values providers reject: - `clip.wave` (audio/wave) emitted `wave`, but the accepted format is `wav` - a file with no extension (`recording`) emitted the whole filename, since splitting a dotless string yields a single-element array, so the existing non-empty guard never fired - `clip.mpeg` (audio/mpeg) emitted `mpeg` rather than `mp3` All of these pass MIME validation and then fail at the provider. Derive the format from the MIME type instead, mirroring the canonicalization already in STTService, and fall back to the filename extension only when it is itself a supported format. Throws when neither source yields one rather than sending an unsupported value. Co-Authored-By: Claude Opus 5 (1M context) * fix: route opted-in custom endpoint media to the provider path In unified upload mode an upload carries no tool resource, so its delivery path is inferred by resolveDefaultLLMDeliveryPath. That resolver judged audio and video for any named endpoint with isMediaSupportedProvider, which lists only Google, Vertex and OpenRouter, so a custom endpoint that opted into media through supportedMimeTypes still recorded video as none and audio as text, and the encoder branch was never reached. The upload resolver already holds the endpoint's file config, so it now passes the endpoint's supportedMimeTypes through, and an explicit match on an OpenAI-compatible or custom endpoint counts as provider-capable, mirroring isConfiguredProviderMediaType on the encoder side. The inherited default list is still not an opt-in, and known providers whose encoders emit nothing for media are unchanged. BaseClient re-resolves per turn with the same function, so the turn path follows. * fix: scope the media opt-in to custom endpoints that run as OpenAI The opt-in applied to the built-in openAI and azureOpenAI endpoints too, but the picker and drag-drop only open media for custom endpoints, so that was a route the client could not send to. It is now limited to endpoints that are not a known provider. A custom endpoint may also declare provider: anthropic, in which case initialization runs it as Anthropic and the encoders emit no media part for it. getCustomEndpointProvider reads that declaration off the app config, the upload callers pass it as endpointProvider, the agent encoder and BaseClient pass the agent's resolved provider, and an endpoint known to run as something other than OpenAI keeps its previous route: video stays off the model path and audio still reaches transcription. * fix: reject unsupported provider audio during upload --------- Co-authored-by: Renato Garita Figueiredo Co-authored-by: Claude Opus 5 (1M context) --- api/app/clients/BaseClient.js | 7 + api/server/routes/files/files.agents.test.js | 27 +++ api/server/routes/files/images.js | 2 + api/server/services/Files/process.js | 3 + .../Chat/Input/Files/AttachFileMenu.tsx | 18 +- .../Files/__tests__/AttachFileMenu.spec.tsx | 40 ++++ .../Files/__tests__/useFileHandling.test.ts | 36 ++- client/src/locales/en/translation.json | 1 + .../__tests__/getViableUploadOptions.spec.ts | 27 +++ client/src/utils/files.ts | 10 +- packages/api/src/agents/files/encode.ts | 1 + packages/api/src/files/encode/audio.spec.ts | 211 ++++++++++++++++++ packages/api/src/files/encode/audio.ts | 27 ++- packages/api/src/files/encode/utils.spec.ts | 116 +++++++++- packages/api/src/files/encode/utils.ts | 108 ++++++++- packages/api/src/files/encode/video.spec.ts | 130 +++++++++++ packages/api/src/files/encode/video.ts | 12 +- packages/api/src/files/upload/errors.ts | 9 + packages/api/src/files/upload/routing.spec.ts | 144 +++++++++++- packages/api/src/files/upload/routing.ts | 32 ++- packages/api/src/types/files.ts | 4 +- packages/api/src/utils/files.spec.ts | 9 + packages/api/src/utils/files.ts | 4 + .../data-provider/src/file-config.spec.ts | 30 +++ packages/data-provider/src/file-config.ts | 12 + .../src/resolve-llm-delivery-path.spec.ts | 116 ++++++++++ .../src/resolve-llm-delivery-path.ts | 80 ++++++- 27 files changed, 1184 insertions(+), 32 deletions(-) create mode 100644 packages/api/src/files/encode/audio.spec.ts create mode 100644 packages/api/src/files/encode/video.spec.ts create mode 100644 packages/api/src/files/upload/errors.ts diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 5ec6e7db960..81f89987604 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -45,6 +45,7 @@ const { resolveUploadLLMDeliveryPath, isSpeechProviderConfigured, resolveUseResponsesApi, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logViolation } = require('~/cache'); @@ -1826,6 +1827,12 @@ class BaseClient { endpointConfig: this._endpointFileConfig, fileConfig: this._mergedFileConfig, endpoint: this._deliveryEndpoint, + endpointProvider: + this.options.agent?.provider ?? + getCustomEndpointProvider( + this.options.req?.config?.endpoints?.custom, + this._deliveryEndpoint, + ), useResponsesApi: this.usesResponsesApi(), sttConfigured: isSpeechProviderConfigured(this.options.req?.config?.speech?.stt), }); diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.js index 4323a28e817..45b082601d1 100644 --- a/api/server/routes/files/files.agents.test.js +++ b/api/server/routes/files/files.agents.test.js @@ -427,6 +427,33 @@ describe('File Routes - Agent Files Endpoint', () => { return testApp; }; + it.each([ + [false, 'application/json'], + [true, 'text/event-stream'], + ])( + 'rejects unsupported provider audio before persistence (legacy=%s, accept=%s)', + async (legacyFileUploadUX, accept) => { + const testApp = createAppWithUser( + otherUserId, + SystemRoles.USER, + { + fileConfig: { + endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } }, + }, + }, + { originalname: 'clip.wma', mimetype: 'audio/wma' }, + ); + const response = await request(testApp).post('/files').set('Accept', accept).send({ + endpoint: 'MyGateway', + file_id: uuidv4(), + }); + expect(response.status).toBe(415); + expect(response.body.message).toBe('com_error_files_provider_audio_format'); + expect(processAgentFileUpload).not.toHaveBeenCalled(); + expect(fs.unlink).toHaveBeenCalledWith('/tmp/test.txt'); + }, + ); + it('inspects the canonical sanitized filename used by upload processing', async () => { await createAgent({ id: agentCustomId, diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.js index 7e5512f0cd1..435027a3b0c 100644 --- a/api/server/routes/files/images.js +++ b/api/server/routes/files/images.js @@ -23,6 +23,7 @@ const { resolveUploadLLMDeliveryPath, isResponsesApiUpload, isSpeechProviderConfigured, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { processAgentFileUpload, @@ -107,6 +108,7 @@ router.post('/', async (req, res) => { endpointConfig: getEndpointFileConfig({ fileConfig, endpoint: effectiveEndpoint }), fileConfig, endpoint: effectiveEndpoint, + endpointProvider: getCustomEndpointProvider(req.config?.endpoints?.custom, effectiveEndpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi), sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt), }); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 0a5705e60c3..7ddf9ab7295 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -25,6 +25,7 @@ const { isMessageFileUpload, isResponsesApiUpload, isSpeechProviderConfigured, + getCustomEndpointProvider, } = require('librechat-data-provider'); const { logger, runAsSystem } = require('@librechat/data-schemas'); const { @@ -494,6 +495,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStr endpointConfig, fileConfig, endpoint: configEndpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, configEndpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), }); @@ -810,6 +812,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => { endpointConfig, fileConfig, endpoint, + endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint), useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi), sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt), }); diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 71f89b03ee4..ead2d57bc1d 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -20,6 +20,7 @@ import { Providers, EToolResources, EModelEndpoint, + isExplicitMimeConfig, getConfiguredMimeAccept, bedrockDocumentMimeTypes, defaultAgentCapabilities, @@ -52,7 +53,8 @@ type FileUploadType = | 'document' | 'image_document' | 'image_document_extended' - | 'image_document_video_audio'; + | 'image_document_video_audio' + | 'image_document_video_audio_configured'; /** What each provider upload path can actually send, used to scope the picker filter to selectable files. */ const fileTypeCapabilities: Record = { @@ -68,6 +70,10 @@ const fileTypeCapabilities: Record = { categories: ['image', 'document', 'audio', 'video'], documentMimeTypes: ['application/pdf'], }, + /** Custom endpoint with an admin-configured allowlist: the config decides, including video/audio. */ + image_document_video_audio_configured: { + categories: ['image', 'document', 'audio', 'video'], + }, }; interface AttachFileMenuProps { @@ -164,6 +170,10 @@ const AttachFileMenu = ({ inputRef.current.accept = `image/*,.heif,.heic,${bedrockDocumentExtensions}`; } else if (fileType === 'image_document_video_audio') { inputRef.current.accept = 'image/*,.heif,.heic,.pdf,application/pdf,video/*,audio/*'; + } else if (fileType === 'image_document_video_audio_configured') { + /** Only reached with an explicit allowlist the accept string cannot represent; leave the + * picker open so it never hides a file the admin allowed (backend still enforces). */ + inputRef.current.accept = ''; } else { inputRef.current.accept = ''; } @@ -237,6 +247,11 @@ const AttachFileMenu = ({ endpointType === EModelEndpoint.bedrock ) { fileType = 'image_document_extended'; + } else if ( + endpointType === EModelEndpoint.custom && + isExplicitMimeConfig(endpointFileConfig?.supportedMimeTypes) + ) { + fileType = 'image_document_video_audio_configured'; } onAction(fileType); }, @@ -324,6 +339,7 @@ const AttachFileMenu = ({ handleUploadClick, setEphemeralAgent, sharePointEnabled, + endpointFileConfig?.supportedMimeTypes, codeAllowedByAgent, fileSearchAllowedByAgent, setIsSharePointDialogOpen, diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx index 13d60564e41..7fcd7ba32bf 100644 --- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx @@ -208,6 +208,46 @@ describe('AttachFileMenu', () => { }); }); + describe('Upload to Provider picker filter', () => { + /** `accept` is reset right after `click()`, so capture it at click time. */ + const captureAcceptOnClick = () => { + const seen: string[] = []; + const spy = jest.spyOn(HTMLInputElement.prototype, 'click').mockImplementation(function ( + this: HTMLInputElement, + ) { + seen.push(this.accept); + }); + return { seen, spy }; + }; + + it('keeps images and PDFs only for a custom endpoint with the inherited default config', () => { + const { seen, spy } = captureAcceptOnClick(); + setupMocks({ provider: 'MyGateway' }); + renderMenu({ endpointType: EModelEndpoint.custom, endpointFileConfig: {} }); + openMenu(); + fireEvent.click(screen.getByText('Upload to Provider')); + expect(seen).toEqual(['image/*,.heif,.heic,.pdf,application/pdf']); + spy.mockRestore(); + }); + + it('adds video when a custom endpoint config explicitly allows it', () => { + const { seen, spy } = captureAcceptOnClick(); + setupMocks({ provider: 'MyGateway' }); + renderMenu({ + endpointType: EModelEndpoint.custom, + endpointFileConfig: { + supportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/], + }, + }); + openMenu(); + fireEvent.click(screen.getByText('Upload to Provider')); + expect(seen).toHaveLength(1); + expect(seen[0]).toContain('video/*'); + expect(seen[0]).not.toContain('audio/*'); + spy.mockRestore(); + }); + }); + describe('Upload to Provider vs Upload Image', () => { it('shows "Upload to Provider" when endpointType is custom (resolved from agent provider)', () => { setupMocks({ provider: 'Moonshot' }); diff --git a/client/src/hooks/Files/__tests__/useFileHandling.test.ts b/client/src/hooks/Files/__tests__/useFileHandling.test.ts index bcbc2e1b564..e741324f7dc 100644 --- a/client/src/hooks/Files/__tests__/useFileHandling.test.ts +++ b/client/src/hooks/Files/__tests__/useFileHandling.test.ts @@ -1,4 +1,4 @@ -import { renderHook, act } from '@testing-library/react'; +import { renderHook, act, waitFor } from '@testing-library/react'; import { megabyte, Constants, @@ -210,6 +210,40 @@ describe('useFileHandling', () => { const loadHook = async () => (await import('../useFileHandling')).default; + it('removes rejected provider audio and localizes the upload error before retry', async () => { + const consoleLog = jest.spyOn(console, 'log').mockImplementation(() => undefined); + const useFileHandling = await loadHook(); + const { result } = renderHook(() => useFileHandling()); + const recovery = jest.fn(); + await act(async () => { + await result.current.handleFiles( + [new File(['audio'], 'clip.wma', { type: 'audio/wma' })], + undefined, + { onError: recovery }, + ); + }); + const body = mockMutate.mock.calls[0][0] as FormData; + const fileId = body.get('file_id'); + act(() => + mockUploadOptions.onError?.( + { + response: { status: 415, data: { message: 'com_error_files_provider_audio_format' } }, + }, + body, + ), + ); + expect(mockDeleteFileById).toHaveBeenCalledWith(fileId); + await waitFor(() => + expect(mockLocalize).toHaveBeenCalledWith('com_error_files_provider_audio_format'), + ); + expect(recovery).toHaveBeenCalledWith(fileId); + await act(async () => { + await result.current.handleFiles([new File(['audio'], 'clip.wav', { type: 'audio/wav' })]); + }); + expect(mockMutate).toHaveBeenCalledTimes(2); + consoleLog.mockRestore(); + }); + describe('endpointOverride', () => { it('clears the loading state when file validation throws', async () => { const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined); diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 64de2fe31b1..6ca1ba26a7a 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1089,6 +1089,7 @@ "com_ui_attach_error_size": "File size limit exceeded for endpoint:", "com_ui_attach_error_total_size": "Total file size limit exceeded for endpoint:", "com_ui_attach_error_type": "Unsupported file type for endpoint:", + "com_error_files_provider_audio_format": "This audio format cannot be attached directly to this model. Convert the file to WAV or MP3 and upload it again, or choose Upload as Text if transcription is configured.", "com_ui_attach_remove": "Remove file", "com_ui_attach_warn_endpoint": "Non-Assistant files may be ignored without a compatible tool", "com_ui_attached_image": "Attached image", diff --git a/client/src/utils/__tests__/getViableUploadOptions.spec.ts b/client/src/utils/__tests__/getViableUploadOptions.spec.ts index 261ea70e401..6d2ea63580d 100644 --- a/client/src/utils/__tests__/getViableUploadOptions.spec.ts +++ b/client/src/utils/__tests__/getViableUploadOptions.spec.ts @@ -132,6 +132,33 @@ describe('getViableUploadOptions', () => { expect(getViableUploadOptions([file(XLSX, 'report.xlsx')], ctx)).toEqual([undefined]); }); + it('offers direct attach for a video when the custom config explicitly allows video', () => { + const ctx = baseCtx({ + provider: 'MyGateway', + endpoint: 'MyGateway', + endpointType: 'custom', + fileSearchEnabled: false, + codeEnabled: false, + contextEnabled: false, + endpointSupportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/], + }); + expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([undefined]); + expect(getViableUploadOptions([file('audio/wav', 'tone.wav')], ctx)).toEqual([]); + }); + + it('does not offer video for a custom endpoint that inherits the default config', () => { + const ctx = baseCtx({ + provider: 'MyGateway', + endpoint: 'MyGateway', + endpointType: 'custom', + fileSearchEnabled: false, + codeEnabled: false, + contextEnabled: false, + endpointSupportedMimeTypes: undefined, + }); + expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([]); + }); + it('does not treat a non-permissive custom config as broad provider support', () => { const ctx = baseCtx({ provider: 'MyGateway', diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts index 62e77dacafd..857742f98af 100644 --- a/client/src/utils/files.ts +++ b/client/src/utils/files.ts @@ -16,7 +16,7 @@ import { EModelEndpoint, retrievalMimeTypes, isBedrockDocumentType, - isPermissiveMimeConfig, + isExplicitMimeConfig, codeInterpreterMimeTypes, isDocumentSupportedProvider, fileConfig as defaultFileConfig, @@ -522,12 +522,12 @@ const isProviderAttachType = (type: string, ctx: UploadOptionContext): boolean = isDocumentSupportedProvider(currentProvider) || isAzureWithResponsesApi ) { - /** Custom endpoints that the admin opened up (permissive config) honor that allowlist, - * matching the file picker; an inherited default config is not treated as opened up. */ + /** Custom endpoints with an admin-configured allowlist honor it for direct attach (this is + * how video/audio get opted in for an OpenAI-compatible gateway), matching the file picker + * and the server-side encoders; an inherited default config is not treated as opened up. */ if ( ctx.endpointType === EModelEndpoint.custom && - ctx.endpointSupportedMimeTypes != null && - isPermissiveMimeConfig(ctx.endpointSupportedMimeTypes) + isExplicitMimeConfig(ctx.endpointSupportedMimeTypes) ) { return checkType(type, ctx.endpointSupportedMimeTypes); } diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.ts index 63b273bd76e..d4de921577a 100644 --- a/packages/api/src/agents/files/encode.ts +++ b/packages/api/src/agents/files/encode.ts @@ -103,6 +103,7 @@ export function createRunFileMessageEncoder( endpointConfig, fileConfig, endpoint, + endpointProvider: agent.provider, useResponsesApi, sttConfigured: isSpeechProviderConfigured(deps.req.config?.speech?.stt), }), diff --git a/packages/api/src/files/encode/audio.spec.ts b/packages/api/src/files/encode/audio.spec.ts new file mode 100644 index 00000000000..e795042fd6f --- /dev/null +++ b/packages/api/src/files/encode/audio.spec.ts @@ -0,0 +1,211 @@ +import { Providers } from '@librechat/agents'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; +import { encodeAndFormatAudios } from './audio'; + +jest.mock('~/files/validation', () => ({ + validateAudio: jest.fn(), +})); + +jest.mock('./utils', () => ({ + ...jest.requireActual('./utils'), + getFileStream: jest.fn(), + getConfiguredFileSizeLimit: jest.fn(), + isConfiguredProviderMediaType: jest.fn(), +})); + +jest.mock('./memoryGuard', () => ({ + runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()), +})); + +import { validateAudio } from '~/files/validation'; +import { getFileStream, isConfiguredProviderMediaType } from './utils'; +import { Types } from 'mongoose'; + +const mockedValidateAudio = validateAudio as jest.MockedFunction; +const mockedGetFileStream = getFileStream as jest.MockedFunction; +const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction< + typeof isConfiguredProviderMediaType +>; + +const createMockFile = (type = 'audio/wav', filename = 'tone.wav'): IMongoFile => + ({ + _id: new Types.ObjectId(), + file_id: 'audio-1', + filename, + filepath: `/uploads/${filename}`, + type, + bytes: 1024, + source: 'local', + user: 'user-1', + object: 'file', + usage: 0, + createdAt: new Date(), + updatedAt: new Date(), + }) as unknown as IMongoFile; + +const req = { config: {} } as unknown as ServerRequest; +const getStrategyFunctions = jest.fn(); + +describe('encodeAndFormatAudios - provider formatting', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedValidateAudio.mockResolvedValue({ isValid: true }); + mockedIsConfigured.mockReturnValue(false); + const file = createMockFile(); + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }); + + it('emits a Google media block for google', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.GOOGLE }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([{ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }]); + }); + + it('emits an OpenAI-compatible input_audio block for openrouter without configuration', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).not.toHaveBeenCalled(); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + + it('drops the audio for a custom endpoint that is not configured for audio', async () => { + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).toHaveBeenCalledWith( + req, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + 'audio/wav', + ); + expect(result.audios).toEqual([]); + expect(result.files).toHaveLength(1); + }); + + it('emits an input_audio block for a custom endpoint whose config allows audio', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + + it('never emits a block for providers without an audio path, even when configured', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [createMockFile()], + { provider: Providers.ANTHROPIC }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([]); + }); + + describe('input_audio format canonicalization', () => { + /** Re-points the `getFileStream` mock at a specific file for one test. */ + const stageFile = (file: IMongoFile) => { + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }; + + const formatFor = async (file: IMongoFile) => { + stageFile(file); + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatAudios( + req, + [file], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + return (result.audios[0] as { input_audio?: { format?: string } })?.input_audio?.format; + }; + + it.each([ + ['audio/wave', 'clip.wave', 'wav'], + ['audio/x-wav', 'clip.x-wav', 'wav'], + ['audio/mpeg', 'clip.mpeg', 'mp3'], + ['audio/mpeg3', 'clip.mpeg3', 'mp3'], + ['audio/x-m4a', 'clip.x-m4a', 'm4a'], + ['audio/vorbis', 'clip.vorbis', 'ogg'], + ['audio/x-flac', 'clip.x-flac', 'flac'], + ])('canonicalizes %s to a provider-accepted format', async (type, filename, expected) => { + expect(await formatFor(createMockFile(type, filename))).toBe(expected); + }); + + it('derives the format from the MIME type when the filename has no extension', async () => { + expect(await formatFor(createMockFile('audio/mpeg', 'recording'))).toBe('mp3'); + }); + + it('prefers the canonical MIME format over a mismatched extension', async () => { + expect(await formatFor(createMockFile('audio/mpeg', 'clip.wav'))).toBe('mp3'); + }); + + it('falls back to a supported filename extension for MIME types with no mapping', async () => { + expect(await formatFor(createMockFile('audio/aac', 'clip.aac'))).toBe('aac'); + }); + + it('throws rather than emitting an unsupported format', async () => { + /** `audio/wma` passes MIME validation but has no provider-accepted format. */ + stageFile(createMockFile('audio/wma', 'recording')); + mockedIsConfigured.mockReturnValue(true); + await expect( + encodeAndFormatAudios( + req, + [createMockFile('audio/wma', 'recording')], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ), + ).rejects.toThrow(/supported audio format/i); + }); + + it('canonicalizes for openrouter as well', async () => { + stageFile(createMockFile('audio/wave', 'clip.wave')); + const result = await encodeAndFormatAudios( + req, + [createMockFile('audio/wave', 'clip.wave')], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(result.audios).toEqual([ + { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } }, + ]); + }); + }); +}); diff --git a/packages/api/src/files/encode/audio.ts b/packages/api/src/files/encode/audio.ts index fa155cd3b3e..ac51c8f7727 100644 --- a/packages/api/src/files/encode/audio.ts +++ b/packages/api/src/files/encode/audio.ts @@ -1,11 +1,13 @@ import { Providers } from '@librechat/agents'; -import { isDocumentSupportedProvider } from 'librechat-data-provider'; +import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest, StrategyFunctions, AudioResult } from '~/types'; import { getFileStream, + getAudioFormat, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, + isConfiguredProviderMediaType, } from './utils'; import { validateAudio } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; @@ -15,7 +17,7 @@ import { runGuardedEncode } from './memoryGuard'; * @param req - The request object * @param files - Array of audio files * @param params - Object containing provider and optional endpoint - * @param params.provider - The provider to format for (currently only google is supported) + * @param params.provider - The provider to format for * @param params.endpoint - Optional endpoint name for file config lookup * @param getStrategyFunctions - Function to get strategy functions * @returns Promise that resolves to audio and file metadata @@ -91,13 +93,22 @@ export async function encodeAndFormatAudios( mimeType: file.type, data: content, }); - } else if (provider === Providers.OPENROUTER) { - // Extract format from filename extension (e.g., 'audio.mp3' -> 'mp3') - // OpenRouter expects format values like: wav, mp3, aiff, aac, ogg, flac, m4a, pcm16, pcm24 - // Note: MIME types don't always match (e.g., 'audio/mpeg' is mp3, not mpeg), so that is why we are using the file extension instead - const format = file.filename.split('.').pop()?.toLowerCase(); + } else if ( + provider === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isConfiguredProviderMediaType(req, { provider, endpoint }, file.type)) + ) { + /** OpenAI-compatible `input_audio` part (OpenRouter, vLLM, and other gateways). + * Custom endpoints only get here when the admin listed audio types in the + * endpoint's `supportedMimeTypes`, since not every gateway accepts audio. */ + /** Canonicalized from the MIME type: the accepted MIME list contains aliases + * (`audio/wave`, `audio/mpeg`) that are not valid format values, and the filename + * may carry no extension at all. */ + const format = getAudioFormat(file.type, file.filename); if (!format) { - throw new Error(`Could not extract audio format from filename: ${file.filename}`); + throw new Error( + `Could not determine a supported audio format for "${file.filename}" (${file.type})`, + ); } result.audios.push({ type: 'input_audio', diff --git a/packages/api/src/files/encode/utils.spec.ts b/packages/api/src/files/encode/utils.spec.ts index 24367409c7a..d4882d952dd 100644 --- a/packages/api/src/files/encode/utils.spec.ts +++ b/packages/api/src/files/encode/utils.spec.ts @@ -1,6 +1,13 @@ import { Readable } from 'node:stream'; +import { Providers } from '@librechat/agents'; +import { audioMimeTypes } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; -import { AttachmentObjectNotFoundError, getFileStream } from './utils'; +import { + AttachmentObjectNotFoundError, + getAudioFormat, + getFileStream, + isConfiguredProviderMediaType, +} from './utils'; const file = { file_id: 'file-1', @@ -84,3 +91,110 @@ describe('getFileStream', () => { expect(getDownloadStream).toHaveBeenCalledWith(req, legacyUrl); }); }); + +/** Uses the real data-provider merge logic so the "inherited default" identity check is exercised. */ +const reqWith = (fileConfig: unknown): ServerRequest => + ({ config: fileConfig === undefined ? undefined : { fileConfig } }) as unknown as ServerRequest; + +describe('isConfiguredProviderMediaType', () => { + const params = { provider: Providers.OPENAI, endpoint: 'MyGateway' }; + + it('is false without any fileConfig', () => { + expect(isConfiguredProviderMediaType(reqWith(undefined), params, 'video/mp4')).toBe(false); + }); + + it('is false when the endpoint only inherits the built-in default list', () => { + const req = reqWith({ endpoints: { OtherEndpoint: { fileLimit: 3 } } }); + expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(false); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false); + }); + + it('is true when the endpoint config explicitly lists the media type', () => { + const req = reqWith({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'application/pdf', 'video/.*'] } }, + }); + expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(true); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false); + }); + + it('is true for a permissive config', () => { + const req = reqWith({ endpoints: { MyGateway: { supportedMimeTypes: ['.*'] } } }); + expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(true); + }); + + it('falls back to the provider key when no endpoint is given', () => { + const req = reqWith({ endpoints: { openAI: { supportedMimeTypes: ['audio/.*'] } } }); + expect(isConfiguredProviderMediaType(req, { provider: Providers.OPENAI }, 'audio/wav')).toBe( + true, + ); + }); +}); + +describe('getAudioFormat', () => { + it.each([ + ['audio/mp3', 'mp3'], + ['audio/mpeg', 'mp3'], + ['audio/mpeg3', 'mp3'], + ['audio/wav', 'wav'], + ['audio/wave', 'wav'], + ['audio/x-wav', 'wav'], + ['audio/ogg', 'ogg'], + ['audio/vorbis', 'ogg'], + ['audio/mp4', 'm4a'], + ['audio/x-m4a', 'm4a'], + ['audio/flac', 'flac'], + ['audio/x-flac', 'flac'], + ])('maps %s to %s regardless of the filename', (mimeType, expected) => { + expect(getAudioFormat(mimeType, 'clip.bogus')).toBe(expected); + expect(getAudioFormat(mimeType, 'recording')).toBe(expected); + }); + + it('is case-insensitive for the MIME type', () => { + expect(getAudioFormat('AUDIO/WAVE', 'clip.wave')).toBe('wav'); + }); + + it('falls back to a supported extension when the MIME type is unmapped', () => { + expect(getAudioFormat('audio/unknown', 'clip.mp3')).toBe('mp3'); + expect(getAudioFormat('audio/unknown', 'clip.PCM16')).toBe('pcm16'); + }); + + it('rejects an unsupported extension rather than passing it through', () => { + expect(getAudioFormat('audio/unknown', 'clip.wave')).toBeUndefined(); + expect(getAudioFormat('audio/unknown', 'clip.exe')).toBeUndefined(); + }); + + it('returns undefined when the filename has no extension and the MIME is unmapped', () => { + expect(getAudioFormat('audio/wma', 'recording')).toBeUndefined(); + expect(getAudioFormat('', '')).toBeUndefined(); + }); + + it('does not treat a leading-dot filename as an extension', () => { + expect(getAudioFormat('audio/unknown', '.mp3')).toBeUndefined(); + }); + + /** Guards against the accepted-MIME list drifting ahead of the format mapping. */ + it('resolves a format for every accepted audio MIME type that has one', () => { + const accepted = [ + 'audio/mp3', + 'audio/mpeg', + 'audio/mpeg3', + 'audio/wav', + 'audio/wave', + 'audio/x-wav', + 'audio/ogg', + 'audio/vorbis', + 'audio/mp4', + 'audio/m4a', + 'audio/x-m4a', + 'audio/flac', + 'audio/x-flac', + 'audio/webm', + 'audio/aac', + 'audio/opus', + ]; + for (const mimeType of accepted) { + expect(audioMimeTypes.test(mimeType)).toBe(true); + expect(getAudioFormat(mimeType, 'recording')).toBeDefined(); + } + }); +}); diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.ts index 7d7ac21a3a7..9c39ce2c13c 100644 --- a/packages/api/src/files/encode/utils.ts +++ b/packages/api/src/files/encode/utils.ts @@ -1,6 +1,11 @@ import getStream from 'get-stream'; import { Providers } from '@librechat/agents'; -import { FileSources, mergeFileConfig, getEndpointFileConfig } from 'librechat-data-provider'; +import { + FileSources, + mergeFileConfig, + isExplicitMimeConfig, + getEndpointFileConfig, +} from 'librechat-data-provider'; import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types'; import type { StoredFileRef } from '~/storage/path'; import { resolveDownloadPath } from '~/storage/path'; @@ -70,6 +75,107 @@ export const getConfiguredFileSizeLimit = ( return endpointConfig?.fileSizeLimit; }; +/** + * Whether the admin explicitly allowed `mimeType` for this endpoint via + * `fileConfig.endpoints..supportedMimeTypes`. The inherited default list does + * not count as opting in, mirroring the client's picker and drag-drop logic, so an + * OpenAI-compatible endpoint only receives video/audio parts when configured for them. + * @param req - The server request object containing config + * @param params - Object containing provider and optional endpoint + * @param params.provider - The provider to look up + * @param params.endpoint - Optional endpoint name for lookup + * @param mimeType - The MIME type of the file being attached + * @returns True when the endpoint config explicitly matches the MIME type + */ +export const isConfiguredProviderMediaType = ( + req: ServerRequest, + params: { + provider: Providers; + endpoint?: string; + }, + mimeType: string, +): boolean => { + if (!req.config?.fileConfig) { + return false; + } + const { provider, endpoint } = params; + const fileConfig = mergeFileConfig(req.config.fileConfig); + const endpointConfig = getEndpointFileConfig({ + fileConfig, + endpoint: endpoint ?? provider, + }); + const types = endpointConfig?.supportedMimeTypes; + if (!isExplicitMimeConfig(types)) { + return false; + } + return fileConfig.checkType?.(mimeType, types) ?? false; +}; + +/** + * Maps accepted audio MIME types to the format token OpenAI-compatible providers expect. + * Mirrors the canonicalization in `api/server/services/Files/Audio/STTService.js`: the + * accepted MIME list carries aliases (`audio/wave`, `audio/x-wav`, `audio/mpeg`) whose + * names are not themselves valid format values. + */ +const audioMimeToFormat: Record = { + 'audio/mp3': 'mp3', + 'audio/mpeg': 'mp3', + 'audio/mpeg3': 'mp3', + 'audio/wav': 'wav', + 'audio/wave': 'wav', + 'audio/x-wav': 'wav', + 'audio/ogg': 'ogg', + 'audio/vorbis': 'ogg', + 'audio/opus': 'ogg', + 'audio/mp4': 'm4a', + 'audio/m4a': 'm4a', + 'audio/x-m4a': 'm4a', + 'audio/flac': 'flac', + 'audio/x-flac': 'flac', + 'audio/webm': 'webm', + 'audio/aac': 'aac', +}; + +/** + * Format values OpenAI-compatible providers accept for an `input_audio` part. + * Used to validate an extension-derived fallback so an arbitrary filename suffix + * never reaches the provider as a format. + */ +const supportedAudioFormats = new Set([ + 'wav', + 'mp3', + 'aiff', + 'aac', + 'ogg', + 'flac', + 'm4a', + 'webm', + 'pcm16', + 'pcm24', +]); + +/** + * Resolves the `input_audio.format` value for a file. + * + * Prefers the MIME type, which is validated upstream against `audioMimeTypes`, because + * the filename extension is user-controlled: it may be absent (`recording`), an alias + * that is not a valid format (`clip.wave`), or disagree with the actual content. Falls + * back to the extension only when it is itself a supported format. + * @param mimeType - The file's MIME type + * @param filename - The original filename + * @returns The canonical format, or undefined when neither source yields a supported one + */ +export const getAudioFormat = (mimeType: string, filename: string): string | undefined => { + const fromMime = audioMimeToFormat[mimeType?.toLowerCase()]; + if (fromMime) { + return fromMime; + } + const parts = filename?.toLowerCase().split('.') ?? []; + /** `parts[0]` must be non-empty so a dotfile (`.mp3`) is not read as an extension. */ + const extension = parts.length > 1 && parts[0] ? parts[parts.length - 1] : undefined; + return extension && supportedAudioFormats.has(extension) ? extension : undefined; +}; + /** * Processes a file by downloading and encoding it to base64 * @param req - Express request object diff --git a/packages/api/src/files/encode/video.spec.ts b/packages/api/src/files/encode/video.spec.ts new file mode 100644 index 00000000000..06da3028d0f --- /dev/null +++ b/packages/api/src/files/encode/video.spec.ts @@ -0,0 +1,130 @@ +import { Providers } from '@librechat/agents'; +import type { IMongoFile } from '@librechat/data-schemas'; +import type { ServerRequest } from '~/types'; +import { encodeAndFormatVideos } from './video'; + +jest.mock('~/files/validation', () => ({ + validateVideo: jest.fn(), +})); + +jest.mock('./utils', () => ({ + getFileStream: jest.fn(), + getConfiguredFileSizeLimit: jest.fn(), + isConfiguredProviderMediaType: jest.fn(), +})); + +jest.mock('./memoryGuard', () => ({ + runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()), +})); + +import { validateVideo } from '~/files/validation'; +import { getFileStream, isConfiguredProviderMediaType } from './utils'; +import { Types } from 'mongoose'; + +const mockedValidateVideo = validateVideo as jest.MockedFunction; +const mockedGetFileStream = getFileStream as jest.MockedFunction; +const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction< + typeof isConfiguredProviderMediaType +>; + +const createMockFile = (type = 'video/mp4'): IMongoFile => + ({ + _id: new Types.ObjectId(), + file_id: 'video-1', + filename: 'clip.mp4', + filepath: '/uploads/clip.mp4', + type, + bytes: 1024, + source: 'local', + user: 'user-1', + object: 'file', + usage: 0, + createdAt: new Date(), + updatedAt: new Date(), + }) as unknown as IMongoFile; + +const req = { config: {} } as unknown as ServerRequest; +const getStrategyFunctions = jest.fn(); + +describe('encodeAndFormatVideos - provider formatting', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedValidateVideo.mockResolvedValue({ isValid: true }); + mockedIsConfigured.mockReturnValue(false); + const file = createMockFile(); + mockedGetFileStream.mockResolvedValue({ + file, + content: 'AAAA', + metadata: { + file_id: file.file_id, + filepath: file.filepath, + source: file.source, + filename: file.filename, + type: file.type, + }, + }); + }); + + it('emits a Google media block for google', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.GOOGLE }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([{ type: 'media', mimeType: 'video/mp4', data: 'AAAA' }]); + }); + + it('emits an OpenAI-compatible video_url block for openrouter without configuration', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENROUTER }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).not.toHaveBeenCalled(); + expect(result.videos).toEqual([ + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } }, + ]); + }); + + it('drops the video for a custom endpoint that is not configured for video', async () => { + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(mockedIsConfigured).toHaveBeenCalledWith( + req, + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + 'video/mp4', + ); + expect(result.videos).toEqual([]); + expect(result.files).toHaveLength(1); + }); + + it('emits a video_url block for a custom endpoint whose config allows video', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([ + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } }, + ]); + }); + + it('never emits a block for providers without a video path, even when configured', async () => { + mockedIsConfigured.mockReturnValue(true); + const result = await encodeAndFormatVideos( + req, + [createMockFile()], + { provider: Providers.ANTHROPIC }, + getStrategyFunctions, + ); + expect(result.videos).toEqual([]); + }); +}); diff --git a/packages/api/src/files/encode/video.ts b/packages/api/src/files/encode/video.ts index 52a9331c8d5..2715b36eed9 100644 --- a/packages/api/src/files/encode/video.ts +++ b/packages/api/src/files/encode/video.ts @@ -1,11 +1,12 @@ import { Providers } from '@librechat/agents'; -import { isDocumentSupportedProvider } from 'librechat-data-provider'; +import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest, StrategyFunctions, VideoResult } from '~/types'; import { getFileStream, getConfiguredFileSizeLimit, isAttachmentObjectNotFoundError, + isConfiguredProviderMediaType, } from './utils'; import { validateVideo } from '~/files/validation'; import { runGuardedEncode } from './memoryGuard'; @@ -91,7 +92,14 @@ export async function encodeAndFormatVideos( mimeType: file.type, data: content, }); - } else if (provider === Providers.OPENROUTER) { + } else if ( + provider === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isConfiguredProviderMediaType(req, { provider, endpoint }, file.type)) + ) { + /** OpenAI-compatible `video_url` part (OpenRouter, vLLM, and other gateways). + * Custom endpoints only get here when the admin listed video types in the + * endpoint's `supportedMimeTypes`, since not every gateway accepts video. */ result.videos.push({ type: 'video_url', video_url: { diff --git a/packages/api/src/files/upload/errors.ts b/packages/api/src/files/upload/errors.ts new file mode 100644 index 00000000000..11e579abb8a --- /dev/null +++ b/packages/api/src/files/upload/errors.ts @@ -0,0 +1,9 @@ +/** A direct audio attachment must be representable before the upload is persisted. */ +export class UnsupportedProviderAudioError extends Error { + readonly userErrorStatusCode = 415; + + constructor() { + super('com_error_files_provider_audio_format'); + this.name = 'UnsupportedProviderAudioError'; + } +} diff --git a/packages/api/src/files/upload/routing.spec.ts b/packages/api/src/files/upload/routing.spec.ts index 2d8de0dc8a0..71ae6289ffb 100644 --- a/packages/api/src/files/upload/routing.spec.ts +++ b/packages/api/src/files/upload/routing.spec.ts @@ -1,6 +1,10 @@ -import { EModelEndpoint } from 'librechat-data-provider'; +import { Readable } from 'node:stream'; +import { EModelEndpoint, EToolResources, Providers } from 'librechat-data-provider'; +import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; import { resolveUploadEndpoint, resolveEffectiveToolResource } from './routing'; +import { encodeAndFormatAudios } from '~/files/encode/audio'; +import { UnsupportedProviderAudioError } from './errors'; describe('resolveUploadEndpoint', () => { const req = { user: { id: 'user-1' } } as unknown as ServerRequest; @@ -104,3 +108,141 @@ describe('resolveEffectiveToolResource Responses handling', () => { expect(usable).toBe('context'); }); }); + +describe('direct provider audio preflight', () => { + const makeReq = (legacyFileUploadUX = false, mimetype = 'audio/wma', originalname = 'clip.wma') => + Object.assign({} as ServerRequest, { + file: { + mimetype, + originalname, + fieldname: 'file', + encoding: '7bit', + size: 16, + destination: '/uploads', + filename: originalname, + path: '/uploads/audio-1', + stream: Readable.from(Buffer.alloc(16)), + buffer: Buffer.alloc(16), + }, + config: { + fileConfig: { + endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } }, + }, + }, + }); + + it.each([false, true])( + 'rejects unsupported audio before processing (legacy=%s)', + async (legacy) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(legacy), + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).rejects.toMatchObject({ + message: 'com_error_files_provider_audio_format', + userErrorStatusCode: 415, + }); + }, + ); + + it('uses the resolved agent endpoint and reuses the agent read', async () => { + const req = makeReq(); + const getAgent = jest.fn().mockResolvedValue({ provider: 'MyGateway' }); + const metadata = { endpoint: EModelEndpoint.agents, agent_id: 'agent_saved01' }; + await resolveUploadEndpoint({ req, metadata, getAgent }); + await expect(resolveEffectiveToolResource({ req, metadata, getAgent })).rejects.toBeInstanceOf( + UnsupportedProviderAudioError, + ); + expect(getAgent).toHaveBeenCalledTimes(1); + }); + + it.each([EToolResources.context, EToolResources.ocr, EToolResources.execute_code])( + 'preserves explicit %s uploads', + async (tool_resource) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(true), + metadata: { endpoint: 'MyGateway', tool_resource }, + getAgent: jest.fn(), + }), + ).resolves.toBe( + tool_resource === EToolResources.ocr ? EToolResources.context : tool_resource, + ); + }, + ); + + it('preserves audio explicitly routed to transcription', async () => { + const req = makeReq(); + req.config!.fileConfig!.defaultLLMDeliveryPath = { overrides: { 'audio/*': 'text' } }; + await expect( + resolveEffectiveToolResource({ + req, + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).resolves.toBe(EToolResources.context); + }); + + it.each([EModelEndpoint.google, Providers.VERTEXAI])( + 'does not apply input_audio rules to %s', + async (endpoint) => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(), + metadata: { endpoint }, + getAgent: jest.fn(), + }), + ).resolves.toBeUndefined(); + }, + ); + + it('also rejects unsupported OpenRouter audio', async () => { + await expect( + resolveEffectiveToolResource({ + req: makeReq(), + metadata: { endpoint: Providers.OPENROUTER }, + getAgent: jest.fn(), + }), + ).rejects.toBeInstanceOf(UnsupportedProviderAudioError); + }); + + it.each([ + ['audio/wave', 'clip.wave', 'wav'], + ['audio/mpeg', 'recording', 'mp3'], + ['audio/unknown', 'clip.pcm16', 'pcm16'], + ])( + 'delivers an accepted %s upload through the real encoder', + async (mimetype, originalname, format) => { + const req = makeReq(false, mimetype, originalname); + await expect( + resolveEffectiveToolResource({ + req, + metadata: { endpoint: 'MyGateway' }, + getAgent: jest.fn(), + }), + ).resolves.toBeUndefined(); + const file = { + file_id: 'audio-1', + filename: originalname, + filepath: '/uploads/audio-1', + type: mimetype, + bytes: 16, + source: 'local', + } as IMongoFile; + const result = await encodeAndFormatAudios( + req, + [file], + { provider: Providers.OPENAI, endpoint: 'MyGateway' }, + () => ({ getDownloadStream: async () => Readable.from(Buffer.alloc(16)) }), + ); + expect(result.audios).toEqual([ + { + type: 'input_audio', + input_audio: { data: Buffer.alloc(16).toString('base64'), format }, + }, + ]); + }, + ); +}); diff --git a/packages/api/src/files/upload/routing.ts b/packages/api/src/files/upload/routing.ts index 31f571cbaeb..f6a5e1d7a77 100644 --- a/packages/api/src/files/upload/routing.ts +++ b/packages/api/src/files/upload/routing.ts @@ -1,13 +1,21 @@ import { + Providers, + EModelEndpoint, EToolResources, mergeFileConfig, + isExplicitMimeConfig, + isOpenAILikeProvider, + isKnownProviderIdentifier, isAssistantsEndpoint, isResponsesApiUpload, getEndpointFileConfig, resolveUploadLLMDeliveryPath, isSpeechProviderConfigured, + getCustomEndpointProvider, } from 'librechat-data-provider'; import type { ServerRequest } from '~/types'; +import { UnsupportedProviderAudioError } from './errors'; +import { getAudioFormat } from '~/files/encode/utils'; /** The subset of an agent record this module reads. */ export interface UploadAgent { @@ -27,6 +35,7 @@ export interface UploadMetadata { /** The uploaded file fields routing depends on. */ export interface UploadFile { mimetype: string; + originalname?: string; } export type GetUploadAgent = (params: { id: string }) => Promise; @@ -112,14 +121,35 @@ export async function resolveEffectiveToolResource({ const fileConfig = mergeFileConfig(req.config?.fileConfig); const endpoint = await resolveUploadEndpoint({ req, metadata, getAgent }); const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + const endpointProvider = getCustomEndpointProvider(req.config?.endpoints?.custom, endpoint); + const file = req.file as UploadFile; const path = resolveUploadLLMDeliveryPath({ toolResource: metadata.tool_resource, - mimeType: (req.file as UploadFile).mimetype, + mimeType: file.mimetype, endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi), sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt), }); + const provider = + endpointProvider ?? (isKnownProviderIdentifier(endpoint) ? endpoint : EModelEndpoint.openAI); + const usesInputAudio = + provider?.toLowerCase() === Providers.OPENROUTER || + (isOpenAILikeProvider(provider) && + isExplicitMimeConfig(endpointConfig.supportedMimeTypes) && + fileConfig.checkType?.(file.mimetype, endpointConfig.supportedMimeTypes)); + if ( + path === 'provider' && + endpoint != null && + endpoint !== EModelEndpoint.agents && + !isAssistantsEndpoint(endpoint) && + file.mimetype.startsWith('audio/') && + usesInputAudio && + !getAudioFormat(file.mimetype, file.originalname ?? '') + ) { + throw new UnsupportedProviderAudioError(); + } return path === 'text' ? EToolResources.context : undefined; } diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.ts index b49a1a1f902..4d606f24825 100644 --- a/packages/api/src/types/files.ts +++ b/packages/api/src/types/files.ts @@ -39,7 +39,7 @@ export interface GoogleVideoBlock { data: string; } -/** OpenRouter video block format */ +/** OpenAI-compatible video block format (OpenRouter, vLLM, configured custom endpoints) */ export interface OpenRouterVideoBlock { type: 'video_url'; video_url: { @@ -136,7 +136,7 @@ export interface GoogleAudioBlock { data: string; } -/** OpenRouter audio block format */ +/** OpenAI-compatible audio block format (OpenRouter, vLLM, configured custom endpoints) */ export interface OpenRouterAudioBlock { type: 'input_audio'; input_audio: { diff --git a/packages/api/src/utils/files.spec.ts b/packages/api/src/utils/files.spec.ts index f8910b37c4c..59eb11decc2 100644 --- a/packages/api/src/utils/files.spec.ts +++ b/packages/api/src/utils/files.spec.ts @@ -4,6 +4,7 @@ import { flattenArtifactPath, resolveUploadErrorMessage, } from './files'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; jest.mock('node:crypto', () => { const actualModule = jest.requireActual('node:crypto'); @@ -503,6 +504,14 @@ describe('flattenArtifactPath', () => { }); describe('resolveUploadErrorMessage', () => { + it.each([false, true])( + 'preserves the localized audio preflight error (redaction=%s)', + (redact) => { + expect( + resolveUploadErrorMessage(new UnsupportedProviderAudioError(), undefined, redact), + ).toBe('com_error_files_provider_audio_format'); + }, + ); test('returns default message for null error', () => { expect(resolveUploadErrorMessage(null)).toBe('Error processing file'); }); diff --git a/packages/api/src/utils/files.ts b/packages/api/src/utils/files.ts index b313dc9b6fd..5be59b27aab 100644 --- a/packages/api/src/utils/files.ts +++ b/packages/api/src/utils/files.ts @@ -2,6 +2,7 @@ import path from 'path'; import crypto from 'node:crypto'; import { createReadStream } from 'fs'; import { readFile, stat } from 'fs/promises'; +import { UnsupportedProviderAudioError } from '~/files/upload/errors'; const USER_FACING_UPLOAD_ERRORS = [ ['Invalid file format', 'Invalid file format'], @@ -98,6 +99,9 @@ export function resolveUploadErrorMessage( defaultMessage = 'Error processing file', redactDetails = false, ): string { + if (error instanceof UnsupportedProviderAudioError) { + return error.message; + } const errorMessage = error?.message; if (!errorMessage) { return defaultMessage; diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.ts index 1ee9e035cc8..164e75fb756 100644 --- a/packages/data-provider/src/file-config.spec.ts +++ b/packages/data-provider/src/file-config.spec.ts @@ -9,6 +9,7 @@ import { bedrockDocumentMimeTypes, isAnthropicDocumentType, isPermissiveMimeConfig, + isExplicitMimeConfig, convertStringsToRegex, setFileConfigRegexCompiler, documentParserMimeTypes, @@ -1399,6 +1400,35 @@ describe('getEndpointFileConfig', () => { }); }); +describe('isExplicitMimeConfig', () => { + it('is false for undefined or empty lists', () => { + expect(isExplicitMimeConfig(undefined)).toBe(false); + expect(isExplicitMimeConfig([])).toBe(false); + }); + + it('is false for the built-in default list (inherited, not configured)', () => { + expect(isExplicitMimeConfig(supportedMimeTypes)).toBe(false); + const endpointConfig = getEndpointFileConfig({ + fileConfig: mergeFileConfig({ endpoints: { Other: { fileLimit: 1 } } }), + endpoint: 'MyGateway', + endpointType: 'custom', + }); + expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(false); + }); + + it('is true for an admin-configured list, permissive or not', () => { + const endpointConfig = getEndpointFileConfig({ + fileConfig: mergeFileConfig({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } }, + }), + endpoint: 'MyGateway', + endpointType: 'custom', + }); + expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(true); + expect(isExplicitMimeConfig([/.*/])).toBe(true); + }); +}); + describe('isPermissiveMimeConfig', () => { it('returns true for wildcard .* pattern', () => { expect(isPermissiveMimeConfig([/.*/])).toBe(true); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index b3f55413b93..0890dd24ae2 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -674,6 +674,18 @@ export const isPermissiveMimeConfig = (types?: RegexLike[]): boolean => { return types.some((regex) => regex.test('x-librechat/x-probe')); }; +/** + * Detects whether an endpoint's `supportedMimeTypes` were set by the admin rather than inherited + * from the built-in default list. Inheritance is signaled by referential identity with + * `supportedMimeTypes`, which `mergeWithDefault` preserves for unconfigured endpoints. + */ +export const isExplicitMimeConfig = (types?: RegexLike[]): types is RegexLike[] => { + if (!types || types.length === 0) { + return false; + } + return types !== supportedMimeTypes; +}; + /** The kind of content a provider upload path can actually send to the model. */ export type MimeUploadCategory = 'image' | 'document' | 'audio' | 'video'; diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts index 520e60a63f1..b62c3a26f90 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts @@ -1,11 +1,15 @@ import type { TDefaultLLMDeliveryPathConfig } from './file-config'; +import type { TEndpoint } from './config'; import { isNativelyReadableText, canToolResourceConsume, resolveUploadDestination, + getCustomEndpointProvider, resolveDefaultLLMDeliveryPath, + resolveUploadLLMDeliveryPath, SYSTEM_LLM_DELIVERY_DEFAULTS, } from './resolve-llm-delivery-path'; +import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config'; describe('resolveDefaultLLMDeliveryPath', () => { it('should return system default for images when no config provided', () => { @@ -300,6 +304,97 @@ describe('resolveDefaultLLMDeliveryPath', () => { ); }); + describe('media a custom endpoint opted into', () => { + /* The encoders emit OpenAI-format media parts for an OpenAI-compatible endpoint only + * when the admin listed the type in its `supportedMimeTypes`, so the route has to + * agree: an explicit match is provider-capable, the inherited default list is not. */ + const explicit = [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/, /^audio\/wav$/]; + const resolve = (mimeType: string, endpoint: string, types?: RegExp[]) => + resolveDefaultLLMDeliveryPath( + mimeType, + undefined, + undefined, + endpoint, + undefined, + true, + types, + ); + + it('keeps an explicitly allowed type on the provider path for a custom endpoint', () => { + expect(resolve('video/mp4', 'MyGateway', explicit)).toBe('provider'); + expect(resolve('audio/wav', 'MyGateway', explicit)).toBe('provider'); + }); + + it('still downgrades a media type the allowlist does not name', () => { + expect(resolve('audio/mpeg', 'MyGateway', explicit)).toBe('text'); + }); + + it('does not read the inherited default list as an opt-in', () => { + expect(resolve('video/mp4', 'MyGateway', supportedMimeTypes)).toBe('none'); + expect(resolve('video/mp4', 'MyGateway', [])).toBe('none'); + }); + + it('does not opt in a built-in endpoint, which the client offers no media for', () => { + /* Anthropic and Bedrock encoders have no media branch at all, and OpenAI/Azure are + * left out because the picker and drag-drop only open media for custom endpoints: + * a route the client cannot send to is a capability with no entry point. */ + expect(resolve('video/mp4', 'openAI', explicit)).toBe('none'); + expect(resolve('video/mp4', 'azureOpenAI', explicit)).toBe('none'); + expect(resolve('video/mp4', 'anthropic', explicit)).toBe('none'); + expect(resolve('video/mp4', 'bedrock', explicit)).toBe('none'); + }); + + it('keeps a custom endpoint that runs as Anthropic on its previous route', () => { + /* A custom endpoint may declare `provider: anthropic`, and the encoders emit + * OpenAI-format parts only, so the opt-in would deliver nothing there. Audio keeps + * its transcription route and video stays off the model path. */ + const endpointConfig = { supportedMimeTypes: explicit }; + const anthropic = { mimeType: 'video/mp4', endpointConfig, endpoint: 'MyClaude' }; + expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'anthropic' })).toBe( + 'none', + ); + expect( + resolveUploadLLMDeliveryPath({ + ...anthropic, + mimeType: 'audio/wav', + endpointProvider: 'anthropic', + sttConfigured: true, + }), + ).toBe('text'); + expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'openAI' })).toBe( + 'provider', + ); + expect(resolveUploadLLMDeliveryPath(anthropic)).toBe('provider'); + }); + + it('reaches the upload resolver through the merged endpoint config', () => { + /* The real merge, so the identity check that separates a configured list from the + * inherited default is exercised the way the upload route exercises it. */ + const fileConfig = mergeFileConfig({ + endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } }, + }); + const configured = getEndpointFileConfig({ fileConfig, endpoint: 'MyGateway' }); + const inherited = getEndpointFileConfig({ fileConfig, endpoint: 'OtherGateway' }); + + expect( + resolveUploadLLMDeliveryPath({ + mimeType: 'video/mp4', + endpointConfig: configured, + fileConfig, + endpoint: 'MyGateway', + }), + ).toBe('provider'); + expect( + resolveUploadLLMDeliveryPath({ + mimeType: 'video/mp4', + endpointConfig: inherited, + fileConfig, + endpoint: 'OtherGateway', + }), + ).toBe('none'); + }); + }); + it('leaves media alone when no endpoint is resolved at all', () => { /* An ephemeral agent reports no usable endpoint, which is not the same as naming one * we cannot identify. */ @@ -545,6 +640,27 @@ describe('resolveUploadDestination', () => { }); }); +describe('getCustomEndpointProvider', () => { + const custom = [ + { name: 'My Claude', provider: 'anthropic' }, + { name: 'Ollama', provider: 'anthropic' }, + { name: 'MyGateway' }, + ] as Array>>; + + it('returns the declared dialect for a custom endpoint, matching the normalized name', () => { + expect(getCustomEndpointProvider(custom, 'My Claude')).toBe('anthropic'); + /* The same normalization the file config lookup applies to endpoint names. */ + expect(getCustomEndpointProvider(custom, 'ollama')).toBe('anthropic'); + }); + + it('returns nothing for an endpoint without a dialect, an unknown one, or no config', () => { + expect(getCustomEndpointProvider(custom, 'MyGateway')).toBeUndefined(); + expect(getCustomEndpointProvider(custom, 'Other')).toBeUndefined(); + expect(getCustomEndpointProvider(undefined, 'My Claude')).toBeUndefined(); + expect(getCustomEndpointProvider(custom, undefined)).toBeUndefined(); + }); +}); + describe('isNativelyReadableText', () => { it('admits the application types whose payload is text', () => { /* Kept in step with the textual set in the content-protection code. Missing one sends diff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.ts index bb6148efaf7..bccf7f566cf 100644 --- a/packages/data-provider/src/resolve-llm-delivery-path.ts +++ b/packages/data-provider/src/resolve-llm-delivery-path.ts @@ -1,13 +1,57 @@ import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config'; -import type { EndpointFileConfig, FileConfig } from './types/files'; +import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files'; +import type { TEndpoint } from './config'; +import { + retrievalMimeTypes, + isExplicitMimeConfig, + isBedrockDocumentType, + codeInterpreterMimeTypes, + fileConfig as baseFileConfig, +} from './file-config'; import { EModelEndpoint, - isDocumentSupportedProvider, + isOpenAILikeProvider, isKnownProviderIdentifier, isMediaSupportedProvider, + isDocumentSupportedProvider, } from './schemas'; -import { retrievalMimeTypes, isBedrockDocumentType, codeInterpreterMimeTypes } from './file-config'; import { EToolResources } from './types/assistants'; +import { normalizeEndpointName } from './utils'; + +/** + * The native provider a custom endpoint declares, when it declares one. A custom endpoint + * speaks OpenAI's API unless its config names another dialect, and the upload route needs + * that answer for the same reason request initialization does: the media encoders emit + * OpenAI-format parts, so a custom endpoint running as Anthropic receives none. + */ +export function getCustomEndpointProvider( + customEndpoints: Array>> | undefined, + endpoint?: string | null, +): string | undefined { + if (!customEndpoints || !endpoint) { + return undefined; + } + const normalized = normalizeEndpointName(endpoint); + return customEndpoints.find((config) => normalizeEndpointName(config.name ?? '') === normalized) + ?.provider; +} + +/** A custom endpoint emits OpenAI-format media parts only for the types the admin listed + * in its `supportedMimeTypes`; the inherited default list is not an opt-in. A name that + * is not a known provider is a custom endpoint. Mirrors `isConfiguredProviderMediaType` + * on the encoder side, so the route and the encoder agree on which uploads the provider + * actually receives; the built-in endpoints are left out because the client offers no + * media for them. */ +const isConfiguredMediaEndpoint = ( + mimeType: string, + endpoint: string, + supportedMimeTypes?: RegexLike[], +): boolean => { + if (!isExplicitMimeConfig(supportedMimeTypes) || isKnownProviderIdentifier(endpoint)) { + return false; + } + return baseFileConfig.checkType(mimeType, supportedMimeTypes); +}; /** Audio and video reach the model only through the media encoders, which support a * narrower provider set than documents. Images use the broadly supported vision @@ -16,9 +60,13 @@ const isProviderCapable = ( mimeType: string, endpoint: string, useResponsesApi?: boolean, + supportedMimeTypes?: RegexLike[], ): boolean => { if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) { - return isMediaSupportedProvider(endpoint); + return ( + isMediaSupportedProvider(endpoint) || + isConfiguredMediaEndpoint(mimeType, endpoint, supportedMimeTypes) + ); } if (mimeType === 'application/pdf') { /* Azure is out of the document set because it needs the Responses API for native @@ -104,6 +152,7 @@ export function resolveDefaultLLMDeliveryPath( endpoint?: string, useResponsesApi?: boolean, sttConfigured?: boolean, + supportedMimeTypes?: RegexLike[], ): TDefaultLLMDeliveryPath { const wildcard = mimeType.split('/')[0] + '/*'; @@ -149,10 +198,11 @@ export function resolveDefaultLLMDeliveryPath( const namedEndpoint = endpoint != null && endpoint !== EModelEndpoint.agents; const providerKnown = namedEndpoint && isKnownProviderIdentifier(endpoint); /* Media is judged for any named endpoint, identified or not. The media encoders emit a - * payload only for the providers they name, so a custom endpoint gets nothing whatever - * it proxies to, and leaving it on the provider path delivers neither media nor text. - * Documents keep the narrower rule: an unidentified endpoint is usually OpenAI- or - * Anthropic-compatible, both of which do carry them. */ + * payload only for the providers they name, or for an OpenAI-compatible endpoint whose + * admin listed the type in its `supportedMimeTypes`; any other custom endpoint gets + * nothing whatever it proxies to, and leaving it on the provider path delivers neither + * media nor text. Documents keep the narrower rule: an unidentified endpoint is usually + * OpenAI- or Anthropic-compatible, both of which do carry them. */ const isMedia = mimeType.startsWith('audio/') || mimeType.startsWith('video/'); /* Audio's text path is transcription, so on a deployment with no speech provider it is * not recoverable at all. Routing it to text there sends the upload to a service that @@ -164,7 +214,7 @@ export function resolveDefaultLLMDeliveryPath( if ( systemDefault === 'provider' && canJudgeCapability && - !isProviderCapable(mimeType, endpoint as string, useResponsesApi) + !isProviderCapable(mimeType, endpoint as string, useResponsesApi, supportedMimeTypes) ) { /* Downgrading is only useful where text can actually be recovered. Video has no * extraction step: speech-to-text covers audio, and the default text matcher accepts @@ -206,6 +256,7 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi, sttConfigured, }: { @@ -213,12 +264,19 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; + /** The provider the endpoint runs as, when the caller knows it: a custom endpoint's + * declared dialect at upload time, the agent's resolved provider at turn time. */ + endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { if (endpointConfig?.legacyFileUploadUX === true) { return 'provider'; } + /* The media opt-in exists for OpenAI-format parts, so an endpoint known to run as + * something else — a custom endpoint declaring `provider: anthropic` — keeps the + * capability gate it had, where audio still reaches transcription. */ + const runsAsOpenAI = endpointProvider == null || isOpenAILikeProvider(endpointProvider); return resolveDefaultLLMDeliveryPath( mimeType, endpointConfig?.defaultLLMDeliveryPath, @@ -226,6 +284,7 @@ export function resolveDefaultUploadLLMDeliveryPath({ endpoint, useResponsesApi, sttConfigured, + runsAsOpenAI ? endpointConfig?.supportedMimeTypes : undefined, ); } @@ -237,6 +296,7 @@ export function resolveUploadLLMDeliveryPath({ fileConfig, endpoint, useResponsesApi, + endpointProvider, sttConfigured, }: { toolResource?: string | null; @@ -244,6 +304,7 @@ export function resolveUploadLLMDeliveryPath({ endpointConfig?: EndpointFileConfig; fileConfig?: FileConfig; endpoint?: string; + endpointProvider?: string | null; useResponsesApi?: boolean; sttConfigured?: boolean; }): TDefaultLLMDeliveryPath { @@ -258,6 +319,7 @@ export function resolveUploadLLMDeliveryPath({ endpointConfig, fileConfig, endpoint, + endpointProvider, useResponsesApi, sttConfigured, }); From 1396946602daa21f7bd422a7c91025e1b5cc060a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 16:49:19 -0400 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=AA=93=20ci:=20Cull=20Stale=20Codegra?= =?UTF-8?q?ph=20Select=20Runs=20(#15939)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/codegraph-select.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codegraph-select.yml b/.github/workflows/codegraph-select.yml index d76d673a8aa..9257c4bac1d 100644 --- a/.github/workflows/codegraph-select.yml +++ b/.github/workflows/codegraph-select.yml @@ -17,6 +17,10 @@ permissions: contents: read pull-requests: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: select: runs-on: ubuntu-latest From bc12fad6f1ee1b5577dc49966e8a32dde0eab5a8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 18:35:33 -0400 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=95=B9=EF=B8=8F=20feat:=20Run=20Decla?= =?UTF-8?q?red=20Project=20Actions=20on=20Attached=20Workers=20(#15943)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: use declared attached project actions * Complete named action definitions and approval previews * Sort named environment imports * Sort native environment fixture imports --- api/server/services/ToolService.js | 1 + .../Chat/Input/CodeWorkspaceMenu.tsx | 7 ++ .../__tests__/CodeWorkspaceMenu.spec.tsx | 19 ++++ .../Chat/approval/__tests__/preview.test.ts | 10 +++ .../src/components/Chat/approval/preview.ts | 7 +- .../src/agents/__tests__/initialize.test.ts | 6 ++ packages/api/src/agents/execution.spec.ts | 17 ++++ packages/api/src/agents/execution.ts | 9 +- packages/api/src/agents/initialize.ts | 3 + packages/api/src/agents/skills.ts | 9 +- packages/api/src/agents/tools.spec.ts | 21 +++++ packages/api/src/agents/tools.ts | 17 +++- packages/api/src/code/bridge.spec.ts | 38 +++++++- packages/api/src/code/bridge.ts | 7 +- packages/api/src/code/capabilities.spec.ts | 22 +++++ packages/api/src/code/capabilities.ts | 1 + packages/api/src/code/command.spec.ts | 31 +++++++ packages/api/src/code/command.ts | 69 +++++++++++--- .../api/src/code/environment.live.spec.ts | 90 +++++++++++++++++++ .../api/src/code/fixtures/environment.mjs | 55 ++++++++++++ packages/api/src/code/workspace.ts | 1 + packages/data-provider/src/code/workspace.ts | 35 ++++++++ 22 files changed, 454 insertions(+), 21 deletions(-) create mode 100644 packages/api/src/code/environment.live.spec.ts create mode 100644 packages/api/src/code/fixtures/environment.mjs diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 6994c106d3a..3afc2ba13ef 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -2280,6 +2280,7 @@ async function loadToolsForExecution({ authHeaders, baseUrl: codeExecutionContext.baseUrl, workspaceId: codeExecutionContext.codeWorkspace.workspaceId, + environment: codeExecutionContext.codeWorkspace.environment, gitIdentity: agent?.git_identity, maxTimeoutMs: resolveAttachedWorkspaceCommandTimeoutMax( codeExecutionContext.codeEnvironmentConfigSchema, diff --git a/client/src/components/Chat/Input/CodeWorkspaceMenu.tsx b/client/src/components/Chat/Input/CodeWorkspaceMenu.tsx index d71265f96c8..783f1f00a2e 100644 --- a/client/src/components/Chat/Input/CodeWorkspaceMenu.tsx +++ b/client/src/components/Chat/Input/CodeWorkspaceMenu.tsx @@ -123,6 +123,13 @@ function EnvironmentWorkspaces({ {descriptor.name && (

{descriptor.id}

)} + {(descriptor.environment?.repo || descriptor.environment?.ref) && ( +

+ {[descriptor.environment.repo, descriptor.environment.ref] + .filter(Boolean) + .join(' · ')} +

+ )} {selected && (