From ed4ddf2f1a63c15fdf54bdebc09925350c0bf90d Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Tue, 4 Aug 2026 03:33:48 +0000 Subject: [PATCH] feat(server): add MiniMax speech TTS to the Media Center extension Add an optional minimax provider to the media speech_synthesize action. It calls the MiniMax Text to Audio v2 endpoint for the global and China regions, sends voice_setting, audio_setting, pronunciation_dict, and output_format request fields, and validates base_resp.status_code in the response. Co-Authored-By: Claude --- apps/server/src/authorization-center.ts | 11 ++ .../src/extensions/media-center.test.ts | 110 +++++++++++++ apps/server/src/extensions/media-center.ts | 150 +++++++++++++++++- 3 files changed, 265 insertions(+), 6 deletions(-) diff --git a/apps/server/src/authorization-center.ts b/apps/server/src/authorization-center.ts index c285d6653..f682358bc 100644 --- a/apps/server/src/authorization-center.ts +++ b/apps/server/src/authorization-center.ts @@ -64,6 +64,17 @@ const AUTHORIZATION_SERVICES: readonly AuthorizationServiceDefinition[] = [ "Use the iPolloWork media extension actions from trusted runtime code. They keep DASHSCOPE_API_KEY on this device and provide the supported media operations without modifying OpenCode.", }, }, + { + id: "minimax-media", + keys: ["MINIMAX_API_KEY"], + category: "media", + agent: { + capability: "MiniMax speech TTS", + useWhen: "Use when the user asks to synthesize speech or create narration audio with a MiniMax account.", + instruction: + "Use the iPolloWork media extension speech_synthesize action from trusted runtime code. It keeps MINIMAX_API_KEY on this device and calls the MiniMax Text to Audio v2 endpoint without modifying OpenCode.", + }, + }, { id: "volcengine-video", keys: ["ARK_API_KEY"], diff --git a/apps/server/src/extensions/media-center.test.ts b/apps/server/src/extensions/media-center.test.ts index 2a719d8bb..29f0bf46a 100644 --- a/apps/server/src/extensions/media-center.test.ts +++ b/apps/server/src/extensions/media-center.test.ts @@ -433,6 +433,116 @@ describe("Media Center extension", () => { }); }); + test("requires the MiniMax key before synthesizing speech", async () => { + await expect(callMediaExtensionAction(config, env({}), "speech_synthesize", { + provider: "minimax", + text: "hello", + }, {})).rejects.toMatchObject({ code: "minimax_api_key_missing" }); + }); + + test("keeps the MiniMax key server-side while synthesizing speech", async () => { + globalThis.fetch = ((input, init) => { + expect(String(input)).toBe("https://api.minimax.io/v1/t2a_v2"); + expect(init?.headers).toMatchObject({ Authorization: "Bearer mm-test-key-0000" }); + expect(JSON.parse(String(init?.body))).toMatchObject({ + model: "speech-2.8-hd", + text: "hello", + stream: false, + voice_setting: { voice_id: "female-qn-qingse" }, + }); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0, status_msg: "success" }, + data: { audio: "aabb", status: 2 }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + voice: "female-qn-qingse", + }, {}); + + expect(result).toMatchObject({ + ok: true, + extensionId: MEDIA_EXTENSION_ID, + action: "speech_synthesize", + result: { + provider: "minimax", + operation: "speech_synthesize", + output: { + base_resp: { status_code: 0, status_msg: "success" }, + data: { audio: "aabb", status: 2 }, + }, + }, + }); + expect(JSON.stringify(result)).not.toContain("mm-test-key-0000"); + }); + + test("routes MiniMax TTS to the China regional endpoint when requested", async () => { + globalThis.fetch = ((input) => { + expect(String(input)).toBe("https://api.minimaxi.com/v1/t2a_v2"); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0 }, + data: { audio: "aabb" }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + region: "cn_zh", + text: "你好", + }, {}); + + expect(result).toMatchObject({ ok: true, result: { provider: "minimax" } }); + }); + + test("passes MiniMax voice, audio, and pronunciation settings", async () => { + globalThis.fetch = ((_input, init) => { + const body = JSON.parse(String(init?.body)); + expect(body).toMatchObject({ + model: "speech-2.6-hd", + output_format: "mp3", + voice_setting: { voice_id: "male-qn-qingse" }, + audio_setting: { sample_rate: 32000 }, + pronunciation_dict: { text: "你好", pinyin: "ni hao" }, + }); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0 }, + data: { audio: "aabb" }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + model: "speech-2.6-hd", + format: "mp3", + voice: "male-qn-qingse", + audioSetting: { sample_rate: 32000 }, + pronunciationDict: { text: "你好", pinyin: "ni hao" }, + }, {}); + + expect(result).toMatchObject({ ok: true, result: { provider: "minimax" } }); + }); + + test("explains MiniMax non-zero status codes", async () => { + globalThis.fetch = ((_input, _init) => { + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 2004, status_msg: "Invalid parameter" }, + data: {}, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + await expect(callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + }, {})).rejects.toMatchObject({ + status: 502, + code: "minimax_request_failed", + message: "Invalid parameter", + }); + }); + test("uses the asynchronous task endpoint for a digital human", async () => { globalThis.fetch = ((input, init) => { expect(String(input)).toBe("https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/video-synthesis"); diff --git a/apps/server/src/extensions/media-center.ts b/apps/server/src/extensions/media-center.ts index ef02b1809..f283b4e4d 100644 --- a/apps/server/src/extensions/media-center.ts +++ b/apps/server/src/extensions/media-center.ts @@ -26,6 +26,28 @@ const LEGACY_COSYVOICE_V3_PRESET_MIGRATIONS: Record = { longfei: "longanlang_v3", }; +// MiniMax speech TTS (Text to Audio v2) uses one regional endpoint per +// marketplace. The global endpoint serves MiniMax international accounts and +// the China endpoint serves accounts provisioned on the MiniMax China platform. +const MINIMAX_SPEECH_ENDPOINTS: Record = { + global_en: "https://api.minimax.io", + cn_zh: "https://api.minimaxi.com", +}; +const DEFAULT_MINIMAX_SPEECH_REGION = "global_en"; +const DEFAULT_MINIMAX_SPEECH_MODEL = "speech-2.8-hd"; +const MINIMAX_T2A_PATH = "/v1/t2a_v2"; +const MINIMAX_SPEECH_MODELS = [ + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", +]; +const MINIMAX_SPEECH_AUDIO_FORMATS = ["mp3", "wav", "flac", "pcm"]; + type JsonRecord = Record; function roundVoiceoverTime(value: number) { @@ -392,15 +414,19 @@ export const MEDIA_EXTENSION_ACTIONS = [ extensionId: MEDIA_EXTENSION_ID, action: "speech_synthesize", title: "Synthesize speech", - description: "Create speech from text with CosyVoice. The result contains a temporary audio URL from Model Studio.", + description: "Create speech from text with the configured media provider. The result contains provider-specific audio data.", inputSchema: { type: "object", properties: { + provider: { type: "string", enum: ["aliyun-bailian", "minimax"], description: "Optional media provider. Defaults to aliyun-bailian." }, text: { type: "string", description: "Text to synthesize." }, - voice: { type: "string", description: "Optional Model Studio voice name or cloned voice id." }, - model: { type: "string", description: "Optional speech model. Defaults to cosyvoice-v3-flash." }, - format: { type: "string", description: "Optional audio format, for example wav or mp3." }, + voice: { type: "string", description: "Optional provider voice name, cloned voice id, or MiniMax voice_id." }, + model: { type: "string", description: `Optional speech model. Defaults to cosyvoice-v3-flash for Alibaba and speech-2.8-hd for MiniMax. MiniMax models: ${MINIMAX_SPEECH_MODELS.join(", ")}.` }, + format: { type: "string", description: `Optional audio format, for example mp3 or wav. MiniMax supports ${MINIMAX_SPEECH_AUDIO_FORMATS.join(", ")}.` }, sampleRate: { type: "number", description: "Optional output sample rate in Hz." }, + region: { type: "string", enum: ["global_en", "cn_zh"], description: "Optional MiniMax regional endpoint. Defaults to global_en." }, + audioSetting: { type: "object", description: "Optional MiniMax audio_setting overrides, for example sample_rate or volume." }, + pronunciationDict: { type: "object", description: "Optional MiniMax pronunciation_dict text-to-pinyin overrides." }, }, required: ["text"], additionalProperties: false, @@ -835,6 +861,98 @@ async function resolveBailianCredentials(env: EnvService): Promise<{ apiKey: str return { apiKey, baseUrl: safeProviderBaseUrl(configuredBaseUrl) }; } +function minimaxErrorMessage(payload: unknown): string { + if (!isRecord(payload)) return ""; + const baseResp = isRecord(payload.base_resp) ? payload.base_resp : null; + return readStringField(baseResp, "status_msg") || providerMessage(payload); +} + +function safeMiniMaxBaseUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch { + throw new ApiError(400, "invalid_minimax_base_url", "MINIMAX_BASE_URL must be a valid HTTPS MiniMax endpoint"); + } + const hostname = url.hostname.toLowerCase(); + const isAllowed = hostname === "api.minimax.io" || hostname === "api.minimaxi.com"; + if (url.protocol !== "https:" || !isAllowed || url.username || url.password || url.pathname !== "/" || url.search || url.hash) { + throw new ApiError(400, "invalid_minimax_base_url", "MINIMAX_BASE_URL must be a trusted HTTPS MiniMax origin without a path"); + } + return url.origin; +} + +async function resolveMiniMaxCredentials( + env: EnvService, + region = DEFAULT_MINIMAX_SPEECH_REGION, +): Promise<{ apiKey: string; baseUrl: string }> { + const records = await env.list(); + const values = new Map(records.map((item) => [item.key, item.value.trim()] as const)); + const apiKey = values.get("MINIMAX_API_KEY") || process.env.MINIMAX_API_KEY?.trim() || ""; + if (!apiKey) { + throw new ApiError(400, "minimax_api_key_missing", "MiniMax API key missing. Configure MiniMax media in Authorization Center."); + } + const defaultBaseUrl = MINIMAX_SPEECH_ENDPOINTS[region]; + if (!defaultBaseUrl) { + throw new ApiError(400, "invalid_minimax_region", `MiniMax region must be one of: ${Object.keys(MINIMAX_SPEECH_ENDPOINTS).join(", ")}`); + } + const configuredBaseUrl = values.get("MINIMAX_BASE_URL") || process.env.MINIMAX_BASE_URL?.trim() || defaultBaseUrl; + return { apiKey, baseUrl: safeMiniMaxBaseUrl(configuredBaseUrl) }; +} + +async function requestMiniMaxTextToSpeech(input: { + apiKey: string; + baseUrl: string; + model: string; + text: string; + voice?: string; + format?: string; + audioSetting?: JsonRecord; + pronunciationDict?: JsonRecord; +}): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), BAILIAN_REQUEST_TIMEOUT_MS); + let response: Response; + try { + response = await fetch(endpoint(input.baseUrl, MINIMAX_T2A_PATH), { + method: "POST", + headers: { + Authorization: `Bearer ${input.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: input.model, + text: input.text, + stream: false, + ...(input.voice ? { voice_setting: { voice_id: input.voice } } : {}), + ...(input.format ? { output_format: input.format } : {}), + ...(input.audioSetting && Object.keys(input.audioSetting).length ? { audio_setting: input.audioSetting } : {}), + ...(input.pronunciationDict && Object.keys(input.pronunciationDict).length ? { pronunciation_dict: input.pronunciationDict } : {}), + }), + signal: controller.signal, + }); + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + throw new ApiError(504, "minimax_timeout", "MiniMax TTS did not respond before the request timed out."); + } + throw new ApiError(502, "minimax_unreachable", "Could not reach MiniMax. Check the network and try again."); + } finally { + clearTimeout(timeout); + } + + const payload: unknown = await response.json().catch(() => null); + const statusCode = isRecord(payload) && isRecord(payload.base_resp) + ? readOptionalNumber(payload.base_resp, "status_code") + : undefined; + if (!response.ok) { + throw new ApiError(response.status, "minimax_request_failed", minimaxErrorMessage(payload) || `MiniMax TTS request failed (HTTP ${response.status}).`); + } + if (statusCode !== undefined && statusCode !== 0) { + throw new ApiError(502, "minimax_request_failed", minimaxErrorMessage(payload) || `MiniMax TTS request failed (status_code ${statusCode}).`); + } + return payload; +} + function endpoint(baseUrl: string, path: string): string { return `${baseUrl}${path}`; } @@ -1045,11 +1163,31 @@ export async function callMediaExtensionAction( }; } - const { apiKey, baseUrl } = await resolveBailianCredentials(env); + const provider = readStringField(args, "provider") || "aliyun-bailian"; + let apiKey: string; + let baseUrl: string; + if (provider === "minimax") { + ({ apiKey, baseUrl } = await resolveMiniMaxCredentials(env, readStringField(args, "region") || DEFAULT_MINIMAX_SPEECH_REGION)); + } else { + ({ apiKey, baseUrl } = await resolveBailianCredentials(env)); + } let result: unknown; switch (action) { case "speech_synthesize": { const text = requireString(args, "text"); + if (provider === "minimax") { + result = await requestMiniMaxTextToSpeech({ + apiKey, + baseUrl, + text, + model: readStringField(args, "model") || DEFAULT_MINIMAX_SPEECH_MODEL, + voice: readStringField(args, "voice"), + format: readStringField(args, "format"), + audioSetting: readRecord(args, "audioSetting"), + pronunciationDict: readRecord(args, "pronunciationDict"), + }); + break; + } const model = readStringField(args, "model") || COSYVOICE_V3_FLASH; const voice = readStringField(args, "voice"); const input: JsonRecord = { @@ -1342,7 +1480,7 @@ export async function callMediaExtensionAction( extensionId: MEDIA_EXTENSION_ID, action, result: { - provider: "aliyun-bailian", + provider, operation: action, ...(isRecord(result) && typeof result.taskId === "string" ? { taskId: result.taskId } : {}), output: result,