diff --git a/apps/server/src/authorization-center.ts b/apps/server/src/authorization-center.ts index c285d6653..b789b968b 100644 --- a/apps/server/src/authorization-center.ts +++ b/apps/server/src/authorization-center.ts @@ -4,6 +4,7 @@ import { createAliyunOssV4Request, createS3V4Request } from "./object-storage-si export const AUTHORIZATION_SERVICE_IDS = [ "openai-images", "aliyun-bailian", + "minimax-media", "volcengine-video", "aliyun-oss", "wasabi", @@ -64,6 +65,17 @@ const AUTHORIZATION_SERVICES: readonly AuthorizationServiceDefinition[] = [ "Use the iPolloWork media extension actions from trusted runtime code. They keep DASHSCOPE_API_KEY on this device and provide the supported media operations without modifying OpenCode.", }, }, + { + id: "minimax-media", + keys: ["MINIMAX_API_KEY"], + category: "media", + agent: { + capability: "MiniMax speech TTS", + useWhen: "Use when the user asks to synthesize speech or create narration audio with a MiniMax account.", + instruction: + "Use the iPolloWork media extension speech_synthesize action from trusted runtime code. It keeps MINIMAX_API_KEY on this device and calls the MiniMax Text to Audio v2 endpoint without modifying OpenCode.", + }, + }, { id: "volcengine-video", keys: ["ARK_API_KEY"], @@ -218,6 +230,8 @@ export async function testAuthorizationService( return fetchAuthorizationTest("https://dashscope.aliyuncs.com/compatible-mode/v1/models", { headers: { Authorization: `Bearer ${resolved.values.DASHSCOPE_API_KEY}` }, }); + case "minimax-media": + return { ok: true, detail: "MiniMax API key saved. Speech synthesis verifies it when used." }; case "volcengine-video": return fetchAuthorizationTest( "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks?page_num=1&page_size=1", diff --git a/apps/server/src/extensions/media-center.test.ts b/apps/server/src/extensions/media-center.test.ts index d72175b94..f4c68f460 100644 --- a/apps/server/src/extensions/media-center.test.ts +++ b/apps/server/src/extensions/media-center.test.ts @@ -44,7 +44,7 @@ test("describes workspace speech synthesis as an installed iPolloWork capability const speechActions = MEDIA_EXTENSION_ACTIONS.filter((action) => action.action.startsWith("speech_synthesize")); expect(speechActions).toHaveLength(3); for (const action of speechActions) { - expect(action.description).toContain("Built-in iPolloWork CosyVoice action"); + expect(action.description).toContain("Built-in iPolloWork"); expect(action.description.toLowerCase()).toContain("without"); expect(action.description.toLowerCase()).toContain("external cli"); } @@ -726,6 +726,160 @@ describe("Media Center extension", () => { }); }); + test("requires the MiniMax key before synthesizing speech", async () => { + await expect(callMediaExtensionAction(config, env({}), "speech_synthesize", { + provider: "minimax", + text: "hello", + }, {})).rejects.toMatchObject({ code: "minimax_api_key_missing" }); + }); + + test("keeps the MiniMax key server-side while synthesizing speech", async () => { + globalThis.fetch = ((input, init) => { + expect(String(input)).toBe("https://api.minimax.io/v1/t2a_v2"); + expect(init?.headers).toMatchObject({ Authorization: "Bearer mm-test-key-0000" }); + expect(JSON.parse(String(init?.body))).toMatchObject({ + model: "speech-2.8-hd", + text: "hello", + stream: false, + output_format: "hex", + voice_setting: { voice_id: "female-qn-qingse" }, + }); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0, status_msg: "success" }, + data: { audio: "aabb", status: 2 }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + voice: "female-qn-qingse", + }, {}); + + expect(result).toMatchObject({ + ok: true, + extensionId: MEDIA_EXTENSION_ID, + action: "speech_synthesize", + result: { + provider: "minimax", + operation: "speech_synthesize", + output: { + base_resp: { status_code: 0, status_msg: "success" }, + data: { audio: "aabb", status: 2 }, + }, + }, + }); + expect(JSON.stringify(result)).not.toContain("mm-test-key-0000"); + }); + + test("routes MiniMax TTS to the China regional endpoint when requested", async () => { + globalThis.fetch = ((input) => { + expect(String(input)).toBe("https://api.minimaxi.com/v1/t2a_v2"); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0 }, + data: { audio: "aabb" }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + region: "cn_zh", + text: "hello", + }, {}); + + expect(result).toMatchObject({ ok: true, result: { provider: "minimax" } }); + }); + + test("passes MiniMax voice, audio, and pronunciation settings", async () => { + globalThis.fetch = ((_input, init) => { + const body = JSON.parse(String(init?.body)); + expect(body).toMatchObject({ + model: "speech-2.6-hd", + language_boost: "auto", + output_format: "url", + voice_setting: { voice_id: "English_expressive_narrator", speed: 1.1, emotion: "happy" }, + audio_setting: { sample_rate: 32000, bitrate: 128000, format: "mp3", channel: 1 }, + pronunciation_dict: { tone: ["Omg/Oh my god"] }, + voice_modify: { pitch: 1, intensity: 2, timbre: -1 }, + subtitle_enable: true, + }); + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0 }, + data: { audio: "aabb" }, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + const result = await callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + model: "speech-2.6-hd", + format: "mp3", + outputFormat: "url", + languageBoost: "auto", + voiceSetting: { voice_id: "English_expressive_narrator", speed: 1.1, emotion: "happy" }, + audioSetting: { sample_rate: 32000, bitrate: 128000, channel: 1 }, + pronunciationDict: { tone: ["Omg/Oh my god"] }, + voiceModify: { pitch: 1, intensity: 2, timbre: -1 }, + subtitleEnable: true, + }, {}); + + expect(result).toMatchObject({ ok: true, result: { provider: "minimax" } }); + }); + + test("explains MiniMax non-zero status codes", async () => { + globalThis.fetch = ((_input: Parameters[0], _init?: Parameters[1]) => { + return Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 2004, status_msg: "Invalid parameter" }, + data: {}, + }), { status: 200, headers: { "content-type": "application/json" } })); + }) as typeof fetch; + + await expect(callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + }, {})).rejects.toMatchObject({ + status: 502, + code: "minimax_request_failed", + message: "Invalid parameter", + }); + }); + + test("rejects unsupported MiniMax speech options before requesting audio", async () => { + let requested = false; + globalThis.fetch = ((_input: Parameters[0], _init?: Parameters[1]) => { + requested = true; + throw new Error("MiniMax must not be called"); + }) as unknown as typeof fetch; + + for (const [options, code] of [ + [{ model: "unsupported-model" }, "invalid_minimax_model"], + [{ format: "aac" }, "invalid_minimax_audio_format"], + [{ outputFormat: "binary" }, "invalid_minimax_output_format"], + ] as const) { + await expect(callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + ...options, + }, {})).rejects.toMatchObject({ code }); + } + expect(requested).toBe(false); + }); + + test("rejects a successful MiniMax response without audio data", async () => { + globalThis.fetch = ((_input, _init) => Promise.resolve(new Response(JSON.stringify({ + base_resp: { status_code: 0, status_msg: "success" }, + data: { status: 2 }, + }), { status: 200, headers: { "content-type": "application/json" } }))) as typeof fetch; + + await expect(callMediaExtensionAction(config, env({ MINIMAX_API_KEY: "mm-test-key-0000" }), "speech_synthesize", { + provider: "minimax", + text: "hello", + }, {})).rejects.toMatchObject({ + status: 502, + code: "minimax_response_invalid", + }); + }); + test("uses the asynchronous task endpoint for a digital human", async () => { globalThis.fetch = ((input, init) => { expect(String(input)).toBe("https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/video-synthesis"); diff --git a/apps/server/src/extensions/media-center.ts b/apps/server/src/extensions/media-center.ts index 507804ce3..4f3076491 100644 --- a/apps/server/src/extensions/media-center.ts +++ b/apps/server/src/extensions/media-center.ts @@ -30,6 +30,29 @@ const LEGACY_COSYVOICE_V3_PRESET_MIGRATIONS: Record = { longfei: "longanlang_v3", }; +// MiniMax speech TTS (Text to Audio v2) uses one regional endpoint per +// marketplace. The global endpoint serves MiniMax international accounts and +// the China endpoint serves accounts provisioned on the MiniMax China platform. +const MINIMAX_SPEECH_ENDPOINTS: Record = { + global_en: "https://api.minimax.io", + cn_zh: "https://api.minimaxi.com", +}; +const DEFAULT_MINIMAX_SPEECH_REGION = "global_en"; +const DEFAULT_MINIMAX_SPEECH_MODEL = "speech-2.8-hd"; +const MINIMAX_T2A_PATH = "/v1/t2a_v2"; +const MINIMAX_SPEECH_MODELS = [ + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", +]; +const MINIMAX_SPEECH_AUDIO_FORMATS = ["mp3", "wav", "flac", "pcm"]; +const MINIMAX_OUTPUT_FORMATS = ["hex", "url"]; + type JsonRecord = Record; const voiceoverAudioCache = new Map(); @@ -553,15 +576,24 @@ export const MEDIA_EXTENSION_ACTIONS = [ extensionId: MEDIA_EXTENSION_ID, action: "speech_synthesize", title: "Synthesize speech", - description: "Built-in iPolloWork CosyVoice action. Create speech without installing or authenticating an external CLI; the result contains a temporary audio URL from Model Studio.", + description: "Built-in iPolloWork speech synthesis action. Create speech with the configured media provider without installing or authenticating an external CLI; the result contains provider-specific audio data.", inputSchema: { type: "object", properties: { + provider: { type: "string", enum: ["aliyun-bailian", "minimax"], description: "Optional media provider. Defaults to aliyun-bailian." }, text: { type: "string", description: "Text to synthesize." }, - voice: { type: "string", description: "Optional Model Studio voice name or cloned voice id." }, - model: { type: "string", description: "Optional speech model. Defaults to cosyvoice-v3-flash." }, - format: { type: "string", description: "Optional audio format, for example wav or mp3." }, + voice: { type: "string", description: "Optional provider voice name, cloned voice id, or MiniMax voice_id." }, + model: { type: "string", description: `Optional speech model. Defaults to cosyvoice-v3-flash for Alibaba and speech-2.8-hd for MiniMax. MiniMax models: ${MINIMAX_SPEECH_MODELS.join(", ")}.` }, + format: { type: "string", description: `Optional encoded audio format. MiniMax supports ${MINIMAX_SPEECH_AUDIO_FORMATS.join(", ")}.` }, sampleRate: { type: "number", description: "Optional output sample rate in Hz." }, + region: { type: "string", enum: ["global_en", "cn_zh"], description: "Optional MiniMax regional endpoint. Defaults to global_en." }, + outputFormat: { type: "string", enum: ["hex", "url"], description: "Optional MiniMax response representation. Defaults to hex." }, + languageBoost: { type: "string", description: "Optional MiniMax language_boost value, for example auto or English." }, + voiceSetting: { type: "object", description: "Optional MiniMax voice_setting overrides, including voice_id, speed, vol, pitch, or emotion." }, + audioSetting: { type: "object", description: "Optional MiniMax audio_setting overrides, for example sample_rate, bitrate, format, or channel." }, + pronunciationDict: { type: "object", description: "Optional MiniMax pronunciation_dict custom pronunciation overrides." }, + voiceModify: { type: "object", description: "Optional MiniMax voice_modify effect settings." }, + subtitleEnable: { type: "boolean", description: "Optional MiniMax subtitle_enable flag." }, }, required: ["text"], additionalProperties: false, @@ -1220,6 +1252,114 @@ async function resolveBailianCredentials(env: EnvService): Promise<{ apiKey: str return { apiKey, baseUrl: safeProviderBaseUrl(configuredBaseUrl) }; } +function minimaxErrorMessage(payload: unknown): string { + if (!isRecord(payload)) return ""; + const baseResp = isRecord(payload.base_resp) ? payload.base_resp : null; + return readStringField(baseResp, "status_msg") || providerMessage(payload) || ""; +} + +async function resolveMiniMaxCredentials( + env: EnvService, + region = DEFAULT_MINIMAX_SPEECH_REGION, +): Promise<{ apiKey: string; baseUrl: string }> { + const records = await env.list(); + const values = new Map(records.map((item) => [item.key, item.value.trim()] as const)); + const apiKey = values.get("MINIMAX_API_KEY") || process.env.MINIMAX_API_KEY?.trim() || ""; + if (!apiKey) { + throw new ApiError(400, "minimax_api_key_missing", "MiniMax API key missing. Configure MiniMax media in Authorization Center."); + } + const defaultBaseUrl = MINIMAX_SPEECH_ENDPOINTS[region]; + if (!defaultBaseUrl) { + throw new ApiError(400, "invalid_minimax_region", `MiniMax region must be one of: ${Object.keys(MINIMAX_SPEECH_ENDPOINTS).join(", ")}`); + } + return { apiKey, baseUrl: defaultBaseUrl }; +} + +async function requestMiniMaxTextToSpeech(input: { + apiKey: string; + baseUrl: string; + model: string; + text: string; + voice?: string; + format?: string; + outputFormat?: string; + languageBoost?: string; + voiceSetting?: JsonRecord; + audioSetting?: JsonRecord; + pronunciationDict?: JsonRecord; + voiceModify?: JsonRecord; + subtitleEnable?: boolean; +}): Promise { + if (!MINIMAX_SPEECH_MODELS.includes(input.model)) { + throw new ApiError(400, "invalid_minimax_model", `MiniMax speech model must be one of: ${MINIMAX_SPEECH_MODELS.join(", ")}`); + } + if (input.format && !MINIMAX_SPEECH_AUDIO_FORMATS.includes(input.format)) { + throw new ApiError(400, "invalid_minimax_audio_format", `MiniMax audio format must be one of: ${MINIMAX_SPEECH_AUDIO_FORMATS.join(", ")}`); + } + const outputFormat = input.outputFormat || "hex"; + if (!MINIMAX_OUTPUT_FORMATS.includes(outputFormat)) { + throw new ApiError(400, "invalid_minimax_output_format", `MiniMax output format must be one of: ${MINIMAX_OUTPUT_FORMATS.join(", ")}`); + } + const voiceSetting = { + ...(input.voiceSetting || {}), + ...(input.voice ? { voice_id: input.voice } : {}), + }; + const audioSetting = { + ...(input.audioSetting || {}), + ...(input.format ? { format: input.format } : {}), + }; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), BAILIAN_REQUEST_TIMEOUT_MS); + let response: Response; + try { + response = await mediaProviderFetch(endpoint(input.baseUrl, MINIMAX_T2A_PATH), { + method: "POST", + headers: { + Authorization: `Bearer ${input.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: input.model, + text: input.text, + stream: false, + output_format: outputFormat, + ...(input.languageBoost ? { language_boost: input.languageBoost } : {}), + ...(Object.keys(voiceSetting).length ? { voice_setting: voiceSetting } : {}), + ...(Object.keys(audioSetting).length ? { audio_setting: audioSetting } : {}), + ...(input.pronunciationDict && Object.keys(input.pronunciationDict).length ? { pronunciation_dict: input.pronunciationDict } : {}), + ...(input.voiceModify && Object.keys(input.voiceModify).length ? { voice_modify: input.voiceModify } : {}), + ...(input.subtitleEnable === undefined ? {} : { subtitle_enable: input.subtitleEnable }), + }), + signal: controller.signal, + }); + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + throw new ApiError(504, "minimax_timeout", "MiniMax TTS did not respond before the request timed out."); + } + throw new ApiError(502, "minimax_unreachable", "Could not reach MiniMax. Check the network and try again."); + } finally { + clearTimeout(timeout); + } + + const payload: unknown = await response.json().catch(() => null); + const baseResp = isRecord(payload) && isRecord(payload.base_resp) ? payload.base_resp : null; + const statusCode = readOptionalNumber(baseResp, "status_code"); + if (!response.ok) { + throw new ApiError(response.status, "minimax_request_failed", minimaxErrorMessage(payload) || `MiniMax TTS request failed (HTTP ${response.status}).`); + } + if (statusCode === undefined) { + throw new ApiError(502, "minimax_response_invalid", "MiniMax TTS did not return base_resp.status_code."); + } + if (statusCode !== 0) { + throw new ApiError(502, "minimax_request_failed", minimaxErrorMessage(payload) || `MiniMax TTS request failed (status_code ${statusCode}).`); + } + const data = isRecord(payload) ? readRecord(payload, "data") : {}; + if (!readStringField(data, "audio")) { + throw new ApiError(502, "minimax_response_invalid", "MiniMax TTS did not return audio data."); + } + return payload; +} + function endpoint(baseUrl: string, path: string): string { return `${baseUrl}${path}`; } @@ -1525,11 +1665,40 @@ export async function callMediaExtensionAction( }; } - const { apiKey, baseUrl } = await resolveBailianCredentials(env); + const provider = readStringField(args, "provider") || "aliyun-bailian"; + let apiKey: string; + let baseUrl: string; + if (provider === "minimax") { + ({ apiKey, baseUrl } = await resolveMiniMaxCredentials(env, readStringField(args, "region") || DEFAULT_MINIMAX_SPEECH_REGION)); + } else { + ({ apiKey, baseUrl } = await resolveBailianCredentials(env)); + } let result: unknown; switch (action) { case "speech_synthesize": { const text = requireString(args, "text"); + if (provider === "minimax") { + const sampleRate = readOptionalNumber(args, "sampleRate"); + result = await requestMiniMaxTextToSpeech({ + apiKey, + baseUrl, + text, + model: readStringField(args, "model") || DEFAULT_MINIMAX_SPEECH_MODEL, + voice: readStringField(args, "voice"), + format: readStringField(args, "format"), + outputFormat: readStringField(args, "outputFormat"), + languageBoost: readStringField(args, "languageBoost"), + voiceSetting: readRecord(args, "voiceSetting"), + audioSetting: { + ...(sampleRate === undefined ? {} : { sample_rate: sampleRate }), + ...readRecord(args, "audioSetting"), + }, + pronunciationDict: readRecord(args, "pronunciationDict"), + voiceModify: readRecord(args, "voiceModify"), + subtitleEnable: readOptionalBoolean(args, "subtitleEnable"), + }); + break; + } const model = readStringField(args, "model") || COSYVOICE_V3_FLASH; const voice = readStringField(args, "voice"); const input: JsonRecord = { @@ -1873,7 +2042,7 @@ export async function callMediaExtensionAction( extensionId: MEDIA_EXTENSION_ID, action, result: { - provider: "aliyun-bailian", + provider, operation: action, ...(isRecord(result) && typeof result.taskId === "string" ? { taskId: result.taskId } : {}), output: result,