From 87b40cc2d087c584f913b8807a4358a50a15405f Mon Sep 17 00:00:00 2001 From: adhikjoshi Date: Fri, 20 Feb 2026 13:47:41 +0530 Subject: [PATCH 1/2] feat: Add ModelsLab TTS engine - Added ModelsLabTTSClient in src/engines/modelslab.ts - Registered in factory.ts, factory-browser.ts, index.ts, browser.ts - Added 'modelslab' to SupportedTTS/SupportedBrowserTTS unions - Added 'modelslab' to UnifiedVoice provider union in types.ts - Supports 9 voices (madison, tara, leah, jess, mia, zoe, leo, dan, zac) - Handles async audio generation with polling - Works in both Node.js and browser environments via getFetch() --- src/browser.ts | 1 + src/engines/modelslab.ts | 267 +++++++++++++++++++++++++++++++++++++++ src/factory-browser.ts | 6 + src/factory.ts | 6 + src/index.ts | 1 + src/types.ts | 1 + 6 files changed, 282 insertions(+) create mode 100644 src/engines/modelslab.ts diff --git a/src/browser.ts b/src/browser.ts index 29c025a..6dd3e40 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -23,6 +23,7 @@ export { WitAITTSClient } from "./engines/witai"; export { SherpaOnnxWasmTTSClient } from "./engines/sherpaonnx-wasm"; export { EspeakBrowserTTSClient } from "./engines/espeak-wasm"; export { UpliftAITTSClient } from "./engines/upliftai"; +export { ModelsLabTTSClient } from "./engines/modelslab"; // Browser-compatible factory (excludes server-only engines) export { createBrowserTTSClient } from "./factory-browser"; diff --git a/src/engines/modelslab.ts b/src/engines/modelslab.ts new file mode 100644 index 0000000..ef42aef --- /dev/null +++ b/src/engines/modelslab.ts @@ -0,0 +1,267 @@ +import { AbstractTTSClient } from "../core/abstract-tts"; +import * as SSMLUtils from "../core/ssml-utils"; +import * as SpeechMarkdown from "../markdown/converter"; +import type { SpeakOptions, TTSCredentials, UnifiedVoice } from "../types"; +import { getFetch } from "../utils/fetch-utils"; + +/** + * ModelsLab TTS Client Credentials + */ +export interface ModelsLabTTSCredentials extends TTSCredentials { + /** ModelsLab API key (also reads MODELSLAB_API_KEY env var) */ + apiKey?: string; +} + +/** + * Extended speak options for ModelsLab TTS + */ +export interface ModelsLabTTSOptions extends SpeakOptions { + /** Language descriptor e.g. "american english", "british english" */ + language?: string; + /** Speed multiplier (default 1.0) */ + speed?: number; + /** Enable emotion tags in the prompt (English only) */ + emotion?: boolean; +} + +/** Static list of available voices */ +const MODELSLAB_VOICES: UnifiedVoice[] = [ + // Emotion-capable female voices + { id: "madison", name: "Madison", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "tara", name: "Tara", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "leah", name: "Leah", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "jess", name: "Jess", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "mia", name: "Mia", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "zoe", name: "Zoe", gender: "Female", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + // Emotion-capable male voices + { id: "leo", name: "Leo", gender: "Male", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "dan", name: "Dan", gender: "Male", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, + { id: "zac", name: "Zac", gender: "Male", provider: "modelslab", languageCodes: [{ bcp47: "en-US", iso639_3: "eng", display: "English (US)" }] }, +]; + +const API_URL = "https://modelslab.com/api/v6/voice/text_to_speech"; +const DEFAULT_VOICE = "madison"; +const DEFAULT_LANGUAGE = "american english"; +const POLL_INTERVAL_MS = 2000; +const MAX_POLL_ATTEMPTS = 20; + +/** + * ModelsLab TTS Client + * + * Provides text-to-speech via the ModelsLab Voice API. + * API docs: https://docs.modelslab.com/voice-cloning/text-to-speech + * + * @example + * ```ts + * const client = new ModelsLabTTSClient({ apiKey: "your-api-key" }); + * await client.synthToFile("Hello world!", "output.mp3"); + * ``` + */ +export class ModelsLabTTSClient extends AbstractTTSClient { + private apiKey: string; + private defaultLanguage: string; + private defaultSpeed: number; + protected sampleRate = 24000; + + constructor(credentials: ModelsLabTTSCredentials = {}) { + super(credentials); + this.apiKey = + credentials.apiKey || + (typeof process !== "undefined" ? process.env.MODELSLAB_API_KEY ?? "" : ""); + this.defaultLanguage = DEFAULT_LANGUAGE; + this.defaultSpeed = 1.0; + if (!this.voiceId) { + this.voiceId = DEFAULT_VOICE; + } + } + + /** Check if credentials are present */ + async checkCredentials(): Promise { + if (!this.apiKey) { + console.error("ModelsLab API key is required. Set MODELSLAB_API_KEY or pass apiKey."); + return false; + } + return true; + } + + protected getRequiredCredentials(): string[] { + return ["apiKey"]; + } + + protected async _getVoices(): Promise { + return MODELSLAB_VOICES; + } + + /** + * Synthesize text to audio bytes (Uint8Array). + * Handles async generation — polls until audio is ready. + */ + async synthToBytes(text: string, options: ModelsLabTTSOptions = {}): Promise { + const { audioStream } = await this.synthToBytestream(text, options); + const reader = audioStream.getReader(); + const chunks: Uint8Array[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + const totalLen = chunks.reduce((n, c) => n + c.length, 0); + const out = new Uint8Array(totalLen); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; + } + + /** + * Synthesize text to a ReadableStream of audio chunks. + */ + async synthToBytestream( + text: string, + options: ModelsLabTTSOptions = {} + ): Promise<{ + audioStream: ReadableStream; + wordBoundaries: Array<{ text: string; offset: number; duration: number }>; + }> { + let processedText = text; + + // Convert SpeechMarkdown → SSML → plain text if needed + if (options.useSpeechMarkdown && SpeechMarkdown.isSpeechMarkdown(processedText)) { + const ssml = await SpeechMarkdown.toSSML(processedText); + processedText = SSMLUtils.stripSSML(ssml); + } else if (SSMLUtils.isSSML(processedText)) { + // ModelsLab doesn't support SSML — strip tags + processedText = SSMLUtils.stripSSML(processedText); + } + + const voiceId = options.voice || this.voiceId || DEFAULT_VOICE; + this.voiceId = voiceId; + + const speed = options.speed ?? this.defaultSpeed; + const language = options.language ?? this.defaultLanguage; + + const audioBytes = await this._synthesize(processedText, voiceId, language, speed, options.emotion ?? false); + + const audioStream = new ReadableStream({ + start(controller) { + controller.enqueue(audioBytes); + controller.close(); + }, + }); + + return { audioStream, wordBoundaries: [] }; + } + + /** Internal: call ModelsLab API and return audio bytes. */ + private async _synthesize( + text: string, + voiceId: string, + language: string, + speed: number, + emotion: boolean + ): Promise { + const fetch = getFetch(); + + const resp = await fetch(API_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + key: this.apiKey, + prompt: text, + language, + voice_id: voiceId, + speed, + emotion, + }), + }); + + if (!resp.ok) { + throw new Error(`ModelsLab API error: ${resp.status} ${resp.statusText}`); + } + + const data = (await resp.json()) as { + status: string; + output?: string[]; + fetch_result?: string; + link?: string; + message?: string; + }; + + if (data.status === "error") { + throw new Error(`ModelsLab TTS error: ${data.message ?? JSON.stringify(data)}`); + } + + let audioUrl: string | undefined; + + if (data.status === "success" && data.output?.length) { + audioUrl = data.output[0]; + } else if (data.status === "processing") { + const fetchUrl = data.fetch_result ?? data.link; + if (!fetchUrl) { + throw new Error("ModelsLab returned processing status with no fetch URL"); + } + audioUrl = await this._poll(fetchUrl, fetch); + } else { + throw new Error(`Unexpected ModelsLab status: ${data.status}`); + } + + if (!audioUrl) { + throw new Error("ModelsLab returned no audio URL"); + } + + return this._downloadAudio(audioUrl, fetch); + } + + /** Poll the fetch_result URL until audio is ready. */ + private async _poll( + fetchUrl: string, + fetch: ReturnType + ): Promise { + for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt++) { + await this._sleep(POLL_INTERVAL_MS); + + const resp = await fetch(fetchUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ key: this.apiKey }), + }); + + if (!resp.ok) continue; + + const data = (await resp.json()) as { + status: string; + output?: string[]; + message?: string; + }; + + if (data.status === "success" && data.output?.length) { + return data.output[0]; + } + if (data.status === "error") { + throw new Error(`ModelsLab poll error: ${data.message}`); + } + } + throw new Error(`ModelsLab audio generation timed out after ${MAX_POLL_ATTEMPTS} attempts`); + } + + /** Download audio from URL and return as Uint8Array. */ + private async _downloadAudio( + url: string, + fetch: ReturnType + ): Promise { + const resp = await fetch(url); + if (!resp.ok) { + throw new Error(`Failed to download audio: ${resp.status} ${resp.statusText}`); + } + const buf = await resp.arrayBuffer(); + return new Uint8Array(buf); + } + + private _sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} + +export default ModelsLabTTSClient; diff --git a/src/factory-browser.ts b/src/factory-browser.ts index f6bcaa0..80749c1 100644 --- a/src/factory-browser.ts +++ b/src/factory-browser.ts @@ -10,6 +10,7 @@ import { SherpaOnnxWasmTTSClient } from "./engines/sherpaonnx-wasm.js"; import { WatsonTTSClient } from "./engines/watson.js"; import { WitAITTSClient } from "./engines/witai.js"; import { UpliftAITTSClient } from "./engines/upliftai.js"; +import { ModelsLabTTSClient } from "./engines/modelslab.js"; import type { TTSCredentials } from "./types"; // Import MockTTSClient for testing @@ -37,6 +38,7 @@ export type SupportedBrowserTTS = | "watson" | "witai" | "upliftai" + | "modelslab" | "sherpaonnx-wasm" | "espeak-wasm" | "mock"; @@ -113,6 +115,10 @@ export function createBrowserTTSClient(engine: SupportedBrowserTTS, credentials? return applyProperties( new UpliftAITTSClient(credentials as import("./engines/upliftai").UpliftAITTSCredentials) ); + case "modelslab": + return applyProperties( + new ModelsLabTTSClient(credentials as import("./engines/modelslab").ModelsLabTTSCredentials) + ); case "sherpaonnx-wasm": return applyProperties(new SherpaOnnxWasmTTSClient(credentials as any)); case "espeak-wasm": diff --git a/src/factory.ts b/src/factory.ts index e2b10e7..44318fb 100644 --- a/src/factory.ts +++ b/src/factory.ts @@ -8,6 +8,7 @@ import { OpenAITTSClient } from "./engines/openai.js"; import { PlayHTTTSClient } from "./engines/playht.js"; import { PollyTTSClient } from "./engines/polly.js"; import { UpliftAITTSClient } from "./engines/upliftai.js"; +import { ModelsLabTTSClient } from "./engines/modelslab.js"; import { SherpaOnnxWasmTTSClient } from "./engines/sherpaonnx-wasm.js"; import { SherpaOnnxTTSClient } from "./engines/sherpaonnx.js"; import { WatsonTTSClient } from "./engines/watson.js"; @@ -40,6 +41,7 @@ export type SupportedTTS = | "watson" | "witai" | "upliftai" + | "modelslab" | "sherpaonnx" | "sherpaonnx-wasm" | "espeak" @@ -119,6 +121,10 @@ export function createTTSClient(engine: SupportedTTS, credentials?: TTSCredentia return applyProperties( new UpliftAITTSClient(credentials as import("./engines/upliftai").UpliftAITTSCredentials) ); + case "modelslab": + return applyProperties( + new ModelsLabTTSClient(credentials as import("./engines/modelslab").ModelsLabTTSCredentials) + ); case "sherpaonnx": return applyProperties(new SherpaOnnxTTSClient(credentials as any)); case "sherpaonnx-wasm": diff --git a/src/index.ts b/src/index.ts index e590daf..96a64ff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,7 @@ export { EspeakBrowserTTSClient, EspeakWasmTTSClient } from "./engines/espeak-wa export { WatsonTTSClient } from "./engines/watson"; export { WitAITTSClient } from "./engines/witai"; export { UpliftAITTSClient } from "./engines/upliftai"; +export { ModelsLabTTSClient } from "./engines/modelslab"; export { SAPITTSClient } from "./engines/sapi"; // Type exports diff --git a/src/types.ts b/src/types.ts index c36dd51..4132912 100644 --- a/src/types.ts +++ b/src/types.ts @@ -108,6 +108,7 @@ export type UnifiedVoice = { | "playht" | "openai" | "upliftai" + | "modelslab" | "sherpa" | "sherpaonnx" | "sherpaonnx-wasm" From e5fd90b4abb385fc4b86939fe58ff99be0050667 Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 20 Feb 2026 09:09:10 +0000 Subject: [PATCH 2/2] fixes to build and ci Added @types/node in devDependencies. Needed because TypeScript build failed (TS2688: Cannot find type definition file for 'node') after clean install. emoved one malformed lock entry (node_modules/sherpa-onnx-node/node_modules/sherpa-onnx-linux-arm64 without a version) that caused npm install/ci to crash with Invalid Version:. Lockfile also got small metadata churn from npm re-resolution (peer flags/versions), but no runtime code changed there. Simplified dynamic loading and improved fallback conversion for emphasis/prosody syntax. Needed because tests were failing in speech-markdown-converter.test.ts and throwing Jest teardown import errors. --- package-lock.json | 24 ++++++----------- package.json | 1 + src/markdown/converter.ts | 56 +++++++++++---------------------------- 3 files changed, 25 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7f93303..eb75dbe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "@rollup/plugin-typescript": "^12.3.0", "@types/jest": "^30.0.0", "@types/mock-fs": "^4.13.4", + "@types/node": "^24.5.2", "@types/node-fetch": "^2.6.13", "cross-env": "^10.1.0", "decompress": "^4.2.1", @@ -850,7 +851,6 @@ "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/generator": "^7.28.6", @@ -3536,13 +3536,13 @@ } }, "node_modules/@types/node": { - "version": "22.15.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.21.tgz", - "integrity": "sha512-EV/37Td6c+MgKAbkcLG6vqZ2zEYHD7bvSrzqqs2RIhbA6w3x+Dqz8MZM3sP6kGTeLrdoOgKZe+Xja7tUB2DNkQ==", + "version": "24.10.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz", + "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", "dev": true, "license": "MIT", "dependencies": { - "undici-types": "~6.21.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/node-fetch": { @@ -4298,7 +4298,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6648,7 +6647,6 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -8351,7 +8349,6 @@ "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8849,7 +8846,6 @@ "integrity": "sha512-PggGy4dhwx5qaW+CKBilA/98Ql9keyfnb7lh4SR6shQ91QQQi1ORJ1v4UinkdP2i87OBs9AQFooQylcrrRfIcg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -9072,9 +9068,6 @@ "sherpa-onnx-win-x64": "^1.12.23" } }, - "node_modules/sherpa-onnx-node/node_modules/sherpa-onnx-linux-arm64": { - "optional": true - }, "node_modules/sherpa-onnx-win-ia32": { "version": "1.12.23", "resolved": "https://registry.npmjs.org/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.12.23.tgz", @@ -9938,7 +9931,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -9998,9 +9990,9 @@ } }, "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "dev": true, "license": "MIT" }, diff --git a/package.json b/package.json index f044592..f90edfe 100644 --- a/package.json +++ b/package.json @@ -138,6 +138,7 @@ "@rollup/plugin-typescript": "^12.3.0", "@types/jest": "^30.0.0", "@types/mock-fs": "^4.13.4", + "@types/node": "^24.5.2", "@types/node-fetch": "^2.6.13", "cross-env": "^10.1.0", "decompress": "^4.2.1", diff --git a/src/markdown/converter.ts b/src/markdown/converter.ts index ca4c540..6c294ae 100644 --- a/src/markdown/converter.ts +++ b/src/markdown/converter.ts @@ -93,58 +93,20 @@ async function loadSpeechMarkdown() { let module: any = null; - // In Node.js environments, try require() first as it's more reliable in bundled contexts if (isNode) { try { - // Try to use require directly (works in most Node.js bundled contexts) const requireFn = typeof require !== "undefined" ? require : undefined; if (requireFn) { module = requireFn("speechmarkdown-js"); } } catch { - // require failed, try dynamic import below - } - - // If require didn't work, try createRequire with better fallback handling - if (!module) { - try { - const { createRequire } = await new Function("m", "return import(m)")("node:module"); - let metaUrl: string | undefined; - - // Try multiple methods to get a valid URL for createRequire - try { - metaUrl = new Function("return import.meta.url")(); - } catch { - // import.meta.url not available, try alternatives - metaUrl = undefined; - } - - // Use multiple fallback strategies for createRequire - const fallbackPaths = [ - metaUrl, - typeof __filename !== "undefined" ? `file://${__filename}` : undefined, - `${process.cwd().replace(/\\/g, "/")}/index.js`, - `${process.cwd().replace(/\\/g, "/")}/package.json`, - ].filter(Boolean); - - for (const fallbackPath of fallbackPaths) { - try { - const requireFn = createRequire(fallbackPath as string); - module = requireFn("speechmarkdown-js"); - if (module) break; - } catch {} - } - } catch { - // createRequire failed, will try dynamic import next - } + // Fallback to dynamic import below } } - // If Node.js methods didn't work or we're in a browser, try dynamic import if (!module) { try { - const dynamicImport: any = new Function("m", "return import(m)"); - module = await dynamicImport("speechmarkdown-js"); + module = await import("speechmarkdown-js"); } catch { // Dynamic import failed } @@ -172,6 +134,20 @@ function convertSpeechMarkdownFallback(markdown: string): string { out = out.replace(/\[break:"([^"]+)"\]/g, ''); // [500ms] or [500s] -> out = out.replace(/\[(\d+)m?s\]/g, ''); + // ++text++ -> text + out = out.replace(/\+\+([\s\S]+?)\+\+/g, '$1'); + // (text)[rate:'x-slow'] or (text)[rate:"x-slow"] -> prosody rate + out = out.replace(/\(([\s\S]+?)\)\[rate:['"]([^'"]+)['"]\]/g, '$1'); + // (text)[pitch:'high'] or (text)[pitch:"high"] -> prosody pitch + out = out.replace( + /\(([\s\S]+?)\)\[pitch:['"]([^'"]+)['"]\]/g, + '$1' + ); + // (text)[volume:'loud'] or (text)[volume:"loud"] -> prosody volume + out = out.replace( + /\(([\s\S]+?)\)\[volume:['"]([^'"]+)['"]\]/g, + '$1' + ); return out; }