diff --git a/.changeset/add-speechify-plugin.md b/.changeset/add-speechify-plugin.md new file mode 100644 index 000000000..72ff73c75 --- /dev/null +++ b/.changeset/add-speechify-plugin.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents-plugin-speechify": patch +--- + +feat(speechify): add Speechify TTS plugin with streaming and word-level timestamps diff --git a/README.md b/README.md index e708dcf6e..d3aa9b27f 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Currently, only the following plugins are supported: | [@livekit/agents-plugin-resemble](https://www.npmjs.com/package/@livekit/agents-plugin-resemble) | TTS | | [@livekit/agents-plugin-rime](https://www.npmjs.com/package/@livekit/agents-plugin-rime) | TTS | | [@livekit/agents-plugin-inworld](https://www.npmjs.com/package/@livekit/agents-plugin-inworld) | STT, TTS | +| [@livekit/agents-plugin-speechify](https://www.npmjs.com/package/@livekit/agents-plugin-speechify) | TTS | | [@livekit/agents-plugin-silero](https://www.npmjs.com/package/@livekit/agents-plugin-silero) | VAD | | [@livekit/agents-plugin-livekit](https://www.npmjs.com/package/@livekit/agents-plugin-livekit) | EOU | | [@livekit/agents-plugin-anam](https://www.npmjs.com/package/@livekit/agents-plugin-anam) | Avatar | diff --git a/plugins/speechify/CHANGELOG.md b/plugins/speechify/CHANGELOG.md new file mode 100644 index 000000000..62cf44533 --- /dev/null +++ b/plugins/speechify/CHANGELOG.md @@ -0,0 +1 @@ +# @livekit/agents-plugin-speechify diff --git a/plugins/speechify/README.md b/plugins/speechify/README.md new file mode 100644 index 000000000..a4f60502a --- /dev/null +++ b/plugins/speechify/README.md @@ -0,0 +1,17 @@ + +# Speechify plugin for LiveKit Agents + +The Agents Framework is designed for building realtime, programmable +participants that run on servers. Use it to create conversational, multi-modal +voice agents that can see, hear, and understand. + +This package contains the Speechify plugin, which provides streaming text-to-speech +with word-level timestamps. Refer to the [documentation](https://docs.livekit.io/agents/overview/) +for information on how to use it, or browse the [API +reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_speechify.html). +See the [repository](https://github.com/livekit/agents-js) for more information +about the framework as a whole. diff --git a/plugins/speechify/api-extractor.json b/plugins/speechify/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/speechify/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/speechify/package.json b/plugins/speechify/package.json new file mode 100644 index 000000000..aab77e5f1 --- /dev/null +++ b/plugins/speechify/package.json @@ -0,0 +1,52 @@ +{ + "name": "@livekit/agents-plugin-speechify", + "version": "1.5.0", + "description": "Speechify plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-openai": "workspace:*", + "@livekit/agents-plugins-test": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "@speechify/api": "^3.0.1" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/speechify/src/index.ts b/plugins/speechify/src/index.ts new file mode 100644 index 000000000..cd06b682b --- /dev/null +++ b/plugins/speechify/src/index.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export * from './models.js'; +export * from './tts.js'; + +class SpeechifyPlugin extends Plugin { + constructor() { + super({ + title: 'speechify', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new SpeechifyPlugin()); diff --git a/plugins/speechify/src/models.ts b/plugins/speechify/src/models.ts new file mode 100644 index 000000000..6d54a43e1 --- /dev/null +++ b/plugins/speechify/src/models.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +export type TTSModels = 'simba-english' | 'simba-multilingual' | 'simba-3.0' | 'simba-3.2'; diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts new file mode 100644 index 000000000..1043a5b6c --- /dev/null +++ b/plugins/speechify/src/tts.ts @@ -0,0 +1,313 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APIError, + APIStatusError, + AudioByteStream, + USERDATA_TIMED_TRANSCRIPT, + createTimedString, + shortuuid, + tokenize, + tts, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import type { Speechify } from '@speechify/api'; +import { SpeechifyClient, SpeechifyError } from '@speechify/api'; +import type { TTSModels } from './models.js'; + +const NUM_CHANNELS = 1; +const SAMPLE_RATE = 24000; +const AUDIO_FORMAT: Speechify.GetSpeechRequest.AudioFormat = 'pcm'; +const DEFAULT_VOICE_ID = 'dominic_32'; +const DEFAULT_MODEL: TTSModels = 'simba-3.2'; + +export interface TTSOptions { + voiceId: string; + model?: TTSModels; + language?: string; + loudnessNormalization?: boolean; + textNormalization?: boolean; + apiKey?: string; + baseUrl?: string; + tokenizer?: tokenize.SentenceTokenizer; +} + +const defaultOptions = (): Omit => ({ + voiceId: DEFAULT_VOICE_ID, + model: DEFAULT_MODEL, +}); + +const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeechRequest => { + const request: Speechify.GetSpeechRequest = { + audio_format: AUDIO_FORMAT, + input: text, + voice_id: opts.voiceId, + }; + if (opts.model) request.model = opts.model; + if (opts.language) request.language = opts.language; + if (opts.loudnessNormalization !== undefined || opts.textNormalization !== undefined) { + request.options = { + loudness_normalization: opts.loudnessNormalization, + text_normalization: opts.textNormalization, + }; + } + return request; +}; + +const toError = (e: unknown): Error => { + if (e instanceof APIError) { + return e; + } + if (e instanceof SpeechifyError) { + return new APIStatusError({ message: e.message, options: { statusCode: e.statusCode ?? -1 } }); + } + return new APIConnectionError({ message: e instanceof Error ? e.message : String(e) }); +}; + +export class TTS extends tts.TTS { + label = 'speechify.TTS'; + #opts: TTSOptions; + #client: SpeechifyClient; + #tokenizer: tokenize.SentenceTokenizer; + + /** + * Create a new instance of Speechify TTS. + * + * @remarks + * `apiKey` must be set, either via the constructor or the `SPEECHIFY_API_KEY` + * environment variable. + * + * Synthesis uses the Speechify `/audio/speech` endpoint, which returns raw PCM + * (24 kHz mono) plus word-level speech marks. `stream()` chunks input into + * sentences and issues one request per sentence, emitting audio and aligned + * word timestamps as each sentence completes. + * + * Defaults to the `dominic_32` voice and the `simba-3.2` model. The voice must + * support the chosen model; see the `/v1/voices` endpoint. + */ + constructor(opts: Partial = {}) { + const merged = { ...defaultOptions(), ...opts }; + + super(SAMPLE_RATE, NUM_CHANNELS, { streaming: true, alignedTranscript: true }); + + this.#opts = merged; + this.#tokenizer = merged.tokenizer ?? new tokenize.basic.SentenceTokenizer(); + + const token = merged.apiKey ?? process.env.SPEECHIFY_API_KEY; + if (!token) { + throw new Error( + 'Speechify API key is required, whether as an argument or as $SPEECHIFY_API_KEY', + ); + } + this.#client = new SpeechifyClient({ token, baseUrl: merged.baseUrl }); + } + + get model(): string { + return this.#opts.model ?? 'unknown'; + } + + get provider(): string { + return 'Speechify'; + } + + get options(): TTSOptions { + return this.#opts; + } + + get tokenizer(): tokenize.SentenceTokenizer { + return this.#tokenizer; + } + + /** @internal */ + async _synthesize( + text: string, + opts: TTSOptions, + offsetSeconds: number, + params: { abortSignal: AbortSignal; timeoutInSeconds?: number }, + ): Promise<{ audio: Buffer; timed: ReturnType[] }> { + const response = await this.#client.audio.speech(buildSpeechRequest(text, opts), params); + return { + audio: Buffer.from(response.audio_data, 'base64'), + timed: timedStringsFromMarks(response.speech_marks, offsetSeconds), + }; + } + + updateOptions(opts: Partial>) { + this.#opts = { ...this.#opts, ...opts }; + } + + synthesize( + text: string, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ): tts.ChunkedStream { + return new ChunkedStream(this, text, this.#opts, connOptions, abortSignal); + } + + stream(options?: { connOptions?: APIConnectOptions }): tts.SynthesizeStream { + return new SynthesizeStream(this, this.#opts, options?.connOptions); + } +} + +const timedStringsFromMarks = ( + marks: Speechify.SpeechMarks | undefined, + offsetSeconds: number, +): ReturnType[] => { + if (!marks?.chunks) return []; + const out: ReturnType[] = []; + for (const chunk of marks.chunks) { + if (!chunk.value || chunk.start_time === undefined) continue; + out.push( + createTimedString({ + text: chunk.value, + startTime: chunk.start_time / 1000 + offsetSeconds, + endTime: chunk.end_time !== undefined ? chunk.end_time / 1000 + offsetSeconds : undefined, + }), + ); + } + return out; +}; + +export class ChunkedStream extends tts.ChunkedStream { + label = 'speechify.ChunkedStream'; + #opts: TTSOptions; + #tts: TTS; + #timeoutInSeconds?: number; + + constructor( + ttsInstance: TTS, + text: string, + opts: TTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, ttsInstance, connOptions, abortSignal); + this.#tts = ttsInstance; + this.#opts = opts; + this.#timeoutInSeconds = + connOptions?.timeoutMs !== undefined ? connOptions.timeoutMs / 1000 : undefined; + } + + protected async run() { + const requestId = shortuuid(); + const bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS); + + try { + const { audio, timed } = await this.#tts._synthesize(this.inputText, this.#opts, 0, { + abortSignal: this.abortSignal, + timeoutInSeconds: this.#timeoutInSeconds, + }); + let attached = false; + + const putFrames = (frames: ReturnType) => { + for (const frame of frames) { + if (!attached && timed.length > 0) { + frame.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; + attached = true; + } + this.queue.put({ requestId, frame, final: false, segmentId: requestId }); + } + }; + + putFrames(bstream.write(audio)); + putFrames(bstream.flush()); + this.queue.close(); + } catch (e) { + if (this.abortSignal.aborted) return; + throw toError(e); + } + } +} + +export class SynthesizeStream extends tts.SynthesizeStream { + label = 'speechify.SynthesizeStream'; + #opts: TTSOptions; + #tts: TTS; + + constructor(ttsInstance: TTS, opts: TTSOptions, connOptions?: APIConnectOptions) { + super(ttsInstance, connOptions); + this.#tts = ttsInstance; + this.#opts = opts; + } + + protected async run() { + const sentenceStream = this.#tts.tokenizer.stream(); + // Total audio duration produced so far, used as the timestamp offset for + // the next sentence's word marks. Advances by each sentence's full frame + // duration right after synthesis, so it stays accurate even though the + // last frame of each sentence is deferred by the buffer-one loop below. + let offsetSeconds = 0; + + // Buffer-one deferral across the WHOLE run: the base class only records + // one pending text per reply, so `final: true` must be emitted exactly + // once on the last frame of the entire stream (mirrors elevenlabs/cartesia). + let lastFrame: AudioFrame | undefined; + let lastRequestId: string | undefined; + const sendFrame = (final: boolean) => { + if (!lastFrame || !lastRequestId) return; + this.queue.put({ + requestId: lastRequestId, + frame: lastFrame, + final, + segmentId: lastRequestId, + }); + lastFrame = undefined; + }; + + const forwardInput = async () => { + for await (const input of this.input) { + if (this.abortController.signal.aborted) break; + if (input === SynthesizeStream.FLUSH_SENTINEL) { + sentenceStream.flush(); + } else { + sentenceStream.pushText(input); + } + } + sentenceStream.endInput(); + }; + + const synthesizeSentence = async (text: string) => { + const requestId = shortuuid(); + const bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS); + + this.markStarted(); + const { audio, timed } = await this.#tts._synthesize(text, this.#opts, offsetSeconds, { + abortSignal: this.abortSignal, + timeoutInSeconds: this.connOptions.timeoutMs / 1000, + }); + + const frames = [...bstream.write(audio), ...bstream.flush()]; + if (timed.length > 0 && frames.length > 0) { + frames[0]!.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; + } + offsetSeconds += frames.reduce((sum, f) => sum + f.samplesPerChannel / f.sampleRate, 0); + + for (const frame of frames) { + sendFrame(false); + lastFrame = frame; + lastRequestId = requestId; + } + }; + + const consume = async () => { + for await (const ev of sentenceStream) { + if (this.abortController.signal.aborted) break; + const text = ev.token.trim(); + if (!text) continue; + await synthesizeSentence(text); + } + sendFrame(true); + this.queue.put(SynthesizeStream.END_OF_STREAM); + }; + + try { + await Promise.all([forwardInput(), consume()]); + } catch (e) { + if (this.abortSignal.aborted) return; + throw toError(e); + } + } +} diff --git a/plugins/speechify/tsconfig.json b/plugins/speechify/tsconfig.json new file mode 100644 index 000000000..eed8cb31e --- /dev/null +++ b/plugins/speechify/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-speechify", + "entryPointStrategy": "resolve", + "readme": "none", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/speechify/tsup.config.ts b/plugins/speechify/tsup.config.ts new file mode 100644 index 000000000..8ca20961f --- /dev/null +++ b/plugins/speechify/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f01e7345..9b5078b9f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1327,6 +1327,34 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/speechify: + dependencies: + '@speechify/api': + specifier: ^3.0.1 + version: 3.0.1 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugin-openai': + specifier: workspace:* + version: link:../openai + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.30 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/tavus: dependencies: livekit-server-sdk: @@ -2713,6 +2741,10 @@ packages: '@rushstack/ts-command-line@4.21.0': resolution: {integrity: sha512-z38FLUCn8M9FQf19gJ9eltdwkvc47PxvJmVZS6aKwbBAa3Pis3r3A+ZcBCVPNb9h/Tbga+i0tHdzoSGUoji9GQ==} + '@speechify/api@3.0.1': + resolution: {integrity: sha512-bEqbyvwGW0A/cgq+ZsQJVP3Hm4/0Bm1sniVtPLBh7vDNG2f/YY8y8sweWojoUnhIeIAL6bHBde43+VZLA2MYpA==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -5678,7 +5710,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.4.1 + debug: 4.4.3 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -5758,7 +5790,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.1 + debug: 4.4.3 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -6521,6 +6553,8 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@speechify/api@3.0.1': {} + '@standard-schema/spec@1.1.0': {} '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': @@ -6647,7 +6681,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.9.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.9.3) - debug: 4.4.1 + debug: 4.4.3 eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.9.3) optionalDependencies: @@ -6661,7 +6695,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.1 + debug: 4.4.3 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -7360,7 +7394,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: - debug: 4.4.1 + debug: 4.4.3 enhanced-resolve: 5.16.1 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) @@ -9077,7 +9111,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.1 + debug: 4.4.3 esbuild: 0.25.2 joycon: 3.1.1 picocolors: 1.1.1 @@ -9105,7 +9139,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.1 + debug: 4.4.3 esbuild: 0.25.2 joycon: 3.1.1 picocolors: 1.1.1 diff --git a/turbo.json b/turbo.json index b3a1c44c5..b68291059 100644 --- a/turbo.json +++ b/turbo.json @@ -87,6 +87,7 @@ "RUNWAY_AVATAR_PRESET_ID", "SARVAM_API_KEY", "SONIOX_API_KEY", + "SPEECHIFY_API_KEY", "SIP_PARTICIPANT_IDENTITY", "SIP_PHONE_NUMBER", "LK_OPENAI_DEBUG",