Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ POLLY_AWS_ACCESS_KEY=your-aws-secret

# ElevenLabs
ELEVENLABS_API_KEY=your-elevenlabs-key
# Optional model override. Defaults to eleven_multilingual_v2 (<break> markup).
# Use eleven_v3 for audio tags ([whispers], [pause], "/IPA/") — v3 parses no SSML.
# ELEVENLABS_MODEL_ID=eleven_v3
# Optional model override. Defaults to eleven_v3 (audio tags: [whispers],
# [pause], "/IPA/"). Use eleven_multilingual_v2 / eleven_flash_v2_5 for
# <break> markup or longer character limits.
# ELEVENLABS_MODEL_ID=eleven_multilingual_v2

# Wit.ai
WITAI_TOKEN=your-witai-token
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Active development. Engine constructors, the C ABI, and the offline test suite r
- **Voice List**: Engines with "API" can enumerate voices from the provider's API.
- **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable).
- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, ElevenLabs gets model-matched prompt markup (see below), others get Alexa SSML.
- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `<break time>` prompt markup (≤3s, clamped); `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2).
- **ElevenLabs markup**: ElevenLabs parses no SSML documents. The default model is `eleven_v3`, so SpeechMarkdown renders as audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`); set the `modelId` credential to a pre-v3 model (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) to get `<break time>` prompt markup (≤3s, clamped) instead. The dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2).

## Rust API

Expand Down
32 changes: 22 additions & 10 deletions src/cloud_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -807,13 +807,15 @@ fn build_config(id: &str, creds: &HashMap<String, String>) -> Option<CloudConfig
.unwrap_or_else(|| "21m00Tcm4TlvDq8ikWAM".into());
// Model selection matters for the SpeechMarkdown dialect:
// eleven_v3* parses no SSML (audio tags only), pre-v3 models
// understand <break> but read audio tags aloud. Unrecognized
// model IDs surface as API errors rather than being masked.
// understand <break> but read audio tags aloud. v3 is the
// default — the most capable model, and the dialects keep the
// markup correct for it. Unrecognized model IDs surface as
// API errors rather than being masked.
let model = creds
.get("modelId")
.filter(|m| !m.is_empty())
.cloned()
.unwrap_or_else(|| "eleven_multilingual_v2".into());
.unwrap_or_else(|| "eleven_v3".into());
Some(CloudConfig {
synth_url: format!("https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"),
auth_header: "xi-api-key".into(),
Expand Down Expand Up @@ -5068,16 +5070,26 @@ mod tests {

#[test]
fn test_elevenlabs_model_id_from_creds() {
// Default model stays multilingual_v2 (pre-v3 dialect).
// Default model is eleven_v3, and that default must select the
// v3 audio-tag dialect — the invariant that makes SpeechMarkdown
// correct out of the box.
let cfg = build_config("elevenlabs", &engine_creds("elevenlabs")).unwrap();
assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2"));
assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3"));
assert_eq!(
elevenlabs_smd_platform("elevenlabs", cfg.model_default.as_deref()),
"elevenlabs-v3"
);

// modelId credential overrides it (v3 needs this: audio tags
// require eleven_v3, which parses no SSML at all).
// modelId credential overrides it (e.g. a pre-v3 model when
// <break> markup or long-form character limits are wanted).
let mut c = engine_creds("elevenlabs");
c.insert("modelId".into(), "eleven_v3".into());
c.insert("modelId".into(), "eleven_multilingual_v2".into());
let cfg = build_config("elevenlabs", &c).unwrap();
assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3"));
assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2"));
assert_eq!(
elevenlabs_smd_platform("elevenlabs", cfg.model_default.as_deref()),
"elevenlabs"
);

let mut c = engine_creds("elevenlabs");
c.insert("modelId".into(), "eleven_flash_v2_5".into());
Expand All @@ -5088,7 +5100,7 @@ mod tests {
let mut c = engine_creds("elevenlabs");
c.insert("modelId".into(), String::new());
let cfg = build_config("elevenlabs", &c).unwrap();
assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2"));
assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3"));
}

#[test]
Expand Down
10 changes: 4 additions & 6 deletions tests/elevenlabs_timestamps_fallback.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
//! Offline test for the ElevenLabs `/with-timestamps` degrade path: when
//! Offline test for the `ElevenLabs` `/with-timestamps` degrade path: when
//! the endpoint variant is rejected (a model that doesn't support it),
//! speak() must retry the plain synthesis endpoint and deliver estimated
//! `speak()` must retry the plain synthesis endpoint and deliver estimated
//! boundaries instead of failing the call.
//!
//! Uses a `std::net::TcpListener` mock so no network access or API key is
//! needed. The MP3 fixture is 0.4s of silence, regenerated with:
//! ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 0.4 -q:a 9 silence.mp3
//! `ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 0.4 -q:a 9 silence.mp3`

use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};

use rust_tts_wrapper::engine::TtsEngine;
use rust_tts_wrapper::factory::create_engine;

const SILENCE_MP3: &[u8] = include_bytes!("fixtures/silence.mp3");
Expand Down Expand Up @@ -40,8 +39,7 @@ fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]
.lines()
.find(|l| l.to_ascii_lowercase().starts_with("content-length"))
.and_then(|l| l.split(':').nth(1))
.map(|v| v.trim().parse().expect("content-length"))
.unwrap_or(0);
.map_or(0, |v| v.trim().parse().expect("content-length"));
let mut received = read - header_end - 4;
while received < content_length {
let n = stream.read(&mut buf[read..]).expect("read body");
Expand Down
6 changes: 5 additions & 1 deletion tests/live_cloud.rs.template
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,11 @@ fn elevenlabs_real_alignment_groups_into_words() {
eprintln!("skipping: ELEVENLABS_API_KEY not set");
return;
}
let creds = format!(r#"{{"apiKey":"{key}"}}"#);
// Pinned to a pre-v3 model: this test verifies the /with-timestamps
// JSON alignment path specifically. (The default model is eleven_v3;
// if it rejects the variant, speak() degrades to estimated
// boundaries — covered separately.)
let creds = format!(r#"{{"apiKey":"{key}","modelId":"eleven_multilingual_v2"}}"#);
let engine = create_engine("elevenlabs", &creds).expect("elevenlabs");
let sink = Arc::new(Mutex::new(BoundarySink::new()));
let s = sink.clone();
Expand Down
Loading