ElevenLabs: model-aware SpeechMarkdown dialects, modelId, voice_settings.speed

.env.example

@@ -22,6 +22,9 @@ POLLY_AWS_ACCESS_KEY=your-aws-secret
 
 # ElevenLabs
 ELEVENLABS_API_KEY=your-elevenlabs-key
+# Optional model override. Defaults to eleven_multilingual_v2 (<break> markup).
+# Use eleven_v3 for audio tags ([whispers], [pause], "/IPA/") — v3 parses no SSML.
+# ELEVENLABS_MODEL_ID=eleven_v3
 
 # Wit.ai
 WITAI_TOKEN=your-witai-token

Cargo.toml

@@ -97,7 +97,7 @@ sherpa-onnx-models = { version = "0.1", optional = true }
 # Pin sys to match the main crate (cargo update can drift them apart).
 sherpa-onnx-sys = { version = "=1.13.5", optional = true }
 base64 = { version = "0.22", optional = true }
-speechmarkdown-rust = { version = "0.4.13", optional = true }
+speechmarkdown-rust = { version = "0.4.14", optional = true }
 anyhow = "1"
 tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"], optional = true }
 uuid = { version = "1.23.2", features = ["v4"], optional = true }

README.md

@@ -49,7 +49,8 @@ cargo test --all-features
 Active development. Engine constructors, the C ABI, and the offline test suite run in CI on Linux, macOS, and Windows. Live cloud API calls are not exercised in CI — see `tests/live_cloud.rs.template` (copy to `tests/live_cloud.rs`, gitignored) and `.env.example` for running them locally with your own credentials. Live SherpaOnnx synthesis IS exercised in CI by the `sherpaonnx-live.yml` workflow (downloads small VITS/Matcha/Kokoro models and runs `tests/sherpaonnx_live.rs`), triggered on PRs touching `src/sherpaonnx_engine.rs` and available as a manual `workflow_dispatch`.
 - **Voice List**: Engines with "API" can enumerate voices from the provider's API.
 - **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable).
-- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, others get Alexa SSML.
+- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, ElevenLabs gets model-matched prompt markup (see below), others get Alexa SSML.
+- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `<break time>` prompt markup (≤3s, clamped); `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) — the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2).
 
 ## Rust API
 

src/cloud_engine.rs

@@ -805,11 +805,20 @@ fn build_config(id: &str, creds: &HashMap<String, String>) -> Option<CloudConfig
                 .get("voiceId")
                 .cloned()
                 .unwrap_or_else(|| "21m00Tcm4TlvDq8ikWAM".into());
+            // Model selection matters for the SpeechMarkdown dialect:
+            // eleven_v3* parses no SSML (audio tags only), pre-v3 models
+            // understand <break> but read audio tags aloud. Unrecognized
+            // model IDs surface as API errors rather than being masked.
+            let model = creds
+                .get("modelId")
+                .filter(|m| !m.is_empty())
+                .cloned()
+                .unwrap_or_else(|| "eleven_multilingual_v2".into());
             Some(CloudConfig {
                 synth_url: format!("https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"),
                 auth_header: "xi-api-key".into(),
                 model_param: Some("model_id".into()),
-                model_default: Some("eleven_multilingual_v2".into()),
+                model_default: Some(model),
                 text_field: "text".into(),
                 voices_url: Some("https://api.elevenlabs.io/v1/voices".into()),
                 provider_id: "elevenlabs".into(),
@@ -1406,6 +1415,20 @@ fn parse_google_timepoints(
     boundaries
 }
 
+/// Pick the SpeechMarkdown platform selector for a provider/model pair.
+///
+/// ElevenLabs markup is model-dependent: `eleven_v3*` parses no SSML and
+/// needs the audio-tag dialect; every other model (and every other
+/// provider) maps to `provider` unchanged (the caller's provider id is
+/// itself the selector for azure/google/the Alexa fallback).
+fn elevenlabs_smd_platform<'a>(provider: &'a str, model: Option<&str>) -> &'a str {
+    if provider == "elevenlabs" && model.is_some_and(|m| m.starts_with("eleven_v3")) {
+        "elevenlabs-v3"
+    } else {
+        provider
+    }
+}
+
 /// Parse ElevenLabs alignment payload into `(word, start_sec, end_sec)` tuples.
 ///
 /// ElevenLabs returns per-character timing in `alignment`:
@@ -1833,7 +1856,19 @@ impl TtsEngine for CloudEngine {
         mut on_boundary: Option<crate::engine::OnBoundaryCallback>,
         _on_mark: Option<crate::engine::OnMarkCallback>,
     ) -> TtsResult<()> {
-        let (original_text, is_ssml) = preprocess_speech_markdown(text, &self.config.provider_id);
+        // SpeechMarkdown platform selector: ElevenLabs needs the dialect
+        // that matches the requested model (v3 audio tags vs pre-v3
+        // <break> markup) — the other dialect gets read aloud or ignored.
+        let smd_platform = elevenlabs_smd_platform(
+            &self.config.provider_id,
+            self.config.model_default.as_deref(),
+        );
+        // Caller-facing text for word-boundary offset mapping: when
+        // SpeechMarkdown was reformatted (rather than passed through or
+        // converted to SSML), injected ElevenLabs tags shift offsets, so
+        // search the user's original input — the spoken words live there.
+        let user_text = text;
+        let (original_text, is_ssml) = preprocess_speech_markdown(text, smd_platform);
 
         // When the caller passed W3C SSML (via tts_speak_ssml), adapt per engine:
         //  - Azure/Edge: pass through (their WS/REST paths handle SSML natively)
@@ -1883,6 +1918,12 @@ impl TtsEngine for CloudEngine {
             text = original_text;
         }
 
+        // Boundary word search target (see `user_text` above): plain or
+        // dialect-reformatted input maps against what the caller passed;
+        // SSML paths keep searching the processed string (previously
+        // existing behavior).
+        let boundary_search_text: &str = if is_ssml { text.as_str() } else { user_text };
+
         // WebSocket approach: Azure when word boundaries are requested, or
         // Edge always (Edge is WS-only — it has no REST synth endpoint).
         // Edge reuses the identical Azure "Turn" protocol; only the URL/auth
@@ -2347,6 +2388,24 @@ impl TtsEngine for CloudEngine {
                     );
                 }
             }
+            // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal)
+            // onto the deterministic voice_settings.speed API parameter
+            // (valid range 0.7–1.2; clamped). Only sent for explicit
+            // non-default rates. pitch/volume have no API equivalent
+            // (v3 models: use audio tags). Inserted before extra_body so
+            // a config-supplied voice_settings object (stability,
+            // similarity, …) takes precedence over the derived one.
+            if self.config.provider_id == "elevenlabs"
+                && rate > 0.0
+                && (rate - 1.0).abs() > f32::EPSILON
+                && !self.config.extra_body.contains_key("voice_settings")
+            {
+                let speed = rate.clamp(0.7, 1.2);
+                body.insert(
+                    "voice_settings".to_string(),
+                    serde_json::json!({ "speed": speed }),
+                );
+            }
             for (k, v) in &self.config.extra_body {
                 body.insert(k.clone(), v.clone());
             }
@@ -2394,7 +2453,7 @@ impl TtsEngine for CloudEngine {
                     let mut search_from = 0usize;
                     for (word, start, end) in parse_elevenlabs_alignment(alignment) {
                         #[allow(clippy::cast_possible_truncation)]
-                        let char_offset = text[search_from..]
+                        let char_offset = boundary_search_text[search_from..]
                             .find(&word)
                             .map_or(-1, |pos| (search_from + pos) as i32);
 
@@ -4882,7 +4941,7 @@ mod tests {
         for input in &probe_inputs {
             let (azure_ssml, azure_ok) = preprocess_speech_markdown(input, "azure");
             let (google_ssml, google_ok) = preprocess_speech_markdown(input, "google");
-            let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "elevenlabs");
+            let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "openai");
 
             assert!(azure_ok, "azure failed to parse: {input:?}");
             assert!(google_ok, "google failed to parse: {input:?}");
@@ -4904,23 +4963,38 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_speechmarkdown_elevenlabs_dialects() {
+        use crate::engine::preprocess_speech_markdown;
+        // Pre-v3 dialect: <break> prompt markup, not SSML (no <speak>
+        // wrapper, is_ssml false so speak() sends it verbatim).
+        let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs");
+        assert!(!is_ssml, "elevenlabs dialect must not be flagged as SSML");
+        assert_eq!(out, "Hello <break time=\"2s\"/> world");
+
+        // v3 dialect: audio tags; no XML the model would read aloud.
+        let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs-v3");
+        assert!(!is_ssml);
+        assert_eq!(out, "Hello [long pause] world");
+
+        let (out, _) = preprocess_speech_markdown("(secret)[whisper]", "elevenlabs-v3");
+        assert_eq!(out, "[whispers] secret");
+
+        let (out, _) = preprocess_speech_markdown("(speech)/spitʃ/", "elevenlabs-v3");
+        assert_eq!(out, "\"/spitʃ/\"");
+    }
+
     #[test]
     fn test_speechmarkdown_other_providers_detect_input() {
         use crate::engine::preprocess_speech_markdown;
-        // ElevenLabs, OpenAI, Cartesia, Murf, etc. all go through the
-        // Alexa fallback. They don't actually consume SSML — the result is
-        // discarded by the JSON-body branch in speak() — but detection
-        // must still flag the input as SpeechMarkdown so callers querying
-        // `is_ssml` get a truthful answer.
-        for provider in [
-            "openai",
-            "elevenlabs",
-            "cartesia",
-            "murf",
-            "deepgram",
-            "witai",
-            "xai",
-        ] {
+        // OpenAI, Cartesia, Murf, etc. all go through the Alexa fallback.
+        // They don't actually consume SSML — the result is discarded by the
+        // JSON-body branch in speak() — but detection must still flag the
+        // input as SpeechMarkdown so callers querying `is_ssml` get a
+        // truthful answer. ElevenLabs is NOT in this list: it gets its own
+        // dialects, which are prompt markup, not SSML (see
+        // test_speechmarkdown_elevenlabs_dialects).
+        for provider in ["openai", "cartesia", "murf", "deepgram", "witai", "xai"] {
             let (_ssml, is_ssml) =
                 preprocess_speech_markdown("Hello (world)[emphasis:\"strong\"]", provider);
             assert!(
@@ -4952,6 +5026,56 @@ mod tests {
         assert!(url.ends_with("/text-to-speech/21m00Tcm4TlvDq8ikWAM/with-timestamps"));
     }
 
+    #[test]
+    fn test_elevenlabs_model_id_from_creds() {
+        // Default model stays multilingual_v2 (pre-v3 dialect).
+        let cfg = build_config("elevenlabs", &engine_creds("elevenlabs")).unwrap();
+        assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2"));
+
+        // modelId credential overrides it (v3 needs this: audio tags
+        // require eleven_v3, which parses no SSML at all).
+        let mut c = engine_creds("elevenlabs");
+        c.insert("modelId".into(), "eleven_v3".into());
+        let cfg = build_config("elevenlabs", &c).unwrap();
+        assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3"));
+
+        let mut c = engine_creds("elevenlabs");
+        c.insert("modelId".into(), "eleven_flash_v2_5".into());
+        let cfg = build_config("elevenlabs", &c).unwrap();
+        assert_eq!(cfg.model_default.as_deref(), Some("eleven_flash_v2_5"));
+
+        // Empty modelId falls back to the default.
+        let mut c = engine_creds("elevenlabs");
+        c.insert("modelId".into(), String::new());
+        let cfg = build_config("elevenlabs", &c).unwrap();
+        assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2"));
+    }
+
+    #[test]
+    fn test_elevenlabs_dialect_follows_model() {
+        // The production predicate used by speak(): eleven_v3* → audio-tag
+        // dialect, anything else → pre-v3 <break> markup. Asserted directly
+        // against the real helper so a flip fails here, not just live.
+        assert_eq!(
+            elevenlabs_smd_platform("elevenlabs", Some("eleven_v3")),
+            "elevenlabs-v3"
+        );
+        assert_eq!(
+            elevenlabs_smd_platform("elevenlabs", Some("eleven_v3_conversational")),
+            "elevenlabs-v3"
+        );
+        assert_eq!(
+            elevenlabs_smd_platform("elevenlabs", Some("eleven_multilingual_v2")),
+            "elevenlabs"
+        );
+        assert_eq!(
+            elevenlabs_smd_platform("elevenlabs", Some("eleven_flash_v2")),
+            "elevenlabs"
+        );
+        assert_eq!(elevenlabs_smd_platform("azure", Some("eleven_v3")), "azure");
+        assert_eq!(elevenlabs_smd_platform("elevenlabs", None), "elevenlabs");
+    }
+
     // ===== Auth-header composition per provider =====
     //
     // speak() builds the final header value as `format!("{}{}", prefix, api_key)`.

src/engine.rs

@@ -37,6 +37,11 @@ pub type OnErrorCallback<'a> = &'a mut dyn FnMut(&str);
 /// `platform` picks the SSML flavour:
 ///   - `"azure"` → MicrosoftAzure
 ///   - `"google"` → GoogleAssistant
+///   - `"elevenlabs"` → ElevenLabs pre-v3 prompt markup (`<break>` tags,
+///     no SSML document). Not SSML: `is_ssml` is returned false so engines
+///     pass the markup through instead of stripping it.
+///   - `"elevenlabs-v3"` → Eleven v3 audio-tag dialect (`[whispers]`,
+///     `[pause]`, native slash IPA). Eleven v3 parses no SSML at all.
 ///   - `"sapi"` / `"avsynth"` / anything else → AmazonAlexa (the closest
 ///     generic SSML baseline; SAPI's own parser accepts the subset that
 ///     speechmarkdown-rust emits for Alexa)
@@ -64,15 +69,24 @@ pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool)
         // few elements, which the engine boundary strips.
         "azure" | "edge" => Platform::MicrosoftAzure,
         "google" => Platform::GoogleAssistant,
+        // ElevenLabs prompt markup, not SSML: the dialects must reach the
+        // API verbatim (stripping would drop every break/tag, and v3
+        // models read stray XML aloud).
+        "elevenlabs" => Platform::ElevenLabs,
+        "elevenlabs-v3" => Platform::ElevenLabsV3,
         // floravox parses the generic (Alexa-baseline) SSML dialect
         // natively; the floravox engine normalizes vendor-specific
         // elements (e.g. whisper's <amazon:effect>) on its side — as
         // does everything else via the default.
         _ => Platform::AmazonAlexa,
     };
 
+    let is_elevenlabs_dialect = matches!(platform, Platform::ElevenLabs | Platform::ElevenLabsV3);
+
     match SpeechMarkdownParser::to_ssml(text, platform) {
-        Ok(ssml) => (ssml, true),
+        // The ElevenLabs dialects are prompt text, not SSML: flag them so
+        // engines send the string as-is rather than treating it as SSML.
+        Ok(ssml) => (ssml, !is_elevenlabs_dialect),
         Err(_) => (text.to_string(), false),
     }
 }