fix(elevenlabs): populate SpeechData.confidence from Scribe logprobs

livekit-plugins/livekit-plugins-elevenlabs/livekit/plugins/elevenlabs/stt.py

@@ -17,6 +17,7 @@
 import asyncio
 import base64
 import json
+import math
 import os
 import weakref
 from dataclasses import dataclass
@@ -48,6 +49,25 @@
 AUTHORIZATION_HEADER = "xi-api-key"
 
 
+def _speech_confidence(words: list[dict[str, Any]] | None) -> float:
+    """Aggregate ElevenLabs per-word logprobs into a [0, 1] transcription confidence.
+
+    Scribe returns a natural-log probability (``logprob``) per token; we average the
+    spoken-word logprobs and exponentiate to a probability (the geometric mean of the
+    token probabilities). Returns ``0.0`` when no per-word logprobs are available.
+    """
+    if not words:
+        return 0.0
+    logprobs = [
+        w["logprob"]
+        for w in words
+        if w.get("type") == "word" and isinstance(w.get("logprob"), (int, float))
+    ]
+    if not logprobs:
+        return 0.0
+    return min(1.0, max(0.0, math.exp(sum(logprobs) / len(logprobs))))
+
+
 class VADOptions(TypedDict, total=False):
     vad_silence_threshold_secs: float | None
     """Silence threshold in seconds for VAD. Default to 1.5"""
@@ -284,6 +304,7 @@ def _transcription_to_speech_event(
                     speaker_id=speaker_id,
                     start_time=start_time,
                     end_time=end_time,
+                    confidence=_speech_confidence(words),
                     words=[
                         TimedString(
                             text=word.get("text", ""),
@@ -591,6 +612,7 @@ def _process_stream_event(self, data: dict) -> None:
             text=text,
             start_time=start_time + self.start_time_offset,
             end_time=end_time + self.start_time_offset,
+            confidence=_speech_confidence(words),
         )
         if words:
             speech_data.words = [

tests/test_plugin_elevenlabs_stt.py

@@ -167,3 +167,43 @@ def test_trace_id_from_headers() -> None:
     assert trace_id_from_headers(CIMultiDict({"X-Trace-Id": "trace-1"})) == "trace-1"
     assert trace_id_from_headers(CIMultiDict()) is None
     assert trace_id_from_headers(None) is None
+
+
+def test_speech_confidence_from_word_logprobs() -> None:
+    # confident transcription: word logprobs near 0 -> confidence near 1 (spacing tokens ignored)
+    words = [
+        {"type": "word", "logprob": -0.01},
+        {"type": "spacing", "logprob": -2.0},
+        {"type": "word", "logprob": -0.05},
+    ]
+    assert 0.9 < elevenlabs_stt._speech_confidence(words) <= 1.0
+
+
+def test_speech_confidence_flags_low_quality_transcription() -> None:
+    # uncertain transcription (very negative logprobs) -> low confidence
+    words = [{"type": "word", "logprob": -2.5}, {"type": "word", "logprob": -3.0}]
+    assert elevenlabs_stt._speech_confidence(words) < 0.2
+
+
+def test_speech_confidence_defaults_to_zero_without_logprobs() -> None:
+    # no words, or words without logprobs (e.g. non-timestamped commit) -> default 0.0
+    assert elevenlabs_stt._speech_confidence(None) == 0.0
+    assert elevenlabs_stt._speech_confidence([{"text": "hi", "start": 0.1, "end": 0.4}]) == 0.0
+
+
+def test_committed_transcript_sets_confidence() -> None:
+    stream = _new_stream(server_vad={"vad_silence_threshold_secs": 0.5})
+
+    stream._process_stream_event(
+        {
+            "message_type": "committed_transcript",
+            "text": "hello",
+            "words": [
+                {"text": "hello", "start": 0.1, "end": 0.4, "type": "word", "logprob": -0.02}
+            ],
+        }
+    )
+
+    final = stream._event_ch.events[1]
+    assert final.type == stt.SpeechEventType.FINAL_TRANSCRIPT
+    assert final.alternatives[0].confidence > 0.9