[pull] main from livekit:main

CHANGELOG.md

@@ -1,5 +1,11 @@
 # github.com/livekit/protocol
 
+## 1.50.3
+
+### Patch Changes
+
+- agent simulation: homogenizes Turn/Job/Run LLM/TTS/STT/Conversation metrics shapes - [#1673](https://github.com/livekit/protocol/pull/1673) ([@Bobronium](https://github.com/Bobronium))
+
 ## 1.50.2
 
 ## 1.50.1

package.json

@@ -1,7 +1,7 @@
 {
   "name": "github.com/livekit/protocol",
   "private": true,
-  "version": "1.50.2",
+  "version": "1.50.3",
   "scripts": {
     "changeset": "changeset",
     "ci:publish": "pnpm --filter @livekit/protocol run build && changeset publish"

packages/javascript/CHANGELOG.md

@@ -1,5 +1,7 @@
 # @livekit/protocol
 
+## 1.50.3
+
 ## 1.50.2
 
 ### Patch Changes

packages/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/protocol",
-  "version": "1.50.2",
+  "version": "1.50.3",
   "description": "",
   "type": "module",
   "require": "dist/index.cjs",

protobufs/livekit_agent_simulation.proto

@@ -86,10 +86,10 @@ message SimulationRun {
     // members that ran (a missing member renormalizes the rest):
     //   accuracy   = task_completion x3 + stt.keyterm_recall + (1 - stt.wer) + tts.enunciation_score
     //   experience = conversation.turn_taking_score x2 + tts.naturalness_score + llm.conciseness_score
-    optional float accuracy_score = 1;
-    optional float experience_score = 2;
+    optional float accuracy_score = 1;    // absent when the simulator spoiled the call
+    optional float experience_score = 2;  // absent when the simulator spoiled the call
     // The accuracy anchor: the scenario verdict as 1/0, or the fraction of the declared target state the call reached.
-    optional float task_completion = 3;
+    optional float task_completion = 3;   // absent when the simulator spoiled the call
 
     STT stt = 4;
     LLM llm = 5;
@@ -103,65 +103,64 @@ message SimulationRun {
     string judge_model = 10;            // text judge for judged scores; "" if none ran
     string audio_judge_model = 11;      // audio (LALM) judge; "" if none ran
     bool has_remote_session = 12;       // false = waveform-only capture (e.g. SIP)
-    google.protobuf.Timestamp t0 = 13;  // call start (first speech edge); turn times relative to it
+    google.protobuf.Timestamp t0 = 13;  // audio only; call start (first speech edge); turn times relative to it
 
-    // The agent's perception of the caller.
+    // The agent's perception of the caller. Audio only.
     message STT {
-      optional float wer = 1;                       // word error rate, pooled: word_errors / words
-      optional uint32 words = 2;                    // pooling stats: run WER = sum errors / sum words
-      optional uint32 word_errors = 3;
-      optional float cer = 4;                       // character error rate, pooled
-      optional uint32 chars = 9;
-      optional uint32 char_errors = 10;
-      optional float keyterm_recall = 5;            // recall of uttered key entities (names, IDs, amounts)
-      optional uint32 keyterms_uttered = 6;
-      optional uint32 keyterms_recognized = 7;
-      optional uint32 transcription_latency_ms = 8; // agent-reported STT/endpointing delay, mean
+      optional float wer = 1;                       // needs the RemoteSession; word error rate, pooled: word_errors / words
+      optional uint32 words = 2;                    // needs the RemoteSession; pooling stats: run WER = sum errors / sum words
+      optional uint32 word_errors = 3;              // needs the RemoteSession
+      optional float cer = 4;                       // needs the RemoteSession; character error rate, pooled
+      optional uint32 chars = 9;                    // needs the RemoteSession
+      optional uint32 char_errors = 10;             // needs the RemoteSession
+      optional float keyterm_recall = 5;            // needs the RemoteSession; recall of uttered key entities (names, IDs, amounts)
+      optional uint32 keyterms_uttered = 6;         // needs the RemoteSession
+      optional uint32 keyterms_recognized = 7;      // needs the RemoteSession
+      optional uint32 transcription_latency_ms = 8; // needs the RemoteSession; agent-reported STT/endpointing delay, mean
     }
 
     message LLM {
-      optional uint32 ttft_ms = 1;            // time to first token, mean (per-turn in turns)
-      optional uint32 ttfs_ms = 2;            // time to first sentence — the smallest speakable unit
-      optional float tokens_per_second = 3;   // decode rate, mean
-      optional float conciseness_score = 4;   // judged 0-1: brief enough for voice
+      optional uint32 ttft_ms = 1;            // needs the RemoteSession; time to first token, mean (per-turn in turns)
+      optional uint32 ttfs_ms = 2;            // needs the RemoteSession; time to first sentence (the smallest speakable unit)
+      optional float tokens_per_second = 3;   // needs the RemoteSession; decode rate, mean (per-turn in turns.llm)
+      optional float conciseness_score = 4;   // audio only; needs the audio judge; judged 0-1: brief enough for voice
     }
 
-    // The agent's voice.
+    // The agent's voice. audio only.
     message TTS {
-      // Time to first AUDIO — the user-relevant signal. Some providers stream
-      // bytes before actual speech, so byte-level TTFB alone over-promises;
-      // it is kept below for disambiguation.
-      optional uint32 ttfa_ms = 1;
-      optional uint32 ttfb_ms = 2;             // provider byte-level TTFB, agent-reported
-      optional float wer = 3;                  // word error rate, pooled: word_errors / words
-      optional uint32 words = 7;
-      optional uint32 word_errors = 8;
-      optional float cer = 9;                  // character error rate, pooled
-      optional uint32 chars = 10;
-      optional uint32 char_errors = 11;
+      optional uint32 ttfa_ms = 1;             // needs the RemoteSession
+      optional uint32 ttfb_ms = 2;             // needs the RemoteSession; provider byte-level TTFB
+      optional float wer = 3;                  // needs the RemoteSession; word error rate, pooled: word_errors / words
+      optional uint32 words = 7;               // needs the RemoteSession
+      optional uint32 word_errors = 8;         // needs the RemoteSession
+      optional float cer = 9;                  // needs the RemoteSession; character error rate, pooled
+      optional uint32 chars = 10;              // needs the RemoteSession
+      optional uint32 char_errors = 11;        // needs the RemoteSession
       optional float speech_rate_wpm = 4;      // speaking rate; conversational English ~110-150
-      optional float naturalness_score = 5;    // judged 0-1: prosody / expressiveness
-      optional float enunciation_score = 6;    // judged 0-1: key entities audibly intact
+      optional uint32 heard_words = 12;        // pooling stats for speech_rate_wpm: raw heard words (not WER tokens)
+      optional uint32 speech_ms = 13;          // agent voiced time, sum of raw VAD segments
+      optional float naturalness_score = 5;    // needs the audio judge; judged 0-1: prosody / expressiveness
+      optional float enunciation_score = 6;    // needs the audio judge; judged 0-1: key entities audibly intact
     }
 
-    // End-to-end floor dynamics, measured off the audio timeline — the ground
-    // truth of what the caller hears, independent of agent self-reports.
+    // Audio only.
     message Conversation {
-      optional float turn_taking_score = 1;         // 0-1: latency curve + cut-in/barge-in/missed-turn penalties
-      optional int32 response_latency_p50_ms = 2;   // floor-transfer offset; a gap is > 0, an overlap < 0
+      optional float turn_taking_score = 1;              // 0-1: latency curve + cut-in/barge-in/missed-turn penalties
+      optional int32 response_latency_p50_ms = 2;        // floor-transfer offset; a gap is > 0, an overlap < 0
       optional int32 response_latency_p95_ms = 3;
       optional int32 response_latency_p99_ms = 4;
-      optional uint32 agent_yield_latency_ms = 5;   // how long the agent talks after a barge-in, mean
-      optional uint32 eot_misprediction_count = 6;  // agent started before the caller's turn ended
-      optional float overlap_ratio = 7;             // overlapping speech / total speech
-      optional uint32 overlap_speech_ms = 8;        // pooling stats for overlap_ratio
+      optional int32 response_latency_ms = 16;           // one turn's floor-transfer offset, cut-in if negative; job/run serve the percentiles
+      optional uint32 agent_yield_latency_ms = 5;        // agent audio continuing past a barge-in — per turn: on the barging persona row (presence = barge-in); job: mean
+      optional uint32 eot_misprediction_count = 6;       // agent started before the caller's turn ended; 0/1 per turn
+      optional float overlap_ratio = 7;                  // overlapping speech / total speech
+      optional uint32 overlap_speech_ms = 8;             // pooling stats for overlap_ratio
       optional uint32 total_speech_ms = 9;
-      optional uint32 silence_total_ms = 10;        // dead air within the conversation
-      optional uint32 awkward_silence_count = 11;   // gaps past the natural-pause threshold
-      optional uint32 unanswered_persona_turns = 12;             // the simulated party spoke, the agent never responded
-      optional uint32 false_interruption_count = 13;             // agent paused for a non-interruption
-      optional uint32 false_interruption_unrecovered_count = 14; // of those, never resumed
-      optional uint32 agent_reported_e2e_latency_ms = 15;        // the agent's own claim, mean
+      optional uint32 silence_total_ms = 10;             // dead air within the conversation
+      optional uint32 awkward_silence_count = 11;        // gaps past the natural-pause threshold; 0/1 per turn
+      optional uint32 unanswered_persona_turns = 12;             // the simulated party spoke, the agent never responded; 0/1 per turn
+      optional uint32 false_interruption_count = 13;             // needs the RemoteSession; agent paused for a non-interruption
+      optional uint32 false_interruption_unrecovered_count = 14; // needs the RemoteSession; of those, never resumed
+      optional uint32 agent_reported_e2e_latency_ms = 15;        // needs the RemoteSession; the agent's own claim, mean
     }
 
     // Was the call spoiled by the simulator, not the agent? Diagnostic only —
@@ -172,24 +171,18 @@ message SimulationRun {
       optional bool late_termination = 2;   // simulator dragged on after the goal was met
     }
 
+    // Rows exist for text and audio;
     message Turn {
-      uint32 index = 1;  // 1-based, conversation order
-      livekit.agent.ChatRole role = 2; // ASSISTANT — the agent under test, USER — simulator persona.
-      optional uint32 start_ms = 3;  // relative to t0; unset if the turn could not be aligned to the audio
-      optional uint32 end_ms = 4;
-      // Agent-turn measurements; a negative response latency is a cut-in (the
-      // agent started before the caller's turn ended).
-      optional int32 response_latency_ms = 5;
-      optional uint32 transcription_delay_ms = 6;
-      optional uint32 llm_ttft_ms = 7;
-      optional uint32 llm_ttfs_ms = 8;
-      optional uint32 tts_ttfa_ms = 9;
-      optional uint32 tts_ttfb_ms = 10;
-      optional uint32 agent_reported_e2e_latency_ms = 11;
-      optional float conciseness_score = 12;
-      optional float naturalness_score = 13;
-      optional float enunciation_score = 14;  // judged 0-1: this turn's key entities audibly intact
-      repeated string flags = 15;  // judge failure tags ("verbosity_or_filler", ...)
+      uint32 index = 1;                 // 1-based, conversation order
+      livekit.agent.ChatRole role = 2;  // ASSISTANT — the agent under test, USER — simulator persona.
+      optional uint32 start_ms = 3;     // audio only; relative to t0; unset if the turn could not be aligned to the audio
+      optional uint32 end_ms = 4;       // audio only
+
+      // Job groups hold means/pools of these; a field with no per-turn measurement stays unset here.
+      STT stt = 5;                      // transcription_latency_ms
+      LLM llm = 6;                      // ttft_ms, ttfs_ms, tokens_per_second, conciseness_score
+      TTS tts = 7;                      // ttfa_ms, ttfb_ms, naturalness_score, enunciation_score
+      Conversation conversation = 8;    // response_latency_ms, agent_yield_latency_ms, agent_reported_e2e_latency_ms, 0/1 counts
     }
   }
 

sip/token.go

@@ -31,13 +31,15 @@ type SIPTokenParams struct {
 	ParticipantAttributes map[string]string
 	RoomPreset            string
 	RoomConfig            *livekit.RoomConfiguration
+	RoomAdmin             bool
 }
 
 func BuildSIPToken(params SIPTokenParams) (string, error) {
 	t := true
 	at := auth.NewAccessToken(params.APIKey, params.APISecret).
 		SetVideoGrant(&auth.VideoGrant{
 			RoomJoin:             true,
+			RoomAdmin:            params.RoomAdmin,
 			Room:                 params.RoomName,
 			CanSubscribe:         &t,
 			CanPublish:           &t,