diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ef96a78a..1177ee7f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # github.com/livekit/protocol +## 1.50.3 + +### Patch Changes + +- agent simulation: homogenizes Turn/Job/Run LLM/TTS/STT/Conversation metrics shapes - [#1673](https://github.com/livekit/protocol/pull/1673) ([@Bobronium](https://github.com/Bobronium)) + ## 1.50.2 ## 1.50.1 diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index a036492f6..50a2d5807 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -895,10 +895,10 @@ type SimulationRun_JobMetrics struct { // // accuracy = task_completion x3 + stt.keyterm_recall + (1 - stt.wer) + tts.enunciation_score // experience = conversation.turn_taking_score x2 + tts.naturalness_score + llm.conciseness_score - AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` - ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` + AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` // absent when the simulator spoiled the call + ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` // absent when the simulator spoiled the call // The accuracy anchor: the scenario verdict as 1/0, or the fraction of the declared target state the call reached. - TaskCompletion *float32 `protobuf:"fixed32,3,opt,name=task_completion,json=taskCompletion,proto3,oneof" json:"task_completion,omitempty"` + TaskCompletion *float32 `protobuf:"fixed32,3,opt,name=task_completion,json=taskCompletion,proto3,oneof" json:"task_completion,omitempty"` // absent when the simulator spoiled the call Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,4,opt,name=stt,proto3" json:"stt,omitempty"` Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,5,opt,name=llm,proto3" json:"llm,omitempty"` Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,6,opt,name=tts,proto3" json:"tts,omitempty"` @@ -909,7 +909,7 @@ type SimulationRun_JobMetrics struct { JudgeModel string `protobuf:"bytes,10,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // text judge for judged scores; "" if none ran AudioJudgeModel string `protobuf:"bytes,11,opt,name=audio_judge_model,json=audioJudgeModel,proto3" json:"audio_judge_model,omitempty"` // audio (LALM) judge; "" if none ran HasRemoteSession bool `protobuf:"varint,12,opt,name=has_remote_session,json=hasRemoteSession,proto3" json:"has_remote_session,omitempty"` // false = waveform-only capture (e.g. SIP) - T0 *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=t0,proto3" json:"t0,omitempty"` // call start (first speech edge); turn times relative to it + T0 *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=t0,proto3" json:"t0,omitempty"` // audio only; call start (first speech edge); turn times relative to it unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1436,19 +1436,19 @@ func (x *SimulationRun_Job_Usage) GetAudioTurnsCount() int32 { return 0 } -// The agent's perception of the caller. +// The agent's perception of the caller. Audio only. type SimulationRun_JobMetrics_STT struct { state protoimpl.MessageState `protogen:"open.v1"` - Wer *float32 `protobuf:"fixed32,1,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // word error rate, pooled: word_errors / words - Words *uint32 `protobuf:"varint,2,opt,name=words,proto3,oneof" json:"words,omitempty"` // pooling stats: run WER = sum errors / sum words - WordErrors *uint32 `protobuf:"varint,3,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` - Cer *float32 `protobuf:"fixed32,4,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // character error rate, pooled - Chars *uint32 `protobuf:"varint,9,opt,name=chars,proto3,oneof" json:"chars,omitempty"` - CharErrors *uint32 `protobuf:"varint,10,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` - KeytermRecall *float32 `protobuf:"fixed32,5,opt,name=keyterm_recall,json=keytermRecall,proto3,oneof" json:"keyterm_recall,omitempty"` // recall of uttered key entities (names, IDs, amounts) - KeytermsUttered *uint32 `protobuf:"varint,6,opt,name=keyterms_uttered,json=keytermsUttered,proto3,oneof" json:"keyterms_uttered,omitempty"` - KeytermsRecognized *uint32 `protobuf:"varint,7,opt,name=keyterms_recognized,json=keytermsRecognized,proto3,oneof" json:"keyterms_recognized,omitempty"` - TranscriptionLatencyMs *uint32 `protobuf:"varint,8,opt,name=transcription_latency_ms,json=transcriptionLatencyMs,proto3,oneof" json:"transcription_latency_ms,omitempty"` // agent-reported STT/endpointing delay, mean + Wer *float32 `protobuf:"fixed32,1,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word error rate, pooled: word_errors / words + Words *uint32 `protobuf:"varint,2,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession; pooling stats: run WER = sum errors / sum words + WordErrors *uint32 `protobuf:"varint,3,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession + Cer *float32 `protobuf:"fixed32,4,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; character error rate, pooled + Chars *uint32 `protobuf:"varint,9,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession + CharErrors *uint32 `protobuf:"varint,10,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession + KeytermRecall *float32 `protobuf:"fixed32,5,opt,name=keyterm_recall,json=keytermRecall,proto3,oneof" json:"keyterm_recall,omitempty"` // needs the RemoteSession; recall of uttered key entities (names, IDs, amounts) + KeytermsUttered *uint32 `protobuf:"varint,6,opt,name=keyterms_uttered,json=keytermsUttered,proto3,oneof" json:"keyterms_uttered,omitempty"` // needs the RemoteSession + KeytermsRecognized *uint32 `protobuf:"varint,7,opt,name=keyterms_recognized,json=keytermsRecognized,proto3,oneof" json:"keyterms_recognized,omitempty"` // needs the RemoteSession + TranscriptionLatencyMs *uint32 `protobuf:"varint,8,opt,name=transcription_latency_ms,json=transcriptionLatencyMs,proto3,oneof" json:"transcription_latency_ms,omitempty"` // needs the RemoteSession; agent-reported STT/endpointing delay, mean unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1555,10 +1555,10 @@ func (x *SimulationRun_JobMetrics_STT) GetTranscriptionLatencyMs() uint32 { type SimulationRun_JobMetrics_LLM struct { state protoimpl.MessageState `protogen:"open.v1"` - TtftMs *uint32 `protobuf:"varint,1,opt,name=ttft_ms,json=ttftMs,proto3,oneof" json:"ttft_ms,omitempty"` // time to first token, mean (per-turn in turns) - TtfsMs *uint32 `protobuf:"varint,2,opt,name=ttfs_ms,json=ttfsMs,proto3,oneof" json:"ttfs_ms,omitempty"` // time to first sentence — the smallest speakable unit - TokensPerSecond *float32 `protobuf:"fixed32,3,opt,name=tokens_per_second,json=tokensPerSecond,proto3,oneof" json:"tokens_per_second,omitempty"` // decode rate, mean - ConcisenessScore *float32 `protobuf:"fixed32,4,opt,name=conciseness_score,json=concisenessScore,proto3,oneof" json:"conciseness_score,omitempty"` // judged 0-1: brief enough for voice + TtftMs *uint32 `protobuf:"varint,1,opt,name=ttft_ms,json=ttftMs,proto3,oneof" json:"ttft_ms,omitempty"` // needs the RemoteSession; time to first token, mean (per-turn in turns) + TtfsMs *uint32 `protobuf:"varint,2,opt,name=ttfs_ms,json=ttfsMs,proto3,oneof" json:"ttfs_ms,omitempty"` // needs the RemoteSession; time to first sentence (the smallest speakable unit) + TokensPerSecond *float32 `protobuf:"fixed32,3,opt,name=tokens_per_second,json=tokensPerSecond,proto3,oneof" json:"tokens_per_second,omitempty"` // needs the RemoteSession; decode rate, mean (per-turn in turns.llm) + ConcisenessScore *float32 `protobuf:"fixed32,4,opt,name=conciseness_score,json=concisenessScore,proto3,oneof" json:"conciseness_score,omitempty"` // audio only; needs the audio judge; judged 0-1: brief enough for voice unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1621,23 +1621,22 @@ func (x *SimulationRun_JobMetrics_LLM) GetConcisenessScore() float32 { return 0 } -// The agent's voice. +// The agent's voice. audio only. type SimulationRun_JobMetrics_TTS struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Time to first AUDIO — the user-relevant signal. Some providers stream - // bytes before actual speech, so byte-level TTFB alone over-promises; - // it is kept below for disambiguation. - TtfaMs *uint32 `protobuf:"varint,1,opt,name=ttfa_ms,json=ttfaMs,proto3,oneof" json:"ttfa_ms,omitempty"` - TtfbMs *uint32 `protobuf:"varint,2,opt,name=ttfb_ms,json=ttfbMs,proto3,oneof" json:"ttfb_ms,omitempty"` // provider byte-level TTFB, agent-reported - Wer *float32 `protobuf:"fixed32,3,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // word error rate, pooled: word_errors / words - Words *uint32 `protobuf:"varint,7,opt,name=words,proto3,oneof" json:"words,omitempty"` - WordErrors *uint32 `protobuf:"varint,8,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` - Cer *float32 `protobuf:"fixed32,9,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // character error rate, pooled - Chars *uint32 `protobuf:"varint,10,opt,name=chars,proto3,oneof" json:"chars,omitempty"` - CharErrors *uint32 `protobuf:"varint,11,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` - SpeechRateWpm *float32 `protobuf:"fixed32,4,opt,name=speech_rate_wpm,json=speechRateWpm,proto3,oneof" json:"speech_rate_wpm,omitempty"` // speaking rate; conversational English ~110-150 - NaturalnessScore *float32 `protobuf:"fixed32,5,opt,name=naturalness_score,json=naturalnessScore,proto3,oneof" json:"naturalness_score,omitempty"` // judged 0-1: prosody / expressiveness - EnunciationScore *float32 `protobuf:"fixed32,6,opt,name=enunciation_score,json=enunciationScore,proto3,oneof" json:"enunciation_score,omitempty"` // judged 0-1: key entities audibly intact + state protoimpl.MessageState `protogen:"open.v1"` + TtfaMs *uint32 `protobuf:"varint,1,opt,name=ttfa_ms,json=ttfaMs,proto3,oneof" json:"ttfa_ms,omitempty"` // needs the RemoteSession + TtfbMs *uint32 `protobuf:"varint,2,opt,name=ttfb_ms,json=ttfbMs,proto3,oneof" json:"ttfb_ms,omitempty"` // needs the RemoteSession; provider byte-level TTFB + Wer *float32 `protobuf:"fixed32,3,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word error rate, pooled: word_errors / words + Words *uint32 `protobuf:"varint,7,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession + WordErrors *uint32 `protobuf:"varint,8,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession + Cer *float32 `protobuf:"fixed32,9,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; character error rate, pooled + Chars *uint32 `protobuf:"varint,10,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession + CharErrors *uint32 `protobuf:"varint,11,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession + SpeechRateWpm *float32 `protobuf:"fixed32,4,opt,name=speech_rate_wpm,json=speechRateWpm,proto3,oneof" json:"speech_rate_wpm,omitempty"` // speaking rate; conversational English ~110-150 + HeardWords *uint32 `protobuf:"varint,12,opt,name=heard_words,json=heardWords,proto3,oneof" json:"heard_words,omitempty"` // pooling stats for speech_rate_wpm: raw heard words (not WER tokens) + SpeechMs *uint32 `protobuf:"varint,13,opt,name=speech_ms,json=speechMs,proto3,oneof" json:"speech_ms,omitempty"` // agent voiced time, sum of raw VAD segments + NaturalnessScore *float32 `protobuf:"fixed32,5,opt,name=naturalness_score,json=naturalnessScore,proto3,oneof" json:"naturalness_score,omitempty"` // needs the audio judge; judged 0-1: prosody / expressiveness + EnunciationScore *float32 `protobuf:"fixed32,6,opt,name=enunciation_score,json=enunciationScore,proto3,oneof" json:"enunciation_score,omitempty"` // needs the audio judge; judged 0-1: key entities audibly intact unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1735,6 +1734,20 @@ func (x *SimulationRun_JobMetrics_TTS) GetSpeechRateWpm() float32 { return 0 } +func (x *SimulationRun_JobMetrics_TTS) GetHeardWords() uint32 { + if x != nil && x.HeardWords != nil { + return *x.HeardWords + } + return 0 +} + +func (x *SimulationRun_JobMetrics_TTS) GetSpeechMs() uint32 { + if x != nil && x.SpeechMs != nil { + return *x.SpeechMs + } + return 0 +} + func (x *SimulationRun_JobMetrics_TTS) GetNaturalnessScore() float32 { if x != nil && x.NaturalnessScore != nil { return *x.NaturalnessScore @@ -1749,25 +1762,25 @@ func (x *SimulationRun_JobMetrics_TTS) GetEnunciationScore() float32 { return 0 } -// End-to-end floor dynamics, measured off the audio timeline — the ground -// truth of what the caller hears, independent of agent self-reports. +// Audio only. type SimulationRun_JobMetrics_Conversation struct { state protoimpl.MessageState `protogen:"open.v1"` TurnTakingScore *float32 `protobuf:"fixed32,1,opt,name=turn_taking_score,json=turnTakingScore,proto3,oneof" json:"turn_taking_score,omitempty"` // 0-1: latency curve + cut-in/barge-in/missed-turn penalties ResponseLatencyP50Ms *int32 `protobuf:"varint,2,opt,name=response_latency_p50_ms,json=responseLatencyP50Ms,proto3,oneof" json:"response_latency_p50_ms,omitempty"` // floor-transfer offset; a gap is > 0, an overlap < 0 ResponseLatencyP95Ms *int32 `protobuf:"varint,3,opt,name=response_latency_p95_ms,json=responseLatencyP95Ms,proto3,oneof" json:"response_latency_p95_ms,omitempty"` ResponseLatencyP99Ms *int32 `protobuf:"varint,4,opt,name=response_latency_p99_ms,json=responseLatencyP99Ms,proto3,oneof" json:"response_latency_p99_ms,omitempty"` - AgentYieldLatencyMs *uint32 `protobuf:"varint,5,opt,name=agent_yield_latency_ms,json=agentYieldLatencyMs,proto3,oneof" json:"agent_yield_latency_ms,omitempty"` // how long the agent talks after a barge-in, mean - EotMispredictionCount *uint32 `protobuf:"varint,6,opt,name=eot_misprediction_count,json=eotMispredictionCount,proto3,oneof" json:"eot_misprediction_count,omitempty"` // agent started before the caller's turn ended + ResponseLatencyMs *int32 `protobuf:"varint,16,opt,name=response_latency_ms,json=responseLatencyMs,proto3,oneof" json:"response_latency_ms,omitempty"` // one turn's floor-transfer offset, cut-in if negative; job/run serve the percentiles + AgentYieldLatencyMs *uint32 `protobuf:"varint,5,opt,name=agent_yield_latency_ms,json=agentYieldLatencyMs,proto3,oneof" json:"agent_yield_latency_ms,omitempty"` // agent audio continuing past a barge-in — per turn: on the barging persona row (presence = barge-in); job: mean + EotMispredictionCount *uint32 `protobuf:"varint,6,opt,name=eot_misprediction_count,json=eotMispredictionCount,proto3,oneof" json:"eot_misprediction_count,omitempty"` // agent started before the caller's turn ended; 0/1 per turn OverlapRatio *float32 `protobuf:"fixed32,7,opt,name=overlap_ratio,json=overlapRatio,proto3,oneof" json:"overlap_ratio,omitempty"` // overlapping speech / total speech OverlapSpeechMs *uint32 `protobuf:"varint,8,opt,name=overlap_speech_ms,json=overlapSpeechMs,proto3,oneof" json:"overlap_speech_ms,omitempty"` // pooling stats for overlap_ratio TotalSpeechMs *uint32 `protobuf:"varint,9,opt,name=total_speech_ms,json=totalSpeechMs,proto3,oneof" json:"total_speech_ms,omitempty"` SilenceTotalMs *uint32 `protobuf:"varint,10,opt,name=silence_total_ms,json=silenceTotalMs,proto3,oneof" json:"silence_total_ms,omitempty"` // dead air within the conversation - AwkwardSilenceCount *uint32 `protobuf:"varint,11,opt,name=awkward_silence_count,json=awkwardSilenceCount,proto3,oneof" json:"awkward_silence_count,omitempty"` // gaps past the natural-pause threshold - UnansweredPersonaTurns *uint32 `protobuf:"varint,12,opt,name=unanswered_persona_turns,json=unansweredPersonaTurns,proto3,oneof" json:"unanswered_persona_turns,omitempty"` // the simulated party spoke, the agent never responded - FalseInterruptionCount *uint32 `protobuf:"varint,13,opt,name=false_interruption_count,json=falseInterruptionCount,proto3,oneof" json:"false_interruption_count,omitempty"` // agent paused for a non-interruption - FalseInterruptionUnrecoveredCount *uint32 `protobuf:"varint,14,opt,name=false_interruption_unrecovered_count,json=falseInterruptionUnrecoveredCount,proto3,oneof" json:"false_interruption_unrecovered_count,omitempty"` // of those, never resumed - AgentReportedE2ELatencyMs *uint32 `protobuf:"varint,15,opt,name=agent_reported_e2e_latency_ms,json=agentReportedE2eLatencyMs,proto3,oneof" json:"agent_reported_e2e_latency_ms,omitempty"` // the agent's own claim, mean + AwkwardSilenceCount *uint32 `protobuf:"varint,11,opt,name=awkward_silence_count,json=awkwardSilenceCount,proto3,oneof" json:"awkward_silence_count,omitempty"` // gaps past the natural-pause threshold; 0/1 per turn + UnansweredPersonaTurns *uint32 `protobuf:"varint,12,opt,name=unanswered_persona_turns,json=unansweredPersonaTurns,proto3,oneof" json:"unanswered_persona_turns,omitempty"` // the simulated party spoke, the agent never responded; 0/1 per turn + FalseInterruptionCount *uint32 `protobuf:"varint,13,opt,name=false_interruption_count,json=falseInterruptionCount,proto3,oneof" json:"false_interruption_count,omitempty"` // needs the RemoteSession; agent paused for a non-interruption + FalseInterruptionUnrecoveredCount *uint32 `protobuf:"varint,14,opt,name=false_interruption_unrecovered_count,json=falseInterruptionUnrecoveredCount,proto3,oneof" json:"false_interruption_unrecovered_count,omitempty"` // needs the RemoteSession; of those, never resumed + AgentReportedE2ELatencyMs *uint32 `protobuf:"varint,15,opt,name=agent_reported_e2e_latency_ms,json=agentReportedE2eLatencyMs,proto3,oneof" json:"agent_reported_e2e_latency_ms,omitempty"` // needs the RemoteSession; the agent's own claim, mean unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1830,6 +1843,13 @@ func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyP99Ms() int32 return 0 } +func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyMs() int32 { + if x != nil && x.ResponseLatencyMs != nil { + return *x.ResponseLatencyMs + } + return 0 +} + func (x *SimulationRun_JobMetrics_Conversation) GetAgentYieldLatencyMs() uint32 { if x != nil && x.AgentYieldLatencyMs != nil { return *x.AgentYieldLatencyMs @@ -1962,27 +1982,20 @@ func (x *SimulationRun_JobMetrics_Simulator) GetLateTermination() bool { return false } +// Rows exist for text and audio; type SimulationRun_JobMetrics_Turn struct { state protoimpl.MessageState `protogen:"open.v1"` Index uint32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` // 1-based, conversation order Role agent.ChatRole `protobuf:"varint,2,opt,name=role,proto3,enum=livekit.agent.ChatRole" json:"role,omitempty"` // ASSISTANT — the agent under test, USER — simulator persona. - StartMs *uint32 `protobuf:"varint,3,opt,name=start_ms,json=startMs,proto3,oneof" json:"start_ms,omitempty"` // relative to t0; unset if the turn could not be aligned to the audio - EndMs *uint32 `protobuf:"varint,4,opt,name=end_ms,json=endMs,proto3,oneof" json:"end_ms,omitempty"` - // Agent-turn measurements; a negative response latency is a cut-in (the - // agent started before the caller's turn ended). - ResponseLatencyMs *int32 `protobuf:"varint,5,opt,name=response_latency_ms,json=responseLatencyMs,proto3,oneof" json:"response_latency_ms,omitempty"` - TranscriptionDelayMs *uint32 `protobuf:"varint,6,opt,name=transcription_delay_ms,json=transcriptionDelayMs,proto3,oneof" json:"transcription_delay_ms,omitempty"` - LlmTtftMs *uint32 `protobuf:"varint,7,opt,name=llm_ttft_ms,json=llmTtftMs,proto3,oneof" json:"llm_ttft_ms,omitempty"` - LlmTtfsMs *uint32 `protobuf:"varint,8,opt,name=llm_ttfs_ms,json=llmTtfsMs,proto3,oneof" json:"llm_ttfs_ms,omitempty"` - TtsTtfaMs *uint32 `protobuf:"varint,9,opt,name=tts_ttfa_ms,json=ttsTtfaMs,proto3,oneof" json:"tts_ttfa_ms,omitempty"` - TtsTtfbMs *uint32 `protobuf:"varint,10,opt,name=tts_ttfb_ms,json=ttsTtfbMs,proto3,oneof" json:"tts_ttfb_ms,omitempty"` - AgentReportedE2ELatencyMs *uint32 `protobuf:"varint,11,opt,name=agent_reported_e2e_latency_ms,json=agentReportedE2eLatencyMs,proto3,oneof" json:"agent_reported_e2e_latency_ms,omitempty"` - ConcisenessScore *float32 `protobuf:"fixed32,12,opt,name=conciseness_score,json=concisenessScore,proto3,oneof" json:"conciseness_score,omitempty"` - NaturalnessScore *float32 `protobuf:"fixed32,13,opt,name=naturalness_score,json=naturalnessScore,proto3,oneof" json:"naturalness_score,omitempty"` - EnunciationScore *float32 `protobuf:"fixed32,14,opt,name=enunciation_score,json=enunciationScore,proto3,oneof" json:"enunciation_score,omitempty"` // judged 0-1: this turn's key entities audibly intact - Flags []string `protobuf:"bytes,15,rep,name=flags,proto3" json:"flags,omitempty"` // judge failure tags ("verbosity_or_filler", ...) - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + StartMs *uint32 `protobuf:"varint,3,opt,name=start_ms,json=startMs,proto3,oneof" json:"start_ms,omitempty"` // audio only; relative to t0; unset if the turn could not be aligned to the audio + EndMs *uint32 `protobuf:"varint,4,opt,name=end_ms,json=endMs,proto3,oneof" json:"end_ms,omitempty"` // audio only + // Job groups hold means/pools of these; a field with no per-turn measurement stays unset here. + Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,5,opt,name=stt,proto3" json:"stt,omitempty"` // transcription_latency_ms + Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,6,opt,name=llm,proto3" json:"llm,omitempty"` // ttft_ms, ttfs_ms, tokens_per_second, conciseness_score + Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,7,opt,name=tts,proto3" json:"tts,omitempty"` // ttfa_ms, ttfb_ms, naturalness_score, enunciation_score + Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,8,opt,name=conversation,proto3" json:"conversation,omitempty"` // response_latency_ms, agent_yield_latency_ms, agent_reported_e2e_latency_ms, 0/1 counts + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_Turn) Reset() { @@ -2043,79 +2056,30 @@ func (x *SimulationRun_JobMetrics_Turn) GetEndMs() uint32 { return 0 } -func (x *SimulationRun_JobMetrics_Turn) GetResponseLatencyMs() int32 { - if x != nil && x.ResponseLatencyMs != nil { - return *x.ResponseLatencyMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetTranscriptionDelayMs() uint32 { - if x != nil && x.TranscriptionDelayMs != nil { - return *x.TranscriptionDelayMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetLlmTtftMs() uint32 { - if x != nil && x.LlmTtftMs != nil { - return *x.LlmTtftMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetLlmTtfsMs() uint32 { - if x != nil && x.LlmTtfsMs != nil { - return *x.LlmTtfsMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetTtsTtfaMs() uint32 { - if x != nil && x.TtsTtfaMs != nil { - return *x.TtsTtfaMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetTtsTtfbMs() uint32 { - if x != nil && x.TtsTtfbMs != nil { - return *x.TtsTtfbMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetAgentReportedE2ELatencyMs() uint32 { - if x != nil && x.AgentReportedE2ELatencyMs != nil { - return *x.AgentReportedE2ELatencyMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Turn) GetConcisenessScore() float32 { - if x != nil && x.ConcisenessScore != nil { - return *x.ConcisenessScore +func (x *SimulationRun_JobMetrics_Turn) GetStt() *SimulationRun_JobMetrics_STT { + if x != nil { + return x.Stt } - return 0 + return nil } -func (x *SimulationRun_JobMetrics_Turn) GetNaturalnessScore() float32 { - if x != nil && x.NaturalnessScore != nil { - return *x.NaturalnessScore +func (x *SimulationRun_JobMetrics_Turn) GetLlm() *SimulationRun_JobMetrics_LLM { + if x != nil { + return x.Llm } - return 0 + return nil } -func (x *SimulationRun_JobMetrics_Turn) GetEnunciationScore() float32 { - if x != nil && x.EnunciationScore != nil { - return *x.EnunciationScore +func (x *SimulationRun_JobMetrics_Turn) GetTts() *SimulationRun_JobMetrics_TTS { + if x != nil { + return x.Tts } - return 0 + return nil } -func (x *SimulationRun_JobMetrics_Turn) GetFlags() []string { +func (x *SimulationRun_JobMetrics_Turn) GetConversation() *SimulationRun_JobMetrics_Conversation { if x != nil { - return x.Flags + return x.Conversation } return nil } @@ -2823,7 +2787,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "suggestion\x18\x02 \x01(\tR\n" + "suggestion\x12\x14\n" + - "\x05label\x18\x03 \x01(\tR\x05label\"\x91?\n" + + "\x05label\x18\x03 \x01(\tR\x05label\"\xf5<\n" + "\rSimulationRun\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + "\n" + @@ -2873,7 +2837,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x10STATUS_COMPLETED\x10\x02\x12\x11\n" + "\rSTATUS_FAILED\x10\x03\x12\x14\n" + "\x10STATUS_CANCELLED\x10\x04J\x04\b\t\x10\n" + - "\x1a\xc9\"\n" + + "\x1a\xad \n" + "\n" + "JobMetrics\x12*\n" + "\x0eaccuracy_score\x18\x01 \x01(\x02H\x00R\raccuracyScore\x88\x01\x01\x12.\n" + @@ -2925,7 +2889,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "\b_ttfs_msB\x14\n" + "\x12_tokens_per_secondB\x14\n" + - "\x12_conciseness_score\x1a\x9e\x04\n" + + "\x12_conciseness_score\x1a\x84\x05\n" + "\x03TTS\x12\x1c\n" + "\attfa_ms\x18\x01 \x01(\rH\x00R\x06ttfaMs\x88\x01\x01\x12\x1c\n" + "\attfb_ms\x18\x02 \x01(\rH\x01R\x06ttfbMs\x88\x01\x01\x12\x15\n" + @@ -2938,10 +2902,13 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + " \x01(\rH\x06R\x05chars\x88\x01\x01\x12$\n" + "\vchar_errors\x18\v \x01(\rH\aR\n" + "charErrors\x88\x01\x01\x12+\n" + - "\x0fspeech_rate_wpm\x18\x04 \x01(\x02H\bR\rspeechRateWpm\x88\x01\x01\x120\n" + - "\x11naturalness_score\x18\x05 \x01(\x02H\tR\x10naturalnessScore\x88\x01\x01\x120\n" + - "\x11enunciation_score\x18\x06 \x01(\x02H\n" + - "R\x10enunciationScore\x88\x01\x01B\n" + + "\x0fspeech_rate_wpm\x18\x04 \x01(\x02H\bR\rspeechRateWpm\x88\x01\x01\x12$\n" + + "\vheard_words\x18\f \x01(\rH\tR\n" + + "heardWords\x88\x01\x01\x12 \n" + + "\tspeech_ms\x18\r \x01(\rH\n" + + "R\bspeechMs\x88\x01\x01\x120\n" + + "\x11naturalness_score\x18\x05 \x01(\x02H\vR\x10naturalnessScore\x88\x01\x01\x120\n" + + "\x11enunciation_score\x18\x06 \x01(\x02H\fR\x10enunciationScore\x88\x01\x01B\n" + "\n" + "\b_ttfa_msB\n" + "\n" + @@ -2952,32 +2919,37 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x04_cerB\b\n" + "\x06_charsB\x0e\n" + "\f_char_errorsB\x12\n" + - "\x10_speech_rate_wpmB\x14\n" + + "\x10_speech_rate_wpmB\x0e\n" + + "\f_heard_wordsB\f\n" + + "\n" + + "_speech_msB\x14\n" + "\x12_naturalness_scoreB\x14\n" + - "\x12_enunciation_score\x1a\x86\n" + + "\x12_enunciation_score\x1a\xd3\n" + "\n" + "\fConversation\x12/\n" + "\x11turn_taking_score\x18\x01 \x01(\x02H\x00R\x0fturnTakingScore\x88\x01\x01\x12:\n" + "\x17response_latency_p50_ms\x18\x02 \x01(\x05H\x01R\x14responseLatencyP50Ms\x88\x01\x01\x12:\n" + "\x17response_latency_p95_ms\x18\x03 \x01(\x05H\x02R\x14responseLatencyP95Ms\x88\x01\x01\x12:\n" + - "\x17response_latency_p99_ms\x18\x04 \x01(\x05H\x03R\x14responseLatencyP99Ms\x88\x01\x01\x128\n" + - "\x16agent_yield_latency_ms\x18\x05 \x01(\rH\x04R\x13agentYieldLatencyMs\x88\x01\x01\x12;\n" + - "\x17eot_misprediction_count\x18\x06 \x01(\rH\x05R\x15eotMispredictionCount\x88\x01\x01\x12(\n" + - "\roverlap_ratio\x18\a \x01(\x02H\x06R\foverlapRatio\x88\x01\x01\x12/\n" + - "\x11overlap_speech_ms\x18\b \x01(\rH\aR\x0foverlapSpeechMs\x88\x01\x01\x12+\n" + - "\x0ftotal_speech_ms\x18\t \x01(\rH\bR\rtotalSpeechMs\x88\x01\x01\x12-\n" + + "\x17response_latency_p99_ms\x18\x04 \x01(\x05H\x03R\x14responseLatencyP99Ms\x88\x01\x01\x123\n" + + "\x13response_latency_ms\x18\x10 \x01(\x05H\x04R\x11responseLatencyMs\x88\x01\x01\x128\n" + + "\x16agent_yield_latency_ms\x18\x05 \x01(\rH\x05R\x13agentYieldLatencyMs\x88\x01\x01\x12;\n" + + "\x17eot_misprediction_count\x18\x06 \x01(\rH\x06R\x15eotMispredictionCount\x88\x01\x01\x12(\n" + + "\roverlap_ratio\x18\a \x01(\x02H\aR\foverlapRatio\x88\x01\x01\x12/\n" + + "\x11overlap_speech_ms\x18\b \x01(\rH\bR\x0foverlapSpeechMs\x88\x01\x01\x12+\n" + + "\x0ftotal_speech_ms\x18\t \x01(\rH\tR\rtotalSpeechMs\x88\x01\x01\x12-\n" + "\x10silence_total_ms\x18\n" + - " \x01(\rH\tR\x0esilenceTotalMs\x88\x01\x01\x127\n" + - "\x15awkward_silence_count\x18\v \x01(\rH\n" + - "R\x13awkwardSilenceCount\x88\x01\x01\x12=\n" + - "\x18unanswered_persona_turns\x18\f \x01(\rH\vR\x16unansweredPersonaTurns\x88\x01\x01\x12=\n" + - "\x18false_interruption_count\x18\r \x01(\rH\fR\x16falseInterruptionCount\x88\x01\x01\x12T\n" + - "$false_interruption_unrecovered_count\x18\x0e \x01(\rH\rR!falseInterruptionUnrecoveredCount\x88\x01\x01\x12E\n" + - "\x1dagent_reported_e2e_latency_ms\x18\x0f \x01(\rH\x0eR\x19agentReportedE2eLatencyMs\x88\x01\x01B\x14\n" + + " \x01(\rH\n" + + "R\x0esilenceTotalMs\x88\x01\x01\x127\n" + + "\x15awkward_silence_count\x18\v \x01(\rH\vR\x13awkwardSilenceCount\x88\x01\x01\x12=\n" + + "\x18unanswered_persona_turns\x18\f \x01(\rH\fR\x16unansweredPersonaTurns\x88\x01\x01\x12=\n" + + "\x18false_interruption_count\x18\r \x01(\rH\rR\x16falseInterruptionCount\x88\x01\x01\x12T\n" + + "$false_interruption_unrecovered_count\x18\x0e \x01(\rH\x0eR!falseInterruptionUnrecoveredCount\x88\x01\x01\x12E\n" + + "\x1dagent_reported_e2e_latency_ms\x18\x0f \x01(\rH\x0fR\x19agentReportedE2eLatencyMs\x88\x01\x01B\x14\n" + "\x12_turn_taking_scoreB\x1a\n" + "\x18_response_latency_p50_msB\x1a\n" + "\x18_response_latency_p95_msB\x1a\n" + - "\x18_response_latency_p99_msB\x19\n" + + "\x18_response_latency_p99_msB\x16\n" + + "\x14_response_latency_msB\x19\n" + "\x17_agent_yield_latency_msB\x1a\n" + "\x18_eot_misprediction_countB\x10\n" + "\x0e_overlap_ratioB\x14\n" + @@ -2993,37 +2965,18 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x11early_termination\x18\x01 \x01(\bH\x00R\x10earlyTermination\x88\x01\x01\x12.\n" + "\x10late_termination\x18\x02 \x01(\bH\x01R\x0flateTermination\x88\x01\x01B\x14\n" + "\x12_early_terminationB\x13\n" + - "\x11_late_termination\x1a\xeb\x06\n" + + "\x11_late_termination\x1a\x9c\x03\n" + "\x04Turn\x12\x14\n" + "\x05index\x18\x01 \x01(\rR\x05index\x12+\n" + "\x04role\x18\x02 \x01(\x0e2\x17.livekit.agent.ChatRoleR\x04role\x12\x1e\n" + "\bstart_ms\x18\x03 \x01(\rH\x00R\astartMs\x88\x01\x01\x12\x1a\n" + - "\x06end_ms\x18\x04 \x01(\rH\x01R\x05endMs\x88\x01\x01\x123\n" + - "\x13response_latency_ms\x18\x05 \x01(\x05H\x02R\x11responseLatencyMs\x88\x01\x01\x129\n" + - "\x16transcription_delay_ms\x18\x06 \x01(\rH\x03R\x14transcriptionDelayMs\x88\x01\x01\x12#\n" + - "\vllm_ttft_ms\x18\a \x01(\rH\x04R\tllmTtftMs\x88\x01\x01\x12#\n" + - "\vllm_ttfs_ms\x18\b \x01(\rH\x05R\tllmTtfsMs\x88\x01\x01\x12#\n" + - "\vtts_ttfa_ms\x18\t \x01(\rH\x06R\tttsTtfaMs\x88\x01\x01\x12#\n" + - "\vtts_ttfb_ms\x18\n" + - " \x01(\rH\aR\tttsTtfbMs\x88\x01\x01\x12E\n" + - "\x1dagent_reported_e2e_latency_ms\x18\v \x01(\rH\bR\x19agentReportedE2eLatencyMs\x88\x01\x01\x120\n" + - "\x11conciseness_score\x18\f \x01(\x02H\tR\x10concisenessScore\x88\x01\x01\x120\n" + - "\x11naturalness_score\x18\r \x01(\x02H\n" + - "R\x10naturalnessScore\x88\x01\x01\x120\n" + - "\x11enunciation_score\x18\x0e \x01(\x02H\vR\x10enunciationScore\x88\x01\x01\x12\x14\n" + - "\x05flags\x18\x0f \x03(\tR\x05flagsB\v\n" + + "\x06end_ms\x18\x04 \x01(\rH\x01R\x05endMs\x88\x01\x01\x127\n" + + "\x03stt\x18\x05 \x01(\v2%.livekit.SimulationRun.JobMetrics.STTR\x03stt\x127\n" + + "\x03llm\x18\x06 \x01(\v2%.livekit.SimulationRun.JobMetrics.LLMR\x03llm\x127\n" + + "\x03tts\x18\a \x01(\v2%.livekit.SimulationRun.JobMetrics.TTSR\x03tts\x12R\n" + + "\fconversation\x18\b \x01(\v2..livekit.SimulationRun.JobMetrics.ConversationR\fconversationB\v\n" + "\t_start_msB\t\n" + - "\a_end_msB\x16\n" + - "\x14_response_latency_msB\x19\n" + - "\x17_transcription_delay_msB\x0e\n" + - "\f_llm_ttft_msB\x0e\n" + - "\f_llm_ttfs_msB\x0e\n" + - "\f_tts_ttfa_msB\x0e\n" + - "\f_tts_ttfb_msB \n" + - "\x1e_agent_reported_e2e_latency_msB\x14\n" + - "\x12_conciseness_scoreB\x14\n" + - "\x12_naturalness_scoreB\x14\n" + - "\x12_enunciation_scoreB\x11\n" + + "\a_end_msB\x11\n" + "\x0f_accuracy_scoreB\x13\n" + "\x11_experience_scoreB\x12\n" + "\x10_task_completion\x1a\xcf\x04\n" + @@ -3236,32 +3189,36 @@ var file_livekit_agent_simulation_proto_depIdxs = []int32{ 22, // 29: livekit.SimulationRun.RunMetrics.tts:type_name -> livekit.SimulationRun.JobMetrics.TTS 23, // 30: livekit.SimulationRun.RunMetrics.conversation:type_name -> livekit.SimulationRun.JobMetrics.Conversation 42, // 31: livekit.SimulationRun.JobMetrics.Turn.role:type_name -> livekit.agent.ChatRole - 6, // 32: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup - 0, // 33: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode - 43, // 34: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest - 4, // 35: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun - 1, // 36: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status - 44, // 37: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination - 4, // 38: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun - 44, // 39: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination - 5, // 40: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario - 26, // 41: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request - 28, // 42: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request - 30, // 43: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request - 32, // 44: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request - 34, // 45: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request - 38, // 46: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request - 27, // 47: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response - 29, // 48: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response - 31, // 49: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response - 33, // 50: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response - 35, // 51: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response - 39, // 52: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response - 47, // [47:53] is the sub-list for method output_type - 41, // [41:47] is the sub-list for method input_type - 41, // [41:41] is the sub-list for extension type_name - 41, // [41:41] is the sub-list for extension extendee - 0, // [0:41] is the sub-list for field type_name + 20, // 32: livekit.SimulationRun.JobMetrics.Turn.stt:type_name -> livekit.SimulationRun.JobMetrics.STT + 21, // 33: livekit.SimulationRun.JobMetrics.Turn.llm:type_name -> livekit.SimulationRun.JobMetrics.LLM + 22, // 34: livekit.SimulationRun.JobMetrics.Turn.tts:type_name -> livekit.SimulationRun.JobMetrics.TTS + 23, // 35: livekit.SimulationRun.JobMetrics.Turn.conversation:type_name -> livekit.SimulationRun.JobMetrics.Conversation + 6, // 36: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup + 0, // 37: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode + 43, // 38: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest + 4, // 39: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun + 1, // 40: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status + 44, // 41: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination + 4, // 42: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun + 44, // 43: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination + 5, // 44: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario + 26, // 45: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request + 28, // 46: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request + 30, // 47: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request + 32, // 48: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request + 34, // 49: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request + 38, // 50: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request + 27, // 51: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response + 29, // 52: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response + 31, // 53: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response + 33, // 54: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response + 35, // 55: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response + 39, // 56: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response + 51, // [51:57] is the sub-list for method output_type + 45, // [45:51] is the sub-list for method input_type + 45, // [45:45] is the sub-list for extension type_name + 45, // [45:45] is the sub-list for extension extendee + 0, // [0:45] is the sub-list for field type_name } func init() { file_livekit_agent_simulation_proto_init() } diff --git a/livekit/livekit_agent_simulation.twirp.go b/livekit/livekit_agent_simulation.twirp.go index 7d1fca026..b2d752787 100644 --- a/livekit/livekit_agent_simulation.twirp.go +++ b/livekit/livekit_agent_simulation.twirp.go @@ -1912,221 +1912,216 @@ func (s *agentSimulationServer) PathPrefix() string { } var twirpFileDescriptor9 = []byte{ - // 3446 bytes of a gzipped FileDescriptorProto + // 3365 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x1b, 0x49, - 0x76, 0x37, 0x3f, 0x45, 0x3e, 0x8a, 0x1f, 0x2a, 0xc9, 0x32, 0xdd, 0xb3, 0x9e, 0xb1, 0xe5, 0x99, - 0x1d, 0xc3, 0x9e, 0x91, 0x1d, 0xcd, 0x3a, 0xb3, 0xf2, 0xec, 0x20, 0x11, 0x25, 0xda, 0x92, 0x23, - 0xc9, 0x4a, 0x91, 0xc2, 0x66, 0x27, 0x08, 0x1a, 0xcd, 0xee, 0x32, 0xd5, 0x72, 0xb3, 0x9b, 0xe9, - 0xaa, 0xb6, 0xac, 0x05, 0x72, 0x0d, 0x16, 0xc8, 0x69, 0x6f, 0x39, 0x2d, 0x72, 0x19, 0x04, 0xc9, - 0x35, 0xd7, 0x00, 0x39, 0xe7, 0x14, 0xe4, 0x92, 0x53, 0xfe, 0x80, 0x20, 0x39, 0xe7, 0x1e, 0xd4, - 0xab, 0xea, 0x0f, 0x7e, 0x49, 0x9a, 0x20, 0x73, 0xcb, 0x8d, 0xf5, 0xde, 0xef, 0xd5, 0xe7, 0xef, - 0xbd, 0x7a, 0x5d, 0x8f, 0xf0, 0xb1, 0xe7, 0xbe, 0x67, 0xef, 0x5c, 0x61, 0x5a, 0x43, 0xe6, 0x0b, - 0x93, 0xbb, 0xa3, 0xc8, 0xb3, 0x84, 0x1b, 0xf8, 0x9b, 0xe3, 0x30, 0x10, 0x01, 0x59, 0xd2, 0x7a, - 0xe3, 0x01, 0x02, 0x9e, 0x4e, 0xc1, 0x19, 0xe7, 0x09, 0xd6, 0xf8, 0x64, 0x18, 0x04, 0x43, 0x8f, - 0x3d, 0xc5, 0xd6, 0x20, 0x7a, 0xfb, 0x54, 0xb8, 0x23, 0xc6, 0x85, 0x35, 0x1a, 0x6b, 0xc0, 0xdd, - 0xd8, 0xda, 0xf6, 0x82, 0xc8, 0x51, 0x7d, 0x68, 0xd5, 0x5a, 0xac, 0x1a, 0x05, 0x0e, 0xf3, 0xb8, - 0x92, 0x6e, 0xfc, 0x6b, 0x01, 0xd6, 0x7a, 0xc9, 0x94, 0x68, 0xe4, 0xf7, 0xa2, 0xd1, 0xc8, 0x0a, - 0x2f, 0xc9, 0x3a, 0x94, 0xc7, 0x16, 0xe7, 0xcc, 0x69, 0xe7, 0xee, 0xe7, 0x1e, 0x95, 0xa8, 0x6e, - 0x49, 0xf9, 0x5b, 0xcb, 0xf5, 0x98, 0xd3, 0xce, 0x2b, 0xb9, 0x6a, 0x91, 0x7b, 0x00, 0xc3, 0xc0, - 0xf5, 0x87, 0xe6, 0x05, 0xf3, 0xbc, 0x76, 0xe1, 0x7e, 0xee, 0x51, 0x95, 0x56, 0x51, 0xf2, 0x4b, - 0xe6, 0x79, 0x52, 0x2d, 0x02, 0xd3, 0x1d, 0x8d, 0xc3, 0xe0, 0x3d, 0x6b, 0x17, 0x95, 0x5a, 0x04, - 0x07, 0x4a, 0x40, 0xbe, 0x81, 0xb2, 0xcb, 0x79, 0xc4, 0x78, 0xbb, 0x74, 0xbf, 0xf0, 0xa8, 0xb6, - 0xf5, 0x70, 0x53, 0xcf, 0x76, 0x73, 0xde, 0xe4, 0x36, 0x0f, 0x24, 0x96, 0x6a, 0x13, 0xf2, 0xc7, - 0xb0, 0x6c, 0x9f, 0x59, 0xc2, 0x3c, 0x73, 0xb9, 0x08, 0xc2, 0xcb, 0x76, 0x19, 0xbb, 0xd8, 0xbc, - 0xba, 0x8b, 0xdd, 0x33, 0x4b, 0xec, 0x2b, 0x83, 0xae, 0x2f, 0xc2, 0x4b, 0x5a, 0xb3, 0x53, 0x89, - 0xf1, 0x1d, 0xb4, 0xa6, 0x01, 0xa4, 0x05, 0x85, 0x77, 0xec, 0x12, 0xb7, 0xa3, 0x4a, 0xe5, 0x4f, - 0xf2, 0x0c, 0x4a, 0xef, 0x2d, 0x2f, 0x62, 0xb8, 0x15, 0xb5, 0x2d, 0x23, 0x19, 0x51, 0xed, 0xbb, - 0xec, 0x61, 0x37, 0xf0, 0x05, 0xfb, 0x20, 0xa8, 0x02, 0xbe, 0xc8, 0xff, 0x3c, 0x67, 0x98, 0x50, - 0xc2, 0xf9, 0x93, 0xfb, 0x50, 0x73, 0x18, 0xb7, 0x43, 0x77, 0x2c, 0xe7, 0xa6, 0x3b, 0xce, 0x8a, - 0xc8, 0xc7, 0x00, 0x3c, 0x1a, 0x0e, 0x19, 0x47, 0x40, 0x1e, 0x01, 0x19, 0x09, 0x59, 0x83, 0x92, - 0x67, 0x0d, 0x58, 0xbc, 0xdf, 0xaa, 0xb1, 0xf1, 0xdb, 0x3f, 0x80, 0xfa, 0xc4, 0x9a, 0x49, 0x03, - 0xf2, 0xae, 0xa3, 0x07, 0xc8, 0xbb, 0x78, 0x58, 0xe3, 0x30, 0x38, 0x67, 0xb6, 0x30, 0x5d, 0x47, - 0xf7, 0x5b, 0xd5, 0x92, 0x03, 0x87, 0x3c, 0x87, 0x32, 0x17, 0x96, 0x88, 0x38, 0xf6, 0xdb, 0xd8, - 0xba, 0x37, 0x7f, 0x2b, 0x37, 0x7b, 0x08, 0xa2, 0x1a, 0x4c, 0x9e, 0xc0, 0x8a, 0x22, 0x6d, 0x76, - 0x55, 0xea, 0xa8, 0x5b, 0xa8, 0xd8, 0xcb, 0x2c, 0x6d, 0x0d, 0x4a, 0x2c, 0x0c, 0x83, 0xb0, 0x5d, - 0x52, 0x53, 0xc7, 0x06, 0xd9, 0x06, 0xb0, 0x43, 0x66, 0x09, 0xe6, 0x98, 0x96, 0x68, 0x97, 0xf5, - 0xb6, 0x2a, 0xd6, 0x6f, 0xc6, 0xac, 0xdf, 0xec, 0xc7, 0xac, 0xa7, 0x55, 0x8d, 0xde, 0x11, 0x64, - 0x13, 0x8a, 0xe7, 0xc1, 0x80, 0xb7, 0x97, 0xf0, 0xf4, 0x8d, 0x05, 0x53, 0x7e, 0x1d, 0x0c, 0x28, - 0xe2, 0xe4, 0x1e, 0xa8, 0xd9, 0xfa, 0xd6, 0x88, 0xb5, 0xab, 0x6a, 0x0f, 0x50, 0x72, 0x6c, 0x8d, - 0x18, 0xf9, 0x16, 0x1a, 0xdc, 0x66, 0xbe, 0x15, 0xba, 0x81, 0x39, 0x0c, 0x83, 0x68, 0xdc, 0x06, - 0x9c, 0xcd, 0x7a, 0xda, 0xb1, 0x56, 0xbf, 0x92, 0x5a, 0x5a, 0xe7, 0xd9, 0x26, 0x79, 0x0e, 0x15, - 0xe6, 0x3b, 0x6a, 0x19, 0xb5, 0x6b, 0x97, 0xb1, 0x84, 0xd8, 0x1d, 0x41, 0x3e, 0x82, 0xea, 0x79, - 0x30, 0x30, 0xed, 0x20, 0xf2, 0x45, 0x7b, 0x19, 0x1d, 0xac, 0x72, 0x1e, 0x0c, 0x76, 0x65, 0x9b, - 0x3c, 0x80, 0x65, 0xe5, 0x84, 0x5a, 0x5f, 0x47, 0x7d, 0x4d, 0xc9, 0x12, 0x88, 0xf2, 0x47, 0x0d, - 0x69, 0x28, 0x88, 0x92, 0x29, 0xc8, 0xe7, 0xd0, 0xf4, 0xa3, 0x51, 0x26, 0x0e, 0xf1, 0x76, 0x13, - 0x51, 0x0d, 0x3f, 0x1a, 0xa5, 0x9b, 0xc5, 0xc9, 0x16, 0x94, 0x22, 0x6e, 0x0d, 0x59, 0xbb, 0x85, - 0xf3, 0xff, 0xc9, 0x82, 0x1d, 0x3d, 0x95, 0x18, 0xaa, 0xa0, 0x92, 0xd2, 0x76, 0xe0, 0xdb, 0x51, - 0x18, 0x32, 0xdf, 0xbe, 0x6c, 0xaf, 0xa8, 0xe1, 0x33, 0x22, 0xf2, 0x04, 0x8a, 0x32, 0x00, 0xb5, - 0x09, 0x32, 0xeb, 0xce, 0x9c, 0x4e, 0x8f, 0x02, 0x87, 0x51, 0x04, 0x91, 0x6f, 0x60, 0x69, 0xc4, - 0x44, 0xe8, 0xda, 0xbc, 0xbd, 0x8a, 0x93, 0x78, 0xb0, 0x60, 0x12, 0x34, 0xf2, 0x8f, 0x14, 0x90, - 0xc6, 0x16, 0x72, 0x2f, 0xb8, 0x72, 0x76, 0xf3, 0xd7, 0x5c, 0x38, 0xed, 0xb5, 0xfb, 0xb9, 0x47, - 0xcb, 0xb4, 0xa6, 0x65, 0xdf, 0x71, 0xe1, 0x18, 0xff, 0x56, 0x82, 0xc2, 0xeb, 0x60, 0x30, 0xe3, - 0x1f, 0xdb, 0x89, 0x03, 0xe4, 0x71, 0x9a, 0x0f, 0x16, 0xb3, 0x69, 0xda, 0x09, 0x36, 0x60, 0xd9, - 0xf5, 0xb9, 0x08, 0x23, 0x5b, 0xed, 0xad, 0xf2, 0xcc, 0x09, 0x59, 0xca, 0xfd, 0x62, 0x96, 0xfb, - 0x5f, 0x02, 0x51, 0x84, 0x64, 0x1f, 0xc6, 0xcc, 0x16, 0xfa, 0x6c, 0x94, 0x7b, 0x28, 0xc7, 0xea, - 0x66, 0x14, 0xa9, 0xef, 0x97, 0x33, 0xbe, 0x4f, 0x08, 0x14, 0x85, 0x35, 0x54, 0x5e, 0x50, 0xa5, - 0xf8, 0x5b, 0x92, 0x2a, 0x0c, 0x82, 0x91, 0x22, 0x7a, 0x05, 0xd1, 0x15, 0x29, 0x40, 0x9e, 0x6f, - 0x03, 0x70, 0x61, 0x85, 0xda, 0xe3, 0xe0, 0x7a, 0x8f, 0xd3, 0xe8, 0x1d, 0xf1, 0xbf, 0xe5, 0xf8, - 0x1d, 0x58, 0xc2, 0xe9, 0xb8, 0x0e, 0x32, 0xbc, 0x4a, 0xcb, 0xb2, 0x79, 0xe0, 0x90, 0xdf, 0x8f, - 0x09, 0x57, 0xc7, 0xce, 0xee, 0x5f, 0xb1, 0xe9, 0x13, 0xa4, 0xcb, 0xb0, 0xa4, 0x71, 0x25, 0x4b, - 0x5e, 0x07, 0x83, 0x69, 0x96, 0x18, 0x7f, 0x06, 0x25, 0xec, 0x8c, 0x3c, 0x82, 0x96, 0x8c, 0xd4, - 0xa6, 0x88, 0x42, 0x9f, 0x6b, 0xf7, 0x51, 0x57, 0x5f, 0x43, 0xca, 0xfb, 0x52, 0xac, 0x3c, 0xe8, - 0x31, 0xac, 0x58, 0x91, 0xe3, 0x06, 0x13, 0x50, 0x75, 0x1b, 0x36, 0x51, 0x91, 0x62, 0x37, 0x02, - 0x28, 0x2b, 0x82, 0x10, 0x02, 0x8d, 0x5e, 0x7f, 0xa7, 0x7f, 0xda, 0x33, 0x4f, 0xba, 0xc7, 0x7b, - 0x07, 0xc7, 0xaf, 0x5a, 0xb7, 0x32, 0x32, 0x7a, 0x7a, 0x7c, 0x2c, 0x65, 0x39, 0xb2, 0x06, 0x2d, - 0x2d, 0xdb, 0x7d, 0x73, 0x74, 0x72, 0xd8, 0xed, 0x77, 0xf7, 0x5a, 0x79, 0xb2, 0x02, 0x75, 0x2d, - 0x7d, 0xb9, 0x73, 0x70, 0xd8, 0xdd, 0x6b, 0x15, 0xb2, 0xc0, 0x9d, 0xe3, 0xdd, 0xee, 0xa1, 0x94, - 0x16, 0x5f, 0x17, 0x2b, 0xd5, 0x16, 0x18, 0xff, 0xbc, 0x01, 0x90, 0xae, 0x96, 0x3c, 0x86, 0x86, - 0x65, 0xdb, 0x51, 0x68, 0xd9, 0x97, 0x26, 0xb7, 0x83, 0x90, 0xe1, 0xca, 0xf2, 0xfb, 0xb7, 0x68, - 0x3d, 0x96, 0xf7, 0xa4, 0xf8, 0x37, 0xb9, 0x1c, 0xd9, 0x84, 0x96, 0x24, 0x60, 0xe8, 0x32, 0xdf, - 0x66, 0x1a, 0x9d, 0x47, 0x74, 0x8e, 0x36, 0x53, 0x4d, 0x82, 0xff, 0x02, 0x9a, 0xc2, 0xe2, 0xef, - 0x4c, 0x3b, 0x18, 0x8d, 0x3d, 0x86, 0x31, 0xbf, 0x80, 0xf0, 0x3c, 0x6d, 0x48, 0xc5, 0x6e, 0x22, - 0x97, 0xe8, 0xaf, 0xa1, 0xc0, 0x85, 0x40, 0xe2, 0xd7, 0xb6, 0x3e, 0xbb, 0xf6, 0x9c, 0x36, 0x7b, - 0xfd, 0x3e, 0x95, 0x16, 0xd2, 0xd0, 0xf3, 0x46, 0xe8, 0x0e, 0x37, 0x32, 0x3c, 0x3c, 0x3c, 0xa2, - 0xd2, 0x42, 0x1a, 0x0a, 0xc1, 0xf5, 0x5d, 0x72, 0x03, 0xc3, 0x7e, 0xbf, 0x47, 0xa5, 0x05, 0xa1, - 0xb0, 0x6c, 0x07, 0xfe, 0x7b, 0x16, 0x72, 0x84, 0xb5, 0x97, 0xb0, 0x87, 0xcd, 0xeb, 0x7b, 0xd8, - 0xcd, 0x58, 0xd1, 0x89, 0x3e, 0xc8, 0x01, 0x54, 0x75, 0xe0, 0x0d, 0x42, 0x74, 0xc5, 0xda, 0xd6, - 0x93, 0x1b, 0x6c, 0x42, 0x6c, 0x42, 0x53, 0x6b, 0xf2, 0x0b, 0x28, 0x21, 0xff, 0xda, 0x55, 0xbc, - 0xf0, 0x7e, 0x7a, 0x83, 0x95, 0x45, 0xa1, 0x4f, 0x95, 0x11, 0xf9, 0x04, 0x6a, 0xe7, 0x91, 0x33, - 0x64, 0x2a, 0x1b, 0x44, 0xbf, 0xaf, 0x52, 0x40, 0x91, 0x8c, 0xbf, 0x5e, 0x4a, 0xf2, 0x2c, 0xac, - 0x86, 0x30, 0x45, 0xf2, 0xd7, 0x29, 0xf6, 0x0b, 0x20, 0x67, 0x16, 0x37, 0x43, 0x36, 0x0a, 0x04, - 0x8b, 0x53, 0x56, 0x74, 0xee, 0x0a, 0x6d, 0x9d, 0x59, 0x9c, 0xa2, 0xa2, 0xa7, 0xe4, 0xe4, 0x31, - 0xe4, 0xc5, 0x33, 0xed, 0xe3, 0x57, 0x05, 0x8c, 0xbc, 0x78, 0x66, 0xfc, 0x6d, 0x11, 0x0a, 0xbd, - 0x7e, 0x9f, 0xdc, 0x86, 0xc2, 0x05, 0x0b, 0x13, 0xd6, 0xca, 0x86, 0x64, 0xd3, 0x5d, 0x28, 0x5d, - 0x04, 0xa1, 0xa3, 0xc2, 0x74, 0x7d, 0x3f, 0x47, 0x55, 0x53, 0xaa, 0x3e, 0x85, 0x9a, 0xfc, 0x6d, - 0x62, 0x6c, 0x55, 0x61, 0xb8, 0xbe, 0x9f, 0xa7, 0x20, 0x85, 0x5d, 0x94, 0x49, 0xd4, 0x6d, 0x28, - 0xd8, 0x4c, 0xc5, 0xe1, 0xfc, 0x7e, 0x81, 0xca, 0x86, 0xee, 0xd7, 0x3e, 0xb3, 0x42, 0x8e, 0x69, - 0x41, 0x7d, 0xbf, 0x48, 0x55, 0x53, 0xf7, 0x2b, 0x7f, 0xc7, 0xfd, 0x02, 0x02, 0x4a, 0x14, 0xa4, - 0x30, 0xed, 0xf7, 0x31, 0x34, 0xde, 0xb1, 0x4b, 0xc1, 0xc2, 0x91, 0x19, 0x32, 0xdb, 0xf2, 0x3c, - 0x24, 0x6e, 0x7e, 0xbf, 0x4c, 0xeb, 0x5a, 0x4e, 0x51, 0xac, 0x1d, 0x4e, 0xcb, 0xb8, 0x19, 0x09, - 0xc1, 0x42, 0xe6, 0x20, 0x5b, 0xeb, 0xfb, 0x4b, 0xb4, 0x19, 0x6b, 0x4e, 0x95, 0x42, 0xe2, 0x7f, - 0x06, 0xab, 0x09, 0x3e, 0x64, 0x76, 0x30, 0xf4, 0xdd, 0x5f, 0x33, 0x07, 0xe9, 0x59, 0xdf, 0xaf, - 0x50, 0x12, 0x2b, 0x69, 0xa2, 0x93, 0x56, 0xdf, 0x42, 0x5b, 0x84, 0x96, 0x9f, 0x24, 0x60, 0xa6, - 0x67, 0x09, 0x79, 0x21, 0x9b, 0x23, 0x8e, 0x44, 0xac, 0xef, 0x57, 0xe9, 0xfa, 0x04, 0xe2, 0x50, - 0x01, 0x8e, 0xe4, 0x82, 0x3a, 0x65, 0x28, 0x9a, 0x17, 0x2c, 0xec, 0x54, 0xa0, 0x6c, 0xe2, 0x1e, - 0x77, 0x1a, 0xb0, 0x6c, 0x66, 0x76, 0x18, 0x11, 0xb6, 0x46, 0xe0, 0x6e, 0x21, 0x22, 0xb3, 0x57, - 0x9d, 0x15, 0x68, 0x9a, 0x93, 0xbb, 0xd2, 0x59, 0x85, 0x15, 0x73, 0x7a, 0xf1, 0x9d, 0x75, 0x58, - 0x33, 0xe7, 0xac, 0xb0, 0xf3, 0x11, 0xdc, 0x35, 0x17, 0xad, 0xc1, 0xf8, 0x8f, 0x1c, 0x14, 0x0e, - 0x0f, 0x8f, 0xc8, 0x4f, 0x60, 0x49, 0x88, 0xb7, 0x42, 0x2e, 0x2b, 0x87, 0xcb, 0xba, 0x45, 0xcb, - 0x52, 0x80, 0xcb, 0xd0, 0x5a, 0x2e, 0xb5, 0x31, 0x65, 0xa4, 0x96, 0x2b, 0xed, 0x53, 0x58, 0x11, - 0xc1, 0x3b, 0xe6, 0x73, 0x73, 0xcc, 0x42, 0x93, 0x33, 0x3b, 0xf0, 0x9d, 0x24, 0x98, 0x35, 0x95, - 0xea, 0x84, 0x85, 0x3d, 0x54, 0x48, 0x83, 0x67, 0xb0, 0x22, 0x73, 0x1b, 0x97, 0x33, 0x9f, 0x71, - 0xae, 0x83, 0x65, 0x4c, 0xa6, 0x56, 0x46, 0x15, 0x47, 0xcb, 0x0e, 0x40, 0xc5, 0xd4, 0xf3, 0x8b, - 0x7f, 0xcb, 0xd9, 0x74, 0xd6, 0x80, 0x98, 0x33, 0x63, 0xa3, 0x74, 0x66, 0x00, 0xe3, 0x77, 0x45, - 0x28, 0xf4, 0xfb, 0x3d, 0xbd, 0x18, 0x6b, 0x7a, 0xa9, 0x56, 0x76, 0xa9, 0x83, 0xe9, 0xa5, 0x0e, - 0x8e, 0x62, 0xe2, 0x4b, 0x87, 0x8a, 0x17, 0x37, 0xe3, 0x50, 0x8a, 0x4d, 0x85, 0xc5, 0x0e, 0x55, - 0xd1, 0x9e, 0x31, 0xdf, 0xa1, 0xaa, 0xd8, 0x6f, 0x69, 0xd6, 0xa1, 0x94, 0xbf, 0x94, 0x17, 0x3b, - 0x54, 0x4d, 0x33, 0x7f, 0xca, 0xa1, 0x9e, 0x40, 0x93, 0x8f, 0x19, 0xb3, 0xcf, 0xcc, 0xd0, 0x12, - 0xcc, 0xbc, 0x18, 0x8f, 0xf4, 0x3e, 0x57, 0x68, 0x5d, 0x29, 0xa8, 0x25, 0xd8, 0x2f, 0xc7, 0x23, - 0x7d, 0x2c, 0xbe, 0x25, 0xa2, 0xd0, 0xf2, 0x32, 0xc7, 0xa2, 0x1c, 0xb0, 0x4a, 0x5b, 0x19, 0x55, - 0x72, 0x89, 0x3d, 0x83, 0x15, 0xe6, 0x47, 0xbe, 0xed, 0x62, 0xdc, 0xd4, 0x16, 0x65, 0xb4, 0x00, - 0xda, 0xca, 0xa8, 0xa6, 0x0f, 0xd2, 0xca, 0x1c, 0xa4, 0xdc, 0xeb, 0xff, 0x13, 0x47, 0x21, 0xd0, - 0x32, 0xa7, 0x56, 0x8b, 0x54, 0x98, 0x59, 0x14, 0x4a, 0x67, 0x26, 0x6e, 0xfc, 0x25, 0xc0, 0x72, - 0xf6, 0x12, 0x42, 0x62, 0x47, 0xa1, 0x6f, 0x0a, 0xeb, 0x9d, 0xfc, 0x44, 0x9f, 0x4c, 0x01, 0x9a, - 0x52, 0xd5, 0x47, 0x4d, 0xb2, 0x1f, 0x2f, 0xe0, 0x4e, 0xc8, 0xf8, 0x38, 0xf0, 0x39, 0x4b, 0x9c, - 0x6c, 0xfc, 0xfc, 0x59, 0x4c, 0xa6, 0xd2, 0x7e, 0x8e, 0xae, 0xc5, 0x00, 0x1d, 0x27, 0x4e, 0x9e, - 0x3f, 0x53, 0xd4, 0x9a, 0x6b, 0xbb, 0xfd, 0x5c, 0xda, 0x16, 0xd0, 0x36, 0x3f, 0x6b, 0xbb, 0xfd, - 0xfc, 0x2a, 0xdb, 0x6d, 0x69, 0x5b, 0x44, 0xdb, 0xc2, 0x1c, 0xdb, 0x6d, 0x65, 0xfb, 0x73, 0x58, - 0x57, 0xf9, 0xf3, 0xa5, 0xcb, 0x3c, 0x27, 0x1b, 0xdf, 0x4a, 0x9a, 0xab, 0xab, 0xa8, 0xff, 0x95, - 0x54, 0x67, 0x83, 0x1b, 0xf9, 0x06, 0xee, 0xb0, 0x40, 0x98, 0x23, 0x97, 0x8f, 0x43, 0xe6, 0xb8, - 0x98, 0xa5, 0xeb, 0xb4, 0xae, 0xac, 0xe3, 0xfb, 0x6d, 0x16, 0x88, 0xa3, 0xac, 0x1e, 0xd3, 0x3b, - 0x69, 0xfc, 0x08, 0xea, 0xc1, 0x7b, 0x16, 0x7a, 0xd6, 0x58, 0x1e, 0x96, 0x1b, 0xa0, 0xeb, 0xc8, - 0x48, 0xbf, 0xac, 0xc5, 0x54, 0x4a, 0x75, 0x78, 0x89, 0x91, 0xfa, 0x74, 0x93, 0xd8, 0xbb, 0x44, - 0x9b, 0x5a, 0xd5, 0x43, 0xcd, 0x51, 0x4c, 0x7a, 0x11, 0x08, 0xcb, 0xcb, 0xc0, 0xab, 0x3a, 0xca, - 0xd7, 0x51, 0x91, 0x05, 0x7f, 0x09, 0x2d, 0xee, 0x7a, 0x98, 0xb4, 0x29, 0xa3, 0x51, 0xec, 0x6d, - 0x55, 0xda, 0xd0, 0x9a, 0xbe, 0x54, 0x28, 0xf8, 0xd7, 0x70, 0xdb, 0xba, 0x78, 0x77, 0x61, 0x85, - 0x8e, 0x19, 0x9b, 0xa9, 0x15, 0x2b, 0x07, 0x04, 0xba, 0xaa, 0xd5, 0x3d, 0xa5, 0x4d, 0xd6, 0xfb, - 0x2d, 0xb4, 0x23, 0xdf, 0xf2, 0xf9, 0x85, 0x8c, 0xd5, 0x32, 0x58, 0xf1, 0xc0, 0xb7, 0x54, 0x2a, - 0x8c, 0x57, 0x7e, 0x7d, 0xbf, 0x46, 0xd7, 0x53, 0xc4, 0x89, 0x02, 0x60, 0x4a, 0xac, 0xcd, 0xdf, - 0x5a, 0x1e, 0x67, 0xa6, 0xeb, 0x0b, 0x16, 0x86, 0xd1, 0x38, 0xb3, 0xd9, 0x75, 0x34, 0x5f, 0xa6, - 0xeb, 0x88, 0x38, 0xc8, 0x00, 0x92, 0xd1, 0xfb, 0xf0, 0xe9, 0x1c, 0xf3, 0xc8, 0x97, 0xd7, 0xc4, - 0x7b, 0x9c, 0x51, 0xfa, 0xe1, 0x5b, 0xdf, 0xaf, 0xd3, 0x07, 0x33, 0x5d, 0x9d, 0xa6, 0xd8, 0xa4, - 0xd7, 0x2e, 0xdc, 0x53, 0xd4, 0x09, 0xd9, 0x38, 0xc0, 0x6f, 0x21, 0xb6, 0xc5, 0xb2, 0x0c, 0x6a, - 0x62, 0x77, 0x0d, 0x7a, 0x17, 0x61, 0x54, 0xa3, 0xba, 0x5b, 0x6c, 0xe2, 0x92, 0xc4, 0x20, 0x3e, - 0xed, 0x67, 0x1d, 0x03, 0xda, 0xe6, 0x02, 0x67, 0x5a, 0xa0, 0x43, 0x67, 0x59, 0xa4, 0x93, 0xce, - 0xd0, 0xb9, 0x0b, 0x77, 0xcc, 0xf9, 0x64, 0x47, 0xb3, 0x05, 0x6c, 0xee, 0xb4, 0xa0, 0x61, 0x4e, - 0x90, 0x15, 0xa7, 0x3c, 0x43, 0x4a, 0x0c, 0x40, 0x53, 0xcc, 0xc3, 0xab, 0x7a, 0x9a, 0x60, 0x9d, - 0x36, 0xac, 0x9b, 0x73, 0x69, 0x84, 0x97, 0xf5, 0x22, 0x9e, 0xa0, 0x72, 0x11, 0x0b, 0x3a, 0x9f, - 0xc3, 0x67, 0xe6, 0x4d, 0xce, 0xb8, 0x73, 0x1f, 0x3e, 0x36, 0xaf, 0x3c, 0x36, 0xe3, 0xaf, 0x73, - 0x50, 0x4d, 0x92, 0x67, 0x0c, 0xf2, 0x56, 0xe8, 0x5d, 0x9a, 0x32, 0xb3, 0x70, 0x7d, 0x2b, 0x79, - 0x75, 0xab, 0xec, 0xdf, 0xa2, 0x2d, 0x54, 0xf5, 0x53, 0x8d, 0x4e, 0xcd, 0x64, 0x6f, 0x13, 0x06, - 0x79, 0x34, 0xc8, 0xd1, 0xa6, 0xd4, 0x4c, 0xe2, 0x55, 0x38, 0x9e, 0x1e, 0x02, 0x77, 0x6e, 0xba, - 0x1b, 0xe3, 0xbf, 0xca, 0x50, 0x94, 0x4e, 0x21, 0xbf, 0xe3, 0x5d, 0xdf, 0x61, 0x1f, 0xd4, 0x1d, - 0x4e, 0x55, 0x83, 0x3c, 0x81, 0x62, 0x18, 0x78, 0x4c, 0xbf, 0x3f, 0xdc, 0x99, 0xf3, 0xb2, 0x48, - 0x03, 0x8f, 0x51, 0x04, 0x91, 0x8f, 0xa1, 0x82, 0x5f, 0xe5, 0x71, 0x88, 0x95, 0x99, 0xc0, 0x12, - 0x4a, 0x94, 0xaf, 0x1b, 0x50, 0x66, 0xbe, 0x13, 0x07, 0x51, 0xcc, 0x93, 0x99, 0xef, 0x28, 0xdd, - 0x57, 0xb0, 0x3a, 0x43, 0x32, 0x1d, 0x32, 0x65, 0xa4, 0x5e, 0x99, 0x8a, 0xb6, 0xca, 0x68, 0x1b, - 0x26, 0x53, 0x45, 0xd3, 0x61, 0x9e, 0x85, 0x76, 0x65, 0x9d, 0x37, 0xac, 0x4d, 0xe8, 0xf7, 0xa4, - 0x5a, 0x99, 0x3e, 0x84, 0x9a, 0xe7, 0x8d, 0xe2, 0x1c, 0x48, 0xe7, 0x19, 0x45, 0x5a, 0xf5, 0xbc, - 0x51, 0x3f, 0x49, 0xd3, 0x52, 0x10, 0x4f, 0x63, 0x64, 0x29, 0x06, 0xf1, 0x04, 0x24, 0x04, 0x8f, - 0x2f, 0x61, 0x1d, 0x19, 0xcb, 0xb4, 0x2a, 0x04, 0xef, 0x27, 0x59, 0x50, 0x0a, 0x1a, 0xa4, 0x01, - 0x71, 0x29, 0x06, 0xe9, 0x64, 0xe8, 0x5a, 0xf7, 0xaf, 0xe9, 0xa8, 0x7b, 0xa5, 0xfb, 0xcf, 0xcf, - 0x06, 0x97, 0xe3, 0xb4, 0x63, 0x4e, 0x36, 0x38, 0x3f, 0x51, 0xa9, 0xc7, 0x69, 0xc7, 0x8d, 0x13, - 0x95, 0x06, 0x5a, 0xd4, 0xe6, 0x26, 0x2a, 0x92, 0x5f, 0x6f, 0x3d, 0x6b, 0x28, 0x63, 0x58, 0xe1, - 0x51, 0x95, 0xaa, 0x46, 0xa7, 0x06, 0x55, 0x33, 0xe6, 0x4c, 0xa7, 0x0a, 0x4b, 0xa6, 0x22, 0x08, - 0xe6, 0xde, 0x73, 0xf8, 0x80, 0x01, 0x67, 0xfe, 0x91, 0x63, 0xf6, 0x92, 0x39, 0xd2, 0x6c, 0x9b, - 0xc7, 0xed, 0xcc, 0x41, 0x65, 0xdb, 0x98, 0x31, 0x5d, 0xeb, 0xc6, 0xf3, 0xd3, 0xe0, 0x1f, 0x92, - 0x11, 0xe1, 0xa7, 0xc7, 0xe4, 0x0b, 0x08, 0x7a, 0xe5, 0xf4, 0x43, 0x87, 0x0a, 0x7c, 0x93, 0xaf, - 0x19, 0xc6, 0xbf, 0x14, 0x01, 0xd2, 0xf7, 0xc5, 0x1f, 0xf5, 0x2d, 0xe5, 0xf7, 0x80, 0x24, 0x8f, - 0xce, 0x63, 0x8b, 0x73, 0x4c, 0xff, 0x92, 0x24, 0xbd, 0x15, 0xeb, 0x4e, 0x2c, 0xce, 0x65, 0xba, - 0xfb, 0xff, 0x0f, 0x2a, 0xe4, 0x1e, 0xc0, 0x79, 0x30, 0xe0, 0xea, 0x46, 0x52, 0x31, 0x80, 0x56, - 0xa5, 0x04, 0x53, 0x1d, 0xf2, 0x10, 0xea, 0xa8, 0x1e, 0x31, 0x8b, 0x47, 0xf2, 0xc3, 0x1a, 0x03, - 0x00, 0x5d, 0x96, 0xc2, 0x23, 0x2d, 0x23, 0xcf, 0x60, 0x0d, 0x41, 0xc9, 0xdb, 0x8a, 0xf9, 0xd6, - 0x8a, 0x3c, 0x9d, 0x09, 0x51, 0x22, 0x75, 0xc9, 0x25, 0xf2, 0x52, 0x6a, 0x6e, 0x4c, 0xa7, 0xdb, - 0xb0, 0x6a, 0xce, 0x1e, 0xa8, 0xf1, 0xdf, 0x05, 0x28, 0xef, 0x62, 0xe1, 0xc2, 0xf8, 0xf7, 0x3c, - 0x2c, 0x51, 0xf6, 0xe7, 0x11, 0xe3, 0x62, 0xaa, 0x2a, 0x93, 0x9b, 0xae, 0xca, 0x4c, 0x16, 0x2c, - 0xf2, 0xd3, 0x05, 0x8b, 0x39, 0xef, 0xfa, 0xc5, 0xb9, 0xef, 0xfa, 0xeb, 0x50, 0x0e, 0xd9, 0x50, - 0x1e, 0x40, 0x59, 0x3f, 0xbf, 0x62, 0x8b, 0x74, 0x66, 0x2a, 0x1e, 0x4b, 0x57, 0x55, 0x3c, 0x24, - 0xe1, 0x27, 0x6a, 0x1e, 0x92, 0x8d, 0x9f, 0x4d, 0xbe, 0xff, 0x57, 0xf4, 0xb7, 0x42, 0x56, 0xa8, - 0x12, 0x5b, 0x55, 0x04, 0xa8, 0xde, 0xa0, 0x08, 0x80, 0x9b, 0x3d, 0x39, 0x31, 0xf5, 0xc1, 0x94, - 0x76, 0xf9, 0xba, 0x58, 0x29, 0xb4, 0x8a, 0x74, 0xb6, 0xfa, 0x64, 0xfc, 0x55, 0x0e, 0x2a, 0x54, - 0x87, 0x33, 0xf2, 0x18, 0x56, 0xd2, 0xdd, 0x31, 0xc3, 0xc8, 0x4f, 0xb7, 0xb9, 0xc9, 0xb3, 0xdc, - 0x3b, 0x70, 0x48, 0x0f, 0xd6, 0xc7, 0x21, 0xe3, 0xee, 0xd0, 0x97, 0xc9, 0x4b, 0xc0, 0x65, 0x70, - 0xc2, 0x53, 0xd2, 0xb5, 0xbe, 0xb4, 0x24, 0x76, 0x12, 0xc3, 0x4e, 0x02, 0x2e, 0xf4, 0x51, 0xd2, - 0xb5, 0xf1, 0x1c, 0xa9, 0xf1, 0x37, 0x39, 0x58, 0xdd, 0x0d, 0xfc, 0xb7, 0x6e, 0x38, 0xea, 0x05, - 0x51, 0x68, 0xb3, 0xd3, 0xb1, 0x17, 0x58, 0x8e, 0xf1, 0x17, 0x37, 0xe6, 0xc0, 0xdc, 0x25, 0xe4, - 0xe7, 0x2f, 0xe1, 0x73, 0x68, 0xda, 0x81, 0xc3, 0x4c, 0xe6, 0x8b, 0xf0, 0x72, 0x1c, 0xb8, 0xbe, - 0xd0, 0xc5, 0x88, 0x86, 0x14, 0x77, 0x13, 0xa9, 0x01, 0xe9, 0x1e, 0x19, 0xbf, 0xcd, 0x41, 0xe1, - 0x15, 0x13, 0x46, 0xff, 0xc7, 0x98, 0x92, 0xf1, 0xb3, 0xcc, 0x69, 0x3c, 0x82, 0x42, 0x18, 0xa9, - 0xfc, 0x6b, 0x82, 0x63, 0x59, 0x13, 0x2a, 0x21, 0xc6, 0x3f, 0xe5, 0xa1, 0x78, 0xe8, 0x72, 0x61, - 0xfc, 0x63, 0xee, 0xc6, 0xb3, 0x7a, 0x31, 0x55, 0xc1, 0xb9, 0xba, 0x84, 0xb9, 0x7f, 0x2b, 0xae, - 0xdf, 0x48, 0x76, 0xfe, 0x02, 0x60, 0x6c, 0x0d, 0x99, 0x7a, 0x8f, 0xc1, 0x3d, 0xab, 0x6d, 0xb5, - 0x13, 0xfb, 0xbe, 0x94, 0x9e, 0x58, 0x43, 0x9d, 0xc8, 0xed, 0xe7, 0x68, 0x55, 0xa2, 0x51, 0x2c, - 0x73, 0x40, 0x79, 0x99, 0xaa, 0xbe, 0x3a, 0x75, 0xa8, 0x99, 0x69, 0x4f, 0xc6, 0x87, 0x09, 0x2e, - 0x16, 0xc3, 0xc8, 0xe7, 0xed, 0x1c, 0x3e, 0xde, 0x2e, 0x5a, 0x3e, 0x62, 0xc8, 0x1f, 0x42, 0xd3, - 0x67, 0x1f, 0x44, 0xa6, 0x2b, 0x4d, 0xc2, 0x85, 0x93, 0xa2, 0x75, 0x69, 0x70, 0x12, 0x4f, 0xcb, - 0x38, 0x87, 0xf2, 0xae, 0xe5, 0xdb, 0xcc, 0xfb, 0x91, 0xce, 0x35, 0xcb, 0xa0, 0x1f, 0xb9, 0xa0, - 0xf2, 0x7d, 0x2e, 0xa9, 0xa8, 0xdc, 0x85, 0xdb, 0x93, 0x15, 0x15, 0xf3, 0xf4, 0xe4, 0xf0, 0xcd, - 0xce, 0x5e, 0xeb, 0x16, 0xb9, 0x0d, 0x2b, 0x5a, 0xf5, 0xaa, 0x7b, 0xdc, 0xa5, 0x3b, 0x7d, 0x55, - 0x5b, 0x99, 0xad, 0xb7, 0xe4, 0xc9, 0x3a, 0x10, 0x2d, 0xeb, 0x9d, 0x1e, 0x1d, 0xed, 0xd0, 0x83, - 0xef, 0xa4, 0xbc, 0x30, 0xb7, 0x0e, 0x53, 0x9c, 0xad, 0xc3, 0x94, 0xe6, 0xd6, 0x61, 0xca, 0xaf, - 0x8b, 0x95, 0x4a, 0xab, 0x4a, 0x97, 0x74, 0xb5, 0x71, 0xe3, 0xfb, 0x02, 0x54, 0xe2, 0xf0, 0x99, - 0x96, 0xee, 0x72, 0xd9, 0xd2, 0xdd, 0x74, 0xe5, 0x30, 0x3f, 0xa7, 0x72, 0x38, 0xbf, 0x46, 0x58, - 0x58, 0x54, 0x23, 0x7c, 0xaa, 0xab, 0x81, 0x45, 0x64, 0xd9, 0x47, 0x33, 0x81, 0x7c, 0xb3, 0x6f, - 0x0d, 0xb9, 0xfa, 0xfb, 0x83, 0x2a, 0x15, 0x1a, 0x50, 0x89, 0x38, 0x0b, 0x1d, 0x4b, 0x58, 0xba, - 0xf2, 0x98, 0xb4, 0x8d, 0xbf, 0xcb, 0xc1, 0x8a, 0xba, 0xb5, 0x5e, 0x86, 0xc1, 0x48, 0xbf, 0xe6, - 0x1b, 0xbf, 0xba, 0x31, 0xa1, 0x32, 0x75, 0xbf, 0xfc, 0x44, 0xdd, 0x2f, 0xbd, 0x90, 0x0a, 0xd9, - 0x0b, 0xc9, 0xd8, 0xce, 0xf8, 0xcb, 0x97, 0x50, 0x89, 0xef, 0x00, 0x1d, 0x32, 0x56, 0x66, 0x56, - 0x43, 0x13, 0x88, 0xf1, 0x35, 0x54, 0x93, 0xa5, 0xcd, 0xf9, 0xe3, 0xc6, 0x5a, 0xf6, 0x8f, 0x1b, - 0xd5, 0xcc, 0x9f, 0x33, 0x36, 0xfa, 0x50, 0x9f, 0xb8, 0xe5, 0x08, 0x81, 0x22, 0xde, 0xb7, 0xca, - 0x1a, 0x7f, 0x93, 0xa7, 0x50, 0x8d, 0x47, 0x92, 0xc7, 0x54, 0x98, 0x3f, 0x9b, 0x14, 0xb3, 0xf1, - 0x0f, 0x39, 0x20, 0xa9, 0x67, 0xef, 0xb9, 0x7c, 0x6c, 0x09, 0xfb, 0xec, 0x07, 0x5d, 0x48, 0xb7, - 0xa1, 0x7c, 0x1e, 0x0c, 0xd2, 0xcd, 0x2b, 0x9d, 0x07, 0x83, 0x03, 0x67, 0x62, 0x5f, 0x0a, 0xd7, - 0xee, 0x4b, 0x72, 0xf1, 0x16, 0x6f, 0x70, 0xf1, 0x3e, 0x7e, 0x0b, 0x8d, 0x49, 0x39, 0xf9, 0x04, - 0x3e, 0xea, 0x1d, 0x1c, 0x9d, 0x1e, 0xee, 0xf4, 0x0f, 0xde, 0x1c, 0x9b, 0x47, 0x6f, 0xf6, 0xba, - 0xe6, 0xe9, 0x71, 0xef, 0xa4, 0xbb, 0x7b, 0xf0, 0xf2, 0xa0, 0x2b, 0xfd, 0xae, 0x0d, 0x6b, 0xd3, - 0x80, 0x7e, 0xf7, 0x4f, 0xfa, 0xad, 0x1c, 0x3a, 0xeb, 0x94, 0x66, 0xe7, 0x74, 0xef, 0xe0, 0x4d, - 0x2b, 0xbf, 0xf5, 0xbb, 0x12, 0x34, 0x77, 0x24, 0x77, 0xd3, 0xd1, 0x88, 0x03, 0xab, 0x8a, 0x6b, - 0x93, 0x7f, 0x64, 0x59, 0x94, 0x6e, 0x2a, 0xec, 0x66, 0x7c, 0xd1, 0xfe, 0xf4, 0x3a, 0x98, 0x66, - 0xd5, 0x6f, 0x72, 0x70, 0x2f, 0xbe, 0x90, 0x13, 0x60, 0xf6, 0x6a, 0x26, 0x5b, 0x8b, 0x7a, 0x9a, - 0xbd, 0xc6, 0x93, 0xd1, 0xbf, 0xfa, 0x41, 0x36, 0x7a, 0x2a, 0x7f, 0x0a, 0xad, 0x57, 0x4c, 0x4c, - 0xae, 0x76, 0x63, 0x41, 0x47, 0xaf, 0x98, 0x48, 0x06, 0x7b, 0x78, 0x25, 0x46, 0x77, 0x6e, 0x02, - 0x91, 0x17, 0xe8, 0x04, 0x82, 0x93, 0x45, 0xa6, 0x12, 0x9a, 0xf4, 0xff, 0xe9, 0xd5, 0x20, 0x3d, - 0x80, 0x3c, 0x2e, 0xbc, 0x60, 0x6e, 0x78, 0x5c, 0x88, 0xbd, 0xfe, 0xb8, 0x62, 0x98, 0x1e, 0xe5, - 0x3d, 0xdc, 0xd5, 0xa4, 0xd0, 0x7c, 0xce, 0x04, 0x22, 0xf2, 0x64, 0x36, 0xba, 0xcd, 0x44, 0xab, - 0x64, 0xc4, 0x2f, 0x6e, 0x06, 0x56, 0xe3, 0x76, 0x5e, 0x7e, 0xf7, 0x70, 0xe8, 0x8a, 0xb3, 0x68, - 0xb0, 0x69, 0x07, 0xa3, 0xf8, 0x0f, 0x7a, 0xea, 0x4f, 0x78, 0x76, 0xe0, 0xc5, 0x82, 0xbf, 0xcf, - 0xd7, 0x0f, 0xdd, 0xf7, 0xec, 0x8f, 0x30, 0x27, 0x0c, 0x44, 0xf0, 0x9f, 0xf9, 0x86, 0x6e, 0xbf, - 0x78, 0x81, 0x82, 0x41, 0x19, 0x4d, 0xbe, 0xfa, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x52, 0xb5, - 0x07, 0x92, 0x13, 0x28, 0x00, 0x00, + 0x76, 0x37, 0x3f, 0x45, 0x3e, 0x8a, 0x1f, 0x2a, 0xc9, 0x32, 0xdd, 0xb3, 0x9e, 0x91, 0x3d, 0x33, + 0x3b, 0x86, 0x3d, 0x23, 0x2b, 0x9a, 0x75, 0x66, 0xed, 0x99, 0x01, 0x22, 0xca, 0xb4, 0x49, 0x47, + 0x92, 0x95, 0x22, 0x85, 0xc9, 0x4e, 0x10, 0x34, 0x5a, 0xdd, 0x65, 0xaa, 0x65, 0xb2, 0x8b, 0xe9, + 0xaa, 0xb6, 0xac, 0x05, 0x72, 0xcb, 0x61, 0x81, 0x9c, 0x72, 0xcb, 0x21, 0x08, 0x72, 0x59, 0x04, + 0x09, 0x90, 0x53, 0xae, 0x01, 0xf2, 0x27, 0x04, 0x41, 0x80, 0x9c, 0xf2, 0x07, 0x04, 0xb9, 0xef, + 0x3d, 0xa8, 0x57, 0xd5, 0xcd, 0xe6, 0x97, 0xac, 0x9d, 0xcd, 0xdc, 0x72, 0x63, 0xbf, 0xf7, 0x7b, + 0xaf, 0xab, 0x5e, 0xfd, 0xde, 0x7b, 0xd5, 0x55, 0x84, 0x0f, 0x87, 0xfe, 0x5b, 0xf6, 0xc6, 0x97, + 0xb6, 0x33, 0x60, 0x81, 0xb4, 0x85, 0x3f, 0x8a, 0x86, 0x8e, 0xf4, 0x79, 0xb0, 0x3d, 0x0e, 0xb9, + 0xe4, 0x64, 0xc5, 0xe8, 0xad, 0xbb, 0x08, 0x78, 0x34, 0x03, 0x67, 0x42, 0x24, 0x58, 0xeb, 0xa3, + 0x01, 0xe7, 0x83, 0x21, 0x7b, 0x84, 0x4f, 0xa7, 0xd1, 0xeb, 0x47, 0xd2, 0x1f, 0x31, 0x21, 0x9d, + 0xd1, 0xd8, 0x00, 0x6e, 0xc7, 0xd6, 0xee, 0x90, 0x47, 0x9e, 0xf6, 0x61, 0x54, 0x1b, 0xb1, 0x6a, + 0xc4, 0x3d, 0x36, 0x14, 0x5a, 0x7a, 0xef, 0xdf, 0x73, 0xb0, 0xd1, 0x4b, 0x86, 0x44, 0xa3, 0xa0, + 0x17, 0x8d, 0x46, 0x4e, 0x78, 0x49, 0x36, 0xa1, 0x38, 0x76, 0x84, 0x60, 0x5e, 0x33, 0xb3, 0x95, + 0xb9, 0x5f, 0xa0, 0xe6, 0x49, 0xc9, 0x5f, 0x3b, 0xfe, 0x90, 0x79, 0xcd, 0xac, 0x96, 0xeb, 0x27, + 0x72, 0x07, 0x60, 0xc0, 0xfd, 0x60, 0x60, 0x5f, 0xb0, 0xe1, 0xb0, 0x99, 0xdb, 0xca, 0xdc, 0x2f, + 0xd3, 0x32, 0x4a, 0xbe, 0x63, 0xc3, 0xa1, 0x52, 0x4b, 0x6e, 0xfb, 0xa3, 0x71, 0xc8, 0xdf, 0xb2, + 0x66, 0x5e, 0xab, 0x25, 0xef, 0x6a, 0x01, 0xf9, 0x1a, 0x8a, 0xbe, 0x10, 0x11, 0x13, 0xcd, 0xc2, + 0x56, 0xee, 0x7e, 0x65, 0xf7, 0xe3, 0x6d, 0x33, 0xda, 0xed, 0x45, 0x83, 0xdb, 0xee, 0x2a, 0x2c, + 0x35, 0x26, 0xe4, 0x8f, 0x60, 0xd5, 0x3d, 0x73, 0xa4, 0x7d, 0xe6, 0x0b, 0xc9, 0xc3, 0xcb, 0x66, + 0x11, 0x5d, 0x6c, 0x5f, 0xed, 0x62, 0xff, 0xcc, 0x91, 0x1d, 0x6d, 0xd0, 0x0e, 0x64, 0x78, 0x49, + 0x2b, 0xee, 0x44, 0x62, 0x7d, 0x0f, 0x8d, 0x59, 0x00, 0x69, 0x40, 0xee, 0x0d, 0xbb, 0xc4, 0x70, + 0x94, 0xa9, 0xfa, 0x49, 0x76, 0xa0, 0xf0, 0xd6, 0x19, 0x46, 0x0c, 0x43, 0x51, 0xd9, 0xb5, 0x92, + 0x37, 0xea, 0xb8, 0x2b, 0x0f, 0xfb, 0x3c, 0x90, 0xec, 0x9d, 0xa4, 0x1a, 0xf8, 0x34, 0xfb, 0xf3, + 0x8c, 0x65, 0x43, 0x01, 0xc7, 0x4f, 0xb6, 0xa0, 0xe2, 0x31, 0xe1, 0x86, 0xfe, 0x58, 0x8d, 0xcd, + 0x38, 0x4e, 0x8b, 0xc8, 0x87, 0x00, 0x22, 0x1a, 0x0c, 0x98, 0x40, 0x40, 0x16, 0x01, 0x29, 0x09, + 0xd9, 0x80, 0xc2, 0xd0, 0x39, 0x65, 0x71, 0xbc, 0xf5, 0xc3, 0xbd, 0xdf, 0x7c, 0x03, 0xd5, 0xa9, + 0x39, 0x93, 0x1a, 0x64, 0x7d, 0xcf, 0xbc, 0x20, 0xeb, 0xe3, 0x62, 0x8d, 0x43, 0x7e, 0xce, 0x5c, + 0x69, 0xfb, 0x9e, 0xf1, 0x5b, 0x36, 0x92, 0xae, 0x47, 0x1e, 0x43, 0x51, 0x48, 0x47, 0x46, 0x02, + 0xfd, 0xd6, 0x76, 0xef, 0x2c, 0x0e, 0xe5, 0x76, 0x0f, 0x41, 0xd4, 0x80, 0xc9, 0x43, 0x58, 0xd3, + 0xa4, 0x4d, 0xcf, 0x4a, 0x2f, 0x75, 0x03, 0x15, 0xcf, 0x52, 0x53, 0xdb, 0x80, 0x02, 0x0b, 0x43, + 0x1e, 0x36, 0x0b, 0x7a, 0xe8, 0xf8, 0x40, 0x9e, 0x00, 0xb8, 0x21, 0x73, 0x24, 0xf3, 0x6c, 0x47, + 0x36, 0x8b, 0x26, 0xac, 0x9a, 0xf5, 0xdb, 0x31, 0xeb, 0xb7, 0xfb, 0x31, 0xeb, 0x69, 0xd9, 0xa0, + 0xf7, 0x24, 0xd9, 0x86, 0xfc, 0x39, 0x3f, 0x15, 0xcd, 0x15, 0x5c, 0x7d, 0x6b, 0xc9, 0x90, 0x5f, + 0xf2, 0x53, 0x8a, 0x38, 0x15, 0x03, 0x3d, 0xda, 0xc0, 0x19, 0xb1, 0x66, 0x59, 0xc7, 0x00, 0x25, + 0x47, 0xce, 0x88, 0x91, 0x6f, 0xa1, 0x26, 0x5c, 0x16, 0x38, 0xa1, 0xcf, 0xed, 0x41, 0xc8, 0xa3, + 0x71, 0x13, 0x70, 0x34, 0x9b, 0x13, 0xc7, 0x46, 0xfd, 0x42, 0x69, 0x69, 0x55, 0xa4, 0x1f, 0xc9, + 0x63, 0x28, 0xb1, 0xc0, 0xd3, 0xd3, 0xa8, 0xbc, 0x77, 0x1a, 0x2b, 0x88, 0xdd, 0x93, 0xe4, 0x03, + 0x28, 0x9f, 0xf3, 0x53, 0xdb, 0xe5, 0x51, 0x20, 0x9b, 0xab, 0x98, 0x60, 0xa5, 0x73, 0x7e, 0xba, + 0xaf, 0x9e, 0xc9, 0x5d, 0x58, 0xd5, 0x49, 0x68, 0xf4, 0x55, 0xd4, 0x57, 0xb4, 0x2c, 0x81, 0xe8, + 0x7c, 0x34, 0x90, 0x9a, 0x86, 0x68, 0x99, 0x86, 0x7c, 0x06, 0xf5, 0x20, 0x1a, 0xa5, 0xea, 0x90, + 0x68, 0xd6, 0x11, 0x55, 0x0b, 0xa2, 0xd1, 0x24, 0x58, 0x82, 0xec, 0x42, 0x21, 0x12, 0xce, 0x80, + 0x35, 0x1b, 0x38, 0xfe, 0x9f, 0x2c, 0x89, 0xe8, 0x89, 0xc2, 0x50, 0x0d, 0x55, 0x94, 0x76, 0x79, + 0xe0, 0x46, 0x61, 0xc8, 0x02, 0xf7, 0xb2, 0xb9, 0xa6, 0x5f, 0x9f, 0x12, 0x91, 0x87, 0x90, 0x57, + 0x05, 0xa8, 0x49, 0x90, 0x59, 0xb7, 0x16, 0x38, 0x3d, 0xe4, 0x1e, 0xa3, 0x08, 0x22, 0x5f, 0xc3, + 0xca, 0x88, 0xc9, 0xd0, 0x77, 0x45, 0x73, 0x1d, 0x07, 0x71, 0x77, 0xc9, 0x20, 0x68, 0x14, 0x1c, + 0x6a, 0x20, 0x8d, 0x2d, 0x54, 0x2c, 0x84, 0x4e, 0x76, 0xfb, 0x97, 0x42, 0x7a, 0xcd, 0x8d, 0xad, + 0xcc, 0xfd, 0x55, 0x5a, 0x31, 0xb2, 0xef, 0x85, 0xf4, 0xac, 0xff, 0x2c, 0x40, 0xee, 0x25, 0x3f, + 0x9d, 0xcb, 0x8f, 0x27, 0x49, 0x02, 0x64, 0x71, 0x98, 0x77, 0x97, 0xb3, 0x69, 0x36, 0x09, 0xee, + 0xc1, 0xaa, 0x1f, 0x08, 0x19, 0x46, 0xae, 0x8e, 0xad, 0xce, 0xcc, 0x29, 0xd9, 0x84, 0xfb, 0xf9, + 0x34, 0xf7, 0xbf, 0x00, 0xa2, 0x09, 0xc9, 0xde, 0x8d, 0x99, 0x2b, 0xcd, 0xda, 0xe8, 0xf4, 0xd0, + 0x89, 0xd5, 0x4e, 0x29, 0x26, 0xb9, 0x5f, 0x4c, 0xe5, 0x3e, 0x21, 0x90, 0x97, 0xce, 0x40, 0x67, + 0x41, 0x99, 0xe2, 0x6f, 0x45, 0xaa, 0x90, 0xf3, 0x91, 0x26, 0x7a, 0x09, 0xd1, 0x25, 0x25, 0x40, + 0x9e, 0x3f, 0x01, 0x10, 0xd2, 0x09, 0x4d, 0xc6, 0xc1, 0xfb, 0x33, 0xce, 0xa0, 0xf7, 0xe4, 0x0f, + 0xe5, 0xf8, 0x2d, 0x58, 0xc1, 0xe1, 0xf8, 0x1e, 0x32, 0xbc, 0x4c, 0x8b, 0xea, 0xb1, 0xeb, 0x91, + 0xdf, 0x8f, 0x09, 0x57, 0x45, 0x67, 0x5b, 0x57, 0x04, 0x7d, 0x8a, 0x74, 0x29, 0x96, 0xd4, 0xae, + 0x64, 0xc9, 0x4b, 0x7e, 0x3a, 0xcb, 0x12, 0xeb, 0x4f, 0xa1, 0x80, 0xce, 0xc8, 0x7d, 0x68, 0xa8, + 0x4a, 0x6d, 0xcb, 0x28, 0x0c, 0x84, 0x49, 0x1f, 0xdd, 0xfa, 0x6a, 0x4a, 0xde, 0x57, 0x62, 0x9d, + 0x41, 0x0f, 0x60, 0xcd, 0x89, 0x3c, 0x9f, 0x4f, 0x41, 0x75, 0x37, 0xac, 0xa3, 0x62, 0x82, 0xbd, + 0xc7, 0xa1, 0xa8, 0x09, 0x42, 0x08, 0xd4, 0x7a, 0xfd, 0xbd, 0xfe, 0x49, 0xcf, 0x3e, 0x6e, 0x1f, + 0x3d, 0xeb, 0x1e, 0xbd, 0x68, 0xdc, 0x48, 0xc9, 0xe8, 0xc9, 0xd1, 0x91, 0x92, 0x65, 0xc8, 0x06, + 0x34, 0x8c, 0x6c, 0xff, 0xd5, 0xe1, 0xf1, 0x41, 0xbb, 0xdf, 0x7e, 0xd6, 0xc8, 0x92, 0x35, 0xa8, + 0x1a, 0xe9, 0xf3, 0xbd, 0xee, 0x41, 0xfb, 0x59, 0x23, 0x97, 0x06, 0xee, 0x1d, 0xed, 0xb7, 0x0f, + 0x94, 0x34, 0xff, 0x32, 0x5f, 0x2a, 0x37, 0xc0, 0xfa, 0xa7, 0x2d, 0x80, 0xc9, 0x6c, 0xc9, 0x03, + 0xa8, 0x39, 0xae, 0x1b, 0x85, 0x8e, 0x7b, 0x69, 0x0b, 0x97, 0x87, 0x0c, 0x67, 0x96, 0xed, 0xdc, + 0xa0, 0xd5, 0x58, 0xde, 0x53, 0xe2, 0x5f, 0x65, 0x32, 0x64, 0x1b, 0x1a, 0x8a, 0x80, 0xa1, 0xcf, + 0x02, 0x97, 0x19, 0x74, 0x16, 0xd1, 0x19, 0x5a, 0x9f, 0x68, 0x12, 0xfc, 0xe7, 0x50, 0x97, 0x8e, + 0x78, 0x63, 0xbb, 0x7c, 0x34, 0x1e, 0x32, 0xac, 0xf9, 0x39, 0x84, 0x67, 0x69, 0x4d, 0x29, 0xf6, + 0x13, 0xb9, 0x42, 0x7f, 0x05, 0x39, 0x21, 0x25, 0x12, 0xbf, 0xb2, 0xfb, 0xe9, 0x7b, 0xd7, 0x69, + 0xbb, 0xd7, 0xef, 0x53, 0x65, 0xa1, 0x0c, 0x87, 0xc3, 0x11, 0xa6, 0xc3, 0xb5, 0x0c, 0x0f, 0x0e, + 0x0e, 0xa9, 0xb2, 0x50, 0x86, 0x52, 0x0a, 0xd3, 0x4b, 0xae, 0x61, 0xd8, 0xef, 0xf7, 0xa8, 0xb2, + 0x20, 0x14, 0x56, 0x5d, 0x1e, 0xbc, 0x65, 0xa1, 0x40, 0x58, 0x73, 0x05, 0x3d, 0x6c, 0xbf, 0xdf, + 0xc3, 0x7e, 0xca, 0x8a, 0x4e, 0xf9, 0x20, 0x5d, 0x28, 0x9b, 0xc2, 0xcb, 0x43, 0x4c, 0xc5, 0xca, + 0xee, 0xc3, 0x6b, 0x04, 0x21, 0x36, 0xa1, 0x13, 0x6b, 0xf2, 0x0d, 0x14, 0x90, 0x7f, 0xcd, 0x32, + 0x36, 0xbc, 0x9f, 0x5e, 0x63, 0x66, 0x51, 0x18, 0x50, 0x6d, 0x44, 0x3e, 0x82, 0xca, 0x79, 0xe4, + 0x0d, 0x98, 0xde, 0x0d, 0x62, 0xde, 0x97, 0x29, 0xa0, 0x48, 0xd5, 0xdf, 0xe1, 0x84, 0xe4, 0x69, + 0x58, 0x05, 0x61, 0x9a, 0xe4, 0x2f, 0x27, 0xd8, 0xcf, 0x81, 0x9c, 0x39, 0xc2, 0x0e, 0xd9, 0x88, + 0x4b, 0x16, 0x6f, 0x59, 0x31, 0xb9, 0x4b, 0xb4, 0x71, 0xe6, 0x08, 0x8a, 0x8a, 0x9e, 0x96, 0x93, + 0x07, 0x90, 0x95, 0x3b, 0x26, 0xc7, 0xaf, 0x2a, 0x18, 0x59, 0xb9, 0x63, 0xfd, 0x7d, 0x1e, 0x72, + 0xbd, 0x7e, 0x9f, 0xdc, 0x84, 0xdc, 0x05, 0x0b, 0x13, 0xd6, 0xaa, 0x07, 0xc5, 0xa6, 0xdb, 0x50, + 0xb8, 0xe0, 0xa1, 0xa7, 0xcb, 0x74, 0xb5, 0x93, 0xa1, 0xfa, 0x51, 0xa9, 0x3e, 0x81, 0x8a, 0xfa, + 0x6d, 0x63, 0x6d, 0xd5, 0x65, 0xb8, 0xda, 0xc9, 0x52, 0x50, 0xc2, 0x36, 0xca, 0x14, 0xea, 0x26, + 0xe4, 0x5c, 0xa6, 0xeb, 0x70, 0xb6, 0x93, 0xa3, 0xea, 0xc1, 0xf8, 0x75, 0xcf, 0x9c, 0x50, 0xe0, + 0xb6, 0xa0, 0xda, 0xc9, 0x53, 0xfd, 0x68, 0xfc, 0xaa, 0xdf, 0xb1, 0x5f, 0x40, 0x40, 0x81, 0x82, + 0x12, 0x4e, 0xfc, 0x3e, 0x80, 0xda, 0x1b, 0x76, 0x29, 0x59, 0x38, 0xb2, 0x43, 0xe6, 0x3a, 0xc3, + 0x21, 0x12, 0x37, 0xdb, 0x29, 0xd2, 0xaa, 0x91, 0x53, 0x14, 0x9b, 0x84, 0x33, 0x32, 0x61, 0x47, + 0x52, 0xb2, 0x90, 0x79, 0xc8, 0xd6, 0x6a, 0x67, 0x85, 0xd6, 0x63, 0xcd, 0x89, 0x56, 0x28, 0xfc, + 0xcf, 0x60, 0x3d, 0xc1, 0x87, 0xcc, 0xe5, 0x83, 0xc0, 0xff, 0x25, 0xf3, 0x90, 0x9e, 0xd5, 0x4e, + 0x89, 0x92, 0x58, 0x49, 0x13, 0x9d, 0xb2, 0xfa, 0x16, 0x9a, 0x32, 0x74, 0x82, 0x64, 0x03, 0x66, + 0x0f, 0x1d, 0xa9, 0x1a, 0xb2, 0x3d, 0x12, 0x48, 0xc4, 0x6a, 0xa7, 0x4c, 0x37, 0xa7, 0x10, 0x07, + 0x1a, 0x70, 0xa8, 0x26, 0xd4, 0x2a, 0x42, 0xde, 0xbe, 0x60, 0x61, 0xab, 0x04, 0x45, 0x1b, 0x63, + 0xdc, 0xaa, 0xc1, 0xaa, 0x9d, 0x8a, 0x30, 0x22, 0x5c, 0x83, 0xc0, 0x68, 0x21, 0x22, 0x15, 0xab, + 0xd6, 0x1a, 0xd4, 0xed, 0xe9, 0xa8, 0xb4, 0xd6, 0x61, 0xcd, 0x9e, 0x9d, 0x7c, 0x6b, 0x13, 0x36, + 0xec, 0x05, 0x33, 0x6c, 0x7d, 0x00, 0xb7, 0xed, 0x65, 0x73, 0xb0, 0xfe, 0x3b, 0x03, 0xb9, 0x83, + 0x83, 0x43, 0xf2, 0x13, 0x58, 0x91, 0xf2, 0xb5, 0x54, 0xd3, 0xca, 0xe0, 0xb4, 0x6e, 0xd0, 0xa2, + 0x12, 0xe0, 0x34, 0x8c, 0x56, 0x28, 0x6d, 0x4c, 0x19, 0xa5, 0x15, 0x5a, 0xfb, 0x08, 0xd6, 0x24, + 0x7f, 0xc3, 0x02, 0x61, 0x8f, 0x59, 0x68, 0x0b, 0xe6, 0xf2, 0xc0, 0x4b, 0x8a, 0x59, 0x5d, 0xab, + 0x8e, 0x59, 0xd8, 0x43, 0x85, 0x32, 0xd8, 0x81, 0x35, 0xb5, 0xb7, 0xf1, 0x05, 0x0b, 0x98, 0x10, + 0xa6, 0x58, 0xc6, 0x64, 0x6a, 0xa4, 0x54, 0x71, 0xb5, 0x6c, 0x01, 0x94, 0x6c, 0x33, 0xbe, 0xf8, + 0xb7, 0x1a, 0x4d, 0x6b, 0x03, 0x88, 0x3d, 0xf7, 0x6e, 0x94, 0xce, 0xbd, 0xc0, 0xfa, 0x8b, 0x02, + 0xe4, 0xfa, 0xfd, 0x9e, 0x99, 0x8c, 0x33, 0x3b, 0x55, 0x27, 0x3d, 0xd5, 0xd3, 0xd9, 0xa9, 0x9e, + 0x1e, 0xc6, 0xc4, 0x57, 0x09, 0x15, 0x4f, 0x6e, 0x2e, 0xa1, 0x34, 0x9b, 0x72, 0xcb, 0x13, 0xaa, + 0x64, 0x32, 0x63, 0x71, 0x42, 0x95, 0xd1, 0x6f, 0x61, 0x3e, 0xa1, 0x74, 0xbe, 0x14, 0x97, 0x27, + 0x54, 0xc5, 0x30, 0x7f, 0x26, 0xa1, 0x1e, 0x42, 0x5d, 0x8c, 0x19, 0x73, 0xcf, 0xec, 0xd0, 0x91, + 0xcc, 0xbe, 0x18, 0x8f, 0x4c, 0x9c, 0x4b, 0xb4, 0xaa, 0x15, 0xd4, 0x91, 0xec, 0xbb, 0xf1, 0xc8, + 0xb8, 0x3c, 0x63, 0x4e, 0xe8, 0x69, 0xa6, 0x62, 0x21, 0x52, 0xf4, 0x06, 0x14, 0x7e, 0x17, 0x4f, + 0x68, 0x0b, 0xca, 0xc6, 0xe5, 0x48, 0x60, 0x39, 0xaa, 0x76, 0x80, 0x96, 0xb4, 0x48, 0x07, 0x69, + 0x07, 0xd6, 0x02, 0x47, 0x46, 0xa1, 0x33, 0x4c, 0x2d, 0xaf, 0x4e, 0xe4, 0x0a, 0x6d, 0xa4, 0x54, + 0x49, 0x33, 0xdc, 0x81, 0x35, 0x16, 0x44, 0x81, 0xeb, 0x63, 0xfd, 0x35, 0x16, 0x45, 0xb4, 0x58, + 0xa5, 0x8d, 0x94, 0x6a, 0x96, 0x10, 0x4e, 0x8a, 0x10, 0x6a, 0xcd, 0xfe, 0x4f, 0x12, 0x8e, 0x40, + 0xc3, 0x9e, 0x89, 0x1a, 0x62, 0x52, 0xc1, 0x69, 0xad, 0x02, 0xd8, 0x49, 0x18, 0x90, 0x70, 0x73, + 0x53, 0x46, 0xe9, 0xdc, 0xb4, 0xac, 0xff, 0x00, 0x58, 0x4d, 0xb7, 0x3a, 0x4c, 0x9f, 0x28, 0x0c, + 0x6c, 0xe9, 0xbc, 0xf1, 0x83, 0xc1, 0xcc, 0x46, 0xa3, 0xae, 0x54, 0x7d, 0xd4, 0x24, 0xd1, 0x7a, + 0x0a, 0xb7, 0x42, 0x26, 0xc6, 0x3c, 0x10, 0x2c, 0x49, 0xe5, 0xf1, 0xe3, 0x9d, 0x98, 0xb2, 0x85, + 0x4e, 0x86, 0x6e, 0xc4, 0x00, 0x53, 0x8d, 0x8e, 0x1f, 0xef, 0xe8, 0xb5, 0x59, 0x68, 0xfb, 0xe4, + 0xb1, 0xb2, 0xcd, 0xa1, 0x6d, 0x76, 0xde, 0xf6, 0xc9, 0xe3, 0xab, 0x6c, 0x9f, 0x28, 0xdb, 0x3c, + 0xda, 0xe6, 0x16, 0xd8, 0x3e, 0xd1, 0xb6, 0x5f, 0xc2, 0xfa, 0x9c, 0xed, 0x48, 0xe0, 0x37, 0x52, + 0xa1, 0x93, 0xa7, 0x6b, 0x33, 0x76, 0xda, 0xe8, 0xe7, 0xb0, 0xa9, 0xb7, 0xf6, 0x97, 0x3e, 0x1b, + 0x7a, 0x69, 0xbb, 0x82, 0xe9, 0x1f, 0xeb, 0xa8, 0xff, 0x85, 0x52, 0x4f, 0x59, 0x7e, 0x0d, 0xb7, + 0x18, 0x97, 0xf6, 0xc8, 0x17, 0xe3, 0x90, 0x79, 0x3e, 0x7e, 0x40, 0x98, 0x1d, 0x67, 0xd1, 0xa4, + 0xd2, 0x4d, 0xc6, 0xe5, 0x61, 0x5a, 0x8f, 0x3b, 0x4f, 0x65, 0x7c, 0x1f, 0xaa, 0xfc, 0x2d, 0x0b, + 0x87, 0xce, 0x58, 0xad, 0xbf, 0xcf, 0x31, 0xab, 0xb3, 0x9d, 0x15, 0xba, 0x6a, 0xc4, 0x54, 0x49, + 0x4d, 0xe5, 0x8b, 0x91, 0x93, 0x9c, 0x28, 0x99, 0x8e, 0x52, 0x37, 0xaa, 0x5e, 0x2a, 0x35, 0x1e, + 0x42, 0x5d, 0x72, 0xe9, 0x0c, 0x53, 0xf0, 0xb2, 0x49, 0xb3, 0x2a, 0x2a, 0xd2, 0xe0, 0x2f, 0xa0, + 0x21, 0xfc, 0x21, 0xee, 0x27, 0xb5, 0xd1, 0x28, 0x2e, 0x04, 0x40, 0x6b, 0x46, 0xd3, 0x57, 0x0a, + 0x0d, 0xff, 0x0a, 0x6e, 0x3a, 0x17, 0x6f, 0x2e, 0x14, 0x47, 0x63, 0x33, 0x3d, 0x63, 0x5d, 0x1b, + 0x2a, 0x74, 0xdd, 0xa8, 0x7b, 0x5a, 0x9b, 0xcc, 0xf7, 0x5b, 0x68, 0x46, 0x81, 0x13, 0x88, 0x0b, + 0xd5, 0x46, 0x54, 0x1d, 0x15, 0x3c, 0x70, 0xf4, 0x2e, 0xdd, 0x14, 0x81, 0x55, 0xba, 0x39, 0x41, + 0x1c, 0x6b, 0x00, 0xee, 0xd6, 0x8d, 0xf9, 0x6b, 0x67, 0x28, 0x98, 0xed, 0x07, 0x92, 0x85, 0x61, + 0x34, 0x4e, 0x05, 0x5b, 0xd7, 0x87, 0x2a, 0xdd, 0x44, 0x44, 0x37, 0x05, 0x48, 0xde, 0xde, 0x87, + 0x4f, 0x16, 0x98, 0x47, 0x81, 0xea, 0x60, 0x6f, 0x71, 0x44, 0x93, 0x6f, 0xf2, 0x6a, 0xa7, 0x46, + 0xef, 0xce, 0xb9, 0x3a, 0x99, 0x60, 0x13, 0xaf, 0x6d, 0xb8, 0xa3, 0xa9, 0x13, 0xb2, 0x31, 0xc7, + 0xcf, 0x34, 0xb6, 0x3b, 0xc5, 0xbc, 0x3a, 0xba, 0xab, 0xd3, 0xdb, 0x08, 0xa3, 0x06, 0xd5, 0xde, + 0x9d, 0x62, 0xa0, 0xee, 0x2f, 0xb3, 0xc9, 0xd9, 0xb2, 0xa0, 0x69, 0x2f, 0xc9, 0xc0, 0x25, 0x3a, + 0xcc, 0xb0, 0x65, 0x3a, 0x95, 0x41, 0xd8, 0xbd, 0x17, 0x64, 0x48, 0xeb, 0x36, 0xdc, 0xb2, 0x17, + 0x27, 0x01, 0xba, 0x5b, 0xc2, 0xf2, 0x56, 0x03, 0x6a, 0xf6, 0x14, 0x89, 0x71, 0x2a, 0x73, 0x64, + 0xc5, 0x5a, 0x37, 0xc3, 0x48, 0xdc, 0x5d, 0xcc, 0x12, 0xaf, 0xd5, 0x84, 0x4d, 0x7b, 0x21, 0xbd, + 0x70, 0x7f, 0xb1, 0x8c, 0x3f, 0xa8, 0x5c, 0xc6, 0x8e, 0xd6, 0x67, 0xf0, 0xa9, 0x7d, 0x9d, 0xb5, + 0x6f, 0x6d, 0xc1, 0x87, 0xf6, 0x95, 0xcb, 0x69, 0xfd, 0x75, 0x06, 0xca, 0xc9, 0x7e, 0x1f, 0xfb, + 0x89, 0x13, 0x0e, 0x2f, 0x6d, 0xb5, 0x19, 0xf2, 0x03, 0x27, 0x39, 0x28, 0x2c, 0x75, 0x6e, 0xd0, + 0x06, 0xaa, 0xfa, 0x13, 0x8d, 0xd9, 0x4d, 0x2a, 0x6f, 0x53, 0x06, 0x59, 0x34, 0xc8, 0xd0, 0xba, + 0xd2, 0x4c, 0xe3, 0x75, 0x6d, 0x9f, 0x7d, 0x05, 0x46, 0x6e, 0xd6, 0x8d, 0xf5, 0x37, 0x39, 0xc8, + 0xab, 0x64, 0x21, 0x1b, 0x50, 0xf0, 0x03, 0x8f, 0xbd, 0xd3, 0xdb, 0x0e, 0xaa, 0x1f, 0xc8, 0x43, + 0xc8, 0x87, 0x7c, 0xc8, 0xcc, 0x91, 0xc9, 0xad, 0x05, 0x87, 0xa1, 0x94, 0x0f, 0x19, 0x45, 0x10, + 0xf9, 0x10, 0x4a, 0x78, 0x90, 0x10, 0xd7, 0x6b, 0xb5, 0x79, 0x59, 0x41, 0x89, 0xae, 0x01, 0x16, + 0x14, 0x59, 0xe0, 0xc5, 0x15, 0x19, 0xb7, 0xf6, 0x2c, 0xf0, 0xe2, 0xfa, 0x80, 0xdf, 0x90, 0x85, + 0x1f, 0xfa, 0x0d, 0x59, 0xfc, 0xa1, 0xdf, 0x90, 0x2b, 0xbf, 0xf3, 0x37, 0x64, 0xe9, 0x77, 0xff, + 0x86, 0x6c, 0x55, 0xa0, 0x6c, 0xc7, 0xb1, 0x6b, 0x95, 0x61, 0xc5, 0xd6, 0x81, 0xc2, 0xed, 0xf5, + 0xf4, 0x57, 0x3e, 0x2e, 0xe3, 0xec, 0xc7, 0xbc, 0xce, 0x94, 0xe9, 0x2f, 0x76, 0xeb, 0xdf, 0xf2, + 0x00, 0x93, 0x33, 0xb4, 0x1f, 0xf5, 0xbc, 0xe0, 0xf7, 0x80, 0x24, 0x07, 0xab, 0x63, 0x47, 0x08, + 0xdc, 0x9a, 0x24, 0x1b, 0xd1, 0x46, 0xac, 0x3b, 0x76, 0x84, 0x50, 0x5b, 0xba, 0xff, 0x3f, 0x34, + 0x20, 0x77, 0x00, 0xce, 0xf9, 0xa9, 0xd0, 0x25, 0x4c, 0xb7, 0x59, 0x5a, 0x56, 0x12, 0xec, 0x99, + 0xe4, 0x63, 0xa8, 0xa2, 0x7a, 0xc4, 0x1c, 0x11, 0xa9, 0x8f, 0x47, 0x6c, 0xad, 0x74, 0x55, 0x09, + 0x0f, 0x8d, 0x8c, 0xec, 0xc0, 0x06, 0x82, 0x92, 0xf3, 0x03, 0xfb, 0xb5, 0x13, 0x0d, 0x4d, 0x4b, + 0xa5, 0x44, 0xe9, 0x92, 0xaa, 0xf3, 0x5c, 0x69, 0xae, 0x4d, 0xa7, 0x9b, 0xb0, 0x6e, 0xcf, 0x2f, + 0xa8, 0xf5, 0x9b, 0x1c, 0x14, 0xf7, 0xf1, 0x70, 0xde, 0xfa, 0xaf, 0x2c, 0xac, 0x50, 0xf6, 0x67, + 0x11, 0x13, 0x72, 0xe6, 0xe6, 0x21, 0x33, 0x7b, 0xf3, 0x30, 0x7d, 0x28, 0x9f, 0x9d, 0x3d, 0x94, + 0x5f, 0x70, 0x76, 0x9d, 0x5f, 0x78, 0x76, 0xbd, 0x09, 0xc5, 0x90, 0x0d, 0xd4, 0x02, 0x14, 0xcd, + 0x11, 0x23, 0x3e, 0x91, 0xd6, 0xdc, 0xa9, 0xfe, 0xca, 0x55, 0xa7, 0xfa, 0x8a, 0xf0, 0x53, 0xe7, + 0xfa, 0x8a, 0x8d, 0x9f, 0x4e, 0x9f, 0x71, 0x97, 0xcc, 0x4e, 0x35, 0x2d, 0xd4, 0x3b, 0x24, 0x7d, + 0xd0, 0x5d, 0xbe, 0xc6, 0x41, 0x37, 0x06, 0x7b, 0x7a, 0x60, 0x7a, 0x33, 0x3f, 0x71, 0xf9, 0x32, + 0x5f, 0xca, 0x35, 0xf2, 0x74, 0xfe, 0x86, 0xc5, 0xfa, 0xcb, 0x0c, 0x94, 0xa8, 0x69, 0xb8, 0xe4, + 0x01, 0xac, 0x4d, 0xa2, 0x63, 0x87, 0x51, 0x30, 0x09, 0x73, 0x5d, 0xa4, 0xb9, 0xd7, 0xf5, 0x48, + 0x0f, 0x36, 0xc7, 0x21, 0x13, 0xfe, 0x20, 0x50, 0xdd, 0x8e, 0x0b, 0xd5, 0x94, 0x70, 0x95, 0xcc, + 0x7d, 0xd6, 0xe4, 0xda, 0xe7, 0x38, 0x86, 0x1d, 0x73, 0x21, 0xcd, 0x52, 0xd2, 0x8d, 0xf1, 0x02, + 0xa9, 0xf5, 0x77, 0x19, 0x58, 0xdf, 0xe7, 0xc1, 0x6b, 0x3f, 0x1c, 0xf5, 0x78, 0x14, 0xba, 0xec, + 0x64, 0x3c, 0xe4, 0x8e, 0x67, 0xfd, 0xf9, 0xb5, 0x39, 0xb0, 0x70, 0x0a, 0xd9, 0xc5, 0x53, 0xf8, + 0x0c, 0xea, 0x2e, 0xf7, 0x98, 0xcd, 0x02, 0x19, 0x5e, 0x8e, 0xb9, 0x1f, 0x48, 0x73, 0xe0, 0x5e, + 0x53, 0xe2, 0x76, 0x22, 0xb5, 0x60, 0x12, 0x23, 0xeb, 0xaf, 0x32, 0x90, 0x7b, 0xc1, 0xa4, 0xd5, + 0xff, 0x31, 0x86, 0x64, 0xfd, 0x2c, 0xb5, 0x1a, 0xf7, 0x21, 0x17, 0x46, 0xba, 0x61, 0x4f, 0x71, + 0x2c, 0x6d, 0x42, 0x15, 0xc4, 0xfa, 0xd7, 0x2c, 0xe4, 0x0f, 0x7c, 0x21, 0xad, 0x7f, 0xc9, 0x5c, + 0x7b, 0x54, 0x4f, 0x67, 0x6e, 0x29, 0xae, 0xbe, 0xa6, 0xeb, 0xdc, 0x88, 0xef, 0x28, 0x14, 0x3b, + 0xbf, 0x01, 0x18, 0x3b, 0x03, 0xa6, 0xcf, 0x1c, 0x30, 0x66, 0x95, 0xdd, 0x66, 0x62, 0xdf, 0x57, + 0xd2, 0x63, 0x67, 0x60, 0x3a, 0x7f, 0x27, 0x43, 0xcb, 0x0a, 0x8d, 0x62, 0xb5, 0x69, 0x50, 0x5d, + 0x47, 0xfb, 0x6a, 0x55, 0xa1, 0x62, 0x4f, 0x3c, 0x59, 0xef, 0xa6, 0xb8, 0x98, 0x0f, 0xa3, 0x40, + 0x34, 0x33, 0x78, 0x40, 0xb9, 0x6c, 0xfa, 0x88, 0x21, 0x7f, 0x00, 0xf5, 0x80, 0xbd, 0x93, 0x29, + 0x57, 0x86, 0x84, 0x4b, 0x07, 0x45, 0xab, 0xca, 0xe0, 0x38, 0x1e, 0x96, 0x75, 0x0e, 0xc5, 0x7d, + 0x27, 0x70, 0xd9, 0xf0, 0x47, 0x5a, 0xd7, 0x34, 0x83, 0x7e, 0xe4, 0x4b, 0x83, 0x5f, 0x67, 0x92, + 0x5b, 0x83, 0xdb, 0x70, 0x73, 0xfa, 0xd6, 0xc0, 0x3e, 0x39, 0x3e, 0x78, 0xb5, 0xf7, 0xac, 0x71, + 0x83, 0xdc, 0x84, 0x35, 0xa3, 0x7a, 0xd1, 0x3e, 0x6a, 0xd3, 0xbd, 0xbe, 0xbe, 0x3f, 0x98, 0xbf, + 0x53, 0xc8, 0x92, 0x4d, 0x20, 0x46, 0xd6, 0x3b, 0x39, 0x3c, 0xdc, 0xa3, 0xdd, 0xef, 0x95, 0x3c, + 0xb7, 0xf0, 0xae, 0x21, 0x3f, 0x7f, 0xd7, 0x50, 0x58, 0x78, 0xd7, 0x50, 0x7c, 0x99, 0x2f, 0x95, + 0x1a, 0x65, 0xba, 0x62, 0x6e, 0xd4, 0xee, 0xfd, 0x3a, 0x07, 0xa5, 0xb8, 0x7c, 0x4e, 0xae, 0xa7, + 0x32, 0xe9, 0xeb, 0xa9, 0xd9, 0xdb, 0xb1, 0xec, 0x82, 0xdb, 0xb1, 0xc5, 0xf7, 0x60, 0xb9, 0x65, + 0xf7, 0x60, 0x8f, 0xcc, 0x8d, 0x57, 0x1e, 0x59, 0xf6, 0xc1, 0x5c, 0x21, 0xdf, 0xee, 0x3b, 0x03, + 0xa1, 0xaf, 0xf8, 0xf5, 0x75, 0x98, 0x05, 0xa5, 0x48, 0xb0, 0xd0, 0x73, 0xa4, 0x63, 0x6e, 0xd7, + 0x92, 0x67, 0xeb, 0x1f, 0x32, 0xb0, 0xa6, 0xbb, 0xd6, 0xf3, 0x90, 0x8f, 0xcc, 0x89, 0xb5, 0xf5, + 0x8b, 0x6b, 0x13, 0x2a, 0x75, 0xb7, 0x95, 0x9d, 0xba, 0xdb, 0x9a, 0x34, 0xa4, 0x5c, 0xba, 0x21, + 0x59, 0x4f, 0x52, 0xf9, 0xf2, 0x05, 0x94, 0xe2, 0x1e, 0x60, 0x4a, 0xc6, 0xda, 0xdc, 0x6c, 0x68, + 0x02, 0xb1, 0xbe, 0x82, 0x72, 0x32, 0xb5, 0x05, 0x7f, 0x4e, 0xd8, 0x48, 0xff, 0x39, 0xa1, 0x9c, + 0xfa, 0x03, 0xc2, 0xbd, 0x3e, 0x54, 0xa7, 0xba, 0x1c, 0x21, 0x90, 0xc7, 0x7e, 0xab, 0xad, 0xf1, + 0x37, 0x79, 0x04, 0xe5, 0xf8, 0x4d, 0x6a, 0x99, 0x72, 0x8b, 0x47, 0x33, 0xc1, 0xdc, 0xfb, 0xe7, + 0x0c, 0x90, 0x49, 0x66, 0x3f, 0xf3, 0xc5, 0xd8, 0x91, 0xee, 0xd9, 0x6f, 0xd5, 0x90, 0x6e, 0x42, + 0xf1, 0x9c, 0x9f, 0x4e, 0x82, 0x57, 0x38, 0xe7, 0xa7, 0x5d, 0x6f, 0x2a, 0x2e, 0xb9, 0xf7, 0xc6, + 0x25, 0x69, 0xbc, 0xf9, 0x6b, 0x34, 0xde, 0x07, 0xaf, 0xa1, 0x36, 0x2d, 0x27, 0x1f, 0xc1, 0x07, + 0xbd, 0xee, 0xe1, 0xc9, 0xc1, 0x5e, 0xbf, 0xfb, 0xea, 0xc8, 0x3e, 0x7c, 0xf5, 0xac, 0x6d, 0x9f, + 0x1c, 0xf5, 0x8e, 0xdb, 0xfb, 0xdd, 0xe7, 0xdd, 0xb6, 0xca, 0xbb, 0x26, 0x6c, 0xcc, 0x02, 0xfa, + 0xed, 0x3f, 0xee, 0x37, 0x32, 0x98, 0xac, 0x33, 0x9a, 0xbd, 0x93, 0x67, 0xdd, 0x57, 0x8d, 0xec, + 0xee, 0xdf, 0x16, 0xa0, 0xbe, 0xa7, 0xb8, 0x3b, 0x79, 0x1b, 0xf1, 0x60, 0x5d, 0x73, 0x6d, 0xfa, + 0xcf, 0x1a, 0xcb, 0xb6, 0x9b, 0x1a, 0xbb, 0x1d, 0x37, 0xda, 0x9f, 0xbe, 0x0f, 0x66, 0x58, 0xf5, + 0xab, 0x0c, 0xdc, 0x89, 0x1b, 0x72, 0x02, 0x4c, 0xb7, 0x66, 0xb2, 0xbb, 0xcc, 0xd3, 0x7c, 0x1b, + 0x4f, 0xde, 0xfe, 0xe5, 0x6f, 0x65, 0x63, 0x86, 0xf2, 0x27, 0xd0, 0x78, 0xc1, 0xe4, 0xf4, 0x6c, + 0xef, 0x2d, 0x71, 0xf4, 0x82, 0xc9, 0xe4, 0x65, 0x1f, 0x5f, 0x89, 0x31, 0xce, 0x6d, 0x20, 0xaa, + 0x81, 0x4e, 0x21, 0x04, 0x59, 0x66, 0xaa, 0xa0, 0x89, 0xff, 0x4f, 0xae, 0x06, 0x99, 0x17, 0xa8, + 0xe5, 0xc2, 0x06, 0x73, 0xcd, 0xe5, 0x42, 0xec, 0xfb, 0x97, 0x2b, 0x86, 0x99, 0xb7, 0xbc, 0x85, + 0xdb, 0x86, 0x14, 0x86, 0xcf, 0xa9, 0x42, 0x44, 0x1e, 0xce, 0x57, 0xb7, 0xb9, 0x6a, 0x95, 0xbc, + 0xf1, 0xf3, 0xeb, 0x81, 0xf5, 0x7b, 0x5b, 0xcf, 0xbf, 0xff, 0x78, 0xe0, 0xcb, 0xb3, 0xe8, 0x74, + 0xdb, 0xe5, 0xa3, 0xf8, 0x4f, 0x68, 0xfa, 0x8f, 0x66, 0x2e, 0x1f, 0xc6, 0x82, 0x7f, 0xcc, 0x56, + 0x0f, 0xfc, 0xb7, 0xec, 0x0f, 0x71, 0x4f, 0xc8, 0x25, 0xff, 0x9f, 0x6c, 0xcd, 0x3c, 0x3f, 0x7d, + 0x8a, 0x82, 0xd3, 0x22, 0x9a, 0x7c, 0xf9, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x59, 0x32, 0xc0, + 0x0f, 0xf7, 0x26, 0x00, 0x00, } diff --git a/package.json b/package.json index 72182b1fb..f3803d5f5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "github.com/livekit/protocol", "private": true, - "version": "1.50.2", + "version": "1.50.3", "scripts": { "changeset": "changeset", "ci:publish": "pnpm --filter @livekit/protocol run build && changeset publish" diff --git a/packages/javascript/CHANGELOG.md b/packages/javascript/CHANGELOG.md index c220aed4b..bf1663bab 100644 --- a/packages/javascript/CHANGELOG.md +++ b/packages/javascript/CHANGELOG.md @@ -1,5 +1,7 @@ # @livekit/protocol +## 1.50.3 + ## 1.50.2 ### Patch Changes diff --git a/packages/javascript/package.json b/packages/javascript/package.json index 153b1c82b..c46d54725 100644 --- a/packages/javascript/package.json +++ b/packages/javascript/package.json @@ -1,6 +1,6 @@ { "name": "@livekit/protocol", - "version": "1.50.2", + "version": "1.50.3", "description": "", "type": "module", "require": "dist/index.cjs", diff --git a/protobufs/livekit_agent_simulation.proto b/protobufs/livekit_agent_simulation.proto index c8f8a9847..fdd57806b 100644 --- a/protobufs/livekit_agent_simulation.proto +++ b/protobufs/livekit_agent_simulation.proto @@ -86,10 +86,10 @@ message SimulationRun { // members that ran (a missing member renormalizes the rest): // accuracy = task_completion x3 + stt.keyterm_recall + (1 - stt.wer) + tts.enunciation_score // experience = conversation.turn_taking_score x2 + tts.naturalness_score + llm.conciseness_score - optional float accuracy_score = 1; - optional float experience_score = 2; + optional float accuracy_score = 1; // absent when the simulator spoiled the call + optional float experience_score = 2; // absent when the simulator spoiled the call // The accuracy anchor: the scenario verdict as 1/0, or the fraction of the declared target state the call reached. - optional float task_completion = 3; + optional float task_completion = 3; // absent when the simulator spoiled the call STT stt = 4; LLM llm = 5; @@ -103,65 +103,64 @@ message SimulationRun { string judge_model = 10; // text judge for judged scores; "" if none ran string audio_judge_model = 11; // audio (LALM) judge; "" if none ran bool has_remote_session = 12; // false = waveform-only capture (e.g. SIP) - google.protobuf.Timestamp t0 = 13; // call start (first speech edge); turn times relative to it + google.protobuf.Timestamp t0 = 13; // audio only; call start (first speech edge); turn times relative to it - // The agent's perception of the caller. + // The agent's perception of the caller. Audio only. message STT { - optional float wer = 1; // word error rate, pooled: word_errors / words - optional uint32 words = 2; // pooling stats: run WER = sum errors / sum words - optional uint32 word_errors = 3; - optional float cer = 4; // character error rate, pooled - optional uint32 chars = 9; - optional uint32 char_errors = 10; - optional float keyterm_recall = 5; // recall of uttered key entities (names, IDs, amounts) - optional uint32 keyterms_uttered = 6; - optional uint32 keyterms_recognized = 7; - optional uint32 transcription_latency_ms = 8; // agent-reported STT/endpointing delay, mean + optional float wer = 1; // needs the RemoteSession; word error rate, pooled: word_errors / words + optional uint32 words = 2; // needs the RemoteSession; pooling stats: run WER = sum errors / sum words + optional uint32 word_errors = 3; // needs the RemoteSession + optional float cer = 4; // needs the RemoteSession; character error rate, pooled + optional uint32 chars = 9; // needs the RemoteSession + optional uint32 char_errors = 10; // needs the RemoteSession + optional float keyterm_recall = 5; // needs the RemoteSession; recall of uttered key entities (names, IDs, amounts) + optional uint32 keyterms_uttered = 6; // needs the RemoteSession + optional uint32 keyterms_recognized = 7; // needs the RemoteSession + optional uint32 transcription_latency_ms = 8; // needs the RemoteSession; agent-reported STT/endpointing delay, mean } message LLM { - optional uint32 ttft_ms = 1; // time to first token, mean (per-turn in turns) - optional uint32 ttfs_ms = 2; // time to first sentence — the smallest speakable unit - optional float tokens_per_second = 3; // decode rate, mean - optional float conciseness_score = 4; // judged 0-1: brief enough for voice + optional uint32 ttft_ms = 1; // needs the RemoteSession; time to first token, mean (per-turn in turns) + optional uint32 ttfs_ms = 2; // needs the RemoteSession; time to first sentence (the smallest speakable unit) + optional float tokens_per_second = 3; // needs the RemoteSession; decode rate, mean (per-turn in turns.llm) + optional float conciseness_score = 4; // audio only; needs the audio judge; judged 0-1: brief enough for voice } - // The agent's voice. + // The agent's voice. audio only. message TTS { - // Time to first AUDIO — the user-relevant signal. Some providers stream - // bytes before actual speech, so byte-level TTFB alone over-promises; - // it is kept below for disambiguation. - optional uint32 ttfa_ms = 1; - optional uint32 ttfb_ms = 2; // provider byte-level TTFB, agent-reported - optional float wer = 3; // word error rate, pooled: word_errors / words - optional uint32 words = 7; - optional uint32 word_errors = 8; - optional float cer = 9; // character error rate, pooled - optional uint32 chars = 10; - optional uint32 char_errors = 11; + optional uint32 ttfa_ms = 1; // needs the RemoteSession + optional uint32 ttfb_ms = 2; // needs the RemoteSession; provider byte-level TTFB + optional float wer = 3; // needs the RemoteSession; word error rate, pooled: word_errors / words + optional uint32 words = 7; // needs the RemoteSession + optional uint32 word_errors = 8; // needs the RemoteSession + optional float cer = 9; // needs the RemoteSession; character error rate, pooled + optional uint32 chars = 10; // needs the RemoteSession + optional uint32 char_errors = 11; // needs the RemoteSession optional float speech_rate_wpm = 4; // speaking rate; conversational English ~110-150 - optional float naturalness_score = 5; // judged 0-1: prosody / expressiveness - optional float enunciation_score = 6; // judged 0-1: key entities audibly intact + optional uint32 heard_words = 12; // pooling stats for speech_rate_wpm: raw heard words (not WER tokens) + optional uint32 speech_ms = 13; // agent voiced time, sum of raw VAD segments + optional float naturalness_score = 5; // needs the audio judge; judged 0-1: prosody / expressiveness + optional float enunciation_score = 6; // needs the audio judge; judged 0-1: key entities audibly intact } - // End-to-end floor dynamics, measured off the audio timeline — the ground - // truth of what the caller hears, independent of agent self-reports. + // Audio only. message Conversation { - optional float turn_taking_score = 1; // 0-1: latency curve + cut-in/barge-in/missed-turn penalties - optional int32 response_latency_p50_ms = 2; // floor-transfer offset; a gap is > 0, an overlap < 0 + optional float turn_taking_score = 1; // 0-1: latency curve + cut-in/barge-in/missed-turn penalties + optional int32 response_latency_p50_ms = 2; // floor-transfer offset; a gap is > 0, an overlap < 0 optional int32 response_latency_p95_ms = 3; optional int32 response_latency_p99_ms = 4; - optional uint32 agent_yield_latency_ms = 5; // how long the agent talks after a barge-in, mean - optional uint32 eot_misprediction_count = 6; // agent started before the caller's turn ended - optional float overlap_ratio = 7; // overlapping speech / total speech - optional uint32 overlap_speech_ms = 8; // pooling stats for overlap_ratio + optional int32 response_latency_ms = 16; // one turn's floor-transfer offset, cut-in if negative; job/run serve the percentiles + optional uint32 agent_yield_latency_ms = 5; // agent audio continuing past a barge-in — per turn: on the barging persona row (presence = barge-in); job: mean + optional uint32 eot_misprediction_count = 6; // agent started before the caller's turn ended; 0/1 per turn + optional float overlap_ratio = 7; // overlapping speech / total speech + optional uint32 overlap_speech_ms = 8; // pooling stats for overlap_ratio optional uint32 total_speech_ms = 9; - optional uint32 silence_total_ms = 10; // dead air within the conversation - optional uint32 awkward_silence_count = 11; // gaps past the natural-pause threshold - optional uint32 unanswered_persona_turns = 12; // the simulated party spoke, the agent never responded - optional uint32 false_interruption_count = 13; // agent paused for a non-interruption - optional uint32 false_interruption_unrecovered_count = 14; // of those, never resumed - optional uint32 agent_reported_e2e_latency_ms = 15; // the agent's own claim, mean + optional uint32 silence_total_ms = 10; // dead air within the conversation + optional uint32 awkward_silence_count = 11; // gaps past the natural-pause threshold; 0/1 per turn + optional uint32 unanswered_persona_turns = 12; // the simulated party spoke, the agent never responded; 0/1 per turn + optional uint32 false_interruption_count = 13; // needs the RemoteSession; agent paused for a non-interruption + optional uint32 false_interruption_unrecovered_count = 14; // needs the RemoteSession; of those, never resumed + optional uint32 agent_reported_e2e_latency_ms = 15; // needs the RemoteSession; the agent's own claim, mean } // Was the call spoiled by the simulator, not the agent? Diagnostic only — @@ -172,24 +171,18 @@ message SimulationRun { optional bool late_termination = 2; // simulator dragged on after the goal was met } + // Rows exist for text and audio; message Turn { - uint32 index = 1; // 1-based, conversation order - livekit.agent.ChatRole role = 2; // ASSISTANT — the agent under test, USER — simulator persona. - optional uint32 start_ms = 3; // relative to t0; unset if the turn could not be aligned to the audio - optional uint32 end_ms = 4; - // Agent-turn measurements; a negative response latency is a cut-in (the - // agent started before the caller's turn ended). - optional int32 response_latency_ms = 5; - optional uint32 transcription_delay_ms = 6; - optional uint32 llm_ttft_ms = 7; - optional uint32 llm_ttfs_ms = 8; - optional uint32 tts_ttfa_ms = 9; - optional uint32 tts_ttfb_ms = 10; - optional uint32 agent_reported_e2e_latency_ms = 11; - optional float conciseness_score = 12; - optional float naturalness_score = 13; - optional float enunciation_score = 14; // judged 0-1: this turn's key entities audibly intact - repeated string flags = 15; // judge failure tags ("verbosity_or_filler", ...) + uint32 index = 1; // 1-based, conversation order + livekit.agent.ChatRole role = 2; // ASSISTANT — the agent under test, USER — simulator persona. + optional uint32 start_ms = 3; // audio only; relative to t0; unset if the turn could not be aligned to the audio + optional uint32 end_ms = 4; // audio only + + // Job groups hold means/pools of these; a field with no per-turn measurement stays unset here. + STT stt = 5; // transcription_latency_ms + LLM llm = 6; // ttft_ms, ttfs_ms, tokens_per_second, conciseness_score + TTS tts = 7; // ttfa_ms, ttfb_ms, naturalness_score, enunciation_score + Conversation conversation = 8; // response_latency_ms, agent_yield_latency_ms, agent_reported_e2e_latency_ms, 0/1 counts } } diff --git a/sip/token.go b/sip/token.go index 4d3f6bc1d..199401069 100644 --- a/sip/token.go +++ b/sip/token.go @@ -31,6 +31,7 @@ type SIPTokenParams struct { ParticipantAttributes map[string]string RoomPreset string RoomConfig *livekit.RoomConfiguration + RoomAdmin bool } func BuildSIPToken(params SIPTokenParams) (string, error) { @@ -38,6 +39,7 @@ func BuildSIPToken(params SIPTokenParams) (string, error) { at := auth.NewAccessToken(params.APIKey, params.APISecret). SetVideoGrant(&auth.VideoGrant{ RoomJoin: true, + RoomAdmin: params.RoomAdmin, Room: params.RoomName, CanSubscribe: &t, CanPublish: &t,